WebScraper | .NET library to scrape content from the Internet | Scraper library

import requests
from bs4 import BeautifulSoup, Comment
import pandas as pd
import re

url = 'https://www.baseball-reference.com/register/league.cgi?code=NWDS&class=Smr'
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36'}

# Get links
response = requests.get(url, headers=headers)
soup = BeautifulSoup(response.text, 'html.parser')
yearLinks = soup.find_all('th', {'data-stat':'year_ID'})

links = {}
for year in yearLinks:
    if year.find('a', href=True):
        links[year.text] = 'https://www.baseball-reference.com' + year.find('a', href=True)['href']

final_df = {'batting':[], 'pitching':[]}
for year, link in links.items():
    print(year)
    response = requests.get(link, headers=headers)
    soup = BeautifulSoup(response.text, 'html.parser')

    if soup.find_all('th', {'data-stat':'team_ID'}):
        team_links = soup.find_all('th', {'data-stat':'team_ID'})
        
    else:
        team_links = []
        comments = soup.find_all(string=lambda text: isinstance(text, Comment))
        for each in comments:
            if 'th' in str(each):
                try:
                    soupAlpha = BeautifulSoup(str(each), 'html.parser').find_all('th', {'data-stat':'team_ID'})
                    if soupAlpha != []:
                        team_links += soupAlpha
                except:
                    continue
                    
    teamLinks = {}
    for team_link in team_links:
        if team_link.find('a', href=True):
            teamLinks[team_link.text] = 'https://www.baseball-reference.com' + team_link.find('a', href=True)['href']
            
    for team, teamLink in teamLinks.items():
        print(f'\t{team}')
        response = requests.get(teamLink, headers=headers)
        soup = BeautifulSoup(response.text, 'html.parser')
        
        batting_table = pd.read_html(response.text, attrs = {'id': 'team_batting'})[0]
        batting_table['Year'] = year
        batting_table['Team'] = team
        
        print(f'\t\t{team} - batting stats')
        
        comments = soup.find_all(string=lambda text: isinstance(text, Comment))
        for each in comments:
            if 'table' in str(each):
                try:
                    pitching_table = pd.read_html(str(each), attrs = {'id': 'team_pitching'})[0]
                    batting_table['Year'] = year
                    batting_table['Team'] = team
                    
                    print(f'\t\t{team} - pitching stats')
                    break
                except:
                    continue
                
        final_df['batting'].append(batting_table)
        final_df['pitching'].append(pitching_table)
            
batting = pd.concat(final_df['batting'], axis=0)     
pitching = pd.concat(final_df['pitching'], axis=0)