import requestsfrom bs4 import BeautifulSoupimport csvdef getHTMLText(url): try: r = requests.get(url, timeout=30) r.raise_for_status() r.encoding = r.apparent_encoding return r.text except: return ""def fillUnivList(ulist, html): soup = BeautifulSoup(html, "html.parser") # 匹配原代码下的模块，在匹配其下面的所有模块，找到数据对应位置 items = soup.find("tbody").find_all("tr") for item in items: # 找到tr模块下的| 模块，分别提取排名、学校名称、国家、国家/地区排名、总分 tds = item.find_all("td") # 提取排名数据 rank = tds[0].text.strip() # 提取学校名称 school_name = tds[1].text.strip() # 提取国家 country = tds[2].text.strip() # 提取国家/地区排名 country_rank = tds[3].text.strip() # 提取总分 total_score = tds[4].text.strip() # 存入ulist列表中 ulist.append([rank, school_name, country, country_rank, total_score])def printUnivList(ulist, num): f=open('2021.csv','w',encoding='utf-8') csv_writer=csv.writer(f) csv_writer.writerow(["排名", "学校名称", "国家", "国家排名", "总分"]) for i in range(num): u = ulist[i] csv_writer.writerow([u[0], u[1], u[2], u[3], u[4]]) f.close()def main(): uinfo = [] url = 'https://www.shanghairanking.cn/rankings/arwu/2021' html = getHTMLText(url) fillUnivList(uinfo, html) printUnivList(uinfo, 30) # 20 univsmain()
|
