使用Python爬取百度热搜

 1 import datetime
 2 import re
 3 import time
 4 
 5 import requests
 6 from bs4 import BeautifulSoup
 7 
 8 
 9 def demo():
10     # 获取响应对象
11     response = requests.get("https://top.baidu.com/board")
12     # 根据""格式获取解析对象
13     doc = BeautifulSoup(response.content.decode(), "xml")
14     # 根据规则获取要抓取的内容
15     title = doc.findAll(attrs={'class': "c-single-text-ellipsis"})
16     # 根据正则表达式切割内容
17     json_str = re.findall(r'\s\w*', str(title))
18     # 获取当前时间对象
19     now = datetime.datetime.now()
20     titles = []
21     with open(
22             file="G:/Code/Python/HotNews/" + str(now.date()) + "-" + str(now.hour) + "-" + str(now.minute) + ".txt",
23             mode="w+",
24             encoding="utf8") as f:
25 
26         for js in json_str:
27             jstr = str(js).strip()
28             if "class" != jstr and jstr != '\n' and titles.count(jstr) == 0:
29                 titles.append(jstr)
30 
31         for item in titles:
32             write(item, f)
33     f.close()
34 
35 
36 def write(file, f):
37     if len(str(file)) > 0:
38         f.writelines(file)
39         f.write("\n")
40 
41 
42 if __name__ == "__main__":
43     while True:
44         demo()
45         time.sleep(60)