1.确定爬取的标签
获取排名 x.find('em')
获取title
问题：
-
三年前用urllib2，现在不这么用了
-
itemlist = soup.find_all("div",class_="item") 找到所以class为item的标签存入列表
-
url = i.find('a',class_='').get('href') 找到a标签的href属性的内容
-
title = i.find('span',class_='title').text
-
正则表达式分割 aa11a123/bbb2(sjd,下提取出任意位数数字+/ 以及任意位数数字+(
正则表达式为 [0-9]一位数字，取值范围是0-9； [0-9]+,任意位数字，取值范围为0-9，+代表多个
[0-9]+/ n位数字/ 123/
[0-9]+[/,(] n位数字/以及n位数字( 123/ 2(
python里面的写法规范是 r'正则表达式'
patt = r'[0-9]+[/,(]'
#任意位数的数字+后面跟一位/或者（符号
match = re.findall(patt,infoTmp) infoTmp是待处理的字符串
html里不是这么写
-
爬取过程中第一个网页，li里面的信息是动态加载的，用目前的方法爬不下来。
-
电脑装了phpstudy，自带mysql，跟本地mysql冲突，到本地mysql的安装目录下，bin文件夹下，cmd管理员模式运行cd C:\D\mysql-8.0.19-winx64\bin mysqld.exe -install 出现：Service successfully installed. 接下来启动mysql：net start mysql
-
Navicat Premium 出现2059错误，改了mysql的密码
-
navicat过期，注册，下破解机，注意选择破解版本是navicat for mysql的，破解时断网，下注册机关360.参见 navicat15 for mysql激活
-
数据库添加 的sql写法
tumpe = (movie_range,title,infoTmp,rating_num,inq,url)
sql='insert into movie(movie_range,title,info,rating_num,inq,url) values("%s","%s","%s","%s","%s","%s")'%tumpe
# "%s" 不能写成%s，会报错
sql='''update movie set country="%s",year="%s",
film_Genres="%s",director_actor="%s"
where movie_range = "%s"
'''%tumpe
#!/usr/bin/python
#coding: UTF-8
from urllib.request import Request,urlopen
from urllib.error import URLError, HTTPError
from bs4 import BeautifulSoup
import pymysql
import json
import re
def getConn():
conn= pymysql.connect(
host='localhost',
port = 3306,
user='root',
passwd='password',
db ='douban',
)
return conn
def getContent(url):
req = Request(url)
#增加header头信息
req.add_header('User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36')
try:
response = urlopen(req)
buff = response.read()
html = buff.decode("utf8")
response.close()
except HTTPError as e:
print('The server couldn\'t ful ll the request.')
print('Error code: ', e.code)
except URLError as e:
print('reason:%s' % e.reason)
return html
def saveContent(content, url):
soup = BeautifulSoup(content,"html.parser")
# print (soup)
itemlist = soup.find_all("div",class_="item")
conn = getConn()
cursor = conn.cursor()
num = 0
for i in itemlist:
num += 1
print (num)
movie_range = (i.find('em')).text
url = i.find('a',class_='').get('href')
title = i.find('span',class_='title').text
bd = i.find('div',class_='bd')
info = bd.find('p',class_='').text
infoTmp = info.replace('\xa0','').replace(' ','').replace('\n','')
rating_num = i.find('span',class_='rating_num').text
inq = i.find('span',class_='inq').text
# #--------数据入库---------
tumpe = (movie_range,title,infoTmp,rating_num,inq,url)
sql='insert into movie(movie_range,title,info,rating_num,inq,url) values("%s","%s","%s","%s","%s","%s")'%tumpe
# "%s" 不能写成%s，会报错
cursor.execute(sql)
conn.commit()
cursor.close() #关闭游标
conn.close() #关闭连接
#---------已经入库的信息，根据id作为主键添加年份，国家，电影类型等信息
def saveyear(content, url):
soup = BeautifulSoup(content,"html.parser")
itemlist = soup.find_all("div",class_="item")
conn = getConn()
cursor = conn.cursor()
num = 0
for i in itemlist:
num += 1
print (num)
movie_range = (i.find('em')).text
#作为主键
bd = i.find('div',class_='bd')
info = bd.find('p',class_='').text
infoTmp = info.replace('\xa0','').replace(' ','').replace('\n','')
patt = r'[0-9]+[/,(]'
#任意位数的数字+后面跟一位/或者（符号
match = re.findall(patt,infoTmp)
#导演: 奥利维·那卡什 / 艾力克·托兰达 Toledano主...2011/法国/剧情 喜剧
director_actor = infoTmp.split(match[0])[0]
year = match[0].replace('/','')
country_type = infoTmp.split(match[0])[1]
country = country_type.split('/')[0]
film_Genres = country_type.split('/')[1]
tumpe = (country,year,film_Genres,director_actor,movie_range)
# #--------数据入库---------
sql='''update movie set country="%s",year="%s",
film_Genres="%s",director_actor="%s"
where movie_range = "%s"
'''%tumpe
# "%s" 不能写成%s，会报错
cursor.execute(sql)
conn.commit()
cursor.close() #关闭游标
conn.close() #关闭连接
def geturl():
for i in range(0,10):
j = 25*i
print (j)
url = 'https://movie.douban.com/top250?start=%s&filter='%j
print(url)
content = getContent(url)
# saveContent(content, url)
saveyear(content,url)
geturl()