python网络小说爬虫程序的一些笔记，都是过程稿
#结构化的文本文件转换为dict, 输入为一个本地文件，每行两列信息；输出为dict def file2dict(filepath): listDict = dict() os.chdir(".") with open(filepath, encoding='utf-8') as f_menulist: try: lines = f_menulist.readlines() #读取全部内容 ，并以列表方式返回 filesum = len(lines) #一共有多少章，即目录文件有多少行 print("Total: " + str(filesum) + " links in menulist") #读取全部或者一定的行数 linesToRead = filesum #linesToRead = 13 #一次性会读取几章（menulist列表里多少行）行号相减再+1 linesStart = 0 #从哪一行开始读取。从第一行开始则写0. for i in range(linesStart, linesStart+linesToRead): #linesToRead #获取网址和每章标题 currentline = lines[i].split("|") currenturl = currentline[0] currenttitle = currentline[1].strip()#[0:len(currentline[1])-1] #NOT inliude CR/LF listDict[currenturl] = currenttitle finally: f_menulist.close() return listDict
1 print(h) 2 listDict = file2dict(basedir + menufile) #一个本地文件转成dict 3 # print(listDict) 4 errcache = "" 5 lendic = 105#len(listDict) #这里也可以指定一个数字，表示只读取n行就停止 6 i = 0 7 firstchap = 1 8 for key,value in listDict.items(): #成功的写下文件，失败的列出列表 9 i = i + 1 10 time.sleep(1.0)# Don't access website so frecquently 11 singlecontent = tryurl(key) #key是url，value是章节名称 12 uf = get_file_by_urllib(key) #以url的文件名作为落地文件名 13 uf = os.path.basename(uf) 14 #print(uf) 15 16 if singlecontent[0]==200: 17 print(i, "/", lendic, "-----------", uf, "-------------") 18 # print(singlecontent[1][100:250], "\n") 19 soup = BeautifulSoup(singlecontent[1], 'lxml') 20 contentPart = soup.find(id='content') #含有正文的那段div 21 # print(contentPart.text) 22 fixedtxt = fixtxt(contentPart.text) #清洗正文 23 # print(fixedtxt) 24 stxt = "" + value + "\n" + fixedtxt 25 h,f = htmlhead("大魏能臣-黑男爵") 26 if firstchap==1: #html的文件头 27 saveHtml(basedir+'out.xhtml', h) 28 firstchap = 0 29 30 if i==lendic: #最后一章html的文件尾 31 saveHtml(basedir+'out.xhtml', stxt) 32 saveHtml(basedir+"out.xhtml", f) 33 else: 34 saveHtml(basedir+'out.xhtml', stxt+chapdiv()) #chapdiv是sigil的章节分割标识 35 else: 36 errcache = errcache + key + "|" + value + "\n" 37 print("XXXXXXXXXXX", singlecontent[1], "XXXXXXXXXXXXX") 38 if i==lendic: #include 55th line in listfile 39 print(errcache) 40 break 41 menuDict = GetMenuHtml(myheader, menuurl) #从一个网址里提取各个章节的url和章节名称 42 saveTxt2File("menulist.txt", menuDict, urldomain) #保存在文件里
def fixtitle(str):
ts = str.find("第",0,1)
#标题不是以第字开头
if ts==-1 :
str = "第"+str
te = str.find("章")
if te==-1:
a=1
else:
a=1
#main
for i in range(linesStart, linesStart+linesToRead): #linesToRead
#获取网址和标题
currentline = lines[i].split("|")
currenturl = currentline[0]
currenttitle = currentline[1].strip()#[0:len(currentline[1])-1] #NOT inliude cr/lf
print("Line:"+str(i+1) + "***hacking: " + currenturl)
StatusCode, htmlcontent = tryurl(currenturl)
#print(type(htmlcontent))
j = 1.2 #计数器，返回非200，则sleep增加0.2秒，上限为2秒.尝试3次都不行，就退出
while not(StatusCode==200): #get return correctly
if j>3:
sys.exit()
else:
j = j + 0.2
print(str(j) + " " + str(StatusCode))
time.sleep(0.8+j)
StatusCode, htmlcontent = tryurl(currenturl)
#清洗替换
htmlcontent = htmlcontent.replace("
", "\r\n")
htmlcontent = htmlcontent.replace(" ", "")
htmlcontent = htmlcontent.replace("","")
htmlcontent = htmlcontent.replace("","")
#saveHtml("1", htmlcontent)
soup = BeautifulSoup(htmlcontent, features="lxml")
#htmlcontent = soup.prettify()
print('Finish：' + currenttitle + "\n") #print the title of chapter
#开始提取正文
txtcontent = ""
allli = ""
results = soup.select('#content')
#print("len:"+str(len(results)))
#print(type(t))
allli = allli + currenttitle + "\n" + results[0].text + "\n"
#print(showsinthistype)
#print(showsinthistype)
#saveHtml("2", showsinthistype)
#remove BOM if exist
# txtcontent = fixtxt(txtcontent, bc, patterns)
# print(txtcontent)
#save current chapter to file
thischap = allli + "\n\n"
#保存单独的文章成一个文件
saveHtml(author+"_"+bookname+"_"+str(linesStart+1)+"-"+str(linesStart+linesToRead), thischap)
#allchap = allchap + (esulstle+"\n\n"+thischap).encode('utf-8')
#allchap = allchap + thischap
time.sleep(1)
#所有文章合并在一起并保存成文件
#allchap = allchap + "\n\n"
#saveHtml(author+"_"+bookname+"_"+str(linesStart+1)+"-"+str(linesStart+linesToRead), allchap)
print("Done!")
start_directory = r'.'
os.startfile(start_directory)
--------------------
1 #read config info from a ini file 2 config = configparser.ConfigParser() 3 cfgfilename = 'cfg.ini' #this ini file should save as ascii tpye. Not UTF-8 4 config.read(cfgfilename) 5 6 author = config.get('article','author') 7 bookname = config.get('article','bookname') 8 9 menuurl = config.get('menu','menuurl') 10 print(menuurl) 11 12 baseurl = config.get('baseurl', 'baseurl') 13 14 print("cfg.ini: "+author+"_"+bookname+"_"+menuurl); 15 16 #用于清理文本的正则表达式 17 patterns = [] 18 #patterns.append( re.compile(r"