爬虫-广东人事考试网
广东人事考试网登录网址：https://ggfw.gdhrss.gov.cn/sydwbk/index.do
查看报名统计详情竟然不支持搜索和筛选，想看下报名人数还挺费劲，而且要查看岗位信息还要另外单独去查询。。
中间省略N字的吐槽，开搞
1.第一步先登录，打开查看报名统计详情
2.F12打开开发人员工具，点开网络，然后开始记录网络日志。
如上图点击查询按钮，会发现只有一个请求，请求方法POST，请求URL: https://ggfw.gdhrss.gov.cn/sydwbk/exam/details/spQuery.do
表单数据有四个bfa001、bab301、pages、rows
到这里思路就很清晰了，因为我们需要爬取全部数据，那就只需要bfa001、bab301这两项就好了，那就先试一下看看返回结果
1 import os 2 import sys 3 import requests 4 5 url = 'https://ggfw.gdhrss.gov.cn/sydwbk/exam/details/spQuery.do' 6 data = {'bfa001':'2210841','bab301':'05'} 7 headers = { 8 'user-agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36', 9 'Cookie':'JSESSIONID=AKnYcHe', # 这里复制自己的 10 } 11 r = requests.post(url,data=data, headers = headers) 12 13 with open('C:/Users/lenovo/Desktop/html.txt','w',encoding='utf-8') as f: 14 f.write(r.text) 15 os.system('C:/Users/lenovo/Desktop/html.txt')
返回json如下
{ "rows": [{ "aab004": "佛山市三水区公路养护中心", "aab019": 1, "aab119": 31, "aae036": 1648546541000, "bab301": "05", "bfe301": "2210841050001", "bfe3a4": "综合室工作岗位", "id": { "bfa001": "2210841", "bfz315": "87365" } }, { "aab004": "佛山市三水区西南街道社区卫生服务中心", "aab019": 3, "aab119": 3, "aae036": 1648546541000, "bab301": "05", "bfe301": "2210841050002", "bfe3a4": "临床医师", "id": { "bfa001": "2210841", "bfz315": "86909" } }, ... 中间省略 ... { "aab004": "佛山市三水区大塘镇宣传文体旅游办公室（佛山市三水区大塘镇教育办公室）", "aab019": 3, "aab119": 57, "aae036": 1648546541000, "bab301": "05", "bfe301": "2210841050128", "bfe3a4": "数学教师", "id": { "bfa001": "2210841", "bfz315": "87384" } } ], "total": 128 }
为了搞懂bfa001、bab301，那点开审查元素搜索一下，找到两项
从上图可看出bab301是城市代码
这里可以看出bfa001=2210841是固定值，结合json结果每一列对应含义都找到了
然后再完善一下，数据存到sqlite，完整代码如下：
#!/usr/bin/python #coding:utf-8 import re import requests import json import urllib import os import sys import random import time from datetime import datetime import sqlite3 from flask import Flask from flask_sqlalchemy import SQLAlchemy def Randomheader(): user_agent_list = [ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/44.0.2403.155 Safari/537.36',] UserAgent=random.choice(user_agent_list) return { 'user-agent':UserAgent, 'Cookie':'JSESSIONID=AKnYcHeRTlEK', # 这里复制自己的 'Host': 'ggfw.gdhrss.gov.cn', 'Origin': 'https://ggfw.gdhrss.gov.cn', 'Connection': 'keep-alive', } app = Flask(__name__) app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///db.sqlite3' app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = True app.config['SQLALCHEMY_COMMIT_ON_TEARDOWN'] = True db = SQLAlchemy(app) class Exam(db.Model): id = db.Column(db.String(255), primary_key=True) city = db.Column(db.String(255), primary_key=True) gwdm = db.Column(db.String(255), primary_key=True) zpdw = db.Column(db.String(255)) zpgw = db.Column(db.String(255)) pyrs = db.Column(db.Integer) bmrs = db.Column(db.Integer) tjsj = db.Column(db.DateTime) def __repr__(self): return f'{self.bmrs}\t{self.gwdm} {self.zpdw} {self.zpgw}' def get_exam_by_city(city_code): url = 'https://ggfw.gdhrss.gov.cn/sydwbk/exam/details/spQuery.do' data = {'bfa001':'2210841','bab301':city_code} r = requests.post(url,data=data, headers = Randomheader()) try: rows = json.loads(r.text) exam_list = [] for row in rows['rows']: exam = Exam.query.filter_by(id=row['id']['bfz315'],city=row['bab301'],gwdm=row['bfe301']).first() if exam: exam.bmrs=row['aab119'] exam.tjsj=datetime.fromtimestamp(row['aae036']/1000) else: exam = Exam(id=row['id']['bfz315'],city=row['bab301'], gwdm=row['bfe301'], zpdw=row['aab004'], zpgw=row['bfe3a4'], pyrs=row['aab019'], bmrs=row['aab119'], tjsj=datetime.fromtimestamp(row['aae036']/1000) ) if exam.id and exam.city and exam.gwdm: exam_list.append(exam) db.session.add_all(exam_list) db.session.commit() print(f"【{city_code}】爬取数目={rows['total']} 数据库数目={Exam.query.filter_by(city=city_code).count()}") except Exception as ex: print(repr(ex)) finally: db.session.close() if __name__ == '__main__': db.create_all() get_exam_by_city('99') # 省直 for i in range(21): time.sleep(10) # 防止访问过于频繁 get_exam_by_city('%02d'% (i+1))
3.至此，招聘信息和岗位报名人数拿到了，但是没有岗位详情。
招聘公告 附件有公开招聘岗位表
把Excel表下载下来导入到数据库，结合上面的Exam表来查询。
导入完整代码如下：
#!/usr/bin/python #coding:utf-8 import re import requests import json import urllib import os import sys import random import time from datetime import datetime import sqlite3 from flask import Flask from flask_sqlalchemy import SQLAlchemy import openpyxl app = Flask(__name__) app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///db.sqlite3' app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = True app.config['SQLALCHEMY_COMMIT_ON_TEARDOWN'] = True db = SQLAlchemy(app) class Job(db.Model): city_name = db.Column(db.String(255)) dwbh = db.Column(db.String(255), primary_key=True) gwdm = db.Column(db.String(255), primary_key=True) zpdw = db.Column(db.String(255)) zpgw = db.Column(db.String(255)) gwjj = db.Column(db.String(255)) gwdj = db.Column(db.String(255)) xlyq = db.Column(db.String(255)) xwyq = db.Column(db.String(255)) zyyq = db.Column(db.String(255)) gzjl = db.Column(db.String(255)) def __repr__(self): return f'{self.dwbh}\t{self.gwdm} {self.zpdw} {self.zpgw}' if __name__ == '__main__': db.create_all() wb = openpyxl.load_workbook('C:/Users/lenovo/Desktop/附件1：公开招聘岗位表.xlsx') sheet = wb.worksheets[0] row_index = 0 job_list = [] for row in sheet.iter_rows(): row_index = row_index + 1 row_str = str(row_index) job = Job( city_name = sheet['A'+row_str].value, dwbh = sheet['B'+row_str].value, gwdm = sheet['C'+row_str].value, zpdw = sheet['D'+row_str].value, zpgw = sheet['E'+row_str].value, gwjj = sheet['F'+row_str].value, gwdj = sheet['G'+row_str].value, xlyq = sheet['K'+row_str].value, xwyq = sheet['L'+row_str].value, zyyq = sheet['N'+row_str].value, gzjl = sheet['U'+row_str].value, ) if job.dwbh and job.gwdm and str.isdigit(job.gwdm): job_list.append(job) db.session.add_all(job_list) db.session.commit()
注意：Excel表第一行（就是合并单元格那行）删除后保存。
有需要的话，还可以先筛选Excel再导入。
数据库查询代码如下：
我这里只查看('省直','广州','佛山')三个考区，按岗位人均报名人数升序排序。
select b.city_name 考区,a.zpdw 招聘单位,a.zpgw 招聘岗位,a.gwdm 岗位代码,b.gwdj 岗位等级,b.gwjj 岗位简介,b.xlyq 学历要求,b.xwyq 学位要求,b.zyyq 专业要求,b.gzjl 工作经历,a.pyrs 聘用人数 ,a.bmrs 报名人数,a.tjsj 更新时间 from exam a,job b where a.gwdm=b.gwdm and b.city_name in ('省直','广州','佛山') order by a.bmrs/a.pyrs;