|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
|
pool = await aiomysql.create_pool(host=self.redis_ip, port=3306,
user='root', password='123',
db='laxiao', loop=loop, charset='utf8',
autocommit=True)
async with pool.acquire() as sqlconn:
async with sqlconn.cursor() as cur:
try:
insert_sql = 'insert into laxiao(cid,company,captical,valuetype,person,tel,province,city,postcode,business,num,product) values("{}","{}","{}","{}","{}","{}","{}","{}","{}","{}","{}","{}")'.format(
item['cid'], item['名称'], item['资本'], item['类型'], item['联系人'], item['电话'], item['省份'], item['城市'],
item['邮编'], item['行业'], item['人数'], item['产品'])
await cur.execute(insert_sql)
print(item['cid'], 'done')
except Exception as e:
print(e)
mongoconn = AsyncIOMotorClient('mongodb://'+self.redis_ip+':27017')
db = mongoconn.laxiao
collection = db.company
await collection.update({'cid': item['cid']}, {'$set': item}, True)
print(item['cid'], 'mongo-done')
conn = await aioredis.create_redis('redis://'+self.redis_ip, db=self.redis_db, loop=loop)
isdo = await conn.hget("seen_list", seen_list_id)
if (isdo != None):
continue
await conn.hset("seen_list", seen_list_id, '1')
if await conn.lpop('waitting_urls') == None:
await asyncio.sleep(5)
wait conn.lpush('waitting_urls', url)
|