爬虫——简单处理js中嵌入的json数据
看了群里一个人提问道https://www.amazon.com/，商品分类那里无法用xpath拿得到列表。遂对其研究。
通过抓包工具可以得知，原始数据存在于js代码中，我的方式是手动解析js，从里面抠出来json数据获取
这是数据包：
window.$Nav && $Nav.when("data").run(function(data) { data({"wishlistContent":{"template":{"name":"itemList","data":{"items":[{"text":"创建一个心愿单","url":"/gp/registry/wishlist?ie=UTF8&ref_=nav_wishlist_create&triggerElementID=createList"},{"text":"查找其他人的心愿单","url":"/gp/registry/search?ie=UTF8&ref_=nav_wishlist_find"},{"text":"查找礼品","url":"/gcx/Gifts-for-Adults/gfhz/?_encoding=UTF8&ref_=nav_wishlist_gf"},{"subtext":"安装亚马逊助理","text":"保存Web中的商品","url":"/gp/BIT?ie=UTF8&bitCampaignCode=a0032&ref_=nav_wishlist_bit_v2_a0032"},{"text":"婚庆礼品登记清单","url":"/wedding/home?_encoding=UTF8&ref_=nav_wishlist_wr"},{"text":"婴儿礼品登记清单","url":"/baby-reg/homepage?_encoding=UTF8&ref_=nav_wishlist_br"},{"text":"亲友馈赠","url":"/gp/gift-central/organizer?ie=UTF8&ref_=nav_wishlist_fafgift"},{"text":"AmazonSmile慈善列表","url":"/gp/clpf?ie=UTF8&ref_=nav_wishlist_smi_se_ya_lll_ll"},{"text":"Pantry 清单","url":"/gp/pantry/yourlists?ie=UTF8&ref_=nav_wishlist_pntry_gno"},{"text":"我的中心","url":"/stream/saves/items?_encoding=UTF8&ref_=nav_wishlist_strm_in_youtique_lists"},{"text":"探索创意列表","url":"/ideas?_encoding=UTF8&ref_=nav_wishlist_idea_nav"},{"text":"探索展室","url":"/showroom?_encoding=UTF8&ref_=nav_wishlist_srm_your_desk_wl"},{"text":"参加家庭风格测验","url":"/stylequiz/?_encoding=UTF8&ref_=nav_wishlist_sq_ya_yl"},{}]}},"wlTriggers":"57042:165723:180559:159521"},"yourAccountContent":{"template":{"name":"itemList","data":{"items":[{"text":"我的账户","url":"/gp/css/homepage.html?ie=UTF8&ref_=nav_youraccount_ya"},{"text":"我的订单","url":"/gp/css/order-history?ie=UTF8&ref_=nav_youraccount_orders","id":"nav_prefetch_yourorders"},{"text":"我的心愿单","url":"/gp/registry/wishlist?ie=UTF8&ref_=nav_youraccount_wl&requiresSignIn=1"},{"text":"为我推荐","url":"/gp/yourstore?ie=UTF8&ref_=nav_youraccount_recs"},{"text":"我的”订购与节省“商品","url":"/gp/subscribe-and-save/manager/viewsubscriptions?ie=UTF8&ref_=nav_youraccount_sns"},{"text":"成员资格和订阅","url":"/yourmembershipsandsubscriptions?_encoding=UTF8&ref_=nav_youraccount_digital_subscriptions"},{"text":"我的服务请求","url":"/localservices/ya/servicerequests?_encoding=UTF8&ref_=nav_youraccount_desktop_vas_requestlist"},{"text":"我的车库","url":"/gp/your-garage/?ie=UTF8&ref_=nav_youraccount_au_pf_as_GNO"},{"text":"您的运动粉丝商店","url":"/fanshop?_encoding=UTF8&ref_=nav_youraccount_yfs"},{"text":"我的宠物","url":"/yourpets?_encoding=UTF8&ref_=nav_youraccount_pet_profiles"},{"text":"管理我的内容和设备","url":"/mn/dcw/myx.html?_encoding=UTF8&ref_=nav_youraccount_myk"},{"text":"我的音乐库","url":"/gp/dmusic/mp3/player?ie=UTF8&ref_=nav_youraccount_cldplyr"},{"text":"我的Amazon Drive","url":"/clouddrive?_encoding=UTF8&ref_=nav_youraccount_clddrv"},{"text":"Prime 即时录像","url":"/Prime-Instant-Video/s/browse?_encoding=UTF8&node=2676882011&ref_=nav_youraccount_piv"},{"text":"我的 Kindle Unlimited","url":"/gp/kindle/ku/ku_central?ie=UTF8&ref_=nav_youraccount_ku"},{"text":"我的播放列表","url":"/gp/video/watchlist?ie=UTF8&ref_=nav_youraccount_ywl"},{"text":"我的录像库","url":"/gp/video/library?ie=UTF8&ref_=nav_youraccount_yvl"},{"text":"我的应用程序和设备","url":"/gp/mas/your-account/myapps?ie=UTF8&ref_=nav_youraccount_aad"},{}]}},"signInHtml":"","wlTriggers":"205695:159521"},"shopAllContent":{"template":{"name":"itemList","data":{"items":[{"text":"Prime视频","url":"/s/browse?_encoding=UTF8&node=2858778011&ref_=nav_shopall-export_nav_mw_sbd_intl_video"},{"text":"音乐、CD和乙烯基塑料唱片","url":"/s/browse?_encoding=UTF8&node=5174&ref_=nav_shopall-export_nav_mw_sbd_intl_cd"},{"text":"数字音乐","url":"/s/browse?_encoding=UTF8&node=163856011&ref_=nav_shopall-export_nav_mw_sbd_intl_music"},{"text":"Kindle 商店","url":"/s/browse?_encoding=UTF8&node=133140011&ref_=nav_shopall-export_nav_mw_sbd_intl_kindle"},{"text":"工艺品","url":"/s/browse?_encoding=UTF8&node=4954955011&ref_=nav_shopall-export_nav_mw_sbd_intl_arts"},{"text":"汽车","url":"/s/browse?_encoding=UTF8&node=2562090011&ref_=nav_shopall-export_nav_mw_sbd_intl_automotive"},{"text":"婴儿用品","url":"/s/browse?_encoding=UTF8&node=16225005011&ref_=nav_shopall-export_nav_mw_sbd_intl_baby"},{"text":"美容和个人护理","url":"/s/browse?_encoding=UTF8&node=16225006011&ref_=nav_shopall-export_nav_mw_sbd_intl_beauty"},{"text":"图书","url":"/s/browse?_encoding=UTF8&node=283155&ref_=nav_shopall-export_nav_mw_sbd_intl_books"},{"text":"计算机","url":"/s/browse?_encoding=UTF8&node=16225007011&ref_=nav_shopall-export_nav_mw_sbd_intl_computers"},{"text":"电子产品","url":"/s/browse?_encoding=UTF8&node=16225009011&ref_=nav_shopall-export_nav_mw_sbd_intl_electronics"},{"text":"女士时尚","url":"/s/browse?_encoding=UTF8&node=16225018011&ref_=nav_shopall-export_nav_mw_sbd_intl_womenfasion"},{"text":"男士时尚","url":"/s/browse?_encoding=UTF8&node=16225019011&ref_=nav_shopall-export_nav_mw_sbd_intl_menfasion"},{"text":"女童时尚","url":"/s/browse?_encoding=UTF8&node=16225020011&ref_=nav_shopall-export_nav_mw_sbd_intl_girlfasion"},{"text":"男童时尚","url":"/s/browse?_encoding=UTF8&node=16225021011&ref_=nav_shopall-export_nav_mw_sbd_intl_boyfasion"},{"text":"健康和家居用品","url":"/s/browse?_encoding=UTF8&node=16225010011&ref_=nav_shopall-export_nav_mw_sbd_intl_health"},{"text":"家居厨房用品","url":"/s/browse?_encoding=UTF8&node=16225011011&ref_=nav_shopall-export_nav_mw_sbd_intl_kitchen"},{"text":"工业科技","url":"/s/browse?_encoding=UTF8&node=16225012011&ref_=nav_shopall-export_nav_mw_sbd_intl_industrial"},{"text":"行李箱包","url":"/s/browse?_encoding=UTF8&node=16225017011&ref_=nav_shopall-export_nav_mw_sbd_intl_luggage"},{"text":"电影和电视","url":"/s/browse?_encoding=UTF8&node=2625373011&ref_=nav_shopall-export_nav_mw_sbd_intl_movies"},{"text":"宠物用品","url":"/s/browse?_encoding=UTF8&node=16225013011&ref_=nav_shopall-export_nav_mw_sbd_intl_pet"},{"text":"软件","url":"/s/browse?_encoding=UTF8&node=16225008011&ref_=nav_shopall-export_nav_mw_sbd_intl_software"},{"text":"运动与户外用品","url":"/s/browse?_encoding=UTF8&node=16225014011&ref_=nav_shopall-export_nav_mw_sbd_intl_sports"},{"text":"工具与家居装饰","url":"/s/browse?_encoding=UTF8&node=256643011&ref_=nav_shopall-export_nav_mw_sbd_intl_tools"},{"text":"玩具与游戏","url":"/s/browse?_encoding=UTF8&node=16225015011&ref_=nav_shopall-export_nav_mw_sbd_intl_toys"},{"text":"视频游戏","url":"/s/browse?_encoding=UTF8&node=16225016011&ref_=nav_shopall-export_nav_mw_sbd_intl_video_games"},{"text":"销售和优惠","url":"/s/browse?_encoding=UTF8&node=15529609011&ref_=nav_shopall-export_nav_mw_sbd_intl_sales"}]}}},"cartContent":{"html":"
"},"signinContent":{"html":""},"accountListContent":{"html":""},"templates":{"asin-promo":" 
","discoveryPanelList":"<# var renderItems = function(items) { #> <# jQuery.each(items, function (i, item) { #> 
<# }); #> View all orders <# }; #><# renderItems(items); #>","itemList":"<# var hasColumns = (function () { var checkColumns = function (_items) { if (!_items) { return false; } for (var i=0; i<_items.length; i++) { if (_items[i].columnBreak || (_items[i].items && checkColumns(_items[i].items))) { return true; } } return false; }; return checkColumns(items);}()); #><# if(hasColumns) { #> <# if(items[0].image && items[0].image.src) { #> <# } else if (items[0].greeting) { #> <# } else { #> <# } #> <# } #> <# if(item.dividerBefore) { #> <# } #> <# if(item.text || item.content) { #> <# if(item.url) { #> <# if(item.panelKey) { #> nav-hasPanel <# } #> <# if(item.items) { #> nav-title <# } #> <# if(item.decorate == 'carat') { #> nav-carat <# } #> <# if(item.decorate == 'nav-action-button') { #> nav-action-button <# } #> nav-item' <# if(item.extra) { #> <#=item.extra #> <# } #> <# if(item.id) { #> id='<#=item.id #>' <# } #> <# if(item.dataNavRole) { #> data-nav-role='<#=item.dataNavRole #>' <# } #> <# if(item.dataNavRef) { #> data-nav-ref='<#=item.dataNavRef #>' <# } #> <# if(item.panelKey) { #> data-nav-panelkey='<#=item.panelKey #>' role='navigation' aria-label='<#=item.text#>' <# } #> <# if(item.subtextKey) { #> data-nav-subtextkey='<#=item.subtextKey #>' <# } #> <# if(item.image && item.image.height > 16) { #> style='line-height:<#=item.image.height #>px;' <# } #> > <# if(item.decorate == 'carat') { #> <# } #> <# if(item.image && item.image.src) { #> 
<# } #> <# if(item.text) { #> <# } else if (item.content) { #> <# } #> <# if(item.subtext) { #> <# } #> <# if(item.url) { #> <# } else {#> <# } #> <# } #> <# if(item.image && item.image.src) { #> <# if(item.url) { #> <# } #> ![]()
id='<#=item.id #>' <# } #> src='<#=item.image.src #>' <# if (item.alt) { #> alt='<#= item.alt #>'<# } #>/> <# if(item.url) { #> <# } #> <# } #> <# if(item.items) { #> <# } #> <# }); #><# }; #><# renderItems(items); #><# if(hasColumns) { #> <# } #>","notificationsList":"<# jQuery.each(items || [], function (i, item) { #> <# if (item.dismissId) { #> <# } #> <# if (item.image && item.image.src) { #> 
alt='<#= item.image.alt #>'<# } #> <# if (item.image.title) { #> title='<#= item.image.title #>'<# } #>/> <# } #> <# if (item.heading) { #> <# } #> <# jQuery.each(item.content || [], function (j, itemContent) { #> <# if (itemContent.url) { #> <# } else { #> <# } #> <# }); #> <# }); #>","discoveryPanelSummary":" <# jQuery.each(items || [], function (i, item) { #> 
<# }); #>","htmlList":" <# jQuery.each(items, function (i, item) { #> <# }); #>","subnav":"<# if (obj && obj.type === 'vertical') { #> <# jQuery.each(obj.rows, function (i, row) { #> <# if (row.flyoutElement === 'button') { #> <# } else if (row.flyoutElement === 'list' && row.list) { #> <# jQuery.each(row.list, function (j, list) { #> <# }); #> <# } else if (row.flyoutElement === 'link') { #> <# if (row.topSpacer) { #> <# } #> <# } #> <# }); #><# } else if (obj) { #> <# } #>","wishlist":"<# jQuery.each(wishlist, function (i, item) { #> <#=item.name #> <# }); #>","cart":"<# jQuery.each(items, function (i, item) { #> <# if (i%2==1) { #> <# } #><# }); #>"}}); });
这是结果：
Prime视频 /s/browse?_encoding=UTF8&node=2858778011&ref_=nav_shopall-export_nav_mw_sbd_intl_video
音乐、CD和乙烯基塑料唱片 /s/browse?_encoding=UTF8&node=5174&ref_=nav_shopall-export_nav_mw_sbd_intl_cd
数字音乐 /s/browse?_encoding=UTF8&node=163856011&ref_=nav_shopall-export_nav_mw_sbd_intl_music
Kindle 商店 /s/browse?_encoding=UTF8&node=133140011&ref_=nav_shopall-export_nav_mw_sbd_intl_kindle
工艺品 /s/browse?_encoding=UTF8&node=4954955011&ref_=nav_shopall-export_nav_mw_sbd_intl_arts
汽车 /s/browse?_encoding=UTF8&node=2562090011&ref_=nav_shopall-export_nav_mw_sbd_intl_automotive
婴儿用品 /s/browse?_encoding=UTF8&node=16225005011&ref_=nav_shopall-export_nav_mw_sbd_intl_baby
美容和个人护理 /s/browse?_encoding=UTF8&node=16225006011&ref_=nav_shopall-export_nav_mw_sbd_intl_beauty
图书 /s/browse?_encoding=UTF8&node=283155&ref_=nav_shopall-export_nav_mw_sbd_intl_books
计算机 /s/browse?_encoding=UTF8&node=16225007011&ref_=nav_shopall-export_nav_mw_sbd_intl_computers
电子产品 /s/browse?_encoding=UTF8&node=16225009011&ref_=nav_shopall-export_nav_mw_sbd_intl_electronics
女士时尚 /s/browse?_encoding=UTF8&node=16225018011&ref_=nav_shopall-export_nav_mw_sbd_intl_womenfasion
男士时尚 /s/browse?_encoding=UTF8&node=16225019011&ref_=nav_shopall-export_nav_mw_sbd_intl_menfasion
女童时尚 /s/browse?_encoding=UTF8&node=16225020011&ref_=nav_shopall-export_nav_mw_sbd_intl_girlfasion
男童时尚 /s/browse?_encoding=UTF8&node=16225021011&ref_=nav_shopall-export_nav_mw_sbd_intl_boyfasion
健康和家居用品 /s/browse?_encoding=UTF8&node=16225010011&ref_=nav_shopall-export_nav_mw_sbd_intl_health
家居厨房用品 /s/browse?_encoding=UTF8&node=16225011011&ref_=nav_shopall-export_nav_mw_sbd_intl_kitchen
工业科技 /s/browse?_encoding=UTF8&node=16225012011&ref_=nav_shopall-export_nav_mw_sbd_intl_industrial
行李箱包 /s/browse?_encoding=UTF8&node=16225017011&ref_=nav_shopall-export_nav_mw_sbd_intl_luggage
电影和电视 /s/browse?_encoding=UTF8&node=2625373011&ref_=nav_shopall-export_nav_mw_sbd_intl_movies
宠物用品 /s/browse?_encoding=UTF8&node=16225013011&ref_=nav_shopall-export_nav_mw_sbd_intl_pet
软件 /s/browse?_encoding=UTF8&node=16225008011&ref_=nav_shopall-export_nav_mw_sbd_intl_software
运动与户外用品 /s/browse?_encoding=UTF8&node=16225014011&ref_=nav_shopall-export_nav_mw_sbd_intl_sports
工具与家居装饰 /s/browse?_encoding=UTF8&node=256643011&ref_=nav_shopall-export_nav_mw_sbd_intl_tools
玩具与游戏 /s/browse?_encoding=UTF8&node=16225015011&ref_=nav_shopall-export_nav_mw_sbd_intl_toys
视频游戏 /s/browse?_encoding=UTF8&node=16225016011&ref_=nav_shopall-export_nav_mw_sbd_intl_video_games
销售和优惠 /s/browse?_encoding=UTF8&node=15529609011&ref_=nav_shopall-export_nav_mw_sbd_intl_sales
源码：
#! /usr/bin/env python
# # -*- coding: utf-8 -*-
import requests
import re
import json
headers = {
'User-Agent': '',
'Accept-Language': 'zh-CN'
}
req = requests.get("https://www.amazon.com/", headers=headers, verify=False)
content = req.text
keyword = 'wishlistContent'
# 解析列表
def handle(s, keyword):
if s.find(keyword) < 0:
return False
b = s.find('data({') + 5
e = s.rfind('}); });') + 1
j = json.loads(s[b:e])
items = j['shopAllContent']['template']['data']['items']
for item in items:
print(item['text'], item['url'])
return True
pattern = re.compile(r']*>((?!).)*', re.DOTALL)
pos = 0
while True:
match = pattern.search(content, pos)
if not match:
break
s = match.start()
e = match.end()
if handle(content[s:e], keyword):
break
pos = e