爬取某鹅数据
原创大约 1 分钟
依然使用DrissionPage来抓取。
'''
基于DrissionPage实现某鹅视频评论数据的爬取
'''
from DrissionPage import ChromiumPage
import json
import base64
# 打开浏览器
page = ChromiumPage()
# 监听响应网址
page.listen.start("trpc.universal_backend_service.page_server_rpc.PageServer/GetPageData?video_appid=1000005&vversion_name=1.0.0&")
page.get('https://v.qq.com/x/cover/75m13e64doz91ul/a0017zmel91.html')
i = 0
# 自动翻页
while True:
# 等待请求加载完毕，并拿到数据
print(f"=========== 开始滚动第{i + 1}次 ===========")
page.scroll.to_bottom()
i += 1
page.wait(2)
if not page.scroll.to_bottom():
break
resp = page.listen.wait().response
jsonp = resp.body
data = jsonp['data']['module_list_datas']
for item in data:
complex_json = item['module_datas'][0]['item_data_lists']['item_datas'][0]
jsonp = json.loads(complex_json['complex_json'])
# decoded_string = base64.b64decode(jsonp)
# 拿到某个用户评论相关的所有数据
# 得到用户名和用户评论信息
username = base64.b64decode(jsonp['user']['base']['name']).decode('utf-8')
comment = base64.b64decode(jsonp['content']['content']).decode('utf-8')
print(f'{username} --------- {comment}')爬出来的某条评论数据的格式如下。
{
"base": {
"id": "100001100538898",
"time": "1616761565",
"seq": "ZTkzNDc2MDYtOTZhMS00OTMwLWIyOGEtMzQyNzMwMmJjN2Ex",
"scene": 1,
"platfrom": 3,
"commentInfo": {
"targetid": "MTAxMjA0NDI2Ng==",
"commentid": "Njc4MTE4OTUwMjk0MjA4MzQ4NQ=="
},
"relation": {
"rootId": "100001100538898",
"parentId": "100001100538898",
"rootUserid": "721047711",
"parentUserid": "721047711"
},
"shareInfo": {},
"msgId": {
"id": "MTAwMDAxMTAwNTM4ODk4"
}
},
"user": {
"base": {
"id": "NzIxMDQ3NzEx",
"type": "USER_TYPE_USER",
"name": "5L2z5YG25aSp5oiQ",
"image": "aHR0cHM6Ly90dnBpYy5ndGltZy5jbi9oZWFkLzY1OTA5MDYxYWFkYjJhMjcyY2Y1ZWU2MGJhNmY1NDg1NDYzYWFmY2Y3MzAzYTliNTAyZjg0ZDUzOTI1ZDRlZWE1ZjNkZDQzMS83MjE=",
"headFrame": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vd3VwbG9hZC94eS9zdGFyX3RoZW1lLzkxalU5Z2tmLnBuZw=="
},
"account": {
"accountType": 1,
"accountId": "NzIxMDQ3NzEx"
},
"degrees": {
"degrees": [{
"name": "5YiY6Z2S5LqRLeaJq+avkjM=",
"icon": "aHR0cHM6Ly9wdXVpLnFwaWMuY24vbWVkaWFfaW1nL2xlbmEvUElDZWZqdm80Xzk2XzMwNi8w",
"url": "aHR0cHM6Ly9tLnYucXEuY29tL2FjdGl2aXR5L2NyZWF0ZS9tZWRhbC9tZWRhbC1kZXRhaWwuaHRtbD9oaWRldGl0bGViYXI9MSZpT1NVc2VXS1dlYlZpZXc9MSZpc19vd25lcj0xJm1lZGFsPW1lZGFsX2lkJTNENjRkMWUxOWM0OTRiMWZjMTRjM2Q4ZmM5JTI2bGV2ZWwlM0QwJm92c2Nyb2xsPTAmdnVpZD03MjEwNDc3MTE=",
"type": "DEGREE_TYPE_COMMUNITY_MEDAL",
"iconAspectRatio": 3.1875
}]
},
"actStatus": {}
},
"content": {
"content": "5omT5q276L+Z5Liq6aKR6YGT55qE57qq5b2V54mH"
},
"relate": {
"relateVideo": {
"videoBase": {
"vid": "bTAxMzR6dnc1ZXU=",
"cid": "NzVtMTNlNjRkb3o5MXVs"
},
"videoAttr": {
"title": "57qq5b2V54mHIOmTgeiFleaymeeah+aZruS6rA==",
"url": "aHR0cDovL3B1dWkucXBpYy5jbi92cGljX2NvdmVyL20wMTM0enZ3NWV1L20wMTM0enZ3NWV1X2h6LmpwZy82NDA=",
"time": 3401,
"size": "120540400",
"ratio": 1.25,
"pioneerTagIds": [""],
"picNew": "aHR0cDovL3B1dWkucXBpYy5jbi92cGljX2NvdmVyL20wMTM0enZ3NWV1L20wMTM0enZ3NWV1X2h6LmpwZy82NDA="
},
"relatedLong": {}
},
"artificialQuality": {}
},
"nums": {
"likes": "2",
"exposures": "362"
},
"ext": {
"msgDatakey": "aWQ9MTAwMDAxMTAwNTM4ODk4JnRpbWU9MTYxNjc2MTU2NSZzY2VuZT0mZnRpZD0yMTAwMDc0Mjc0JnN0YXJpZD0mdGFyZ2V0aWQ9MTAxMjA0NDI2NiZidWlkPSZidXN1YmlkPSZjaT02NzgxMTg5NTAyOTQyMDgzNDg1Jm1zZ190eXBlPTAmbXNnX2lkPTEwMDAwMTEwMDUzODg5OCZ0b3BpY2lkPSZ2aWQ9bTAxMzR6dnc1ZXUmY2lkPTc1bTEzZTY0ZG96OTF1bCZjdmlkPWEwMDE3em1lbDkxJmNjaWQ9NzVtMTNlNjRkb3o5MXVsJmdyYW50a2V5PWJ1aWQlM0QlMjZzdnJ0eXBlJTNEMSUyNmlkJTNEMjEwMDA3NDI3NA=="
},
"interaction": {
"userPraiseStatus": "PRAISE_STATUS_UN_PRAISED",
"userAttitudeInfo": {
"itemList": [{
"attType": "3",
"lottieUrl": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS9iOWU5ZjYxNjI0NjE0MzQ3MTkxLnppcA==",
"lottieMoveTime": "800",
"imageUrl": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS9jMTUyOGQxNjI0NjA2NzA2MDIwLndlYnA=",
"title": "55yf5qOS",
"count": "2",
"imageUrlStatic": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS80OTA5MzIxNjI0NjE0MzIzMTI3LnBuZw==",
"selectedImageUrl": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS9jMTUyOGQxNjI0NjA2NzA2MDIwLndlYnA=",
"selectedImageUrlStatic": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS80OTA5MzIxNjI0NjE0MzIzMTI3LnBuZw=="
}, {
"attType": "4",
"lottieUrl": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS9hOTRlYjkxNjI0NjE0MzAzNzA1LnppcA==",
"lottieMoveTime": "800",
"imageUrl": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS85ZGE5OTMxNjI0NjA2NzIxNTcxLndlYnA=",
"title": "55Sf5rCU",
"imageUrlStatic": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS83MThkMDMxNjI0NjEzOTAyOTk3LnBuZw==",
"selectedImageUrl": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS85ZGE5OTMxNjI0NjA2NzIxNTcxLndlYnA=",
"selectedImageUrlStatic": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS83MThkMDMxNjI0NjEzOTAyOTk3LnBuZw=="
}, {
"attType": "5",
"lottieUrl": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS9iYTQ2ZmYxNjI0NjE0MjY4MTMzLnppcA==",
"lottieMoveTime": "800",
"imageUrl": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS9mNGY5NDYxNjI0NTkxNzAzNzY3LndlYnA=",
"title": "5b+D56KO",
"imageUrlStatic": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS8yNGU5MDkxNjI0NjE0MjQyOTI4LnBuZw==",
"selectedImageUrl": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS9mNGY5NDYxNjI0NTkxNzAzNzY3LndlYnA=",
"selectedImageUrlStatic": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS8yNGU5MDkxNjI0NjE0MjQyOTI4LnBuZw=="
}, {
"attType": "1",
"lottieUrl": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS80ZDIzNWYxNjI0NjE0MjA3NzIyLnppcA==",
"lottieMoveTime": "800",
"imageUrl": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS83YjdlZTUxNjI0NjA2NTYwOTUyLndlYnA=",
"title": "5ZCD55Oc",
"imageUrlStatic": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS9lMTc4ZDUxNjI0NjE0MTg3MjE3LnBuZw==",
"selectedImageUrl": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS83YjdlZTUxNjI0NjA2NTYwOTUyLndlYnA=",
"selectedImageUrlStatic": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS9lMTc4ZDUxNjI0NjE0MTg3MjE3LnBuZw=="
}, {
"attType": "2",
"lottieUrl": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS8wMmY3ZjcxNjI0NjE0MTY4MzU3LnppcA==",
"lottieMoveTime": "800",
"imageUrl": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS84ZGU3ZmUxNjI0NjA2NjI2MjM1LndlYnA=",
"title": "6Iac5ouc",
"imageUrlStatic": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS8zMGNiYzYxNjI0NjE0MTM2MTMzLnBuZw==",
"selectedImageUrl": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS84ZGU3ZmUxNjI0NjA2NjI2MjM1LndlYnA=",
"selectedImageUrlStatic": "aHR0cHM6Ly92ZmlsZXMuZ3RpbWcuY24vdnVwbG9hZC8yMDIxMDYyNS8zMGNiYzYxNjI0NjE0MTM2MTMzLnBuZw=="
}]
}
},
"forwardInfo": {},
"gradeInfo": {
"detail": {}
},
"area": {}
}感谢支持
更多内容，请移步《超级个体》。