怎么实现抓取同行网站访客号码

importrequests

import json

importpandasas pd

import time

# getRtVisitor.json

session= requests.Session() # 创建一个session对象

headers = {undefined

'accept':'*/*',

'accept-encoding':'gzip, deflate, br',

'accept-language':'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',

'cookie':'t=4a09a3c800e9591a2c198b124d80e02a; cna=G2veFsG5YX0CAQ5pIwfTWHFB; lgc=\u5927\u5468\u4F73; tracknick=\u5927\u5468\u4F73; tg=0; _euacm_ac_l_uid_=1639181234; 1639181234_euacm_ac_c_uid_=1639181234; 1639181234_euacm_ac_rs_uid_=1639181234; _portal_version_=new; cc_gray=1; thw=cn; mt=ci=84_1; _euacm_ac_rs_sid_=155550734; enc=Rk6EFG1Zi/5sBqHEJADmGhcHg/8HVbpOVQDb72MK8zmamaAE7C23fINFdlW5BT/NfkRKZLPkB8gKw+j0cK0hig==; hng=CN|zh-CN|CNY|156; cookie2=1a5b17269a2a570500957e7dcce65c45; _tb_token_=35e1e66785eb7; _samesite_flag_=true; sgcookie=Ep88649ag1i3VPa4P6akw; unb=1639181234; uc3=vt3=F8dBxd9nWWLAbiRWGZE=&lg2=UtASsssmOIJ0bQ==&nk2=1z8nwQRn&id2=Uoe3fo1fFq44Zg==; csg=aa9633c3; cookie17=Uoe3fo1fFq44Zg==; dnk=\u5927\u5468\u4F73; skt=69bf89596c5fe0d1; existShop=MTU4NDU4MTc0MQ==; uc4=nk4=0@1fDckZcjfHBEZVI1NQCO3RY=&id4=0@UO+xIxkhtoiLbBQVUeEnpSn1KHbH; _cc_=U+GCWk/7og==; _l_g_=Ug==; sg=佳4e; _nk_=\u5927\u5468\u4F73; cookie1=BYXJ7V2Aq8c/nceEFKLACXxZ7qw2VwJYwgQbxY/eb6A=; tfstk=cKiNBsfOWhKwV3THPur4G7Z13THOZAHinMy7S4xicwaWPJ4GiQfYKvNUL-NJxPf..; v=0; _m_h5_tk=8b49ef5c39700daef25a92f8c520cc65_1584590383214; _m_h5_tk_enc=4d9374ef7e4743b408cb6247793448be; uc1=cookie14=UoTUPvXUU0lRTg==&lng=zh_CN&cookie16=Vq8l+KCLySLZMFWHxqs8fwqnEw==&existShop=true&cookie21=UtASsssmfavZrexPkAwn7A==&tag=8&cookie15=W5iHLLyFOGW7aA==&pas=0; XSRF-TOKEN=673a0ad9-3e96-4fb2-bbd9-af3ffa50a9bc; JSESSIONID=B0A57AC70F717D1231606D512F674ECB; l=dBjCTtrcQuukdAdWBOfgqK_ahxbOrIRbzsPPlS9CCICP9O1wrJp1WZ4V8-8eCnGN36J6R3WhGKB3BqLTFPaOhtikBBrsDOsCydTBR; isg=BAoK5gRhYDd8MOy0NcE2jJ5BW_Cs-45VYmG9hZRDct30R6gBfIkRZ2NxV7ubtwbt',

'referer':'https://sycm.taobao.com/ipoll/visitor.htm?spm=a21ag.7622617.LeftMenu.d181.758a1be9MfheaI',

'sec-fetch-mode':'cors',

'sec-fetch-site':'same-origin',

'user-agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36'

}

def loadPage(page):

strPage=str(page)

strToken='6f5921e86'

strLastPara='1584581855701'

url = 'https://sycm.taobao.com/ipoll/live/visitor/getRtVisitor.json?device=2&limit=20&page=' strPage '&token=' strToken '&type=Y&_=' strLastPara

# https://sycm.taobao.com/ipoll/live/visitor/getRtVisitor.json?device=2&limit=20&page=2&token=97890b4c6&type=Y&_=1584409617907

req = session.get(url, headers=headers) #发起get请求

text = req.text

with open(str(page) ".txt", "w") as f:

f.write(text)

return json.loads(text)

def GetValueFromDic(item, key, defaultValue=None):

if key in item:

return item[key]

if defaultValue is None:

return ''

return defaultValue

def appendList(json_res):

list_res = json_res['data']['data']['list']

for item in list_res:

arr=[]

if item['pageTypeId'] == 5:# 商品分类页

arr.append(item['visitTime'])

arr.append(其它来源)

arr.append('')# 搜索关键字

arr.append(商品分类页)

arr.append('')# 访客位置

arr.append(item['oid'])

else:

arr.append(GetValueFromDic(item, 'visitTime'))

arr.append(GetValueFromDic(item, 'srcGrpName', 其它来源)

arr.append(GetValueFromDic(item, 'preSeKeyword'))

arr.append(GetValueFromDic(item, 'title'))

arr.append(GetValueFromDic(item, 'cityName'))

arr.append(GetValueFromDic(item, 'oid'))

list_excel.append(arr)

# list集合

list_excel=[]

# 数据源

json_res = loadPage(1)

# 解析数据

totalPage = json_res['data']['data']['totalPage']

curPage = json_res['data']['data']['page']

print总页数： totalPage, 现在的页面： curPage)

# 追加数据

appendList(json_res)

# 循环下载

for i in range(totalPage-1):

page = i 2

print(当前页面 page)

json_res = loadPage(page)

appendList(json_res)

# 转换为DataFrame

#df = pd.DataFrame(list_excel, columns=['visitTime', 'srcGrpName', 'preSeKeyword', 'title', 'cityName', 'oid'])

df = pd.DataFrame(list_excel, columns=访问时间， 入店来源， 搜索关键字， 被访页面， 访客位置， 访客编号

# 保存到本地excel

curTime = time.strftie('%Y-%m-%d %H%M%S',time.localtime(time.time()))

df.to_excel('访客-'+curTime+'.xlsx', index=False)

mark

pageTypeId：访客访问页面的类型

具体看这里：item['pageTypeId'] == 5:# 商品分类页

pageTypeId==5，表示用户是从商品分类页来的，或者点了商品分类页页