爬取动态分页数据案例

#url— http://kuaixun.eastmoney.com/ssgs.html
import requests
import json
import re
headers = {
   ‘User-Agent‘:‘Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.12 Safari/537.36‘
}
# 通过对网站的分析发现翻页时，有ajax数据包分析数据包的url发现，翻页改变url中的参数，共20页
url = "http://newsapi.eastmoney.com/kuaixun/v1/getlist_103_ajaxResult_50_{}_.html"
for i in range(1,21):
    url2 = url.format(i)
    page_text = requests.get(url=url2, headers=headers).text
    page_str = re.findall(‘\{.*\}‘,page_text)[0]
    page_dic = json.loads(page_str)
    page_list = page_dic["LivesList"]
    
    content_list = []
    for dic in page_list:
        content = dic["digest"]
        content_list.append(content)
    
    # 分页存储
    with open(f"./第{i}页新闻","w",encoding=‘utf-8‘) as fp:
        fp.write(‘\n‘.join(content_list))
    print(f"第{i}页新闻存储成功")