requests库爬取豆瓣热门国产电视剧数据并保存到本地

 1 import json
 2 import requests
 3 
 4 
 5 class DoubanSpider(object):
 6     """爬取豆瓣热门国产电视剧的数据并保存到本地"""
 7 
 8     def __init__(self):
 9         # url_temp中的start的值是动态的，所以这里用{}替换，方便后面使用format方法
10         self.url_temp = ‘https://movie.douban.com/j/search_subjects?type=tv&tag=%E5%9B%BD%E4%BA%A7%E5%89%A7&sort=recommend&page_limit=20&page_start={}‘
11         self.headers = {
12             ‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36‘,
13         }
14 
15     def pass_url(self, url):  # 发送请求，获取响应
16         print(url)
17         response = requests.get(url, headers=self.headers)
18         return response.content.decode()
19 
20     def get_content_list(self, json_str):  # 提取数据
21         dict_ret = json.loads(json_str)
22         content_list = dict_ret[‘subjects‘]
23         return content_list
24 
25     def save_content_list(self, content_list):  # 保存
26         with open(‘douban.txt‘, ‘a‘, encoding=‘utf-8‘) as f:
27             for content in content_list:
28                 f.write(json.dumps(content, ensure_ascii=False))  # 一部电视剧的信息一行
29                 f.write(‘\n‘)  # 写入换行符进行换行
30         print(‘保存成功！‘)
31 
32 
33     def run(self):  # 实现主要逻辑
34         num = 0
35         while True:
36             # 1. start_url
37             url = self.url_temp.format(num)
38             # 2. 发送请求，获取响应
39             json_str = self.pass_url(url)
40             # 3. 提取数据
41             content_list = self.get_content_list(json_str)
42             # 4. 保存
43             self.save_content_list(content_list)
44             if len(content_list) < 20:
45                 break
46             # 5. 构造下一页url地址，进入循环
47             num += 20  # 每一页有二十条数据
48 
49 
50 if __name__ == ‘__main__‘:
51     douban_spider = DoubanSpider()
52     douban_spider.run()

 1 import json
 2 import requests
 3 
 4 
 5 class DoubanSpider(object):
 6     """爬取豆瓣热门国产电视剧的数据并保存到本地"""
 7     def __init__(self):
 8         # url_temp中的start的值是动态的，所以这里用{}替换，方便后面使用format方法
 9         self.url_temp = ‘https://movie.douban.com/j/search_subjects?type=tv&tag=%E5%9B%BD%E4%BA%A7%E5%89%A7&sort=recommend&page_limit=20&page_start={}‘
10         self.headers = {
11             ‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36‘,
12         }
13         self.num = 0
14 
15     def pass_url(self, url):  # 发送请求，获取响应
16         print(url)
17         response = requests.get(url, headers=self.headers)
18         return response.content.decode()
19 
20     def get_content_list(self, json_str):  # 提取数据
21         dict_ret = json.loads(json_str)
22         content_list = dict_ret[‘subjects‘]
23         return content_list
24 
25     def save_content_list(self, content_list):  # 保存
26         with open(‘douban2.txt‘, ‘a‘, encoding=‘utf-8‘) as f:
27             for content in content_list:
28                 f.write(json.dumps(content, ensure_ascii=False))  # 一部电视剧的信息一行
29                 f.write(‘\n‘)  # 写入换行符进行换行
30         print(‘保存成功！‘)
31 
32 
33     def run(self):  # 实现主要逻辑
34         # 1. start_url
35         url = self.url_temp.format(self.num)
36         # 2. 发送请求，获取响应
37         json_str = self.pass_url(url)
38         # 3. 提取数据
39         content_list = self.get_content_list(json_str)
40         # 4. 保存
41         self.save_content_list(content_list)
42         # 5. 构造下一页url地址，进入循环
43         if len(content_list) == 20:
44             self.num += 20  # 每一页有二十条数据
45             self.run()
46 
47 
48 if __name__ == ‘__main__‘:
49     douban_spider = DoubanSpider()
50     douban_spider.run()