爬取全本小说内容并进行文本分析

from bs4 import BeautifulSoup
import requests

#获取小说各章节的网页地址
def get_urls():
    req = requests.get(url = aim)
    html = req.text
    div_bf = BeautifulSoup(html)
    div = div_bf.find_all(‘div‘, class_ = ‘listmain‘)
    a_bf = BeautifulSoup(str(div[0]))
    a = a_bf.find_all(‘a‘)
    nums = len(a[15:])                                      #删除开头重复章节
    for each in a[15:]:
        names.append(each.string)
        urls.append(main + each.get(‘href‘))
    return nums

#保存小说到本地
def writer(name, path, text):
    write_flag = True
    with open(path, ‘a‘, encoding=‘utf-8‘) as f:
        f.write(name + ‘\n‘)
        f.writelines(text)
        f.write(‘\n\n‘)

#获取小说内容
def get_contents(target):
    req = requests.get(url = target)
    html = req.text
    bf = BeautifulSoup(html)
    texts = bf.find_all(‘div‘, class_ = ‘showtxt‘)
    texts = texts[0].text.replace(‘\xa0‘*8,‘\n\n‘)
    return texts



main = ‘http://www.biqukan.com/‘
aim = ‘http://www.biqukan.com/30_30398/‘
names = []                                                   #章节名
urls = []                                                    #章节链接
nums = get_urls()                                            #章节数

print(‘小说开始下载：‘)
for i in range(nums):
    writer(names[i], ‘小说.txt‘, get_contents(urls[i]))
print(‘小说下载完成‘)

import re
import jieba
import matplotlib.pyplot as plt
from wordcloud import WordCloud
file=open(‘I:\\大四\\python\\大作业\\小说.txt‘,‘r‘,encoding=‘utf-8‘).readlines()
data=‘‘
for i in file:
    data+=‘ ‘.join(jieba.cut(i))+‘ ‘
my_wordcloud = WordCloud(font_path=‘I:\\大四\\python\\py5\\msyh.ttf‘).generate(data)
plt.imshow(my_wordcloud)
plt.axis("off")
plt.show()
my_wordcloud.to_file(path.join("小说.png"))