# 导入包
import requests
import numpy as np
import pandas as pd
import re 
import time
import json
from faker import Factory


def get_weibo_comment(ids, mid, max_page, max_id=0, max_id_type=0):
    """
    功能：获取指定微博的评论数据，数据接口由chrome切换到手机端抓包获取。
    注意事项：此程序每次获取的数量有限制，每次获取之后隔5分钟再抓取即可
    """
    max_id = max_id
    max_id_type = max_id_type

    # 存储数据
    df_all = pd.DataFrame()

    for i in range(1, max_page):
        # 打印进度
        print(‘我正在获取第{}页的评论信息‘.format(i))

        # 获取URL
        url = ‘https://m.weibo.cn/comments/hotflow?id={}&mid={}&max_id={}&max_id_type={}‘.format(ids, mid, max_id, max_id_type)

        # 添加headers
        headers = {
            ‘User-Agent‘: Factory().create().user_agent(),
            ‘Referer‘: ‘https://m.weibo.cn/detail/1591254045309‘,
            ‘cookie‘: ‘复制cookie信息‘,
            ‘X-Requested-With‘: ‘XMLHttpRequest‘,
        }

        # 发起请求
        try:
            r = requests.get(url, headers=headers, timeout=3)
        except Exception as e:
            print(e)
            r = requests.get(url, headers=headers, timeout=3)

        if r.status_code==200:
            # 解析数据
            json_data = json.loads(r.text)

            # 获取数据
            comment_data = json_data[‘data‘][‘data‘]

            created_time = [i.get(‘created_at‘) for i in comment_data]
            text = [i.get(‘text‘) for i in comment_data]
            user_id = [i[‘user‘].get(‘id‘) for i in comment_data]
            screen_name = [i[‘user‘].get(‘screen_name‘) for i in comment_data]
            reply_num = [i.get(‘total_number‘) for i in comment_data]
            like_count = [i.get(‘like_count‘) for i in comment_data]

            # max_id
            max_id = json_data[‘data‘][‘max_id‘]
            # max_id_type
            max_id_type = json_data[‘data‘][‘max_id_type‘]

            # 存储数据
            df_one = pd.DataFrame({
                ‘created_time‘: created_time,
                ‘text‘: text,
                ‘user_id‘: user_id,
                ‘screen_name‘: screen_name,
                ‘reply_num‘: reply_num,
                ‘like_count‘: like_count
            })

            # 追加
            df_all = df_all.append(df_one, ignore_index=True)

            # 休眠一秒
            time.sleep(np.random.uniform(2))

        else:
            print(‘解析出错！打印最后一次的值‘, max_id, max_id_type)
            continue

    return df_all

# 运行函数
if __name__ == ‘__main__‘:
    # 获取一条微博评论
    df = get_weibo_comment(ids=‘4511703964943057‘, mid=‘4511703964943057‘, max_page=200)

df.head()  

# 重复值
df = df.drop_duplicates()

# 转换字典
week_transform = {
    ‘Mon‘: ‘星期一‘,
    ‘Tue‘: ‘星期二‘,
    ‘Wed‘: ‘星期三‘,
    ‘Thu‘: ‘星期四‘,
    ‘Fri‘: ‘星期五‘,
    ‘Sat‘: ‘星期六‘,
    ‘Sun‘: ‘星期日‘
}

# 提取星期
df[‘day_week‘] = df[‘created_time‘].str.split(‘ ‘).apply(lambda x:x[0])
df[‘day_week‘] = df[‘day_week‘].map(week_transform) 

# 提取日期时间
df[‘time‘] = df[‘created_time‘].str.split(‘ ‘).map(lambda x:x[-1]+‘-‘+x[1]+‘-‘+x[2]+‘ ‘+x[3])
df[‘time‘] = df.time.str.replace(‘May‘, ‘05‘).str.replace(‘Jun‘, ‘06‘)

# text 字段处理
pattern = ‘<span.*?</span>|<a.*?</a>‘

df[‘text‘] = [re.sub(pattern, ‘‘, i) for i in df[‘text‘]]

# 删除列
df = df.drop([‘created_time‘, ‘user_id‘, ‘screen_name‘], axis=1)  

df.head() 

#如果去摆地摊该做什么生意?
def get_cut_words(content_series):
    # 读入停用词表
    stop_words = [] 

    with open("stop_words.txt", ‘r‘, encoding=‘utf-8‘) as f:
        lines = f.readlines()
        for line in lines:
            stop_words.append(line.strip())

    # 添加关键词
    my_words = [‘胸口碎大石‘, ‘烤冷面‘, ‘贴膜‘, ‘卖衣服‘, ‘套大鹅‘] 
    for i in my_words:
        jieba.add_word(i) 

    # 定义停用词
    my_stop_words = [‘信公号‘, ‘摆地摊‘, ‘摆摊‘, ‘地摊‘, ‘哈哈哈哈‘, ‘手机‘,
                    ‘这是‘, ‘这是哪‘, ‘哈哈哈‘, ‘真的‘, ‘一千‘, ‘专业‘,
                    ‘有人‘, ‘我要‘, ‘那种‘, ‘只能‘, ‘好吃‘, ‘喜欢‘, ‘城管‘,
                    ‘评论‘, ‘卖点‘, ‘有没有‘, ‘秘籍‘,
                    ]
    stop_words.extend(my_stop_words)               

    # 分词
    word_num = jieba.lcut(content_series.str.cat(sep=‘。‘), cut_all=False)

    # 条件筛选
    word_num_selected = [i for i in word_num if i not in stop_words and len(i)>=2]

    return word_num_selected


# 分词
text = get_cut_words(content_series=df.text)

# 获取top10
shengyi_num = pd.Series(text)
num_top10 = shengyi_num.value_counts()[:10]

# 条形图
bar1 = Bar(init_opts=opts.InitOpts(width=‘1350px‘, height=‘750px‘)) 
bar1.add_xaxis(num_top10.index.tolist())
bar1.add_yaxis(‘‘, num_top10.values.tolist()) 
bar1.set_global_opts(title_opts=opts.TitleOpts(title=‘如果去摆地摊该做什么生意-Top10‘),
                     visualmap_opts=opts.VisualMapOpts(max_=150)
                    ) 
bar1.render() 


# 绘制词云图
stylecloud.gen_stylecloud(text=‘ ‘.join(text), 
                          collocations=False,
                          font_path=r‘?C:\Windows\Fonts\msyh.ttc‘,   # 更换为本机的字体
                          icon_name=‘fas fa-pie-chart‘,
                          size=768,
                          output_name=‘如果去摆地摊该做什么生意.png‘)
Image(filename=‘如果去摆地摊该做什么生意.png‘)