淘宝商品信息定向爬虫

标签（空格分隔）：python

功能描述

（1）目标：获取淘宝搜索页面信息，提取其中商品的名称和价格
（2）技术路线：Requests-Re

接口描述

（1）搜索接口：https://s.taobao.com/search?q=关键词
（2）翻页接口：第二页?https://s.taobao.com/search?q=关键词&s=44
? ? ?第三页?https://s.taobao.com/search?q=关键词&s=88

获取headers和cookie

（1）使用chrome打开淘宝网站，登陆自己的账户，对某一关键词进行搜索
（2）点击F12，打开开发者工具，其中有四个功能模块：元素（ELements）、控制台（Console）、源代码（Sources），网络（Network）。(在此只用上元素和网络)
（3）点击网络（Network），按F5进行刷新，选择第一个search？开头的。右键打开点击copy，选择copy as curl(bash).
（4）点击进入https://curl.trillworks.com/
（5）将copy as curl(bash)获得的代码黏贴进去即可获得headers和cookie。

具体代码

import requests
import re
def gethtml(url): #定义获取url的函数
    try:
        header = {
            'authority': 's.taobao.com',
            'cache-control': 'max-age=0',
            'upgrade-insecure-requests': '1',
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36',
            'sec-fetch-dest': 'document',
            'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
            'sec-fetch-site': 'same-origin',
            'sec-fetch-mode': 'navigate',
            'sec-fetch-user': '?1',
            'referer': 'https://www.taobao.com/',
            'accept-language': 'zh-CN,zh;q=0.9',
            'cookie': 'cna=4moHFvDPaTICAd9oQObLDEW5; hng=CN%7Czh-CN%7CCNY%7C156; thw=cn; miid=98165971917894791; tk_trace=oTRxOWSBNwn9dPyorMJE%2FoPdY8zfvmw%2Fq5hp3Mv2W1rdhguPHlecTIe16MKxpYMNQ840ADWIKl4HCcNnnW9z4BNHYs0RQnk6SOD5109VJeGaMCGWsbQUljTc5orzQDtHW0ZHPcAkexCv82Pr0%2Bir9%2FNMRIeVuVe9GQSnxqidVTIvFNaLMcuNdECSiNxC%2FQnu%2B9nA4%2B%2BbS5q7UOCAIx97S4fBTwWQOrIOY%2BHXE1J7DW1OK2RQ8w8zxnM%2BNA0VO9KcWjs7gUoldcGOtfei6dMNivZhwKs%3D; t=1b1cdfcdb60bc6af993083500fb98809; cookie2=16882b730c4c2dbd47f2401e2afedde6; v=0; _tb_token_=e85e3054eeef7; _samesite_flag_=true; uc3=id2=UU8A4Kx417Je1g%3D%3D&nk2=tuzu8l4xEM3brAQ%3D&vt3=F8dBxdsbA8er0Nab%2BVU%3D&lg2=U%2BGCWk%2F75gdr5Q%3D%3D; csg=7341820c; lgc=%5Cu6B63%5Cu5728%5Cu8F93%5Cu5165hlb; dnk=%5Cu6B63%5Cu5728%5Cu8F93%5Cu5165hlb; skt=bedd109d72058aa3; existShop=MTU4MTM5MzQ0Nw%3D%3D; uc4=nk4=0%40tDrgQwZMVuegvjAqDNS9hkfkATb7Ig%3D%3D&id4=0%40U22HvQYNO%2F0Nj%2F3PFPtqQzjkpxOi; tracknick=%5Cu6B63%5Cu5728%5Cu8F93%5Cu5165hlb; _cc_=UtASsssmfA%3D%3D; tg=0; mt=ci=35_1; enc=0t2WxzAHAyhblJLweXry%2FHSJKiDB1Ti2KY%2BSibq7pvD8pdeNpkwAlyFz%2BK3eBYxTi1V173bRxt4Cnx1tFJ2EQA%3D%3D; alitrackid=www.taobao.com; lastalitrackid=www.taobao.com; _uab_collina=158139905776795738095785; _m_h5_tk=227e68008dcef8ba04d1a39deac684d9_1581439067593; _m_h5_tk_enc=2387782262126954e8f5b8b7df1c4336; uc1=cookie14=UoTUO8RDL2ugXA%3D%3D; JSESSIONID=D28BF681762BC843F16BADEB8B690D5F; l=dBrGrYzrq4ME0mrtBOCNdm-UOR_tjIRAgul2TiRBi_5dr6L_YL_OoS0W4Fp6cjWft8LB46VFKQp9-etui-y06Pt-g3fPGxDc.; isg=BC8v86H6reV7M6oGWX90St-tvkM51IP27-n7WUG8wh6lkE-SSaZTRjbCFoCu6Ftu',
        }
        t=requests.get(url,headers=header)
        t.encoding=t.apparent_encoding
        t.raise_for_status()
        return t.text
    except:
        print('爬取失败')
        return ''

def perpage(list,html):
    try:
        plt=re.findall(r'\"view_price\":\"\d+\.\d*\"',html)#正则表达式
        llt=re.findall(r'\"raw_title\":\".*?\"',html)
        l=len(plt)
        print('该页面有',l,'个商品')
        for i in range(l):
            price=eval(plt[i].split(':')[1])
            title=eval(llt[i].split(':')[1])
            list.append([price,title])
    except:
        print('解析失败')
        return ''



def printlist(list,num):
    geshihua='{0:^20}\t{1:^20}\t{2:^20}'
    print(geshihua.format('序号','价格','商品名称'))
    count = 0
    for g in list:
        try:
            count = count + 1
            if count<=num:
                print(geshihua.format(count,g[0],g[1]))
        except:
            print('打印错误')
            return ''

def main():
    goods=input('请输入需要查询的商品名称:')
    start_url='https://s.taobao.com/search?q='+goods
    width=eval(input('请输入爬取的页码:'))
    num=eval(input('请输入爬取的商品数量：'))
    list=[]
    for j in range(width):
        try:
            url=start_url+'$S='+str(44*j)
            html = gethtml(url)
            perpage(list, html)
        except:
            continue
    printlist(list,num)

main()

爬取淘宝页面信息

原文：https://www.cnblogs.com/HLBBLOG/p/12304183.html