网络爬虫爬取淘宝商品页面数据

import requests
import re
from bs4 import BeautifulSoup
import bs4

‘‘‘
数据线起始页https://s.taobao.com/search?q=%E4%B9%A6%E5%8C%85&imgfile=&commend=all&ssid=s5-e&search_type=item&sourceId=tb.index&spm=a21bo.2017.201856-taobao-item.1&ie=utf8&initiative_id=tbindexz_20170306
数据线第二页https://s.taobao.com/search?q=%E4%B9%A6%E5%8C%85&imgfile=&commend=all&ssid=s5-e&search_type=item&sourceId=tb.index&spm=a21bo.2017.201856-taobao-item.1&ie=utf8&initiative_id=tbindexz_20170306&bcoffset=3&ntoffset=3&p4ppushleft=1%2C48&s=44
数据线第三页https://s.taobao.com/search?q=%E4%B9%A6%E5%8C%85&imgfile=&commend=all&ssid=s5-e&search_type=item&sourceId=tb.index&spm=a21bo.2017.201856-taobao-item.1&ie=utf8&initiative_id=tbindexz_20170306&bcoffset=0&ntoffset=6&p4ppushleft=1%2C48&s=88
‘‘‘
def get_html_text(url):
    try:
        r=requests.get(url,timeout=30)
        r.raise_for_status()
        r.encoding=r.apparent_encoding
        return r.text
    except :
        return ‘该网页请求连接失败‘

#通过脚本语言编写的html代码，不是完整的html语言，直接搜索相对比较简单
#正则表达式原生字符串你还没有理解，最小匹配原则  视频时间11:20
def parse_page(list_info,html):
    try:
        list_price=re.findall(r‘"view_price":"[\d.]*"‘,html)
        list_title=re.findall(r‘"raw_title":".*?"‘,html)
        list_location=re.findall(r‘"item_loc":".*?"‘,html)
        #list_num_payment=re.findall(r‘"view_sales":"u"‘,html)
        for i in range(len(list_price)):
            price=eval(list_price[i].split(‘:‘)[1])
            title=eval(list_title[i].split(‘:‘)[1])
            location=eval(list_location[i].split(‘:‘)[1])
            #num_payment=eval(list_num_payment.split(‘:‘)[1])
            #list_info.append([price,num_payment,location,title])
            list_info.append([price,location,title])
    except :
        print(‘解析网页出现异常‘)

def print_goods_info(list_info):
    #tplt=‘{:4}\t{:8}\t{:8}\t{:12}\t{:20}\t‘
    tplt=‘{:4}\t{:8}\t{:12}\t{:20}\t‘
    #print(tplt.format(‘序号‘,‘商品价格‘,‘付款人数‘,‘发货地址‘,‘商品名称‘))
    print(tplt.format(‘序号‘,‘商品价格‘,‘发货地址‘,‘商品名称‘))
    count=0
    for goods in list_info:
        count+=1
        #print(tplt.format(count,goods[0],goods[1],goods[2],goods[3]))
        print(tplt.format(count,goods[0],goods[1],goods[2]))

if __name__ == ‘__main__‘:
    goods=‘书包‘
    depth=2
    start_url=‘https://s.taobao.com/search?q=‘+goods
    list_info=[]
    for i in range(depth):
        try:
            url=start_url+‘&s=‘+str(44*i)
            html=get_html_text(url)
            parse_page(list_info,html)
        except:
            continue        #如果某一个页面出现了问题，则会跳过该页面的解析，而不会影响程序的整体运行
    print_goods_info(list_info)