记一次查深圳磨房百公里徒步照片历程

import requests
from requests.exceptions import ReadTimeout,HTTPError,RequestException
import re
from bs4 import BeautifulSoup
import os
import urllib.request
import urllib.parse
import string
import time

#爬取图片URL信息数据
def CrawImgInfo(req_url):
    resultdata=[]
    headers={"content-type":"text/html; charset=utf-8"}
    try:
        res=requests.get(req_url,headers=headers)
        if res.status_code==200:
            html=BeautifulSoup(res.text,"html.parser")
            #选取所有照片HTML
            imgItems=html.select(".mdui-container-fluid div.mdui-row-gapless div.mdui-col-xs-4")
            for item in imgItems:
                #获取大图信息页面URL
                imginfo_url=item.select("a")[0].attrs["href"]
                resultdata.append(imginfo_url)
        else:
            print("fail")

    except ReadTimeout:
        print("timeout")
    except HTTPError:
        print("HttpError")
    except RequestException:
        print("httpError") 

    return resultdata

#获取图片的URL
def CrawImgUrl(req_url):
    imgurl=""
    headers={"content-type":"text/html; charset=utf-8"}
    try:
        res=requests.get(req_url,headers=headers)
        if res.status_code==200:
            html=BeautifulSoup(res.text,"html.parser")
            #获取大图的URL
            imgurl=html.select(".mdui-container-fluid div.mdui-row-gapless a")[0].attrs["href"]
        else:
            print("fail")

    except ReadTimeout:
        print("timeout")
    except HTTPError:
        print("HttpError")
    except RequestException:
        print("httpError") 
    return imgurl

#写入数据，保存到本地
def SaveImg(filepath,imgurl):
    #URL含中文，需要编码
    imgurl_encode=urllib.parse.quote(imgurl,safe=string.printable)
    try:
        resp=urllib.request.urlopen(imgurl_encode)
        imgname=imgurl.split("/")[-1]
        if resp.code==200:
            img=resp.read()
            imgfilename=filepath+imgname
            with open(imgfilename,"wb") as f:
                f.write(img)
    except urllib.error.HTTPError as e:
        print("The server conldn\‘t fulfill the request.")
    except urllib.error.URLError as e:
        print(‘We failed to reach a server.‘)


def GetData():
    rootUrl="网站地址"
    pageUrl="翻页地址"
    #实际总页数加1
    pageCount=267
    
    savePath="本地存储地址"
    imgCount=0
    for pageIndex in range(1,pageCount):
        req_url=pageUrl+str(pageIndex)
        curpage_imginfos=CrawImgInfo(req_url)
        if len(curpage_imginfos)>0:
            for item in curpage_imginfos:
                imgUrl=CrawImgUrl(rootUrl+item)
                if len(imgUrl)>0:
                    SaveImg(savePath,imgUrl)
                    imgCount=imgCount+1
                    print("图片：%s 下载完成！(下载总数：%s|%s/%s)" %(imgUrl,imgCount,pageIndex,pageCount-1))
                #暂停指定秒数，对服务器要温柔一些
                time.sleep(1)
         

if __name__ == ‘__main__‘:
    GetData()