发现了一个练习爬虫的网站GlidedSky
过完了第一关,第二关需要爬取1000个网页,爬取到第956个网页时报错:
RecursionError: maximum recursion depth exceeded while calling a Python object
解决方法:
import sys sys.setrecursionlimit(100000)
https://blog.csdn.net/Ren_ger/article/details/85068955
附上我爬取第二关的代码:
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from lxml import etree
import sys
sys.setrecursionlimit(100000)
url=‘http://glidedsky.com/level/web/crawler-basic-2?page=1‘
browser=webdriver.Chrome()
wait=WebDriverWait(browser,10)
browser.get(url)
def index_from_page(page):
print(‘正在爬取第‘,page,‘页‘)
try:
if page<1000:
wait.until(EC.presence_of_element_located((By.XPATH,‘//*[@id="app"]/main/div[1]/div/div‘)))
button=wait.until(EC.element_to_be_clickable((By.XPATH,‘//*[@id="app"]/main/div[1]/ul/li//a[@rel="next"] ‘)))
get_sum(page)
time.sleep(1)
button.click()
index_from_page(page+1)
elif page==1000:
get_sum(page)
except TimeoutException:
index_from_page(page)
n=0
def get_sum(url):
global n
html=browser.page_source
html=etree.HTML(html)
item=html.xpath(‘/html/body/div/main/div[1]/div/div/div/div/text()‘)
for num in item:
num=eval(get_number(num))
n+=num
def get_number(num):
for i in range(2):
num=num.strip(‘ ‘)
num=num.strip(‘\n‘)
return num
a=time.perf_counter()
index_from_page(1)
b=time.perf_counter()
print(‘time:‘,b-a)
原文:https://www.cnblogs.com/fran-py-/p/12329296.html