tobast/mpri-webdam
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add a timeout to a single page retrieval

Browse source
This commit is contained in:
Théophile Bastian 2018-02-26 15:42:36 +01:00
parent e140d4a8a7
commit 67ad232533
1 changed files with 5 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
crawl/crawl.py
View file

@ -289,7 +289,11 @@ async def async_crawler(url, out_tree, crawled, user_agent, session, parent):
crawled.add(simplify_url(url)) crawled.add(simplify_url(url))
parsed_url = urlparse(url) parsed_url = urlparse(url)
print("Crawling {}".format(url)) print("Crawling {}".format(url))
try:
with async_timeout.timeout(3):
html = await PageGetter(session, url, user_agent).get(ssl=False) html = await PageGetter(session, url, user_agent).get(ssl=False)
except asyncio.TimeoutError:
return
new_tasks = [] new_tasks = []