diff --git a/crawl/crawl.py b/crawl/crawl.py index 28564f8..8754530 100644 --- a/crawl/crawl.py +++ b/crawl/crawl.py @@ -289,7 +289,11 @@ async def async_crawler(url, out_tree, crawled, user_agent, session, parent): crawled.add(simplify_url(url)) parsed_url = urlparse(url) print("Crawling {}".format(url)) - html = await PageGetter(session, url, user_agent).get(ssl=False) + try: + with async_timeout.timeout(3): + html = await PageGetter(session, url, user_agent).get(ssl=False) + except asyncio.TimeoutError: + return new_tasks = []