tobast/mpri-webdam
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

If robots.txt file is invalid, abort mission.

Browse source
This commit is contained in:
Rémi Oudin 2018-02-23 10:36:14 +01:00
parent 0e02f22d08
commit f6da179820
1 changed files with 2 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
crawl/crawl.py
View file

@ -117,6 +117,8 @@ class WebsiteScheduler(metaclass=WebsiteSchedulerMeta):
except Exception as e: except Exception as e:
print(e) print(e)
raise e raise e
if not self.robot_parser.default_entry:
self.dead = True
if not self.dead: if not self.dead:
delay = self.robot_parser.crawl_delay(settings.USER_AGENT) delay = self.robot_parser.crawl_delay(settings.USER_AGENT)
if delay is None: if delay is None: