If robots.txt file is invalid, abort mission.

This commit is contained in:
Rémi Oudin 2018-02-23 10:36:14 +01:00
parent 0e02f22d08
commit f6da179820

View file

@ -117,6 +117,8 @@ class WebsiteScheduler(metaclass=WebsiteSchedulerMeta):
except Exception as e:
print(e)
raise e
if not self.robot_parser.default_entry:
self.dead = True
if not self.dead:
delay = self.robot_parser.crawl_delay(settings.USER_AGENT)
if delay is None: