If robots.txt file is invalid, abort mission.

2018-02-23 10:36:14 +01:00 · 2018-02-23 10:36:14 +01:00 · f6da179820
commit f6da179820
parent 0e02f22d08
1 changed files with 2 additions and 0 deletions
--- a/crawl/crawl.py
+++ b/crawl/crawl.py
@ -117,6 +117,8 @@ class WebsiteScheduler(metaclass=WebsiteSchedulerMeta):
        except Exception as e:
            print(e)
            raise e
+        if not self.robot_parser.default_entry:
+            self.dead = True
        if not self.dead:
            delay = self.robot_parser.crawl_delay(settings.USER_AGENT)
            if delay is None: