diff --git a/crawl/crawl.py b/crawl/crawl.py
index 46c7707..ab3b8e1 100644
--- a/crawl/crawl.py
+++ b/crawl/crawl.py
@@ -117,6 +117,8 @@ class WebsiteScheduler(metaclass=WebsiteSchedulerMeta):
         except Exception as e:
             print(e)
             raise e
+        if not self.robot_parser.default_entry:
+            self.dead = True
         if not self.dead:
             delay = self.robot_parser.crawl_delay(settings.USER_AGENT)
             if delay is None: