More robust crawling

crawl/crawl.py

@@ -291,6 +291,8 @@ async def async_crawler(url, output_tree, headers=None):
                         if sample_url not in queued
                         and sample_url not in crawled
                     ]
+            else:
+                print("No html received")
     print(crawled)
     output_tree += crawl_tree
 

histories/models.py

@@ -244,6 +244,6 @@ def generate_history(user, start_time):
                     new_line.full_clean()
                     new_line.save()
                 except ValidationError:
-                    pass
+                    continue
 
     return history