Remove duplicated url in history

This search engine works better than the others

README.md

@@ -1,3 +1,6 @@
 # mpri-webdam
 
-Générer tout plein de faux historiques. Parce qu'il faut bien valider ce cours.
+Generate realistic fake browsing histories for borderline and/or activists
+users, to hide real traffic from global surveillance.
+
+Lacks proper documentation at the moment `:(`

data/search_engine.json

@@ -13,6 +13,13 @@
             "query_pattern":"?q={}"
         }
     },
+    {
+		"searchengine": {
+            "name":"Duckduckgo Lite",
+            "url":"https://duckduckgo.com/lite/",
+            "query_pattern":"?q={}"
+        }
+    },
     {
 		"searchengine": {
             "name":"Qwant",

histories/management/commands/gen_history.py

@@ -0,0 +1,16 @@
+from django.core.management.base import BaseCommand
+from profiles import models as profiles
+from histories.models import generate_history
+from datetime import datetime
+
+
+class Command(BaseCommand):
+    ''' Generates an history and prints the related XML '''
+
+    def add_arguments(self, parser):
+        pass
+
+    def handle(self, *args, **kwargs):
+        prof = profiles.Profile.objects.all()[0]
+        history = generate_history(prof, datetime.now())
+        print(history.to_xml_string())

histories/models.py

@@ -5,6 +5,7 @@ interests, keywords...
 
 from collections import namedtuple
 import random
+import asyncio
 from math import floor
 from xml.etree import ElementTree as ET
 from datetime import datetime
@@ -107,15 +108,18 @@ class History(models.Model):
             output += str(entry) + '\n'
         return output
 
-    async def play_histories(self):
+    async def _handler(self):
+        runner = await TorInstance.create(self.return_history(), self.user.browser_fingerprint.serialize_headers())
+        await runner.run()
+        self.played = True
+        self.save()
+
+    def play_histories(self):
         """ Actually plays the history.
         """
-        self.played = True
-        runner = await TorInstance.create(
-            self.return_history(),
-            self.user.browser_fingerprint.serialize_headers())
-        runner.run()
-        self.save()
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        loop.run_until_complete(asyncio.wait([self._handler()]))
 
     def to_xml(self, xml_root=None):
         ''' Exports the current history to xml '''
@@ -136,6 +140,10 @@ class History(models.Model):
         if standalone:
             return xml_root
 
+    def to_xml_string(self):
+        xml = self.to_xml()
+        return ET.tostring(xml)
+
     @staticmethod
     def from_xml(xml_root):
         ''' Loads an history from an XML file '''
@@ -179,7 +187,6 @@ def generate_partial_history(user, t_start):
     timestamp = t_start
     result = []
     basis = generate_first_url(user)
-    result.append(PartialHistoryEntry(basis, timestamp))
     t_start += 5 * random.weibullvariate(1, 1.5)
     crawler = crawl.CrawlingThread(basis)
     crawler.start()

histories/tor_runner.py

@@ -58,7 +58,9 @@ class TorInstance():
     async def run(self):
         """ Runs the Tor Instance on the history.
         """
-        while (self.history[0][1] - dt.datetime.now()).total_seconds >= 10:
+        while (self.history) and (dt.datetime.combine(self.history[0][1],
+                                                      dt.datetime.min.time()) -
+                                  dt.datetime.now()).total_seconds() >= 10:
             print("Sleeping")
             sleep(10)
         while self.history:
@@ -66,8 +68,9 @@ class TorInstance():
             async with async_timeout.timeout(30):
                 await(self.query(item[0]))
                 now = dt.datetime.now()
-                if now <= self.history[0][1]:
-                    sleep((self.history[0][1] - now).total_seconds())
+                print(self.history[0])
+                if now <= dt.datetime.combine(self.history[0][1], dt.datetime.min.time()):
+                    sleep((dt.datetime.combine(self.history[0][1], dt.datetime.min.time()) - now).total_seconds())
 
 
     def create_session(self):