From 854481dbd384581afa0951289ca3d668a8321a31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Oudin?= Date: Sat, 24 Feb 2018 17:21:41 +0100 Subject: [PATCH] Import utilities --- data/keywords.json | 17 ++++ data/place.json | 26 ++++++ data/search_engine.json | 37 ++++++++ data/website.json | 93 +++++++++++++++++++ histories/models.py | 2 +- .../management/commands/import_keywords.py | 20 ++++ profiles/management/commands/import_places.py | 27 ++++++ 7 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 data/keywords.json create mode 100644 data/place.json create mode 100644 data/search_engine.json create mode 100644 data/website.json create mode 100644 profiles/management/commands/import_keywords.py create mode 100644 profiles/management/commands/import_places.py diff --git a/data/keywords.json b/data/keywords.json new file mode 100644 index 0000000..15a94b5 --- /dev/null +++ b/data/keywords.json @@ -0,0 +1,17 @@ +{ + "list": [ + { "keyword" : "gender queer Paris"}, + {"keyword" : "fabriquer masque manif"}, + {"keyword" : "Se protéger en manif"}, + {"keyword" : "Legal team manif France"}, + {"keyword" : "Guide juridique GAV"}, + {"keyword" : "Échec du capitaisme"}, + {"keyword" : "Bienfait du communisme"}, + {"keyword" : "Le comité invisible"}, + {"keyword" : "À nos enfants"}, + {"keyword" : "Squats sur Paris"}, + {"keyword" : "Local facho à Strasbourg"}, + {"keyword" : "Discrimation sexistes, quelles actions ?"}, + {"keyword" : "Pourquoi la lutte des classes"} + ] +} diff --git a/data/place.json b/data/place.json new file mode 100644 index 0000000..cecbac3 --- /dev/null +++ b/data/place.json @@ -0,0 +1,26 @@ +[ + { + "place" : { + "name" : "Zad NDDL", + "address" : "Notre-Dame-des-landes, 44111", + "lat" : "47.3435", + "lon": "-1.7367" + } + }, + { + "place" : { + "name" : "La Mutinerie", + "address" : "176 - 178 rue Saint Martin, 75003 Paris", + "lat" : "48.8625665", + "lon": "2.3522237" + } + }, + { + "place" : { + "name" : "Zad Bure", + "address" : "2 rue de l'Église, 55290 Bure", + "lat" : "48.502", + "lon": "5.351" + } + } +] diff --git a/data/search_engine.json b/data/search_engine.json new file mode 100644 index 0000000..00796ea --- /dev/null +++ b/data/search_engine.json @@ -0,0 +1,37 @@ +[ + { + "searchengine": { + "name":"Google", + "url":"https://google.com/", + "query_pattern": "search?q={}" + } + }, + { + "searchengine": { + "name":"Duckduckgo", + "url":"https://duckduckgo.com/", + "query_pattern":"?q={}" + } + }, + { + "searchengine": { + "name":"Qwant", + "url":"https://www.qwant.com/", + "query_pattern":"?q={}" + } + }, + { + "searchengine": { + "name":"Qwant lite", + "url":"https://lite.qwant.com/", + "query_pattern":"?q={}" + } + }, + { + "searchengine": { + "name":"Framabee", + "url":"https://framabee.org/", + "query_pattern":"?q={}" + } + } +] diff --git a/data/website.json b/data/website.json new file mode 100644 index 0000000..9e47f61 --- /dev/null +++ b/data/website.json @@ -0,0 +1,93 @@ +[ + "website": { + "name":"emmaclit", + "url":"https://emmaclit.com/", + "keywords": [ + "keyword":"Charge mentale", + "keyword":"Un autre regard", + "keyword":"Un petit poutou", + "keyword":"solidarité", + "keyword":"dédicace" + }, + "notable_pages": [ + "webpage": "https://emmaclit.com/2017/05/09/repartition-des-taches-hommes-femmes/", + "webpage": "https://emmaclit.com/2016/12/01/une-famille-parmi-dautres/", + "webpage": "https://emmaclit.com/2017/09/11/travaille-pourquoi/" + ] + }, + "website": { + "name":"paris-luttes info", + "url":"https//paris-luttes.info/", + "keywords": [ + "keyword":"manifestations", + "keyword":"solidarité immigré·e·s", + "keyword":"grève salariés", + "keyword":"prison", + "keyword":"violence policère" + ], + "notable_pages": [ + "webpage": "https://paris-luttes.info/-analyse-et-reflexion-?lang=fr", + "webpage": "https://paris-luttes.info/comment-publier-sur-paris-luttes-134?lang=fr", + "webpage": "https://paris-luttes.info/pourquoi-et-comment-utiliser-tor-9013?lang=fr" + ] + }, + "website": { + "name":"zad nadir", + "url":"http://zad.nadir.org/", + "keywords": [ + "keyword":"Écologie", + "keyword":"opération césar", + "keyword":"expulsion vinci", + "keyword":"adresse", + "keyword":"la wardine", + "keyword":"route des chicanes", + "keyword":"opposition à l'aéroport Grand Ouest" + ], + "notable_pages": [ + "webpage": "http://zad.nadir.org/spip.php?article86&lang=fr", + "webpage": "http://zad.nadir.org/spip.php?article515&lang=fr", + "webpage": "http://zad.nadir.org/spip.php?rubrique71", + "webpage": "https://zad.nadir.org/spip.php?rubrique70" + ] + }, + "website": { + "name":"Fnac", + "url":"https://www.fnac.com/", + "keywords": [ + "keyword":"smartphone", + "keyword":"SAV", + "keyword":"Macbook", + "keyword":"TV", + "keyword":"PC Gaming", + "keyword":"DVD", + "keyword":"Home Cinema Philips", + "keyword":"Billeterie" + ], + "notable_pages": [ + "webpage": "https://www.fnac.com/Informatique/shi48966/w-4#bl=MMinfo", + "webpage": "https://www.fnac.com/Service/default.aspx#bl=footer", + "webpage": "https://www.fnac.com/Ventes-Flash/shi42245/w-4#bl=marktlink1", + "webpage": "https://www.fnac.com/Home-cinema-barre-de-son-et-enceinte-TV/shi474916/w-4#bl=MMtvh" + ] + }, + "website": { + "name":"Sea Shepherd", + "url":"https://www.seashepherd.fr/", + "keywords": [ + "keyword":"pirates", + "keyword":"Phoques", + "keyword":"Paul Watson", + "keyword":"harponnage", + "keyword":"seal", + "keyword":"Chasse aux dauphins", + "keyword":"participation", + "keyword":"boutique" + ], + "notable_pages": [ + "webpage": "http://www.seashepherd.fr/index.php/qui-sommes-nous", + "webpage": "http://nyamba.seashepherd.info/", + "webpage": "http://seashepherd-shop.com/en/", + "webpage": "http://seashepherd.fr/index.php/qui-sommes-nous/sea-shepherd-france" + ] + } +] diff --git a/histories/models.py b/histories/models.py index 2f33e2e..2f52ae2 100644 --- a/histories/models.py +++ b/histories/models.py @@ -7,7 +7,7 @@ import random from math import floor from django.db import models import profiles.models as profiles -from crawl import crawl +#from crawl import crawl from pinocchio.settings import HISTORY_MIN diff --git a/profiles/management/commands/import_keywords.py b/profiles/management/commands/import_keywords.py new file mode 100644 index 0000000..01a6025 --- /dev/null +++ b/profiles/management/commands/import_keywords.py @@ -0,0 +1,20 @@ +""" Small module that import browser fingerprints into the databose, +based on the data listed in https://huit.re/user-agent-json. +""" + +import json +from django.core.management.base import BaseCommand +from django.db import models +from profiles.models import Keyword + +def import_file(filename): + with open(filename, mode='r') as file: + data = json.load(file) + for _keyword in data["list"]: + keyword = Keyword(text=_keyword.get("keyword", "")) + keyword.save() + + +class Command(BaseCommand): + def handle(self, *args, **kwargs): + import_file("data/keywords.json") diff --git a/profiles/management/commands/import_places.py b/profiles/management/commands/import_places.py new file mode 100644 index 0000000..dba02ae --- /dev/null +++ b/profiles/management/commands/import_places.py @@ -0,0 +1,27 @@ +""" Small module that import browser fingerprints into the databose, +based on the data listed in https://huit.re/user-agent-json. +""" + +import json +from django.core.management.base import BaseCommand +from django.db import models +from profiles.models import Place + +def import_file(filename): + with open(filename, mode='r') as file: + data = json.load(file) + for place in data: + import_place(place["place"]) + +def import_place(_place): + place = Place( + name=_place.get("name", ""), + address=_place.get("address", ""), + lat=float(_place.get("lat", 0)), + lon=float(_place.get("lon", 0)) + ) + place.save() + +class Command(BaseCommand): + def handle(self, *args, **kwargs): + import_file("data/place.json")