From ee32e5385b32522d9442f2015c6baa2acfe8ad9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Oudin?= Date: Sun, 25 Feb 2018 11:49:11 +0100 Subject: [PATCH] Finished data import --- data/events.json | 27 +++++ data/website.json | 114 +++++++++--------- profiles/management/commands/import_events.py | 41 +++++++ .../management/commands/import_website.py | 46 +++++++ 4 files changed, 171 insertions(+), 57 deletions(-) create mode 100644 data/events.json create mode 100644 profiles/management/commands/import_events.py create mode 100644 profiles/management/commands/import_website.py diff --git a/data/events.json b/data/events.json new file mode 100644 index 0000000..f8f894c --- /dev/null +++ b/data/events.json @@ -0,0 +1,27 @@ +[ + { + "name" : "Atelier Anti-Pub", + "date" : "07/03/2018 19:00 UTC", + "place" : { + "name" : "Centre Social Autogéré Vaydom", + "address" : "37 rue Marceau, Ivry-sur-Seine", + "lat" : "48.81787", + "lon" : "2.38032" + } + }, + { + "name" : "Rassemblement de soutien pour Bure", + "date" : "27/02/2018 17:00 UTC", + "place" : { + "name" : "Place Saint-Michel", + "address" : "Place Saint-Michel, 75005 Paris", + "lat" : "48.85374", + "lon" : "2.34455" + } + }, + { + "name" : "Création d'un serveur mail", + "date" : "15/02/2018 12:00 UTC", + "place" : "La Mutinerie" + } +] diff --git a/data/website.json b/data/website.json index 9e47f61..38d83ec 100644 --- a/data/website.json +++ b/data/website.json @@ -1,93 +1,93 @@ [ - "website": { + { "name":"emmaclit", "url":"https://emmaclit.com/", "keywords": [ - "keyword":"Charge mentale", - "keyword":"Un autre regard", - "keyword":"Un petit poutou", - "keyword":"solidarité", - "keyword":"dédicace" - }, + {"keyword":"Charge mentale"}, + {"keyword":"Un autre regard"}, + {"keyword":"Un petit poutou"}, + {"keyword":"solidarité"}, + {"keyword":"dédicace"} + ], "notable_pages": [ - "webpage": "https://emmaclit.com/2017/05/09/repartition-des-taches-hommes-femmes/", - "webpage": "https://emmaclit.com/2016/12/01/une-famille-parmi-dautres/", - "webpage": "https://emmaclit.com/2017/09/11/travaille-pourquoi/" + {"webpage": "https://emmaclit.com/2017/05/09/repartition-des-taches-hommes-femmes/"}, + {"webpage": "https://emmaclit.com/2016/12/01/une-famille-parmi-dautres/"}, + {"webpage": "https://emmaclit.com/2017/09/11/travaille-pourquoi/"} ] }, - "website": { + { "name":"paris-luttes info", "url":"https//paris-luttes.info/", "keywords": [ - "keyword":"manifestations", - "keyword":"solidarité immigré·e·s", - "keyword":"grève salariés", - "keyword":"prison", - "keyword":"violence policère" + {"keyword":"manifestations"}, + {"keyword":"solidarité immigré·e·s"}, + {"keyword":"grève salariés"}, + {"keyword":"prison"}, + {"keyword":"violence policère"} ], "notable_pages": [ - "webpage": "https://paris-luttes.info/-analyse-et-reflexion-?lang=fr", - "webpage": "https://paris-luttes.info/comment-publier-sur-paris-luttes-134?lang=fr", - "webpage": "https://paris-luttes.info/pourquoi-et-comment-utiliser-tor-9013?lang=fr" + {"webpage": "https://paris-luttes.info/-analyse-et-reflexion-?lang=fr"}, + {"webpage": "https://paris-luttes.info/comment-publier-sur-paris-luttes-134?lang=fr"}, + {"webpage": "https://paris-luttes.info/pourquoi-et-comment-utiliser-tor-9013?lang=fr"} ] }, - "website": { + { "name":"zad nadir", "url":"http://zad.nadir.org/", "keywords": [ - "keyword":"Écologie", - "keyword":"opération césar", - "keyword":"expulsion vinci", - "keyword":"adresse", - "keyword":"la wardine", - "keyword":"route des chicanes", - "keyword":"opposition à l'aéroport Grand Ouest" + {"keyword":"Écologie"}, + {"keyword":"opération césar"}, + {"keyword":"expulsion vinci"}, + {"keyword":"adresse"}, + {"keyword":"la wardine"}, + {"keyword":"route des chicanes"}, + {"keyword":"opposition à l'aéroport Grand Ouest"} ], "notable_pages": [ - "webpage": "http://zad.nadir.org/spip.php?article86&lang=fr", - "webpage": "http://zad.nadir.org/spip.php?article515&lang=fr", - "webpage": "http://zad.nadir.org/spip.php?rubrique71", - "webpage": "https://zad.nadir.org/spip.php?rubrique70" + {"webpage": "http://zad.nadir.org/spip.php?article86&lang=fr"}, + {"webpage": "http://zad.nadir.org/spip.php?article515&lang=fr"}, + {"webpage": "http://zad.nadir.org/spip.php?rubrique71"}, + {"webpage": "https://zad.nadir.org/spip.php?rubrique70"} ] }, - "website": { + { "name":"Fnac", "url":"https://www.fnac.com/", "keywords": [ - "keyword":"smartphone", - "keyword":"SAV", - "keyword":"Macbook", - "keyword":"TV", - "keyword":"PC Gaming", - "keyword":"DVD", - "keyword":"Home Cinema Philips", - "keyword":"Billeterie" + {"keyword":"smartphone"}, + {"keyword":"SAV"}, + {"keyword":"Macbook"}, + {"keyword":"TV"}, + {"keyword":"PC Gaming"}, + {"keyword":"DVD"}, + {"keyword":"Home Cinema Philips"}, + {"keyword":"Billeterie"} ], "notable_pages": [ - "webpage": "https://www.fnac.com/Informatique/shi48966/w-4#bl=MMinfo", - "webpage": "https://www.fnac.com/Service/default.aspx#bl=footer", - "webpage": "https://www.fnac.com/Ventes-Flash/shi42245/w-4#bl=marktlink1", - "webpage": "https://www.fnac.com/Home-cinema-barre-de-son-et-enceinte-TV/shi474916/w-4#bl=MMtvh" + {"webpage": "https://www.fnac.com/Informatique/shi48966/w-4#bl=MMinfo"}, + {"webpage": "https://www.fnac.com/Service/default.aspx#bl=footer"}, + {"webpage": "https://www.fnac.com/Ventes-Flash/shi42245/w-4#bl=marktlink1"}, + {"webpage": "https://www.fnac.com/Home-cinema-barre-de-son-et-enceinte-TV/shi474916/w-4#bl=MMtvh"} ] }, - "website": { + { "name":"Sea Shepherd", "url":"https://www.seashepherd.fr/", "keywords": [ - "keyword":"pirates", - "keyword":"Phoques", - "keyword":"Paul Watson", - "keyword":"harponnage", - "keyword":"seal", - "keyword":"Chasse aux dauphins", - "keyword":"participation", - "keyword":"boutique" + {"keyword":"pirates"}, + {"keyword":"Phoques"}, + {"keyword":"Paul Watson"}, + {"keyword":"harponnage"}, + {"keyword":"seal"}, + {"keyword":"Chasse aux dauphins"}, + {"keyword":"participation"}, + {"keyword":"boutique"} ], "notable_pages": [ - "webpage": "http://www.seashepherd.fr/index.php/qui-sommes-nous", - "webpage": "http://nyamba.seashepherd.info/", - "webpage": "http://seashepherd-shop.com/en/", - "webpage": "http://seashepherd.fr/index.php/qui-sommes-nous/sea-shepherd-france" + {"webpage": "http://www.seashepherd.fr/index.php/qui-sommes-nous"}, + {"webpage": "http://nyamba.seashepherd.info/"}, + {"webpage": "http://seashepherd-shop.com/en/"}, + {"webpage": "http://seashepherd.fr/index.php/qui-sommes-nous/sea-shepherd-france"} ] } ] diff --git a/profiles/management/commands/import_events.py b/profiles/management/commands/import_events.py new file mode 100644 index 0000000..80088a1 --- /dev/null +++ b/profiles/management/commands/import_events.py @@ -0,0 +1,41 @@ +""" Small module that import events into the database. +""" + +import json +from datetime import datetime +from django.core.management.base import BaseCommand +from django.db import models +from profiles.models import Place, Event + +def import_file(filename): + with open(filename, mode='r') as file: + data = json.load(file) + for event in data: + import_event(event) + +def import_place(_place): + place = Place( + name=_place.get("name", ""), + address=_place.get("address", ""), + lat=float(_place.get("lat", 0)), + lon=float(_place.get("lon", 0)) + ) + place.save() + return place + +def import_event(_event): + if isinstance(_event["place"], str): + place = Place.objects.get(name=_event["place"]) + else: + place = import_place(_event["place"]) + event = Event( + name=_event.get("name", ""), + date=datetime.strptime(_event.get("date", "01/01/1970 00:00 UTC"), "%d/%m/%Y %H:%M %Z"), + place=place + ) + #print(event) + event.save() + +class Command(BaseCommand): + def handle(self, *args, **kwargs): + import_file("data/events.json") diff --git a/profiles/management/commands/import_website.py b/profiles/management/commands/import_website.py new file mode 100644 index 0000000..ae3c10f --- /dev/null +++ b/profiles/management/commands/import_website.py @@ -0,0 +1,46 @@ +""" Small module that import events into the database. +""" + +import json +from datetime import datetime +from django.core.management.base import BaseCommand +from django.db import models +from profiles.models import Webpage, Website, Keyword + +def import_file(filename): + with open(filename, mode='r') as file: + data = json.load(file) + for website in data: + import_website(website) + +def import_website(_website): + keywords = [] + webpages = [] + for keyword in _website.get("keywords", []): + new_keyword = Keyword( + text=keyword.get("keyword", "") + ) + new_keyword.save() + keywords.append(new_keyword) + for webpage in _website.get("notable_pages",[]): + new_webpage = Webpage( + url=webpage.get("keyword", "") + ) + new_webpage.save() + webpages.append(new_webpage) + website = Website( + name=_website.get("name", ""), + url=_website.get("url", ""), + ) + website.save() + for keyword in keywords: + website.keywords.add(keyword) + for webpage in webpages: + website.notable_pages.add(webpage) + print(website) + #website.save() + + +class Command(BaseCommand): + def handle(self, *args, **kwargs): + import_file("data/website.json")