Finished data import

This commit is contained in:
Rémi Oudin 2018-02-25 11:49:11 +01:00
parent bc7348f677
commit ee32e5385b
4 changed files with 171 additions and 57 deletions

27
data/events.json Normal file
View file

@ -0,0 +1,27 @@
[
{
"name" : "Atelier Anti-Pub",
"date" : "07/03/2018 19:00 UTC",
"place" : {
"name" : "Centre Social Autogéré Vaydom",
"address" : "37 rue Marceau, Ivry-sur-Seine",
"lat" : "48.81787",
"lon" : "2.38032"
}
},
{
"name" : "Rassemblement de soutien pour Bure",
"date" : "27/02/2018 17:00 UTC",
"place" : {
"name" : "Place Saint-Michel",
"address" : "Place Saint-Michel, 75005 Paris",
"lat" : "48.85374",
"lon" : "2.34455"
}
},
{
"name" : "Création d'un serveur mail",
"date" : "15/02/2018 12:00 UTC",
"place" : "La Mutinerie"
}
]

View file

@ -1,93 +1,93 @@
[ [
"website": { {
"name":"emmaclit", "name":"emmaclit",
"url":"https://emmaclit.com/", "url":"https://emmaclit.com/",
"keywords": [ "keywords": [
"keyword":"Charge mentale", {"keyword":"Charge mentale"},
"keyword":"Un autre regard", {"keyword":"Un autre regard"},
"keyword":"Un petit poutou", {"keyword":"Un petit poutou"},
"keyword":"solidarité", {"keyword":"solidarité"},
"keyword":"dédicace" {"keyword":"dédicace"}
}, ],
"notable_pages": [ "notable_pages": [
"webpage": "https://emmaclit.com/2017/05/09/repartition-des-taches-hommes-femmes/", {"webpage": "https://emmaclit.com/2017/05/09/repartition-des-taches-hommes-femmes/"},
"webpage": "https://emmaclit.com/2016/12/01/une-famille-parmi-dautres/", {"webpage": "https://emmaclit.com/2016/12/01/une-famille-parmi-dautres/"},
"webpage": "https://emmaclit.com/2017/09/11/travaille-pourquoi/" {"webpage": "https://emmaclit.com/2017/09/11/travaille-pourquoi/"}
] ]
}, },
"website": { {
"name":"paris-luttes info", "name":"paris-luttes info",
"url":"https//paris-luttes.info/", "url":"https//paris-luttes.info/",
"keywords": [ "keywords": [
"keyword":"manifestations", {"keyword":"manifestations"},
"keyword":"solidarité immigré·e·s", {"keyword":"solidarité immigré·e·s"},
"keyword":"grève salariés", {"keyword":"grève salariés"},
"keyword":"prison", {"keyword":"prison"},
"keyword":"violence policère" {"keyword":"violence policère"}
], ],
"notable_pages": [ "notable_pages": [
"webpage": "https://paris-luttes.info/-analyse-et-reflexion-?lang=fr", {"webpage": "https://paris-luttes.info/-analyse-et-reflexion-?lang=fr"},
"webpage": "https://paris-luttes.info/comment-publier-sur-paris-luttes-134?lang=fr", {"webpage": "https://paris-luttes.info/comment-publier-sur-paris-luttes-134?lang=fr"},
"webpage": "https://paris-luttes.info/pourquoi-et-comment-utiliser-tor-9013?lang=fr" {"webpage": "https://paris-luttes.info/pourquoi-et-comment-utiliser-tor-9013?lang=fr"}
] ]
}, },
"website": { {
"name":"zad nadir", "name":"zad nadir",
"url":"http://zad.nadir.org/", "url":"http://zad.nadir.org/",
"keywords": [ "keywords": [
"keyword":"Écologie", {"keyword":"Écologie"},
"keyword":"opération césar", {"keyword":"opération césar"},
"keyword":"expulsion vinci", {"keyword":"expulsion vinci"},
"keyword":"adresse", {"keyword":"adresse"},
"keyword":"la wardine", {"keyword":"la wardine"},
"keyword":"route des chicanes", {"keyword":"route des chicanes"},
"keyword":"opposition à l'aéroport Grand Ouest" {"keyword":"opposition à l'aéroport Grand Ouest"}
], ],
"notable_pages": [ "notable_pages": [
"webpage": "http://zad.nadir.org/spip.php?article86&lang=fr", {"webpage": "http://zad.nadir.org/spip.php?article86&lang=fr"},
"webpage": "http://zad.nadir.org/spip.php?article515&lang=fr", {"webpage": "http://zad.nadir.org/spip.php?article515&lang=fr"},
"webpage": "http://zad.nadir.org/spip.php?rubrique71", {"webpage": "http://zad.nadir.org/spip.php?rubrique71"},
"webpage": "https://zad.nadir.org/spip.php?rubrique70" {"webpage": "https://zad.nadir.org/spip.php?rubrique70"}
] ]
}, },
"website": { {
"name":"Fnac", "name":"Fnac",
"url":"https://www.fnac.com/", "url":"https://www.fnac.com/",
"keywords": [ "keywords": [
"keyword":"smartphone", {"keyword":"smartphone"},
"keyword":"SAV", {"keyword":"SAV"},
"keyword":"Macbook", {"keyword":"Macbook"},
"keyword":"TV", {"keyword":"TV"},
"keyword":"PC Gaming", {"keyword":"PC Gaming"},
"keyword":"DVD", {"keyword":"DVD"},
"keyword":"Home Cinema Philips", {"keyword":"Home Cinema Philips"},
"keyword":"Billeterie" {"keyword":"Billeterie"}
], ],
"notable_pages": [ "notable_pages": [
"webpage": "https://www.fnac.com/Informatique/shi48966/w-4#bl=MMinfo", {"webpage": "https://www.fnac.com/Informatique/shi48966/w-4#bl=MMinfo"},
"webpage": "https://www.fnac.com/Service/default.aspx#bl=footer", {"webpage": "https://www.fnac.com/Service/default.aspx#bl=footer"},
"webpage": "https://www.fnac.com/Ventes-Flash/shi42245/w-4#bl=marktlink1", {"webpage": "https://www.fnac.com/Ventes-Flash/shi42245/w-4#bl=marktlink1"},
"webpage": "https://www.fnac.com/Home-cinema-barre-de-son-et-enceinte-TV/shi474916/w-4#bl=MMtvh" {"webpage": "https://www.fnac.com/Home-cinema-barre-de-son-et-enceinte-TV/shi474916/w-4#bl=MMtvh"}
] ]
}, },
"website": { {
"name":"Sea Shepherd", "name":"Sea Shepherd",
"url":"https://www.seashepherd.fr/", "url":"https://www.seashepherd.fr/",
"keywords": [ "keywords": [
"keyword":"pirates", {"keyword":"pirates"},
"keyword":"Phoques", {"keyword":"Phoques"},
"keyword":"Paul Watson", {"keyword":"Paul Watson"},
"keyword":"harponnage", {"keyword":"harponnage"},
"keyword":"seal", {"keyword":"seal"},
"keyword":"Chasse aux dauphins", {"keyword":"Chasse aux dauphins"},
"keyword":"participation", {"keyword":"participation"},
"keyword":"boutique" {"keyword":"boutique"}
], ],
"notable_pages": [ "notable_pages": [
"webpage": "http://www.seashepherd.fr/index.php/qui-sommes-nous", {"webpage": "http://www.seashepherd.fr/index.php/qui-sommes-nous"},
"webpage": "http://nyamba.seashepherd.info/", {"webpage": "http://nyamba.seashepherd.info/"},
"webpage": "http://seashepherd-shop.com/en/", {"webpage": "http://seashepherd-shop.com/en/"},
"webpage": "http://seashepherd.fr/index.php/qui-sommes-nous/sea-shepherd-france" {"webpage": "http://seashepherd.fr/index.php/qui-sommes-nous/sea-shepherd-france"}
] ]
} }
] ]

View file

@ -0,0 +1,41 @@
""" Small module that import events into the database.
"""
import json
from datetime import datetime
from django.core.management.base import BaseCommand
from django.db import models
from profiles.models import Place, Event
def import_file(filename):
with open(filename, mode='r') as file:
data = json.load(file)
for event in data:
import_event(event)
def import_place(_place):
place = Place(
name=_place.get("name", ""),
address=_place.get("address", ""),
lat=float(_place.get("lat", 0)),
lon=float(_place.get("lon", 0))
)
place.save()
return place
def import_event(_event):
if isinstance(_event["place"], str):
place = Place.objects.get(name=_event["place"])
else:
place = import_place(_event["place"])
event = Event(
name=_event.get("name", ""),
date=datetime.strptime(_event.get("date", "01/01/1970 00:00 UTC"), "%d/%m/%Y %H:%M %Z"),
place=place
)
#print(event)
event.save()
class Command(BaseCommand):
def handle(self, *args, **kwargs):
import_file("data/events.json")

View file

@ -0,0 +1,46 @@
""" Small module that import events into the database.
"""
import json
from datetime import datetime
from django.core.management.base import BaseCommand
from django.db import models
from profiles.models import Webpage, Website, Keyword
def import_file(filename):
with open(filename, mode='r') as file:
data = json.load(file)
for website in data:
import_website(website)
def import_website(_website):
keywords = []
webpages = []
for keyword in _website.get("keywords", []):
new_keyword = Keyword(
text=keyword.get("keyword", "")
)
new_keyword.save()
keywords.append(new_keyword)
for webpage in _website.get("notable_pages",[]):
new_webpage = Webpage(
url=webpage.get("keyword", "")
)
new_webpage.save()
webpages.append(new_webpage)
website = Website(
name=_website.get("name", ""),
url=_website.get("url", ""),
)
website.save()
for keyword in keywords:
website.keywords.add(keyword)
for webpage in webpages:
website.notable_pages.add(webpage)
print(website)
#website.save()
class Command(BaseCommand):
def handle(self, *args, **kwargs):
import_file("data/website.json")