Finished data import
This commit is contained in:
parent
bc7348f677
commit
ee32e5385b
4 changed files with 171 additions and 57 deletions
27
data/events.json
Normal file
27
data/events.json
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
[
|
||||
{
|
||||
"name" : "Atelier Anti-Pub",
|
||||
"date" : "07/03/2018 19:00 UTC",
|
||||
"place" : {
|
||||
"name" : "Centre Social Autogéré Vaydom",
|
||||
"address" : "37 rue Marceau, Ivry-sur-Seine",
|
||||
"lat" : "48.81787",
|
||||
"lon" : "2.38032"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name" : "Rassemblement de soutien pour Bure",
|
||||
"date" : "27/02/2018 17:00 UTC",
|
||||
"place" : {
|
||||
"name" : "Place Saint-Michel",
|
||||
"address" : "Place Saint-Michel, 75005 Paris",
|
||||
"lat" : "48.85374",
|
||||
"lon" : "2.34455"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name" : "Création d'un serveur mail",
|
||||
"date" : "15/02/2018 12:00 UTC",
|
||||
"place" : "La Mutinerie"
|
||||
}
|
||||
]
|
||||
|
|
@ -1,93 +1,93 @@
|
|||
[
|
||||
"website": {
|
||||
{
|
||||
"name":"emmaclit",
|
||||
"url":"https://emmaclit.com/",
|
||||
"keywords": [
|
||||
"keyword":"Charge mentale",
|
||||
"keyword":"Un autre regard",
|
||||
"keyword":"Un petit poutou",
|
||||
"keyword":"solidarité",
|
||||
"keyword":"dédicace"
|
||||
},
|
||||
{"keyword":"Charge mentale"},
|
||||
{"keyword":"Un autre regard"},
|
||||
{"keyword":"Un petit poutou"},
|
||||
{"keyword":"solidarité"},
|
||||
{"keyword":"dédicace"}
|
||||
],
|
||||
"notable_pages": [
|
||||
"webpage": "https://emmaclit.com/2017/05/09/repartition-des-taches-hommes-femmes/",
|
||||
"webpage": "https://emmaclit.com/2016/12/01/une-famille-parmi-dautres/",
|
||||
"webpage": "https://emmaclit.com/2017/09/11/travaille-pourquoi/"
|
||||
{"webpage": "https://emmaclit.com/2017/05/09/repartition-des-taches-hommes-femmes/"},
|
||||
{"webpage": "https://emmaclit.com/2016/12/01/une-famille-parmi-dautres/"},
|
||||
{"webpage": "https://emmaclit.com/2017/09/11/travaille-pourquoi/"}
|
||||
]
|
||||
},
|
||||
"website": {
|
||||
{
|
||||
"name":"paris-luttes info",
|
||||
"url":"https//paris-luttes.info/",
|
||||
"keywords": [
|
||||
"keyword":"manifestations",
|
||||
"keyword":"solidarité immigré·e·s",
|
||||
"keyword":"grève salariés",
|
||||
"keyword":"prison",
|
||||
"keyword":"violence policère"
|
||||
{"keyword":"manifestations"},
|
||||
{"keyword":"solidarité immigré·e·s"},
|
||||
{"keyword":"grève salariés"},
|
||||
{"keyword":"prison"},
|
||||
{"keyword":"violence policère"}
|
||||
],
|
||||
"notable_pages": [
|
||||
"webpage": "https://paris-luttes.info/-analyse-et-reflexion-?lang=fr",
|
||||
"webpage": "https://paris-luttes.info/comment-publier-sur-paris-luttes-134?lang=fr",
|
||||
"webpage": "https://paris-luttes.info/pourquoi-et-comment-utiliser-tor-9013?lang=fr"
|
||||
{"webpage": "https://paris-luttes.info/-analyse-et-reflexion-?lang=fr"},
|
||||
{"webpage": "https://paris-luttes.info/comment-publier-sur-paris-luttes-134?lang=fr"},
|
||||
{"webpage": "https://paris-luttes.info/pourquoi-et-comment-utiliser-tor-9013?lang=fr"}
|
||||
]
|
||||
},
|
||||
"website": {
|
||||
{
|
||||
"name":"zad nadir",
|
||||
"url":"http://zad.nadir.org/",
|
||||
"keywords": [
|
||||
"keyword":"Écologie",
|
||||
"keyword":"opération césar",
|
||||
"keyword":"expulsion vinci",
|
||||
"keyword":"adresse",
|
||||
"keyword":"la wardine",
|
||||
"keyword":"route des chicanes",
|
||||
"keyword":"opposition à l'aéroport Grand Ouest"
|
||||
{"keyword":"Écologie"},
|
||||
{"keyword":"opération césar"},
|
||||
{"keyword":"expulsion vinci"},
|
||||
{"keyword":"adresse"},
|
||||
{"keyword":"la wardine"},
|
||||
{"keyword":"route des chicanes"},
|
||||
{"keyword":"opposition à l'aéroport Grand Ouest"}
|
||||
],
|
||||
"notable_pages": [
|
||||
"webpage": "http://zad.nadir.org/spip.php?article86&lang=fr",
|
||||
"webpage": "http://zad.nadir.org/spip.php?article515&lang=fr",
|
||||
"webpage": "http://zad.nadir.org/spip.php?rubrique71",
|
||||
"webpage": "https://zad.nadir.org/spip.php?rubrique70"
|
||||
{"webpage": "http://zad.nadir.org/spip.php?article86&lang=fr"},
|
||||
{"webpage": "http://zad.nadir.org/spip.php?article515&lang=fr"},
|
||||
{"webpage": "http://zad.nadir.org/spip.php?rubrique71"},
|
||||
{"webpage": "https://zad.nadir.org/spip.php?rubrique70"}
|
||||
]
|
||||
},
|
||||
"website": {
|
||||
{
|
||||
"name":"Fnac",
|
||||
"url":"https://www.fnac.com/",
|
||||
"keywords": [
|
||||
"keyword":"smartphone",
|
||||
"keyword":"SAV",
|
||||
"keyword":"Macbook",
|
||||
"keyword":"TV",
|
||||
"keyword":"PC Gaming",
|
||||
"keyword":"DVD",
|
||||
"keyword":"Home Cinema Philips",
|
||||
"keyword":"Billeterie"
|
||||
{"keyword":"smartphone"},
|
||||
{"keyword":"SAV"},
|
||||
{"keyword":"Macbook"},
|
||||
{"keyword":"TV"},
|
||||
{"keyword":"PC Gaming"},
|
||||
{"keyword":"DVD"},
|
||||
{"keyword":"Home Cinema Philips"},
|
||||
{"keyword":"Billeterie"}
|
||||
],
|
||||
"notable_pages": [
|
||||
"webpage": "https://www.fnac.com/Informatique/shi48966/w-4#bl=MMinfo",
|
||||
"webpage": "https://www.fnac.com/Service/default.aspx#bl=footer",
|
||||
"webpage": "https://www.fnac.com/Ventes-Flash/shi42245/w-4#bl=marktlink1",
|
||||
"webpage": "https://www.fnac.com/Home-cinema-barre-de-son-et-enceinte-TV/shi474916/w-4#bl=MMtvh"
|
||||
{"webpage": "https://www.fnac.com/Informatique/shi48966/w-4#bl=MMinfo"},
|
||||
{"webpage": "https://www.fnac.com/Service/default.aspx#bl=footer"},
|
||||
{"webpage": "https://www.fnac.com/Ventes-Flash/shi42245/w-4#bl=marktlink1"},
|
||||
{"webpage": "https://www.fnac.com/Home-cinema-barre-de-son-et-enceinte-TV/shi474916/w-4#bl=MMtvh"}
|
||||
]
|
||||
},
|
||||
"website": {
|
||||
{
|
||||
"name":"Sea Shepherd",
|
||||
"url":"https://www.seashepherd.fr/",
|
||||
"keywords": [
|
||||
"keyword":"pirates",
|
||||
"keyword":"Phoques",
|
||||
"keyword":"Paul Watson",
|
||||
"keyword":"harponnage",
|
||||
"keyword":"seal",
|
||||
"keyword":"Chasse aux dauphins",
|
||||
"keyword":"participation",
|
||||
"keyword":"boutique"
|
||||
{"keyword":"pirates"},
|
||||
{"keyword":"Phoques"},
|
||||
{"keyword":"Paul Watson"},
|
||||
{"keyword":"harponnage"},
|
||||
{"keyword":"seal"},
|
||||
{"keyword":"Chasse aux dauphins"},
|
||||
{"keyword":"participation"},
|
||||
{"keyword":"boutique"}
|
||||
],
|
||||
"notable_pages": [
|
||||
"webpage": "http://www.seashepherd.fr/index.php/qui-sommes-nous",
|
||||
"webpage": "http://nyamba.seashepherd.info/",
|
||||
"webpage": "http://seashepherd-shop.com/en/",
|
||||
"webpage": "http://seashepherd.fr/index.php/qui-sommes-nous/sea-shepherd-france"
|
||||
{"webpage": "http://www.seashepherd.fr/index.php/qui-sommes-nous"},
|
||||
{"webpage": "http://nyamba.seashepherd.info/"},
|
||||
{"webpage": "http://seashepherd-shop.com/en/"},
|
||||
{"webpage": "http://seashepherd.fr/index.php/qui-sommes-nous/sea-shepherd-france"}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
|
|
|||
41
profiles/management/commands/import_events.py
Normal file
41
profiles/management/commands/import_events.py
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
""" Small module that import events into the database.
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db import models
|
||||
from profiles.models import Place, Event
|
||||
|
||||
def import_file(filename):
|
||||
with open(filename, mode='r') as file:
|
||||
data = json.load(file)
|
||||
for event in data:
|
||||
import_event(event)
|
||||
|
||||
def import_place(_place):
|
||||
place = Place(
|
||||
name=_place.get("name", ""),
|
||||
address=_place.get("address", ""),
|
||||
lat=float(_place.get("lat", 0)),
|
||||
lon=float(_place.get("lon", 0))
|
||||
)
|
||||
place.save()
|
||||
return place
|
||||
|
||||
def import_event(_event):
|
||||
if isinstance(_event["place"], str):
|
||||
place = Place.objects.get(name=_event["place"])
|
||||
else:
|
||||
place = import_place(_event["place"])
|
||||
event = Event(
|
||||
name=_event.get("name", ""),
|
||||
date=datetime.strptime(_event.get("date", "01/01/1970 00:00 UTC"), "%d/%m/%Y %H:%M %Z"),
|
||||
place=place
|
||||
)
|
||||
#print(event)
|
||||
event.save()
|
||||
|
||||
class Command(BaseCommand):
|
||||
def handle(self, *args, **kwargs):
|
||||
import_file("data/events.json")
|
||||
46
profiles/management/commands/import_website.py
Normal file
46
profiles/management/commands/import_website.py
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
""" Small module that import events into the database.
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db import models
|
||||
from profiles.models import Webpage, Website, Keyword
|
||||
|
||||
def import_file(filename):
|
||||
with open(filename, mode='r') as file:
|
||||
data = json.load(file)
|
||||
for website in data:
|
||||
import_website(website)
|
||||
|
||||
def import_website(_website):
|
||||
keywords = []
|
||||
webpages = []
|
||||
for keyword in _website.get("keywords", []):
|
||||
new_keyword = Keyword(
|
||||
text=keyword.get("keyword", "")
|
||||
)
|
||||
new_keyword.save()
|
||||
keywords.append(new_keyword)
|
||||
for webpage in _website.get("notable_pages",[]):
|
||||
new_webpage = Webpage(
|
||||
url=webpage.get("keyword", "")
|
||||
)
|
||||
new_webpage.save()
|
||||
webpages.append(new_webpage)
|
||||
website = Website(
|
||||
name=_website.get("name", ""),
|
||||
url=_website.get("url", ""),
|
||||
)
|
||||
website.save()
|
||||
for keyword in keywords:
|
||||
website.keywords.add(keyword)
|
||||
for webpage in webpages:
|
||||
website.notable_pages.add(webpage)
|
||||
print(website)
|
||||
#website.save()
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
def handle(self, *args, **kwargs):
|
||||
import_file("data/website.json")
|
||||
Loading…
Reference in a new issue