Finished data import
This commit is contained in:
parent
bc7348f677
commit
ee32e5385b
4 changed files with 171 additions and 57 deletions
27
data/events.json
Normal file
27
data/events.json
Normal file
|
|
@ -0,0 +1,27 @@
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"name" : "Atelier Anti-Pub",
|
||||||
|
"date" : "07/03/2018 19:00 UTC",
|
||||||
|
"place" : {
|
||||||
|
"name" : "Centre Social Autogéré Vaydom",
|
||||||
|
"address" : "37 rue Marceau, Ivry-sur-Seine",
|
||||||
|
"lat" : "48.81787",
|
||||||
|
"lon" : "2.38032"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name" : "Rassemblement de soutien pour Bure",
|
||||||
|
"date" : "27/02/2018 17:00 UTC",
|
||||||
|
"place" : {
|
||||||
|
"name" : "Place Saint-Michel",
|
||||||
|
"address" : "Place Saint-Michel, 75005 Paris",
|
||||||
|
"lat" : "48.85374",
|
||||||
|
"lon" : "2.34455"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name" : "Création d'un serveur mail",
|
||||||
|
"date" : "15/02/2018 12:00 UTC",
|
||||||
|
"place" : "La Mutinerie"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
@ -1,93 +1,93 @@
|
||||||
[
|
[
|
||||||
"website": {
|
{
|
||||||
"name":"emmaclit",
|
"name":"emmaclit",
|
||||||
"url":"https://emmaclit.com/",
|
"url":"https://emmaclit.com/",
|
||||||
"keywords": [
|
"keywords": [
|
||||||
"keyword":"Charge mentale",
|
{"keyword":"Charge mentale"},
|
||||||
"keyword":"Un autre regard",
|
{"keyword":"Un autre regard"},
|
||||||
"keyword":"Un petit poutou",
|
{"keyword":"Un petit poutou"},
|
||||||
"keyword":"solidarité",
|
{"keyword":"solidarité"},
|
||||||
"keyword":"dédicace"
|
{"keyword":"dédicace"}
|
||||||
},
|
],
|
||||||
"notable_pages": [
|
"notable_pages": [
|
||||||
"webpage": "https://emmaclit.com/2017/05/09/repartition-des-taches-hommes-femmes/",
|
{"webpage": "https://emmaclit.com/2017/05/09/repartition-des-taches-hommes-femmes/"},
|
||||||
"webpage": "https://emmaclit.com/2016/12/01/une-famille-parmi-dautres/",
|
{"webpage": "https://emmaclit.com/2016/12/01/une-famille-parmi-dautres/"},
|
||||||
"webpage": "https://emmaclit.com/2017/09/11/travaille-pourquoi/"
|
{"webpage": "https://emmaclit.com/2017/09/11/travaille-pourquoi/"}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"website": {
|
{
|
||||||
"name":"paris-luttes info",
|
"name":"paris-luttes info",
|
||||||
"url":"https//paris-luttes.info/",
|
"url":"https//paris-luttes.info/",
|
||||||
"keywords": [
|
"keywords": [
|
||||||
"keyword":"manifestations",
|
{"keyword":"manifestations"},
|
||||||
"keyword":"solidarité immigré·e·s",
|
{"keyword":"solidarité immigré·e·s"},
|
||||||
"keyword":"grève salariés",
|
{"keyword":"grève salariés"},
|
||||||
"keyword":"prison",
|
{"keyword":"prison"},
|
||||||
"keyword":"violence policère"
|
{"keyword":"violence policère"}
|
||||||
],
|
],
|
||||||
"notable_pages": [
|
"notable_pages": [
|
||||||
"webpage": "https://paris-luttes.info/-analyse-et-reflexion-?lang=fr",
|
{"webpage": "https://paris-luttes.info/-analyse-et-reflexion-?lang=fr"},
|
||||||
"webpage": "https://paris-luttes.info/comment-publier-sur-paris-luttes-134?lang=fr",
|
{"webpage": "https://paris-luttes.info/comment-publier-sur-paris-luttes-134?lang=fr"},
|
||||||
"webpage": "https://paris-luttes.info/pourquoi-et-comment-utiliser-tor-9013?lang=fr"
|
{"webpage": "https://paris-luttes.info/pourquoi-et-comment-utiliser-tor-9013?lang=fr"}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"website": {
|
{
|
||||||
"name":"zad nadir",
|
"name":"zad nadir",
|
||||||
"url":"http://zad.nadir.org/",
|
"url":"http://zad.nadir.org/",
|
||||||
"keywords": [
|
"keywords": [
|
||||||
"keyword":"Écologie",
|
{"keyword":"Écologie"},
|
||||||
"keyword":"opération césar",
|
{"keyword":"opération césar"},
|
||||||
"keyword":"expulsion vinci",
|
{"keyword":"expulsion vinci"},
|
||||||
"keyword":"adresse",
|
{"keyword":"adresse"},
|
||||||
"keyword":"la wardine",
|
{"keyword":"la wardine"},
|
||||||
"keyword":"route des chicanes",
|
{"keyword":"route des chicanes"},
|
||||||
"keyword":"opposition à l'aéroport Grand Ouest"
|
{"keyword":"opposition à l'aéroport Grand Ouest"}
|
||||||
],
|
],
|
||||||
"notable_pages": [
|
"notable_pages": [
|
||||||
"webpage": "http://zad.nadir.org/spip.php?article86&lang=fr",
|
{"webpage": "http://zad.nadir.org/spip.php?article86&lang=fr"},
|
||||||
"webpage": "http://zad.nadir.org/spip.php?article515&lang=fr",
|
{"webpage": "http://zad.nadir.org/spip.php?article515&lang=fr"},
|
||||||
"webpage": "http://zad.nadir.org/spip.php?rubrique71",
|
{"webpage": "http://zad.nadir.org/spip.php?rubrique71"},
|
||||||
"webpage": "https://zad.nadir.org/spip.php?rubrique70"
|
{"webpage": "https://zad.nadir.org/spip.php?rubrique70"}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"website": {
|
{
|
||||||
"name":"Fnac",
|
"name":"Fnac",
|
||||||
"url":"https://www.fnac.com/",
|
"url":"https://www.fnac.com/",
|
||||||
"keywords": [
|
"keywords": [
|
||||||
"keyword":"smartphone",
|
{"keyword":"smartphone"},
|
||||||
"keyword":"SAV",
|
{"keyword":"SAV"},
|
||||||
"keyword":"Macbook",
|
{"keyword":"Macbook"},
|
||||||
"keyword":"TV",
|
{"keyword":"TV"},
|
||||||
"keyword":"PC Gaming",
|
{"keyword":"PC Gaming"},
|
||||||
"keyword":"DVD",
|
{"keyword":"DVD"},
|
||||||
"keyword":"Home Cinema Philips",
|
{"keyword":"Home Cinema Philips"},
|
||||||
"keyword":"Billeterie"
|
{"keyword":"Billeterie"}
|
||||||
],
|
],
|
||||||
"notable_pages": [
|
"notable_pages": [
|
||||||
"webpage": "https://www.fnac.com/Informatique/shi48966/w-4#bl=MMinfo",
|
{"webpage": "https://www.fnac.com/Informatique/shi48966/w-4#bl=MMinfo"},
|
||||||
"webpage": "https://www.fnac.com/Service/default.aspx#bl=footer",
|
{"webpage": "https://www.fnac.com/Service/default.aspx#bl=footer"},
|
||||||
"webpage": "https://www.fnac.com/Ventes-Flash/shi42245/w-4#bl=marktlink1",
|
{"webpage": "https://www.fnac.com/Ventes-Flash/shi42245/w-4#bl=marktlink1"},
|
||||||
"webpage": "https://www.fnac.com/Home-cinema-barre-de-son-et-enceinte-TV/shi474916/w-4#bl=MMtvh"
|
{"webpage": "https://www.fnac.com/Home-cinema-barre-de-son-et-enceinte-TV/shi474916/w-4#bl=MMtvh"}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"website": {
|
{
|
||||||
"name":"Sea Shepherd",
|
"name":"Sea Shepherd",
|
||||||
"url":"https://www.seashepherd.fr/",
|
"url":"https://www.seashepherd.fr/",
|
||||||
"keywords": [
|
"keywords": [
|
||||||
"keyword":"pirates",
|
{"keyword":"pirates"},
|
||||||
"keyword":"Phoques",
|
{"keyword":"Phoques"},
|
||||||
"keyword":"Paul Watson",
|
{"keyword":"Paul Watson"},
|
||||||
"keyword":"harponnage",
|
{"keyword":"harponnage"},
|
||||||
"keyword":"seal",
|
{"keyword":"seal"},
|
||||||
"keyword":"Chasse aux dauphins",
|
{"keyword":"Chasse aux dauphins"},
|
||||||
"keyword":"participation",
|
{"keyword":"participation"},
|
||||||
"keyword":"boutique"
|
{"keyword":"boutique"}
|
||||||
],
|
],
|
||||||
"notable_pages": [
|
"notable_pages": [
|
||||||
"webpage": "http://www.seashepherd.fr/index.php/qui-sommes-nous",
|
{"webpage": "http://www.seashepherd.fr/index.php/qui-sommes-nous"},
|
||||||
"webpage": "http://nyamba.seashepherd.info/",
|
{"webpage": "http://nyamba.seashepherd.info/"},
|
||||||
"webpage": "http://seashepherd-shop.com/en/",
|
{"webpage": "http://seashepherd-shop.com/en/"},
|
||||||
"webpage": "http://seashepherd.fr/index.php/qui-sommes-nous/sea-shepherd-france"
|
{"webpage": "http://seashepherd.fr/index.php/qui-sommes-nous/sea-shepherd-france"}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
|
||||||
41
profiles/management/commands/import_events.py
Normal file
41
profiles/management/commands/import_events.py
Normal file
|
|
@ -0,0 +1,41 @@
|
||||||
|
""" Small module that import events into the database.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
from datetime import datetime
|
||||||
|
from django.core.management.base import BaseCommand
|
||||||
|
from django.db import models
|
||||||
|
from profiles.models import Place, Event
|
||||||
|
|
||||||
|
def import_file(filename):
|
||||||
|
with open(filename, mode='r') as file:
|
||||||
|
data = json.load(file)
|
||||||
|
for event in data:
|
||||||
|
import_event(event)
|
||||||
|
|
||||||
|
def import_place(_place):
|
||||||
|
place = Place(
|
||||||
|
name=_place.get("name", ""),
|
||||||
|
address=_place.get("address", ""),
|
||||||
|
lat=float(_place.get("lat", 0)),
|
||||||
|
lon=float(_place.get("lon", 0))
|
||||||
|
)
|
||||||
|
place.save()
|
||||||
|
return place
|
||||||
|
|
||||||
|
def import_event(_event):
|
||||||
|
if isinstance(_event["place"], str):
|
||||||
|
place = Place.objects.get(name=_event["place"])
|
||||||
|
else:
|
||||||
|
place = import_place(_event["place"])
|
||||||
|
event = Event(
|
||||||
|
name=_event.get("name", ""),
|
||||||
|
date=datetime.strptime(_event.get("date", "01/01/1970 00:00 UTC"), "%d/%m/%Y %H:%M %Z"),
|
||||||
|
place=place
|
||||||
|
)
|
||||||
|
#print(event)
|
||||||
|
event.save()
|
||||||
|
|
||||||
|
class Command(BaseCommand):
|
||||||
|
def handle(self, *args, **kwargs):
|
||||||
|
import_file("data/events.json")
|
||||||
46
profiles/management/commands/import_website.py
Normal file
46
profiles/management/commands/import_website.py
Normal file
|
|
@ -0,0 +1,46 @@
|
||||||
|
""" Small module that import events into the database.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
from datetime import datetime
|
||||||
|
from django.core.management.base import BaseCommand
|
||||||
|
from django.db import models
|
||||||
|
from profiles.models import Webpage, Website, Keyword
|
||||||
|
|
||||||
|
def import_file(filename):
|
||||||
|
with open(filename, mode='r') as file:
|
||||||
|
data = json.load(file)
|
||||||
|
for website in data:
|
||||||
|
import_website(website)
|
||||||
|
|
||||||
|
def import_website(_website):
|
||||||
|
keywords = []
|
||||||
|
webpages = []
|
||||||
|
for keyword in _website.get("keywords", []):
|
||||||
|
new_keyword = Keyword(
|
||||||
|
text=keyword.get("keyword", "")
|
||||||
|
)
|
||||||
|
new_keyword.save()
|
||||||
|
keywords.append(new_keyword)
|
||||||
|
for webpage in _website.get("notable_pages",[]):
|
||||||
|
new_webpage = Webpage(
|
||||||
|
url=webpage.get("keyword", "")
|
||||||
|
)
|
||||||
|
new_webpage.save()
|
||||||
|
webpages.append(new_webpage)
|
||||||
|
website = Website(
|
||||||
|
name=_website.get("name", ""),
|
||||||
|
url=_website.get("url", ""),
|
||||||
|
)
|
||||||
|
website.save()
|
||||||
|
for keyword in keywords:
|
||||||
|
website.keywords.add(keyword)
|
||||||
|
for webpage in webpages:
|
||||||
|
website.notable_pages.add(webpage)
|
||||||
|
print(website)
|
||||||
|
#website.save()
|
||||||
|
|
||||||
|
|
||||||
|
class Command(BaseCommand):
|
||||||
|
def handle(self, *args, **kwargs):
|
||||||
|
import_file("data/website.json")
|
||||||
Loading…
Reference in a new issue