Import utilities

This commit is contained in:
Rémi Oudin 2018-02-24 17:21:41 +01:00
parent 5decd205fb
commit 854481dbd3
7 changed files with 221 additions and 1 deletions

17
data/keywords.json Normal file
View file

@ -0,0 +1,17 @@
{
"list": [
{ "keyword" : "gender queer Paris"},
{"keyword" : "fabriquer masque manif"},
{"keyword" : "Se protéger en manif"},
{"keyword" : "Legal team manif France"},
{"keyword" : "Guide juridique GAV"},
{"keyword" : "Échec du capitaisme"},
{"keyword" : "Bienfait du communisme"},
{"keyword" : "Le comité invisible"},
{"keyword" : "À nos enfants"},
{"keyword" : "Squats sur Paris"},
{"keyword" : "Local facho à Strasbourg"},
{"keyword" : "Discrimation sexistes, quelles actions ?"},
{"keyword" : "Pourquoi la lutte des classes"}
]
}

26
data/place.json Normal file
View file

@ -0,0 +1,26 @@
[
{
"place" : {
"name" : "Zad NDDL",
"address" : "Notre-Dame-des-landes, 44111",
"lat" : "47.3435",
"lon": "-1.7367"
}
},
{
"place" : {
"name" : "La Mutinerie",
"address" : "176 - 178 rue Saint Martin, 75003 Paris",
"lat" : "48.8625665",
"lon": "2.3522237"
}
},
{
"place" : {
"name" : "Zad Bure",
"address" : "2 rue de l'Église, 55290 Bure",
"lat" : "48.502",
"lon": "5.351"
}
}
]

37
data/search_engine.json Normal file
View file

@ -0,0 +1,37 @@
[
{
"searchengine": {
"name":"Google",
"url":"https://google.com/",
"query_pattern": "search?q={}"
}
},
{
"searchengine": {
"name":"Duckduckgo",
"url":"https://duckduckgo.com/",
"query_pattern":"?q={}"
}
},
{
"searchengine": {
"name":"Qwant",
"url":"https://www.qwant.com/",
"query_pattern":"?q={}"
}
},
{
"searchengine": {
"name":"Qwant lite",
"url":"https://lite.qwant.com/",
"query_pattern":"?q={}"
}
},
{
"searchengine": {
"name":"Framabee",
"url":"https://framabee.org/",
"query_pattern":"?q={}"
}
}
]

93
data/website.json Normal file
View file

@ -0,0 +1,93 @@
[
"website": {
"name":"emmaclit",
"url":"https://emmaclit.com/",
"keywords": [
"keyword":"Charge mentale",
"keyword":"Un autre regard",
"keyword":"Un petit poutou",
"keyword":"solidarité",
"keyword":"dédicace"
},
"notable_pages": [
"webpage": "https://emmaclit.com/2017/05/09/repartition-des-taches-hommes-femmes/",
"webpage": "https://emmaclit.com/2016/12/01/une-famille-parmi-dautres/",
"webpage": "https://emmaclit.com/2017/09/11/travaille-pourquoi/"
]
},
"website": {
"name":"paris-luttes info",
"url":"https//paris-luttes.info/",
"keywords": [
"keyword":"manifestations",
"keyword":"solidarité immigré·e·s",
"keyword":"grève salariés",
"keyword":"prison",
"keyword":"violence policère"
],
"notable_pages": [
"webpage": "https://paris-luttes.info/-analyse-et-reflexion-?lang=fr",
"webpage": "https://paris-luttes.info/comment-publier-sur-paris-luttes-134?lang=fr",
"webpage": "https://paris-luttes.info/pourquoi-et-comment-utiliser-tor-9013?lang=fr"
]
},
"website": {
"name":"zad nadir",
"url":"http://zad.nadir.org/",
"keywords": [
"keyword":"Écologie",
"keyword":"opération césar",
"keyword":"expulsion vinci",
"keyword":"adresse",
"keyword":"la wardine",
"keyword":"route des chicanes",
"keyword":"opposition à l'aéroport Grand Ouest"
],
"notable_pages": [
"webpage": "http://zad.nadir.org/spip.php?article86&lang=fr",
"webpage": "http://zad.nadir.org/spip.php?article515&lang=fr",
"webpage": "http://zad.nadir.org/spip.php?rubrique71",
"webpage": "https://zad.nadir.org/spip.php?rubrique70"
]
},
"website": {
"name":"Fnac",
"url":"https://www.fnac.com/",
"keywords": [
"keyword":"smartphone",
"keyword":"SAV",
"keyword":"Macbook",
"keyword":"TV",
"keyword":"PC Gaming",
"keyword":"DVD",
"keyword":"Home Cinema Philips",
"keyword":"Billeterie"
],
"notable_pages": [
"webpage": "https://www.fnac.com/Informatique/shi48966/w-4#bl=MMinfo",
"webpage": "https://www.fnac.com/Service/default.aspx#bl=footer",
"webpage": "https://www.fnac.com/Ventes-Flash/shi42245/w-4#bl=marktlink1",
"webpage": "https://www.fnac.com/Home-cinema-barre-de-son-et-enceinte-TV/shi474916/w-4#bl=MMtvh"
]
},
"website": {
"name":"Sea Shepherd",
"url":"https://www.seashepherd.fr/",
"keywords": [
"keyword":"pirates",
"keyword":"Phoques",
"keyword":"Paul Watson",
"keyword":"harponnage",
"keyword":"seal",
"keyword":"Chasse aux dauphins",
"keyword":"participation",
"keyword":"boutique"
],
"notable_pages": [
"webpage": "http://www.seashepherd.fr/index.php/qui-sommes-nous",
"webpage": "http://nyamba.seashepherd.info/",
"webpage": "http://seashepherd-shop.com/en/",
"webpage": "http://seashepherd.fr/index.php/qui-sommes-nous/sea-shepherd-france"
]
}
]

View file

@ -7,7 +7,7 @@ import random
from math import floor from math import floor
from django.db import models from django.db import models
import profiles.models as profiles import profiles.models as profiles
from crawl import crawl #from crawl import crawl
from pinocchio.settings import HISTORY_MIN from pinocchio.settings import HISTORY_MIN

View file

@ -0,0 +1,20 @@
""" Small module that import browser fingerprints into the databose,
based on the data listed in https://huit.re/user-agent-json.
"""
import json
from django.core.management.base import BaseCommand
from django.db import models
from profiles.models import Keyword
def import_file(filename):
with open(filename, mode='r') as file:
data = json.load(file)
for _keyword in data["list"]:
keyword = Keyword(text=_keyword.get("keyword", ""))
keyword.save()
class Command(BaseCommand):
def handle(self, *args, **kwargs):
import_file("data/keywords.json")

View file

@ -0,0 +1,27 @@
""" Small module that import browser fingerprints into the databose,
based on the data listed in https://huit.re/user-agent-json.
"""
import json
from django.core.management.base import BaseCommand
from django.db import models
from profiles.models import Place
def import_file(filename):
with open(filename, mode='r') as file:
data = json.load(file)
for place in data:
import_place(place["place"])
def import_place(_place):
place = Place(
name=_place.get("name", ""),
address=_place.get("address", ""),
lat=float(_place.get("lat", 0)),
lon=float(_place.get("lon", 0))
)
place.save()
class Command(BaseCommand):
def handle(self, *args, **kwargs):
import_file("data/place.json")