Import utilities
This commit is contained in:
parent
5decd205fb
commit
854481dbd3
|
|
@ -0,0 +1,17 @@
|
|||
{
|
||||
"list": [
|
||||
{ "keyword" : "gender queer Paris"},
|
||||
{"keyword" : "fabriquer masque manif"},
|
||||
{"keyword" : "Se protéger en manif"},
|
||||
{"keyword" : "Legal team manif France"},
|
||||
{"keyword" : "Guide juridique GAV"},
|
||||
{"keyword" : "Échec du capitaisme"},
|
||||
{"keyword" : "Bienfait du communisme"},
|
||||
{"keyword" : "Le comité invisible"},
|
||||
{"keyword" : "À nos enfants"},
|
||||
{"keyword" : "Squats sur Paris"},
|
||||
{"keyword" : "Local facho à Strasbourg"},
|
||||
{"keyword" : "Discrimation sexistes, quelles actions ?"},
|
||||
{"keyword" : "Pourquoi la lutte des classes"}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
[
|
||||
{
|
||||
"place" : {
|
||||
"name" : "Zad NDDL",
|
||||
"address" : "Notre-Dame-des-landes, 44111",
|
||||
"lat" : "47.3435",
|
||||
"lon": "-1.7367"
|
||||
}
|
||||
},
|
||||
{
|
||||
"place" : {
|
||||
"name" : "La Mutinerie",
|
||||
"address" : "176 - 178 rue Saint Martin, 75003 Paris",
|
||||
"lat" : "48.8625665",
|
||||
"lon": "2.3522237"
|
||||
}
|
||||
},
|
||||
{
|
||||
"place" : {
|
||||
"name" : "Zad Bure",
|
||||
"address" : "2 rue de l'Église, 55290 Bure",
|
||||
"lat" : "48.502",
|
||||
"lon": "5.351"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
[
|
||||
{
|
||||
"searchengine": {
|
||||
"name":"Google",
|
||||
"url":"https://google.com/",
|
||||
"query_pattern": "search?q={}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"searchengine": {
|
||||
"name":"Duckduckgo",
|
||||
"url":"https://duckduckgo.com/",
|
||||
"query_pattern":"?q={}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"searchengine": {
|
||||
"name":"Qwant",
|
||||
"url":"https://www.qwant.com/",
|
||||
"query_pattern":"?q={}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"searchengine": {
|
||||
"name":"Qwant lite",
|
||||
"url":"https://lite.qwant.com/",
|
||||
"query_pattern":"?q={}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"searchengine": {
|
||||
"name":"Framabee",
|
||||
"url":"https://framabee.org/",
|
||||
"query_pattern":"?q={}"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
[
|
||||
"website": {
|
||||
"name":"emmaclit",
|
||||
"url":"https://emmaclit.com/",
|
||||
"keywords": [
|
||||
"keyword":"Charge mentale",
|
||||
"keyword":"Un autre regard",
|
||||
"keyword":"Un petit poutou",
|
||||
"keyword":"solidarité",
|
||||
"keyword":"dédicace"
|
||||
},
|
||||
"notable_pages": [
|
||||
"webpage": "https://emmaclit.com/2017/05/09/repartition-des-taches-hommes-femmes/",
|
||||
"webpage": "https://emmaclit.com/2016/12/01/une-famille-parmi-dautres/",
|
||||
"webpage": "https://emmaclit.com/2017/09/11/travaille-pourquoi/"
|
||||
]
|
||||
},
|
||||
"website": {
|
||||
"name":"paris-luttes info",
|
||||
"url":"https//paris-luttes.info/",
|
||||
"keywords": [
|
||||
"keyword":"manifestations",
|
||||
"keyword":"solidarité immigré·e·s",
|
||||
"keyword":"grève salariés",
|
||||
"keyword":"prison",
|
||||
"keyword":"violence policère"
|
||||
],
|
||||
"notable_pages": [
|
||||
"webpage": "https://paris-luttes.info/-analyse-et-reflexion-?lang=fr",
|
||||
"webpage": "https://paris-luttes.info/comment-publier-sur-paris-luttes-134?lang=fr",
|
||||
"webpage": "https://paris-luttes.info/pourquoi-et-comment-utiliser-tor-9013?lang=fr"
|
||||
]
|
||||
},
|
||||
"website": {
|
||||
"name":"zad nadir",
|
||||
"url":"http://zad.nadir.org/",
|
||||
"keywords": [
|
||||
"keyword":"Écologie",
|
||||
"keyword":"opération césar",
|
||||
"keyword":"expulsion vinci",
|
||||
"keyword":"adresse",
|
||||
"keyword":"la wardine",
|
||||
"keyword":"route des chicanes",
|
||||
"keyword":"opposition à l'aéroport Grand Ouest"
|
||||
],
|
||||
"notable_pages": [
|
||||
"webpage": "http://zad.nadir.org/spip.php?article86&lang=fr",
|
||||
"webpage": "http://zad.nadir.org/spip.php?article515&lang=fr",
|
||||
"webpage": "http://zad.nadir.org/spip.php?rubrique71",
|
||||
"webpage": "https://zad.nadir.org/spip.php?rubrique70"
|
||||
]
|
||||
},
|
||||
"website": {
|
||||
"name":"Fnac",
|
||||
"url":"https://www.fnac.com/",
|
||||
"keywords": [
|
||||
"keyword":"smartphone",
|
||||
"keyword":"SAV",
|
||||
"keyword":"Macbook",
|
||||
"keyword":"TV",
|
||||
"keyword":"PC Gaming",
|
||||
"keyword":"DVD",
|
||||
"keyword":"Home Cinema Philips",
|
||||
"keyword":"Billeterie"
|
||||
],
|
||||
"notable_pages": [
|
||||
"webpage": "https://www.fnac.com/Informatique/shi48966/w-4#bl=MMinfo",
|
||||
"webpage": "https://www.fnac.com/Service/default.aspx#bl=footer",
|
||||
"webpage": "https://www.fnac.com/Ventes-Flash/shi42245/w-4#bl=marktlink1",
|
||||
"webpage": "https://www.fnac.com/Home-cinema-barre-de-son-et-enceinte-TV/shi474916/w-4#bl=MMtvh"
|
||||
]
|
||||
},
|
||||
"website": {
|
||||
"name":"Sea Shepherd",
|
||||
"url":"https://www.seashepherd.fr/",
|
||||
"keywords": [
|
||||
"keyword":"pirates",
|
||||
"keyword":"Phoques",
|
||||
"keyword":"Paul Watson",
|
||||
"keyword":"harponnage",
|
||||
"keyword":"seal",
|
||||
"keyword":"Chasse aux dauphins",
|
||||
"keyword":"participation",
|
||||
"keyword":"boutique"
|
||||
],
|
||||
"notable_pages": [
|
||||
"webpage": "http://www.seashepherd.fr/index.php/qui-sommes-nous",
|
||||
"webpage": "http://nyamba.seashepherd.info/",
|
||||
"webpage": "http://seashepherd-shop.com/en/",
|
||||
"webpage": "http://seashepherd.fr/index.php/qui-sommes-nous/sea-shepherd-france"
|
||||
]
|
||||
}
|
||||
]
|
||||
|
|
@ -7,7 +7,7 @@ import random
|
|||
from math import floor
|
||||
from django.db import models
|
||||
import profiles.models as profiles
|
||||
from crawl import crawl
|
||||
#from crawl import crawl
|
||||
from pinocchio.settings import HISTORY_MIN
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
""" Small module that import browser fingerprints into the databose,
|
||||
based on the data listed in https://huit.re/user-agent-json.
|
||||
"""
|
||||
|
||||
import json
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db import models
|
||||
from profiles.models import Keyword
|
||||
|
||||
def import_file(filename):
|
||||
with open(filename, mode='r') as file:
|
||||
data = json.load(file)
|
||||
for _keyword in data["list"]:
|
||||
keyword = Keyword(text=_keyword.get("keyword", ""))
|
||||
keyword.save()
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
def handle(self, *args, **kwargs):
|
||||
import_file("data/keywords.json")
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
""" Small module that import browser fingerprints into the databose,
|
||||
based on the data listed in https://huit.re/user-agent-json.
|
||||
"""
|
||||
|
||||
import json
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db import models
|
||||
from profiles.models import Place
|
||||
|
||||
def import_file(filename):
|
||||
with open(filename, mode='r') as file:
|
||||
data = json.load(file)
|
||||
for place in data:
|
||||
import_place(place["place"])
|
||||
|
||||
def import_place(_place):
|
||||
place = Place(
|
||||
name=_place.get("name", ""),
|
||||
address=_place.get("address", ""),
|
||||
lat=float(_place.get("lat", 0)),
|
||||
lon=float(_place.get("lon", 0))
|
||||
)
|
||||
place.save()
|
||||
|
||||
class Command(BaseCommand):
|
||||
def handle(self, *args, **kwargs):
|
||||
import_file("data/place.json")
|
||||
Loading…
Reference in New Issue