From 2732e4115f559f8c0caf11137d9ce8f34139e766 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Bastian?= Date: Fri, 23 Feb 2018 13:32:32 +0100 Subject: [PATCH 1/2] =?UTF-8?q?Add=20RDF=20models=20export=20classes=20?= =?UTF-8?q?=E2=80=94=20untested?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also add a dependency to https://github.com/tobast/RDFSerializer/ --- .gitignore | 1 + profiles/models_rdf.py | 126 +++++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 + 3 files changed, 129 insertions(+) create mode 100644 profiles/models_rdf.py diff --git a/.gitignore b/.gitignore index e5cb34c..be7c368 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,4 @@ venv/ # Django stuff db.sqlite3 +_vimrc_local.vim diff --git a/profiles/models_rdf.py b/profiles/models_rdf.py new file mode 100644 index 0000000..3a2e2bd --- /dev/null +++ b/profiles/models_rdf.py @@ -0,0 +1,126 @@ +""" RDF serialization class for profile models """ + +import rdfserializer as rdf +from rdfserializer import RDFModelSerialiser as RDFModelSerializer +# ^ This was hurting my eyes way too much +from rdfserializer import SCHEMA as schema + +import models as profile_models + + +class RdfWebpage(RDFModelSerializer): + """ RDF serializer for Webpage """ + + _type = schema.WebPage + model = profile_models.Webpage + entries = [ + rdf.RDFSimpleField(schema.url, 'url'), + ] + + +class RdfWebsite(RDFModelSerializer): + """ RDF serializer for Website """ + + _type = schema.WebSite + model = profile_models.Website + entries = [ + rdf.RDFSimpleField(schema.name, 'name'), + rdf.RDFSimpleField(schema.url, 'url'), + rdf.RDFManyField(schema.keywords, 'keywords', + lambda keyword: keyword.text), + rdf.RDFManyLinker(schema.hasPart, 'notable_pages', RdfWebpage), + ] + + +class RdfPlace(RDFModelSerializer): + """ RDF serializer for Place """ + + _type = schema.Place + model = profile_models.Place + entries = [ + rdf.RDFSimpleField(schema.name, 'name'), + rdf.RDFSimpleField(schema.address, 'address'), + rdf.RDFSimpleField(schema.latitude, 'lat'), + rdf.RDFSimpleField(schema.longitude, 'lon'), + ] + + +class RdfEvent(RDFModelSerializer): + """ RDF serializer for Event """ + + _type = schema.Event + model = profile_models.Event + entries = [ + rdf.RDFSimpleField(schema.name, 'name'), + rdf.RDFSimpleField(schema.startDate, 'date'), + rdf.RDFLeftBinder(schema.location, 'place', RdfPlace), + ] + + +class RdfBrowserFingerprint(RDFModelSerializer): + """ RDF serializer for BrowserFingerprint """ + + _type = schema.Intangible + model = profile_models.BrowserFingerprint + entries = [ + rdf.RDFSimpleField(schema.description, 'description'), + rdf.RDFSimpleField('useragent', 'useragent'), + rdf.RDFSimpleField('appname', 'appname'), + rdf.RDFSimpleField('appversion', 'appversion'), + rdf.RDFSimpleField('platform', 'platform'), + rdf.RDFSimpleField('vendor', 'vendor'), + rdf.RDFSimpleField('vendorsub', 'vendorsub'), + rdf.RDFSimpleField('buildID', 'buildID'), + rdf.RDFSimpleField('oscpu', 'oscpu'), + rdf.RDFSimpleField('accept_encoding', 'accept_encoding'), + rdf.RDFSimpleField('accept_default', 'accept_default'), + rdf.RDFSimpleField('accept_lang', 'accept_lang'), + rdf.RDFSimpleField('pixeldepth', 'pixeldepth'), + rdf.RDFSimpleField('colordepth', 'colordepth'), + rdf.RDFSimpleField('screens', 'screens'), + ] + + +class RdfSearchEngine(RDFModelSerializer): + """ RDF serializer for SearchEngine """ + + _type = schema.WebSite + model = profile_models.SearchEngine + entries = [ + rdf.RDFSimpleField(schema.url, 'url'), + rdf.RDFSimpleField(schema.name, 'name'), + rdf.RDFSimpleField('query_pattern', 'query_pattern'), + ] + + +class RdfInterest(RDFModelSerializer): + """ RDF serializer for Interest """ + + Interesttype = 'interest' + model = profile_models.Interest + entries = [ + rdf.RDFSimpleField(schema.name, 'name'), + rdf.RDFManyField(schema.keywords, 'keywords', + lambda keyword: keyword.text), + rdf.RDFManyLinker(schema.location, 'places', RdfPlace), + rdf.RDFManyLinker(schema.website, 'websites', RdfWebsite), + rdf.RDFManyLinker(schema.event, 'events', RdfEvent), + ] + + +class RdfProfile(RDFModelSerializer): + """ RDF serializer for Profile """ + + _type = schema.Person + model = profile_models.Profile + entries = [ + rdf.RDFSimpleField('nickname', 'nick'), + rdf.RDFSimpleField(schema.given_name, 'first_name'), + rdf.RDFSimpleField(schema.family_name, 'last_name'), + rdf.RDFSimpleField(schema.email, 'email'), + rdf.RDFSimpleField('uses_urls', 'uses_urls'), + rdf.RDFManyLinker('interest', 'interests', RdfInterest), + rdf.RDFLeftBinder('search_engine', 'search_engine', RdfSearchEngine), + rdf.RDFLeftBinder('browser_fingerprint', 'browser_fingerprint', + RdfBrowserFingerprint) + ] diff --git a/requirements.txt b/requirements.txt index 480760f..813dfd9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,5 @@ pycares==2.3.0 pytz==2017.3 yarl==1.1.1 beautifulsoup4==4.6.0 +rdflib==4.2.2 +git+https://github.com/tobast/RDFSerializer.git From c3bcdea1eb00a2a2ca1b56e630dedd37ce17b869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Bastian?= Date: Sun, 25 Feb 2018 13:34:44 +0100 Subject: [PATCH 2/2] Add tentative export to RDF --- profiles/management/commands/exportrdf.py | 27 +++++++++++++ profiles/models_rdf.py | 47 +++++++++++++---------- 2 files changed, 53 insertions(+), 21 deletions(-) create mode 100644 profiles/management/commands/exportrdf.py diff --git a/profiles/management/commands/exportrdf.py b/profiles/management/commands/exportrdf.py new file mode 100644 index 0000000..5786afd --- /dev/null +++ b/profiles/management/commands/exportrdf.py @@ -0,0 +1,27 @@ +from django.core.management.base import BaseCommand +from profiles.models_rdf import RdfProfile +from profiles import models + + +class Command(BaseCommand): + ''' Exports database models to RDF ''' + + def add_arguments(self, parser): + pass + + def handle(self, *args, **kwargs): + exported_models = [ + models.Keyword, + models.Webpage, + models.Website, + models.Place, + models.Event, + models.BrowserFingerprint, + models.SearchEngine, + models.Interest, + models.Profile, + ] + output_xml = RdfProfile().serialize( + # models=exported_models, + ) + self.stdout.write(output_xml) diff --git a/profiles/models_rdf.py b/profiles/models_rdf.py index 3a2e2bd..6e142f8 100644 --- a/profiles/models_rdf.py +++ b/profiles/models_rdf.py @@ -4,8 +4,12 @@ import rdfserializer as rdf from rdfserializer import RDFModelSerialiser as RDFModelSerializer # ^ This was hurting my eyes way too much from rdfserializer import SCHEMA as schema +from rdflib.namespace import Namespace -import models as profile_models +import profiles.models as profile_models + + +LOCAL_NS = Namespace('local:') class RdfWebpage(RDFModelSerializer): @@ -64,20 +68,20 @@ class RdfBrowserFingerprint(RDFModelSerializer): model = profile_models.BrowserFingerprint entries = [ rdf.RDFSimpleField(schema.description, 'description'), - rdf.RDFSimpleField('useragent', 'useragent'), - rdf.RDFSimpleField('appname', 'appname'), - rdf.RDFSimpleField('appversion', 'appversion'), - rdf.RDFSimpleField('platform', 'platform'), - rdf.RDFSimpleField('vendor', 'vendor'), - rdf.RDFSimpleField('vendorsub', 'vendorsub'), - rdf.RDFSimpleField('buildID', 'buildID'), - rdf.RDFSimpleField('oscpu', 'oscpu'), - rdf.RDFSimpleField('accept_encoding', 'accept_encoding'), - rdf.RDFSimpleField('accept_default', 'accept_default'), - rdf.RDFSimpleField('accept_lang', 'accept_lang'), - rdf.RDFSimpleField('pixeldepth', 'pixeldepth'), - rdf.RDFSimpleField('colordepth', 'colordepth'), - rdf.RDFSimpleField('screens', 'screens'), + rdf.RDFSimpleField(LOCAL_NS.useragent, 'useragent'), + rdf.RDFSimpleField(LOCAL_NS.appname, 'appname'), + rdf.RDFSimpleField(LOCAL_NS.appversion, 'appversion'), + rdf.RDFSimpleField(LOCAL_NS.platform, 'platform'), + rdf.RDFSimpleField(LOCAL_NS.vendor, 'vendor'), + rdf.RDFSimpleField(LOCAL_NS.vendorsub, 'vendorsub'), + rdf.RDFSimpleField(LOCAL_NS.buildID, 'buildID'), + rdf.RDFSimpleField(LOCAL_NS.oscpu, 'oscpu'), + rdf.RDFSimpleField(LOCAL_NS.accept_encoding, 'accept_encoding'), + rdf.RDFSimpleField(LOCAL_NS.accept_default, 'accept_default'), + rdf.RDFSimpleField(LOCAL_NS.accept_lang, 'accept_lang'), + rdf.RDFSimpleField(LOCAL_NS.pixeldepth, 'pixeldepth'), + rdf.RDFSimpleField(LOCAL_NS.colordepth, 'colordepth'), + rdf.RDFSimpleField(LOCAL_NS.screens, 'screens'), ] @@ -89,7 +93,7 @@ class RdfSearchEngine(RDFModelSerializer): entries = [ rdf.RDFSimpleField(schema.url, 'url'), rdf.RDFSimpleField(schema.name, 'name'), - rdf.RDFSimpleField('query_pattern', 'query_pattern'), + rdf.RDFSimpleField(LOCAL_NS.query_pattern, 'query_pattern'), ] @@ -114,13 +118,14 @@ class RdfProfile(RDFModelSerializer): _type = schema.Person model = profile_models.Profile entries = [ - rdf.RDFSimpleField('nickname', 'nick'), + rdf.RDFSimpleField(LOCAL_NS.nickname, 'nick'), rdf.RDFSimpleField(schema.given_name, 'first_name'), rdf.RDFSimpleField(schema.family_name, 'last_name'), rdf.RDFSimpleField(schema.email, 'email'), - rdf.RDFSimpleField('uses_urls', 'uses_urls'), - rdf.RDFManyLinker('interest', 'interests', RdfInterest), - rdf.RDFLeftBinder('search_engine', 'search_engine', RdfSearchEngine), - rdf.RDFLeftBinder('browser_fingerprint', 'browser_fingerprint', + rdf.RDFSimpleField(LOCAL_NS.uses_urls, 'uses_urls'), + rdf.RDFManyLinker(LOCAL_NS.interest, 'interests', RdfInterest), + rdf.RDFLeftBinder(LOCAL_NS.search_engine, 'search_engine', + RdfSearchEngine), + rdf.RDFLeftBinder(LOCAL_NS.browser_fingerprint, 'browser_fingerprint', RdfBrowserFingerprint) ]