Add crawler settings and requirements

2018-02-20 12:48:16 +01:00 · 2018-02-20 12:48:16 +01:00 · c05c2561d2
parent bef1fca5b9
commit c05c2561d2
2 changed files with 13 additions and 0 deletions
--- a/pinocchio/settings.py
+++ b/pinocchio/settings.py
@ -100,3 +100,5 @@ USE_TZ = True
 # https://docs.djangoproject.com/en/2.0/howto/static-files/

 STATIC_URL = '/static/'
+
+USER_AGENT = 'UnaffiliatedBot/0.1'
--- a/requirements.txt
+++ b/requirements.txt
@ -1,2 +1,13 @@
+aiodns==1.1.1
+aiohttp==3.0.1
+async-timeout==2.0.0
+attrs==17.4.0
+cchardet==2.1.1
+chardet==3.0.4
 Django==2.0.1
+idna==2.6
+idna-ssl==1.0.0
+multidict==4.1.0
+pycares==2.3.0
 pytz==2017.3
+yarl==1.1.1