tobast
/
mpri-webdam
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix function calls

This commit is contained in:
Théophile Bastian 2018-02-26 11:56:02 +01:00
parent 3e5fc2f9b3
commit 02e91bb2b7
1 changed files with 8 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
crawl/crawl.py
View File

@ -234,7 +234,8 @@ class PageGetter:
async def async_print(url):
""" Debug function to follow what's actually happening """
async with aiohttp.ClientSession() as session:
html = await PageGetter(session, url).get(ssl=False)
html = await PageGetter(session, url,
settings.USER_AGENT).get(ssl=False)
print('GOT {}HTML for {}'.format(
'None ' if html is None else '',
@ -244,9 +245,11 @@ async def async_print(url):
async def async_crawler(url, queue, headers=None):
if headers is None:
headers = {
'User-Agent': settings.USER_AGENT,
}
headers = {}
if 'User-Agent' not in headers:
headers['User-Agent'] = settings.USER_AGENT
user_agent = headers['User-Agent']
queued = [url]
crawled = []
@ -259,7 +262,7 @@ async def async_crawler(url, queue, headers=None):
return crawled
parsed_url = urlparse(url)
print("Crawling {}".format(url))
html = await PageGetter(session, url).get(ssl=False)
html = await PageGetter(session, url, user_agent).get(ssl=False)
if html:
new_urls = url_getter(
html,