tobast/mpri-webdam
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add mailto adresses to forbidden list

Browse source
This commit is contained in:
Rémi Oudin 2018-02-24 15:41:46 +01:00
parent e56c088632
commit d19c2e8216
1 changed files with 1 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
crawl/crawl.py
View file

@ -71,7 +71,7 @@ def url_getter(html, current_page, root_url):
# Works only with python >= 3.6 # Works only with python >= 3.6
links_list = list(dict.fromkeys(links_list)) links_list = list(dict.fromkeys(links_list))
forbidden_words = ['login', 'agreement'] forbidden_words = ['login', 'agreement', 'mailto']
links_list = [link for link in links_list if not any(word in link.lower() links_list = [link for link in links_list if not any(word in link.lower()
for word in for word in
forbidden_words)] forbidden_words)]