CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [title] fix None title

pull/604/head
Terrtia 2023-08-23 11:46:37 +02:00
parent 46c721590d
commit 2145eb7b8a
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
bin/lib/crawlers.py
View File

@ -234,7 +234,9 @@ def extract_title_from_html(html):
soup = BeautifulSoup(html, 'html.parser') soup = BeautifulSoup(html, 'html.parser')
title = soup.title title = soup.title
if title: if title:
return str(title.string) title = title.string
if title:
return str(title)
return '' return ''
def extract_description_from_html(html): def extract_description_from_html(html):
@ -2022,4 +2024,4 @@ if __name__ == '__main__':
# _reprocess_all_hars_cookie_name() # _reprocess_all_hars_cookie_name()
# _reprocess_all_hars_etag() # _reprocess_all_hars_etag()
# _gzip_all_hars() # _gzip_all_hars()
_reprocess_all_hars_hhhashs() # _reprocess_all_hars_hhhashs()