CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [helper dump_crawler] fix empty dict

pull/422/head
Terrtia 2019-06-26 11:42:19 +02:00
parent 5377369032
commit 06ab66ff57
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
1 changed files with 7 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
bin/helper/dump_crawler_history.py
View File

@ -54,9 +54,10 @@ for date in date_range:
print(domain)
h = HiddenServices(domain, 'onion')
item_core = h.get_domain_crawled_core_item()
l_pastes = h.get_last_crawled_pastes(item_root=item_core['root_item'])
res = h.create_domain_basic_archive(l_pastes)
filename = os.path.join(save_path, '{}'.format(domain))
with open(filename, 'wb') as f:
shutil.copyfileobj(res, f)
print('done')
if 'root_item' in item_core:
l_pastes = h.get_last_crawled_pastes(item_root=item_core['root_item'])
res = h.create_domain_basic_archive(l_pastes)
filename = os.path.join(save_path, '{}'.format(domain))
with open(filename, 'wb') as f:
shutil.copyfileobj(res, f)
print('done')