CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [crawler] filter lookup tags

pull/607/merge
terrtia 2024-10-07 14:53:13 +02:00
parent 83e11082b5
commit d9fc014a1d
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
bin/lib/crawlers.py
View File

@ -87,6 +87,11 @@ def api_get_onion_lookup(domain): # TODO check if object process done ???
meta['first_seen'] = meta['first_seen'].replace('/', '-') meta['first_seen'] = meta['first_seen'].replace('/', '-')
meta['last_seen'] = meta['last_check'].replace('/', '-') meta['last_seen'] = meta['last_check'].replace('/', '-')
meta['languages'] = list(meta['languages']) meta['languages'] = list(meta['languages'])
tags = []
for tag in meta['tags']:
if tag.startswith('dark-web:') or tag.startswith('infoleak:'):
tags.append(tag)
meta['tags'] = tags
del meta['domain'] del meta['domain']
del meta['last_check'] del meta['last_check']
del meta['type'] del meta['type']