CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [crawler] fix new crawled item id

pull/569/head
Terrtia 2021-07-14 15:48:17 +02:00
parent 25813a7287
commit 7a652b5195
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/lib/crawlers.py
View File

@ -775,6 +775,8 @@ def create_item_metadata(item_id, domain, url, port, item_father):
r_serv_metadata.sadd('paste_children:{}'.format(item_father), item_id) r_serv_metadata.sadd('paste_children:{}'.format(item_father), item_id)
def create_item_id(item_dir, domain): def create_item_id(item_dir, domain):
# remove /
domain = domain.replace('/', '_')
if len(domain) > 215: if len(domain) > 215:
UUID = domain[-215:]+str(uuid.uuid4()) UUID = domain[-215:]+str(uuid.uuid4())
else: else: