CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [urlextract importer] fix parent map + replace set JSON queue by list

pull/497/head
Terrtia 2020-05-06 16:57:13 +02:00
parent bae2fd79df
commit 8c864fdba0
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
2 changed files with 5 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bin/import/importer.py
View File

@ -46,10 +46,10 @@ def get_json_importer_list():
def add_json_to_json_queue(json_item): def add_json_to_json_queue(json_item):
json_item = json.dumps(json_item) json_item = json.dumps(json_item)
r_serv_db.sadd('importer:json:item', json_item) r_serv_db.rpush('importer:json:item', json_item)
def get_json_item_to_import(): def get_json_item_to_import():
return r_serv_db.spop('importer:json:item') return r_serv_db.lpop('importer:json:item')
def get_json_receiver_class(feeder_name_in): def get_json_receiver_class(feeder_name_in):
global importer_list global importer_list

9
bin/lib/item_basic.py
View File

@ -32,12 +32,9 @@ def add_item_parent_by_parent_id(parent_type, parent_id, item_id):
add_item_parent(parent_item_id, item_id) add_item_parent(parent_item_id, item_id)
def add_item_parent(parent_item_id, item_id): def add_item_parent(parent_item_id, item_id):
if not exist_item(parent_item_id): r_serv_metadata.hset('paste_metadata:{}'.format(item_id), 'father', parent_item_id)
return False r_serv_metadata.sadd('paste_children:{}'.format(parent_item_id), item_id)
else: return True
r_serv_metadata.hset('paste_metadata:{}'.format(item_id), 'father', parent_item_id)
r_serv_metadata.sadd('paste_children:{}'.format(parent_item_id), item_id)
return True
def add_map_obj_id_item_id(obj_id, item_id, obj_type): def add_map_obj_id_item_id(obj_id, item_id, obj_type):
if obj_type == 'twitter_id': if obj_type == 'twitter_id':