CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

fix: [urlexport importer] fix item name + redis config

pull/497/head
Terrtia 2020-05-03 15:40:43 +02:00
parent b4cb14798c
commit 64fae5d7df
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
2 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
bin/import/ail_json_importer/Ail_feeder_urlextract.py
View File

@ -33,11 +33,17 @@ class Ail_feeder_urlextract(Default_json):
# use twitter timestamp ? # use twitter timestamp ?
item_date = datetime.date.today().strftime("%Y/%m/%d") item_date = datetime.date.today().strftime("%Y/%m/%d")
item_id = str(self.json_item['meta']['twitter:url-extracted']) item_id = str(self.json_item['meta']['twitter:url-extracted'])
item_id = item_id.split('//')
if len(item_id) > 1:
item_id = ''.join(item_id[1:])
else:
item_id = item_id[0]
item_id = item_id.replace('/', '_')
if len(item_id) > 215: if len(item_id) > 215:
item_id = '{}{}.gz'.format(item_id[:215], str(uuid.uuid4())) item_id = '{}{}.gz'.format(item_id[:215], str(uuid.uuid4()))
else: else:
item_id = '{}{}.gz'.format(item_id, str(uuid.uuid4())) item_id = '{}{}.gz'.format(item_id, str(uuid.uuid4()))
return os.path.join('urlextract', item_date, item_id) + '.gz' return os.path.join('urlextract', item_date, item_id)
# # TODO: # # TODO:
def process_json_meta(self, process, item_id): def process_json_meta(self, process, item_id):

2
bin/lib/item_basic.py
View File

@ -11,6 +11,8 @@ config_loader = ConfigLoader.ConfigLoader()
# get and sanityze PASTE DIRECTORY # get and sanityze PASTE DIRECTORY
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/' PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
PASTES_FOLDER = os.path.join(os.path.realpath(PASTES_FOLDER), '') PASTES_FOLDER = os.path.join(os.path.realpath(PASTES_FOLDER), '')
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
config_loader = None config_loader = None
def exist_item(item_id): def exist_item(item_id):