AIL-framework/bin/lib/Decoded.py

93 lines
2.4 KiB
Python
Executable File

#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import redis
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
import Item
import ConfigLoader
config_loader = ConfigLoader.ConfigLoader()
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
config_loader = None
def get_decoded_item_type(sha1_string):
'''
Retun the estimed type of a given decoded item.
:param sha1_string: sha1_string
'''
return r_serv_metadata.hget('metadata_hash:{}'.format(sha1_string), 'estimated_type')
def get_decoded_items_list(sha1_string):
return r_serv_metadata.zrange('nb_seen_hash:{}'.format(sha1_string), 0, -1)
def get_item_decoded(item_id):
'''
Retun all decoded item of a given item id.
:param item_id: item id
'''
res = r_serv_metadata.smembers('hash_paste:{}'.format(item_id))
if res:
return list(res)
else:
return []
def get_domain_decoded_item(domain):
'''
Retun all decoded item of a given domain.
:param domain: crawled domain
'''
res = r_serv_metadata.smembers('hash_domain:{}'.format(domain))
if res:
return list(res)
else:
return []
def get_decoded_domain_item(sha1_string):
'''
Retun all domain of a given decoded item.
:param sha1_string: sha1_string
'''
res = r_serv_metadata.smembers('domain_hash:{}'.format(sha1_string))
if res:
return list(res)
else:
return []
def get_decoded_correlated_object(sha1_string, correlation_objects=[]):
'''
Retun all correlation of a given sha1.
:param sha1_string: sha1
:type sha1_string: str
:return: a dict of all correlation for a given sha1
:rtype: dict
'''
if correlation_objects is None:
correlation_objects = Correlation.get_all_correlation_objects()
decoded_correlation = {}
for correlation_object in correlation_objects:
if correlation_object == 'paste':
res = get_decoded_items_list(sha1_string)
elif correlation_object == 'domain':
res = get_decoded_domain_item(sha1_string)
else:
res = None
if res:
decoded_correlation[correlation_object] = res
return decoded_correlation
def save_domain_decoded(domain, sha1_string):
r_serv_metadata.sadd('hash_domain:{}'.format(domain), sha1_string) # domain - hash map
r_serv_metadata.sadd('domain_hash:{}'.format(sha1_string), domain) # hash - domain ma