diff --git a/bin/DB_KVROCKS_MIGRATION.py b/bin/DB_KVROCKS_MIGRATION.py index 12cbcd55..2cd65219 100755 --- a/bin/DB_KVROCKS_MIGRATION.py +++ b/bin/DB_KVROCKS_MIGRATION.py @@ -612,8 +612,10 @@ def domain_migration(): def get_estimated_type(decoded_id): return r_serv_metadata.hget(f'metadata_hash:{decoded_id}', 'estimated_type') -def get_decoded_items_list_by_decoder(decoder_type, decoded_id): ################### - #return r_serv_metadata.zrange('nb_seen_hash:{}'.format(sha1_string), 0, -1) +def get_hash_size(decoded_id): + return r_serv_metadata.hget(f'metadata_hash:{decoded_id}', 'size') + +def get_decoded_items_list_by_decoder(decoder_type, decoded_id): return r_serv_metadata.zrange(f'{decoder_type}_hash:{decoded_id}', 0, -1) def get_decodeds_tags(decoded_id): @@ -621,10 +623,10 @@ def get_decodeds_tags(decoded_id): def decodeds_migration(): print('Decoded MIGRATION...') - decoder_names = ['base64', 'binary', 'hexadecimal'] + algo_names = ['base64', 'binary', 'hexadecimal'] Decodeds._delete_old_json_descriptor() - for decoded_id in Decodeds.get_all_decodeds(): + for decoded_id in Decodeds.get_all_decodeds_files(): mimetype = get_estimated_type(decoded_id) # ignore invalid object if mimetype is None: @@ -633,19 +635,23 @@ def decodeds_migration(): print(decoded_id) decoded = Decodeds.Decoded(decoded_id) - filepath = decoded.get_filepath(mimetype=mimetype) - decoded._save_meta(filepath, mimetype) + decoded._add_create() + decoded.set_mimetype(mimetype) + + size = get_hash_size(decoded_id) + if not size: + filepath = decoded.get_filepath(mimetype=mimetype) + size = os.path.getsize(filepath) + decoded.set_size(size) for tag in get_decodeds_tags(decoded_id): decoded.add_tag(tag) - for decoder_type in decoder_names: - for item_id in get_decoded_items_list_by_decoder(decoder_type, decoded_id): - print(item_id, decoder_type) + for algo in algo_names: + for item_id in get_decoded_items_list_by_decoder(algo, decoded_id): + print(item_id, algo) date = get_item_date(item_id) - #for decoder_type in : - - decoded.add(decoder_type, date, item_id, mimetype) + decoded.add(algo, date, item_id, mimetype=mimetype) ############################### # # @@ -868,7 +874,7 @@ if __name__ == '__main__': # items_migration() #crawler_migration() # domain_migration() # TO TEST ########################### - #decodeds_migration() + decodeds_migration() # screenshots_migration() # subtypes_obj_migration() # ail_2_ail_migration() @@ -876,7 +882,7 @@ if __name__ == '__main__': # investigations_migration() # statistics_migration() - cves_migration() + # cves_migration() # custom tags # crawler queues + auto_crawlers diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index aceda52a..e1fbeb4e 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -96,7 +96,10 @@ def is_obj_correlated(obj_type, subtype, obj_id, obj2_type, subtype2, obj2_id): subtype = '' if subtype2 is None: subtype2 = '' - return r_metadata.sismember(f'correlation:obj:{obj_type}:{subtype}:{obj2_type}:{obj_id}', f'{subtype2}:{obj2_id}') + try: + return r_metadata.sismember(f'correlation:obj:{obj_type}:{subtype}:{obj2_type}:{obj_id}', f'{subtype2}:{obj2_id}') + except: + return False def add_obj_correlation(obj1_type, subtype1, obj1_id, obj2_type, subtype2, obj2_id): if subtype1 is None: diff --git a/bin/lib/data_retention_engine.py b/bin/lib/data_retention_engine.py index 3fe961ae..602bc018 100755 --- a/bin/lib/data_retention_engine.py +++ b/bin/lib/data_retention_engine.py @@ -27,7 +27,7 @@ config_loader = None # DOMAIN -> subtype = domain type # TAG -> type = "TAG" -# TAG -> subtype = "OBJ:"tag +# TAG -> subtype = f"OBJ:{tag}" def load_obj_date_first_last(): # LOAD FIRST DATE diff --git a/bin/lib/objects/Cves.py b/bin/lib/objects/Cves.py index 619d5809..6c84675c 100755 --- a/bin/lib/objects/Cves.py +++ b/bin/lib/objects/Cves.py @@ -84,7 +84,7 @@ class Cve(AbstractDaterangeObject): # 'Modified' return json_response else: - return {'error': 'cve search error'} # TODO + return {'error': f'{response.status_code}'} # TODO ADD SEARCH FUNCTION @@ -122,5 +122,14 @@ def api_get_cves_meta_by_daterange(date_from, date_to): date = Date.sanitise_date_range(date_from, date_to) return get_cves_meta(get_cves_by_daterange(date['date_from'], date['date_to']), options=['sparkline']) +def get_cve_graphline(cve_id): + cve = Cve(cve_id) + graphline = [] + if cve.exists(): + nb_day = 30 + for date in Date.get_previous_date_list(nb_day): + graphline.append({'date': f'{date[0:4]}-{date[4:6]}-{date[6:8]}', 'value': cve.get_nb_seen_by_date(date)}) + return graphline + # if __name__ == '__main__': diff --git a/bin/lib/objects/Decodeds.py b/bin/lib/objects/Decodeds.py index 9535ae86..8d0bf2de 100755 --- a/bin/lib/objects/Decodeds.py +++ b/bin/lib/objects/Decodeds.py @@ -8,12 +8,14 @@ import zipfile from flask import url_for from io import BytesIO +from pymisp import MISPObject sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## from lib.ConfigLoader import ConfigLoader -from lib.objects.abstract_object import AbstractObject -from lib.item_basic import is_crawled, get_item_domain - +from lib.objects.abstract_daterange_object import AbstractDaterangeObject from packages import Date sys.path.append('../../configs/keys') @@ -22,17 +24,17 @@ try: if vt_key != '': VT_TOKEN = vt_key VT_ENABLED = True - #print('VT submission is enabled') + # print('VT submission is enabled') else: VT_ENABLED = False - #print('VT submission is disabled') + # print('VT submission is disabled') except: VT_TOKEN = None VT_ENABLED = False - #print('VT submission is disabled') + # print('VT submission is disabled') config_loader = ConfigLoader() -r_metadata = config_loader.get_db_conn("Kvrocks_Objects") +r_objects = config_loader.get_db_conn("Kvrocks_Objects") r_metadata = config_loader.get_redis_conn("ARDB_Metadata") HASH_DIR = config_loader.get_config_str('Directories', 'hash') @@ -46,7 +48,7 @@ config_loader = None # # TODO: COMPLETE CLASS -class Decoded(AbstractObject): +class Decoded(AbstractDaterangeObject): """ AIL Decoded Object. (strings) """ @@ -54,9 +56,6 @@ class Decoded(AbstractObject): def __init__(self, id): super(Decoded, self).__init__('decoded', id) - def exists(self): - return r_metadata.exists(f'metadata_hash:{self.id}') - # def get_ail_2_ail_payload(self): # payload = {'raw': self.get_gzip_content(b64=True), # 'compress': 'gzip'} @@ -74,9 +73,10 @@ class Decoded(AbstractObject): url = f'{baseurl}/correlation/show?type={self.type}&id={self.id}' return url - def get_svg_icon(self): - file_type = self.get_estimated_type() - file_type = file_type.split('/')[0] + def get_svg_icon(self, mimetype=None): + if not mimetype: + mimetype = self.get_mimetype() + file_type = mimetype.split('/')[0] if file_type == 'application': icon = '\uf15b' elif file_type == 'audio': @@ -87,19 +87,28 @@ class Decoded(AbstractObject): icon = '\uf15c' else: icon = '\uf249' - return {'style': 'fas', 'icon': icon, 'color': '#88CCEE', 'radius':5} + return {'style': 'fas', 'icon': icon, 'color': '#88CCEE', 'radius': 5} ''' Return the estimated type of a given decoded item. :param sha1_string: sha1_string ''' - def get_estimated_type(self): - return r_metadata.hget(f'metadata_hash:{self.id}', 'estimated_type') + def get_mimetype(self): + return r_objects.hget(f'meta:{self.type}:{self.id}', 'mime') + + def set_mimetype(self, mimetype): + return r_objects.hset(f'meta:{self.type}:{self.id}', 'mime', mimetype) + + def get_size(self): + return r_objects.hget(f'meta:{self.type}:{self.id}', 'size') + + def set_size(self, size): + return r_objects.hset(f'meta:{self.type}:{self.id}', 'size', int(size)) def get_rel_path(self, mimetype=None): if not mimetype: - mimetype = self.get_estimated_type() + mimetype = self.get_mimetype() return os.path.join(HASH_DIR, mimetype, self.id[0:2], self.id) def get_filepath(self, mimetype=None): @@ -112,25 +121,24 @@ class Decoded(AbstractObject): return file_content def get_zip_content(self): - mimetype = self.get_estimated_type() + # mimetype = self.get_estimated_type() zip_content = BytesIO() with zipfile.ZipFile(zip_content, "w") as zf: # TODO: Fix password - #zf.setpassword(b"infected") - zf.writestr( self.id, self.get_content().getvalue()) + # zf.setpassword(b"infected") + zf.writestr(self.id, self.get_content().getvalue()) zip_content.seek(0) return zip_content - def get_misp_object(self): obj_attrs = [] obj = MISPObject('file') obj.first_seen = self.get_first_seen() obj.last_seen = self.get_last_seen() - obj_attrs.append( obj.add_attribute('sha1', value=self.id) ) - obj_attrs.append( obj.add_attribute('mimetype', value=self.get_estimated_type()) ) - obj_attrs.append( obj.add_attribute('malware-sample', value=self.id, data=self.get_content()) ) + obj_attrs.append( obj.add_attribute('sha1', value=self.id)) + obj_attrs.append( obj.add_attribute('mimetype', value=self.get_mimetype())) + obj_attrs.append( obj.add_attribute('malware-sample', value=self.id, data=self.get_content())) for obj_attr in obj_attrs: for tag in self.get_tags(): obj_attr.add_tag(tag) @@ -139,176 +147,84 @@ class Decoded(AbstractObject): ############################################################################ ############################################################################ ############################################################################ + def get_decoders(self): return ['base64', 'binary', 'hexadecimal'] - def get_first_seen(self): - res = r_metadata.hget(f'metadata_hash:{self.id}', 'first_seen') - if res: - return int(res) - else: - return 99999999 - - def get_last_seen(self): - res = r_metadata.hget(f'metadata_hash:{self.id}', 'last_seen') - if res: - return int(res) - else: - return 0 - - def set_first_seen(self, date): - r_metadata.hset(f'metadata_hash:{self.id}', 'first_seen', date) - - def set_last_seen(self, date): - r_metadata.hset(f'metadata_hash:{self.id}', 'last_seen', date) - - def update_daterange(self, date): - first_seen = self.get_first_seen() - last_seen = self.get_last_seen() - if date < first_seen: - self.set_first_seen(date) - if date > last_seen: - self.set_last_seen(date) - def get_meta(self, options=set()): - meta = {'id': self.id, - 'subtype': self.subtype, - 'tags': self.get_tags()} + meta = self._get_meta(options=options) + meta['id'] = self.id + if 'mimetype' in options: + meta['mimetype'] = self.get_mimetype() + if 'icon' in options: + if 'mimetype' in meta: + mimetype = meta['mimetype'] + else: + mimetype = None + meta['icon'] = self.get_svg_icon(mimetype=mimetype) + if 'size' in options: + meta['size'] = self.get_size() + if 'tags' in options: + meta['tags'] = self.get_tags() + if 'vt' in options: + meta['vt'] = self.get_meta_vt() return meta def get_meta_vt(self): - meta = {} - meta['link'] = r_metadata.hget(f'metadata_hash:{self.id}', 'vt_link') - meta['report'] = r_metadata.hget(f'metadata_hash:{self.id}', 'vt_report') - return meta + link = r_objects.hget(f'meta:{self.type}:{self.id}', 'vt_link') + report = r_objects.hget(f'meta:{self.type}:{self.id}', 'vt_report') + if link or report: + return {'link': link, 'report': report} + else: + return {} def guess_mimetype(self, bytes_content): return magic.from_buffer(bytes_content, mime=True) - def _save_meta(self, filepath, mimetype): - # create hash metadata - r_metadata.hset(f'metadata_hash:{self.id}', 'size', os.path.getsize(filepath)) - r_metadata.hset(f'metadata_hash:{self.id}', 'estimated_type', mimetype) - r_metadata.sadd('hash_all_type', mimetype) #################################################### rename ???? + # avoid counting the same hash multiple time on the same item + # except if different encoding - def save_file(self, content, mimetype): ##################################################### + def is_seen_this_day(self, date): + return bool(self.get_nb_seen_by_date(Date.get_today_date_str())) + + def save_file(self, b_content, mimetype): # TODO TEST ME filepath = self.get_filepath(mimetype=mimetype) if os.path.isfile(filepath): - #print('File already exist') + # print('File already exist') return False # create dir dirname = os.path.dirname(filepath) if not os.path.exists(dirname): os.makedirs(dirname) with open(filepath, 'wb') as f: - f.write(file_content) + f.write(b_content) + + # create meta + self._add_create() + self.set_mimetype(mimetype) + self.set_size(os.path.getsize(filepath)) - # create hash metadata - self._save_meta(filepath, mimetype) return True - # avoid counting the same hash multiple time on the same item - # except if defferent encoding + def add(self, algo_name, date, obj_id, mimetype=None): + self._add(date, obj_id) + if not mimetype: + mimetype = self.get_mimetype() - def is_seen_this_day(self, date): - for decoder in get_decoders_names(): - if r_metadata.zscore(f'{decoder}_date:{date}', self.id): - return True - return False + is_new_decoded = r_objects.sadd(f'decoded:algo:{algo_name}:{date}', self.id) # filter by algo + sparkline + # uniq decoded in this algo today + if int(is_new_decoded): + r_objects.zincrby(f'decoded:algos:{date}', 1, algo_name) # pie chart - def add(self, decoder_name, date, obj_id, mimetype): + # mimetype -> decodeds + is_new_decoded = r_objects.sadd(f'decoded:mime:{mimetype}:{date}', self.id) # filter by mimetype + # uniq decoded in this mimetype today + if int(is_new_decoded): + r_objects.zincrby(f'decoded:mime:{date}', 1, mimetype) # TDO ADD OPTION TO CALC IF NOT EXISTS + r_objects.sadd('decoded:mimetypes', mimetype) - if not self.is_seen_this_day(date): - # mimetype - r_metadata.zincrby(f'decoded:mimetype:{date}', 1, mimetype) - r_metadata.sadd(f'decoded:mimetypes', mimetype) - - # filter hash encoded in the same object - if not self.is_correlated('item', None, obj_id): - - r_metadata.hincrby(f'metadata_hash:{self.id}', f'{decoder_name}_decoder', 1) - r_metadata.zincrby(f'{decoder_name}_type:{mimetype}', 1, date) - - r_metadata.incrby(f'{decoder_name}_decoded:{date}', 1) - r_metadata.zincrby(f'{decoder_name}_date:{date}', 1, self.id) - - - self.update_daterange(date) - - # First Hash for this decoder this day - - - - - - - - # Correlations - self.add_correlation('item', '', obj_id) - # domain - if is_crawled(obj_id): - domain = get_item_domain(obj_id) - self.add_correlation('domain', '', domain) - - - # Filter duplicates ###################################################################### - # Filter on item + hash for this day - - # filter Obj Duplicate - - # first time we see this day - # iterate on all decoder - - - - - ###################################################################### - - # first time we see this hash today - - - # mimetype # # # # # # # # - r_metadata.zincrby(f'decoded:mimetype:{date}', 1, mimetype) - - # create hash metadata - r_metadata.sadd(f'decoded:mimetypes', mimetype) - - - - # # TODO: DUPLICATES + check fields - def add(self, decoder_name, date, obj_id, mimetype): - self.update_daterange(date) - - r_metadata.incrby(f'{decoder_type}_decoded:{date}', 1) - r_metadata.zincrby(f'{decoder_type}_date:{date}', 1, self.id) - - r_metadata.hincrby(f'metadata_hash:{self.id}', f'{decoder_type}_decoder', 1) - r_metadata.zincrby(f'{decoder_type}_type:{mimetype}', 1, date) # # TODO: # DUP1 - - ################################################################ # TODO: REMOVE ????????????????????????????????? - r_metadata.zincrby(f'{decoder_type}_hash:{self.id}', 1, obj_id) # number of b64 on this item - - - # first time we see this hash encoding on this item - if not r_metadata.zscore(f'{decoder_type}_hash:{self.id}', obj_id): - - # create hash metadata - r_metadata.sadd(f'hash_{decoder_type}_all_type', mimetype) - - # first time we see this hash encoding today - if not r_metadata.zscore(f'{decoder_type}_date:{date}', self.id): - r_metadata.zincrby(f'{decoder_type}_type:{mimetype}', 1, date) # # TODO: # DUP1 - - - # Correlations - self.add_correlation('item', '', obj_id) - # domain - if is_crawled(obj_id): - domain = get_item_domain(obj_id) - self.add_correlation('domain', '', domain) - - - # NB of MIMETYPE / DAY -> ALL HASH OR UNIQ HASH ?????? + # algo + mimetype -> Decodeds + # -> sinter with r_objects.sunion(f'decoded:algo:{algo_name}:{date}') # # TODO: ADD items def create(self, content, date, mimetype=None): @@ -324,17 +240,14 @@ class Decoded(AbstractObject): ####################################################################################### ####################################################################################### - ####################################################################################### - ####################################################################################### - def is_vt_enabled(self): return VT_ENABLED def set_vt_report(self, report): - r_metadata.hset(f'metadata_hash:{self.id}', 'vt_report', report) + r_objects.hset(f'meta:{self.type}:{self.id}', 'vt_report', report) def set_meta_vt(self, link, report): - r_metadata.hset(f'metadata_hash:{self.id}', 'vt_link', link) + r_objects.hset(f'meta:{self.type}:{self.id}', 'vt_link', link) self.set_vt_report(report) def refresh_vt_report(self): @@ -371,17 +284,68 @@ class Decoded(AbstractObject): link = json_response['permalink'].split('analysis')[0] + 'analysis/' self.set_meta_vt(link, 'Please Refresh') - ############################################################################ - ############################################################################ +############################################################################ -def get_decoders_names(): +def is_vt_enabled(): + return VT_ENABLED + +def get_all_decodeds(): + return r_objects.smembers(f'decoded:all') + +def get_algos(): return ['base64', 'binary', 'hexadecimal'] def get_all_mimetypes(): - return r_metadata.smembers(f'decoded:mimetypes') + return r_objects.smembers('decoded:mimetypes') + +def get_nb_decodeds_by_date(date): + return r_objects.zcard(f'decoded:date:{date}') + +def get_decodeds_by_date(date): + return r_objects.zrange(f'decoded:date:{date}', 0, -1) + +def get_algo_decodeds_by_date(date, algo): + return r_objects.smembers(f'decoded:algo:{algo}:{date}') + +def get_mimetype_decodeds_by_date(date, mimetype): + return r_objects.smembers(f'decoded:mime:{mimetype}:{date}') + +def get_algo_mimetype_decodeds_by_date(date, algo, mimetype): + return r_objects.sinter(f'decoded:algo:{algo}:{date}', f'decoded:mime:{mimetype}:{date}') + +def get_decodeds_by_daterange(date_from, date_to, algo=None, mimetype=None): + decodeds = set() + if not algo and not mimetype: + for date in Date.substract_date(date_from, date_to): + decodeds = decodeds | set(get_decodeds_by_date(date)) + elif algo and not mimetype: + for date in Date.substract_date(date_from, date_to): + decodeds = decodeds | get_algo_decodeds_by_date(date, algo) + elif mimetype and not algo: + for date in Date.substract_date(date_from, date_to): + decodeds = decodeds | get_mimetype_decodeds_by_date(date, mimetype) + elif algo and mimetype: + for date in Date.substract_date(date_from, date_to): + decodeds = decodeds | get_algo_mimetype_decodeds_by_date(date, algo, mimetype) + return decodeds + +def sanitise_algo(algo): + if algo in get_algos(): + return algo + else: + return None + +def sanitise_mimetype(mimetype): + if mimetype: + if r_objects.sismember('decoded:mimetypes', mimetype): + return mimetype + else: + return None + +############################################################################ def sanityze_decoder_names(decoder_name): - if decoder_name not in Decodeds.get_decoders_names(): + if decoder_name not in Decodeds.get_algos(): return None else: return decoder_name @@ -389,11 +353,12 @@ def sanityze_decoder_names(decoder_name): def sanityze_mimetype(mimetype): if mimetype == 'All types': return None - elif not r_metadata.sismember('hash_all_type', mimetype): + elif not r_objects.sismember(f'decoded:mimetypes', mimetype): return None else: return mimetype +# TODO def pie_chart_mimetype_json(date_from, date_to, mimetype, decoder_name): if mimetype: all_mimetypes = [mimetype] @@ -404,33 +369,121 @@ def pie_chart_mimetype_json(date_from, date_to, mimetype, decoder_name): for mimet in all_mimetypes: pass +# TODO def pie_chart_decoder_json(date_from, date_to, mimetype): - all_decoder = get_decoders_names() + all_algos = get_algos() date_range = Date.substract_date(date_from, date_to) if not date_range: date_range.append(Date.get_today_date_str()) - nb_decoder = {} + nb_algos = {} for date in date_range: - for decoder_name in all_decoder: - if not mimetype: - nb = r_metadata.get(f'{decoder_name}_decoded:{date}') - if nb is None: - nb = 0 - else: - nb = int(nb) + for algo_name in all_algos: + # if not mimetype: + nb = r_objects.zscore(f'decoded:algos:{date}', algo_name) + # TODO mimetype necoding per day + # else: + # nb = r_metadata.zscore(f'{algo_name}_type:{mimetype}', date) + if nb is None: + nb = 0 else: - nb = r_metadata.zscore(f'{decoder_name}_type:{mimetype}', date) - nb_decoder[decoder_name] = nb_decoder.get(decoder_name, 0) + nb + nb = int(nb) + nb_algos[algo_name] = nb_algos.get(algo_name, 0) + nb pie_chart = [] - for decoder_name in all_decoder: - pie_chart.append({'name': decoder_name, 'value': nb_decoder[decoder_name]}) + for algo_name in all_algos: + pie_chart.append({'name': algo_name, 'value': nb_algos[algo_name]}) return pie_chart +# TODO FILTER BY ALGO +def pie_chart_mimetype_json(date_from, date_to, algo): + date_range = Date.substract_date(date_from, date_to) + if not date_range: + date_range.append(Date.get_today_date_str()) + mimetypes = {} + if len(date_range) == 1: + mimes = r_objects.zrange(f'decoded:mime:{date_range[0]}', 0, -1, withscores=True) + for t_mime in mimes: + mime, nb = t_mime + mimetypes[mime] = int(nb) + else: + mimetypes = {} + for date in date_range: + mimes = r_objects.zrange(f'decoded:mime:{date}', 0, -1, withscores=True) + for t_mime in mimes: + mime, nb = t_mime + mimetypes[mime] = mimetypes.get(mime, 0) + int(nb) + top5_mimes = sorted(mimetypes, key=mimetypes.get, reverse=True)[:5] + pie_chart = [] + for mime in top5_mimes: + pie_chart.append({'name': mime, 'value': mimetypes[mime]}) + return pie_chart + +def barchart_range_json(date_from, date_to, mimetype=None): + date_range = Date.substract_date(date_from, date_to) + if not date_range: + date_range.append(Date.get_today_date_str()) + barchart = [] + if mimetype: + for date in date_range: + range_day = {'date': f'{date[0:4]}-{date[4:6]}-{date[6:8]}'} + nb_day = r_objects.scard(f'decoded:mime:{mimetype}:{date}') + range_day[mimetype] = nb_day + barchart.append(range_day) + else: + # algo by mimetype, date = mimetype + if len(date_range) == 1: + mimes = r_objects.zrange(f'decoded:mime:{date_range[0]}', 0, -1, withscores=True) + # TODO + # UNION + # f'decoded:algo:{algo_name}:{date}' + # f'decoded:mime:{mimetype}:{date}' + for t_mime in mimes: + mime, nb = t_mime + range_day = {'date': mime, 'mimetype': nb} + barchart.append(range_day) + # mimetypes by date + else: + mimetypes = set() + for date in date_range: + range_day = {'date': f'{date[0:4]}-{date[4:6]}-{date[6:8]}'} + mimes = r_objects.zrange(f'decoded:mime:{date}', 0, -1, withscores=True) + for t_mime in mimes: + mime, nb = t_mime + mimetypes.add(mime) + range_day[mime] = int(nb) + barchart.append(range_day) + if not mimetypes: + mimetypes.add('No Data') + for row in barchart: + for mime in mimetypes: + if mime not in row: + row[mime] = 0 + return barchart + +def graphline_json(decoded_id): + decoded = Decoded(decoded_id) + graphline = [] + if decoded.exists(): + nb_day = 30 + for date in Date.get_previous_date_list(nb_day): + graphline.append({'date': f'{date[0:4]}-{date[4:6]}-{date[6:8]}', 'value': decoded.get_nb_seen_by_date(date)}) + return graphline + def api_pie_chart_decoder_json(date_from, date_to, mimetype): mimetype = sanityze_mimetype(mimetype) date = Date.sanitise_date_range(date_from, date_to) return pie_chart_decoder_json(date['date_from'], date['date_to'], mimetype) +def api_pie_chart_mimetype_json(date_from, date_to, algo): + algo = sanitise_algo(algo) + date = Date.sanitise_date_range(date_from, date_to) + return pie_chart_mimetype_json(date['date_from'], date['date_to'], algo) + +def api_barchart_range_json(date_from, date_to, mimetype): + date = Date.sanitise_date_range(date_from, date_to) + if mimetype: + mimetype = sanityze_mimetype(mimetype) + return barchart_range_json(date['date_from'], date['date_to'], mimetype=mimetype) + def _delete_old_json_descriptor(): decodeds = [] hash_dir = os.path.join(os.environ['AIL_HOME'], HASH_DIR) @@ -441,15 +494,17 @@ def _delete_old_json_descriptor(): os.remove(decoded_path) return decodeds -def get_all_decodeds(): + +# TODO +def get_all_decodeds_files(): decodeds = [] hash_dir = os.path.join(os.environ['AIL_HOME'], HASH_DIR) if not hash_dir.endswith("/"): hash_dir = f"{hash_dir}/" for root, dirs, files in os.walk(hash_dir): for file in files: - decoded_path = f'{root}{file}' + # decoded_path = f'{root}{file}' decodeds.append(file) return decodeds - #if __name__ == '__main__': +# if __name__ == '__main__': diff --git a/bin/lib/objects/abstract_daterange_object.py b/bin/lib/objects/abstract_daterange_object.py index fd721da6..5613e5c0 100755 --- a/bin/lib/objects/abstract_daterange_object.py +++ b/bin/lib/objects/abstract_daterange_object.py @@ -65,7 +65,7 @@ class AbstractDaterangeObject(AbstractObject, ABC): return last_seen def get_nb_seen(self): - return r_object.hget(f'meta:{self.type}:{self.id}', 'nb') + return self.get_nb_correlation('item') def get_nb_seen_by_date(self, date): nb = r_object.zscore(f'{self.type}:date:{date}', self.id) @@ -108,20 +108,23 @@ class AbstractDaterangeObject(AbstractObject, ABC): sparkline.append(self.get_nb_seen_by_date(date)) return sparkline + def _add_create(self): + r_object.sadd(f'{self.type}:all', self.id) + + # TODO don't increase nb if same hash in item with different encoding + # if hash already in item def _add(self, date, item_id): if not self.exists(): + self._add_create(date) self.set_first_seen(date) self.set_last_seen(date) - r_object.sadd(f'{self.type}:all', self.id) else: self.update_daterange(date) update_obj_date(date, self.type) # NB Object seen by day - print(f'{self.type}:date:{date}', 1, self.id) - r_object.zincrby(f'{self.type}:date:{date}', 1, self.id) - # NB Object seen - r_object.hincrby(f'meta:{self.type}:{self.id}', 'nb', 1) + if not self.is_correlated('item', '', item_id): # if decoded not already in object + r_object.zincrby(f'{self.type}:date:{date}', 1, self.id) # Correlations self.add_correlation('item', '', item_id) diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index 76b3f04d..0b8b78ee 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -19,7 +19,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from lib import Tag from lib import Duplicate -from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, exists_obj_correlation, is_obj_correlated +from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations from lib.Tracker import is_obj_tracked, get_obj_all_trackers, delete_obj_trackers @@ -219,6 +219,9 @@ class AbstractObject(ABC): """ return get_correlations(self.type, self.subtype, self.id) + def get_nb_correlation(self, correl_type): + return get_nb_correlation_by_correl_type(self.type, self.get_subtype(r_str=True), self.id, correl_type) + def get_nb_correlations(self, filter_types=[]): return get_nb_correlations(self.type, self.subtype, self.id, filter_types=filter_types) diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 03c09ea6..db5c0abe 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -97,6 +97,21 @@ def get_object_svg(obj_type, subtype, id): obj = get_object(obj_type, subtype, id) return obj.get_svg_icon() +## TAGS ## +def get_obj_tags(obj_type, subtype, id): + obj = get_object(obj_type, subtype, id) + return obj.get_tags() + +def add_obj_tag(obj_type, subtype, id, tag): + obj = get_object(obj_type, subtype, id) + obj.add_tag(tag) + +def add_obj_tags(obj_type, subtype, id, tags): + obj = get_object(obj_type, subtype, id) + for tag in tags: + obj.add_tag(tag) +# -TAGS- # + def get_object_meta(obj_type, subtype, id, options=[], flask_context=False): obj = get_object(obj_type, subtype, id) meta = obj.get_meta(options=options) @@ -117,6 +132,7 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False): meta['icon'] = obj.get_svg_icon() if subtype or obj_type == 'cve': meta['sparkline'] = obj.get_sparkline() + meta['cve_search'] = obj.get_cve_search() if subtype == 'bitcoin' and related_btc: meta["related_btc"] = btc_ail.get_bitcoin_info(obj.id) if obj.get_type() == 'decoded': diff --git a/bin/modules/Decoder.py b/bin/modules/Decoder.py index 85449498..b14b5879 100755 --- a/bin/modules/Decoder.py +++ b/bin/modules/Decoder.py @@ -9,15 +9,10 @@ ################################## # Import External packages ################################## -import time import os import base64 from hashlib import sha1 -import magic -import json -import datetime import re -import signal import sys sys.path.append(os.environ['AIL_BIN']) @@ -27,49 +22,27 @@ sys.path.append(os.environ['AIL_BIN']) from modules.abstract_module import AbstractModule from lib.ConfigLoader import ConfigLoader from lib.objects.Items import Item -from lib.objects import Decodeds +from lib.objects.Decodeds import Decoded config_loader = ConfigLoader() -serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") hex_max_execution_time = config_loader.get_config_int("Hex", "max_execution_time") binary_max_execution_time = config_loader.get_config_int("Binary", "max_execution_time") base64_max_execution_time = config_loader.get_config_int("Base64", "max_execution_time") config_loader = None -##################################################### -##################################################### - -# # TODO: use regex_helper -class TimeoutException(Exception): - pass - -def timeout_handler(signum, frame): - raise TimeoutException - - -# # TODO: # FIXME: Remove signal -> replace with regex_helper -signal.signal(signal.SIGALRM, timeout_handler) - - -##################################################### -#################################################### - class Decoder(AbstractModule): """ Decoder module for AIL framework """ - # TODO to lambda expr def hex_decoder(self, hexStr): - #hexStr = ''.join( hex_string.split(" ") ) + # hexStr = ''.join( hex_string.split(" ") ) return bytes(bytearray([int(hexStr[i:i+2], 16) for i in range(0, len(hexStr), 2)])) - # TODO to lambda expr def binary_decoder(self, binary_string): return bytes(bytearray([int(binary_string[i:i+8], 2) for i in range(0, len(binary_string), 8)])) - # TODO to lambda expr def base64_decoder(self, base64_string): return base64.b64decode(base64_string) @@ -86,18 +59,20 @@ class Decoder(AbstractModule): cmp_regex_base64 = re.compile(regex_base64) # map decoder function - self.decoder_function = {'binary': self.binary_decoder, 'hexadecimal': self.hex_decoder, 'base64': self.base64_decoder} + self.decoder_function = {'binary': self.binary_decoder, + 'hexadecimal': self.hex_decoder, + 'base64': self.base64_decoder} # list all decoder with regex, - decoder_binary = {'name': 'binary', 'regex': cmp_regex_binary, 'encoded_min_size': 300, 'max_execution_time': binary_max_execution_time} - decoder_hexadecimal = {'name': 'hexadecimal', 'regex': cmp_regex_hex, 'encoded_min_size': 300, 'max_execution_time': hex_max_execution_time} - decoder_base64 = {'name': 'base64', 'regex': cmp_regex_base64, 'encoded_min_size': 40, 'max_execution_time': base64_max_execution_time} + decoder_binary = {'name': 'binary', 'regex': cmp_regex_binary, + 'encoded_min_size': 300, 'max_execution_time': binary_max_execution_time} + decoder_hexadecimal = {'name': 'hexadecimal', 'regex': cmp_regex_hex, + 'encoded_min_size': 300, 'max_execution_time': hex_max_execution_time} + decoder_base64 = {'name': 'base64', 'regex': cmp_regex_base64, + 'encoded_min_size': 40, 'max_execution_time': base64_max_execution_time} self.decoder_order = [decoder_base64, decoder_binary, decoder_hexadecimal, decoder_base64] - for decoder in self.decoder_order: - serv_metadata.sadd('all_decoder', decoder['name']) - # Waiting time in seconds between to message processed self.pending_seconds = 1 @@ -110,63 +85,47 @@ class Decoder(AbstractModule): content = item.get_content() date = item.get_date() - for decoder in self.decoder_order: # add threshold and size limit - # max execution time on regex - signal.alarm(decoder['max_execution_time']) + for decoder in self.decoder_order: + find = False + dname = decoder['name'] - try: - encoded_list = decoder['regex'].findall(content) - except TimeoutException: - encoded_list = [] - self.process.incr_module_timeout_statistic() # add encoder type - self.redis_logger.debug(f"{item.id} processing timeout") - continue - else: - signal.alarm(0) - - if len(encoded_list) > 0: - content = self.decode_string(content, item.id, date, encoded_list, decoder['name'], decoder['encoded_min_size']) - - def decode_string(self, content, item_id, date, encoded_list, decoder_name, encoded_min_size): - find = False - for encoded in encoded_list: - if len(encoded) >= encoded_min_size: - decoded_file = self.decoder_function[decoder_name](encoded) - find = True - - sha1_string = sha1(decoded_file).hexdigest() - decoded = Decoded(sha1_string) - - mimetype = decoded.guess_mimetype(decoded_file) - if not mimetype: - print(item_id) - print(sha1_string) - raise Exception(f'Invalid mimetype: {sha1_string} {item_id}') - - decoded.create(content, date) - decoded.add(decoder_name, date, item_id, mimetype) - - save_item_relationship(sha1_string, item_id) ################################ - - # remove encoded from item content + encodeds = self.regex_findall(decoder['regex'], item.id, content) + # PERF remove encoded from item content + for encoded in encodeds: content = content.replace(encoded, '', 1) + encodeds = set(encodeds) - self.redis_logger.debug(f'{item_id} : {decoder_name} - {mimetype}') - print(f'{item_id} : {decoder_name} - {mimetype}') - if find: - self.redis_logger.info(f'{decoder_name} decoded') - print(f'{decoder_name} decoded') + for encoded in encodeds: + find = False + if len(encoded) >= decoder['encoded_min_size']: + decoded_file = self.decoder_function[dname](encoded) - # Send to Tags - msg = f'infoleak:automatic-detection="{decoder_name}";{item_id}' - self.send_message_to_queue(msg, 'Tags') + sha1_string = sha1(decoded_file).hexdigest() + decoded = Decoded(sha1_string) - # perf: remove encoded from item content - return content + if not decoded.exists(): + mimetype = decoded.guess_mimetype(decoded_file) + if not mimetype: + print(sha1_string, item.id) + raise Exception(f'Invalid mimetype: {decoded.id} {item.id}') + decoded.save_file(decoded_file, mimetype) + else: + mimetype = decoded.get_mimetype() + decoded.add(dname, date, item.id, mimetype=mimetype) + + # DEBUG + self.redis_logger.debug(f'{item.id} : {dname} - {decoded.id} - {mimetype}') + print(f'{item.id} : {dname} - {decoded.id} - {mimetype}') + + if find: + self.redis_logger.info(f'{item.id} - {dname}') + print(f'{item.id} - {dname}') + + # Send to Tags + msg = f'infoleak:automatic-detection="{dname}";{item.id}' + self.send_message_to_queue(msg, 'Tags') if __name__ == '__main__': - - # # TODO: TEST ME module = Decoder() module.run() diff --git a/bin/modules/abstract_module.py b/bin/modules/abstract_module.py index b3f584ac..64f454a0 100644 --- a/bin/modules/abstract_module.py +++ b/bin/modules/abstract_module.py @@ -87,7 +87,7 @@ class AbstractModule(ABC): def regex_finditer(self, regex, obj_id, content): return regex_helper.regex_finditer(self.r_cache_key, regex, obj_id, content, max_time=self.max_execution_time) - def regex_findall(self, regex, id, content): + def regex_findall(self, regex, id, content, r_set=False): """ regex findall helper (force timeout) :param regex: compiled regex @@ -96,7 +96,7 @@ class AbstractModule(ABC): ex: send_to_queue(item_id, 'Global') """ - return regex_helper.regex_findall(self.module_name, self.r_cache_key, regex, id, content, max_time=self.max_execution_time) + return regex_helper.regex_findall(self.module_name, self.r_cache_key, regex, id, content, max_time=self.max_execution_time, r_set=r_set) def run(self): """ diff --git a/bin/packages/Date.py b/bin/packages/Date.py index 86646399..bc68caf3 100644 --- a/bin/packages/Date.py +++ b/bin/packages/Date.py @@ -183,3 +183,42 @@ def sanitise_date_range(date_from, date_to, separator='', date_type='str'): date_from = date_to date_to = res return {"date_from": date_from, "date_to": date_to} + +def sanitise_daterange(date_from, date_to, separator='', date_type='str'): + ''' + Check/Return a correct date_form and date_to + ''' + if not date_from and date_to: + date_from = date_to + elif not date_to and date_from: + date_to = date_from + elif not date_to and not date_from: + date = datetime.date.today().strftime("%Y%m%d") + return date, date + + if date_type == 'str': + # remove separators + if len(date_from) == 10: + date_from = date_from[0:4] + date_from[5:7] + date_from[8:10] + if len(date_to) == 10: + date_to = date_to[0:4] + date_to[5:7] + date_to[8:10] + + if not validate_str_date(date_from, separator=separator): + date_from = datetime.date.today().strftime("%Y%m%d") + if not validate_str_date(date_to, separator=separator): + date_to = datetime.date.today().strftime("%Y%m%d") + else: # datetime + if isinstance(date_from, datetime.datetime): + date_from = date_from.strftime("%Y%m%d") + else: + date_from = datetime.date.today().strftime("%Y%m%d") + if isinstance(date_to, datetime.datetime): + date_to = date_to.strftime("%Y%m%d") + else: + date_to = datetime.date.today().strftime("%Y%m%d") + + if int(date_from) > int(date_to): + res = date_from + date_from = date_to + date_to = res + return date_from, date_to diff --git a/var/www/blueprints/objects_cve.py b/var/www/blueprints/objects_cve.py index a1178568..7d4ac1a2 100644 --- a/var/www/blueprints/objects_cve.py +++ b/var/www/blueprints/objects_cve.py @@ -83,5 +83,15 @@ def objects_cve_search(): else: return redirect(cve.get_link(flask_context=True)) +@objects_cve.route("/objects/cve/graphline/json", methods=['GET']) +@login_required +@login_read_only +def objects_cve_graphline_json(): + cve_id = request.args.get('id') + cve = Cves.Cve(cve_id) + if not cve.exists(): + abort(404) + return jsonify(Cves.get_cve_graphline(cve_id)) + # ============= ROUTES ============== diff --git a/var/www/blueprints/objects_decoded.py b/var/www/blueprints/objects_decoded.py index 0a7c041e..b6623407 100644 --- a/var/www/blueprints/objects_decoded.py +++ b/var/www/blueprints/objects_decoded.py @@ -20,10 +20,12 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from lib.objects import Decodeds - +from packages import Date # ============ BLUEPRINT ============ -objects_decoded = Blueprint('objects_decoded', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/decoded')) +objects_decoded = Blueprint('objects_decoded', __name__, + template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/decoded')) + # ============ VARIABLES ============ @@ -33,9 +35,60 @@ objects_decoded = Blueprint('objects_decoded', __name__, template_folder=os.path # ============= ROUTES ============== -# # TODO: # FIXME: CHECK IF OBJ EXIST +@objects_decoded.route("/object/decodeds", methods=['GET', 'POST']) +@login_required +@login_read_only +def decodeds_dashboard(): + if request.method == 'POST': + date_from = request.form.get('date_from') + date_to = request.form.get('date_to') + mimetype = request.form.get('mimetype') + algo = request.form.get('algo') + show_decoded = request.form.get('show_decoded') + return redirect( + url_for('objects_decoded.decodeds_dashboard', date_from=date_from, date_to=date_to, mimetype=mimetype, + algo=algo, show=show_decoded)) + else: + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + mimetype = request.args.get('mimetype') + algo = request.args.get('algo') + show_decoded = request.args.get('show') + if show_decoded: + show_decoded = True -@objects_decoded.route("/object/decoded/download") #completely shows the paste in a new tab + if mimetype == 'All types': + mimetype = None + if algo == 'All encoding': + algo = None + + algo = Decodeds.sanitise_algo(algo) + mimetype = Decodeds.sanitise_mimetype(mimetype) + date_from, date_to = Date.sanitise_daterange(date_from, date_to) + metas = [] + if show_decoded: + decodeds = Decodeds.get_decodeds_by_daterange(date_from, date_to, algo=algo, mimetype=mimetype) + metas = [] + for decoded_id in decodeds: + decoded = Decodeds.Decoded(decoded_id) + metas.append(decoded.get_meta(options={'sparkline', 'mimetype', 'icon', 'size', 'vt'})) + + # TODO GET PIE CHARTS + + return render_template("decoded/decodeds_dashboard.html", metas=metas, vt_enabled=Decodeds.is_vt_enabled(), + date_from=date_from, date_to=date_to, algo=algo, mimetype=mimetype, + algos=Decodeds.get_algos(), show_decoded=show_decoded, + mimetypes=Decodeds.get_all_mimetypes()) + +@objects_decoded.route("/object/decodeds/search", methods=['POST']) +@login_required +@login_read_only +def decodeds_search(): + decoded_id = request.form.get('object_id') + print(decoded_id) + return redirect(url_for('correlation.show_correlation', type='decoded', id=decoded_id)) + +@objects_decoded.route("/object/decoded/download") @login_required @login_read_only def decoded_download(): @@ -51,7 +104,8 @@ def decoded_download(): else: abort(404) -@objects_decoded.route("/object/decoded/send_to_vt") #completely shows the paste in a new tab + +@objects_decoded.route("/object/decoded/send_to_vt") @login_required @login_read_only def send_to_vt(): @@ -66,7 +120,8 @@ def send_to_vt(): else: abort(404) -@objects_decoded.route("/object/decoded/refresh_vt_report") #completely shows the paste in a new tab + +@objects_decoded.route("/object/decoded/refresh_vt_report") @login_required @login_read_only def refresh_vt_report(): @@ -81,18 +136,42 @@ def refresh_vt_report(): else: abort(404) -@objects_decoded.route("/object/decoded/decoder_pie_chart_json", methods=['GET']) + +# TODO +@objects_decoded.route("/object/decoded/algo_pie_chart/json", methods=['GET']) @login_required @login_read_only def decoder_pie_chart_json(): date_from = request.args.get('date_from') date_to = request.args.get('date_to') - mimetype = request.args.get('type') + mimetype = request.args.get('mimetype') return jsonify(Decodeds.api_pie_chart_decoder_json(date_from, date_to, mimetype)) +# TODO +@objects_decoded.route("/object/decoded/mimetype_pie_chart/json", methods=['GET']) +@login_required +@login_read_only +def mimetype_pie_chart_json(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + algo = request.args.get('algo') + return jsonify(Decodeds.api_pie_chart_mimetype_json(date_from, date_to, algo)) +@objects_decoded.route("/object/decoded/barchart/json", methods=['GET']) +@login_required +@login_read_only +def barchart_json(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + mimetype = request.args.get('mimetype') + return jsonify(Decodeds.api_barchart_range_json(date_from, date_to , mimetype)) - - - -#####################################################3 +@objects_decoded.route("/object/decoded/graphline/json", methods=['GET']) +@login_required +@login_read_only +def graphline_json(): + decoded_id = request.args.get('id') + decoded = Decodeds.Decoded(decoded_id) + if not decoded: + abort(404) + return jsonify(Decodeds.graphline_json(decoded_id)) diff --git a/var/www/modules/hashDecoded/Flask_hashDecoded.py b/var/www/modules/hashDecoded/Flask_hashDecoded.py index 03fcc453..e2fba72e 100644 --- a/var/www/modules/hashDecoded/Flask_hashDecoded.py +++ b/var/www/modules/hashDecoded/Flask_hashDecoded.py @@ -59,46 +59,6 @@ def substract_date(date_from, date_to): l_date.append( date.strftime('%Y%m%d') ) return l_date -def list_sparkline_values(date_range_sparkline, hash): - sparklines_value = [] - for date_day in date_range_sparkline: - nb_seen_this_day = r_serv_metadata.zscore('hash_date:'+date_day, hash) - if nb_seen_this_day is None: - nb_seen_this_day = 0 - sparklines_value.append(int(nb_seen_this_day)) - return sparklines_value - -def get_file_icon(estimated_type): - file_type = estimated_type.split('/')[0] - # set file icon - if file_type == 'application': - file_icon = 'fa-file ' - elif file_type == 'audio': - file_icon = 'fa-file-audio ' - elif file_type == 'image': - file_icon = 'fa-file-image' - elif file_type == 'text': - file_icon = 'fa-file-alt' - else: - file_icon = 'fa-sticky-note' - - return file_icon - -def get_file_icon_text(estimated_type): - file_type = estimated_type.split('/')[0] - # set file icon - if file_type == 'application': - file_icon_text = '\uf15b' - elif file_type == 'audio': - file_icon_text = '\uf1c7' - elif file_type == 'image': - file_icon_text = '\uf1c5' - elif file_type == 'text': - file_icon_text = '\uf15c' - else: - file_icon_text = '\uf249' - - return file_icon_text def get_icon(correlation_type, type_id): icon_text = 'fas fa-sticky-note' @@ -182,16 +142,6 @@ def list_sparkline_type_id_values(date_range_sparkline, correlation_type, type_i sparklines_value.append(int(nb_seen_this_day)) return sparklines_value -def get_all_keys_id_from_item(correlation_type, item_path): - all_keys_id_dump = set() - if item_path is not None: - for type_id in get_all_types_id(correlation_type): - res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_type, type_id, item_path)) - for key_id in res: - all_keys_id_dump.add( (key_id, type_id) ) - return all_keys_id_dump - - def get_correlation_type_search_endpoint(correlation_type): if correlation_type == 'pgpdump': endpoint = 'hashDecoded.all_pgpdump_search' @@ -379,345 +329,7 @@ def correlation_type_range_type_json(correlation_type, date_from, date_to): return jsonify(range_type) # ============= ROUTES ============== -@hashDecoded.route("/hashDecoded/all_hash_search", methods=['POST']) -@login_required -@login_read_only -def all_hash_search(): - date_from = request.form.get('date_from') - date_to = request.form.get('date_to') - type = request.form.get('type') - encoding = request.form.get('encoding') - show_decoded_files = request.form.get('show_decoded_files') - return redirect(url_for('hashDecoded.hashDecoded_page', date_from=date_from, date_to=date_to, type=type, encoding=encoding, show_decoded_files=show_decoded_files)) -@hashDecoded.route("/hashDecoded/", methods=['GET']) -@login_required -@login_read_only -def hashDecoded_page(): - date_from = request.args.get('date_from') - date_to = request.args.get('date_to') - type = request.args.get('type') - encoding = request.args.get('encoding') - show_decoded_files = request.args.get('show_decoded_files') - - if type == 'All types': - type = None - - if encoding == 'All encoding': - encoding = None - - #date_from = '20180628' or date_from = '2018-06-28' - #date_to = '20180628' or date_to = '2018-06-28' - - # verify file type input - if type is not None: - #retrieve + char - type = type.replace(' ', '+') - if type not in r_serv_metadata.smembers('hash_all_type'): - type = None - - all_encoding = r_serv_metadata.smembers('all_decoder') - # verify encoding input - if encoding is not None: - if encoding not in all_encoding: - encoding = None - - date_range = [] - if date_from is not None and date_to is not None: - #change format - try: - if len(date_from) != 8: - date_from = date_from[0:4] + date_from[5:7] + date_from[8:10] - date_to = date_to[0:4] + date_to[5:7] + date_to[8:10] - date_range = substract_date(date_from, date_to) - except: - pass - - if not date_range: - date_range.append(datetime.date.today().strftime("%Y%m%d")) - date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8] - date_to = date_from - - else: - date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8] - date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8] - - # display day type bar chart - if len(date_range) == 1 and type is None: - daily_type_chart = True - daily_date = date_range[0] - else: - daily_type_chart = False - daily_date = None - - l_64 = set() - if show_decoded_files: - show_decoded_files = True - for date in date_range: - if encoding is None: - l_hash = r_serv_metadata.zrange('hash_date:' +date, 0, -1) - else: - l_hash = r_serv_metadata.zrange(encoding+'_date:' +date, 0, -1) - if l_hash: - for hash in l_hash: - l_64.add(hash) - - num_day_sparkline = 6 - date_range_sparkline = get_date_range(num_day_sparkline) - - b64_metadata = [] - l_64 = list(l_64) - for hash in l_64: - # select requested base 64 type - estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type') - if type is not None: - if estimated_type is not None: - if estimated_type != type: - continue - - first_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'first_seen') - last_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'last_seen') - nb_seen_in_paste = r_serv_metadata.hget('metadata_hash:'+hash, 'nb_seen_in_all_pastes') - size = r_serv_metadata.hget('metadata_hash:'+hash, 'size') - - if hash is not None and first_seen is not None and \ - last_seen is not None and \ - nb_seen_in_paste is not None and \ - size is not None: - - file_icon = get_file_icon(estimated_type) - - if r_serv_metadata.hexists('metadata_hash:'+hash, 'vt_link'): - b64_vt = True - b64_vt_link = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_link') - b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report') - else: - b64_vt = False - b64_vt_link = '' - b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report') - # hash never refreshed - if b64_vt_report is None: - b64_vt_report = '' - - sparklines_value = list_sparkline_values(date_range_sparkline, hash) - - b64_metadata.append( (file_icon, estimated_type, hash, nb_seen_in_paste, size, first_seen, last_seen, b64_vt, b64_vt_link, b64_vt_report, sparklines_value) ) - - l_type = sorted(r_serv_metadata.smembers('hash_all_type')) - - return render_template("hashDecoded.html", l_64=b64_metadata, vt_enabled=vt_enabled, l_type=l_type, type=type, daily_type_chart=daily_type_chart, daily_date=daily_date, - encoding=encoding, all_encoding=all_encoding, date_from=date_from, date_to=date_to, show_decoded_files=show_decoded_files) - - - - - - -@hashDecoded.route('/hashDecoded/hash_by_type_json') -@login_required -@login_read_only -def hash_by_type_json(): - type = request.args.get('type') - - #retrieve + char - type = type.replace(' ', '+') - - num_day_type = 30 - date_range = get_date_range(num_day_type) - - #verify input - if type in r_serv_metadata.smembers('hash_all_type'): - type_value = [] - all_decoder = r_serv_metadata.smembers('all_decoder') - - range_decoder = [] - for date in date_range: - day_decoder = {} - day_decoder['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8] - for decoder in all_decoder: - num_day_decoder = r_serv_metadata.zscore(decoder+'_type:'+type, date) - if num_day_decoder is None: - num_day_decoder = 0 - day_decoder[decoder]= num_day_decoder - range_decoder.append(day_decoder) - - - - return jsonify(range_decoder) - else: - return jsonify() - - - -@hashDecoded.route('/hashDecoded/top5_type_json') -@login_required -@login_read_only -def top5_type_json(): - date_from = request.args.get('date_from') - date_to = request.args.get('date_to') - - typ = request.args.get('type') - decoder = request.args.get('encoding') - - if decoder == 'All encoding' or decoder is None: - all_decoder = r_serv_metadata.smembers('all_decoder') - else: - if not r_serv_metadata.sismember('all_decoder', decoder): - return jsonify({'Error': 'This decoder do not exist'}) - else: - all_decoder = [decoder] - - if typ == 'All types' or typ is None or typ=='None': - all_type = r_serv_metadata.smembers('hash_all_type') - else: - typ = typ.replace(' ', '+') - if not r_serv_metadata.sismember('hash_all_type', typ): - return jsonify({'Error': 'This type do not exist'}) - else: - all_type = [typ] - - date_range = [] - if date_from is not None and date_to is not None: - #change format - try: - if len(date_from) != 8: - date_from = date_from[0:4] + date_from[5:7] + date_from[8:10] - date_to = date_to[0:4] + date_to[5:7] + date_to[8:10] - date_range = substract_date(date_from, date_to) - except: - pass - - if not date_range: - date_range.append(datetime.date.today().strftime("%Y%m%d")) - - # TODO replace with ZUNIONSTORE - nb_types_decoded = {} - for date in date_range: - for typ in all_type: - for decoder in all_decoder: - nb_decoded = r_serv_metadata.zscore(f'{decoder}_type:{typ}', date) # daily_type key:date mimetype 3 - if nb_decoded is not None: - if typ in nb_types_decoded: - nb_types_decoded[typ] = nb_types_decoded[typ] + int(nb_decoded) - else: - nb_types_decoded[typ] = int(nb_decoded) - - to_json = [] - top5_types = sorted(nb_types_decoded, key=nb_types_decoded.get, reverse=True)[:5] - for typ in top5_types: - to_json.append({'name': typ, 'value': nb_types_decoded[typ]}) - return jsonify(to_json) - - -@hashDecoded.route('/hashDecoded/daily_type_json') -@login_required -@login_read_only -def daily_type_json(): - date = request.args.get('date') - - daily_type = set() - l_b64 = r_serv_metadata.zrange('hash_date:' +date, 0, -1) - for hash in l_b64: - estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type') - if estimated_type is not None: - daily_type.add(estimated_type) - - type_value = [] - for day_type in daily_type: - num_day_type = r_serv_metadata.zscore('hash_type:'+day_type, date) - type_value.append({ 'date' : day_type, 'value' : int( num_day_type )}) - - return jsonify(type_value) - - -@hashDecoded.route('/hashDecoded/range_type_json') -@login_required -@login_read_only -def range_type_json(): - date_from = request.args.get('date_from') - date_to = request.args.get('date_to') - - date_range = [] - if date_from is not None and date_to is not None: - #change format - if len(date_from) != 8: - date_from = date_from[0:4] + date_from[5:7] + date_from[8:10] - date_to = date_to[0:4] + date_to[5:7] + date_to[8:10] - date_range = substract_date(date_from, date_to) - - if not date_range: - date_range.append(datetime.date.today().strftime("%Y%m%d")) - - all_type = set() - for date in date_range: - l_hash = r_serv_metadata.zrange('hash_date:' +date, 0, -1) - if l_hash: - for hash in l_hash: - estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type') - if estimated_type: - all_type.add(estimated_type) - - range_type = [] - - list_decoder = r_serv_metadata.smembers('all_decoder') - for date in date_range: - if len(date_range) == 1: - if date==date_from and date==date_to: - for type in all_type: - day_type = {} - day_type['date']= type - for decoder in list_decoder: - num_day_decoder = r_serv_metadata.zscore(decoder+'_type:'+type, date) - if num_day_decoder is None: - num_day_decoder = 0 - day_type[decoder]= num_day_decoder - range_type.append(day_type) - else: - range_type = '' - else: - day_type = {} - day_type['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8] - for type in all_type: - num_day_type = 0 - for decoder in list_decoder: - num_day_type_decoder = r_serv_metadata.zscore(decoder+'_type:'+type, date) - if num_day_type_decoder is not None: - num_day_type += num_day_type_decoder - day_type[type]= num_day_type - range_type.append(day_type) - - return jsonify(range_type) - - -@hashDecoded.route('/hashDecoded/hash_graph_line_json') -@login_required -@login_read_only -def hash_graph_line_json(): - hash = request.args.get('hash') - date_from = request.args.get('date_from') - date_to = request.args.get('date_to') - - if date_from is None or date_to is None: - nb_days_seen_in_pastes = 30 - else: - # # TODO: # FIXME: - nb_days_seen_in_pastes = 30 - - date_range_seen_in_pastes = get_date_range(nb_days_seen_in_pastes) - - # verify input - if r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type') is not None: - json_seen_in_paste = [] - for date in date_range_seen_in_pastes: - nb_seen_this_day = r_serv_metadata.zscore('hash_date:'+date, hash) - if nb_seen_this_day is None: - nb_seen_this_day = 0 - date = date[0:4] + '-' + date[4:6] + '-' + date[6:8] - json_seen_in_paste.append({'date': date, 'value': int(nb_seen_this_day)}) - - return jsonify(json_seen_in_paste) - else: - return jsonify() ############################ PGPDump ############################ diff --git a/var/www/modules/hashDecoded/templates/DaysCorrelation.html b/var/www/modules/hashDecoded/templates/DaysCorrelation.html index 5bbed4c4..8f8dd2e9 100644 --- a/var/www/modules/hashDecoded/templates/DaysCorrelation.html +++ b/var/www/modules/hashDecoded/templates/DaysCorrelation.html @@ -594,8 +594,7 @@ function barchart_type(url, id) { .attr("transform", "rotate(-20)" ); {% else %} .attr("transform", "rotate(-70)" ) - .attr("class", "bar") - .on("click", function (d) { window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}"+'?date_from='+d+'&date_to='+d }); + .attr("class", "bar"); {% endif %} svg.append("g") @@ -617,12 +616,6 @@ function barchart_type(url, id) { .attr("width", x.bandwidth()) .attr("y", function(d) { return y(d.value); }) .attr("height", function(d) { return height - y(d.value); }) - {% if type %} - .on("click", function(d){ window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}" +'?type={{type}}&date_from='+ d.date +'&date_to='+ d.date; }); - {% endif %} - {% if daily_type_chart %} - .on("click", function(d){ window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}" +'?type='+d.date+'&date_from={{ daily_date }}&date_to={{ daily_date }}'; }); - {% endif %} data.forEach(function(d) { diff --git a/var/www/modules/hashDecoded/templates/header_hashDecoded.html b/var/www/modules/hashDecoded/templates/header_hashDecoded.html index 69fb9da9..17515c9e 100644 --- a/var/www/modules/hashDecoded/templates/header_hashDecoded.html +++ b/var/www/modules/hashDecoded/templates/header_hashDecoded.html @@ -1 +1 @@ -
  • hashesDecoded
    • +
  • hashesDecoded
    • diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 7c6c9362..6bb4c9fa 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -270,9 +270,9 @@ $(document).ready(function(){ {% elif dict_object["object_type"] == "cryptocurrency" %} all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hashDecoded.cryptocurrency_graph_line_json') }}?type_id={{dict_object["metadata"]["type_id"]}}&key_id={{dict_object["correlation_id"]}}"); {% elif dict_object["object_type"] == "decoded" %} - all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hashDecoded.hash_graph_line_json') }}?hash={{dict_object["correlation_id"]}}"); + all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('objects_decoded.graphline_json') }}?id={{dict_object["correlation_id"]}}"); {% elif dict_object["object_type"] == "cve" %} - all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hashDecoded.hash_graph_line_json') }}?hash={{dict_object["correlation_id"]}}"); + all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('objects_cve.objects_cve_graphline_json') }}?id={{dict_object["correlation_id"]}}"); {% endif %} all_graph.onResize(); }); diff --git a/var/www/templates/decoded/decodeds_dashboard.html b/var/www/templates/decoded/decodeds_dashboard.html new file mode 100644 index 00000000..29c602d8 --- /dev/null +++ b/var/www/templates/decoded/decodeds_dashboard.html @@ -0,0 +1,703 @@ + + + + + Decoded - AIL + + + + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'sidebars/sidebar_objects.html' %} + +
    + +
    +
    +
    +
    +
    +
    +
    Search Decoded by name:
    + +
    +
    + + +
    +
    +
    +
    +
    + +
    + +
    +
    +
    Select a date range :
    +
    +
    +
    + +
    +
    +
    + +
    +
    Encoding :
    + +
    File Type :
    + +
    + + +
    + +
    +
    +
    + +
    +
    +
    +
    +
    +
    + + + + {% if metas|length != 0 %} + {% if date_from|string == date_to|string %} +

    {{ date_from }} Decoded files:

    + {% else %} +

    {{ date_from }} to {{ date_to }} Decoded files:

    + {% endif %} + + + + + + + + + + + + + + + {% for meta in metas %} + + + + + + + + + + + {% endfor %} + +
    estimated typehashfirst seenlast seennb itemsizeVirus TotalSparkline
    + + + + {{ meta['icon']['icon'] }} + +   {{ meta['mimetype'] }} + {{ meta['id'] }}{{ meta['first_seen'] }}{{ meta['last_seen'] }}{{ meta['nb_seen'] }}{{ meta['size'] }} + {% if vt_enabled %} + {% if not meta['vt'] %} + + + + {% else %} +  VT Report + {% endif %} + + {% else %} + Virus Total submission is disabled + {% endif %} + +
    + {% else %} + {% if show_decoded %} + {% if date_from|string == date_to|string %} +

    {{ date_from }}, No Decoded

    + {% else %} +

    {{ date_from }} to {{ date_to }}, No Decodeds

    + {% endif %} + {% endif %} + {% endif %} +
    + +
    +
    + + + + + + + + + + + + + + + + + + + + diff --git a/var/www/templates/objects/cve/CveDaterange.html b/var/www/templates/objects/cve/CveDaterange.html index 861c70e7..1b477b0f 100644 --- a/var/www/templates/objects/cve/CveDaterange.html +++ b/var/www/templates/objects/cve/CveDaterange.html @@ -564,7 +564,6 @@ function barchart_type(url, id) { {% else %} .attr("transform", "rotate(-70)" ) .attr("class", "bar") - .on("click", function (d) { window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}"+'?date_from='+d+'&date_to='+d }); {% endif %} svg.append("g") @@ -586,12 +585,6 @@ function barchart_type(url, id) { .attr("width", x.bandwidth()) .attr("y", function(d) { return y(d.value); }) .attr("height", function(d) { return height - y(d.value); }) - {% if type %} - .on("click", function(d){ window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}" +'?type={{type}}&date_from='+ d.date +'&date_to='+ d.date; }); - {% endif %} - {% if daily_type_chart %} - .on("click", function(d){ window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}" +'?type='+d.date+'&date_from={{ daily_date }}&date_to={{ daily_date }}'; }); - {% endif %} data.forEach(function(d) { diff --git a/var/www/templates/sidebars/sidebar_objects.html b/var/www/templates/sidebars/sidebar_objects.html index 9a1650bf..f8508137 100644 --- a/var/www/templates/sidebars/sidebar_objects.html +++ b/var/www/templates/sidebars/sidebar_objects.html @@ -35,7 +35,7 @@
  • - + Decoded