From 47e13431873483604642bcfb412f20f53bf120ad Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 22 Jun 2023 16:09:18 +0200 Subject: [PATCH 01/45] fix: [crawler] same capture uuid if a domain is already crawled --- bin/lib/crawlers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 300edb66..6f6bf5b7 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -1239,7 +1239,8 @@ class CrawlerCapture: def create(self, task_uuid): if self.exists(): - raise Exception(f'Error: Capture {self.uuid} already exists') + print(f'Capture {self.uuid} already exists') # TODO LOGS + return None launch_time = int(time.time()) r_crawler.hset(f'crawler:task:{task_uuid}', 'capture', self.uuid) r_crawler.hset('crawler:captures:tasks', self.uuid, task_uuid) From f1f33d6c1b4d1b1a5171b8d0be1efed1a28bde17 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 22 Jun 2023 16:34:14 +0200 Subject: [PATCH 02/45] chg: [pystemon importer] fix gzipped pastes --- bin/importer/PystemonImporter.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bin/importer/PystemonImporter.py b/bin/importer/PystemonImporter.py index 536801ba..d7a260f5 100755 --- a/bin/importer/PystemonImporter.py +++ b/bin/importer/PystemonImporter.py @@ -35,7 +35,7 @@ class PystemonImporter(AbstractImporter): print(item_id) if item_id: print(item_id) - full_item_path = os.path.join(self.dir_pystemon, item_id) # TODO SANITIZE PATH + full_item_path = os.path.join(self.dir_pystemon, item_id) # TODO SANITIZE PATH # Check if pystemon file exists if not os.path.isfile(full_item_path): print(f'Error: {full_item_path}, file not found') @@ -47,7 +47,12 @@ class PystemonImporter(AbstractImporter): if not content: return None - return self.create_message(item_id, content, source='pystemon') + if full_item_path[-3:] == '.gz': + gzipped = True + else: + gzipped = False + + return self.create_message(item_id, content, gzipped=gzipped, source='pystemon') except IOError as e: print(f'Error: {full_item_path}, IOError') From a0686eefcf005a4db17b19f498d0cfa9bcdbc18d Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 26 Jun 2023 11:24:04 +0200 Subject: [PATCH 03/45] fix: [pystemon importer] fix base64 encoding --- bin/importer/abstract_importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/importer/abstract_importer.py b/bin/importer/abstract_importer.py index e5155775..1c4b458d 100755 --- a/bin/importer/abstract_importer.py +++ b/bin/importer/abstract_importer.py @@ -89,7 +89,7 @@ class AbstractImporter(ABC): # TODO ail queues if not gzipped: content = self.b64_gzip(content) elif not b64: - content = self.b64(gzipped) + content = self.b64(content) if not content: return None if isinstance(content, bytes): From b4d536f174b4f8976e056709a220880f050aa4ec Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 26 Jun 2023 16:28:31 +0200 Subject: [PATCH 04/45] chg: [investigation] add object comment --- bin/lib/Investigations.py | 21 +++++++++++++++---- bin/lib/objects/Items.py | 7 +++---- bin/lib/objects/Screenshots.py | 2 +- bin/lib/objects/abstract_daterange_object.py | 7 ++++--- bin/lib/objects/abstract_object.py | 2 +- var/www/blueprints/investigations_b.py | 15 ++++++++++--- .../investigations/view_investigation.html | 18 ++++++++++------ .../modals/investigations_register_obj.html | 16 ++++++++++---- 8 files changed, 62 insertions(+), 26 deletions(-) diff --git a/bin/lib/Investigations.py b/bin/lib/Investigations.py index 1944d00f..9c6def0f 100755 --- a/bin/lib/Investigations.py +++ b/bin/lib/Investigations.py @@ -235,18 +235,27 @@ class Investigation(object): objs.append(dict_obj) return objs + def get_objects_comment(self, obj_global_id): + return r_tracking.hget(f'investigations:objs:comment:{self.uuid}', obj_global_id) + + def set_objects_comment(self, obj_global_id, comment): + if comment: + r_tracking.hset(f'investigations:objs:comment:{self.uuid}', obj_global_id, comment) + # # TODO: def register_object(self, Object): in OBJECT CLASS - def register_object(self, obj_id, obj_type, subtype): + def register_object(self, obj_id, obj_type, subtype, comment=''): r_tracking.sadd(f'investigations:objs:{self.uuid}', f'{obj_type}:{subtype}:{obj_id}') r_tracking.sadd(f'obj:investigations:{obj_type}:{subtype}:{obj_id}', self.uuid) + if comment: + self.set_objects_comment(f'{obj_type}:{subtype}:{obj_id}', comment) timestamp = int(time.time()) self.set_last_change(timestamp) - def unregister_object(self, obj_id, obj_type, subtype): r_tracking.srem(f'investigations:objs:{self.uuid}', f'{obj_type}:{subtype}:{obj_id}') r_tracking.srem(f'obj:investigations:{obj_type}:{subtype}:{obj_id}', self.uuid) + r_tracking.hdel(f'investigations:objs:comment:{self.uuid}', f'{obj_type}:{subtype}:{obj_id}') timestamp = int(time.time()) self.set_last_change(timestamp) @@ -351,7 +360,7 @@ def get_investigations_selector(): for investigation_uuid in get_all_investigations(): investigation = Investigation(investigation_uuid) name = investigation.get_info() - l_investigations.append({"id":investigation_uuid, "name": name}) + l_investigations.append({"id": investigation_uuid, "name": name}) return l_investigations #{id:'8dc4b81aeff94a9799bd70ba556fa345',name:"Paris"} @@ -453,7 +462,11 @@ def api_register_object(json_dict): if subtype == 'None': subtype = '' obj_id = json_dict.get('id', '').replace(' ', '') - res = investigation.register_object(obj_id, obj_type, subtype) + + comment = json_dict.get('comment', '') + # if comment: + # comment = escape(comment) + res = investigation.register_object(obj_id, obj_type, subtype, comment=comment) return res, 200 def api_unregister_object(json_dict): diff --git a/bin/lib/objects/Items.py b/bin/lib/objects/Items.py index 2e35497e..03c6f2cd 100755 --- a/bin/lib/objects/Items.py +++ b/bin/lib/objects/Items.py @@ -264,10 +264,9 @@ class Item(AbstractObject): """ if options is None: options = set() - meta = {'id': self.id, - 'date': self.get_date(separator=True), - 'source': self.get_source(), - 'tags': self.get_tags(r_list=True)} + meta = self.get_default_meta(tags=True) + meta['date'] = self.get_date(separator=True) + meta['source'] = self.get_source() # optional meta fields if 'content' in options: meta['content'] = self.get_content() diff --git a/bin/lib/objects/Screenshots.py b/bin/lib/objects/Screenshots.py index 19ae3754..26f8543f 100755 --- a/bin/lib/objects/Screenshots.py +++ b/bin/lib/objects/Screenshots.py @@ -88,7 +88,7 @@ class Screenshot(AbstractObject): return obj def get_meta(self, options=set()): - meta = {'id': self.id} + meta = self.get_default_meta() meta['img'] = get_screenshot_rel_path(self.id) ######### # TODO: Rename ME ?????? meta['tags'] = self.get_tags(r_list=True) if 'tags_safe' in options: diff --git a/bin/lib/objects/abstract_daterange_object.py b/bin/lib/objects/abstract_daterange_object.py index b96c5ec4..5ec103d0 100755 --- a/bin/lib/objects/abstract_daterange_object.py +++ b/bin/lib/objects/abstract_daterange_object.py @@ -82,9 +82,10 @@ class AbstractDaterangeObject(AbstractObject, ABC): return int(nb) def _get_meta(self, options=[]): - meta_dict = {'first_seen': self.get_first_seen(), - 'last_seen': self.get_last_seen(), - 'nb_seen': self.get_nb_seen()} + meta_dict = self.get_default_meta() + meta_dict['first_seen'] = self.get_first_seen() + meta_dict['last_seen'] = self.get_last_seen() + meta_dict['nb_seen'] = self.get_nb_seen() if 'sparkline' in options: meta_dict['sparkline'] = self.get_sparkline() return meta_dict diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index cb7595ad..2423a294 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -62,7 +62,7 @@ class AbstractObject(ABC): def get_default_meta(self, tags=False): dict_meta = {'id': self.get_id(), 'type': self.get_type(), - 'subtype': self.get_subtype()} + 'subtype': self.get_subtype(r_str=True)} if tags: dict_meta['tags'] = self.get_tags() return dict_meta diff --git a/var/www/blueprints/investigations_b.py b/var/www/blueprints/investigations_b.py index 8c1d592b..cf3cf688 100644 --- a/var/www/blueprints/investigations_b.py +++ b/var/www/blueprints/investigations_b.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* -''' +""" Blueprint Flask: ail_investigations -''' +""" import os import sys @@ -54,7 +54,13 @@ def show_investigation(): investigation_uuid = request.args.get("uuid") investigation = Investigations.Investigation(investigation_uuid) metadata = investigation.get_metadata(r_str=True) - objs = ail_objects.get_objects_meta(investigation.get_objects(), flask_context=True) + objs = [] + for obj in investigation.get_objects(): + obj_meta = ail_objects.get_object_meta(obj["type"], obj["subtype"], obj["id"], flask_context=True) + comment = investigation.get_objects_comment(f'{obj["type"]}:{obj["subtype"]}:{obj["id"]}') + if comment: + obj_meta['comment'] = comment + objs.append(obj_meta) return render_template("view_investigation.html", bootstrap_label=bootstrap_label, metadata=metadata, investigation_objs=objs) @@ -169,10 +175,13 @@ def register_investigation(): object_type = request.args.get('type') object_subtype = request.args.get('subtype') object_id = request.args.get('id') + comment = request.args.get('comment') for investigation_uuid in investigations_uuid: input_dict = {"uuid": investigation_uuid, "id": object_id, "type": object_type, "subtype": object_subtype} + if comment: + input_dict["comment"] = comment res = Investigations.api_register_object(input_dict) if res[1] != 200: return create_json_response(res[0], res[1]) diff --git a/var/www/templates/investigations/view_investigation.html b/var/www/templates/investigations/view_investigation.html index 4cfcd06e..3848b736 100644 --- a/var/www/templates/investigations/view_investigation.html +++ b/var/www/templates/investigations/view_investigation.html @@ -12,8 +12,8 @@ - - + + @@ -125,11 +125,12 @@ - - - + + + - + + @@ -156,6 +157,11 @@ {{ tag }} {% endfor %} + - + @@ -80,6 +80,10 @@ {% endif %} + {% if dict_uuid['description'] %} +
{{ dict_uuid['description'] }}
+ {% endif %} +
{% for tag in dict_uuid['tags'] %} @@ -145,6 +149,10 @@ {% endif %} + {% if dict_uuid['description'] %} +
{{ dict_uuid['description'] }}
+ {% endif %} +
{% for tag in dict_uuid['tags'] %} From fce63d52a0a20170943d68b4d8de9ec957747c5e Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 29 Jun 2023 09:07:00 +0200 Subject: [PATCH 06/45] fix: [updater] remove old ARDB env --- bin/LAUNCH.sh | 2 +- update/bin/Update_Redis.sh | 4 +--- update/default_update/Update.sh | 6 +++--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 547cd76f..1de7f301 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -27,7 +27,7 @@ fi export PATH=$AIL_VENV/bin:$PATH export PATH=$AIL_HOME:$PATH export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH +export PATH=$AIL_KVROCKS:$PATH export PATH=$AIL_BIN:$PATH export PATH=$AIL_FLASK:$PATH diff --git a/update/bin/Update_Redis.sh b/update/bin/Update_Redis.sh index 238d53f7..dc4d394d 100755 --- a/update/bin/Update_Redis.sh +++ b/update/bin/Update_Redis.sh @@ -2,13 +2,11 @@ [ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; [ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_BIN. Run the script from the virtual environment." && exit 1; [ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; export PATH=$AIL_HOME:$PATH export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH export PATH=$AIL_BIN:$PATH export PATH=$AIL_FLASK:$PATH diff --git a/update/default_update/Update.sh b/update/default_update/Update.sh index 189ae846..c14c4dac 100755 --- a/update/default_update/Update.sh +++ b/update/default_update/Update.sh @@ -7,13 +7,13 @@ fi [ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; [ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_KVROCKS" ] && echo "Needs the env var AIL_KVROCKS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_BIN. Run the script from the virtual environment." && exit 1; [ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; export PATH=$AIL_HOME:$PATH export PATH=$AIL_REDIS:$PATH -export PATH=$AIL_ARDB:$PATH +export PATH=AIL_KVROCKS:$PATH export PATH=$AIL_BIN:$PATH export PATH=$AIL_FLASK:$PATH From 279ec00990ed0cb5104a89f6dbe1516bb831dca9 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 29 Jun 2023 11:40:57 +0200 Subject: [PATCH 07/45] fix: [correlation card decoded meta] mimetype + size --- bin/lib/objects/ail_objects.py | 2 ++ var/www/templates/correlation/metadata_card_decoded.html | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index df598a70..e64d952f 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -177,6 +177,8 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False): if subtype == 'bitcoin' and related_btc: meta["related_btc"] = btc_ail.get_bitcoin_info(obj.id) if obj.get_type() == 'decoded': + meta['mimetype'] = obj.get_mimetype() + meta['size'] = obj.get_size() meta["vt"] = obj.get_meta_vt() meta["vt"]["status"] = obj.is_vt_enabled() # TAGS MODAL diff --git a/var/www/templates/correlation/metadata_card_decoded.html b/var/www/templates/correlation/metadata_card_decoded.html index da57cb21..47a6627d 100644 --- a/var/www/templates/correlation/metadata_card_decoded.html +++ b/var/www/templates/correlation/metadata_card_decoded.html @@ -35,7 +35,7 @@ {{ dict_object["metadata_card"]["icon"]["icon"] }} - {{ dict_object["metadata"]["file_type"] }} + {{ dict_object["metadata"]["mimetype"] }}
From f545daaa4360af236b2b3312bc387b46852a8a0e Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 29 Jun 2023 11:47:13 +0200 Subject: [PATCH 08/45] fix: [correlation card decoded meta] mimetype + size --- .../templates/correlation/metadata_card_decoded.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/var/www/templates/correlation/metadata_card_decoded.html b/var/www/templates/correlation/metadata_card_decoded.html index 47a6627d..1e292a80 100644 --- a/var/www/templates/correlation/metadata_card_decoded.html +++ b/var/www/templates/correlation/metadata_card_decoded.html @@ -35,12 +35,12 @@ {{ dict_object["metadata_card"]["icon"]["icon"] }} - {{ dict_object["metadata"]["mimetype"] }} + {{ dict_object["metadata_card"]["mimetype"] }} - - - - + + + +
TypeIdTypeId TagsComment
+ {% if 'comment' in object %} + {{ object['comment']}} + {% endif %} + diff --git a/var/www/templates/modals/investigations_register_obj.html b/var/www/templates/modals/investigations_register_obj.html index edac2853..50a6ca02 100644 --- a/var/www/templates/modals/investigations_register_obj.html +++ b/var/www/templates/modals/investigations_register_obj.html @@ -14,7 +14,10 @@
- +
+ + +
@@ -55,8 +58,13 @@ $('#investigations_register_obj_modal').on('shown.bs.modal', function () { }); function Register_Obj() { - var uuids = linvestigations.getValue(); - // TODO: REQUEST - window.location.replace("{{ url_for('investigations_b.register_investigation') }}?uuids=" + uuids + "&type={{ obj_type }}&subtype={{ obj_subtype }}&id={{ obj_id }}"); + var uuids = linvestigations.getValue(); + var comment = $('#inv_obj_comment').val(); + // TODO: REQUEST + var url = "{{ url_for('investigations_b.register_investigation') }}?uuids=" + uuids + "&type={{ obj_type }}&subtype={{ obj_subtype }}&id={{ obj_id }}" + if (comment) { + url += "&comment=" + comment; + } + window.location.replace(url); } From d6a24c035778333f6d31bb42d125716d30d00f27 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 29 Jun 2023 08:48:50 +0200 Subject: [PATCH 05/45] fix: [hunter + misp export] fix misp event json export + retro hunt date search and description --- bin/lib/Tracker.py | 4 ++-- var/www/blueprints/hunters.py | 2 +- var/www/blueprints/import_export.py | 1 + var/www/templates/hunter/trackersManagement.html | 10 +++++++++- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 2a5336ad..b7ca5e77 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -650,14 +650,14 @@ def get_user_trackers_meta(user_id, tracker_type=None): metas = [] for tracker_uuid in get_user_trackers(user_id, tracker_type=tracker_type): tracker = Tracker(tracker_uuid) - metas.append(tracker.get_meta(options={'mails', 'sparkline', 'tags'})) + metas.append(tracker.get_meta(options={'description', 'mails', 'sparkline', 'tags'})) return metas def get_global_trackers_meta(tracker_type=None): metas = [] for tracker_uuid in get_global_trackers(tracker_type=tracker_type): tracker = Tracker(tracker_uuid) - metas.append(tracker.get_meta(options={'mails', 'sparkline', 'tags'})) + metas.append(tracker.get_meta(options={'description', 'mails', 'sparkline', 'tags'})) return metas def get_users_trackers_meta(): diff --git a/var/www/blueprints/hunters.py b/var/www/blueprints/hunters.py index b2a2e30b..9a2b6c3e 100644 --- a/var/www/blueprints/hunters.py +++ b/var/www/blueprints/hunters.py @@ -172,7 +172,7 @@ def show_tracker(): typo_squatting = set() if date_from: - date_from, date_to = Date.sanitise_daterange(meta['first_seen'], meta['last_seen']) + date_from, date_to = Date.sanitise_daterange(date_from, date_to) objs = tracker.get_objs_by_daterange(date_from, date_to) meta['objs'] = ail_objects.get_objects_meta(objs, flask_context=True) else: diff --git a/var/www/blueprints/import_export.py b/var/www/blueprints/import_export.py index 312fe0be..bb28d080 100644 --- a/var/www/blueprints/import_export.py +++ b/var/www/blueprints/import_export.py @@ -163,6 +163,7 @@ def objects_misp_export_post(): MISPExporter.delete_user_misp_objects_to_export(user_id) if not export: event_uuid = event[10:46] + event = f'{{"Event": {event}}}' # TODO ADD JAVASCRIPT REFRESH PAGE IF RESP == 200 return send_file(io.BytesIO(event.encode()), as_attachment=True, download_name=f'ail_export_{event_uuid}.json') diff --git a/var/www/templates/hunter/trackersManagement.html b/var/www/templates/hunter/trackersManagement.html index 6bd1cfe4..89407245 100644 --- a/var/www/templates/hunter/trackersManagement.html +++ b/var/www/templates/hunter/trackersManagement.html @@ -60,7 +60,7 @@ 		Tracker First seen Last seenEmail notificationEmails sparkline
{{ dict_object["metadata"]['first_seen'] }} {{ dict_object["metadata"]['last_seen'] }}{{ dict_object["metadata"]['first_seen'] }}{{ dict_object["metadata"]['last_seen'] }}{{ dict_object["metadata"]['size'] }}{{ dict_object["metadata"]['nb_seen'] }}{{ dict_object["metadata_card"]['first_seen'] }}{{ dict_object["metadata_card"]['last_seen'] }}{{ dict_object["metadata_card"]['size'] }}{{ dict_object["metadata_card"]['nb_seen'] }}
From fa8b4a67d963a1abe8719973f29c3b4d496b9b56 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 29 Jun 2023 15:02:57 +0200 Subject: [PATCH 09/45] fix: [correlation tags] fix tag all objects --- bin/lib/objects/ail_objects.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index e64d952f..c54b7dd4 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -336,7 +336,7 @@ def get_obj_correlations(obj_type, subtype, obj_id): return obj.get_correlations() def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max): - if len(objs) < nb_max or nb_max == -1: + if len(objs) < nb_max or nb_max == 0: if lvl == 0: objs.add((obj_type, subtype, obj_id)) @@ -356,6 +356,7 @@ def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0, return objs def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300): + print(nb_max) objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max) # print(objs) for obj_tuple in objs: From 0fa0984ec4ffa3ed19383aef1dd071a8a1e5313b Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 30 Jun 2023 16:19:38 +0200 Subject: [PATCH 10/45] fix: [updater] fix db checker --- bin/LAUNCH.sh | 3 --- update/default_update/Update.sh | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 1de7f301..00c224e4 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -685,9 +685,6 @@ while [ "$1" != "" ]; do -lrv | --launchRedisVerify ) launch_redis; wait_until_redis_is_ready; ;; - -lav | --launchARDBVerify ) launch_ardb; - wait_until_ardb_is_ready; - ;; -lkv | --launchKVORCKSVerify ) launch_kvrocks; wait_until_kvrocks_is_ready; ;; diff --git a/update/default_update/Update.sh b/update/default_update/Update.sh index c14c4dac..ef881805 100755 --- a/update/default_update/Update.sh +++ b/update/default_update/Update.sh @@ -25,7 +25,7 @@ bash ${AIL_BIN}/LAUNCH.sh -ks wait echo "" -bash ${AIL_BIN}/LAUNCH.sh -lav +bash ${AIL_BIN}/LAUNCH.sh -lkv wait echo "" From 35f678245fc5cd53a26d7a2d8d7e23a5780bada1 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 30 Jun 2023 16:22:30 +0200 Subject: [PATCH 11/45] fix: [decoded] fix download file --- bin/lib/objects/Decodeds.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/lib/objects/Decodeds.py b/bin/lib/objects/Decodeds.py index 001f7dfd..fb194be1 100755 --- a/bin/lib/objects/Decodeds.py +++ b/bin/lib/objects/Decodeds.py @@ -138,7 +138,7 @@ class Decoded(AbstractDaterangeObject): with open(filepath, 'rb') as f: content = f.read() return content - elif r_str == 'bytesio': + elif r_type == 'bytesio': with open(filepath, 'rb') as f: content = BytesIO(f.read()) return content @@ -149,7 +149,7 @@ class Decoded(AbstractDaterangeObject): with zipfile.ZipFile(zip_content, "w") as zf: # TODO: Fix password # zf.setpassword(b"infected") - zf.writestr(self.id, self.get_content().getvalue()) + zf.writestr(self.id, self.get_content(r_type='bytesio').getvalue()) zip_content.seek(0) return zip_content From 450ebdd7899c3a1bef6059469f5c9eeafc2f743e Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 6 Jul 2023 11:26:32 +0200 Subject: [PATCH 12/45] chg: [etag] add new etag object --- bin/crawlers/Crawler.py | 5 + bin/lib/ail_core.py | 2 +- bin/lib/correlations_engine.py | 3 +- bin/lib/crawlers.py | 46 +- bin/lib/objects/Etags.py | 121 ++++ bin/lib/objects/ail_objects.py | 5 +- var/www/Flask_server.py | 2 + var/www/blueprints/correlation.py | 3 + var/www/blueprints/objects_cookie_name.py | 2 - var/www/blueprints/objects_etag.py | 86 +++ .../correlation/metadata_card_etag.html | 173 +++++ .../correlation/show_correlation.html | 6 + .../templates/objects/etag/EtagDaterange.html | 602 ++++++++++++++++++ .../templates/sidebars/sidebar_objects.html | 6 + 14 files changed, 1046 insertions(+), 16 deletions(-) create mode 100755 bin/lib/objects/Etags.py create mode 100644 var/www/blueprints/objects_etag.py create mode 100644 var/www/templates/correlation/metadata_card_etag.html create mode 100644 var/www/templates/objects/etag/EtagDaterange.html diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index be615993..2b2f35c6 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -17,6 +17,7 @@ from lib import ail_logger from lib import crawlers from lib.ConfigLoader import ConfigLoader from lib.objects import CookiesNames +from lib.objects import Etags from lib.objects.Domains import Domain from lib.objects.Items import Item from lib.objects import Screenshots @@ -288,6 +289,10 @@ class Crawler(AbstractModule): print(cookie_name) cookie = CookiesNames.create(cookie_name) cookie.add(self.date.replace('/', ''), self.domain.id) + for etag_content in crawlers.extract_etag_from_har(entries['har']): + print(etag_content) + etag = Etags.create(etag_content) + etag.add(self.date.replace('/', ''), self.domain.id) # Next Children entries_children = entries.get('children') diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index c52db274..e0fd3a17 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -15,7 +15,7 @@ config_loader = ConfigLoader() r_serv_db = config_loader.get_db_conn("Kvrocks_DB") config_loader = None -AIL_OBJECTS = sorted({'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'favicon', 'item', 'pgp', +AIL_OBJECTS = sorted({'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'item', 'pgp', 'screenshot', 'title', 'username'}) def get_ail_uuid(): diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index 8e29837d..1a2081ac 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -45,7 +45,8 @@ CORRELATION_TYPES_BY_OBJ = { "cryptocurrency": ["domain", "item"], "cve": ["domain", "item"], "decoded": ["domain", "item"], - "domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "favicon", "item", "pgp", "title", "screenshot", "username"], + "domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "etag", "favicon", "item", "pgp", "title", "screenshot", "username"], + "etag": ["domain"], "favicon": ["domain", "item"], # TODO Decoded "item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], "pgp": ["domain", "item"], diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 6f6bf5b7..7c98537e 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -264,6 +264,7 @@ def extract_author_from_html(html): if keywords: return keywords['content'] return '' + # # # - - # # # @@ -299,9 +300,10 @@ def get_all_har_ids(): except (TypeError, ValueError): pass - for file in [f for f in os.listdir(today_root_dir) if os.path.isfile(os.path.join(today_root_dir, f))]: - har_id = os.path.relpath(os.path.join(today_root_dir, file), HAR_DIR) - har_ids.append(har_id) + if os.path.exists(today_root_dir): + for file in [f for f in os.listdir(today_root_dir) if os.path.isfile(os.path.join(today_root_dir, f))]: + har_id = os.path.relpath(os.path.join(today_root_dir, file), HAR_DIR) + har_ids.append(har_id) for ydir in sorted(dirs_year, reverse=False): search_dear = os.path.join(HAR_DIR, ydir) @@ -312,14 +314,13 @@ def get_all_har_ids(): har_ids.append(har_id) return har_ids -def extract_cookies_names_from_har_by_har_id(har_id): +def get_har_content(har_id): har_path = os.path.join(HAR_DIR, har_id) with open(har_path) as f: try: - har_content = json.loads(f.read()) + return json.loads(f.read()) except json.decoder.JSONDecodeError: - har_content = {} - return extract_cookies_names_from_har(har_content) + return {} def extract_cookies_names_from_har(har): cookies = set() @@ -334,18 +335,41 @@ def extract_cookies_names_from_har(har): cookies.add(name) return cookies -def _reprocess_all_hars(): +def _reprocess_all_hars_cookie_name(): from lib.objects import CookiesNames for har_id in get_all_har_ids(): domain = har_id.split('/')[-1] domain = domain[:-41] date = har_id.split('/') date = f'{date[-4]}{date[-3]}{date[-2]}' - for cookie_name in extract_cookies_names_from_har_by_har_id(har_id): + for cookie_name in extract_cookies_names_from_har(get_har_content(har_id)): print(domain, date, cookie_name) cookie = CookiesNames.create(cookie_name) cookie.add(date, domain) +def extract_etag_from_har(har): # TODO check response url + etags = set() + for entrie in har.get('log', {}).get('entries', []): + for header in entrie.get('response', {}).get('headers', []): + if header.get('name') == 'etag': + # print(header) + etag = header.get('value') + if etag: + etags.add(etag) + return etags + +def _reprocess_all_hars_etag(): + from lib.objects import Etags + for har_id in get_all_har_ids(): + domain = har_id.split('/')[-1] + domain = domain[:-41] + date = har_id.split('/') + date = f'{date[-4]}{date[-3]}{date[-2]}' + for etag_content in extract_etag_from_har(get_har_content(har_id)): + print(domain, date, etag_content) + etag = Etags.create(etag_content) + etag.add(date, domain) + # # # - - # # # ################################################################################ @@ -1913,5 +1937,5 @@ load_blacklist() # temp_url = '' # r = extract_favicon_from_html(content, temp_url) # print(r) -# _reprocess_all_hars() - +# _reprocess_all_hars_cookie_name() +# _reprocess_all_hars_etag() diff --git a/bin/lib/objects/Etags.py b/bin/lib/objects/Etags.py new file mode 100755 index 00000000..eb41f68c --- /dev/null +++ b/bin/lib/objects/Etags.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys + +from hashlib import sha256 +from flask import url_for + +from pymisp import MISPObject + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ConfigLoader import ConfigLoader +from lib.objects.abstract_daterange_object import AbstractDaterangeObject, AbstractDaterangeObjects + +config_loader = ConfigLoader() +r_objects = config_loader.get_db_conn("Kvrocks_Objects") +baseurl = config_loader.get_config_str("Notifications", "ail_domain") +config_loader = None + +# TODO NEW ABSTRACT OBJECT -> daterange for all objects ???? + +class Etag(AbstractDaterangeObject): + """ + AIL Etag Object. + """ + + def __init__(self, obj_id): + super(Etag, self).__init__('etag', obj_id) + + # def get_ail_2_ail_payload(self): + # payload = {'raw': self.get_gzip_content(b64=True), + # 'compress': 'gzip'} + # return payload + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + # # TODO: + pass + + def get_content(self, r_type='str'): + if r_type == 'str': + return self._get_field('content') + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('correlation.show_correlation', type=self.type, id=self.id) + else: + url = f'{baseurl}/correlation/show?type={self.type}&id={self.id}' + return url + + # TODO # CHANGE COLOR + def get_svg_icon(self): + return {'style': 'fas', 'icon': '\uf02b', 'color': '#556F65', 'radius': 5} + + def get_misp_object(self): + obj_attrs = [] + obj = MISPObject('etag') + first_seen = self.get_first_seen() + last_seen = self.get_last_seen() + if first_seen: + obj.first_seen = first_seen + if last_seen: + obj.last_seen = last_seen + if not first_seen or not last_seen: + self.logger.warning( + f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}') + + obj_attrs.append(obj.add_attribute('etag', value=self.get_content())) + for obj_attr in obj_attrs: + for tag in self.get_tags(): + obj_attr.add_tag(tag) + return obj + + def get_nb_seen(self): + return self.get_nb_correlation('domain') + + def get_meta(self, options=set()): + meta = self._get_meta(options=options) + meta['id'] = self.id + meta['tags'] = self.get_tags(r_list=True) + meta['content'] = self.get_content() + return meta + + def add(self, date, obj_id): # date = HAR Date + self._add(date, 'domain', '', obj_id) + + def create(self, content, _first_seen=None, _last_seen=None): + if not isinstance(content, str): + content = content.decode() + self._set_field('content', content) + self._create() + + +def create(content): + if isinstance(content, str): + content = content.encode() + obj_id = sha256(content).hexdigest() + etag = Etag(obj_id) + if not etag.exists(): + etag.create(content) + return etag + + +class Etags(AbstractDaterangeObjects): + """ + Etags Objects + """ + def __init__(self): + super().__init__('etag', Etag) + + def sanitize_id_to_search(self, name_to_search): + return name_to_search # TODO + + +# if __name__ == '__main__': +# name_to_search = '98' +# print(search_cves_by_name(name_to_search)) diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index c54b7dd4..4279c776 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -18,6 +18,7 @@ from lib.objects import CookiesNames from lib.objects.Cves import Cve from lib.objects.Decodeds import Decoded, get_all_decodeds_objects, get_nb_decodeds_objects from lib.objects.Domains import Domain +from lib.objects import Etags from lib.objects.Favicons import Favicon from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects from lib.objects import Pgps @@ -57,6 +58,8 @@ def get_object(obj_type, subtype, id): return CookiesNames.CookieName(id) elif obj_type == 'cve': return Cve(id) + elif obj_type == 'etag': + return Etags.Etag(id) elif obj_type == 'favicon': return Favicon(id) elif obj_type == 'screenshot': @@ -168,7 +171,7 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False): obj = get_object(obj_type, subtype, id) meta = obj.get_meta() meta['icon'] = obj.get_svg_icon() - if subtype or obj_type == 'cookie-name' or obj_type == 'cve' or obj_type == 'title' or obj_type == 'favicon': + if subtype or obj_type == 'cookie-name' or obj_type == 'cve' or obj_type == 'etag' or obj_type == 'title' or obj_type == 'favicon': meta['sparkline'] = obj.get_sparkline() if obj_type == 'cve': meta['cve_search'] = obj.get_cve_search() diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index cc110c35..b40b5cc8 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -51,6 +51,7 @@ from blueprints.objects_decoded import objects_decoded from blueprints.objects_subtypes import objects_subtypes from blueprints.objects_title import objects_title from blueprints.objects_cookie_name import objects_cookie_name +from blueprints.objects_etag import objects_etag Flask_dir = os.environ['AIL_FLASK'] @@ -106,6 +107,7 @@ app.register_blueprint(objects_decoded, url_prefix=baseUrl) app.register_blueprint(objects_subtypes, url_prefix=baseUrl) app.register_blueprint(objects_title, url_prefix=baseUrl) app.register_blueprint(objects_cookie_name, url_prefix=baseUrl) +app.register_blueprint(objects_etag, url_prefix=baseUrl) # ========= =========# diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index f6e7feda..d5bb1c82 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -83,6 +83,9 @@ def show_correlation(): correl_option = request.form.get('CookieNameCheck') if correl_option: filter_types.append('cookie-name') + correl_option = request.form.get('EtagCheck') + if correl_option: + filter_types.append('etag') correl_option = request.form.get('CveCheck') if correl_option: filter_types.append('cve') diff --git a/var/www/blueprints/objects_cookie_name.py b/var/www/blueprints/objects_cookie_name.py index ab111ff2..06d6743a 100644 --- a/var/www/blueprints/objects_cookie_name.py +++ b/var/www/blueprints/objects_cookie_name.py @@ -45,8 +45,6 @@ def objects_cookies_names(): else: dict_objects = {} - print(dict_objects) - return render_template("CookieNameDaterange.html", date_from=date_from, date_to=date_to, dict_objects=dict_objects, show_objects=show_objects) diff --git a/var/www/blueprints/objects_etag.py b/var/www/blueprints/objects_etag.py new file mode 100644 index 00000000..ad2b24fd --- /dev/null +++ b/var/www/blueprints/objects_etag.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... +''' + +import os +import sys + +from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort +from flask_login import login_required, current_user + +# Import Role_Manager +from Role_Manager import login_admin, login_analyst, login_read_only + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.objects import Etags +from packages import Date + +# ============ BLUEPRINT ============ +objects_etag = Blueprint('objects_etag', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/etag')) + +# ============ VARIABLES ============ +bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] + + +# ============ FUNCTIONS ============ +@objects_etag.route("/objects/etags", methods=['GET']) +@login_required +@login_read_only +def objects_etags(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + show_objects = request.args.get('show_objects') + date = Date.sanitise_date_range(date_from, date_to) + date_from = date['date_from'] + date_to = date['date_to'] + + if show_objects: + dict_objects = Etags.Etags().api_get_meta_by_daterange(date_from, date_to) + else: + dict_objects = {} + + return render_template("EtagDaterange.html", date_from=date_from, date_to=date_to, + dict_objects=dict_objects, show_objects=show_objects) + +@objects_etag.route("/objects/etag/post", methods=['POST']) +@login_required +@login_read_only +def objects_etags_post(): + date_from = request.form.get('date_from') + date_to = request.form.get('date_to') + show_objects = request.form.get('show_objects') + return redirect(url_for('objects_etag.objects_etags', date_from=date_from, date_to=date_to, show_objects=show_objects)) + +@objects_etag.route("/objects/etag/range/json", methods=['GET']) +@login_required +@login_read_only +def objects_etag_range_json(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + date = Date.sanitise_date_range(date_from, date_to) + date_from = date['date_from'] + date_to = date['date_to'] + return jsonify(Etags.Etags().api_get_chart_nb_by_daterange(date_from, date_to)) + +# @objects_etag.route("/objects/etag/search", methods=['POST']) +# @login_required +# @login_read_only +# def objects_etags_names_search(): +# to_search = request.form.get('object_id') +# +# # TODO SANITIZE ID +# # TODO Search all +# cve = Cves.Cve(to_search) +# if not cve.exists(): +# abort(404) +# else: +# return redirect(cve.get_link(flask_context=True)) + +# ============= ROUTES ============== + diff --git a/var/www/templates/correlation/metadata_card_etag.html b/var/www/templates/correlation/metadata_card_etag.html new file mode 100644 index 00000000..cc599227 --- /dev/null +++ b/var/www/templates/correlation/metadata_card_etag.html @@ -0,0 +1,173 @@ + + + +{% with modal_add_tags=dict_object['metadata_card']['add_tags_modal']%} + {% include 'modals/add_tags.html' %} +{% endwith %} + +{% include 'modals/edit_tag.html' %} + +
+
+

{{ dict_object["metadata"]["content"] }}

+
{{ dict_object["correlation_id"] }}
+
    +
  • +
    +
    + + + + + + + + + + + + + + + + + +
    Object typeFirst seenLast seenNb seen
    + + + + {{ dict_object["metadata_card"]["icon"]["icon"] }} + + + {{ dict_object["object_type"] }} + {{ dict_object["metadata"]['first_seen'] }}{{ dict_object["metadata"]['last_seen'] }}{{ dict_object["metadata"]['nb_seen'] }}
    +
    +
    +
    +
    +
    +
    • + +
  • +
    +
    + Tags: + {% for tag in dict_object["metadata"]['tags'] %} + + {% endfor %} + +
    +
    • +
+ + {% with obj_type='etag', obj_id=dict_object['correlation_id'], obj_subtype='' %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + +
+
+ + + + + + diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 95aa922c..326b637f 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -117,6 +117,8 @@ {% include 'correlation/metadata_card_title.html' %} {% elif dict_object["object_type"] == "cookie-name" %} {% include 'correlation/metadata_card_cookie_name.html' %} + {% elif dict_object["object_type"] == "etag" %} + {% include 'correlation/metadata_card_etag.html' %} {% elif dict_object["object_type"] == "item" %} {% include 'correlation/metadata_card_item.html' %} {% endif %} @@ -211,6 +213,10 @@ +
+ + +
diff --git a/var/www/templates/objects/etag/EtagDaterange.html b/var/www/templates/objects/etag/EtagDaterange.html new file mode 100644 index 00000000..9fc7f3e7 --- /dev/null +++ b/var/www/templates/objects/etag/EtagDaterange.html @@ -0,0 +1,602 @@ + + + + + Etags - AIL + + + + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'sidebars/sidebar_objects.html' %} + +
+ +
+
+
+ +{# {% include 'etag/block_etag_search.html' %}#} + +
+ + +
+ +
+
+
Select a date range :
+
+
+
+ +
+
+
+ +
+
+ + +
+ +
+
+
+ +
+
+
+
+
+
+ + {% if dict_objects %} + {% if date_from|string == date_to|string %} +

{{ date_from }} Etags Name:

+ {% else %} +

{{ date_from }} to {{ date_to }} Etags Name:

+ {% endif %} + + + + + + + + + + + + {% for obj_id in dict_objects %} + + + + + + + + {% endfor %} + +
First SeenLast SeenTotalLast days
{{ dict_objects[obj_id]['content'] }}{{ dict_objects[obj_id]['first_seen'] }}{{ dict_objects[obj_id]['last_seen'] }}{{ dict_objects[obj_id]['nb_seen'] }}
+ + + {% else %} + {% if show_objects %} + {% if date_from|string == date_to|string %} +

{{ date_from }}, No Etag Name

+ {% else %} +

{{ date_from }} to {{ date_to }}, No Etag Name

+ {% endif %} + {% endif %} + {% endif %} +
+ +
+
+ + + + + + + + + + + + + + + + + diff --git a/var/www/templates/sidebars/sidebar_objects.html b/var/www/templates/sidebars/sidebar_objects.html index 89239701..5f7b00b5 100644 --- a/var/www/templates/sidebars/sidebar_objects.html +++ b/var/www/templates/sidebars/sidebar_objects.html @@ -40,6 +40,12 @@ Cookie Name +
  • + + + Etag + +
  • From 482fc21b5e93f37982c2c580fb4e710eeac4d9ff Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 7 Jul 2023 16:29:32 +0200 Subject: [PATCH 13/45] chg: [correlation] correlation graph, add an option to hide an object/node by pressing H + reset correlation graph --- bin/lib/correlations_engine.py | 12 ++++-- bin/lib/objects/ail_objects.py | 17 +++++--- var/www/blueprints/correlation.py | 35 +++++++++++++-- .../correlation/show_correlation.html | 43 ++++++++++++++++++- 4 files changed, 91 insertions(+), 16 deletions(-) diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index 1a2081ac..39ccaa4e 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -170,18 +170,18 @@ def get_obj_str_id(obj_type, subtype, obj_id): subtype = '' return f'{obj_type}:{subtype}:{obj_id}' -def get_correlations_graph_nodes_links(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, flask_context=False): +def get_correlations_graph_nodes_links(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, objs_hidden=set(), flask_context=False): links = set() nodes = set() meta = {'complete': True, 'objs': set()} obj_str_id = get_obj_str_id(obj_type, subtype, obj_id) - _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=filter_types, previous_str_obj='') + _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, previous_str_obj='') return obj_str_id, nodes, links, meta -def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=[], previous_str_obj=''): +def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, level, max_nodes, filter_types=[], objs_hidden=set(), previous_str_obj=''): obj_str_id = get_obj_str_id(obj_type, subtype, obj_id) meta['objs'].add(obj_str_id) nodes.add(obj_str_id) @@ -192,6 +192,10 @@ def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, for str_obj in obj_correlations[correl_type]: subtype2, obj2_id = str_obj.split(':', 1) obj2_str_id = get_obj_str_id(correl_type, subtype2, obj2_id) + # filter objects to hide + if obj2_str_id in objs_hidden: + continue + meta['objs'].add(obj2_str_id) if obj2_str_id == previous_str_obj: @@ -205,5 +209,5 @@ def _get_correlations_graph_node(links, nodes, meta, obj_type, subtype, obj_id, if level > 0: next_level = level - 1 - _get_correlations_graph_node(links, nodes, meta, correl_type, subtype2, obj2_id, next_level, max_nodes, filter_types=filter_types, previous_str_obj=obj_str_id) + _get_correlations_graph_node(links, nodes, meta, correl_type, subtype2, obj2_id, next_level, max_nodes, filter_types=filter_types, objs_hidden=objs_hidden, previous_str_obj=obj_str_id) diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 4279c776..e0eeda63 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -338,7 +338,7 @@ def get_obj_correlations(obj_type, subtype, obj_id): obj = get_object(obj_type, subtype, obj_id) return obj.get_correlations() -def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max): +def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max, objs_hidden): if len(objs) < nb_max or nb_max == 0: if lvl == 0: objs.add((obj_type, subtype, obj_id)) @@ -351,16 +351,17 @@ def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lv for obj2_type in correlations: for str_obj in correlations[obj2_type]: obj2_subtype, obj2_id = str_obj.split(':', 1) - _get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, filter_types, lvl, nb_max) + if get_obj_global_id(obj2_type, obj2_subtype, obj2_id) in objs_hidden: + continue # filter object to hide + _get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, filter_types, lvl, nb_max, objs_hidden) -def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0, nb_max=300): +def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0, nb_max=300, objs_hidden=set()): objs = set() - _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max) + _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, filter_types, lvl, nb_max, objs_hidden) return objs -def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300): - print(nb_max) - objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max) +def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300, objs_hidden=set()): + objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max, objs_hidden=objs_hidden) # print(objs) for obj_tuple in objs: obj1_type, subtype1, id1 = obj_tuple @@ -422,10 +423,12 @@ def create_correlation_graph_nodes(nodes_set, obj_str_id, flask_context=True): def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, + objs_hidden=set(), flask_context=False): obj_str_id, nodes, links, meta = correlations_engine.get_correlations_graph_nodes_links(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=level, + objs_hidden=objs_hidden, flask_context=flask_context) # print(meta) meta['objs'] = list(meta['objs']) diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index d5bb1c82..594dc660 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -61,6 +61,13 @@ def sanitise_level(level): level = 2 return level +def sanitise_objs_hidden(objs_hidden): + if objs_hidden: + objs_hidden = set(objs_hidden.split(',')) # TODO sanitize objects + else: + objs_hidden = set() + return objs_hidden + # ============= ROUTES ============== @correlation.route('/correlation/show', methods=['GET', 'POST']) @login_required @@ -130,6 +137,10 @@ def show_correlation(): max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes')) mode = sanitise_graph_mode(request.args.get('mode')) level = sanitise_level(request.args.get('level')) + objs_hidden = sanitise_objs_hidden(request.args.get('hidden')) + obj_to_hide = request.args.get('hide') + if obj_to_hide: + objs_hidden.add(obj_to_hide) related_btc = bool(request.args.get('related_btc', False)) @@ -139,17 +150,24 @@ def show_correlation(): if not ail_objects.exists_obj(obj_type, subtype, obj_id): return abort(404) # object exist - else: - dict_object = {"object_type": obj_type, - "correlation_id": obj_id, + else: # TODO remove old dict key + dict_object = {"type": obj_type, + "id": obj_id, + "object_type": obj_type, "max_nodes": max_nodes, "mode": mode, "level": level, "filter": filter_types, "filter_str": ",".join(filter_types), + "hidden": objs_hidden, "hidden_str": ",".join(objs_hidden), + + "correlation_id": obj_id, "metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id, options={'tags'}, flask_context=True), "nb_correl": ail_objects.get_obj_nb_correlations(obj_type, subtype, obj_id) } if subtype: + dict_object["subtype"] = subtype dict_object["metadata"]['type_id'] = subtype + else: + dict_object["subtype"] = '' dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id, related_btc=related_btc) return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label, tags_selector_data=Tag.get_tags_selector_data()) @@ -194,9 +212,15 @@ def graph_node_json(): max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes')) level = sanitise_level(request.args.get('level')) + hidden = request.args.get('hidden') + if hidden: + hidden = set(hidden.split(',')) + else: + hidden = set() + filter_types = ail_objects.sanitize_objs_types(request.args.get('filter', '').split(',')) - json_graph = ail_objects.get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=True) + json_graph = ail_objects.get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=level, objs_hidden=hidden, flask_context=True) #json_graph = Correlate_object.get_graph_node_object_correlation(obj_type, obj_id, 'union', correlation_names, correlation_objects, requested_correl_type=subtype, max_nodes=max_nodes) return jsonify(json_graph) @@ -224,6 +248,7 @@ def correlation_tags_add(): nb_max = sanitise_nb_max_nodes(request.form.get('tag_nb_max')) level = sanitise_level(request.form.get('tag_level')) filter_types = ail_objects.sanitize_objs_types(request.form.get('tag_filter', '').split(',')) + hidden = sanitise_objs_hidden(request.form.get('tag_hidden')) if not ail_objects.exists_obj(obj_type, subtype, obj_id): return abort(404) @@ -252,9 +277,11 @@ def correlation_tags_add(): if tags: ail_objects.obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=filter_types, + objs_hidden=hidden, lvl=level + 1, nb_max=nb_max) return redirect(url_for('correlation.show_correlation', type=obj_type, subtype=subtype, id=obj_id, level=level, max_nodes=nb_max, + hidden=hidden, hidden_str=",".join(hidden), filter=",".join(filter_types))) diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 326b637f..b243bd35 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -164,6 +164,11 @@  Resize Graph + + +
     Graph Incomplete, Max Nodes Reached.
    @@ -180,6 +185,14 @@
    • +

    Press H on an object / node to hide it.

    + {% if dict_object["hidden"] %} +
    Hidden objects:
    + {% for obj_hidden in dict_object["hidden"] %} + {{ obj_hidden }}
    + {% endfor %} + {% endif %} +
    @@ -343,6 +356,7 @@ + {% include 'tags/block_tags_selector.html' %} + {% endfor %} + +
    + + + + {% with obj_type='hhhash', obj_id=dict_object['correlation_id'], obj_subtype='' %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + + + + + + + + + diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index b243bd35..11a85cd7 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -119,6 +119,8 @@ {% include 'correlation/metadata_card_cookie_name.html' %} {% elif dict_object["object_type"] == "etag" %} {% include 'correlation/metadata_card_etag.html' %} + {% elif dict_object["object_type"] == "hhhash" %} + {% include 'correlation/metadata_card_hhhash.html' %} {% elif dict_object["object_type"] == "item" %} {% include 'correlation/metadata_card_item.html' %} {% endif %} @@ -230,6 +232,10 @@ +
    + + +
    diff --git a/var/www/templates/objects/hhhash/HHHashDaterange.html b/var/www/templates/objects/hhhash/HHHashDaterange.html new file mode 100644 index 00000000..79e12238 --- /dev/null +++ b/var/www/templates/objects/hhhash/HHHashDaterange.html @@ -0,0 +1,602 @@ + + + + + HHHashs - AIL + + + + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'sidebars/sidebar_objects.html' %} + +
    + +
    +
    +
    + +{# {% include 'hhhash/block_hhhash_search.html' %}#} + +
    + + +
    + +
    +
    +
    Select a date range :
    +
    +
    +
    + +
    +
    +
    + +
    +
    + + +
    + +
    +
    +
    + +
    +
    +
    +
    +
    +
    + + {% if dict_objects %} + {% if date_from|string == date_to|string %} +

    {{ date_from }} HHHashs Name:

    + {% else %} +

    {{ date_from }} to {{ date_to }} HHHashs Name:

    + {% endif %} + + + + + + + + + + + + {% for obj_id in dict_objects %} + + + + + + + + {% endfor %} + +
    First SeenLast SeenTotalLast days
    {{ dict_objects[obj_id]['content'] }}{{ dict_objects[obj_id]['first_seen'] }}{{ dict_objects[obj_id]['last_seen'] }}{{ dict_objects[obj_id]['nb_seen'] }}
    + + + {% else %} + {% if show_objects %} + {% if date_from|string == date_to|string %} +

    {{ date_from }}, No HHHash Name

    + {% else %} +

    {{ date_from }} to {{ date_to }}, No HHHash Name

    + {% endif %} + {% endif %} + {% endif %} +
    + +
    +
    + + + + + + + + + + + + + + + + + diff --git a/var/www/templates/sidebars/sidebar_objects.html b/var/www/templates/sidebars/sidebar_objects.html index 5f7b00b5..12b5abc0 100644 --- a/var/www/templates/sidebars/sidebar_objects.html +++ b/var/www/templates/sidebars/sidebar_objects.html @@ -46,6 +46,12 @@ Etag     +
  • + + + HHHash + +
  • From fe2769308b41600bceb5fd17e861aa0b98dfc57d Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 18 Jul 2023 14:30:00 +0200 Subject: [PATCH 25/45] fix: [crawler] fix down domain + domain redirection history --- bin/crawlers/Crawler.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index ab972c68..7f2c3df9 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -60,6 +60,7 @@ class Crawler(AbstractModule): self.root_item = None self.date = None self.items_dir = None + self.original_domain = None self.domain = None # TODO Replace with warning list ??? @@ -190,6 +191,7 @@ class Crawler(AbstractModule): print(domain) self.domain = Domain(domain) + self.original_domain = Domain(domain) epoch = int(time.time()) parent_id = task.get_parent() @@ -212,12 +214,20 @@ class Crawler(AbstractModule): # Origin + History + tags if self.root_item: self.domain.set_last_origin(parent_id) - self.domain.add_history(epoch, root_item=self.root_item) # Tags for tag in task.get_tags(): self.domain.add_tag(tag) - elif self.domain.was_up(): - self.domain.add_history(epoch, root_item=epoch) + self.domain.add_history(epoch, root_item=self.root_item) + + if self.domain != self.original_domain: + self.original_domain.update_daterange(self.date.replace('/', '')) + if self.root_item: + self.original_domain.set_last_origin(parent_id) + # Tags + for tag in task.get_tags(): + self.domain.add_tag(tag) + self.original_domain.add_history(epoch, root_item=self.root_item) + crawlers.update_last_crawled_domain(self.original_domain.get_domain_type(), self.original_domain.id, epoch) crawlers.update_last_crawled_domain(self.domain.get_domain_type(), self.domain.id, epoch) print('capture:', capture.uuid, 'completed') From 68dffcd26b53229525ffe649526e3cef669dbcef Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 25 Jul 2023 15:57:11 +0200 Subject: [PATCH 26/45] chg: [api crawler] fix response + add cookiejar, proxy and frequency parameters --- bin/lib/crawlers.py | 10 +- bin/modules/Onion.py | 8 +- tools/crawler_add_task.py | 120 +++++++++++++++++++++++ var/www/modules/restApi/Flask_restApi.py | 5 +- 4 files changed, 134 insertions(+), 9 deletions(-) create mode 100755 tools/crawler_add_task.py diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 571af921..3e61ed88 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -1723,14 +1723,16 @@ def api_add_crawler_task(data, user_id=None): if frequency: # TODO verify user - return create_schedule(frequency, user_id, url, depth=depth_limit, har=har, screenshot=screenshot, header=None, - cookiejar=cookiejar_uuid, proxy=proxy, user_agent=None, tags=tags), 200 + task_uuid = create_schedule(frequency, user_id, url, depth=depth_limit, har=har, screenshot=screenshot, header=None, + cookiejar=cookiejar_uuid, proxy=proxy, user_agent=None, tags=tags) else: # TODO HEADERS # TODO USER AGENT - return create_task(url, depth=depth_limit, har=har, screenshot=screenshot, header=None, + task_uuid = create_task(url, depth=depth_limit, har=har, screenshot=screenshot, header=None, cookiejar=cookiejar_uuid, proxy=proxy, user_agent=None, tags=tags, - parent='manual', priority=90), 200 + parent='manual', priority=90) + + return {'uuid': task_uuid}, 200 #### #### diff --git a/bin/modules/Onion.py b/bin/modules/Onion.py index 5e76d0fa..303b5bdc 100755 --- a/bin/modules/Onion.py +++ b/bin/modules/Onion.py @@ -42,7 +42,8 @@ class Onion(AbstractModule): self.faup = crawlers.get_faup() # activate_crawler = p.config.get("Crawler", "activate_crawler") - + self.har = config_loader.get_config_boolean('Crawler', 'default_har') + self.screenshot = config_loader.get_config_boolean('Crawler', 'default_screenshot') self.onion_regex = r"((http|https|ftp)?(?:\://)?([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.onion)(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*)" # self.i2p_regex = r"((http|https|ftp)?(?:\://)?([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.i2p)(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*)" @@ -90,8 +91,9 @@ class Onion(AbstractModule): if onion_urls: if crawlers.is_crawler_activated(): - for domain in domains: # TODO LOAD DEFAULT SCREENSHOT + HAR - task_uuid = crawlers.create_task(domain, parent=item.get_id(), priority=0) + for domain in domains: + task_uuid = crawlers.create_task(domain, parent=item.get_id(), priority=0, + har=self.har, screenshot=self.screenshot) if task_uuid: print(f'{domain} added to crawler queue: {task_uuid}') else: diff --git a/tools/crawler_add_task.py b/tools/crawler_add_task.py new file mode 100755 index 00000000..f785f167 --- /dev/null +++ b/tools/crawler_add_task.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +DIR/File Importer Helper +================ + +Import Content + +""" + +import argparse +import os +from pyail import PyAIL +import sys + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ConfigLoader import ConfigLoader + +def check_frequency(value): + value = int(value) + if value <= 0: + raise argparse.ArgumentTypeError(f'Error: Invalid frequency {value}') + + +if __name__ == "__main__": + + # TODO add c argument for config file + parser = argparse.ArgumentParser(description='Directory or file importer') + parser.add_argument('-u', '--url', type=str, help='URL to crawl', required=True) + parser.add_argument('-k', '--key', type=str, help='AIL API Key', required=True) + parser.add_argument('-a', '--ail', type=str, help='AIL URL') + parser.add_argument('-d', '--depth', type=int, default=1, help='Depth limit') # TODO improve me + parser.add_argument('--cookiejar', type=str, help='Cookiejar uuid') + parser.add_argument('-p', '--proxy', type=str, help='Proxy address to use, "web" and "tor" can be used as shortcut (web is used by default is )') + + group = parser.add_mutually_exclusive_group() + group.add_argument('--har', dest='har', action='store_true', help='Save HAR') + group.add_argument('--no-har', dest='har', action='store_false', help='Don\'t save HAR') + parser.set_defaults(har=None) + + group = parser.add_mutually_exclusive_group() + group.add_argument('--screenshot', dest='screenshot', action='store_true', help='Save screenshot') + group.add_argument('--no-screenshot', dest='screenshot', action='store_false', help='Don\'t save screenshot') + parser.set_defaults(screenshot=None) + + group = parser.add_argument_group('Frequency, create a regular crawler/scheduler') + group.add_argument('-f', '--frequency', type=str, choices=['monthly', 'weekly', 'daily', 'hourly'], + help='monthly, weekly, daily or hourly frequency or specify a custom one with the others arguments') + group.add_argument('--minutes', type=int, help='frequency in minutes') + group.add_argument('--hours', type=int, help='frequency in hours') + group.add_argument('--days', type=int, help='frequency in days') + group.add_argument('--weeks', type=int, help='frequency in weeks') + group.add_argument('--months', type=int, help='frequency in months') + + args = parser.parse_args() + + if not args.url and not args.key: + parser.print_help() + sys.exit(0) + + # Load crawler default config + config_loader = ConfigLoader() + har = args.har + if har is None: + har = config_loader.get_config_boolean('Crawler', 'default_har') + screenshot = args.screenshot + if screenshot is None: + screenshot = config_loader.get_config_boolean('Crawler', 'default_screenshot') + + if args.depth: + depth = args.depth + if depth < 0: + raise argparse.ArgumentTypeError(f'Error: Invalid depth {depth}') + else: + depth = 1 + + # frequency + frequency = {} + if args.frequency: + if args.frequency in ['monthly', 'weekly', 'daily', 'hourly']: + frequency = args.frequency + else: + raise argparse.ArgumentTypeError('Invalid frequency') + elif args.minutes or args.hours or args.days or args.weeks or args.months: + if args.minutes: + check_frequency(args.minutes) + frequency['minutes'] = args.minutes + if args.hours: + check_frequency(args.hours) + frequency['hours'] = args.hours + if args.days: + check_frequency(args.days) + frequency['days'] = args.days + if args.weeks: + check_frequency(args.weeks) + frequency['weeks'] = args.weeks + if args.months: + check_frequency(args.months) + frequency['months'] = args.months + if not frequency: + frequency = None + + proxy = args.proxy + + if args.cookiejar: + cookiejar = args.cookiejar + else: + cookiejar = None + + ail = args.ail + if not ail: + ail = 'https://localhost:7000/' + + client = PyAIL(ail, args.key, ssl=False) + r = client.crawl_url(args.url, har=har, screenshot=screenshot, depth_limit=depth, frequency=frequency, + cookiejar=cookiejar, proxy=proxy) + print(r) diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index e233bb00..0c95d0a0 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* -''' +""" Flask functions and routes for the rest api -''' +""" import os import re @@ -508,6 +508,7 @@ def get_item_cryptocurrency_bitcoin(): # # # # # # # # # # # # # # CRAWLER # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TODO: ADD RESULT JSON Response +# @restApi.route("api/v1/crawler/task/add", methods=['POST']) @restApi.route("api/v1/add/crawler/task", methods=['POST']) @token_required('analyst') def add_crawler_task(): From 6a30d6149bcecd0044f2aa61582b5a2dbc53b48b Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 25 Jul 2023 16:04:45 +0200 Subject: [PATCH 27/45] fix: [tools crawler] improve helper --- tools/crawler_add_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/crawler_add_task.py b/tools/crawler_add_task.py index f785f167..455a12fc 100755 --- a/tools/crawler_add_task.py +++ b/tools/crawler_add_task.py @@ -46,7 +46,7 @@ if __name__ == "__main__": group.add_argument('--no-screenshot', dest='screenshot', action='store_false', help='Don\'t save screenshot') parser.set_defaults(screenshot=None) - group = parser.add_argument_group('Frequency, create a regular crawler/scheduler') + group = parser.add_argument_group('Frequency, create a regular crawler/scheduler. one shot if not specified') group.add_argument('-f', '--frequency', type=str, choices=['monthly', 'weekly', 'daily', 'hourly'], help='monthly, weekly, daily or hourly frequency or specify a custom one with the others arguments') group.add_argument('--minutes', type=int, help='frequency in minutes') From 4f79944b2452073ab197486c551a09d83217830f Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 25 Jul 2023 16:11:36 +0200 Subject: [PATCH 28/45] fix: [tools crawler] fix helper --- tools/crawler_add_task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/crawler_add_task.py b/tools/crawler_add_task.py index 455a12fc..6b59242a 100755 --- a/tools/crawler_add_task.py +++ b/tools/crawler_add_task.py @@ -28,13 +28,13 @@ def check_frequency(value): if __name__ == "__main__": # TODO add c argument for config file - parser = argparse.ArgumentParser(description='Directory or file importer') + parser = argparse.ArgumentParser(description='Send an URL to the crawler - Create a crawler task') parser.add_argument('-u', '--url', type=str, help='URL to crawl', required=True) parser.add_argument('-k', '--key', type=str, help='AIL API Key', required=True) parser.add_argument('-a', '--ail', type=str, help='AIL URL') parser.add_argument('-d', '--depth', type=int, default=1, help='Depth limit') # TODO improve me parser.add_argument('--cookiejar', type=str, help='Cookiejar uuid') - parser.add_argument('-p', '--proxy', type=str, help='Proxy address to use, "web" and "tor" can be used as shortcut (web is used by default is )') + parser.add_argument('-p', '--proxy', type=str, help='Proxy address to use, "web" and "tor" can be used as shortcut (web is used by default if the domain isn\'t an onion)') group = parser.add_mutually_exclusive_group() group.add_argument('--har', dest='har', action='store_true', help='Save HAR') From fc477ebcb0308b65f0d7b86b38d8153539b15421 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Wed, 26 Jul 2023 10:14:03 +0200 Subject: [PATCH 29/45] Update crawler_add_task.py --- tools/crawler_add_task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/crawler_add_task.py b/tools/crawler_add_task.py index 6b59242a..5d9604fe 100755 --- a/tools/crawler_add_task.py +++ b/tools/crawler_add_task.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ -DIR/File Importer Helper +Send an URL to the crawler - Create a crawler task ================ -Import Content +Import URL to be crawled by AIL and then analysed """ From 4eb5b4c4b9aeabf353a30c29d2d744ce3d09191a Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 28 Jul 2023 11:10:21 +0200 Subject: [PATCH 30/45] fix: [MailExporter] improve logs + SMTP over SSL: add an option to validate self signed CA This exporter use STARTTLS by defaults. If the SMTP server does not support the STARTTLS extension, the client will switch to SSL instead. --- bin/exporter/MailExporter.py | 33 +++++++++++++++++++++++++++------ configs/core.cfg.sample | 4 ++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/bin/exporter/MailExporter.py b/bin/exporter/MailExporter.py index 4b2d4a3a..c4d3f5b5 100755 --- a/bin/exporter/MailExporter.py +++ b/bin/exporter/MailExporter.py @@ -8,9 +8,12 @@ Import Content """ import os +import logging +import logging.config import sys from abc import ABC +from ssl import create_default_context import smtplib from email.mime.multipart import MIMEMultipart @@ -22,17 +25,22 @@ sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## +from lib import ail_logger from exporter.abstract_exporter import AbstractExporter from lib.ConfigLoader import ConfigLoader # from lib.objects.abstract_object import AbstractObject # from lib.Tracker import Tracker +logging.config.dictConfig(ail_logger.get_config(name='modules')) + class MailExporter(AbstractExporter, ABC): - def __init__(self, host=None, port=None, password=None, user='', sender=''): + def __init__(self, host=None, port=None, password=None, user='', sender='', cert_required=None, ca_file=None): super().__init__() config_loader = ConfigLoader() + self.logger = logging.getLogger(f'{self.__class__.__name__}') + if host: self.host = host self.port = port @@ -45,6 +53,15 @@ class MailExporter(AbstractExporter, ABC): self.pw = config_loader.get_config_str("Notifications", "sender_pw") if self.pw == 'None': self.pw = None + if cert_required is not None: + self.cert_required = bool(cert_required) + self.ca_file = ca_file + else: + self.cert_required = config_loader.get_config_boolean("Notifications", "cert_required") + if self.cert_required: + self.ca_file = config_loader.get_config_str("Notifications", "ca_file") + else: + self.ca_file = None if user: self.user = user else: @@ -67,8 +84,12 @@ class MailExporter(AbstractExporter, ABC): smtp_server = smtplib.SMTP(self.host, self.port) smtp_server.starttls() except smtplib.SMTPNotSupportedError: - print("The server does not support the STARTTLS extension.") - smtp_server = smtplib.SMTP_SSL(self.host, self.port) + self.logger.info(f"The server {self.host}:{self.port} does not support the STARTTLS extension.") + if self.cert_required: + context = create_default_context(cafile=self.ca_file) + else: + context = None + smtp_server = smtplib.SMTP_SSL(self.host, self.port, context=context) smtp_server.ehlo() if self.user is not None: @@ -80,7 +101,7 @@ class MailExporter(AbstractExporter, ABC): return smtp_server # except Exception as err: # traceback.print_tb(err.__traceback__) - # logger.warning(err) + # self.logger.warning(err) def _export(self, recipient, subject, body): mime_msg = MIMEMultipart() @@ -95,8 +116,8 @@ class MailExporter(AbstractExporter, ABC): smtp_client.quit() # except Exception as err: # traceback.print_tb(err.__traceback__) - # logger.warning(err) - print(f'Send notification: {subject} to {recipient}') + # self.logger.warning(err) + self.logger.info(f'Send notification: {subject} to {recipient}') class MailExporterTracker(MailExporter): diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index bb9054fc..9bdf40a4 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -45,6 +45,10 @@ sender = sender@example.com sender_host = smtp.example.com sender_port = 1337 sender_pw = None +# Only needed for SSL if the mail server don't support TLS (used by default). use this option to validate the server certificate. +cert_required = False +# Only needed for the SSL if you want to validate your self signed certificate for SSL +ca_file = # Only needed when the credentials for email server needs a username instead of an email address #sender_user = sender sender_user = From 64252ddf752bacb4cb9c362c3213b81a7de7fdb5 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 31 Jul 2023 16:00:31 +0200 Subject: [PATCH 31/45] fix: [Tracker] edit tracker filters --- bin/lib/Tracker.py | 8 ++++++++ configs/core.cfg.sample | 4 ++-- var/www/templates/hunter/tracker_add.html | 10 +++++++--- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index b7ca5e77..f1588137 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -207,6 +207,13 @@ class Tracker: if filters: self._set_field('filters', json.dumps(filters)) + def del_filters(self, tracker_type, to_track): + filters = self.get_filters() + for obj_type in filters: + r_tracker.srem(f'trackers:objs:{tracker_type}:{obj_type}', to_track) + r_tracker.srem(f'trackers:uuid:{tracker_type}:{to_track}', f'{self.uuid}:{obj_type}') + r_tracker.hdel(f'tracker:{self.uuid}', 'filters') + def get_tracked(self): return self._get_field('tracked') @@ -513,6 +520,7 @@ class Tracker: self._set_mails(mails) # Filters + self.del_filters(old_type, old_to_track) if not filters: filters = {} for obj_type in get_objects_tracked(): diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index 9bdf40a4..3278033f 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -45,9 +45,9 @@ sender = sender@example.com sender_host = smtp.example.com sender_port = 1337 sender_pw = None -# Only needed for SSL if the mail server don't support TLS (used by default). use this option to validate the server certificate. +# Only needed for SMTP over SSL if the mail server don't support TLS (used by default). use this option to validate the server certificate. cert_required = False -# Only needed for the SSL if you want to validate your self signed certificate for SSL +# Only needed for SMTP over SSL if you want to validate your self signed certificate for SSL ca_file = # Only needed when the credentials for email server needs a username instead of an email address #sender_user = sender diff --git a/var/www/templates/hunter/tracker_add.html b/var/www/templates/hunter/tracker_add.html index 9233e4ad..7cc690ba 100644 --- a/var/www/templates/hunter/tracker_add.html +++ b/var/www/templates/hunter/tracker_add.html @@ -76,7 +76,7 @@ {#
    • #}
    - +
    {#
    #} {# #} @@ -84,7 +84,7 @@ {#
    #}
    - +
    @@ -101,7 +101,7 @@
    - +
    @@ -351,6 +351,10 @@ $(document).ready(function(){ }); +$(function () { + $('[data-toggle="tooltip"]').tooltip() +}) + function toggle_sidebar(){ if($('#nav_menu').is(':visible')){ $('#nav_menu').hide(); From 1aa0bd8a0ec20c8d6b4c1f7e26123c90b0926656 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 31 Jul 2023 16:25:28 +0200 Subject: [PATCH 32/45] fix: [settings] fix edit user --- bin/lib/Tracker.py | 3 -- bin/lib/Users.py | 8 ++++- var/www/modules/settings/Flask_settings.py | 37 ++++++++++++++-------- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index f1588137..f1ea8905 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -530,9 +530,6 @@ class Tracker: for obj_type in filters: r_tracker.sadd(f'trackers:objs:{tracker_type}:{obj_type}', to_track) r_tracker.sadd(f'trackers:uuid:{tracker_type}:{to_track}', f'{self.uuid}:{obj_type}') - if tracker_type != old_type: - r_tracker.srem(f'trackers:objs:{old_type}:{obj_type}', old_to_track) - r_tracker.srem(f'trackers:uuid:{old_type}:{old_to_track}', f'{self.uuid}:{obj_type}') # Refresh Trackers trigger_trackers_refresh(tracker_type) diff --git a/bin/lib/Users.py b/bin/lib/Users.py index a61830ef..765b1360 100755 --- a/bin/lib/Users.py +++ b/bin/lib/Users.py @@ -247,7 +247,10 @@ class User(UserMixin): self.id = "__anonymous__" def exists(self): - return self.id != "__anonymous__" + if self.id == "__anonymous__": + return False + else: + return r_serv_db.exists(f'ail:user:metadata:{self.id}') # return True or False # def is_authenticated(): @@ -287,3 +290,6 @@ class User(UserMixin): return True else: return False + + def get_role(self): + return r_serv_db.hget(f'ail:user:metadata:{self.id}', 'role') diff --git a/var/www/modules/settings/Flask_settings.py b/var/www/modules/settings/Flask_settings.py index 4316d490..2b1b8826 100644 --- a/var/www/modules/settings/Flask_settings.py +++ b/var/www/modules/settings/Flask_settings.py @@ -19,7 +19,6 @@ sys.path.append(os.environ['AIL_BIN']) from lib import d4 from lib import Users - # ============ VARIABLES ============ import Flask_config @@ -33,7 +32,6 @@ email_regex = Flask_config.email_regex settings = Blueprint('settings', __name__, template_folder='templates') - # ============ FUNCTIONS ============ def check_email(email): @@ -43,6 +41,7 @@ def check_email(email): else: return False + # ============= ROUTES ============== @settings.route("/settings/edit_profile", methods=['GET']) @@ -52,7 +51,8 @@ def edit_profile(): user_metadata = Users.get_user_metadata(current_user.get_id()) admin_level = current_user.is_in_role('admin') return render_template("edit_profile.html", user_metadata=user_metadata, - admin_level=admin_level) + admin_level=admin_level) + @settings.route("/settings/new_token", methods=['GET']) @login_required @@ -61,6 +61,7 @@ def new_token(): Users.generate_new_token(current_user.get_id()) return redirect(url_for('settings.edit_profile')) + @settings.route("/settings/new_token_user", methods=['POST']) @login_required @login_admin @@ -70,6 +71,7 @@ def new_token_user(): Users.generate_new_token(user_id) return redirect(url_for('settings.users_list')) + @settings.route("/settings/create_user", methods=['GET']) @login_required @login_admin @@ -78,14 +80,15 @@ def create_user(): error = request.args.get('error') error_mail = request.args.get('error_mail') role = None - if r_serv_db.exists('user_metadata:{}'.format(user_id)): - role = r_serv_db.hget('user_metadata:{}'.format(user_id), 'role') - else: - user_id = None + if user_id: + user = Users.User(user_id) + if user.exists(): + role = user.get_role() all_roles = Users.get_all_roles() return render_template("create_user.html", all_roles=all_roles, user_id=user_id, user_role=role, - error=error, error_mail=error_mail, - admin_level=True) + error=error, error_mail=error_mail, + admin_level=True) + @settings.route("/settings/create_user_post", methods=['POST']) @login_required @@ -98,17 +101,19 @@ def create_user_post(): all_roles = Users.get_all_roles() - if email and len(email)< 300 and check_email(email) and role: + if email and len(email) < 300 and check_email(email) and role: if role in all_roles: # password set if password1 and password2: - if password1==password2: + if password1 == password2: if Users.check_password_strength(password1): password = password1 else: - return render_template("create_user.html", all_roles=all_roles, error="Incorrect Password", admin_level=True) + return render_template("create_user.html", all_roles=all_roles, error="Incorrect Password", + admin_level=True) else: - return render_template("create_user.html", all_roles=all_roles, error="Passwords don't match", admin_level=True) + return render_template("create_user.html", all_roles=all_roles, error="Passwords don't match", + admin_level=True) # generate password else: password = Users.gen_password() @@ -127,6 +132,7 @@ def create_user_post(): else: return render_template("create_user.html", all_roles=all_roles, error_mail=True, admin_level=True) + @settings.route("/settings/users_list", methods=['GET']) @login_required @login_admin @@ -140,6 +146,7 @@ def users_list(): new_user_dict['password'] = request.args.get('new_user_password') return render_template("users_list.html", all_users=all_users, new_user=new_user_dict, admin_level=True) + @settings.route("/settings/edit_user", methods=['POST']) @login_required @login_admin @@ -147,6 +154,7 @@ def edit_user(): user_id = request.form.get('user_id') return redirect(url_for('settings.create_user', user_id=user_id)) + @settings.route("/settings/delete_user", methods=['POST']) @login_required @login_admin @@ -163,6 +171,7 @@ def passive_dns(): passivedns_enabled = d4.is_passive_dns_enabled() return render_template("passive_dns.html", passivedns_enabled=passivedns_enabled) + @settings.route("/settings/passivedns/change_state", methods=['GET']) @login_required @login_admin @@ -171,11 +180,13 @@ def passive_dns_change_state(): passivedns_enabled = d4.change_passive_dns_state(new_state) return redirect(url_for('settings.passive_dns')) + @settings.route("/settings/ail", methods=['GET']) @login_required @login_admin def ail_configs(): return render_template("ail_configs.html", passivedns_enabled=None) + # ========= REGISTRATION ========= app.register_blueprint(settings, url_prefix=baseUrl) From 14a76a91d983709ffc19d4fc713e73e01d60c029 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 1 Aug 2023 11:07:06 +0200 Subject: [PATCH 33/45] fix: [tags ui] fix galaxy, get number of tags enabled + add toolip helper --- bin/lib/Tag.py | 2 +- var/www/templates/tags/galaxies.html | 8 ++++---- var/www/templates/tags/taxonomies.html | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bin/lib/Tag.py b/bin/lib/Tag.py index 94b2eca4..64850b3c 100755 --- a/bin/lib/Tag.py +++ b/bin/lib/Tag.py @@ -338,7 +338,7 @@ def get_galaxy_meta(galaxy_name, nb_active_tags=False): else: meta['icon'] = f'fas fa-{icon}' if nb_active_tags: - meta['nb_active_tags'] = get_galaxy_nb_tags_enabled(galaxy) + meta['nb_active_tags'] = get_galaxy_nb_tags_enabled(galaxy.type) meta['nb_tags'] = len(get_galaxy_tags(galaxy.type)) return meta diff --git a/var/www/templates/tags/galaxies.html b/var/www/templates/tags/galaxies.html index ff69c2b5..ac0b443c 100644 --- a/var/www/templates/tags/galaxies.html +++ b/var/www/templates/tags/galaxies.html @@ -41,7 +41,7 @@ Namespace Enabled Active Tags - + @@ -51,7 +51,7 @@ {{ galaxy['description'] }} {{ galaxy['namespace'] }} - {% if galaxy['enebled'] %} + {% if galaxy['enabled'] %}
    @@ -69,7 +69,7 @@ -     + @@ -98,7 +98,7 @@ $(document).ready(function(){ $("#nav_galaxies").addClass("active"); - $('#myTable_').DataTable({ "lengthMenu": [ 5, 10, 25, 50, 100 ], "pageLength": 15, "order": [[ 0, "asc" ]] }); + $('#myTable_').DataTable({ "lengthMenu": [ 5, 10, 25, 50, 100 ], "pageLength": 15, "order": [[ 3, "desc" ]] }); }); diff --git a/var/www/templates/tags/taxonomies.html b/var/www/templates/tags/taxonomies.html index eff480b5..ac3346d7 100644 --- a/var/www/templates/tags/taxonomies.html +++ b/var/www/templates/tags/taxonomies.html @@ -41,7 +41,7 @@ Version Enabled Active Tags - + @@ -69,7 +69,7 @@ - + @@ -98,7 +98,7 @@ $(document).ready(function(){ $("#nav_taxonomies").addClass("active"); - $('#myTable_').DataTable({ "lengthMenu": [ 5, 10, 25, 50, 100 ], "pageLength": 15, "order": [[ 0, "asc" ]] }); + $('#myTable_').DataTable({ "lengthMenu": [ 5, 10, 25, 50, 100 ], "pageLength": 15, "order": [[ 3, "desc" ]] }); }); From 9098ab25a6cd782230ef3268d4624473654f5bee Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 1 Aug 2023 11:30:45 +0200 Subject: [PATCH 34/45] chg: [tracker ui] improve show typo squatting button + add tooltip --- var/www/templates/hunter/tracker_show.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/www/templates/hunter/tracker_show.html b/var/www/templates/hunter/tracker_show.html index c2715ccc..62c387b6 100644 --- a/var/www/templates/hunter/tracker_show.html +++ b/var/www/templates/hunter/tracker_show.html @@ -89,8 +89,8 @@ Tracked {% if meta['type'] == 'typosquatting' %} -
    From ac45c2dd61f41cf61b96fb2b1c6b86d6c8ba7685 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 1 Aug 2023 14:30:36 +0200 Subject: [PATCH 35/45] chg: [crawler ui] last crawled domains, show last check timestamp --- var/www/blueprints/crawler_splash.py | 1 + var/www/templates/crawler/crawler_splash/last_crawled.html | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/var/www/blueprints/crawler_splash.py b/var/www/blueprints/crawler_splash.py index 39d84971..1b7f4454 100644 --- a/var/www/blueprints/crawler_splash.py +++ b/var/www/blueprints/crawler_splash.py @@ -272,6 +272,7 @@ def crawlers_last_domains(): domain, epoch = domain_row.split(':', 1) dom = Domains.Domain(domain) meta = dom.get_meta() + meta['last'] = datetime.fromtimestamp(int(epoch)).strftime("%Y/%m/%d %H:%M.%S") meta['epoch'] = epoch meta['status_epoch'] = dom.is_up_by_epoch(epoch) domains.append(meta) diff --git a/var/www/templates/crawler/crawler_splash/last_crawled.html b/var/www/templates/crawler/crawler_splash/last_crawled.html index a097c47a..582e45cb 100644 --- a/var/www/templates/crawler/crawler_splash/last_crawled.html +++ b/var/www/templates/crawler/crawler_splash/last_crawled.html @@ -74,7 +74,7 @@ data-content="epoch: {{domain['epoch']}}
    last status: {{ domain['status'] }}"> {{ domain['domain'] }} {{domain['first_seen']}} - {{domain['last_check']}} + {{domain['last']}} {% if domain['status_epoch'] %}
    From 7d19da0806ec4a353aca085e681ef40d9dd806b2 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 2 Aug 2023 13:50:07 +0200 Subject: [PATCH 36/45] chg: [flask] cleanup, remove unused import --- var/www/Flask_server.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 93d3a47d..c9d35232 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -17,9 +17,6 @@ from flask_login import LoginManager, current_user, login_user, logout_user, log import importlib from os.path import join -# # TODO: put me in lib/Tag -from pytaxonomies import Taxonomies - sys.path.append('./modules/') sys.path.append(os.environ['AIL_BIN']) @@ -255,6 +252,7 @@ for taxonomy in default_taxonomies: Tag.enable_taxonomy_tags(taxonomy) # ========== INITIAL tags auto export ============ +# from pytaxonomies import Taxonomies # taxonomies = Taxonomies() # # infoleak_tags = taxonomies.get('infoleak').machinetags() From 859591b53f0cd5c52864eca94bf4e8dc755f0de3 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 2 Aug 2023 13:51:13 +0200 Subject: [PATCH 37/45] chg: [flask] cleanup --- var/www/Flask_server.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index c9d35232..e6a99350 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -251,17 +251,6 @@ default_taxonomies = ["infoleak", "gdpr", "fpf", "dark-web"] for taxonomy in default_taxonomies: Tag.enable_taxonomy_tags(taxonomy) -# ========== INITIAL tags auto export ============ -# from pytaxonomies import Taxonomies -# taxonomies = Taxonomies() -# -# infoleak_tags = taxonomies.get('infoleak').machinetags() -# infoleak_automatic_tags = [] -# for tag in taxonomies.get('infoleak').machinetags(): -# if tag.split('=')[0][:] == 'infoleak:automatic-detection': -# r_serv_db.sadd('list_export_tags', tag) -# -# r_serv_db.sadd('list_export_tags', 'infoleak:submission="manual"') # ============ MAIN ============ if __name__ == "__main__": From 2691000d0cdf15b90030a12c101af02bf22c9009 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 2 Aug 2023 15:49:12 +0200 Subject: [PATCH 38/45] chg: [telegram fedeer] use meta of the new feeder --- bin/importer/feeders/Telegram.py | 34 ++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/bin/importer/feeders/Telegram.py b/bin/importer/feeders/Telegram.py index 3856c88e..c9c448ef 100755 --- a/bin/importer/feeders/Telegram.py +++ b/bin/importer/feeders/Telegram.py @@ -29,8 +29,8 @@ class TelegramFeeder(DefaultFeeder): def get_item_id(self): # TODO use telegram message date date = datetime.date.today().strftime("%Y/%m/%d") - channel_id = str(self.json_data['meta']['channel_id']) - message_id = str(self.json_data['meta']['message_id']) + channel_id = str(self.json_data['meta']['chat']['id']) + message_id = str(self.json_data['meta']['id']) item_id = f'{channel_id}_{message_id}' item_id = os.path.join('telegram', date, item_id) self.item_id = f'{item_id}.gz' @@ -40,17 +40,21 @@ class TelegramFeeder(DefaultFeeder): """ Process JSON meta field. """ - # channel_id = str(self.json_data['meta']['channel_id']) - # message_id = str(self.json_data['meta']['message_id']) - # telegram_id = f'{channel_id}_{message_id}' - # item_basic.add_map_obj_id_item_id(telegram_id, item_id, 'telegram_id') ######################################### - user = None - if self.json_data['meta'].get('user'): - user = str(self.json_data['meta']['user']) - elif self.json_data['meta'].get('channel'): - user = str(self.json_data['meta']['channel'].get('username')) - if user: - date = item_basic.get_item_date(self.item_id) - username = Username(user, 'telegram') - username.add(date, self.item_id) + # message chat + meta = self.json_data['meta'] + if meta.get('chat'): + if meta['chat'].get('username'): + user = meta['chat']['username'] + if user: + date = item_basic.get_item_date(self.item_id) + username = Username(user, 'telegram') + username.add(date, self.item_id) + # message sender + if meta.get('sender'): + if meta['sender'].get('username'): + user = meta['sender']['username'] + if user: + date = item_basic.get_item_date(self.item_id) + username = Username(user, 'telegram') + username.add(date, self.item_id) return None From bd7aa979bd847a516155bb896d6453a961f63116 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 8 Aug 2023 10:36:58 +0200 Subject: [PATCH 39/45] chg: [module extrator] add debug --- bin/lib/module_extractor.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/lib/module_extractor.py b/bin/lib/module_extractor.py index d4ea6c78..54e4d3ab 100755 --- a/bin/lib/module_extractor.py +++ b/bin/lib/module_extractor.py @@ -104,7 +104,11 @@ def _get_word_regex(word): def convert_byte_offset_to_string(b_content, offset): byte_chunk = b_content[:offset + 1] - string_chunk = byte_chunk.decode() + try: + string_chunk = byte_chunk.decode() + except UnicodeDecodeError as e: + logger.error(f'Yara offset coverter error, {e.reason}\n{b_content}\n{offset}') + string_chunk = b_content offset = len(string_chunk) - 1 return offset From 529a24c191299551ccd90959bea8ae382c0fa5da Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 8 Aug 2023 10:40:44 +0200 Subject: [PATCH 40/45] chg: [module extrator] add debug --- bin/lib/module_extractor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/lib/module_extractor.py b/bin/lib/module_extractor.py index 54e4d3ab..cdb67ab6 100755 --- a/bin/lib/module_extractor.py +++ b/bin/lib/module_extractor.py @@ -107,8 +107,8 @@ def convert_byte_offset_to_string(b_content, offset): try: string_chunk = byte_chunk.decode() except UnicodeDecodeError as e: - logger.error(f'Yara offset coverter error, {e.reason}\n{b_content}\n{offset}') - string_chunk = b_content + logger.error(f'Yara offset converter error, {e.reason}\n{byte_chunk}\n{offset}') + string_chunk = byte_chunk offset = len(string_chunk) - 1 return offset From 4dc5527c1a74586a4fd97a646559844cfdbc8da9 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 8 Aug 2023 11:26:16 +0200 Subject: [PATCH 41/45] fix: [module extractor] fix invalid yara offset --- bin/lib/module_extractor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/lib/module_extractor.py b/bin/lib/module_extractor.py index cdb67ab6..681d666e 100755 --- a/bin/lib/module_extractor.py +++ b/bin/lib/module_extractor.py @@ -106,11 +106,11 @@ def convert_byte_offset_to_string(b_content, offset): byte_chunk = b_content[:offset + 1] try: string_chunk = byte_chunk.decode() + offset = len(string_chunk) - 1 + return offset except UnicodeDecodeError as e: - logger.error(f'Yara offset converter error, {e.reason}\n{byte_chunk}\n{offset}') - string_chunk = byte_chunk - offset = len(string_chunk) - 1 - return offset + logger.error(f'Yara offset converter error, {str(e)}\n{offset}/{len(b_content)}') + return convert_byte_offset_to_string(b_content, offset) # TODO RETRO HUNTS From f05c7b6a93ff5a166d338d9d9fdd6a7825c33918 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 8 Aug 2023 11:27:57 +0200 Subject: [PATCH 42/45] fix: [module extractor] fix invalid yara offset --- bin/lib/module_extractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/lib/module_extractor.py b/bin/lib/module_extractor.py index 681d666e..b6254372 100755 --- a/bin/lib/module_extractor.py +++ b/bin/lib/module_extractor.py @@ -110,7 +110,7 @@ def convert_byte_offset_to_string(b_content, offset): return offset except UnicodeDecodeError as e: logger.error(f'Yara offset converter error, {str(e)}\n{offset}/{len(b_content)}') - return convert_byte_offset_to_string(b_content, offset) + return convert_byte_offset_to_string(b_content, offset - 1) # TODO RETRO HUNTS From 3c1813ba02bec0dac4a0dc677cda771be55a23fe Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 18 Aug 2023 11:05:21 +0200 Subject: [PATCH 43/45] chg: [core] add telegram importer + Chat object + message Object + add timeline engine --- bin/importer/FeederImporter.py | 9 +- bin/importer/feeders/Telegram.py | 110 +++++-- bin/lib/ail_core.py | 16 +- bin/lib/correlations_engine.py | 7 +- bin/lib/objects/Chats.py | 288 ++++++++++++++++++ bin/lib/objects/Items.py | 2 + bin/lib/objects/Messages.py | 268 ++++++++++++++++ bin/lib/objects/UsersAccount.py | 154 ++++++++++ bin/lib/objects/abstract_daterange_object.py | 4 +- bin/lib/objects/abstract_object.py | 56 ++++ bin/lib/objects/ail_objects.py | 3 + bin/lib/timeline_engine.py | 157 ++++++++++ configs/6383.conf | 1 + configs/core.cfg.sample | 5 + var/www/Flask_server.py | 2 + var/www/blueprints/objects_chat.py | 58 ++++ var/www/blueprints/objects_subtypes.py | 6 + .../templates/objects/chat/ChatMessages.html | 190 ++++++++++++ 18 files changed, 1307 insertions(+), 29 deletions(-) create mode 100755 bin/lib/objects/Chats.py create mode 100755 bin/lib/objects/Messages.py create mode 100755 bin/lib/objects/UsersAccount.py create mode 100755 bin/lib/timeline_engine.py create mode 100644 var/www/blueprints/objects_chat.py create mode 100644 var/www/templates/objects/chat/ChatMessages.html diff --git a/bin/importer/FeederImporter.py b/bin/importer/FeederImporter.py index e7a06132..021a52e2 100755 --- a/bin/importer/FeederImporter.py +++ b/bin/importer/FeederImporter.py @@ -87,13 +87,16 @@ class FeederImporter(AbstractImporter): feeder_name = feeder.get_name() print(f'importing: {feeder_name} feeder') - item_id = feeder.get_item_id() + item_id = feeder.get_item_id() # TODO replace me with object global id # process meta if feeder.get_json_meta(): feeder.process_meta() - gzip64_content = feeder.get_gzip64_content() - return f'{feeder_name} {item_id} {gzip64_content}' + if feeder_name == 'telegram': + return item_id # TODO support UI dashboard + else: + gzip64_content = feeder.get_gzip64_content() + return f'{feeder_name} {item_id} {gzip64_content}' class FeederModuleImporter(AbstractModule): diff --git a/bin/importer/feeders/Telegram.py b/bin/importer/feeders/Telegram.py index c9c448ef..52eb0a75 100755 --- a/bin/importer/feeders/Telegram.py +++ b/bin/importer/feeders/Telegram.py @@ -16,9 +16,30 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from importer.feeders.Default import DefaultFeeder +from lib.ConfigLoader import ConfigLoader +from lib.objects.Chats import Chat +from lib.objects import Messages +from lib.objects import UsersAccount from lib.objects.Usernames import Username from lib import item_basic +import base64 +import io +import gzip +def gunzip_bytes_obj(bytes_obj): + gunzipped_bytes_obj = None + try: + in_ = io.BytesIO() + in_.write(bytes_obj) + in_.seek(0) + + with gzip.GzipFile(fileobj=in_, mode='rb') as fo: + gunzipped_bytes_obj = fo.read() + except Exception as e: + print(f'Global; Invalid Gzip file: {e}') + + return gunzipped_bytes_obj + class TelegramFeeder(DefaultFeeder): def __init__(self, json_data): @@ -26,14 +47,17 @@ class TelegramFeeder(DefaultFeeder): self.name = 'telegram' # define item id - def get_item_id(self): - # TODO use telegram message date - date = datetime.date.today().strftime("%Y/%m/%d") - channel_id = str(self.json_data['meta']['chat']['id']) + def get_item_id(self): # TODO rename self.item_id + # Get message date + timestamp = self.json_data['meta']['date']['timestamp'] # TODO CREATE DEFAULT TIMESTAMP + # if self.json_data['meta'].get('date'): + # date = datetime.datetime.fromtimestamp( self.json_data['meta']['date']['timestamp']) + # date = date.strftime('%Y/%m/%d') + # else: + # date = datetime.date.today().strftime("%Y/%m/%d") + chat_id = str(self.json_data['meta']['chat']['id']) message_id = str(self.json_data['meta']['id']) - item_id = f'{channel_id}_{message_id}' - item_id = os.path.join('telegram', date, item_id) - self.item_id = f'{item_id}.gz' + self.item_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) return self.item_id def process_meta(self): @@ -42,19 +66,67 @@ class TelegramFeeder(DefaultFeeder): """ # message chat meta = self.json_data['meta'] + mess_id = self.json_data['meta']['id'] + if meta.get('reply_to'): + reply_to_id = meta['reply_to'] + else: + reply_to_id = None + + timestamp = meta['date']['timestamp'] + date = datetime.datetime.fromtimestamp(timestamp) + date = date.strftime('%Y%m%d') + if meta.get('chat'): - if meta['chat'].get('username'): - user = meta['chat']['username'] - if user: - date = item_basic.get_item_date(self.item_id) - username = Username(user, 'telegram') - username.add(date, self.item_id) + chat = Chat(meta['chat']['id'], 'telegram') + + if meta['chat'].get('username'): # TODO USE ID AND SAVE USERNAME + chat_username = meta['chat']['username'] + + # Chat---Message + chat.add(date, self.item_id) # TODO modify to accept file objects + # message meta ????? who is the user if two user ???? + + if self.json_data.get('translation'): + translation = self.json_data['translation'] + else: + translation = None + decoded = base64.standard_b64decode(self.json_data['data']) + content = gunzip_bytes_obj(decoded) + Messages.create(self.item_id, content, translation=translation) + + chat.add_message(self.item_id, timestamp, mess_id, reply_id=reply_to_id) + else: + chat = None + # message sender - if meta.get('sender'): + if meta.get('sender'): # TODO handle message channel forward + user_id = meta['sender']['id'] + user_account = UsersAccount.UserAccount(user_id, 'telegram') + # UserAccount---Message + user_account.add(date, self.item_id) + # UserAccount---Chat + user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id) + + if meta['sender'].get('firstname'): + user_account.set_first_name(meta['sender']['firstname']) + if meta['sender'].get('lastname'): + user_account.set_last_name(meta['sender']['lastname']) + if meta['sender'].get('phone'): + user_account.set_phone(meta['sender']['phone']) + if meta['sender'].get('username'): - user = meta['sender']['username'] - if user: - date = item_basic.get_item_date(self.item_id) - username = Username(user, 'telegram') - username.add(date, self.item_id) + username = Username(meta['sender']['username'], 'telegram') + user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id) + + # Username---Message + username.add(date, self.item_id) # TODO #################################################################### + if chat: + chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id) + + # if meta.get('fwd_from'): + # if meta['fwd_from'].get('post_author') # user first name + + # TODO reply threads ???? + + return None diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index 75520a2b..eeb83a98 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -15,8 +15,8 @@ config_loader = ConfigLoader() r_serv_db = config_loader.get_db_conn("Kvrocks_DB") config_loader = None -AIL_OBJECTS = sorted({'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'hhhash', 'item', - 'pgp', 'screenshot', 'title', 'username'}) +AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'hhhash', 'item', + 'pgp', 'screenshot', 'title', 'user-account', 'username'}) def get_ail_uuid(): ail_uuid = r_serv_db.get('ail:uuid') @@ -38,9 +38,11 @@ def get_all_objects(): return AIL_OBJECTS def get_objects_with_subtypes(): - return ['cryptocurrency', 'pgp', 'username'] + return ['chat', 'cryptocurrency', 'pgp', 'username'] def get_object_all_subtypes(obj_type): + if obj_type == 'chat': + return ['discord', 'jabber', 'telegram'] if obj_type == 'cryptocurrency': return ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash'] if obj_type == 'pgp': @@ -66,6 +68,14 @@ def get_all_objects_with_subtypes_tuple(): str_objs.append((obj_type, '')) return str_objs +def unpack_obj_global_id(global_id, r_type='tuple'): + if r_type == 'dict': + obj = global_id.split(':', 2) + return {'type': obj[0], 'subtype': obj[1], 'id': obj['2']} + else: # tuple(type, subtype, id) + return global_id.split(':', 2) + + ##-- AIL OBJECTS --## #### Redis #### diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index 609aa8c6..94a06773 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -41,6 +41,7 @@ config_loader = None ################################## CORRELATION_TYPES_BY_OBJ = { + "chat": ["item", "username"], # item ??? "cookie-name": ["domain"], "cryptocurrency": ["domain", "item"], "cve": ["domain", "item"], @@ -49,11 +50,11 @@ CORRELATION_TYPES_BY_OBJ = { "etag": ["domain"], "favicon": ["domain", "item"], # TODO Decoded "hhhash": ["domain"], - "item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], + "item": ["chat", "cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], "pgp": ["domain", "item"], "screenshot": ["domain", "item"], "title": ["domain", "item"], - "username": ["domain", "item"], + "username": ["chat", "domain", "item"], } def get_obj_correl_types(obj_type): @@ -65,6 +66,8 @@ def sanityze_obj_correl_types(obj_type, correl_types): correl_types = set(correl_types).intersection(obj_correl_types) if not correl_types: correl_types = obj_correl_types + if not correl_types: + return [] return correl_types def get_nb_correlation_by_correl_type(obj_type, subtype, obj_id, correl_type): diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py new file mode 100755 index 00000000..438acf51 --- /dev/null +++ b/bin/lib/objects/Chats.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys + +from datetime import datetime + +from flask import url_for +# from pymisp import MISPObject + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import ail_core +from lib.ConfigLoader import ConfigLoader +from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id +from lib.data_retention_engine import update_obj_date +from lib.objects import ail_objects +from lib import item_basic + +from lib.correlations_engine import get_correlation_by_correl_type + +config_loader = ConfigLoader() +baseurl = config_loader.get_config_str("Notifications", "ail_domain") +r_object = config_loader.get_db_conn("Kvrocks_Objects") +r_cache = config_loader.get_redis_conn("Redis_Cache") +config_loader = None + + +################################################################################ +################################################################################ +################################################################################ + +class Chat(AbstractSubtypeObject): # TODO # ID == username ????? + """ + AIL Chat Object. (strings) + """ + + def __init__(self, id, subtype): + super(Chat, self).__init__('chat', id, subtype) + + # def get_ail_2_ail_payload(self): + # payload = {'raw': self.get_gzip_content(b64=True), + # 'compress': 'gzip'} + # return payload + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + # # TODO: + pass + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('correlation.show_correlation', type=self.type, subtype=self.subtype, id=self.id) + else: + url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}' + return url + + def get_svg_icon(self): # TODO + # if self.subtype == 'telegram': + # style = 'fab' + # icon = '\uf2c6' + # elif self.subtype == 'discord': + # style = 'fab' + # icon = '\uf099' + # else: + # style = 'fas' + # icon = '\uf007' + style = 'fas' + icon = '\uf086' + return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5} + + def get_meta(self, options=set()): + meta = self._get_meta(options=options) + meta['id'] = self.id + meta['subtype'] = self.subtype + meta['tags'] = self.get_tags(r_list=True) + return meta + + def get_misp_object(self): + # obj_attrs = [] + # if self.subtype == 'telegram': + # obj = MISPObject('telegram-account', standalone=True) + # obj_attrs.append(obj.add_attribute('username', value=self.id)) + # + # elif self.subtype == 'twitter': + # obj = MISPObject('twitter-account', standalone=True) + # obj_attrs.append(obj.add_attribute('name', value=self.id)) + # + # else: + # obj = MISPObject('user-account', standalone=True) + # obj_attrs.append(obj.add_attribute('username', value=self.id)) + # + # first_seen = self.get_first_seen() + # last_seen = self.get_last_seen() + # if first_seen: + # obj.first_seen = first_seen + # if last_seen: + # obj.last_seen = last_seen + # if not first_seen or not last_seen: + # self.logger.warning( + # f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}') + # + # for obj_attr in obj_attrs: + # for tag in self.get_tags(): + # obj_attr.add_tag(tag) + # return obj + return + + ############################################################################ + ############################################################################ + + # others optional metas, ... -> # TODO ALL meta in hset + + def get_name(self): # get username ???? + pass + + # return username correlation + def get_users(self): # get participants ??? -> passive users ??? + pass + + # def get_last_message_id(self): + # + # return r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'last:message:id') + + def get_obj_message_id(self, obj_id): + if obj_id.endswith('.gz'): + obj_id = obj_id[:-3] + return int(obj_id.split('_')[-1]) + + def _get_message_timestamp(self, obj_global_id): + return r_object.zscore(f'messages:{self.type}:{self.subtype}:{self.id}', obj_global_id) + + def _get_messages(self): + return r_object.zrange(f'messages:{self.type}:{self.subtype}:{self.id}', 0, -1, withscores=True) + + def get_message_meta(self, obj_global_id, parent=True, mess_datetime=None): + obj = ail_objects.get_obj_from_global_id(obj_global_id) + mess_dict = obj.get_meta(options={'content', 'link', 'parent'}) + if mess_dict.get('parent') and parent: + mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False) + mess_dict['username'] = {} + user = obj.get_correlation('username').get('username') + if user: + subtype, user = user.pop().split(':', 1) + mess_dict['username']['type'] = 'telegram' + mess_dict['username']['subtype'] = subtype + mess_dict['username']['id'] = user + else: + mess_dict['username']['id'] = 'UNKNOWN' + + if not mess_datetime: + obj_mess_id = self._get_message_timestamp(obj_global_id) + mess_datetime = datetime.fromtimestamp(obj_mess_id) + mess_dict['date'] = mess_datetime.isoformat(' ') + mess_dict['hour'] = mess_datetime.strftime('%H:%M:%S') + return mess_dict + + + def get_messages(self, start=0, page=1, nb=500): # TODO limit nb returned, # TODO add replies + start = 0 + stop = -1 + # r_object.delete(f'messages:{self.type}:{self.subtype}:{self.id}') + + # TODO chat without username ???? -> chat ID ???? + + messages = {} + curr_date = None + for message in self._get_messages(): + date = datetime.fromtimestamp(message[1]) + date_day = date.strftime('%Y/%m/%d') + if date_day != curr_date: + messages[date_day] = [] + curr_date = date_day + mess_dict = self.get_message_meta(message[0], parent=True, mess_datetime=date) + messages[date_day].append(mess_dict) + return messages + + # Zset with ID ??? id -> item id ??? multiple id == media + text + # id -> media id + # How do we handle reply/thread ??? -> separate with new chats name/id ZSET ??? + # Handle media ??? + + # list of message id -> obj_id + # list of obj_id -> + # abuse parent children ??? + + # def add(self, timestamp, obj_id, mess_id=0, username=None, user_id=None): + # date = # TODO get date from object + # self.update_daterange(date) + # update_obj_date(date, self.type, self.subtype) + # + # + # # daily + # r_object.hincrby(f'{self.type}:{self.subtype}:{date}', self.id, 1) + # # all subtypes + # r_object.zincrby(f'{self.type}_all:{self.subtype}', 1, self.id) + # + # ####################################################################### + # ####################################################################### + # + # # Correlations + # self.add_correlation('item', '', item_id) + # # domain + # if is_crawled(item_id): + # domain = get_item_domain(item_id) + # self.add_correlation('domain', '', domain) + + # TODO kvrocks exception if key don't exists + def get_obj_by_message_id(self, mess_id): + return r_object.hget(f'messages:ids:{self.type}:{self.subtype}:{self.id}', mess_id) + + # importer -> use cache for previous reply SET to_add_id: previously_imported : expire SET key -> 30 mn + def add_message(self, obj_global_id, timestamp, mess_id, reply_id=None): + r_object.hset(f'messages:ids:{self.type}:{self.subtype}:{self.id}', mess_id, obj_global_id) + r_object.zadd(f'messages:{self.type}:{self.subtype}:{self.id}', {obj_global_id: timestamp}) + + if reply_id: + reply_obj = self.get_obj_by_message_id(reply_id) + if reply_obj: + self.add_obj_children(reply_obj, obj_global_id) + else: + self.add_message_cached_reply(reply_id, mess_id) + + # ADD cached replies + for reply_obj in self.get_cached_message_reply(mess_id): + self.add_obj_children(obj_global_id, reply_obj) + + def _get_message_cached_reply(self, message_id): + return r_cache.smembers(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{message_id}') + + def get_cached_message_reply(self, message_id): + objs_global_id = [] + for mess_id in self._get_message_cached_reply(message_id): + obj_global_id = self.get_obj_by_message_id(mess_id) + if obj_global_id: + objs_global_id.append(obj_global_id) + return objs_global_id + + def add_message_cached_reply(self, reply_to_id, message_id): + r_cache.sadd(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_to_id}', message_id) + r_cache.expire(f'messages:ids:{self.type}:{self.subtype}:{self.id}:{reply_to_id}', 600) + + # TODO nb replies = nb son ???? what if it create a onion item ??? -> need source filtering + + +# TODO factorize +def get_all_subtypes(): + return ail_core.get_object_all_subtypes('chat') + +def get_all(): + objs = {} + for subtype in get_all_subtypes(): + objs[subtype] = get_all_by_subtype(subtype) + return objs + +def get_all_by_subtype(subtype): + return get_all_id('chat', subtype) + +# # TODO FILTER NAME + Key + mail +# def sanitize_username_name_to_search(name_to_search, subtype): # TODO FILTER NAME +# +# return name_to_search +# +# def search_usernames_by_name(name_to_search, subtype, r_pos=False): +# usernames = {} +# # for subtype in subtypes: +# r_name = sanitize_username_name_to_search(name_to_search, subtype) +# if not name_to_search or isinstance(r_name, dict): +# # break +# return usernames +# r_name = re.compile(r_name) +# for user_name in get_all_usernames_by_subtype(subtype): +# res = re.search(r_name, user_name) +# if res: +# usernames[user_name] = {} +# if r_pos: +# usernames[user_name]['hl-start'] = res.start() +# usernames[user_name]['hl-end'] = res.end() +# return usernames + + +if __name__ == '__main__': + chat = Chat('test', 'telegram') + r = chat.get_messages() + print(r) diff --git a/bin/lib/objects/Items.py b/bin/lib/objects/Items.py index 03c6f2cd..c2edbb40 100755 --- a/bin/lib/objects/Items.py +++ b/bin/lib/objects/Items.py @@ -288,6 +288,8 @@ class Item(AbstractObject): meta['mimetype'] = self.get_mimetype(content=content) if 'investigations' in options: meta['investigations'] = self.get_investigations() + if 'link' in options: + meta['link'] = self.get_link(flask_context=True) # meta['encoding'] = None return meta diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py new file mode 100755 index 00000000..98cc838f --- /dev/null +++ b/bin/lib/objects/Messages.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import re +import sys +import cld3 +import html2text + +from datetime import datetime + +from pymisp import MISPObject + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ail_core import get_ail_uuid +from lib.objects.abstract_object import AbstractObject +from lib.ConfigLoader import ConfigLoader +from lib.data_retention_engine import update_obj_date, get_obj_date_first +# TODO Set all messages ??? + + +from flask import url_for + +config_loader = ConfigLoader() +r_cache = config_loader.get_redis_conn("Redis_Cache") +r_object = config_loader.get_db_conn("Kvrocks_Objects") +r_content = config_loader.get_db_conn("Kvrocks_Content") +baseurl = config_loader.get_config_str("Notifications", "ail_domain") +config_loader = None + + +# TODO SAVE OR EXTRACT MESSAGE SOURCE FOR ICON ????????? +# TODO iterate on all objects +# TODO also add support for small objects ???? + +# CAN Message exists without CHAT -> no convert it to object + +# ID: source:chat_id:message_id ???? +# +# /!\ handle null chat and message id -> chat = uuid and message = timestamp ??? + + +class Message(AbstractObject): + """ + AIL Message Object. (strings) + """ + + def __init__(self, id): # TODO subtype or use source ???? + super(Message, self).__init__('message', id) # message::< telegram/1692189934.380827/ChatID_MessageID > + + def exists(self): + if self.subtype is None: + return r_object.exists(f'meta:{self.type}:{self.id}') + else: + return r_object.exists(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}') + + def get_source(self): + """ + Returns source/feeder name + """ + l_source = self.id.split('/')[:-4] + return os.path.join(*l_source) + + def get_basename(self): + return os.path.basename(self.id) + + def get_content(self, r_type='str'): # TODO ADD cache # TODO Compress content ??????? + """ + Returns content + """ + content = self._get_field('content') + if r_type == 'str': + return content + elif r_type == 'bytes': + return content.encode() + + def get_date(self): + timestamp = self.get_timestamp() + return datetime.fromtimestamp(timestamp).strftime('%Y%m%d') + + def get_timestamp(self): + dirs = self.id.split('/') + return dirs[-2] + + def get_message_id(self): # TODO optimize + message_id = self.get_basename().rsplit('_', 1)[1] + # if message_id.endswith('.gz'): + # message_id = message_id[:-3] + return message_id + + def get_chat_id(self): # TODO optimize + chat_id = self.get_basename().rsplit('_', 1)[0] + # if chat_id.endswith('.gz'): + # chat_id = chat_id[:-3] + return chat_id + + # Update value on import + # reply to -> parent ? + # reply/comment - > children ? + # nb views + # reactions + # nb fowards + # room ??? + # message from channel ??? + # message media + + def get_translation(self): # TODO support multiple translated languages ????? + """ + Returns translated content + """ + return self._get_field('translated') # TODO multiples translation ... -> use set + + def _set_translation(self, translation): + """ + Set translated content + """ + return self._set_field('translated', translation) # translation by hash ??? -> avoid translating multiple time + + def get_html2text_content(self, content=None, ignore_links=False): + if not content: + content = self.get_content() + h = html2text.HTML2Text() + h.ignore_links = ignore_links + h.ignore_images = ignore_links + return h.handle(content) + + # def get_ail_2_ail_payload(self): + # payload = {'raw': self.get_gzip_content(b64=True)} + # return payload + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('correlation.show_correlation', type=self.type, id=self.id) + else: + url = f'{baseurl}/correlation/show?type={self.type}&id={self.id}' + return url + + def get_svg_icon(self): + return {'style': 'fas', 'icon': 'fa-comment-dots', 'color': '#4dffff', 'radius': 5} + + def get_misp_object(self): # TODO + obj = MISPObject('instant-message', standalone=True) + obj_date = self.get_date() + if obj_date: + obj.first_seen = obj_date + else: + self.logger.warning( + f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={obj_date}') + + # obj_attrs = [obj.add_attribute('first-seen', value=obj_date), + # obj.add_attribute('raw-data', value=self.id, data=self.get_raw_content()), + # obj.add_attribute('sensor', value=get_ail_uuid())] + obj_attrs = [] + for obj_attr in obj_attrs: + for tag in self.get_tags(): + obj_attr.add_tag(tag) + return obj + + # def get_url(self): + # return r_object.hget(f'meta:item::{self.id}', 'url') + + # options: set of optional meta fields + def get_meta(self, options=None): + """ + :type options: set + """ + if options is None: + options = set() + meta = self.get_default_meta(tags=True) + meta['date'] = self.get_date() # TODO replace me by timestamp ?????? + meta['source'] = self.get_source() + # optional meta fields + if 'content' in options: + meta['content'] = self.get_content() + if 'parent' in options: + meta['parent'] = self.get_parent() + if 'investigations' in options: + meta['investigations'] = self.get_investigations() + if 'link' in options: + meta['link'] = self.get_link(flask_context=True) + + # meta['encoding'] = None + return meta + + def _languages_cleaner(self, content=None): + if not content: + content = self.get_content() + # REMOVE URLS + regex = r'\b(?:http://|https://)?(?:[a-zA-Z\d-]{,63}(?:\.[a-zA-Z\d-]{,63})+)(?:\:[0-9]+)*(?:/(?:$|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*\b' + url_regex = re.compile(regex) + urls = url_regex.findall(content) + urls = sorted(urls, key=len, reverse=True) + for url in urls: + content = content.replace(url, '') + # REMOVE PGP Blocks + regex_pgp_public_blocs = r'-----BEGIN PGP PUBLIC KEY BLOCK-----[\s\S]+?-----END PGP PUBLIC KEY BLOCK-----' + regex_pgp_signature = r'-----BEGIN PGP SIGNATURE-----[\s\S]+?-----END PGP SIGNATURE-----' + regex_pgp_message = r'-----BEGIN PGP MESSAGE-----[\s\S]+?-----END PGP MESSAGE-----' + re.compile(regex_pgp_public_blocs) + re.compile(regex_pgp_signature) + re.compile(regex_pgp_message) + res = re.findall(regex_pgp_public_blocs, content) + for it in res: + content = content.replace(it, '') + res = re.findall(regex_pgp_signature, content) + for it in res: + content = content.replace(it, '') + res = re.findall(regex_pgp_message, content) + for it in res: + content = content.replace(it, '') + return content + + def detect_languages(self, min_len=600, num_langs=3, min_proportion=0.2, min_probability=0.7): + languages = [] + ## CLEAN CONTENT ## + content = self.get_html2text_content(ignore_links=True) + content = self._languages_cleaner(content=content) + # REMOVE USELESS SPACE + content = ' '.join(content.split()) + # - CLEAN CONTENT - # + if len(content) >= min_len: + for lang in cld3.get_frequent_languages(content, num_langs=num_langs): + if lang.proportion >= min_proportion and lang.probability >= min_probability and lang.is_reliable: + languages.append(lang) + return languages + + # def translate(self, content=None): # TODO translation plugin + # # TODO get text language + # if not content: + # content = self.get_content() + # translated = argostranslate.translate.translate(content, 'ru', 'en') + # # Save translation + # self._set_translation(translated) + # return translated + + def create(self, content, translation, tags): + self._set_field('content', content) + r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content) + if translation: + self._set_translation(translation) + for tag in tags: + self.add_tag(tag) + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + pass + +def create_obj_id(source, chat_id, message_id, timestamp): + return f'{source}/{timestamp}/{chat_id}_{message_id}' + +# TODO Check if already exists +# def create(source, chat_id, message_id, timestamp, content, tags=[]): +def create(obj_id, content, translation=None, tags=[]): + message = Message(obj_id) + if not message.exists(): + message.create(content, translation, tags) + return message + + +# TODO Encode translation + + +if __name__ == '__main__': + r = 'test' + print(r) diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py new file mode 100755 index 00000000..0355806e --- /dev/null +++ b/bin/lib/objects/UsersAccount.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import re + +from flask import url_for +from pymisp import MISPObject + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import ail_core +from lib.ConfigLoader import ConfigLoader +from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id + +config_loader = ConfigLoader() +baseurl = config_loader.get_config_str("Notifications", "ail_domain") +config_loader = None + + +################################################################################ +################################################################################ +################################################################################ + +class UserAccount(AbstractSubtypeObject): + """ + AIL User Object. (strings) + """ + + def __init__(self, id, subtype): + super(UserAccount, self).__init__('user-account', id, subtype) + + # def get_ail_2_ail_payload(self): + # payload = {'raw': self.get_gzip_content(b64=True), + # 'compress': 'gzip'} + # return payload + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + # # TODO: + pass + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('correlation.show_correlation', type=self.type, subtype=self.subtype, id=self.id) + else: + url = f'{baseurl}/correlation/show?type={self.type}&subtype={self.subtype}&id={self.id}' + return url + + def get_svg_icon(self): # TODO change icon/color + if self.subtype == 'telegram': + style = 'fab' + icon = '\uf2c6' + elif self.subtype == 'twitter': + style = 'fab' + icon = '\uf099' + else: + style = 'fas' + icon = '\uf007' + return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5} + + def get_first_name(self): + return self._get_field('firstname') + + def get_last_name(self): + return self._get_field('lastname') + + def get_phone(self): + return self._get_field('phone') + + def set_first_name(self, firstname): + return self._set_field('firstname', firstname) + + def set_last_name(self, lastname): + return self._set_field('lastname', lastname) + + def set_phone(self, phone): + return self._set_field('phone', phone) + + # TODO REWRITE ADD FUNCTION + + def get_username(self): + return '' + + def get_usernames(self): + usernames = [] + correl = self.get_correlation('username') + for partial_id in correl.get('username', []): + usernames.append(f'username:{partial_id}') + return usernames + + def get_meta(self, options=set()): + meta = self._get_meta(options=options) + meta['id'] = self.id + meta['subtype'] = self.subtype + meta['tags'] = self.get_tags(r_list=True) + if 'username' in options: + meta['username'] = self.get_username() + if 'usernames' in options: + meta['usernames'] = self.get_usernames() + return meta + + def get_misp_object(self): + obj_attrs = [] + if self.subtype == 'telegram': + obj = MISPObject('telegram-account', standalone=True) + obj_attrs.append(obj.add_attribute('username', value=self.id)) + + elif self.subtype == 'twitter': + obj = MISPObject('twitter-account', standalone=True) + obj_attrs.append(obj.add_attribute('name', value=self.id)) + + else: + obj = MISPObject('user-account', standalone=True) + obj_attrs.append(obj.add_attribute('username', value=self.id)) + + first_seen = self.get_first_seen() + last_seen = self.get_last_seen() + if first_seen: + obj.first_seen = first_seen + if last_seen: + obj.last_seen = last_seen + if not first_seen or not last_seen: + self.logger.warning( + f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={first_seen}, last={last_seen}') + + for obj_attr in obj_attrs: + for tag in self.get_tags(): + obj_attr.add_tag(tag) + return obj + +def get_user_by_username(): + pass + +def get_all_subtypes(): + return ail_core.get_object_all_subtypes('user-account') + +def get_all(): + users = {} + for subtype in get_all_subtypes(): + users[subtype] = get_all_by_subtype(subtype) + return users + +def get_all_by_subtype(subtype): + return get_all_id('user-account', subtype) + + +# if __name__ == '__main__': +# name_to_search = 'co' +# subtype = 'telegram' +# print(search_usernames_by_name(name_to_search, subtype)) diff --git a/bin/lib/objects/abstract_daterange_object.py b/bin/lib/objects/abstract_daterange_object.py index 5ec103d0..98aa49c2 100755 --- a/bin/lib/objects/abstract_daterange_object.py +++ b/bin/lib/objects/abstract_daterange_object.py @@ -45,10 +45,10 @@ class AbstractDaterangeObject(AbstractObject, ABC): def exists(self): return r_object.exists(f'meta:{self.type}:{self.id}') - def _get_field(self, field): + def _get_field(self, field): # TODO remove me (NEW in abstract) return r_object.hget(f'meta:{self.type}:{self.id}', field) - def _set_field(self, field, value): + def _set_field(self, field, value): # TODO remove me (NEW in abstract) return r_object.hset(f'meta:{self.type}:{self.id}', field, value) def get_first_seen(self, r_int=False): diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index 2423a294..59a7e968 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -20,6 +20,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from lib import ail_logger from lib import Tag +from lib.ConfigLoader import ConfigLoader from lib import Duplicate from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations @@ -27,6 +28,11 @@ from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers logging.config.dictConfig(ail_logger.get_config(name='ail')) +config_loader = ConfigLoader() +# r_cache = config_loader.get_redis_conn("Redis_Cache") +r_object = config_loader.get_db_conn("Kvrocks_Objects") +config_loader = None + class AbstractObject(ABC): """ Abstract Object @@ -67,6 +73,18 @@ class AbstractObject(ABC): dict_meta['tags'] = self.get_tags() return dict_meta + def _get_field(self, field): + if self.subtype is None: + return r_object.hget(f'meta:{self.type}:{self.id}', field) + else: + return r_object.hget(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', field) + + def _set_field(self, field, value): + if self.subtype is None: + return r_object.hset(f'meta:{self.type}:{self.id}', field, value) + else: + return r_object.hset(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', field, value) + ## Tags ## def get_tags(self, r_list=False): tags = Tag.get_object_tags(self.type, self.id, self.get_subtype(r_str=True)) @@ -198,6 +216,8 @@ class AbstractObject(ABC): else: return [] + ## Correlation ## + def _get_external_correlation(self, req_type, req_subtype, req_id, obj_type): """ Get object correlation @@ -253,3 +273,39 @@ class AbstractObject(ABC): Get object correlations """ delete_obj_correlation(self.type, self.subtype, self.id, type2, subtype2, id2) + + ## -Correlation- ## + + ## Parent ## + + def is_parent(self): + return r_object.exists(f'child:{self.type}:{self.get_subtype(r_str=True)}:{self.id}') + + def is_children(self): + return r_object.hexists(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', 'parent') + + def get_parent(self): + return r_object.hget(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', 'parent') + + def get_children(self): + return r_object.smembers(f'child:{self.type}:{self.get_subtype(r_str=True)}:{self.id}') + + def set_parent(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO ###################### + if not obj_global_id: + if obj_subtype is None: + obj_subtype = '' + obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}' + r_object.hset(f'meta:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', 'parent', obj_global_id) + + def add_children(self, obj_type=None, obj_subtype=None, obj_id=None, obj_global_id=None): # TODO ###################### + if not obj_global_id: + if obj_subtype is None: + obj_subtype = '' + obj_global_id = f'{obj_type}:{obj_subtype}:{obj_id}' + r_object.sadd(f'child:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', obj_global_id) + + def add_obj_children(self, parent_global_id, son_global_id): + r_object.sadd(f'child:{parent_global_id}', son_global_id) + r_object.hset(f'meta:{son_global_id}', 'parent', parent_global_id) + + ## Parent ## diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index f12708fb..cd2f7225 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -13,6 +13,7 @@ from lib import correlations_engine from lib import btc_ail from lib import Tag +from lib.objects import Chats from lib.objects import CryptoCurrencies from lib.objects import CookiesNames from lib.objects.Cves import Cve @@ -55,6 +56,8 @@ def get_object(obj_type, subtype, id): return Domain(id) elif obj_type == 'decoded': return Decoded(id) + elif obj_type == 'chat': + return Chats.Chat(id, subtype) elif obj_type == 'cookie-name': return CookiesNames.CookieName(id) elif obj_type == 'cve': diff --git a/bin/lib/timeline_engine.py b/bin/lib/timeline_engine.py new file mode 100755 index 00000000..405e7a50 --- /dev/null +++ b/bin/lib/timeline_engine.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys + +from uuid import uuid4 + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.ConfigLoader import ConfigLoader + +config_loader = ConfigLoader() +r_meta = config_loader.get_db_conn("Kvrocks_Timeline") +config_loader = None + +# CORRELATION_TYPES_BY_OBJ = { +# "chat": ["item", "username"], # item ??? +# "cookie-name": ["domain"], +# "cryptocurrency": ["domain", "item"], +# "cve": ["domain", "item"], +# "decoded": ["domain", "item"], +# "domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "etag", "favicon", "hhhash", "item", "pgp", "title", "screenshot", "username"], +# "etag": ["domain"], +# "favicon": ["domain", "item"], +# "hhhash": ["domain"], +# "item": ["chat", "cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], +# "pgp": ["domain", "item"], +# "screenshot": ["domain", "item"], +# "title": ["domain", "item"], +# "username": ["chat", "domain", "item"], +# } +# +# def get_obj_correl_types(obj_type): +# return CORRELATION_TYPES_BY_OBJ.get(obj_type) + +# def sanityze_obj_correl_types(obj_type, correl_types): +# obj_correl_types = get_obj_correl_types(obj_type) +# if correl_types: +# correl_types = set(correl_types).intersection(obj_correl_types) +# if not correl_types: +# correl_types = obj_correl_types +# if not correl_types: +# return [] +# return correl_types + +# TODO rename all function + add missing parameters + +def get_bloc_obj_global_id(bloc): + return r_meta.hget('hset:key', bloc) + +def set_bloc_obj_global_id(bloc, global_id): + return r_meta.hset('hset:key', bloc, global_id) + +def get_bloc_timestamp(bloc, position): + return r_meta.zscore('key', f'{position}:{bloc}') + +def add_bloc(global_id, timestamp, end=None): + if end: + timestamp_end = end + else: + timestamp_end = timestamp + new_bloc = str(uuid4()) + r_meta.zadd('key', {f'start:{new_bloc}': timestamp, f'end:{new_bloc}': timestamp_end}) + set_bloc_obj_global_id(new_bloc, global_id) + return new_bloc + +def _update_bloc(bloc, position, timestamp): + r_meta.zadd('key', {f'{position}:{bloc}': timestamp}) + +# score = timestamp +def get_nearest_bloc_inf(timestamp): + return r_meta.zrevrangebyscore('key', timestamp, 0, num=1) + +def get_nearest_bloc_sup(timestamp): + return r_meta.zrangebyscore('key', timestamp, 0, num=1) + +####################################################################################### + +def add_timestamp(timestamp, obj_global_id): + inf = get_nearest_bloc_inf(timestamp) + sup = get_nearest_bloc_sup(timestamp) + if not inf and not sup: + # create new bloc + new_bloc = add_bloc(obj_global_id, timestamp) + return new_bloc + # timestamp < first_seen + elif not inf: + sup_pos, sup_id = inf.split(':') + sup_obj = get_bloc_obj_global_id(sup_pos) + if sup_obj == obj_global_id: + _update_bloc(sup_id, 'start', timestamp) + # create new bloc + else: + new_bloc = add_bloc(obj_global_id, timestamp) + return new_bloc + + # timestamp > first_seen + elif not sup: + inf_pos, inf_id = inf.split(':') + inf_obj = get_bloc_obj_global_id(inf_id) + if inf_obj == obj_global_id: + _update_bloc(inf_id, 'end', timestamp) + # create new bloc + else: + new_bloc = add_bloc(obj_global_id, timestamp) + return new_bloc + + else: + inf_pos, inf_id = inf.split(':') + sup_pos, sup_id = inf.split(':') + inf_obj = get_bloc_obj_global_id(inf_id) + + if inf_id == sup_id: + # reduce bloc + create two new bloc + if obj_global_id != inf_obj: + # get end timestamp + sup_timestamp = get_bloc_timestamp(sup_id, 'end') + # reduce original bloc + _update_bloc(inf_id, 'end', timestamp - 1) + # Insert new bloc + new_bloc = add_bloc(obj_global_id, timestamp) + # Recreate end of the first bloc by a new bloc + add_bloc(inf_obj, timestamp + 1, end=sup_timestamp) + return new_bloc + + # timestamp in existing bloc + else: + return inf_id + + # different blocs: expend sup/inf bloc or create a new bloc if + elif inf_pos == 'end' and sup_pos == 'start': + # Extend inf bloc + if obj_global_id == inf_obj: + _update_bloc(inf_id, 'end', timestamp) + return inf_id + + sup_obj = get_bloc_obj_global_id(sup_pos) + # Extend sup bloc + if obj_global_id == sup_obj: + _update_bloc(sup_id, 'start', timestamp) + return sup_id + + # create new bloc + new_bloc = add_bloc(obj_global_id, timestamp) + return new_bloc + + # inf_pos == 'start' and sup_pos == 'end' + # else raise error ??? + + + + + + diff --git a/configs/6383.conf b/configs/6383.conf index c730003c..a06d4e69 100644 --- a/configs/6383.conf +++ b/configs/6383.conf @@ -663,6 +663,7 @@ namespace.crawl ail_crawlers namespace.db ail_datas namespace.dup ail_dups namespace.obj ail_objs +namespace.tl ail_tls namespace.stat ail_stats namespace.tag ail_tags namespace.track ail_trackers diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index 3278033f..8185b8f7 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -190,6 +190,11 @@ host = localhost port = 6383 password = ail_objs +[Kvrocks_Timeline] +host = localhost +port = 6383 +password = ail_tls + [Kvrocks_Stats] host = localhost port = 6383 diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index e6a99350..c330443b 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -50,6 +50,7 @@ from blueprints.objects_title import objects_title from blueprints.objects_cookie_name import objects_cookie_name from blueprints.objects_etag import objects_etag from blueprints.objects_hhhash import objects_hhhash +from blueprints.objects_chat import objects_chat Flask_dir = os.environ['AIL_FLASK'] @@ -107,6 +108,7 @@ app.register_blueprint(objects_title, url_prefix=baseUrl) app.register_blueprint(objects_cookie_name, url_prefix=baseUrl) app.register_blueprint(objects_etag, url_prefix=baseUrl) app.register_blueprint(objects_hhhash, url_prefix=baseUrl) +app.register_blueprint(objects_chat, url_prefix=baseUrl) # ========= =========# diff --git a/var/www/blueprints/objects_chat.py b/var/www/blueprints/objects_chat.py new file mode 100644 index 00000000..8a1db11f --- /dev/null +++ b/var/www/blueprints/objects_chat.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... +''' + +import os +import sys +import json + +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file +from flask_login import login_required, current_user + +# Import Role_Manager +from Role_Manager import login_admin, login_analyst, login_read_only + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import ail_core +from lib.objects import abstract_subtype_object +from lib.objects import ail_objects +from lib.objects import Chats +from packages import Date + +# ============ BLUEPRINT ============ +objects_chat = Blueprint('objects_chat', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/chat')) + +# ============ VARIABLES ============ +bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] + +def create_json_response(data, status_code): + return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code + +# ============ FUNCTIONS ============ + +# ============= ROUTES ============== + + +@objects_chat.route("/objects/chat/messages", methods=['GET']) +@login_required +@login_read_only +def objects_dashboard_chat(): + chat = request.args.get('id') + subtype = request.args.get('subtype') + chat = Chats.Chat(chat, subtype) + if chat.exists(): + messages = chat.get_messages() + meta = chat.get_meta({'icon'}) + print(meta) + return render_template('ChatMessages.html', meta=meta, messages=messages, bootstrap_label=bootstrap_label) + else: + return abort(404) + + + diff --git a/var/www/blueprints/objects_subtypes.py b/var/www/blueprints/objects_subtypes.py index dc97ffa8..a41066a4 100644 --- a/var/www/blueprints/objects_subtypes.py +++ b/var/www/blueprints/objects_subtypes.py @@ -91,6 +91,12 @@ def subtypes_objects_dashboard(obj_type, f_request): # ============= ROUTES ============== +@objects_subtypes.route("/objects/chats", methods=['GET']) +@login_required +@login_read_only +def objects_dashboard_chat(): + return subtypes_objects_dashboard('chat', request) + @objects_subtypes.route("/objects/cryptocurrencies", methods=['GET']) @login_required @login_read_only diff --git a/var/www/templates/objects/chat/ChatMessages.html b/var/www/templates/objects/chat/ChatMessages.html new file mode 100644 index 00000000..b89a447a --- /dev/null +++ b/var/www/templates/objects/chat/ChatMessages.html @@ -0,0 +1,190 @@ + + + + + Chat Messages - AIL + + + + + + +{# #} + + + + + + + +{# + #} + + + + + + + + {% include 'nav_bar.html' %} + +
    +
    + + {% include 'sidebars/sidebar_objects.html' %} + +
    + +
    +
    +

    {{ meta["id"] }} :

    +
      +
    • +
      +
      + + + + + + + + + + + + + + + + + +
      Object subtypeFirst seenLast seenNb seen
      + + + + {{ meta["icon"]["icon"] }} + + + {{ meta["subtype"] }} + {{ meta['first_seen'] }}{{ meta['last_seen'] }}{{ meta['nb_seen'] }}
      +
      +
      +
      +
      +
      +
      • +
    • +
      +
      + Tags: + {% for tag in meta['tags'] %} + + {% endfor %} + +
      +
      • +
    + + {% with obj_type='chat', obj_id=meta['id'], obj_subtype=meta['subtype'] %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + +
    +
    + +
    +
    + + {% for date in messages %} +

    {{ date }}

    + {% for mess in messages[date] %} + +
    +
    + {{ mess['username']['id'] }} +
    {{ mess['hour'] }}
    +
    +
    +
    {{ mess['username']['id'] }}
    + {% if mess['reply_to'] %} +
    +
    {{ mess['reply_to']['username']['id'] }}
    + 
    {{ mess['reply_to']['content'] }}
    + {% for tag in mess['reply_to']['tags'] %} + {{ tag }} + {% endfor %} +
    {{ mess['reply_to']['date'] }}
    +{#
    #} +{# #} +{# #} +{#
    #} +
    + {% endif %} + 
    {{ mess['content'] }}
    + {% for tag in mess['tags'] %} + {{ tag }} + {% endfor %} +
    + + +
    +
    +
    + + {% endfor %} +
    + {% endfor %} + +
    +
    + +
    + +
    +
    + + + + + + From 843b2d3134e96d8e11cdaaf72044961ee391d65d Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 23 Aug 2023 16:13:20 +0200 Subject: [PATCH 44/45] fix: correlations --- bin/importer/feeders/Telegram.py | 7 +++++-- bin/lib/objects/Chats.py | 12 +++++++++--- bin/lib/objects/Messages.py | 2 +- bin/lib/objects/UsersAccount.py | 4 +--- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/bin/importer/feeders/Telegram.py b/bin/importer/feeders/Telegram.py index 52eb0a75..2900a46d 100755 --- a/bin/importer/feeders/Telegram.py +++ b/bin/importer/feeders/Telegram.py @@ -79,7 +79,7 @@ class TelegramFeeder(DefaultFeeder): if meta.get('chat'): chat = Chat(meta['chat']['id'], 'telegram') - if meta['chat'].get('username'): # TODO USE ID AND SAVE USERNAME + if meta['chat'].get('username'): # SAVE USERNAME chat_username = meta['chat']['username'] # Chat---Message @@ -99,7 +99,7 @@ class TelegramFeeder(DefaultFeeder): chat = None # message sender - if meta.get('sender'): # TODO handle message channel forward + if meta.get('sender'): # TODO handle message channel forward - check if is user user_id = meta['sender']['id'] user_account = UsersAccount.UserAccount(user_id, 'telegram') # UserAccount---Message @@ -117,10 +117,13 @@ class TelegramFeeder(DefaultFeeder): if meta['sender'].get('username'): username = Username(meta['sender']['username'], 'telegram') user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id) + # TODO Update user_account<--->username timeline # Username---Message username.add(date, self.item_id) # TODO #################################################################### + if chat: + # Chat---Username chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id) # if meta.get('fwd_from'): diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index 438acf51..a3d1721c 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -117,9 +117,15 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ????? def get_name(self): # get username ???? pass - # return username correlation - def get_users(self): # get participants ??? -> passive users ??? - pass + # users that send at least a message else participants/spectator + # correlation created by messages + def get_users(self): + users = set() + accounts = self.get_correlation('user-account').get('user-account', []) + for account in accounts: + users.add(account[1:]) + return users + # def get_last_message_id(self): # diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index 98cc838f..302f0d0a 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -91,7 +91,7 @@ class Message(AbstractObject): # message_id = message_id[:-3] return message_id - def get_chat_id(self): # TODO optimize + def get_chat_id(self): # TODO optimize -> use me to tag Chat chat_id = self.get_basename().rsplit('_', 1)[0] # if chat_id.endswith('.gz'): # chat_id = chat_id[:-3] diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py index 0355806e..f4f71d05 100755 --- a/bin/lib/objects/UsersAccount.py +++ b/bin/lib/objects/UsersAccount.py @@ -87,9 +87,7 @@ class UserAccount(AbstractSubtypeObject): def get_usernames(self): usernames = [] - correl = self.get_correlation('username') - for partial_id in correl.get('username', []): - usernames.append(f'username:{partial_id}') + # TODO TIMELINE return usernames def get_meta(self, options=set()): From b32f1102851c9be585f26867a2db97b72fba1348 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 28 Aug 2023 16:29:38 +0200 Subject: [PATCH 45/45] chg: [chat + user-account] correlations + usernames timeline --- bin/crawlers/Crawler.py | 6 +- bin/importer/feeders/Jabber.py | 12 +- bin/importer/feeders/Telegram.py | 46 ++-- bin/importer/feeders/Twitter.py | 8 +- bin/lib/correlations_engine.py | 16 +- bin/lib/crawlers.py | 6 +- bin/lib/objects/Chats.py | 35 ++- bin/lib/objects/Messages.py | 19 +- bin/lib/objects/UsersAccount.py | 15 +- bin/lib/objects/abstract_object.py | 4 +- bin/lib/objects/abstract_subtype_object.py | 18 +- bin/lib/objects/ail_objects.py | 6 + bin/lib/timeline_engine.py | 231 +++++++++++------- bin/modules/Cryptocurrencies.py | 2 +- bin/modules/PgpDump.py | 6 +- bin/modules/Telegram.py | 4 +- .../templates/objects/chat/ChatMessages.html | 18 +- 17 files changed, 279 insertions(+), 173 deletions(-) diff --git a/bin/crawlers/Crawler.py b/bin/crawlers/Crawler.py index 7f2c3df9..d16ad9f7 100755 --- a/bin/crawlers/Crawler.py +++ b/bin/crawlers/Crawler.py @@ -282,7 +282,7 @@ class Crawler(AbstractModule): title_content = crawlers.extract_title_from_html(entries['html']) if title_content: title = Titles.create_title(title_content) - title.add(item.get_date(), item_id) + title.add(item.get_date(), item) # SCREENSHOT if self.screenshot: @@ -306,11 +306,11 @@ class Crawler(AbstractModule): for cookie_name in crawlers.extract_cookies_names_from_har(entries['har']): print(cookie_name) cookie = CookiesNames.create(cookie_name) - cookie.add(self.date.replace('/', ''), self.domain.id) + cookie.add(self.date.replace('/', ''), self.domain) for etag_content in crawlers.extract_etag_from_har(entries['har']): print(etag_content) etag = Etags.create(etag_content) - etag.add(self.date.replace('/', ''), self.domain.id) + etag.add(self.date.replace('/', ''), self.domain) crawlers.extract_hhhash(entries['har'], self.domain.id, self.date.replace('/', '')) # Next Children diff --git a/bin/importer/feeders/Jabber.py b/bin/importer/feeders/Jabber.py index 79d0950f..8c90adfd 100755 --- a/bin/importer/feeders/Jabber.py +++ b/bin/importer/feeders/Jabber.py @@ -17,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from importer.feeders.Default import DefaultFeeder from lib.objects.Usernames import Username -from lib import item_basic +from lib.objects.Items import Item class JabberFeeder(DefaultFeeder): @@ -36,7 +36,7 @@ class JabberFeeder(DefaultFeeder): self.item_id = f'{item_id}.gz' return self.item_id - def process_meta(self): + def process_meta(self): # TODO replace me by message """ Process JSON meta field. """ @@ -44,10 +44,12 @@ class JabberFeeder(DefaultFeeder): # item_basic.add_map_obj_id_item_id(jabber_id, item_id, 'jabber_id') ############################################## to = str(self.json_data['meta']['jabber:to']) fr = str(self.json_data['meta']['jabber:from']) - date = item_basic.get_item_date(item_id) + + item = Item(self.item_id) + date = item.get_date() user_to = Username(to, 'jabber') user_fr = Username(fr, 'jabber') - user_to.add(date, self.item_id) - user_fr.add(date, self.item_id) + user_to.add(date, item) + user_fr.add(date, item) return None diff --git a/bin/importer/feeders/Telegram.py b/bin/importer/feeders/Telegram.py index 2900a46d..2cc6a127 100755 --- a/bin/importer/feeders/Telegram.py +++ b/bin/importer/feeders/Telegram.py @@ -21,7 +21,6 @@ from lib.objects.Chats import Chat from lib.objects import Messages from lib.objects import UsersAccount from lib.objects.Usernames import Username -from lib import item_basic import base64 import io @@ -57,7 +56,7 @@ class TelegramFeeder(DefaultFeeder): # date = datetime.date.today().strftime("%Y/%m/%d") chat_id = str(self.json_data['meta']['chat']['id']) message_id = str(self.json_data['meta']['id']) - self.item_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) + self.item_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) # TODO rename self.item_id return self.item_id def process_meta(self): @@ -68,7 +67,7 @@ class TelegramFeeder(DefaultFeeder): meta = self.json_data['meta'] mess_id = self.json_data['meta']['id'] if meta.get('reply_to'): - reply_to_id = meta['reply_to'] + reply_to_id = meta['reply_to']['id'] else: reply_to_id = None @@ -76,25 +75,24 @@ class TelegramFeeder(DefaultFeeder): date = datetime.datetime.fromtimestamp(timestamp) date = date.strftime('%Y%m%d') + if self.json_data.get('translation'): + translation = self.json_data['translation'] + else: + translation = None + decoded = base64.standard_b64decode(self.json_data['data']) + content = gunzip_bytes_obj(decoded) + message = Messages.create(self.item_id, content, translation=translation) + if meta.get('chat'): chat = Chat(meta['chat']['id'], 'telegram') - if meta['chat'].get('username'): # SAVE USERNAME - chat_username = meta['chat']['username'] + if meta['chat'].get('username'): + chat_username = Username(meta['chat']['username'], 'telegram') + chat.update_username_timeline(chat_username.get_global_id(), timestamp) # Chat---Message - chat.add(date, self.item_id) # TODO modify to accept file objects - # message meta ????? who is the user if two user ???? - - if self.json_data.get('translation'): - translation = self.json_data['translation'] - else: - translation = None - decoded = base64.standard_b64decode(self.json_data['data']) - content = gunzip_bytes_obj(decoded) - Messages.create(self.item_id, content, translation=translation) - - chat.add_message(self.item_id, timestamp, mess_id, reply_id=reply_to_id) + chat.add(date) + chat.add_message(message.get_global_id(), timestamp, mess_id, reply_id=reply_to_id) else: chat = None @@ -103,7 +101,7 @@ class TelegramFeeder(DefaultFeeder): user_id = meta['sender']['id'] user_account = UsersAccount.UserAccount(user_id, 'telegram') # UserAccount---Message - user_account.add(date, self.item_id) + user_account.add(date, obj=message) # UserAccount---Chat user_account.add_correlation(chat.type, chat.get_subtype(r_str=True), chat.id) @@ -116,20 +114,22 @@ class TelegramFeeder(DefaultFeeder): if meta['sender'].get('username'): username = Username(meta['sender']['username'], 'telegram') + # TODO timeline or/and correlation ???? user_account.add_correlation(username.type, username.get_subtype(r_str=True), username.id) - # TODO Update user_account<--->username timeline + user_account.update_username_timeline(username.get_global_id(), timestamp) # Username---Message - username.add(date, self.item_id) # TODO #################################################################### + username.add(date) # TODO # correlation message ??? - if chat: - # Chat---Username - chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id) + # if chat: # TODO Chat---Username correlation ??? + # # Chat---Username + # chat.add_correlation(username.type, username.get_subtype(r_str=True), username.id) # if meta.get('fwd_from'): # if meta['fwd_from'].get('post_author') # user first name # TODO reply threads ???? + # message edit ???? return None diff --git a/bin/importer/feeders/Twitter.py b/bin/importer/feeders/Twitter.py index d5040c65..1c719e73 100755 --- a/bin/importer/feeders/Twitter.py +++ b/bin/importer/feeders/Twitter.py @@ -17,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from importer.feeders.Default import DefaultFeeder from lib.objects.Usernames import Username -from lib import item_basic +from lib.objects.Items import Item class TwitterFeeder(DefaultFeeder): @@ -40,9 +40,9 @@ class TwitterFeeder(DefaultFeeder): ''' # tweet_id = str(self.json_data['meta']['twitter:tweet_id']) # item_basic.add_map_obj_id_item_id(tweet_id, item_id, 'twitter_id') ############################################ - - date = item_basic.get_item_date(self.item_id) + item = Item(self.item_id) + date = item.get_date() user = str(self.json_data['meta']['twitter:id']) username = Username(user, 'twitter') - username.add(date, item_id) + username.add(date, item) return None diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index 94a06773..f7b13f61 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -41,20 +41,22 @@ config_loader = None ################################## CORRELATION_TYPES_BY_OBJ = { - "chat": ["item", "username"], # item ??? + "chat": ["user-account"], # message or direct correlation like cve, bitcoin, ... ??? "cookie-name": ["domain"], - "cryptocurrency": ["domain", "item"], - "cve": ["domain", "item"], - "decoded": ["domain", "item"], + "cryptocurrency": ["domain", "item", "message"], + "cve": ["domain", "item", "message"], + "decoded": ["domain", "item", "message"], "domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "etag", "favicon", "hhhash", "item", "pgp", "title", "screenshot", "username"], "etag": ["domain"], "favicon": ["domain", "item"], # TODO Decoded "hhhash": ["domain"], - "item": ["chat", "cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], - "pgp": ["domain", "item"], + "item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"], # chat ??? + "message": ["cve", "cryptocurrency", "decoded", "pgp", "user-account"], # chat ?? + "pgp": ["domain", "item", "message"], "screenshot": ["domain", "item"], "title": ["domain", "item"], - "username": ["chat", "domain", "item"], + "user-account": ["chat", "message"], + "username": ["domain", "item", "message"], # TODO chat-user/account } def get_obj_correl_types(obj_type): diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 3e61ed88..6387c76f 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -342,7 +342,7 @@ def _reprocess_all_hars_cookie_name(): for cookie_name in extract_cookies_names_from_har(get_har_content(har_id)): print(domain, date, cookie_name) cookie = CookiesNames.create(cookie_name) - cookie.add(date, domain) + cookie.add(date, Domain(domain)) def extract_etag_from_har(har): # TODO check response url etags = set() @@ -365,7 +365,7 @@ def _reprocess_all_hars_etag(): for etag_content in extract_etag_from_har(get_har_content(har_id)): print(domain, date, etag_content) etag = Etags.create(etag_content) - etag.add(date, domain) + etag.add(date, Domain(domain)) def extract_hhhash_by_id(har_id, domain, date): return extract_hhhash(get_har_content(har_id), domain, date) @@ -395,7 +395,7 @@ def extract_hhhash(har, domain, date): # ----- obj = HHHashs.create(hhhash_header, hhhash) - obj.add(date, domain) + obj.add(date, Domain(domain)) hhhashs.add(hhhash) urls.add(url) diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index a3d1721c..bb27413d 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -18,7 +18,7 @@ from lib.ConfigLoader import ConfigLoader from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id from lib.data_retention_engine import update_obj_date from lib.objects import ail_objects -from lib import item_basic +from lib.timeline_engine import Timeline from lib.correlations_engine import get_correlation_by_correl_type @@ -126,6 +126,18 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ????? users.add(account[1:]) return users + def _get_timeline_username(self): + return Timeline(self.get_global_id(), 'username') + + def get_username(self): + return self._get_timeline_username().get_last_obj_id() + + def get_usernames(self): + return self._get_timeline_username().get_objs_ids() + + def update_username_timeline(self, username_global_id, timestamp): + self._get_timeline_username().add_timestamp(timestamp, username_global_id) + # def get_last_message_id(self): # @@ -144,18 +156,21 @@ class Chat(AbstractSubtypeObject): # TODO # ID == username ????? def get_message_meta(self, obj_global_id, parent=True, mess_datetime=None): obj = ail_objects.get_obj_from_global_id(obj_global_id) - mess_dict = obj.get_meta(options={'content', 'link', 'parent'}) + mess_dict = obj.get_meta(options={'content', 'link', 'parent', 'user-account'}) if mess_dict.get('parent') and parent: mess_dict['reply_to'] = self.get_message_meta(mess_dict['parent'], parent=False) - mess_dict['username'] = {} - user = obj.get_correlation('username').get('username') - if user: - subtype, user = user.pop().split(':', 1) - mess_dict['username']['type'] = 'telegram' - mess_dict['username']['subtype'] = subtype - mess_dict['username']['id'] = user + if mess_dict.get('user-account'): + user_account = ail_objects.get_obj_from_global_id(mess_dict['user-account']) + mess_dict['user-account'] = {} + mess_dict['user-account']['type'] = user_account.get_type() + mess_dict['user-account']['subtype'] = user_account.get_subtype(r_str=True) + mess_dict['user-account']['id'] = user_account.get_id() + username = user_account.get_username() + if username: + username = ail_objects.get_obj_from_global_id(username).get_default_meta(link=False) + mess_dict['user-account']['username'] = username # TODO get username at the given timestamp ??? else: - mess_dict['username']['id'] = 'UNKNOWN' + mess_dict['user-account']['id'] = 'UNKNOWN' if not mess_datetime: obj_mess_id = self._get_message_timestamp(obj_global_id) diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index 302f0d0a..b724f854 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -27,7 +27,7 @@ from flask import url_for config_loader = ConfigLoader() r_cache = config_loader.get_redis_conn("Redis_Cache") r_object = config_loader.get_db_conn("Kvrocks_Objects") -r_content = config_loader.get_db_conn("Kvrocks_Content") +# r_content = config_loader.get_db_conn("Kvrocks_Content") baseurl = config_loader.get_config_str("Notifications", "ail_domain") config_loader = None @@ -61,7 +61,7 @@ class Message(AbstractObject): """ Returns source/feeder name """ - l_source = self.id.split('/')[:-4] + l_source = self.id.split('/')[:-2] return os.path.join(*l_source) def get_basename(self): @@ -79,7 +79,7 @@ class Message(AbstractObject): def get_date(self): timestamp = self.get_timestamp() - return datetime.fromtimestamp(timestamp).strftime('%Y%m%d') + return datetime.fromtimestamp(float(timestamp)).strftime('%Y%m%d') def get_timestamp(self): dirs = self.id.split('/') @@ -92,11 +92,16 @@ class Message(AbstractObject): return message_id def get_chat_id(self): # TODO optimize -> use me to tag Chat - chat_id = self.get_basename().rsplit('_', 1)[0] + chat_id = self.get_basename().rsplit('_', 1)[0] # if chat_id.endswith('.gz'): # chat_id = chat_id[:-3] return chat_id + def get_user_account(self): + user_account = self.get_correlation('user-account') + if user_account.get('user-account'): + return f'user-account:{user_account["user-account"].pop()}' + # Update value on import # reply to -> parent ? # reply/comment - > children ? @@ -139,7 +144,7 @@ class Message(AbstractObject): return url def get_svg_icon(self): - return {'style': 'fas', 'icon': 'fa-comment-dots', 'color': '#4dffff', 'radius': 5} + return {'style': 'fas', 'icon': '\uf4ad', 'color': '#4dffff', 'radius': 5} def get_misp_object(self): # TODO obj = MISPObject('instant-message', standalone=True) @@ -181,6 +186,8 @@ class Message(AbstractObject): meta['investigations'] = self.get_investigations() if 'link' in options: meta['link'] = self.get_link(flask_context=True) + if 'user-account' in options: + meta['user-account'] = self.get_user_account() # meta['encoding'] = None return meta @@ -238,7 +245,7 @@ class Message(AbstractObject): def create(self, content, translation, tags): self._set_field('content', content) - r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content) + # r_content.get(f'content:{self.type}:{self.get_subtype(r_str=True)}:{self.id}', content) if translation: self._set_translation(translation) for tag in tags: diff --git a/bin/lib/objects/UsersAccount.py b/bin/lib/objects/UsersAccount.py index f4f71d05..5bc94a9c 100755 --- a/bin/lib/objects/UsersAccount.py +++ b/bin/lib/objects/UsersAccount.py @@ -3,7 +3,7 @@ import os import sys -import re +# import re from flask import url_for from pymisp import MISPObject @@ -15,6 +15,7 @@ sys.path.append(os.environ['AIL_BIN']) from lib import ail_core from lib.ConfigLoader import ConfigLoader from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id +from lib.timeline_engine import Timeline config_loader = ConfigLoader() baseurl = config_loader.get_config_str("Notifications", "ail_domain") @@ -80,15 +81,17 @@ class UserAccount(AbstractSubtypeObject): def set_phone(self, phone): return self._set_field('phone', phone) - # TODO REWRITE ADD FUNCTION + def _get_timeline_username(self): + return Timeline(self.get_global_id(), 'username') def get_username(self): - return '' + return self._get_timeline_username().get_last_obj_id() def get_usernames(self): - usernames = [] - # TODO TIMELINE - return usernames + return self._get_timeline_username().get_objs_ids() + + def update_username_timeline(self, username_global_id, timestamp): + self._get_timeline_username().add_timestamp(timestamp, username_global_id) def get_meta(self, options=set()): meta = self._get_meta(options=options) diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py index 59a7e968..a3f25216 100755 --- a/bin/lib/objects/abstract_object.py +++ b/bin/lib/objects/abstract_object.py @@ -65,12 +65,14 @@ class AbstractObject(ABC): def get_global_id(self): return f'{self.get_type()}:{self.get_subtype(r_str=True)}:{self.get_id()}' - def get_default_meta(self, tags=False): + def get_default_meta(self, tags=False, link=False): dict_meta = {'id': self.get_id(), 'type': self.get_type(), 'subtype': self.get_subtype(r_str=True)} if tags: dict_meta['tags'] = self.get_tags() + if link: + dict_meta['link'] = self.get_link() return dict_meta def _get_field(self, field): diff --git a/bin/lib/objects/abstract_subtype_object.py b/bin/lib/objects/abstract_subtype_object.py index 82bb85f6..007f716b 100755 --- a/bin/lib/objects/abstract_subtype_object.py +++ b/bin/lib/objects/abstract_subtype_object.py @@ -151,7 +151,7 @@ class AbstractSubtypeObject(AbstractObject, ABC): # # - def add(self, date, item_id): + def add(self, date, obj=None): self.update_daterange(date) update_obj_date(date, self.type, self.subtype) # daily @@ -162,20 +162,22 @@ class AbstractSubtypeObject(AbstractObject, ABC): ####################################################################### ####################################################################### - # Correlations - self.add_correlation('item', '', item_id) - # domain - if is_crawled(item_id): - domain = get_item_domain(item_id) - self.add_correlation('domain', '', domain) + if obj: + # Correlations + self.add_correlation(obj.type, obj.get_subtype(r_str=True), obj.get_id()) + if obj.type == 'item': # TODO same for message->chat ??? + item_id = obj.get_id() + # domain + if is_crawled(item_id): + domain = get_item_domain(item_id) + self.add_correlation('domain', '', domain) # TODO:ADD objects + Stats def create(self, first_seen, last_seen): self.set_first_seen(first_seen) self.set_last_seen(last_seen) - def _delete(self): pass diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index cd2f7225..3a19060a 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -23,9 +23,11 @@ from lib.objects import Etags from lib.objects.Favicons import Favicon from lib.objects import HHHashs from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects +from lib.objects.Messages import Message from lib.objects import Pgps from lib.objects.Screenshots import Screenshot from lib.objects import Titles +from lib.objects.UsersAccount import UserAccount from lib.objects import Usernames config_loader = ConfigLoader() @@ -68,6 +70,8 @@ def get_object(obj_type, subtype, id): return Favicon(id) elif obj_type == 'hhhash': return HHHashs.HHHash(id) + elif obj_type == 'message': + return Message(id) elif obj_type == 'screenshot': return Screenshot(id) elif obj_type == 'cryptocurrency': @@ -76,6 +80,8 @@ def get_object(obj_type, subtype, id): return Pgps.Pgp(id, subtype) elif obj_type == 'title': return Titles.Title(id) + elif obj_type == 'user-account': + return UserAccount(id, subtype) elif obj_type == 'username': return Usernames.Username(id, subtype) diff --git a/bin/lib/timeline_engine.py b/bin/lib/timeline_engine.py index 405e7a50..58c222f6 100755 --- a/bin/lib/timeline_engine.py +++ b/bin/lib/timeline_engine.py @@ -46,112 +46,167 @@ config_loader = None # return [] # return correl_types -# TODO rename all function + add missing parameters +class Timeline: -def get_bloc_obj_global_id(bloc): - return r_meta.hget('hset:key', bloc) + def __init__(self, global_id, name): + self.id = global_id + self.name = name -def set_bloc_obj_global_id(bloc, global_id): - return r_meta.hset('hset:key', bloc, global_id) + def _get_block_obj_global_id(self, block): + return r_meta.hget(f'block:{self.id}:{self.name}', block) -def get_bloc_timestamp(bloc, position): - return r_meta.zscore('key', f'{position}:{bloc}') + def _set_block_obj_global_id(self, block, global_id): + return r_meta.hset(f'block:{self.id}:{self.name}', block, global_id) -def add_bloc(global_id, timestamp, end=None): - if end: - timestamp_end = end - else: - timestamp_end = timestamp - new_bloc = str(uuid4()) - r_meta.zadd('key', {f'start:{new_bloc}': timestamp, f'end:{new_bloc}': timestamp_end}) - set_bloc_obj_global_id(new_bloc, global_id) - return new_bloc + def _get_block_timestamp(self, block, position): + return r_meta.zscore(f'line:{self.id}:{self.name}', f'{position}:{block}') -def _update_bloc(bloc, position, timestamp): - r_meta.zadd('key', {f'{position}:{bloc}': timestamp}) + def _get_nearest_bloc_inf(self, timestamp): + inf = r_meta.zrevrangebyscore(f'line:{self.id}:{self.name}', float(timestamp), 0, start=0, num=1, withscores=True) + if inf: + inf, score = inf[0] + if inf.startswith('end'): + inf_key = f'start:{inf[4:]}' + inf_score = r_meta.zscore(f'line:{self.id}:{self.name}', inf_key) + if inf_score == score: + inf = inf_key + return inf + else: + return None -# score = timestamp -def get_nearest_bloc_inf(timestamp): - return r_meta.zrevrangebyscore('key', timestamp, 0, num=1) + def _get_nearest_bloc_sup(self, timestamp): + sup = r_meta.zrangebyscore(f'line:{self.id}:{self.name}', float(timestamp), '+inf', start=0, num=1, withscores=True) + if sup: + sup, score = sup[0] + if sup.startswith('start'): + sup_key = f'end:{sup[6:]}' + sup_score = r_meta.zscore(f'line:{self.id}:{self.name}', sup_key) + if score == sup_score: + sup = sup_key + return sup + else: + return None -def get_nearest_bloc_sup(timestamp): - return r_meta.zrangebyscore('key', timestamp, 0, num=1) + def get_first_obj_id(self): + first = r_meta.zrange(f'line:{self.id}:{self.name}', 0, 0) + if first: # start:block + first = first[0] + if first.startswith('start:'): + first = first[6:] + else: + first = first[4:] + return self._get_block_obj_global_id(first) -####################################################################################### + def get_last_obj_id(self): + last = r_meta.zrevrange(f'line:{self.id}:{self.name}', 0, 0) + if last: # end:block + last = last[0] + if last.startswith('end:'): + last = last[4:] + else: + last = last[6:] + return self._get_block_obj_global_id(last) -def add_timestamp(timestamp, obj_global_id): - inf = get_nearest_bloc_inf(timestamp) - sup = get_nearest_bloc_sup(timestamp) - if not inf and not sup: - # create new bloc - new_bloc = add_bloc(obj_global_id, timestamp) + def get_objs_ids(self): + objs = set() + for block in r_meta.zrange(f'line:{self.id}:{self.name}', 0, -1): + if block: + if block.startswith('start:'): + objs.add(self._get_block_obj_global_id(block[6:])) + return objs + + # def get_objs_ids(self): + # objs = {} + # last_obj_id = None + # for block, timestamp in r_meta.zrange(f'line:{self.id}:{self.name}', 0, -1, withscores=True): + # if block: + # if block.startswith('start:'): + # last_obj_id = self._get_block_obj_global_id(block[6:]) + # objs[last_obj_id] = {'first_seen': timestamp} + # else: + # objs[last_obj_id]['last_seen'] = timestamp + # return objs + + def _update_bloc(self, block, position, timestamp): + r_meta.zadd(f'line:{self.id}:{self.name}', {f'{position}:{block}': timestamp}) + + def _add_bloc(self, obj_global_id, timestamp, end=None): + if end: + timestamp_end = end + else: + timestamp_end = timestamp + new_bloc = str(uuid4()) + r_meta.zadd(f'line:{self.id}:{self.name}', {f'start:{new_bloc}': timestamp, f'end:{new_bloc}': timestamp_end}) + self._set_block_obj_global_id(new_bloc, obj_global_id) return new_bloc - # timestamp < first_seen - elif not inf: - sup_pos, sup_id = inf.split(':') - sup_obj = get_bloc_obj_global_id(sup_pos) - if sup_obj == obj_global_id: - _update_bloc(sup_id, 'start', timestamp) - # create new bloc - else: - new_bloc = add_bloc(obj_global_id, timestamp) + + def add_timestamp(self, timestamp, obj_global_id): + inf = self._get_nearest_bloc_inf(timestamp) + sup = self._get_nearest_bloc_sup(timestamp) + if not inf and not sup: + # create new bloc + new_bloc = self._add_bloc(obj_global_id, timestamp) return new_bloc - - # timestamp > first_seen - elif not sup: - inf_pos, inf_id = inf.split(':') - inf_obj = get_bloc_obj_global_id(inf_id) - if inf_obj == obj_global_id: - _update_bloc(inf_id, 'end', timestamp) - # create new bloc - else: - new_bloc = add_bloc(obj_global_id, timestamp) - return new_bloc - - else: - inf_pos, inf_id = inf.split(':') - sup_pos, sup_id = inf.split(':') - inf_obj = get_bloc_obj_global_id(inf_id) - - if inf_id == sup_id: - # reduce bloc + create two new bloc - if obj_global_id != inf_obj: - # get end timestamp - sup_timestamp = get_bloc_timestamp(sup_id, 'end') - # reduce original bloc - _update_bloc(inf_id, 'end', timestamp - 1) - # Insert new bloc - new_bloc = add_bloc(obj_global_id, timestamp) - # Recreate end of the first bloc by a new bloc - add_bloc(inf_obj, timestamp + 1, end=sup_timestamp) + # timestamp < first_seen + elif not inf: + sup_pos, sup_id = sup.split(':') + sup_obj = self._get_block_obj_global_id(sup_id) + if sup_obj == obj_global_id: + self._update_bloc(sup_id, 'start', timestamp) + # create new bloc + else: + new_bloc = self._add_bloc(obj_global_id, timestamp) return new_bloc - # timestamp in existing bloc - else: - return inf_id - - # different blocs: expend sup/inf bloc or create a new bloc if - elif inf_pos == 'end' and sup_pos == 'start': - # Extend inf bloc - if obj_global_id == inf_obj: - _update_bloc(inf_id, 'end', timestamp) - return inf_id - - sup_obj = get_bloc_obj_global_id(sup_pos) - # Extend sup bloc - if obj_global_id == sup_obj: - _update_bloc(sup_id, 'start', timestamp) - return sup_id - + # timestamp > first_seen + elif not sup: + inf_pos, inf_id = inf.split(':') + inf_obj = self._get_block_obj_global_id(inf_id) + if inf_obj == obj_global_id: + self._update_bloc(inf_id, 'end', timestamp) # create new bloc - new_bloc = add_bloc(obj_global_id, timestamp) - return new_bloc + else: + new_bloc = self._add_bloc(obj_global_id, timestamp) + return new_bloc - # inf_pos == 'start' and sup_pos == 'end' - # else raise error ??? + else: + inf_pos, inf_id = inf.split(':') + sup_pos, sup_id = sup.split(':') + inf_obj = self._get_block_obj_global_id(inf_id) + if inf_id == sup_id: + # reduce bloc + create two new bloc + if obj_global_id != inf_obj: + # get end timestamp + sup_timestamp = self._get_block_timestamp(sup_id, 'end') + # reduce original bloc + self._update_bloc(inf_id, 'end', timestamp - 1) + # Insert new bloc + new_bloc = self._add_bloc(obj_global_id, timestamp) + # Recreate end of the first bloc by a new bloc + self._add_bloc(inf_obj, timestamp + 1, end=sup_timestamp) + return new_bloc + # timestamp in existing bloc + else: + return inf_id + # different blocs: expend sup/inf bloc or create a new bloc if + elif inf_pos == 'end' and sup_pos == 'start': + # Extend inf bloc + if obj_global_id == inf_obj: + self._update_bloc(inf_id, 'end', timestamp) + return inf_id + sup_obj = self._get_block_obj_global_id(sup_id) + # Extend sup bloc + if obj_global_id == sup_obj: + self._update_bloc(sup_id, 'start', timestamp) + return sup_id + # create new bloc + new_bloc = self._add_bloc(obj_global_id, timestamp) + return new_bloc + # inf_pos == 'start' and sup_pos == 'end' + # else raise error ??? diff --git a/bin/modules/Cryptocurrencies.py b/bin/modules/Cryptocurrencies.py index 6197f8a1..318cdd88 100755 --- a/bin/modules/Cryptocurrencies.py +++ b/bin/modules/Cryptocurrencies.py @@ -130,7 +130,7 @@ class Cryptocurrencies(AbstractModule, ABC): if crypto.is_valid_address(): # print(address) is_valid_address = True - crypto.add(date, item_id) + crypto.add(date, item) # Check private key if is_valid_address: diff --git a/bin/modules/PgpDump.py b/bin/modules/PgpDump.py index 0647e897..53c89a19 100755 --- a/bin/modules/PgpDump.py +++ b/bin/modules/PgpDump.py @@ -210,18 +210,18 @@ class PgpDump(AbstractModule): date = item.get_date() for key in self.keys: pgp = Pgps.Pgp(key, 'key') - pgp.add(date, self.item_id) + pgp.add(date, item) print(f' key: {key}') for name in self.names: pgp = Pgps.Pgp(name, 'name') - pgp.add(date, self.item_id) + pgp.add(date, item) print(f' name: {name}') self.tracker_term.compute(name, obj_type='pgp', subtype='name') self.tracker_regex.compute(name, obj_type='pgp', subtype='name') self.tracker_yara.compute(name, obj_type='pgp', subtype='name') for mail in self.mails: pgp = Pgps.Pgp(mail, 'mail') - pgp.add(date, self.item_id) + pgp.add(date, item) print(f' mail: {mail}') self.tracker_term.compute(mail, obj_type='pgp', subtype='mail') self.tracker_regex.compute(mail, obj_type='pgp', subtype='mail') diff --git a/bin/modules/Telegram.py b/bin/modules/Telegram.py index 31d90878..42feaa09 100755 --- a/bin/modules/Telegram.py +++ b/bin/modules/Telegram.py @@ -58,7 +58,7 @@ class Telegram(AbstractModule): user_id = dict_url.get('username') if user_id: username = Username(user_id, 'telegram') - username.add(item_date, item.id) + username.add(item_date, item) print(f'username: {user_id}') invite_hash = dict_url.get('invite_hash') if invite_hash: @@ -73,7 +73,7 @@ class Telegram(AbstractModule): user_id = dict_url.get('username') if user_id: username = Username(user_id, 'telegram') - username.add(item_date, item.id) + username.add(item_date, item) print(f'username: {user_id}') invite_hash = dict_url.get('invite_hash') if invite_hash: diff --git a/var/www/templates/objects/chat/ChatMessages.html b/var/www/templates/objects/chat/ChatMessages.html index b89a447a..aa0bdf35 100644 --- a/var/www/templates/objects/chat/ChatMessages.html +++ b/var/www/templates/objects/chat/ChatMessages.html @@ -120,14 +120,26 @@
    - {{ mess['username']['id'] }} + {{ mess['user-account']['id'] }}
    {{ mess['hour'] }}
    -
    {{ mess['username']['id'] }}
    +
    + {% if mess['user-account']['username'] %} + {{ mess['user-account']['username']['id'] }} + {% else %} + {{ mess['user-account']['id'] }} + {% endif %} +
    {% if mess['reply_to'] %}
    -
    {{ mess['reply_to']['username']['id'] }}
    +
    + {% if mess['reply_to']['user-account']['username'] %} + {{ mess['reply_to']['user-account']['username']['id'] }} + {% else %} + {{ mess['reply_to']['user-account']['id'] }} + {% endif %} + 
    {{ mess['reply_to']['content'] }}
    {% for tag in mess['reply_to']['tags'] %} {{ tag }}