From fc2c3ea08fec98748248feb631ed822dcd35a7cd Mon Sep 17 00:00:00 2001 From: osagit Date: Tue, 7 Sep 2021 11:57:17 +0200 Subject: [PATCH 01/21] fix: error message contains http protocol twice Error Can't connect to AIL Splash Manager, http://https://localhost:7001/ --- bin/lib/crawlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/lib/crawlers.py b/bin/lib/crawlers.py index 23ba0ae1..bc1879b6 100755 --- a/bin/lib/crawlers.py +++ b/bin/lib/crawlers.py @@ -1354,7 +1354,7 @@ def test_ail_crawlers(): # # TODO: test regular domain if not ping_splash_manager(): manager_url = get_splash_manager_url() - error_message = f'Error: Can\'t connect to AIL Splash Manager, http://{manager_url}' + error_message = f'Error: Can\'t connect to AIL Splash Manager, {manager_url}' print(error_message) save_test_ail_crawlers_result(False, error_message) return False From e063dc1ee3938256dc441ecc1da11a4e06eeb953 Mon Sep 17 00:00:00 2001 From: osagit Date: Wed, 8 Sep 2021 10:32:47 +0200 Subject: [PATCH 02/21] fix: inherit AbstractModule to prevent stuck queues regex compiled only at start, not in the loop no duplicate warning string comments --- bin/Cve.py | 89 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 47 insertions(+), 42 deletions(-) diff --git a/bin/Cve.py b/bin/Cve.py index 2a723a74..6e11ca98 100755 --- a/bin/Cve.py +++ b/bin/Cve.py @@ -10,56 +10,61 @@ It apply CVE regexes on paste content and warn if a reference to a CVE is spotte """ +################################## +# Import External packages +################################## import time import re -from pubsublogger import publisher + +################################## +# Import Project packages +################################## +from modules.abstract_module import AbstractModule from packages import Paste -from Helper import Process -def search_cve(message): - filepath, count = message.split() - paste = Paste.Paste(filepath) - content = paste.get_p_content() - # regex to find CVE - reg_cve = re.compile(r'(CVE-)[1-2]\d{1,4}-\d{1,5}') - # list of the regex results in the Paste, may be null - results = set(reg_cve.findall(content)) +class Cve(AbstractModule): + """ + Cve module for AIL framework + """ - # if the list is greater than 2, we consider the Paste may contain a list of cve - if len(results) > 0: - print('{} contains CVEs'.format(paste.p_name)) - publisher.warning('{} contains CVEs'.format(paste.p_name)) + def __init__(self): + super(Cve, self).__init__() + + # regex to find CVE + self.reg_cve = re.compile(r'(CVE-)[1-2]\d{1,4}-\d{1,5}') + + # Waiting time in secondes between to message proccessed + self.pending_seconds = 1 + + # Send module state to logs + self.redis_logger.info(f'Module {self.module_name} initialized') + + + def compute(self, message): + + filepath, count = message.split() + paste = Paste.Paste(filepath) + content = paste.get_p_content() + + # list of the regex results in the Paste, may be null + results = set(self.reg_cve.findall(content)) + + # if the list is positive, we consider the Paste may contain a list of cve + if len(results) > 0: + warning = f'{paste.p_name} contains CVEs' + print(warning) + self.redis_logger.warning(warning) + + msg = f'infoleak:automatic-detection="cve";{filepath}' + # Send to Tags Queue + self.send_message_to_queue(msg, 'Tags') + # Send to Duplicate Queue + self.send_message_to_queue(filepath, 'Duplicate') - msg = 'infoleak:automatic-detection="cve";{}'.format(filepath) - p.populate_set_out(msg, 'Tags') - #Send to duplicate - p.populate_set_out(filepath, 'Duplicate') if __name__ == '__main__': - # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) - # Port of the redis instance used by pubsublogger - publisher.port = 6380 - # Script is the default channel used for the modules. - publisher.channel = 'Script' - # Section name in bin/packages/modules.cfg - config_section = 'Cve' + module = Cve() + module.run() - # Setup the I/O queues - p = Process(config_section) - - # Sent to the logging a description of the module - publisher.info("Run CVE module") - - # Endless loop getting messages from the input queue - while True: - # Get one message from the input queue - message = p.get_from_set() - if message is None: - publisher.debug("{} queue is empty, waiting".format(config_section)) - time.sleep(1) - continue - - # Do something with the message from the queue - search_cve(message) From 06a886732c66db56907380efa73dfc67813dbe64 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Tue, 28 Sep 2021 20:31:19 +0200 Subject: [PATCH 03/21] Add initial UI support for Webhook in tracker --- bin/lib/Tracker.py | 7 +- var/www/modules/hunter/Flask_hunter.py | 7 +- .../hunter/templates/edit_tracker.html | 8 +- .../modules/hunter/templates/showTracker.html | 6 + .../hunter/templates/trackersManagement.html | 391 +++++++++--------- 5 files changed, 227 insertions(+), 192 deletions(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 5310fa73..bf691514 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -106,6 +106,9 @@ def get_tracker_tags(tracker_uuid): def get_tracker_mails(tracker_uuid): return list(r_serv_tracker.smembers('tracker:mail:{}'.format(tracker_uuid))) +def get_tracker_webhook(tracker_uuid): + return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'webhook') + def get_tracker_uuid_sources(tracker_uuid): return list(r_serv_tracker.smembers(f'tracker:sources:{tracker_uuid}')) @@ -129,7 +132,7 @@ def get_tracker_last_seen(tracker_uuid): else: return None -def get_tracker_metedata(tracker_uuid, user_id=False, description=False, level=False, tags=False, mails=False, sources=True, sparkline=False): +def get_tracker_metedata(tracker_uuid, user_id=False, description=False, level=False, tags=False, mails=False, sources=True, sparkline=False, webhook=False): dict_uuid = {} dict_uuid['tracker'] = get_tracker_by_uuid(tracker_uuid) dict_uuid['type'] = get_tracker_type(tracker_uuid) @@ -149,6 +152,8 @@ def get_tracker_metedata(tracker_uuid, user_id=False, description=False, level=F dict_uuid['tags'] = get_tracker_tags(tracker_uuid) if sparkline: dict_uuid['sparkline'] = get_tracker_sparkline(tracker_uuid) + if webhook: + dict_uuid['webhook'] = get_tracker_webhook(tracker_uuid) dict_uuid['uuid'] = tracker_uuid return dict_uuid diff --git a/var/www/modules/hunter/Flask_hunter.py b/var/www/modules/hunter/Flask_hunter.py index a04c7286..0d45c23e 100644 --- a/var/www/modules/hunter/Flask_hunter.py +++ b/var/www/modules/hunter/Flask_hunter.py @@ -96,6 +96,7 @@ def add_tracked_menu(): tracker_type = request.form.get("tracker_type") nb_words = request.form.get("nb_word", 1) description = request.form.get("description", '') + webhook = request.form.get("webhook", '') level = request.form.get("level", 0) tags = request.form.get("tags", []) mails = request.form.get("mails", []) @@ -125,7 +126,7 @@ def add_tracked_menu(): input_dict = {"tracker": tracker, "type": tracker_type, "nb_words": nb_words, "tags": tags, "mails": mails, "sources": sources, - "level": level, "description": description} + "level": level, "description": description, "webhook": webhook} user_id = current_user.get_id() # edit tracker if tracker_uuid: @@ -155,7 +156,7 @@ def edit_tracked_menu(): if res: # invalid access return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - dict_tracker = Tracker.get_tracker_metedata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True) + dict_tracker = Tracker.get_tracker_metedata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True, webhook=True) dict_tracker['tags'] = ' '.join(dict_tracker['tags']) dict_tracker['mails'] = ' '.join(dict_tracker['mails']) @@ -202,7 +203,7 @@ def show_tracker(): if date_to: date_to = date_to.replace('-', '') - tracker_metadata = Tracker.get_tracker_metedata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True, sparkline=True) + tracker_metadata = Tracker.get_tracker_metedata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True, sparkline=True, webhook=True) if tracker_metadata['type'] == 'yara': yara_rule_content = Tracker.get_yara_rule_content(tracker_metadata['tracker']) diff --git a/var/www/modules/hunter/templates/edit_tracker.html b/var/www/modules/hunter/templates/edit_tracker.html index 8d91c54b..279051fb 100644 --- a/var/www/modules/hunter/templates/edit_tracker.html +++ b/var/www/modules/hunter/templates/edit_tracker.html @@ -43,7 +43,7 @@
-
+
@@ -53,6 +53,12 @@
+
+
+
+
+
+
diff --git a/var/www/modules/hunter/templates/showTracker.html b/var/www/modules/hunter/templates/showTracker.html index 0faf20e6..77b1242d 100644 --- a/var/www/modules/hunter/templates/showTracker.html +++ b/var/www/modules/hunter/templates/showTracker.html @@ -74,6 +74,7 @@ Created by First seen Last seen + Webhook URL Tags Email @@ -95,6 +96,11 @@ {{ tracker_metadata['last_seen'][0:4] }}/{{ tracker_metadata['last_seen'][4:6] }}/{{ tracker_metadata['last_seen'][6:8] }} {% endif %} + + {% if tracker_metadata['wehook'] %} + {{ tracker_metadata['wehook'] }} + {% endif %} + {% for tag in tracker_metadata['tags'] %} diff --git a/var/www/modules/hunter/templates/trackersManagement.html b/var/www/modules/hunter/templates/trackersManagement.html index 007d8046..dc9a2545 100644 --- a/var/www/modules/hunter/templates/trackersManagement.html +++ b/var/www/modules/hunter/templates/trackersManagement.html @@ -2,224 +2,241 @@ - - + + - Tracker Management - + Tracker Management + - - - - + + + + - - - - - - + + + + + + - + .btn-link { + color: #000000 + } + + .mouse_pointer { + cursor: pointer; + } + + .lb-md { + font-size: 16px; + } + - {% include 'nav_bar.html' %} +{% include 'nav_bar.html' %} -
-
+
+
- {% include 'hunter/menu_sidebar.html' %} + {% include 'hunter/menu_sidebar.html' %} -
+
-
-
-
Your {{filter_type}} Trackers
-
-
- - - - - - - - - - - - - {% for dict_uuid in user_term %} - - - + + + + + + + {% endfor %} + +
TypeTrackerFirst seenLast seenEmail notificationsparkline
{{dict_uuid['type']}} +
+
+
Your {{ filter_type }} Trackers
+
+
+ + + + + + + + + + + + + + {% for dict_uuid in user_term %} + + + - - - - - - {% endfor %} - -
TypeTrackerFirst seenLast seenWebhook URLEmail notificationsparkline
{{ dict_uuid['type'] }} - {% if dict_uuid['term']%} - {% if dict_uuid['term']|length > 256 %} - {{ dict_uuid['term'][0:256]}}... - {% else %} - {{ dict_uuid['term']}} - {% endif %} + {% if dict_uuid['term'] %} + {% if dict_uuid['term']|length > 256 %} + {{ dict_uuid['term'][0:256] }}... + {% else %} + {{ dict_uuid['term'] }} + {% endif %} {% endif %} -
- {% for tag in dict_uuid['tags'] %} - - {{ tag }} - - {% endfor %} -
- 		- {% if dict_uuid['first_seen'] %} - {{dict_uuid['first_seen'][0:4]}}/{{dict_uuid['first_seen'][4:6]}}/{{dict_uuid['first_seen'][6:8]}} - {% endif %} - - {% if dict_uuid['last_seen'] %} - {{dict_uuid['last_seen'][0:4]}}/{{dict_uuid['last_seen'][4:6]}}/{{dict_uuid['last_seen'][6:8]}} - {% endif %} - - {% for mail in dict_uuid['mails'] %} - {{ mail }}
- {% endfor %} -
-
-
+
+ {% for tag in dict_uuid['tags'] %} + + {{ tag }} + + {% endfor %} +
+ 		+ {% if dict_uuid['first_seen'] %} + {{ dict_uuid['first_seen'][0:4] }}/{{ dict_uuid['first_seen'][4:6] }}/ + {{ dict_uuid['first_seen'][6:8] }} + {% endif %} + + {% if dict_uuid['last_seen'] %} + {{ dict_uuid['last_seen'][0:4] }}/{{ dict_uuid['last_seen'][4:6] }}/ + {{ dict_uuid['last_seen'][6:8] }} + {% endif %} + + {% if dict_uuid['webhook'] %} + {{ dict_uuid['webhook'] }} + {% endif %} + + {% for mail in dict_uuid['mails'] %} + {{ mail }}
+ {% endfor %} +
+
+
-
-
-
Global {{filter_type}} Trackers
-
-
- - - - - - - - - - - - - {% for dict_uuid in global_term %} - - - + + + + + + + {% endfor %} + +
TypeTrackerFirst seenLast seenEmail notificationsparkline
{{dict_uuid['type']}} +
+
+
Global {{ filter_type }} Trackers
+
+
+ + + + + + + + + + + + + + {% for dict_uuid in global_term %} + + + - - - - - - {% endfor %} - -
TypeTrackerFirst seenLast seenWebhook URLEmail notificationsparkline
{{ dict_uuid['type'] }} - {% if dict_uuid['term']%} - {% if dict_uuid['term']|length > 256 %} - {{ dict_uuid['term'][0:256]}}... - {% else %} - {{ dict_uuid['term']}} - {% endif %} + {% if dict_uuid['term'] %} + {% if dict_uuid['term']|length > 256 %} + {{ dict_uuid['term'][0:256] }}... + {% else %} + {{ dict_uuid['term'] }} + {% endif %} {% endif %} -
- {% for tag in dict_uuid['tags'] %} - - {{ tag }} - - {% endfor %} -
- 		- {% if dict_uuid['first_seen'] %} - {{dict_uuid['first_seen'][0:4]}}/{{dict_uuid['first_seen'][4:6]}}/{{dict_uuid['first_seen'][6:8]}} - {% endif %} - - {% if dict_uuid['last_seen'] %} - {{dict_uuid['last_seen'][0:4]}}/{{dict_uuid['last_seen'][4:6]}}/{{dict_uuid['last_seen'][6:8]}} - {% endif %} - - {% for mail in dict_uuid['mails'] %} - {{ mail }}
- {% endfor %} -
-
+
+ {% for tag in dict_uuid['tags'] %} + + {{ tag }} + + {% endfor %} +
+
+ {% if dict_uuid['first_seen'] %} + {{ dict_uuid['first_seen'][0:4] }}/{{ dict_uuid['first_seen'][4:6] }}/ + {{ dict_uuid['first_seen'][6:8] }} + {% endif %} + + {% if dict_uuid['last_seen'] %} + {{ dict_uuid['last_seen'][0:4] }}/{{ dict_uuid['last_seen'][4:6] }}/ + {{ dict_uuid['last_seen'][6:8] }} + {% endif %} + + {% if dict_uuid['webhook'] %} + {{ dict_uuid['webhook'] }} + {% endif %} + + {% for mail in dict_uuid['mails'] %} + {{ mail }}
+ {% endfor %} +
+
+
+ + + + + Create New Tracker + +
- - - - - - Create New Tracker - - -
-
-
+
+
From bc91c202a839fb2002e610afd4c999c919519114 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Tue, 28 Sep 2021 20:48:47 +0200 Subject: [PATCH 04/21] Add more support for Webhook URL --- bin/lib/Tracker.py | 10 +++++++--- var/www/modules/hunter/templates/edit_tracker.html | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index bf691514..20be8cf2 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -374,7 +374,7 @@ def api_validate_tracker_to_add(tracker , tracker_type, nb_words=1): return ({"status": "error", "reason": "Incorrect type"}, 400) return ({"status": "success", "tracker": tracker, "type": tracker_type}, 200) -def create_tracker(tracker, tracker_type, user_id, level, tags, mails, description, dashboard=0, tracker_uuid=None, sources=[]): +def create_tracker(tracker, tracker_type, user_id, level, tags, mails, description, webhook, dashboard=0, tracker_uuid=None, sources=[]): # edit tracker if tracker_uuid: edit_tracker = True @@ -415,6 +415,9 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti if description: r_serv_tracker.hset('tracker:{}'.format(tracker_uuid), 'description', description) + if webhook: + r_serv_tracker.hset('tracker:{}'.format(tracker_uuid), 'webhook', webhook) + # type change if edit_tracker: r_serv_tracker.srem('all:tracker:{}'.format(old_type), old_tracker) @@ -486,7 +489,8 @@ def api_add_tracker(dict_input, user_id): nb_words = dict_input.get('nb_words', 1) description = dict_input.get('description', '') description = escape(description) - + webhook = dict_input.get('webhook', '') + webhook = escape(webhook) res = api_validate_tracker_to_add(tracker , tracker_type, nb_words=nb_words) if res[1]!=200: return res @@ -528,7 +532,7 @@ def api_add_tracker(dict_input, user_id): if is_tracker_in_user_level(tracker, tracker_type, user_id) and not tracker_uuid: return ({"status": "error", "reason": "Tracker already exist"}, 409) - tracker_uuid = create_tracker(tracker , tracker_type, user_id, level, tags, mails, description, tracker_uuid=tracker_uuid, sources=sources) + tracker_uuid = create_tracker(tracker , tracker_type, user_id, level, tags, mails, description, webhook, tracker_uuid=tracker_uuid, sources=sources) return ({'tracker': tracker, 'type': tracker_type, 'uuid': tracker_uuid}, 200) diff --git a/var/www/modules/hunter/templates/edit_tracker.html b/var/www/modules/hunter/templates/edit_tracker.html index 279051fb..b6067ecb 100644 --- a/var/www/modules/hunter/templates/edit_tracker.html +++ b/var/www/modules/hunter/templates/edit_tracker.html @@ -56,7 +56,7 @@
-
+
From 661bcf37f7bf49a28c877751223450b8a88251c2 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Tue, 28 Sep 2021 20:55:41 +0200 Subject: [PATCH 05/21] Fix spelling issue in Webhook --- var/www/modules/hunter/templates/showTracker.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/www/modules/hunter/templates/showTracker.html b/var/www/modules/hunter/templates/showTracker.html index 77b1242d..d16fcc48 100644 --- a/var/www/modules/hunter/templates/showTracker.html +++ b/var/www/modules/hunter/templates/showTracker.html @@ -97,8 +97,8 @@ {% endif %} - {% if tracker_metadata['wehook'] %} - {{ tracker_metadata['wehook'] }} + {% if tracker_metadata['webhook'] %} + {{ tracker_metadata['webhook'] }} {% endif %} From 30d66ef3a8808e5c0911438fbb0330c339e5599e Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Tue, 28 Sep 2021 21:11:20 +0200 Subject: [PATCH 06/21] Add initial support for Webhook in Term Trackers --- bin/packages/Term.py | 3 +++ bin/trackers/Tracker_Term.py | 20 +++++++++++++++----- requirements.txt | 1 + 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/bin/packages/Term.py b/bin/packages/Term.py index 245296ae..5bd91cd0 100755 --- a/bin/packages/Term.py +++ b/bin/packages/Term.py @@ -374,6 +374,9 @@ def get_term_tags(term_uuid): def get_term_mails(term_uuid): return list(r_serv_term.smembers('tracker:mail:{}'.format(term_uuid))) +def get_term_webhook(term_uuid): + return list(r_serv_term.hget('tracker:webhook:{}'.format(term_uuid))) + def add_tracked_item(term_uuid, item_id, item_date): # track item r_serv_term.sadd('tracker:item:{}:{}'.format(term_uuid, item_date), item_id) diff --git a/bin/trackers/Tracker_Term.py b/bin/trackers/Tracker_Term.py index a0986bdf..622aad2a 100755 --- a/bin/trackers/Tracker_Term.py +++ b/bin/trackers/Tracker_Term.py @@ -13,6 +13,8 @@ import os import sys import time import signal +import requests + sys.path.append(os.environ['AIL_BIN']) ################################## @@ -24,21 +26,24 @@ from packages.Item import Item from packages import Term from lib import Tracker - class TimeoutException(Exception): pass + + def timeout_handler(signum, frame): raise TimeoutException + + signal.signal(signal.SIGALRM, timeout_handler) class Tracker_Term(AbstractModule): - mail_body_template = "AIL Framework,\nNew occurrence for tracked term: {}\nitem id: {}\nurl: {}{}" """ Tracker_Term module for AIL framework """ + def __init__(self): super(Tracker_Term, self).__init__() @@ -56,7 +61,6 @@ class Tracker_Term(AbstractModule): self.redis_logger.info(f"Module: {self.module_name} Launched") - def compute(self, item_id): # refresh Tracked term if self.last_refresh_word < Term.get_tracked_term_last_updated_by_type('word'): @@ -88,7 +92,7 @@ class Tracker_Term(AbstractModule): if dict_words_freq: # create token statistics - #for word in dict_words_freq: + # for word in dict_words_freq: # Term.create_token_statistics(item_date, word, dict_words_freq[word]) item_source = item.get_source() @@ -135,8 +139,14 @@ class Tracker_Term(AbstractModule): print(f'S print(item_content)end Mail {mail_subject}') NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) + webhook_to_post = Term.get_term_webhook(term_uuid) + if webhook_to_post: + request_body = dict({"itemId": item_id, "url": self.full_item_url, "type": "Term", "term": term}) + r = requests.post(webhook_to_post, data=request_body) + if (r.status_code >= 400): + raise Exception(f"Webhook request failed for {webhook_to_post}\nReason: {r.reason}") + if __name__ == '__main__': - module = Tracker_Term() module.run() diff --git a/requirements.txt b/requirements.txt index 8dde88c7..cb38df0b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -77,6 +77,7 @@ PySocks>=1.7.1 pycountry>=20.7.3 https://github.com/saffsd/langid.py/archive/master.zip +requests ##### Old packages From 70ccb3cf25dd19cb5be5b6a46f35fc13000c6c0a Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Tue, 28 Sep 2021 22:01:17 +0200 Subject: [PATCH 07/21] Add some changes for webhook --- bin/lib/Tracker.py | 4 ++-- bin/packages/Term.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 20be8cf2..8829e2a4 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -107,7 +107,7 @@ def get_tracker_mails(tracker_uuid): return list(r_serv_tracker.smembers('tracker:mail:{}'.format(tracker_uuid))) def get_tracker_webhook(tracker_uuid): - return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'webhook') + return r_serv_tracker.hget('tracker:webhook:{}'.format(tracker_uuid)) def get_tracker_uuid_sources(tracker_uuid): return list(r_serv_tracker.smembers(f'tracker:sources:{tracker_uuid}')) @@ -472,7 +472,7 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti for source in sources: # escape source ? r_serv_tracker.sadd(f'tracker:sources:{tracker_uuid}', escape(source)) - + r_serv_tracker.sadd(f'tracker:webhook:{tracker_uuid}', webhook) # toggle refresh module tracker list/set r_serv_tracker.set('tracker:refresh:{}'.format(tracker_type), time.time()) if tracker_type != old_type: # toggle old type refresh diff --git a/bin/packages/Term.py b/bin/packages/Term.py index 5bd91cd0..956dd2e7 100755 --- a/bin/packages/Term.py +++ b/bin/packages/Term.py @@ -375,7 +375,7 @@ def get_term_mails(term_uuid): return list(r_serv_term.smembers('tracker:mail:{}'.format(term_uuid))) def get_term_webhook(term_uuid): - return list(r_serv_term.hget('tracker:webhook:{}'.format(term_uuid))) + return r_serv_term.hget('tracker:webhook:{}'.format(term_uuid)) def add_tracked_item(term_uuid, item_id, item_date): # track item From 2319b34dcc062341ac9aa0b253ea83ce85c07a4f Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Tue, 28 Sep 2021 22:22:59 +0200 Subject: [PATCH 08/21] Fix get_term_webhook --- bin/packages/Term.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/packages/Term.py b/bin/packages/Term.py index 956dd2e7..5a495570 100755 --- a/bin/packages/Term.py +++ b/bin/packages/Term.py @@ -375,7 +375,7 @@ def get_term_mails(term_uuid): return list(r_serv_term.smembers('tracker:mail:{}'.format(term_uuid))) def get_term_webhook(term_uuid): - return r_serv_term.hget('tracker:webhook:{}'.format(term_uuid)) + return r_serv_term.hget('tracker:{}'.format(term_uuid), "webhook") def add_tracked_item(term_uuid, item_id, item_date): # track item From 2e880c79cac370b374c407820b4950f075e8a5e4 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Tue, 28 Sep 2021 22:53:01 +0200 Subject: [PATCH 09/21] Add webhook post support in yara and regex trackers --- bin/lib/Tracker.py | 2 +- bin/trackers/Tracker_Regex.py | 8 +++++++- bin/trackers/Tracker_Yara.py | 8 +++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 8829e2a4..06d005a2 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -107,7 +107,7 @@ def get_tracker_mails(tracker_uuid): return list(r_serv_tracker.smembers('tracker:mail:{}'.format(tracker_uuid))) def get_tracker_webhook(tracker_uuid): - return r_serv_tracker.hget('tracker:webhook:{}'.format(tracker_uuid)) + return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'webhook') def get_tracker_uuid_sources(tracker_uuid): return list(r_serv_tracker.smembers(f'tracker:sources:{tracker_uuid}')) diff --git a/bin/trackers/Tracker_Regex.py b/bin/trackers/Tracker_Regex.py index b499e909..78b7baba 100755 --- a/bin/trackers/Tracker_Regex.py +++ b/bin/trackers/Tracker_Regex.py @@ -12,6 +12,7 @@ import os import re import sys import time +import requests sys.path.append(os.environ['AIL_BIN']) ################################## @@ -92,7 +93,12 @@ class Tracker_Regex(AbstractModule): mail_body = Tracker_Regex.mail_body_template.format(tracker, item_id, self.full_item_url, item_id) for mail in mail_to_notify: NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) - + webhook_to_post = Term.get_term_webhook(tracker_uuid) + if webhook_to_post: + request_body = dict({"itemId": item_id, "url": self.full_item_url, "type": "REGEX"}) + r = requests.post(webhook_to_post, data=request_body) + if (r.status_code >= 400): + raise Exception(f"Webhook request failed for {webhook_to_post}\nReason: {r.reason}") if __name__ == "__main__": module = Tracker_Regex() diff --git a/bin/trackers/Tracker_Yara.py b/bin/trackers/Tracker_Yara.py index 5fcd6887..1e62021d 100755 --- a/bin/trackers/Tracker_Yara.py +++ b/bin/trackers/Tracker_Yara.py @@ -14,6 +14,7 @@ import re import sys import time import yara +import requests sys.path.append(os.environ['AIL_BIN']) ################################## @@ -95,7 +96,12 @@ class Tracker_Yara(AbstractModule): self.redis_logger.debug(f'Send Mail {mail_subject}') print(f'Send Mail {mail_subject}') NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) - + webhook_to_post = Term.get_term_webhook(tracker_uuid) + if webhook_to_post: + request_body = dict({"itemId": item_id, "url": self.full_item_url, "type": "YARA"}) + r = requests.post(webhook_to_post, data=request_body) + if (r.status_code >= 400): + raise Exception(f"Webhook request failed for {webhook_to_post}\nReason: {r.reason}") return yara.CALLBACK_CONTINUE From ecc16d10040f19b538d85a20420b954deac8baf9 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Tue, 28 Sep 2021 22:56:28 +0200 Subject: [PATCH 10/21] remove dict from Trackers --- bin/trackers/Tracker_Regex.py | 2 +- bin/trackers/Tracker_Term.py | 2 +- bin/trackers/Tracker_Yara.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/trackers/Tracker_Regex.py b/bin/trackers/Tracker_Regex.py index 78b7baba..3cced9f0 100755 --- a/bin/trackers/Tracker_Regex.py +++ b/bin/trackers/Tracker_Regex.py @@ -95,7 +95,7 @@ class Tracker_Regex(AbstractModule): NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) webhook_to_post = Term.get_term_webhook(tracker_uuid) if webhook_to_post: - request_body = dict({"itemId": item_id, "url": self.full_item_url, "type": "REGEX"}) + request_body = {"itemId": item_id, "url": self.full_item_url, "type": "REGEX"} r = requests.post(webhook_to_post, data=request_body) if (r.status_code >= 400): raise Exception(f"Webhook request failed for {webhook_to_post}\nReason: {r.reason}") diff --git a/bin/trackers/Tracker_Term.py b/bin/trackers/Tracker_Term.py index 622aad2a..487174bf 100755 --- a/bin/trackers/Tracker_Term.py +++ b/bin/trackers/Tracker_Term.py @@ -141,7 +141,7 @@ class Tracker_Term(AbstractModule): webhook_to_post = Term.get_term_webhook(term_uuid) if webhook_to_post: - request_body = dict({"itemId": item_id, "url": self.full_item_url, "type": "Term", "term": term}) + request_body = {"itemId": item_id, "url": self.full_item_url, "type": "Term", "term": term} r = requests.post(webhook_to_post, data=request_body) if (r.status_code >= 400): raise Exception(f"Webhook request failed for {webhook_to_post}\nReason: {r.reason}") diff --git a/bin/trackers/Tracker_Yara.py b/bin/trackers/Tracker_Yara.py index 1e62021d..9cf36ae8 100755 --- a/bin/trackers/Tracker_Yara.py +++ b/bin/trackers/Tracker_Yara.py @@ -98,7 +98,7 @@ class Tracker_Yara(AbstractModule): NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) webhook_to_post = Term.get_term_webhook(tracker_uuid) if webhook_to_post: - request_body = dict({"itemId": item_id, "url": self.full_item_url, "type": "YARA"}) + request_body = {"itemId": item_id, "url": self.full_item_url, "type": "YARA"} r = requests.post(webhook_to_post, data=request_body) if (r.status_code >= 400): raise Exception(f"Webhook request failed for {webhook_to_post}\nReason: {r.reason}") From 52e422604a3bd1327285e2eec7041f571f2e5faa Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Wed, 29 Sep 2021 16:17:08 +0200 Subject: [PATCH 11/21] Fix webhook --- var/www/Flask_server.py | 2 +- var/www/modules/hunter/templates/edit_tracker.html | 2 +- var/www/modules/hunter/templates/showTracker.html | 4 ++-- .../hunter/templates/trackersManagement.html | 13 +------------ 4 files changed, 5 insertions(+), 16 deletions(-) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 32fc2823..5fa0fbfb 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -70,7 +70,7 @@ r_cache = config_loader.get_redis_conn("Redis_Cache") # logs log_dir = os.path.join(os.environ['AIL_HOME'], 'logs') if not os.path.isdir(log_dir): - os.makedirs(logs_dir) + os.makedirs(log_dir) # log_filename = os.path.join(log_dir, 'flask_server.logs') # logger = logging.getLogger() diff --git a/var/www/modules/hunter/templates/edit_tracker.html b/var/www/modules/hunter/templates/edit_tracker.html index b6067ecb..b7ab281b 100644 --- a/var/www/modules/hunter/templates/edit_tracker.html +++ b/var/www/modules/hunter/templates/edit_tracker.html @@ -31,7 +31,7 @@
-
Edit a Tracker
+
{%if dict_tracker%}Edit a{%else%}Create a new{%endif%} Tracker
diff --git a/var/www/modules/hunter/templates/showTracker.html b/var/www/modules/hunter/templates/showTracker.html index d16fcc48..90cdca7b 100644 --- a/var/www/modules/hunter/templates/showTracker.html +++ b/var/www/modules/hunter/templates/showTracker.html @@ -111,8 +111,8 @@ {% for mail in tracker_metadata['mails'] %} - {{ mail }}
- {% endfor %} + {{ mail }}
+ {% endfor %} diff --git a/var/www/modules/hunter/templates/trackersManagement.html b/var/www/modules/hunter/templates/trackersManagement.html index dc9a2545..aedc790d 100644 --- a/var/www/modules/hunter/templates/trackersManagement.html +++ b/var/www/modules/hunter/templates/trackersManagement.html @@ -58,7 +58,6 @@ Tracker First seen Last seen - Webhook URL Email notification sparkline @@ -99,11 +98,6 @@ {{ dict_uuid['last_seen'][6:8] }} {% endif %} - - {% if dict_uuid['webhook'] %} - {{ dict_uuid['webhook'] }} - {% endif %} - {% for mail in dict_uuid['mails'] %} {{ mail }}
@@ -129,7 +123,6 @@ Tracker First seen Last seen - Webhook URL Email notification sparkline @@ -170,11 +163,7 @@ {{ dict_uuid['last_seen'][6:8] }} {% endif %} - - {% if dict_uuid['webhook'] %} - {{ dict_uuid['webhook'] }} - {% endif %} - + {% for mail in dict_uuid['mails'] %} {{ mail }}
From 4b2e9f5cfbfa63f6005eb53641477eb8f5510891 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Thu, 30 Sep 2021 13:37:12 +0200 Subject: [PATCH 12/21] -Fixed the 500 error issue when installing new instance of ail when adding new trackers -Fixed missing arguments -Typo Fixed --- bin/lib/item_basic.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/bin/lib/item_basic.py b/bin/lib/item_basic.py index dde052e2..f9a49260 100755 --- a/bin/lib/item_basic.py +++ b/bin/lib/item_basic.py @@ -83,7 +83,7 @@ def is_father(item_id): def is_children(item_id): return r_serv_metadata.hexists('paste_metadata:{}'.format(item_id), 'father') -def is_root_node(): +def is_root_node(item_id): if is_father(item_id) and not is_children(item_id): return True else: @@ -131,8 +131,8 @@ def _delete_node(item_id): # only if item isn't deleted #if is_crawled(item_id): # r_serv_metadata.hrem('paste_metadata:{}'.format(item_id), 'real_link') - for chidren_id in get_item_children(item_id): - r_serv_metadata.hdel('paste_metadata:{}'.format(chidren_id), 'father') + for children_id in get_item_children(item_id): + r_serv_metadata.hdel('paste_metadata:{}'.format(children_id), 'father') r_serv_metadata.delete('paste_children:{}'.format(item_id)) # delete regular @@ -210,9 +210,12 @@ def _get_dir_source_name(directory, source_name=None, l_sources_name=set(), filt def get_all_items_sources(filter_dir=False, r_list=False): res = _get_dir_source_name(PASTES_FOLDER, filter_dir=filter_dir) - if r_list: - res = list(res) - return res + if res: + if r_list: + res = list(res) + return res + else: + return [] def verify_sources_list(sources): all_sources = get_all_items_sources() From 743dae54b60128f82d9802f8ccfa16ac803e3294 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Thu, 30 Sep 2021 13:40:12 +0200 Subject: [PATCH 13/21] -Fixed "description" arg -Typo Fixed --- bin/lib/Tracker.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 06d005a2..3c4b63ed 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -132,12 +132,12 @@ def get_tracker_last_seen(tracker_uuid): else: return None -def get_tracker_metedata(tracker_uuid, user_id=False, description=False, level=False, tags=False, mails=False, sources=True, sparkline=False, webhook=False): +def get_tracker_metadata(tracker_uuid, user_id=False, description=False, level=False, tags=False, mails=False, sources=True, sparkline=False, webhook=False): dict_uuid = {} + dict_uuid['uuid'] = tracker_uuid dict_uuid['tracker'] = get_tracker_by_uuid(tracker_uuid) dict_uuid['type'] = get_tracker_type(tracker_uuid) dict_uuid['date'] = get_tracker_date(tracker_uuid) - dict_uuid['description'] = get_tracker_description(tracker_uuid) dict_uuid['first_seen'] = get_tracker_first_seen(tracker_uuid) dict_uuid['last_seen'] = get_tracker_last_seen(tracker_uuid) if user_id: @@ -152,9 +152,11 @@ def get_tracker_metedata(tracker_uuid, user_id=False, description=False, level=F dict_uuid['tags'] = get_tracker_tags(tracker_uuid) if sparkline: dict_uuid['sparkline'] = get_tracker_sparkline(tracker_uuid) + if description: + dict_uuid['description'] = get_tracker_description(tracker_uuid) if webhook: dict_uuid['webhook'] = get_tracker_webhook(tracker_uuid) - dict_uuid['uuid'] = tracker_uuid + return dict_uuid # tracker sparkline From ac9df0b9fbda1d42a9924eba0a5bccb3ad5162a6 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Thu, 30 Sep 2021 13:41:10 +0200 Subject: [PATCH 14/21] -Fixed "get_tracker_metedata" typo -Typo Fixed --- var/www/modules/hunter/Flask_hunter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/www/modules/hunter/Flask_hunter.py b/var/www/modules/hunter/Flask_hunter.py index 0d45c23e..0853257f 100644 --- a/var/www/modules/hunter/Flask_hunter.py +++ b/var/www/modules/hunter/Flask_hunter.py @@ -156,7 +156,7 @@ def edit_tracked_menu(): if res: # invalid access return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - dict_tracker = Tracker.get_tracker_metedata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True, webhook=True) + dict_tracker = Tracker.get_tracker_metadata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True, webhook=True) dict_tracker['tags'] = ' '.join(dict_tracker['tags']) dict_tracker['mails'] = ' '.join(dict_tracker['mails']) @@ -203,7 +203,7 @@ def show_tracker(): if date_to: date_to = date_to.replace('-', '') - tracker_metadata = Tracker.get_tracker_metedata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True, sparkline=True, webhook=True) + tracker_metadata = Tracker.get_tracker_metadata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True, sparkline=True, webhook=True) if tracker_metadata['type'] == 'yara': yara_rule_content = Tracker.get_yara_rule_content(tracker_metadata['tracker']) From 912956c73cb6e62ae73dbbb2515db95e2a114fdd Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Thu, 30 Sep 2021 14:20:08 +0200 Subject: [PATCH 15/21] Fixed Webhook integration with Trackers --- bin/trackers/Tracker_Regex.py | 26 ++++++++++++++++++++------ bin/trackers/Tracker_Term.py | 20 +++++++++++++++----- bin/trackers/Tracker_Yara.py | 32 +++++++++++++++++++++----------- 3 files changed, 56 insertions(+), 22 deletions(-) diff --git a/bin/trackers/Tracker_Regex.py b/bin/trackers/Tracker_Regex.py index 3cced9f0..00642445 100755 --- a/bin/trackers/Tracker_Regex.py +++ b/bin/trackers/Tracker_Regex.py @@ -5,7 +5,7 @@ The Tracker_Regex trackers module =================== This Module is used for regex tracking. -It processes every item coming from the global module and test the regexs +It processes every item coming from the global module and test the regex """ import os @@ -76,6 +76,8 @@ class Tracker_Regex(AbstractModule): for tracker_uuid in uuid_list: # Source Filtering item_source = item.get_source() + item_date = item.get_date() + tracker_sources = Tracker.get_tracker_uuid_sources(tracker_uuid) if tracker_sources and item_source not in tracker_sources: continue @@ -93,13 +95,25 @@ class Tracker_Regex(AbstractModule): mail_body = Tracker_Regex.mail_body_template.format(tracker, item_id, self.full_item_url, item_id) for mail in mail_to_notify: NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) + + # Webhook webhook_to_post = Term.get_term_webhook(tracker_uuid) if webhook_to_post: - request_body = {"itemId": item_id, "url": self.full_item_url, "type": "REGEX"} - r = requests.post(webhook_to_post, data=request_body) - if (r.status_code >= 400): - raise Exception(f"Webhook request failed for {webhook_to_post}\nReason: {r.reason}") -if __name__ == "__main__": + json_request = {"trackerId": tracker_uuid, + "itemId": item_id, + "itemURL": self.full_item_url + item_id, + "tracker": tracker, + "itemSource": item_source, + "itemDate": item_date, + "tags": tags_to_add, + "emailNotification": f'{mail_to_notify}', + "trackerType": tracker_type + } + response = requests.post(webhook_to_post, json=json_request) + if response.status_code >= 400: + raise Exception(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}") + +if __name__ == "__main__": module = Tracker_Regex() module.run() diff --git a/bin/trackers/Tracker_Term.py b/bin/trackers/Tracker_Term.py index 487174bf..703dfb4f 100755 --- a/bin/trackers/Tracker_Term.py +++ b/bin/trackers/Tracker_Term.py @@ -119,7 +119,7 @@ class Tracker_Term(AbstractModule): uuid_list = Term.get_term_uuid_list(term, term_type) self.redis_logger.info(f'new tracked term found: {term} in {item_id}') print(f'new tracked term found: {term} in {item_id}') - + item_date = Item.get_date() for term_uuid in uuid_list: tracker_sources = Tracker.get_tracker_uuid_sources(term_uuid) if not tracker_sources or item_source in tracker_sources: @@ -139,12 +139,22 @@ class Tracker_Term(AbstractModule): print(f'S print(item_content)end Mail {mail_subject}') NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) + # Webhook webhook_to_post = Term.get_term_webhook(term_uuid) if webhook_to_post: - request_body = {"itemId": item_id, "url": self.full_item_url, "type": "Term", "term": term} - r = requests.post(webhook_to_post, data=request_body) - if (r.status_code >= 400): - raise Exception(f"Webhook request failed for {webhook_to_post}\nReason: {r.reason}") + json_request = {"trackerId": term_uuid, + "itemId": item_id, + "itemURL": self.full_item_url + item_id, + "term": term, + "itemSource": item_source, + "itemDate": item_date, + "tags": tags_to_add, + "emailNotification": f'{mail_to_notify}', + "trackerType": term_type + } + response = requests.post(webhook_to_post, json=json_request) + if response.status_code >= 400: + raise Exception(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}") if __name__ == '__main__': diff --git a/bin/trackers/Tracker_Yara.py b/bin/trackers/Tracker_Yara.py index 9cf36ae8..43cd179b 100755 --- a/bin/trackers/Tracker_Yara.py +++ b/bin/trackers/Tracker_Yara.py @@ -1,10 +1,8 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* -""" -The Tracker_Yara trackers module -=================== - -""" +################################## +# The Tracker_Yara trackers module +################################## ################################## # Import External packages @@ -25,7 +23,7 @@ from packages import Term from packages.Item import Item from lib import Tracker -import NotificationHelper # # TODO: refractor +import NotificationHelper # # TODO: refactor class Tracker_Yara(AbstractModule): @@ -72,6 +70,7 @@ class Tracker_Yara(AbstractModule): tracker_uuid = data['namespace'] item_id = self.item.get_id() item_source = self.item.get_source() + item_date = self.item.get_date() # Source Filtering tracker_sources = Tracker.get_tracker_uuid_sources(tracker_uuid) @@ -96,16 +95,27 @@ class Tracker_Yara(AbstractModule): self.redis_logger.debug(f'Send Mail {mail_subject}') print(f'Send Mail {mail_subject}') NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) + + # Webhook webhook_to_post = Term.get_term_webhook(tracker_uuid) if webhook_to_post: - request_body = {"itemId": item_id, "url": self.full_item_url, "type": "YARA"} - r = requests.post(webhook_to_post, data=request_body) - if (r.status_code >= 400): - raise Exception(f"Webhook request failed for {webhook_to_post}\nReason: {r.reason}") + json_request = {"trackerId": tracker_uuid, + "itemId": item_id, + "itemURL": self.full_item_url + item_id, + "dataRule": data["rule"], + "itemSource": item_source, + "itemDate": item_date, + "tags": tags_to_add, + "emailNotification": f'{mail_to_notify}', + "trackerType": "yara" + } + response = requests.post(webhook_to_post, json=json_request) + if response.status_code >= 400: + raise Exception(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}") + return yara.CALLBACK_CONTINUE if __name__ == '__main__': - module = Tracker_Yara() module.run() From e1d5e8f2025048ca0c8964055782f4f868f97843 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Fri, 1 Oct 2021 10:02:29 +0200 Subject: [PATCH 16/21] Fixed UI Datatable in showTracker Page Removed "Advanced Search" text from menu misleading button --- .../modules/hunter/templates/showTracker.html | 667 +++++++++--------- var/www/templates/nav_bar.html | 2 +- 2 files changed, 352 insertions(+), 317 deletions(-) diff --git a/var/www/modules/hunter/templates/showTracker.html b/var/www/modules/hunter/templates/showTracker.html index 90cdca7b..162181f5 100644 --- a/var/www/modules/hunter/templates/showTracker.html +++ b/var/www/modules/hunter/templates/showTracker.html @@ -1,365 +1,400 @@ - - + + - AIL Framework - AIL - + AIL Framework - AIL + - - - - - + + + + + - - - - - - - - - - - + + + + + + + + + + + - + -
-
+ + - {% include 'hunter/menu_sidebar.html' %} +{% include 'nav_bar.html' %} -
+
+
-
-
-

- {%if tracker_metadata['description']%} - {{ tracker_metadata['description'] }} - {%endif%} - -

-
- {{ tracker_metadata['uuid'] }} -
-
    -
  • -
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    TypeTrackerDate addedLevelCreated byFirst seenLast seenWebhook URLTags Email
    {{ tracker_metadata['type'] }}{{ tracker_metadata['tracker'] }}{{ tracker_metadata['date'][0:4] }}/{{ tracker_metadata['date'][4:6] }}/{{ tracker_metadata['date'][6:8] }}{{ tracker_metadata['level'] }}{{ tracker_metadata['user_id'] }} - {% if tracker_metadata['first_seen'] %} - {{ tracker_metadata['first_seen'][0:4] }}/{{ tracker_metadata['first_seen'][4:6] }}/{{ tracker_metadata['first_seen'][6:8] }} - {% endif %} - - {% if tracker_metadata['last_seen'] %} - {{ tracker_metadata['last_seen'][0:4] }}/{{ tracker_metadata['last_seen'][4:6] }}/{{ tracker_metadata['last_seen'][6:8] }} - {% endif %} - - {% if tracker_metadata['webhook'] %} - {{ tracker_metadata['webhook'] }} - {% endif %} - - {% for tag in tracker_metadata['tags'] %} - - {{ tag }} - - {% endfor %} + {% include 'hunter/menu_sidebar.html' %} - - {% for mail in tracker_metadata['mails'] %} - {{ mail }}
    - {% endfor %} -
    -
    -
    -
    -
    -
    -
    Sources:
    - {% if tracker_metadata['sources'] %} - {% for sources in tracker_metadata['sources'] %} - {{ sources }}
    - {% endfor %} - {% else %} - All Souces
    - {% endif %} -
    • -
+
-
-
- -
Update this tracker description:
-
-
-
-
- -
+
+
+

+ {% if tracker_metadata['description'] %} + {{ tracker_metadata['description'] }} + {% endif %} + +

+
+ {{ tracker_metadata['uuid'] }} +
+
    +
  • +
    +
    + + + + + + + + + + + {% if tracker_metadata['webhook'] %} + + {% endif %} + + + + + + + + + + + + + + {% if tracker_metadata['webhook'] %} + + {% endif %} + + + + +
    TypeTrackerDate addedAccess LevelCreated byFirst seenLast seenWebhookTags Email
    {{ tracker_metadata['type'] }}{{ tracker_metadata['tracker'] }}{{ tracker_metadata['date'][0:4] }}/{{ tracker_metadata['date'][4:6] }}/{{ tracker_metadata['date'][6:8] }} + {% if tracker_metadata['level'] == "0" %} + Private + {% else %} + Global + {% endif %} + {{ tracker_metadata['user_id'] }} + {% if tracker_metadata['first_seen'] %} + {{ tracker_metadata['first_seen'][0:4] }}/ + {{ tracker_metadata['first_seen'][4:6] }}/ + {{ tracker_metadata['first_seen'][6:8] }} + {% endif %} + + {% if tracker_metadata['last_seen'] %} + {{ tracker_metadata['last_seen'][0:4] }}/ + {{ tracker_metadata['last_seen'][4:6] }}/ + {{ tracker_metadata['last_seen'][6:8] }} + {% endif %} + + Turned ON + + {% for tag in tracker_metadata['tags'] %} + + {{ tag }} + + {% endfor %} - - + + {% for mail in tracker_metadata['mails'] %} + {{ mail }}
    + {% endfor %} +
    +
    +
    +
    +
    +
    +
    Sources:
    + {% if tracker_metadata['sources'] %} + {% for sources in tracker_metadata['sources'] %} + {{ sources }}
    + {% endfor %} + {% else %} + All Souces
    + {% endif %} +
    • +
-
+
+
+ +
Update this tracker description:
+
+
+
+
+ +
-
- - -
All Tags added for this tracker, space separated:
-
-
-
-
- -
+ + - - +
-
+
+
+ +
All Tags added for this tracker, space separated:
+
+
+
+
+ +
-
- - -
All E-Mails to Notify for this tracker, space separated:
-
-
-
-
- -
+ + - - +
-
+
+
+ +
All E-Mails to Notify for this tracker, space separated:
+
+
+
+
+ +
-
- - - - - - -
+ +
- {%if yara_rule_content%} - 



{{ yara_rule_content }}

- {%endif%} +
-
-
+
+ + + + + + +
-
+ {% if yara_rule_content %} +



+ 

{{ yara_rule_content }}

+ {% endif %} -
-
+
+
-
-
-
-
- -
-
-
-
-
- -
-
-
+
- +
+
-
-
+
+
+
+
+ +
+
+
+
+
+ +
+
+
- {%if tracker_metadata['items']%} -
- - - - - - - - + - {% for item in tracker_metadata['items'] %} - - - - - {% endfor %} + + - -
DateItem Id
- {{item['date'][0:4]}}/{{item['date'][4:6]}}/{{item['date'][6:8]}} - - -
{{ item['id'] }}
- -
- {% for tag in item['tags'] %} - - {{ tag }} - - {% endfor %} -
-
-
- {% endif %} + {% if tracker_metadata['items'] %} +
+ + + + + + + + - - - + {% for item in tracker_metadata['items'] %} + + + + + {% endfor %} + + +
DateItem Id
+ {{ item['date'][0:4] }}/{{ item['date'][4:6] }}/{{ item['date'][6:8] }} + + +
{{ item['id'] }}
+ +
+ {% for tag in item['tags'] %} + + {{ tag }} + + {% endfor %} +
+
+
+ {% endif %} + +
+
+
diff --git a/var/www/templates/nav_bar.html b/var/www/templates/nav_bar.html index 257988cc..51db0900 100644 --- a/var/www/templates/nav_bar.html +++ b/var/www/templates/nav_bar.html @@ -45,7 +45,7 @@
- Advanced Search + {#Advanced Search#}
From 09f6d64e1b2d33cdd2dd4fd17bb3f194fbb5c874 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Fri, 1 Oct 2021 10:06:37 +0200 Subject: [PATCH 17/21] Typo Fixed --- var/www/blueprints/hunters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/blueprints/hunters.py b/var/www/blueprints/hunters.py index 1cdde3b7..b4d99b81 100644 --- a/var/www/blueprints/hunters.py +++ b/var/www/blueprints/hunters.py @@ -192,7 +192,7 @@ def get_json_retro_hunt_nb_items_by_date(): if date_from and date_to: res = Tracker.get_retro_hunt_nb_item_by_day([task_uuid], date_from=date_from, date_to=date_to) else: - res = Term.get_retro_hunt_nb_item_by_day([task_uuid]) + res = Tracker.get_retro_hunt_nb_item_by_day([task_uuid]) return jsonify(res) From 1c3ad52f5c38b8c7f1dd4ba423cb40f621c1b7c0 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Fri, 1 Oct 2021 10:16:08 +0200 Subject: [PATCH 18/21] Fixed tracker_metadata --- var/www/modules/hunter/templates/showTracker.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/modules/hunter/templates/showTracker.html b/var/www/modules/hunter/templates/showTracker.html index 162181f5..5f551d24 100644 --- a/var/www/modules/hunter/templates/showTracker.html +++ b/var/www/modules/hunter/templates/showTracker.html @@ -94,7 +94,7 @@ {{ tracker_metadata['tracker'] }} {{ tracker_metadata['date'][0:4] }}/{{ tracker_metadata['date'][4:6] }}/{{ tracker_metadata['date'][6:8] }} - {% if tracker_metadata['level'] == "0" %} + {% if tracker_metadata['level'] == 0 %} Private {% else %} Global From a2b28db32efb4bb9e6522d37a35a6c0c31316d6d Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Mon, 4 Oct 2021 12:55:40 +0200 Subject: [PATCH 19/21] Added a try/catch to handle exceptions Replaced the raise to send message to redis_logger --- bin/trackers/Tracker_Regex.py | 9 ++++++--- bin/trackers/Tracker_Term.py | 10 +++++++--- bin/trackers/Tracker_Yara.py | 10 +++++++--- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/bin/trackers/Tracker_Regex.py b/bin/trackers/Tracker_Regex.py index 00642445..8134c67b 100755 --- a/bin/trackers/Tracker_Regex.py +++ b/bin/trackers/Tracker_Regex.py @@ -109,9 +109,12 @@ class Tracker_Regex(AbstractModule): "emailNotification": f'{mail_to_notify}', "trackerType": tracker_type } - response = requests.post(webhook_to_post, json=json_request) - if response.status_code >= 400: - raise Exception(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}") + try: + response = requests.post(webhook_to_post, json=json_request) + if response.status_code >= 400: + self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}") + except: + self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong") if __name__ == "__main__": diff --git a/bin/trackers/Tracker_Term.py b/bin/trackers/Tracker_Term.py index 703dfb4f..fd90224a 100755 --- a/bin/trackers/Tracker_Term.py +++ b/bin/trackers/Tracker_Term.py @@ -152,9 +152,13 @@ class Tracker_Term(AbstractModule): "emailNotification": f'{mail_to_notify}', "trackerType": term_type } - response = requests.post(webhook_to_post, json=json_request) - if response.status_code >= 400: - raise Exception(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}") + try: + response = requests.post(webhook_to_post, json=json_request) + if response.status_code >= 400: + self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}") + except: + self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong") + if __name__ == '__main__': diff --git a/bin/trackers/Tracker_Yara.py b/bin/trackers/Tracker_Yara.py index 43cd179b..3b23fbee 100755 --- a/bin/trackers/Tracker_Yara.py +++ b/bin/trackers/Tracker_Yara.py @@ -109,9 +109,13 @@ class Tracker_Yara(AbstractModule): "emailNotification": f'{mail_to_notify}', "trackerType": "yara" } - response = requests.post(webhook_to_post, json=json_request) - if response.status_code >= 400: - raise Exception(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}") + try: + response = requests.post(webhook_to_post, json=json_request) + if response.status_code >= 400: + self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}") + except: + self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong") + return yara.CALLBACK_CONTINUE From 768d371e5c0db02bc54ea9fd01c30223fe9a8f65 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Mon, 4 Oct 2021 13:53:28 +0200 Subject: [PATCH 20/21] Type fixed --- var/www/modules/hunter/templates/edit_tracker.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/modules/hunter/templates/edit_tracker.html b/var/www/modules/hunter/templates/edit_tracker.html index b7ab281b..cc3c8f80 100644 --- a/var/www/modules/hunter/templates/edit_tracker.html +++ b/var/www/modules/hunter/templates/edit_tracker.html @@ -43,7 +43,7 @@
-
+
From d7c158726335eec548e99496ee6204dd03e475c8 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Mon, 4 Oct 2021 13:58:04 +0200 Subject: [PATCH 21/21] Webhook unnecessarily line removed Removed unnecessarily parentheses --- bin/lib/Tracker.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 3c4b63ed..5e929b8b 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -474,7 +474,6 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti for source in sources: # escape source ? r_serv_tracker.sadd(f'tracker:sources:{tracker_uuid}', escape(source)) - r_serv_tracker.sadd(f'tracker:webhook:{tracker_uuid}', webhook) # toggle refresh module tracker list/set r_serv_tracker.set('tracker:refresh:{}'.format(tracker_type), time.time()) if tracker_type != old_type: # toggle old type refresh @@ -484,10 +483,10 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti def api_add_tracker(dict_input, user_id): tracker = dict_input.get('tracker', None) if not tracker: - return ({"status": "error", "reason": "Tracker not provided"}, 400) + return {"status": "error", "reason": "Tracker not provided"}, 400 tracker_type = dict_input.get('type', None) if not tracker_type: - return ({"status": "error", "reason": "Tracker type not provided"}, 400) + return {"status": "error", "reason": "Tracker type not provided"}, 400 nb_words = dict_input.get('nb_words', 1) description = dict_input.get('description', '') description = escape(description) @@ -529,14 +528,14 @@ def api_add_tracker(dict_input, user_id): # check if tracker already tracked in global if level==1: if is_tracker_in_global_level(tracker, tracker_type) and not tracker_uuid: - return ({"status": "error", "reason": "Tracker already exist"}, 409) + return {"status": "error", "reason": "Tracker already exist"}, 409 else: if is_tracker_in_user_level(tracker, tracker_type, user_id) and not tracker_uuid: - return ({"status": "error", "reason": "Tracker already exist"}, 409) + return {"status": "error", "reason": "Tracker already exist"}, 409 tracker_uuid = create_tracker(tracker , tracker_type, user_id, level, tags, mails, description, webhook, tracker_uuid=tracker_uuid, sources=sources) - return ({'tracker': tracker, 'type': tracker_type, 'uuid': tracker_uuid}, 200) + return {'tracker': tracker, 'type': tracker_type, 'uuid': tracker_uuid}, 200 ##-- CREATE TRACKER --##