From 1a2aefedf7a6dd670fc39e852e206af686056902 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Wed, 26 Jun 2024 15:06:33 +0200 Subject: [PATCH 1/2] Update README.md --- README.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 1a2c1bd0..8a6fdcad 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ Requirement: ## Installation Notes -For Lacus Crawler installation instructions, refer to the [HOWTO](https://github.com/ail-project/ail-framework/blob/master/HOWTO.md#crawler) +For Lacus Crawler and LibreTranslate installation instructions (if you want to use those features), refer to the [HOWTO](https://github.com/ail-project/ail-framework/blob/master/HOWTO.md#crawler) ## Starting AIL @@ -126,7 +126,6 @@ To start AIL, use the following commands: cd bin/ ./LAUNCH.sh -l ``` - You can access the AIL framework web interface at the following URL: ``` @@ -171,7 +170,6 @@ If you use or reference AIL in an academic paper, you can cite it using the foll ## Screenshots - ### Websites, Forums and Tor Hidden-Services ![Domain CIRCL](./doc/screenshots/domain_circl.png?raw=true "Tor hidden service crawler") @@ -222,11 +220,11 @@ If you use or reference AIL in an academic paper, you can cite it using the foll ``` Copyright (C) 2014 Jules Debra Copyright (c) 2021 Olivier Sagit - Copyright (C) 2014-2023 CIRCL - Computer Incident Response Center Luxembourg (c/o smile, security made in Lëtzebuerg, Groupement d'Intérêt Economique) - Copyright (c) 2014-2023 Raphaël Vinot - Copyright (c) 2014-2023 Alexandre Dulaunoy - Copyright (c) 2016-2023 Sami Mokaddem - Copyright (c) 2018-2023 Thirion Aurélien + Copyright (C) 2014-2024 CIRCL - Computer Incident Response Center Luxembourg (c/o smile, security made in Lëtzebuerg, Groupement d'Intérêt Economique) + Copyright (c) 2014-2024 Raphaël Vinot + Copyright (c) 2014-2024 Alexandre Dulaunoy + Copyright (c) 2016-2024 Sami Mokaddem + Copyright (c) 2018-2024 Thirion Aurélien This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by From ab52db2b561f960b363de7312bf282e756a933e5 Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 10 Jul 2024 15:10:07 +0200 Subject: [PATCH 2/2] fix: [trackers] fix webhook export --- bin/exporter/WebHookExporter.py | 44 +++++++++++++++++++-------------- bin/trackers/Tracker_Regex.py | 4 ++- bin/trackers/Tracker_Yara.py | 4 ++- 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/bin/exporter/WebHookExporter.py b/bin/exporter/WebHookExporter.py index b4391922..49fd00b3 100755 --- a/bin/exporter/WebHookExporter.py +++ b/bin/exporter/WebHookExporter.py @@ -7,6 +7,8 @@ Importer Class Import Content """ +import json +import logging import os import requests import sys @@ -18,10 +20,9 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################# from exporter.abstract_exporter import AbstractExporter +from lib.ail_core import get_ail_uuid -# from ConfigLoader import ConfigLoader -# from lib.objects.abstract_object import AbstractObject -# from lib.Tracker import Tracker +logger = logging.getLogger() class WebHookExporter(AbstractExporter, ABC): def __init__(self, url=''): @@ -35,11 +36,9 @@ class WebHookExporter(AbstractExporter, ABC): try: response = requests.post(self.url, json=data) if response.status_code >= 400: - print(f"Webhook request failed for {self.url}\nReason: {response.reason}") - # self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}") + logger.error(f"Webhook request failed for {self.url}\nReason: {response.reason}") except Exception as e: - print(f"Webhook request failed for {self.url}\nReason: Something went wrong {e}") - # self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong") + logger.error(f"Webhook request failed for {self.url}\nReason: Something went wrong {e}") class WebHookExporterTracker(WebHookExporter): @@ -48,17 +47,26 @@ class WebHookExporterTracker(WebHookExporter): super().__init__(url=url) # TODO Change exported keys - def export(self, tracker, obj): + def export(self, tracker, obj, matches=[]): self.set_url(tracker.get_webhook()) - data = {'trackerId': tracker.get_uuid(), - 'trackerType': tracker.get_type(), - 'tags': tracker.get_tags(), - 'tracker': tracker.get_tracked(), - # object - 'itemId': obj.get_id(), - 'itemURL': obj.get_link()} - # Item - # data['itemDate'] = obj.get_date() - # data["itemSource"] = obj.get_source() + data = {'version': 0, + 'type': 'tracker:match', + 'ail_uuid': get_ail_uuid(), + 'tracker': { + 'uuid': tracker.get_uuid(), + 'type': tracker.get_type(), + 'tags': list(tracker.get_tags()), + 'tracker': tracker.get_tracked(), + }, + 'obj': {'type': obj.get_type(), + 'subtype': obj.get_subtype(r_str=True), + 'id': obj.get_id(), + 'tags': list(obj.get_tags()), + 'url': obj.get_link() + }, + } + if matches: + data['matches'] = matches + # data = json.dumps(data) self._export(data) diff --git a/bin/trackers/Tracker_Regex.py b/bin/trackers/Tracker_Regex.py index c23d1589..9d5f8418 100755 --- a/bin/trackers/Tracker_Regex.py +++ b/bin/trackers/Tracker_Regex.py @@ -133,7 +133,9 @@ class Tracker_Regex(AbstractModule): self.exporters['mail'].export(tracker, obj, matches) if tracker.webhook_export(): - self.exporters['webhook'].export(tracker, obj) + if not matches: + matches = self.extract_matches(re_matches) + self.exporters['webhook'].export(tracker, obj, matches) if __name__ == "__main__": diff --git a/bin/trackers/Tracker_Yara.py b/bin/trackers/Tracker_Yara.py index 29542553..0bf6db4e 100755 --- a/bin/trackers/Tracker_Yara.py +++ b/bin/trackers/Tracker_Yara.py @@ -151,7 +151,9 @@ class Tracker_Yara(AbstractModule): # Webhook if tracker.webhook_export(): - self.exporters['webhook'].export(tracker, self.obj) + if not matches: + matches = self.extract_matches(data) + self.exporters['webhook'].export(tracker, self.obj, matches) return yara.CALLBACK_CONTINUE