From 79a187b039deb2cf9aa39f0aae6bf482635decf6 Mon Sep 17 00:00:00 2001 From: terrtia Date: Tue, 5 Nov 2024 14:10:30 +0100 Subject: [PATCH] chg: [barcode] add and extract barcode object --- bin/LAUNCH.sh | 2 +- bin/lib/ail_core.py | 17 +- bin/lib/chats_viewer.py | 22 +- bin/lib/correlations_engine.py | 17 +- bin/lib/objects/BarCodes.py | 163 +++++ bin/lib/objects/Messages.py | 26 +- bin/lib/objects/ail_objects.py | 3 + .../{QrCodeReader.py => CodeReader.py} | 71 +- configs/modules.cfg | 2 +- tools/reprocess_objects.py | 7 +- var/www/Flask_server.py | 2 + var/www/blueprints/objects_barcode.py | 99 +++ .../chats_explorer/block_message.html | 15 + .../correlation/show_correlation.html | 2 + .../objects/barcode/BarcodeDaterange.html | 604 ++++++++++++++++++ .../objects/barcode/ShowBarcode.html | 68 ++ .../objects/barcode/card_barcode.html | 83 +++ .../templates/sidebars/sidebar_objects.html | 6 + 18 files changed, 1161 insertions(+), 48 deletions(-) create mode 100755 bin/lib/objects/BarCodes.py rename bin/modules/{QrCodeReader.py => CodeReader.py} (63%) create mode 100644 var/www/blueprints/objects_barcode.py create mode 100644 var/www/templates/objects/barcode/BarcodeDaterange.html create mode 100644 var/www/templates/objects/barcode/ShowBarcode.html create mode 100644 var/www/templates/objects/barcode/card_barcode.html diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index edb5b853..6c64fea2 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -280,7 +280,7 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "OcrExtractor" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./OcrExtractor.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "QrCodeReader" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./QrCodeReader.py; read x" + screen -S "Script_AIL" -X screen -t "CodeReader" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./CodeReader.py; read x" sleep 0.1 # TITLES diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index 6e22f795..1af20e30 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -16,16 +16,17 @@ r_serv_db = config_loader.get_db_conn("Kvrocks_DB") r_object = config_loader.get_db_conn("Kvrocks_Objects") config_loader = None -AIL_OBJECTS = sorted({'chat', 'chat-subchannel', 'chat-thread', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', - 'domain', 'dom-hash', 'etag', 'favicon', 'file-name', 'hhhash','item', 'image', 'message', 'ocr', - 'pgp', 'qrcode', 'screenshot', 'title', 'user-account', 'username'}) +AIL_OBJECTS = sorted({'barcode', 'chat', 'chat-subchannel', 'chat-thread', 'cookie-name', 'cve', 'cryptocurrency', + 'decoded', 'domain', 'dom-hash', 'etag', 'favicon', 'file-name', 'hhhash','item', 'image', + 'message', 'ocr', 'pgp', 'qrcode', 'screenshot', 'title', 'user-account', 'username'}) AIL_OBJECTS_WITH_SUBTYPES = {'chat', 'chat-subchannel', 'cryptocurrency', 'pgp', 'username', 'user-account'} # TODO by object TYPE ???? -AIL_OBJECTS_CORRELATIONS_DEFAULT = sorted({'chat', 'chat-subchannel', 'chat-thread', 'cve', 'cryptocurrency', 'decoded', - 'domain', 'dom-hash', 'favicon', 'file-name', 'item', 'image', 'message', - 'ocr', 'pgp', 'qrcode', 'screenshot', 'title', 'user-account', 'username'}) +AIL_OBJECTS_CORRELATIONS_DEFAULT = sorted({'barcode', 'chat', 'chat-subchannel', 'chat-thread', 'cve', 'cryptocurrency', + 'decoded', 'domain', 'dom-hash', 'favicon', 'file-name', 'item', 'image', + 'message', 'ocr', 'pgp', 'qrcode', 'screenshot', 'title', 'user-account', + 'username'}) def get_ail_uuid(): ail_uuid = r_serv_db.get('ail:uuid') @@ -86,10 +87,10 @@ def get_default_correlation_objects(): return AIL_OBJECTS_CORRELATIONS_DEFAULT def get_obj_queued(): - return ['item', 'image', 'message', 'ocr', 'qrcode'] # screenshot ??? + return ['barcode', 'item', 'image', 'message', 'ocr', 'qrcode'] # screenshot ??? def get_objects_tracked(): - return ['decoded', 'item', 'pgp', 'message', 'ocr', 'qrcode', 'title'] + return ['barcode', 'decoded', 'item', 'pgp', 'message', 'ocr', 'qrcode', 'title'] def get_objects_retro_hunted(): return ['decoded', 'item', 'message'] diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 6a910172..4c8cea5e 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -23,6 +23,7 @@ from lib.objects import Chats from lib.objects import ChatSubChannels from lib.objects import ChatThreads from lib.objects import Messages +from lib.objects.BarCodes import Barcode from lib.objects.QrCodes import Qrcode from lib.objects import UsersAccount from lib.objects import Usernames @@ -429,6 +430,12 @@ def get_chat_object_messages_meta(c_messages): temp_chats[meta['forwarded_from']] = chat.get_meta({'icon'}) else: meta['forwarded_from'] = temp_chats[meta['forwarded_from']] + if meta['barcodes']: + barcodes = [] + for q in meta['barcodes']: + obj = Barcode(q) + barcodes.append({'id': obj.id, 'content': obj.get_content(), 'tags': obj.get_tags()}) + meta['qrcodes'] = barcodes if meta['qrcodes']: qrcodes = [] for q in meta['qrcodes']: @@ -746,6 +753,14 @@ def api_get_chat_service_instance(chat_instance_uuid): # return chat_instance.get_meta({'chats'}), 200 return chat_instance.get_meta({'chats_with_messages'}), 200 +def api_get_chats_selector(): + selector = [] + for instance_uuid in get_chat_service_instances(): + for chat_id in ChatServiceInstance(instance_uuid).get_chats(): + chat = Chats.Chat(chat_id, instance_uuid) + selector.append({'id': chat.get_global_id(), 'name': f'{chat.get_chat_instance()}: {chat.get_label()}'}) + return selector + def api_get_chat(chat_id, chat_instance_uuid, translation_target=None, nb=-1, page=-1, messages=True): chat = Chats.Chat(chat_id, chat_instance_uuid) if not chat.exists(): @@ -826,10 +841,15 @@ def api_get_message(message_id, translation_target=None): message = Messages.Message(message_id) if not message.exists(): return {"status": "error", "reason": "Unknown uuid"}, 404 - meta = message.get_meta({'chat', 'content', 'files-names', 'forwarded_from', 'icon', 'images', 'language', 'link', 'parent', 'parent_meta', 'qrcodes', 'reactions', 'thread', 'translation', 'user-account'}, translation_target=translation_target) + meta = message.get_meta({'barcodes', 'chat', 'content', 'files-names', 'forwarded_from', 'icon', 'images', 'language', 'link', 'parent', 'parent_meta', 'qrcodes', 'reactions', 'thread', 'translation', 'user-account'}, translation_target=translation_target) if 'forwarded_from' in meta: chat = get_obj_chat_from_global_id(meta['forwarded_from']) meta['forwarded_from'] = chat.get_meta({'icon'}) + barcodes = [] + for q in meta['barcodes']: + obj = Barcode(q) + barcodes.append({'id': obj.id, 'content': obj.get_content(), 'tags': obj.get_tags()}) + meta['barcodes'] = barcodes qrcodes = [] for q in meta['qrcodes']: qr = Qrcode(q) diff --git a/bin/lib/correlations_engine.py b/bin/lib/correlations_engine.py index 9cf8b5d2..cc14c01f 100755 --- a/bin/lib/correlations_engine.py +++ b/bin/lib/correlations_engine.py @@ -41,26 +41,27 @@ config_loader = None ################################## CORRELATION_TYPES_BY_OBJ = { - "chat": ["chat-subchannel", "chat-thread", "image", "message", "ocr", "user-account"], # message or direct correlation like cve, bitcoin, ... ??? + "barcode": ["chat", "cve", "cryptocurrency", "decoded", "domain", "image", "message", "screenshot"], + "chat": ["barcode", "chat-subchannel", "chat-thread", "image", "message", "ocr", "user-account"], # message or direct correlation like cve, bitcoin, ... ??? "chat-subchannel": ["chat", "chat-thread", "image", "message", "ocr", "user-account"], "chat-thread": ["chat", "chat-subchannel", "image", "message", "ocr", "user-account"], "cookie-name": ["domain"], - "cryptocurrency": ["domain", "item", "message", "ocr", "qrcode"], - "cve": ["domain", "item", "message", "ocr", "qrcode"], - "decoded": ["domain", "item", "message", "ocr", "qrcode"], - "domain": ["cve", "cookie-name", "cryptocurrency", "dom-hash", "decoded", "etag", "favicon", "hhhash", "item", "pgp", "title", "screenshot", "username"], + "cryptocurrency": ["barcode", "domain", "item", "message", "ocr", "qrcode"], + "cve": ["barcode", "domain", "item", "message", "ocr", "qrcode"], + "decoded": ["barcode", "domain", "item", "message", "ocr", "qrcode"], + "domain": ["barcode", "cve", "cookie-name", "cryptocurrency", "dom-hash", "decoded", "etag", "favicon", "hhhash", "item", "pgp", "title", "screenshot", "username"], "dom-hash": ["domain", "item"], "etag": ["domain"], "favicon": ["domain", "item"], # TODO Decoded "file-name": ["chat", "message"], "hhhash": ["domain"], - "image": ["chat", "chat-subchannel", "chat-thread", "message", "ocr", "qrcode", "user-account"], # TODO subchannel + threads ???? + "image": ["barcode", "chat", "chat-subchannel", "chat-thread", "message", "ocr", "qrcode", "user-account"], # TODO subchannel + threads ???? "item": ["cve", "cryptocurrency", "decoded", "domain", "dom-hash", "favicon", "pgp", "screenshot", "title", "username"], # chat ??? - "message": ["chat", "chat-subchannel", "chat-thread", "cve", "cryptocurrency", "decoded", "file-name", "image", "ocr", "pgp", "user-account"], + "message": ["barcode", "chat", "chat-subchannel", "chat-thread", "cve", "cryptocurrency", "decoded", "file-name", "image", "ocr", "pgp", "user-account"], "ocr": ["chat", "chat-subchannel", "chat-thread", "cve", "cryptocurrency", "decoded", "image", "message", "pgp", "user-account"], "pgp": ["domain", "item", "message", "ocr"], "qrcode": ["chat", "cve", "cryptocurrency", "decoded", "domain", "image", "message", "screenshot"], # "chat-subchannel", "chat-thread" ????? - "screenshot": ["domain", "item", "qrcode"], + "screenshot": ["barcode", "domain", "item", "qrcode"], "title": ["domain", "item"], "user-account": ["chat", "chat-subchannel", "chat-thread", "image", "message", "ocr", "username"], "username": ["domain", "item", "message", "user-account"], diff --git a/bin/lib/objects/BarCodes.py b/bin/lib/objects/BarCodes.py new file mode 100755 index 00000000..80d4842d --- /dev/null +++ b/bin/lib/objects/BarCodes.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys + +from hashlib import sha256 +from pymisp import MISPObject + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib.objects.abstract_daterange_object import AbstractDaterangeObject, AbstractDaterangeObjects +from lib.ConfigLoader import ConfigLoader +from packages import Date +# from lib.data_retention_engine import update_obj_date, get_obj_date_first + +from flask import url_for + +config_loader = ConfigLoader() +r_object = config_loader.get_db_conn("Kvrocks_Objects") +r_cache = config_loader.get_redis_conn("Redis_Cache") +baseurl = config_loader.get_config_str("Notifications", "ail_domain") +IMAGE_FOLDER = config_loader.get_files_directory('images') +config_loader = None + +# SET x1,y1:x2,y2:x3,y3:x4,y4:extracted_text + +class Barcode(AbstractDaterangeObject): + """ + AIL Message Object. (strings) + """ + + def __init__(self, id): + super(Barcode, self).__init__('barcode', id) + + def get_content(self, r_type='str'): + """ + Returns content + """ + global_id = self.get_global_id() + content = r_cache.get(f'content:{global_id}') + if not content: + content = self._get_field('content') + # Set Cache + if content: + global_id = self.get_global_id() + r_cache.set(f'content:{global_id}', content) + r_cache.expire(f'content:{global_id}', 300) + if r_type == 'str': + return content + elif r_type == 'bytes': + if content: + return content.encode() + + def get_date(self): # TODO + return Date.get_today_date_str() + + def get_nb_seen(self): + return self.get_nb_correlation('image') + self.get_nb_correlation('screenshot') + + def get_source(self): # TODO + """ + Returns source/feeder name + """ + return 'barcode' + + def get_basename(self): # TODO + return 'barcode' + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('correlation.show_correlation', type=self.type, id=self.id) + else: + url = f'{baseurl}/correlation/show?type={self.type}&id={self.id}' + return url + + def get_svg_icon(self): # TODO + return {'style': 'fas', 'icon': '\uf02a', 'color': 'yellow', 'radius': 5} + + def get_misp_object(self): # TODO + pass + # obj = MISPObject('instant-message', standalone=True) + # obj_date = self.get_date() + # if obj_date: + # obj.first_seen = obj_date + # else: + # self.logger.warning( + # f'Export error, None seen {self.type}:{self.subtype}:{self.id}, first={obj_date}') + # + # # obj_attrs = [obj.add_attribute('first-seen', value=obj_date), + # # obj.add_attribute('raw-data', value=self.id, data=self.get_raw_content()), + # # obj.add_attribute('sensor', value=get_ail_uuid())] + # obj_attrs = [] + # for obj_attr in obj_attrs: + # for tag in self.get_tags(): + # obj_attr.add_tag(tag) + # return obj + + # options: set of optional meta fields + def get_meta(self, options=None): + """ + :type options: set + """ + if options is None: + options = set() + meta = self._get_meta(options=options) + meta['tags'] = self.get_tags() + meta['content'] = self.get_content() + + # optional meta fields + if 'investigations' in options: + meta['investigations'] = self.get_investigations() + if 'link' in options: + meta['link'] = self.get_link(flask_context=True) + if 'icon' in options: + meta['svg_icon'] = self.get_svg_icon() + return meta + + def create(self, content, im_obj, tags=[]): + self._set_field('content', content) + if im_obj.type == 'screenshot': + for date in im_obj.get_dates(): + self._add(date, None) + else: + self._copy_from(im_obj.type, im_obj.get_id()) + + for tag in tags: + self.add_tag(tag) + return self.id + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + r_object.delete(f'barcode:{self.id}') + + +def create(content, im_obj, tags=[]): + if content: + obj_id = sha256(content.encode()).hexdigest() + obj = Barcode(obj_id) + if not obj.exists(): + obj.create(content, im_obj, tags=tags) + return obj + +class Barcodes(AbstractDaterangeObjects): + """ + Barcodes Objects + """ + def __init__(self): + super().__init__('barcode', Barcode) + + def sanitize_id_to_search(self, name_to_search): + return name_to_search # TODO + + +#### API #### +def api_get_barcode(obj_id): + obj = Barcode(obj_id) + if not obj.exists(): + return {"status": "error", "reason": "Unknown barcode"}, 404 + meta = obj.get_meta({'content', 'icon', 'link'}) + return meta, 200 diff --git a/bin/lib/objects/Messages.py b/bin/lib/objects/Messages.py index ee4879db..f9c9288c 100755 --- a/bin/lib/objects/Messages.py +++ b/bin/lib/objects/Messages.py @@ -151,6 +151,12 @@ class Message(AbstractObject): images.append({'id': obj_id, 'ocr': self._get_image_ocr(obj_id)}) return images + def get_barcodes(self): + barcodes = [] + for c in self.get_correlation('barcode').get('barcode', []): + barcodes.append(c[1:]) + return barcodes + def get_qrcodes(self): qrcodes = [] for c in self.get_correlation('qrcode').get('qrcode', []): @@ -306,6 +312,8 @@ class Message(AbstractObject): meta['thread'] = thread if 'images' in options: meta['images'] = self.get_images() + if 'barcodes' in options: + meta['barcodes'] = self.get_barcodes() if 'qrcodes' in options: meta['qrcodes'] = self.get_qrcodes() if 'files-names' in options: @@ -336,16 +344,20 @@ class Message(AbstractObject): ## Language ## - def get_objs_container(self): + def get_root_obj(self): + return self.get_objs_container(root=True) + + def get_objs_container(self, root=False): objs_containers = set() # chat objs_containers.add(self.get_chat()) - subchannel = self.get_subchannel() - if subchannel: - objs_containers.add(subchannel) - thread = self.get_current_thread() - if thread: - objs_containers.add(thread) + if not root: + subchannel = self.get_subchannel() + if subchannel: + objs_containers.add(subchannel) + thread = self.get_current_thread() + if thread: + objs_containers.add(thread) return objs_containers #- Language -# diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py index 9823e928..41259c4d 100755 --- a/bin/lib/objects/ail_objects.py +++ b/bin/lib/objects/ail_objects.py @@ -19,6 +19,7 @@ from lib import Tag from lib import chats_viewer +from lib.objects import BarCodes from lib.objects import Chats from lib.objects import ChatSubChannels from lib.objects import ChatThreads @@ -102,6 +103,8 @@ def get_object(obj_type, subtype, obj_id): return Messages.Message(obj_id) elif obj_type == 'ocr': return Ocrs.Ocr(obj_id) + elif obj_type == 'barcode': + return BarCodes.Barcode(obj_id) elif obj_type == 'qrcode': return QrCodes.Qrcode(obj_id) elif obj_type == 'screenshot': diff --git a/bin/modules/QrCodeReader.py b/bin/modules/CodeReader.py similarity index 63% rename from bin/modules/QrCodeReader.py rename to bin/modules/CodeReader.py index 6a8a863a..090a9c40 100755 --- a/bin/modules/QrCodeReader.py +++ b/bin/modules/CodeReader.py @@ -22,16 +22,17 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from modules.abstract_module import AbstractModule from lib.ConfigLoader import ConfigLoader +from lib.objects import BarCodes from lib.objects import QrCodes -class QrCodeReader(AbstractModule): +class CodeReader(AbstractModule): """ QrCodeReader for AIL framework """ def __init__(self): - super(QrCodeReader, self).__init__() + super(CodeReader, self).__init__() # Waiting time in seconds between to message processed self.pending_seconds = 1 @@ -39,6 +40,8 @@ class QrCodeReader(AbstractModule): config_loader = ConfigLoader() self.r_cache = config_loader.get_redis_conn("Redis_Cache") + self.barcode_type = {'CODABAR', 'CODE39', 'CODE93', 'CODE128', 'EAN8', 'EAN13', 'I25'} # 2 - 5 + # Send module state to logs self.logger.info(f'Module {self.module_name} initialized') @@ -48,26 +51,35 @@ class QrCodeReader(AbstractModule): def add_to_cache(self): self.r_cache.setex(f'qrcode:no:{self.obj.type}:{self.obj.id}', 86400, 0) - def extract_qrcode(self, path): + def extract_codes(self, path): + barcodes = [] + qrcodes = [] qr_codes = False - contents = [] try: image = cv2.cvtColor(cv2.imread(path), cv2.COLOR_BGR2RGB) except cv2.error: self.logger.warning(f'Invalid image: {self.obj.get_global_id()}') - return False, [] + return [], [] try: decodeds = decode(image) for decoded in decodeds: + # print(decoded) if decoded.type == 'QRCODE': qr_codes = True if decoded.data: - contents.append(decoded.data.decode()) + qrcodes.append(decoded.data.decode()) + elif decoded.type in self.barcode_type: + if decoded.data: + rect = decoded.rect + if rect.width and rect.height and decoded.quality > 1: + barcodes.append(decoded.data.decode()) + elif decoded.type: + self.logger.error(f'Unsupported pyzbar code type {decoded.type}: {self.obj.get_global_id()}') except ValueError as e: self.logger.error(f'{e}: {self.obj.get_global_id()}') - if not contents: + if not qrcodes: detector = cv2.QRCodeDetector() try: qr, decodeds, qarray, _ = detector.detectAndDecodeMulti(image) @@ -75,28 +87,31 @@ class QrCodeReader(AbstractModule): qr_codes = True for d in decodeds: if d: - contents.append(d) + qrcodes.append(d) except cv2.error as e: self.logger.error(f'{e}: {self.obj.get_global_id()}') try: data_qr, box, qrcode_image = detector.detectAndDecode(image) if data_qr: - contents.append(data_qr) + qrcodes.append(data_qr) qr_codes = True except cv2.error as e: self.logger.error(f'{e}: {self.obj.get_global_id()}') - if qr_codes and not contents: + if qr_codes and not qrcodes: # # # # 0.5s per image try: qreader = QReader() decoded_text = qreader.detect_and_decode(image=image) for d in decoded_text: - contents.append(d) + qrcodes.append(d) qr_codes = True except ValueError as e: self.logger.error(f'{e}: {self.obj.get_global_id()}') - return qr_codes, contents + if not qr_codes: + self.logger.warning(f'Can notextract qr code: {self.obj.get_global_id()}') + + return barcodes, qrcodes def compute(self, message): obj = self.get_obj() @@ -111,13 +126,12 @@ class QrCodeReader(AbstractModule): # image - screenshot path = self.obj.get_filepath() - is_qrcode, contents = self.extract_qrcode(path) - if not contents: - # print('no qr code detected') + barcodes, qrcodes = self.extract_codes(path) + if not barcodes and not qrcodes: self.add_to_cache() return None - for content in contents: + for content in qrcodes: if not content: continue print(content) @@ -134,11 +148,32 @@ class QrCodeReader(AbstractModule): # TODO only if new ??? self.add_message_to_queue(obj=qr_code, queue='Item') - if is_qrcode or contents: + for content in barcodes: + if not content: + continue + print(content) + barcode = BarCodes.create(content, self.obj) # copy screenshot + image daterange + if not barcode: + print('Error Empty content', self.obj.get_global_id()) + barcode.add(barcode.get_date(), self.obj) + + for obj_type in ['chat', 'domain', 'message']: # TODO ITEM ??? + for c_id in self.obj.get_correlation(obj_type).get(obj_type, []): + o_subtype, o_id = c_id.split(':', 1) + barcode.add_correlation(obj_type, o_subtype, o_id) + + self.add_message_to_queue(obj=barcode, queue='Item') + + if qrcodes: tag = 'infoleak:automatic-detection="qrcode"' self.add_message_to_queue(obj=self.obj, message=tag, queue='Tags') + if barcodes: + tag = 'infoleak:automatic-detection="barcode"' + self.add_message_to_queue(obj=self.obj, message=tag, queue='Tags') + if __name__ == '__main__': - module = QrCodeReader() + module = CodeReader() module.run() + \ No newline at end of file diff --git a/configs/modules.cfg b/configs/modules.cfg index 1639c288..cff40488 100644 --- a/configs/modules.cfg +++ b/configs/modules.cfg @@ -168,7 +168,7 @@ publish = Item ######## IMAGES ######## images + screenshots -[QrCodeReader] +[CodeReader] subscribe = Images publish = Item,Tags diff --git a/tools/reprocess_objects.py b/tools/reprocess_objects.py index a2505a19..683afb4d 100755 --- a/tools/reprocess_objects.py +++ b/tools/reprocess_objects.py @@ -30,16 +30,15 @@ from lib.objects import ail_objects # from modules.Telegram import Telegram from modules.CEDetector import CEDetector +from modules.CodeReader import CodeReader from modules.Languages import Languages from modules.OcrExtractor import OcrExtractor -from modules.QrCodeReader import QrCodeReader MODULES = { 'CEDetector': CEDetector, + 'CodeReader': CodeReader, 'Languages': Languages, - 'OcrExtractor': OcrExtractor, - 'QrCodeReader': QrCodeReader - + 'OcrExtractor': OcrExtractor } def reprocess_message_objects(object_type, module_name=None): diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 6890874f..ec779a92 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -58,6 +58,7 @@ from blueprints.objects_dom_hash import objects_dom_hash from blueprints.chats_explorer import chats_explorer from blueprints.objects_image import objects_image from blueprints.objects_ocr import objects_ocr +from blueprints.objects_barcode import objects_barcode from blueprints.objects_qrcode import objects_qrcode from blueprints.objects_favicon import objects_favicon from blueprints.api_rest import api_rest @@ -143,6 +144,7 @@ app.register_blueprint(objects_dom_hash, url_prefix=baseUrl) app.register_blueprint(chats_explorer, url_prefix=baseUrl) app.register_blueprint(objects_image, url_prefix=baseUrl) app.register_blueprint(objects_ocr, url_prefix=baseUrl) +app.register_blueprint(objects_barcode, url_prefix=baseUrl) app.register_blueprint(objects_qrcode, url_prefix=baseUrl) app.register_blueprint(objects_favicon, url_prefix=baseUrl) app.register_blueprint(api_rest, url_prefix=baseUrl) diff --git a/var/www/blueprints/objects_barcode.py b/var/www/blueprints/objects_barcode.py new file mode 100644 index 00000000..284e873b --- /dev/null +++ b/var/www/blueprints/objects_barcode.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... +''' + +import json +import os +import sys + +from io import BytesIO + +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file, send_from_directory +from flask_login import login_required + +# Import Role_Manager +from Role_Manager import login_admin, login_read_only, no_cache + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from lib import Language +from lib import Tag +from lib.objects import BarCodes + +from packages import Date + +# ============ BLUEPRINT ============ +objects_barcode = Blueprint('objects_barcode', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/barcode')) + +# ============ VARIABLES ============ +bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] + +def create_json_response(data, status_code): + return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code + +# ============ FUNCTIONS ============ + +@objects_barcode.route("/objects/barcodes", methods=['GET']) +@login_required +@login_read_only +def objects_barcodes(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + show_objects = request.args.get('show_objects') + date = Date.sanitise_date_range(date_from, date_to) + date_from = date['date_from'] + date_to = date['date_to'] + + if show_objects: + dict_objects = BarCodes.Barcodes().api_get_meta_by_daterange(date_from, date_to) + else: + dict_objects = {} + + return render_template("BarcodeDaterange.html", date_from=date_from, date_to=date_to, + dict_objects=dict_objects, show_objects=show_objects) + + +@objects_barcode.route("/objects/barcodes/post", methods=['POST']) +@login_required +@login_read_only +def objects_barcodes_post(): + date_from = request.form.get('date_from') + date_to = request.form.get('date_to') + show_objects = request.form.get('show_objects') + return redirect(url_for('objects_barcode.objects_barcodes', date_from=date_from, date_to=date_to, show_objects=show_objects)) + + +@objects_barcode.route("/objects/barcodes/range/json", methods=['GET']) +@login_required +@login_read_only +def objects_barcodes_range_json(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + date = Date.sanitise_date_range(date_from, date_to) + date_from = date['date_from'] + date_to = date['date_to'] + return jsonify(BarCodes.Barcodes().api_get_chart_nb_by_daterange(date_from, date_to)) + + +@objects_barcode.route("/objects/barcode", methods=['GET']) +@login_required +@login_read_only +def object_barcode(): + obj_id = request.args.get('id') + meta = BarCodes.api_get_barcode(obj_id) + if meta[1] != 200: + return create_json_response(meta[0], meta[1]) + else: + meta = meta[0] + return render_template("ShowBarcode.html", meta=meta, + bootstrap_label=bootstrap_label, + ail_tags=Tag.get_modal_add_tags(meta['id'], meta['type'], meta['subtype'])) + + +# ============= ROUTES ============== + diff --git a/var/www/templates/chats_explorer/block_message.html b/var/www/templates/chats_explorer/block_message.html index 849b8bc7..3ae742bc 100644 --- a/var/www/templates/chats_explorer/block_message.html +++ b/var/www/templates/chats_explorer/block_message.html @@ -97,6 +97,21 @@ OCR {% endif %} + {% if message['barcodes'] %} + {% for barcode in message['barcodes'] %} + + + {{ barcode['content'] }} +
+ {% for tag in barcode['tags'] %} + {{ tag }} + {% endfor %} +
+ + + + {% endfor %} + {% endif %} {% if message['qrcodes'] %} {% for qrcode in message['qrcodes'] %} diff --git a/var/www/templates/correlation/show_correlation.html b/var/www/templates/correlation/show_correlation.html index 30f66ab1..fd7608d6 100644 --- a/var/www/templates/correlation/show_correlation.html +++ b/var/www/templates/correlation/show_correlation.html @@ -134,6 +134,8 @@ {% include 'chats_explorer/card_image.html' %} {% elif dict_object["object_type"] == "ocr" %} {% include 'objects/ocr/card_ocr.html' %} + {% elif dict_object["object_type"] == "barcode" %} + {% include 'objects/barcode/card_barcode.html' %} {% elif dict_object["object_type"] == "qrcode" %} {% include 'objects/qrcode/card_qrcode.html' %} {% elif dict_object["object_type"] == "item" %} diff --git a/var/www/templates/objects/barcode/BarcodeDaterange.html b/var/www/templates/objects/barcode/BarcodeDaterange.html new file mode 100644 index 00000000..af528b34 --- /dev/null +++ b/var/www/templates/objects/barcode/BarcodeDaterange.html @@ -0,0 +1,604 @@ + + + + + Barcodes - AIL + + + + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'sidebars/sidebar_objects.html' %} + +
+ +
+
+
+ +{# {% include 'image/block_images_search.html' %}#} + +
+ + +
+ +
+
+
Select a date range :
+
+
+
+ +
+
+
+ +
+
+ + +
+ +
+
+
+ +
+
+
+
+
+
+ + {% if dict_objects %} + {% if date_from|string == date_to|string %} +

{{ date_from }} Barcodes Name:

+ {% else %} +

{{ date_from }} to {{ date_to }} Barcodes Name:

+ {% endif %} + + + + + + + + + + + + {% for obj_id in dict_objects %} + + + + + + + + {% endfor %} + +
First SeenLast SeenTotalLast days
+ {{ dict_objects[obj_id]['content'] }} + {{ dict_objects[obj_id]['first_seen'] }}{{ dict_objects[obj_id]['last_seen'] }}{{ dict_objects[obj_id]['nb_seen'] }}
+ + + {% else %} + {% if show_objects %} + {% if date_from|string == date_to|string %} +

{{ date_from }}, No Barcode

+ {% else %} +

{{ date_from }} to {{ date_to }}, No Barcode

+ {% endif %} + {% endif %} + {% endif %} +
+ +
+
+ + + + + + + + + + + + + + + + + diff --git a/var/www/templates/objects/barcode/ShowBarcode.html b/var/www/templates/objects/barcode/ShowBarcode.html new file mode 100644 index 00000000..7fc87b37 --- /dev/null +++ b/var/www/templates/objects/barcode/ShowBarcode.html @@ -0,0 +1,68 @@ + + + + + Barcode - AIL + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'sidebars/sidebar_objects.html' %} + +
+ + {% with meta=meta, is_correlation=False %} + {% include 'objects/barcode/card_barcode.html' %} + {% endwith %} + +
+ +
+
+ + + + + + diff --git a/var/www/templates/objects/barcode/card_barcode.html b/var/www/templates/objects/barcode/card_barcode.html new file mode 100644 index 00000000..f04c9204 --- /dev/null +++ b/var/www/templates/objects/barcode/card_barcode.html @@ -0,0 +1,83 @@ + + + +{% with modal_add_tags=ail_tags %} + {% include 'modals/add_tags.html' %} +{% endwith %} + +{% include 'modals/edit_tag.html' %} + + + +
+
+ {{ meta["id"] }} : +
{{ meta["content"] }}
+
    +
  • + + + + + + + + +
    + + + + {{ meta["svg_icon"]["icon"] }} + + + {{ meta['type'] }} + First Seen: {% if meta['first_seen'] %}{{ meta['first_seen'][0:4] }}-{{ meta['first_seen'][4:6] }}-{{ meta['first_seen'][6:8] }}{% endif %}Last Seen: {% if meta['last_seen'] %}{{ meta['last_seen'][0:4] }}-{{ meta['last_seen'][4:6] }}-{{ meta['last_seen'][6:8] }}{% endif %}
    +
    • +
  • + + +
    • + +
  • +
    + Tags: + {% for tag in meta['tags'] %} + + {% endfor %} + +
    +
    • +
+ + {% with obj_type='barcode', obj_id=meta['id'], obj_subtype='' %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + + + {% if is_correlation %} + + + + {% else %} + + + + {% endif %} + + +
+
\ No newline at end of file diff --git a/var/www/templates/sidebars/sidebar_objects.html b/var/www/templates/sidebars/sidebar_objects.html index b28f1313..a1fdadb2 100644 --- a/var/www/templates/sidebars/sidebar_objects.html +++ b/var/www/templates/sidebars/sidebar_objects.html @@ -94,6 +94,12 @@ OCR +
  • + + + Barcode + +