From 4552e4a033ece7dbe378afcdbc30b972e8b92500 Mon Sep 17 00:00:00 2001 From: Lesley De Keyser Date: Mon, 10 May 2021 16:51:06 +0200 Subject: [PATCH 1/4] Submit file contents to TheHive --- bin/MISP_The_Hive_feeder.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/bin/MISP_The_Hive_feeder.py b/bin/MISP_The_Hive_feeder.py index 41d18b43..9b555263 100755 --- a/bin/MISP_The_Hive_feeder.py +++ b/bin/MISP_The_Hive_feeder.py @@ -14,6 +14,8 @@ import uuid import redis import time import json +import binascii +import gzip from pubsublogger import publisher from Helper import Process @@ -56,7 +58,8 @@ import thehive4py.exceptions from thehive4py.models import Alert, AlertArtifact from thehive4py.models import Case, CaseTask, CustomFieldHelper - +def is_gzip_file(magic_nuber): + return binascii.hexlify(magic_nuber) == b'1f8b' def create_the_hive_alert(source, item_id, tag): # # TODO: check items status (processed by all modules) @@ -64,9 +67,26 @@ def create_the_hive_alert(source, item_id, tag): # # # TODO: description, add AIL link:show items ? tags = list( r_serv_metadata.smembers('tag:{}'.format(item_id)) ) + path = item_basic.get_item_filepath(item_id) + paste_handle = open(path, 'rb') + paste_data = paste_handle.read() + tmp_path = None + + if is_gzip_file(paste_data[0:2]): # if gzip, create a new file to supply to TheHive + paste_handle.close() # TheHive expects a file handle, that's why we create a new file + tmp_data = gzip.decompress(paste_data) + tmp_path = path + '.unzip' + with open(tmp_path, 'wb+') as f: + f.write(tmp_data) + paste_handle = open(tmp_path, 'rb') + if path.endswith(".gz"): # remove .gz from submitted path to TheHive beause we've decompressed it + path = path[:-3] + + path = os.path.basename(os.path.normpath(path)) + ".txt" # get last part of path, add .txt so it's easier to open when downloaded from TheHive + artifacts = [ AlertArtifact( dataType='uuid-ail', data=r_serv_db.get('ail:uuid') ), - AlertArtifact( dataType='file', data=item_basic.get_item_filepath(item_id), tags=tags ) + AlertArtifact( dataType='file', data=(paste_handle, path), tags=tags ) ] # Prepare the sample Alert @@ -95,6 +115,10 @@ def create_the_hive_alert(source, item_id, tag): except: print('hive connection error') + paste_handle.close() + if tmp_path is not None: # this file has been send to TheHive, we won't ever need it again + os.remove(tmp_path) + def feeder(message, count=0): if flag_the_hive or flag_misp: From 777eed3b98e96252527507f47ae25a37e1dc9e49 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 9 Jun 2021 12:57:30 +0200 Subject: [PATCH 2/4] fix: [UI crawler dashboard] block_domains_name_search: fix domains_types #110 --- var/www/blueprints/crawler_splash.py | 2 ++ var/www/templates/domains/block_domains_name_search.html | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/var/www/blueprints/crawler_splash.py b/var/www/blueprints/crawler_splash.py index 9ad499e0..66511827 100644 --- a/var/www/blueprints/crawler_splash.py +++ b/var/www/blueprints/crawler_splash.py @@ -281,9 +281,11 @@ def domains_search_name(): page = int(page) except: page = 1 + domains_types = request.args.getlist('domain_types') if domains_types: domains_types = domains_types[0].split(',') + domains_types = Domain.sanitize_domain_types(domains_types) l_dict_domains = Domain.api_search_domains_by_name(name, domains_types, domains_metadata=True, page=page) return render_template("domains/domains_result_list.html", template_folder='../../', diff --git a/var/www/templates/domains/block_domains_name_search.html b/var/www/templates/domains/block_domains_name_search.html index ca5e7cbe..6fabc144 100644 --- a/var/www/templates/domains/block_domains_name_search.html +++ b/var/www/templates/domains/block_domains_name_search.html @@ -13,13 +13,13 @@
- +
- + From dfb8ea4378f459f0f05214d340cd58f8f74aa294 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 9 Jun 2021 15:09:06 +0200 Subject: [PATCH 3/4] fix: [UI crawler dashboard] block_languages_search: fix domains_types #110 --- var/www/blueprints/crawler_splash.py | 5 ++++- var/www/templates/domains/block_languages_search.html | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/var/www/blueprints/crawler_splash.py b/var/www/blueprints/crawler_splash.py index 66511827..8fe3ed13 100644 --- a/var/www/blueprints/crawler_splash.py +++ b/var/www/blueprints/crawler_splash.py @@ -260,9 +260,12 @@ def domains_search_languages_get(): page = int(page) except: page = 1 + domains_types = request.args.getlist('domain_types') if domains_types: domains_types = domains_types[0].split(',') + domains_types = Domain.sanitize_domain_types(domains_types) + languages = request.args.getlist('languages') if languages: languages = languages[0].split(',') @@ -281,7 +284,7 @@ def domains_search_name(): page = int(page) except: page = 1 - + domains_types = request.args.getlist('domain_types') if domains_types: domains_types = domains_types[0].split(',') diff --git a/var/www/templates/domains/block_languages_search.html b/var/www/templates/domains/block_languages_search.html index 338e91d0..46f3822c 100644 --- a/var/www/templates/domains/block_languages_search.html +++ b/var/www/templates/domains/block_languages_search.html @@ -17,13 +17,13 @@
- +
- + From 039e70dc0b849c9643015a018d82cbaed2cfea23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thirion=20Aur=C3=A9lien?= Date: Wed, 9 Jun 2021 17:18:04 +0200 Subject: [PATCH 4/4] chg: [test] add new workflow --- .github/workflows/ail_framework_test.yml | 47 ++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/ail_framework_test.yml diff --git a/.github/workflows/ail_framework_test.yml b/.github/workflows/ail_framework_test.yml new file mode 100644 index 00000000..c082c0a9 --- /dev/null +++ b/.github/workflows/ail_framework_test.yml @@ -0,0 +1,47 @@ +# This is a basic workflow to help you get started with Actions + +name: CI + +# Controls when the action will run. +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [ master, dev ] + pull_request: + branches: [ master, dev ] + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + ail_test: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + with: + submodules: 'recursive' + + + # Runs a single command using the runners shell + - name: Install AIL + run: bash installing_deps.sh + + # Runs a set of commands using the runners shell + - name: Launch AIL + run: | + pushd bin + bash LAUNCH.sh -l + + # Runs a set of commands using the runners shell + - name: Run tests + run: bash LAUNCH.sh -t +