diff --git a/.github/workflows/ail_framework_test.yml b/.github/workflows/ail_framework_test.yml new file mode 100644 index 00000000..c082c0a9 --- /dev/null +++ b/.github/workflows/ail_framework_test.yml @@ -0,0 +1,47 @@ +# This is a basic workflow to help you get started with Actions + +name: CI + +# Controls when the action will run. +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [ master, dev ] + pull_request: + branches: [ master, dev ] + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + ail_test: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + with: + submodules: 'recursive' + + + # Runs a single command using the runners shell + - name: Install AIL + run: bash installing_deps.sh + + # Runs a set of commands using the runners shell + - name: Launch AIL + run: | + pushd bin + bash LAUNCH.sh -l + + # Runs a set of commands using the runners shell + - name: Run tests + run: bash LAUNCH.sh -t + diff --git a/bin/MISP_The_Hive_feeder.py b/bin/MISP_The_Hive_feeder.py index 41d18b43..9b555263 100755 --- a/bin/MISP_The_Hive_feeder.py +++ b/bin/MISP_The_Hive_feeder.py @@ -14,6 +14,8 @@ import uuid import redis import time import json +import binascii +import gzip from pubsublogger import publisher from Helper import Process @@ -56,7 +58,8 @@ import thehive4py.exceptions from thehive4py.models import Alert, AlertArtifact from thehive4py.models import Case, CaseTask, CustomFieldHelper - +def is_gzip_file(magic_nuber): + return binascii.hexlify(magic_nuber) == b'1f8b' def create_the_hive_alert(source, item_id, tag): # # TODO: check items status (processed by all modules) @@ -64,9 +67,26 @@ def create_the_hive_alert(source, item_id, tag): # # # TODO: description, add AIL link:show items ? tags = list( r_serv_metadata.smembers('tag:{}'.format(item_id)) ) + path = item_basic.get_item_filepath(item_id) + paste_handle = open(path, 'rb') + paste_data = paste_handle.read() + tmp_path = None + + if is_gzip_file(paste_data[0:2]): # if gzip, create a new file to supply to TheHive + paste_handle.close() # TheHive expects a file handle, that's why we create a new file + tmp_data = gzip.decompress(paste_data) + tmp_path = path + '.unzip' + with open(tmp_path, 'wb+') as f: + f.write(tmp_data) + paste_handle = open(tmp_path, 'rb') + if path.endswith(".gz"): # remove .gz from submitted path to TheHive beause we've decompressed it + path = path[:-3] + + path = os.path.basename(os.path.normpath(path)) + ".txt" # get last part of path, add .txt so it's easier to open when downloaded from TheHive + artifacts = [ AlertArtifact( dataType='uuid-ail', data=r_serv_db.get('ail:uuid') ), - AlertArtifact( dataType='file', data=item_basic.get_item_filepath(item_id), tags=tags ) + AlertArtifact( dataType='file', data=(paste_handle, path), tags=tags ) ] # Prepare the sample Alert @@ -95,6 +115,10 @@ def create_the_hive_alert(source, item_id, tag): except: print('hive connection error') + paste_handle.close() + if tmp_path is not None: # this file has been send to TheHive, we won't ever need it again + os.remove(tmp_path) + def feeder(message, count=0): if flag_the_hive or flag_misp: diff --git a/var/www/blueprints/crawler_splash.py b/var/www/blueprints/crawler_splash.py index 9ad499e0..8fe3ed13 100644 --- a/var/www/blueprints/crawler_splash.py +++ b/var/www/blueprints/crawler_splash.py @@ -260,9 +260,12 @@ def domains_search_languages_get(): page = int(page) except: page = 1 + domains_types = request.args.getlist('domain_types') if domains_types: domains_types = domains_types[0].split(',') + domains_types = Domain.sanitize_domain_types(domains_types) + languages = request.args.getlist('languages') if languages: languages = languages[0].split(',') @@ -281,9 +284,11 @@ def domains_search_name(): page = int(page) except: page = 1 + domains_types = request.args.getlist('domain_types') if domains_types: domains_types = domains_types[0].split(',') + domains_types = Domain.sanitize_domain_types(domains_types) l_dict_domains = Domain.api_search_domains_by_name(name, domains_types, domains_metadata=True, page=page) return render_template("domains/domains_result_list.html", template_folder='../../', diff --git a/var/www/templates/domains/block_domains_name_search.html b/var/www/templates/domains/block_domains_name_search.html index ca5e7cbe..6fabc144 100644 --- a/var/www/templates/domains/block_domains_name_search.html +++ b/var/www/templates/domains/block_domains_name_search.html @@ -13,13 +13,13 @@
- +
- + diff --git a/var/www/templates/domains/block_languages_search.html b/var/www/templates/domains/block_languages_search.html index 338e91d0..46f3822c 100644 --- a/var/www/templates/domains/block_languages_search.html +++ b/var/www/templates/domains/block_languages_search.html @@ -17,13 +17,13 @@
- +
- +