Merge branch 'master' into dev

.github/workflows/ail_framework_test.yml

@@ -0,0 +1,47 @@
+# This is a basic workflow to help you get started with Actions
+
+name: CI
+
+# Controls when the action will run. 
+on:
+  # Triggers the workflow on push or pull request events but only for the master branch
+  push:
+    branches: [ master, dev ]
+  pull_request:
+    branches: [ master, dev ]
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  ail_test:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+    
+    strategy:
+       matrix:
+         python-version: [3.6, 3.7, 3.8, 3.9]
+    
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+          
+          
+      # Runs a single command using the runners shell
+      - name: Install AIL
+        run: bash installing_deps.sh
+
+      # Runs a set of commands using the runners shell
+      - name: Launch AIL
+        run: |
+              pushd bin
+              bash LAUNCH.sh -l
+              
+      # Runs a set of commands using the runners shell
+      - name: Run tests
+        run: bash LAUNCH.sh -t
+      

bin/MISP_The_Hive_feeder.py

@@ -14,6 +14,8 @@ import uuid
 import redis
 import time
 import json
+import binascii
+import gzip
 
 from pubsublogger import publisher
 from Helper import Process
@@ -56,7 +58,8 @@ import thehive4py.exceptions
 from thehive4py.models import Alert, AlertArtifact
 from thehive4py.models import Case, CaseTask, CustomFieldHelper
 
-
+def is_gzip_file(magic_nuber):
+     return binascii.hexlify(magic_nuber) == b'1f8b'
 
 def create_the_hive_alert(source, item_id, tag):
     # # TODO: check items status (processed by all modules)
@@ -64,9 +67,26 @@ def create_the_hive_alert(source, item_id, tag):
     # # # TODO: description, add AIL link:show items ?
     tags = list( r_serv_metadata.smembers('tag:{}'.format(item_id)) )
 
+    path = item_basic.get_item_filepath(item_id)
+    paste_handle = open(path, 'rb')
+    paste_data = paste_handle.read()
+    tmp_path = None
+
+    if is_gzip_file(paste_data[0:2]): # if gzip, create a new file to supply to TheHive
+      paste_handle.close()            # TheHive expects a file handle, that's why we create a new file
+      tmp_data = gzip.decompress(paste_data)
+      tmp_path = path + '.unzip'
+      with open(tmp_path, 'wb+') as f:
+        f.write(tmp_data)
+      paste_handle = open(tmp_path, 'rb')
+      if path.endswith(".gz"): # remove .gz from submitted path to TheHive beause we've decompressed it
+        path = path[:-3]
+
+    path = os.path.basename(os.path.normpath(path)) + ".txt" # get last part of path, add .txt so it's easier to open when downloaded from TheHive
+
     artifacts = [
         AlertArtifact( dataType='uuid-ail', data=r_serv_db.get('ail:uuid') ),
-        AlertArtifact( dataType='file', data=item_basic.get_item_filepath(item_id), tags=tags )
+        AlertArtifact( dataType='file', data=(paste_handle, path), tags=tags )
     ]
 
     # Prepare the sample Alert
@@ -95,6 +115,10 @@ def create_the_hive_alert(source, item_id, tag):
     except:
         print('hive connection error')
 
+    paste_handle.close()
+    if tmp_path is not None: # this file has been send to TheHive, we won't ever need it again
+      os.remove(tmp_path)
+
 def feeder(message, count=0):
 
     if flag_the_hive or flag_misp:

var/www/blueprints/crawler_splash.py

@@ -260,9 +260,12 @@ def domains_search_languages_get():
         page = int(page)
     except:
         page = 1
+
     domains_types = request.args.getlist('domain_types')
     if domains_types:
         domains_types = domains_types[0].split(',')
+    domains_types = Domain.sanitize_domain_types(domains_types)
+
     languages = request.args.getlist('languages')
     if languages:
         languages = languages[0].split(',')
@@ -281,9 +284,11 @@ def domains_search_name():
         page = int(page)
     except:
         page = 1
+
     domains_types = request.args.getlist('domain_types')
     if domains_types:
         domains_types = domains_types[0].split(',')
+    domains_types = Domain.sanitize_domain_types(domains_types)
 
     l_dict_domains = Domain.api_search_domains_by_name(name, domains_types, domains_metadata=True, page=page)
     return render_template("domains/domains_result_list.html", template_folder='../../',

var/www/templates/domains/block_domains_name_search.html

@@ -13,13 +13,13 @@
 
 			<div class="mb-3">
 				<div class="custom-control custom-switch">
-					<input class="custom-control-input" type="checkbox" name="domain_onion_switch" value="" id="domain_onion_switch" {%if 'onion' in domains_types or not domains_types%}checked{%endif%}>
+					<input class="custom-control-input" type="checkbox" name="domain_onion_switch" value="" id="domain_onion_switch" {%if not domains_types%}checked{%elif 'onion' in domains_types%}checked{%endif%}>
 					<label class="custom-control-label" for="domain_onion_switch">
 						<span class="badge badge-danger"><i class="fas fa-user-secret"></i> Onion Domains</span>
 					</label>
 				</div>
 				<div class="custom-control custom-switch">
-					<input class="custom-control-input" type="checkbox" name="domain_regular_switch" value="True" id="domain_regular_switch"{%if 'regular' in domains_types%}checked{%endif%}>
+					<input class="custom-control-input" type="checkbox" name="domain_regular_switch" value="True" id="domain_regular_switch"{%if domains_types%}{%if 'regular' in domains_types%}checked{%endif%}{%endif%}>
 					<label class="custom-control-label" for="domain_regular_switch">
 						<span class="badge badge-warning"><i class="fab fa-html5"></i> Web Domains</span>
 					</label>

var/www/templates/domains/block_languages_search.html

@@ -17,13 +17,13 @@
 
 			<div class="mb-3">
 				<div class="custom-control custom-switch">
-					<input class="custom-control-input" type="checkbox" name="domain_onion_switch" value="" id="domain_onion_switch" {%if 'onion' in domains_types%}checked{%endif%}>
+					<input class="custom-control-input" type="checkbox" name="domain_onion_switch" value="" id="domain_onion_switch" {%if not domains_types%}checked{%elif 'onion' in domains_types%}checked{%endif%}>
 					<label class="custom-control-label" for="domain_onion_switch">
 						<span class="badge badge-danger"><i class="fas fa-user-secret"></i> Onion Domains</span>
 					</label>
 				</div>
 				<div class="custom-control custom-switch">
-					<input class="custom-control-input" type="checkbox" name="domain_regular_switch" value="True" id="domain_regular_switch"{%if 'regular' in domains_types%}checked{%endif%}>
+					<input class="custom-control-input" type="checkbox" name="domain_regular_switch" value="True" id="domain_regular_switch" {%if not domains_types%}checked{%elif 'regular' in domains_types%}checked{%endif%}>
 					<label class="custom-control-label" for="domain_regular_switch">
 						<span class="badge badge-warning"><i class="fab fa-html5"></i> Web Domains</span>
 					</label>