Merge branch 'master' into crawler_manager

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "bin/trackers/yara/ail-yara-rules"]
+	path = bin/trackers/yara/ail-yara-rules
+	url = https://github.com/ail-project/ail-yara-rules.git

bin/LAUNCH.sh

@@ -184,6 +184,8 @@ function launching_scripts {
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "RegexTracker" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexTracker.py; read x"
     sleep 0.1
+    screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x"
+    sleep 0.1
     screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x"
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x"

bin/Update.py

@@ -231,6 +231,15 @@ def get_git_upper_tags_remote(current_tag, is_fork):
             aborting_update()
             sys.exit(0)
 
+def update_submodules():
+    print('{}git submodule update:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
+    process = subprocess.run(['git', 'submodule', 'update'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    if process.returncode == 0:
+        print(process.stdout.decode())
+        print()
+    else:
+        print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
+
 def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_fork):
     print('{}git checkout master:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
     process = subprocess.run(['git', 'checkout', 'master'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -238,6 +247,9 @@ def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_for
     if process.returncode == 0:
         print(process.stdout.decode())
         print()
+
+        update_submodules()
+
         print('{}git pull:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
         process = subprocess.run(['git', 'pull'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 

bin/lib/Tracker.py

@@ -5,6 +5,7 @@ import os
 import sys
 import time
 import redis
+import yara
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
 import ConfigLoader
@@ -14,12 +15,117 @@ config_loader = ConfigLoader.ConfigLoader()
 r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker")
 config_loader = None
 
+def get_tracker_uuid_list(tracker, tracker_type):
+    return list(r_serv_tracker.smembers('all:tracker_uuid:{}:{}'.format(tracker_type, tracker)))
+
+def get_tracker_tags(tracker_uuid):
+    return list(r_serv_tracker.smembers('tracker:tags:{}'.format(tracker_uuid)))
+
+def get_tracker_mails(tracker_uuid):
+    return list(r_serv_tracker.smembers('tracker:mail:{}'.format(tracker_uuid)))
+
 def get_tracker_description(tracker_uuid):
     return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'description')
 
+def add_tracked_item(tracker_uuid, item_id, item_date):
+    # track item
+    r_serv_tracker.sadd('tracker:item:{}:{}'.format(tracker_uuid, item_date), item_id)
+    # track nb item by date
+    r_serv_tracker.zadd('tracker:stat:{}'.format(tracker_uuid), item_date, int(item_date))
+
 def get_email_subject(tracker_uuid):
     tracker_description = get_tracker_description(tracker_uuid)
     if not tracker_description:
         return "AIL framework: Tracker Alert"
     else:
         return 'AIL framework: {}'.format(tracker_description)
+
+def get_tracker_last_updated_by_type(tracker_type):
+    epoch_update = r_serv_tracker.get('tracker:refresh:{}'.format(term_type))
+    if not epoch_update:
+        epoch_update = 0
+    return float(epoch_update)
+
+#### YARA ####
+def get_yara_rules_dir():
+    return os.path.join(os.environ['AIL_BIN'], 'trackers', 'yara')
+
+def get_yara_rules_default_dir():
+    return os.path.join(os.environ['AIL_BIN'], 'trackers', 'yara', 'ail-yara-rules', 'rules')
+
+# # TODO: cache + update
+def get_all_default_yara_rules_types():
+    yara_dir = get_yara_rules_default_dir()
+    all_yara_types = next(os.walk(yara_dir))[1]
+    # save in cache ?
+    return all_yara_types
+
+# # TODO: cache + update
+def get_all_default_yara_files():
+    yara_dir = get_yara_rules_default_dir()
+    all_default_yara_files = {}
+    for rules_type in get_all_default_yara_rules_types():
+        all_default_yara_files[rules_type] = os.listdir(os.path.join(yara_dir, rules_type))
+    return all_default_yara_files
+
+def get_all_default_yara_rules_by_type(yara_types):
+    all_default_yara_files = get_all_default_yara_files()
+    if yara_types in all_default_yara_files:
+        return all_default_yara_files[yara_types]
+    else:
+        return []
+
+def get_all_tracked_yara_files():
+    yara_files = r_serv_tracker.smembers('all:tracker:yara')
+    if not yara_files:
+        yara_files = []
+    return yara_files
+
+def reload_yara_rules():
+    yara_files = get_all_tracked_yara_files()
+    # {uuid: filename}
+    rule_dict = {}
+    for yar_path in yara_files:
+        l_tracker_uuid = get_tracker_uuid_list(yar_path, 'yara')
+        for tracker_uuid in l_tracker_uuid:
+            rule_dict[tracker_uuid] = os.path.join(get_yara_rules_dir(), yar_path)
+    rules = yara.compile(filepaths=rule_dict)
+    return rules
+
+def is_valid_yara_rule(yara_rule):
+    try:
+        yara.compile(source=yara_rule)
+        return True
+    except:
+        return False
+
+def is_valid_default_yara_rule(yara_rule):
+    yara_dir = get_yara_rules_default_dir()
+    filename = os.path.join(yara_dir, yara_rule)
+    filename = os.path.realpath(filename)
+
+    # incorrect filename
+    if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
+        return False
+    else:
+        if os.path.isfile(filename):
+            return True
+        else:
+            return False
+
+def save_yara_rule(yara_rule_type, yara_rule, tracker_uuid=None):
+    if yara_rule_type == 'yara_custom':
+        if not  tracker_uuid:
+            tracker_uuid = str(uuid.uuid4())
+        filename = os.path.join('custom-rules', tracker_uuid + '.yar')
+        with open(os.path.join(get_yara_rules_dir(), filename), 'w') as f:
+            f.write(str(yara_rule))
+    if yara_rule_type == 'yara_default':
+        filename = os.path.join('ail-yara-rules', 'rules', yara_rule)
+    return filename
+##-- YARA --##
+
+
+if __name__ == '__main__':
+    res = is_valid_yara_rule('rule dummy {  }')
+    print(res)

bin/lib/crawlers.py

@@ -430,7 +430,7 @@ def save_crawler_config(crawler_mode, crawler_type, crawler_config, domain, url=
     if crawler_mode == 'manual':
         r_cache.set('crawler_config:{}:{}:{}'.format(crawler_mode, crawler_type, domain), json.dumps(crawler_config))
     elif crawler_mode == 'auto':
-        r_serv_onion.set('crawler_config:{}:{}:{}:{}'.format(crawler_type, crawler_type, domain, url), json.dumps(crawler_config))
+        r_serv_onion.set('crawler_config:{}:{}:{}:{}'.format(crawler_mode, crawler_type, domain, url), json.dumps(crawler_config))
 
 def send_url_to_crawl_in_queue(crawler_mode, crawler_type, url):
     r_serv_onion.sadd('{}_crawler_priority_queue'.format(crawler_type), '{};{}'.format(url, crawler_mode))

bin/lib/simple_correlation.py

@@ -166,20 +166,20 @@ class SimpleCorrelation(object): #social_name
         else:
             return []
 
-    # def get_correlation_all_object(self, correlation_type, correlation_value, correlation_objects=[]):
+    def get_correlation_all_object(self, correlation_value, correlation_objects=[]):
-    #     if not correlation_objects:
+        if not correlation_objects:
-    #         correlation_objects = get_all_correlation_objects()
+            correlation_objects = get_all_correlation_objects()
-    #     correlation_obj = {}
+        correlation_obj = {}
-    #     for correlation_object in correlation_objects:
+        for correlation_object in correlation_objects:
-    #         if correlation_object == 'paste':
+            if correlation_object == 'paste':
-    #             res = self._get_items(correlation_type, correlation_value)
+                res = self._get_items(correlation_value)
-    #         elif correlation_object == 'domain':
+            elif correlation_object == 'domain':
-    #             res = self.get_correlation_obj_domain(correlation_value, correlation_type=correlation_type)
+                res = self.get_correlation_obj_domain(correlation_value)
-    #         else:
+            else:
-    #             res = None
+                res = None
-    #         if res:
+            if res:
-    #             correlation_obj[correlation_object] = res
+                correlation_obj[correlation_object] = res
-    #     return correlation_obj
+        return correlation_obj
 
     def update_correlation_daterange(self, obj_id, date):
         date = int(date)

bin/packages/Term.py

@@ -16,6 +16,7 @@ from textblob import TextBlob
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
 import ConfigLoader
+import Tracker
 
 from flask import escape
 
@@ -219,7 +220,12 @@ def parse_tracked_term_to_add(term , term_type, nb_words=1):
 
             if nb_words > len(words_set):
                 nb_words = len(words_set)
-
+    elif term_type=='yara_custom':
+        if not Tracker.is_valid_yara_rule(term):
+            return ({"status": "error", "reason": "Invalid custom Yara Rule"}, 400)
+    elif term_type=='yara_default':
+        if not Tracker.is_valid_default_yara_rule(term):
+            return ({"status": "error", "reason": "The Yara Rule doesn't exist"}, 400)
     else:
         return ({"status": "error", "reason": "Incorrect type"}, 400)
     return ({"status": "success", "term": term, "type": term_type}, 200)
@@ -228,8 +234,13 @@ def add_tracked_term(term , term_type, user_id, level, tags, mails, description,
 
     term_uuid =  str(uuid.uuid4())
 
+    # YARA
+    if term_type == 'yara_custom' or term_type == 'yara_default':
+        term = Tracker.save_yara_rule(term_type, term, tracker_uuid=term_uuid)
+        term_type = 'yara'
+
     # create metadata
-    r_serv_term.hset('tracker:{}'.format(term_uuid), 'tracked',term)
+    r_serv_term.hset('tracker:{}'.format(term_uuid), 'tracked',term) # # TODO: use hash
     r_serv_term.hset('tracker:{}'.format(term_uuid), 'type', term_type)
     r_serv_term.hset('tracker:{}'.format(term_uuid), 'date', datetime.date.today().strftime("%Y%m%d"))
     r_serv_term.hset('tracker:{}'.format(term_uuid), 'user_id', user_id)
@@ -310,6 +321,10 @@ def delete_term(term_uuid):
         r_serv_term.delete('tracker:item:{}:{}'.format(term_uuid, date))
     r_serv_term.delete('tracker:stat:{}'.format(term_uuid))
 
+    if term_type == 'yara':
+        # # TODO: 
+        pass
+
 def replace_tracker_description(term_uuid, description):
     description = escape(description)
     r_serv_term.hset('tracker:{}'.format(term_uuid), 'description', description)

bin/packages/modules.cfg

@@ -34,6 +34,10 @@ publish = Redis_Tags
 subscribe = Redis_Global
 publish = Redis_Tags
 
+[Tracker_Yara]
+subscribe = Redis_Global
+publish = Redis_Tags
+
 [Tools]
 subscribe = Redis_Global
 publish = Redis_Tags

bin/trackers/Tracker_Yara.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+"""
+Yara trackers
+
+"""
+import os
+import re
+import sys
+import time
+import yara
+
+from pubsublogger import publisher
+#
+# import NotificationHelper
+#
+
+sys.path.append(os.environ['AIL_BIN'])
+from Helper import Process
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
+import Term
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
+import Tracker
+import item_basic
+
+
+full_item_url = "/showsavedpaste/?paste="
+mail_body_template = "AIL Framework,\nNew YARA match: {}\nitem id: {}\nurl: {}{}"
+
+last_refresh = time.time()
+
+def yara_rules_match(data):
+    #print(data)
+    tracker_uuid = data['namespace']
+
+    item_date = item_basic.get_item_date(item_id)
+    Tracker.add_tracked_item(tracker_uuid, item_id, item_date)
+
+    # Tags
+    tags_to_add = Tracker.get_tracker_tags(tracker_uuid)
+    for tag in tags_to_add:
+        msg = '{};{}'.format(tag, item_id)
+        p.populate_set_out(msg, 'Tags')
+
+    # Mails
+    mail_to_notify = Tracker.get_tracker_mails(tracker_uuid)
+    if mail_to_notify:
+        mail_subject = Tracker.get_email_subject(tracker_uuid)
+        mail_body = mail_body_template.format(term, item_id, full_item_url, item_id)
+    for mail in mail_to_notify:
+        NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
+
+    return yara.CALLBACK_CONTINUE
+
+if __name__ == "__main__":
+    publisher.port = 6380
+    publisher.channel = "Script"
+    publisher.info("Script Tracker_Yara started")
+
+    config_section = 'Tracker_Yara'
+    module_name = "Tracker_Yara"
+    p = Process(config_section)
+
+    full_item_url = p.config.get("Notifications", "ail_domain") + full_item_url
+
+    # Load Yara rules
+    rules = Tracker.reload_yara_rules()
+
+    # Regex Frequency
+    while True:
+        item_id = p.get_from_set()
+        if item_id is not None:
+            item_content = item_basic.get_item_content(item_id)
+            yara_match = rules.match(data=item_content, callback=yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
+            if yara_match:
+                print(f'{item_id}: {yara_match}')
+
+            time.sleep(30)
+
+        else:
+            time.sleep(5)
+
+        # refresh YARA list
+        if last_refresh < Tracker.get_tracker_last_updated_by_type('yara'):
+            rules = Tracker.reload_yara_rules()
+            last_refresh = time.time()
+            print('Tracked set refreshed')

bin/trackers/yara/ail-yara-rules

@@ -0,0 +1 @@
+Subproject commit edc390c4a8d93a028e29938e92aacb399e270cc4

bin/trackers/yara/custom-rules/test.yar

@@ -0,0 +1,14 @@
+
+/*
+    Test Rule
+*/
+
+rule certificatestest
+{
+    strings:
+        $ssh_priv = "BEGIN RSA PRIVATE KEY" wide ascii nocase
+        $pem_cert = "BEGIN CERTIFICATE" wide ascii nocase
+
+    condition:
+        any of them
+}

installing_deps.sh

@@ -39,6 +39,10 @@ sudo apt-get install build-essential libffi-dev automake autoconf libtool -qq
 # sflock, gz requirement
 sudo apt-get install p7zip-full -qq
 
+# SUBMODULES #
+git submodule init
+git submodule update
+
 # REDIS #
 test ! -d redis/ && git clone https://github.com/antirez/redis.git
 pushd redis/
@@ -113,7 +117,7 @@ $AIL_HOME/doc/generate_modules_data_flow_graph.sh
 # init update version
 pushd ${AIL_HOME}
 # shallow clone
-git fetch --tags --prune --depth=10000
+git fetch --depth=500 --tags --prune
 git describe --abbrev=0 --tags | tr -d '\n' > ${AIL_HOME}/update/current_version
 echo "AIL current version:"
 git describe --abbrev=0 --tags

requirements.txt

@@ -22,6 +22,8 @@ textblob
 #Tokeniser
 nltk
 
+yara-python
+
 #Crawler
 scrapy
 scrapy-splash

update/v3.1.1/Update.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import time
+import redis
+import argparse
+import datetime
+import configparser
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
+new_version = 'v3.1.1'
+
+if __name__ == '__main__':
+
+    start_deb = time.time()
+
+    config_loader = ConfigLoader.ConfigLoader()
+    r_serv_db = config_loader.get_redis_conn("ARDB_DB")
+    config_loader = None
+
+    #### NEW EXPORTER
+    # remove old tags errors
+    #r_serv_db.delete('mess_not_saved_export')
+
+    # move solo tags to export in tags_db
+    #all_misp_tags = r_serv_db.smembers('whitelist_misp')
+    #all_hive_tags = r_serv_db.smembers('whitelist_hive')
+    # # TODO: save them in tags db
+    #### NEW EXPORTER
+
+    #Set current ail version
+    r_serv_db.set('ail:version', new_version)
+
+    #Set current ail version
+    r_serv_db.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))

update/v3.1.1/Update.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
+
+export PATH=$AIL_HOME:$PATH
+export PATH=$AIL_REDIS:$PATH
+export PATH=$AIL_ARDB:$PATH
+export PATH=$AIL_BIN:$PATH
+export PATH=$AIL_FLASK:$PATH
+
+GREEN="\\033[1;32m"
+DEFAULT="\\033[0;39m"
+
+echo -e $GREEN"Shutting down AIL ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ks
+wait
+
+bash ${AIL_BIN}/LAUNCH.sh -ldbv &
+wait
+echo ""
+
+echo -e $GREEN"Installing YARA ..."$DEFAULT
+pip3 install yara-python
+bash ${AIL_BIN}/LAUNCH.sh -t
+
+# SUBMODULES #
+git submodule init
+git submodule update
+
+echo ""
+echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
+echo ""
+python ${AIL_HOME}/update/v3.1.1/Update.py
+wait
+echo ""
+echo ""
+
+
+echo ""
+echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ks
+wait
+
+exit 0

var/www/modules/hunter/Flask_hunter.py

@@ -4,6 +4,8 @@
 '''
     Flask functions and routes for tracked items
 '''
+import os
+import sys
 import json
 import redis
 import datetime
@@ -14,14 +16,11 @@ from flask import Flask, render_template, jsonify, request, Blueprint, url_for,
 from Role_Manager import login_admin, login_analyst, login_read_only
 from flask_login import login_required, current_user
 
-import re
-from pprint import pprint
-import Levenshtein
-
 # ---------------------------------------------------------------
 
-import Paste
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
 import Term
+import Tracker
 
 # ============ VARIABLES ============
 import Flask_config
@@ -78,6 +77,16 @@ def tracked_menu_regex():
     global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
     return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
 
+@hunter.route("/trackers/yara")
+@login_required
+@login_read_only
+def tracked_menu_yara():
+    filter_type = 'yara'
+    user_id = current_user.get_id()
+    user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type)
+    global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
+    return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
+
 
 @hunter.route("/tracker/add", methods=['GET', 'POST'])
 @login_required
@@ -92,6 +101,18 @@ def add_tracked_menu():
         tags = request.form.get("tags", [])
         mails = request.form.get("mails", [])
 
+        # YARA #
+        if term_type == 'yara':
+            yara_default_rule = request.form.get("yara_default_rule")
+            yara_custom_rule =  request.form.get("yara_custom_rule")
+            if yara_custom_rule:
+                term = yara_custom_rule
+                term_type='yara_custom'
+            else:
+                term = yara_default_rule
+                term_type='yara_default'
+        # #
+
         if level == 'on':
             level = 1
 
@@ -109,7 +130,8 @@ def add_tracked_menu():
             ## TODO: use modal
             return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
     else:
-        return render_template("Add_tracker.html")
+        all_yara_files = Tracker.get_all_default_yara_files()
+        return render_template("Add_tracker.html", all_yara_files=all_yara_files)
 
 @hunter.route("/tracker/show_tracker")
 @login_required
@@ -225,5 +247,12 @@ def get_json_tracker_stats():
         res = Term.get_list_tracked_term_stats_by_day([tracker_uuid])
     return jsonify(res)
 
+# @hunter.route("/tracker/get_all_default_yara_rules_by_type", methods=['GET'])
+# @login_required
+# @login_read_only
+# def get_all_default_yara_rules_by_type():
+#     yara_types = request.args.get('yara_types')
+#     get_all_default_yara_rules_by_types(yara_types)
+
 # ========= REGISTRATION =========
 app.register_blueprint(hunter, url_prefix=baseUrl)

var/www/modules/hunter/templates/Add_tracker.html

@@ -32,7 +32,7 @@
 								<h5 class="card-title">Create a new tracker</h5>
 							</div>
 						  <div class="card-body">
-						    <p class="card-text">Enter a domain and choose what kind of data you want.</p>
+						    <p class="card-text">Select a tracker type.</p>
 
 								<form action="{{ url_for('hunter.add_tracked_menu') }}" method='post'>
 
@@ -74,11 +74,12 @@
 										<option value="word">Word</option>
 										<option value="set">Set</option>
 										<option value="regex">Regex</option>
+										<option value="yara">YARA rule</option>
 									</select>
 
 									<p id="tracker_desc">Terms to track (space separated)</p>
 
-									<div class="row">
+									<div class="row" id="simple_input">
 										<div class="col-12 col-lg-10">
 											<input id="term" name="term" class="form-control" placeholder="Terms to track (space separated)" type="text">
 										</div>
@@ -88,6 +89,24 @@
 									</div>
 
 
+									<div class="" id="yara_rule">
+										<div class="" id="yara_default_rules">
+
+											<select class="custom-select w-100 mb-3" name="yara_default_rule">
+											  <option selected>Select a default rule</option>
+												{% for yara_types in all_yara_files %}
+													{% for yara_file in all_yara_files[yara_types] %}
+													<option value="{{yara_types}}/{{yara_file}}">{{yara_types}} - {{yara_file}}</option>
+													{% endfor %}
+												{% endfor %}
+											</select>
+										</div>
+
+									<div class="row" id="textarea">
+										<textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5"></textarea>
+									</div>
+								</div>
+
 									<br>
 									<button class="btn btn-success mt-2">
 										<i class="fas fa-plus"></i> Add Tracker
@@ -97,8 +116,6 @@
 
 
 
-
-
 						  </div>
 						</div>
 
@@ -119,6 +136,7 @@ $(document).ready(function(){
 	$("#tracker_desc").hide();
 	$("#term").hide();
 	$("#nb_word").hide();
+	$("#yara_rule").hide();
 
 	$('#tracker_type').on('change', function() {
 		var tracker_type = this.value;
@@ -127,16 +145,25 @@ $(document).ready(function(){
 			$("#tracker_desc").show();
 			$("#term").show();
 			$("#nb_word").hide();
+			$("#yara_rule").hide();
 		} else if (tracker_type=="set") {
 			$("#tracker_desc").text("Set of Terms to track (space separated). This tracker is used to check if an item contain one or more terms specified in a set. If an item contain NB unique terms (by default NB of unique keywords = 1), this tracker is triggered. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
 			$("#tracker_desc").show();
 			$("#term").show();
 			$("#nb_word").show();
-		} else {
+			$("#yara_rule").hide();
+		} else if (tracker_type=="regex") {
 			$("#tracker_desc").text("Enter a valid Python regex");
 			$("#tracker_desc").show();
 			$("#term").show();
 			$("#nb_word").hide();
+			$("#yara_rule").hide();
+		} else if (tracker_type=="yara") {
+			$("#tracker_desc").text("Select a default yara rule or create your own rule:");
+			$("#tracker_desc").show();
+			$("#term").hide();
+			$("#nb_word").hide();
+			$("#yara_rule").show();
 		}
 	});
 

var/www/templates/hunter/menu_sidebar.html

@@ -22,19 +22,25 @@
       	    <li class="nav-item">
       	      <a class="nav-link" href="{{url_for('hunter.tracked_menu_word')}}" id="nav_tracker_word">
                 <i class="fas fa-font"></i>
-      	        <span>Tracked Words</span>
+      	        <span>Words</span>
       	      </a>
       	    </li>
             <li class="nav-item">
       	      <a class="nav-link" href="{{url_for('hunter.tracked_menu_set')}}" id="nav_tracker_set">
       	        <i class="fas fa-layer-group"></i>
-      	        <span>Tracked Set</span>
+      	        <span>Set</span>
       	      </a>
       	    </li>
             <li class="nav-item">
       	      <a class="nav-link" href="{{url_for('hunter.tracked_menu_regex')}}" id="nav_tracker_regex">
+      	        <i class="fas fa-ruler-vertical"></i>
+                <span>Regex</span>
+      	      </a>
+      	    </li>
+            <li class="nav-item">
+      	      <a class="nav-link" href="{{url_for('hunter.tracked_menu_yara')}}" id="nav_tracker_yara">
       	        <i class="fas fa-ruler"></i>
-                <span>Tracked Regex</span>
+                <span>YARA</span>
       	      </a>
       	    </li>
           </ul>

var/www/update_thirdparty.sh

@@ -2,6 +2,9 @@
 
 set -e
 
+# submodules
+git submodule update
+
 wget -q http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js
 
 SBADMIN_VERSION='3.3.7'
@@ -90,13 +93,21 @@ wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb61
 wget -q http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js
 wget -q http://canvasjs.com/fdm/chart/ -O temp/canvasjs.zip
 unzip -qq temp/canvasjs.zip -d temp/
-mv temp/jquery.canvasjs.min.js ./static/js/jquery.canvasjs.min.js
+mv temp/canvasjs-2.3.2/Chart\ 2.3.2\ GA\ -\ Stable/jquery.canvasjs.min.js ./static/js/jquery.canvasjs.min.js
 
 wget -q https://jqueryui.com/resources/download/jquery-ui-1.12.1.zip -O temp/jquery-ui.zip
 unzip -qq temp/jquery-ui.zip -d temp/
 mv temp/jquery-ui-1.12.1/jquery-ui.min.js ./static/js/jquery-ui.min.js
 mv temp/jquery-ui-1.12.1/jquery-ui.min.css ./static/css/jquery-ui.min.css
 
+# INSTALL YARA
+YARA_VERSION="4.0.2"
+wget -q https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.zip -O temp/yara.zip
+unzip -qq temp/yara.zip -d temp/
+pushd temp/yara-${YARA_VERSION}
+./bootstrap.sh
+popd
+
 
 rm -rf temp