diff --git a/HOWTO.md b/HOWTO.md
index 629e8fa8..c21a970f 100644
--- a/HOWTO.md
+++ b/HOWTO.md
@@ -27,7 +27,7 @@ Feed data to AIL:
4. Edit your configuration file ```bin/packages/config.cfg``` and modify the pystemonpath path accordingly
-5. Launch pystemon-feeder ``` ./pystemon-feeder.py ```
+5. Launch pystemon-feeder ``` ./bin/feeder/pystemon-feeder.py ```
How to create a new module
@@ -114,6 +114,9 @@ There are two types of installation. You can install a *local* or a *remote* Spl
2. *(Splash host)* To install and setup your tor proxy:
- Install the tor proxy: ``sudo apt-get install tor -y``
(Not required if ``Splah host == AIL host`` - The tor proxy is installed by default in AIL)
+ - Allow Tor to bind to any interface or to the docker interface (by default binds to 127.0.0.1 only) in ``/etc/tor/torrc``
+ ``SOCKSPort 0.0.0.0:9050`` or
+ ``SOCKSPort 172.17.0.1:9050``
- Add the following line ``SOCKSPolicy accept 172.17.0.0/16`` in ``/etc/tor/torrc``
(for a linux docker, the localhost IP is *172.17.0.1*; Should be adapted for other platform)
- Restart the tor proxy: ``sudo service tor restart``
diff --git a/OVERVIEW.md b/OVERVIEW.md
index effb387d..3d3a62ab 100644
--- a/OVERVIEW.md
+++ b/OVERVIEW.md
@@ -26,6 +26,24 @@ ARDB overview
ARDB_DB
* DB 1 - Curve
* DB 2 - TermFreq
+ ----------------------------------------- TERM ----------------------------------------
+
+ SET - 'TrackedRegexSet' term
+
+ HSET - 'TrackedRegexDate' tracked_regex today_timestamp
+
+ SET - 'TrackedSetSet' set_to_add
+
+ HSET - 'TrackedSetDate' set_to_add today_timestamp
+
+ SET - 'TrackedSetTermSet' term
+
+ HSET - 'TrackedTermDate' tracked_regex today_timestamp
+
+ SET - 'TrackedNotificationEmails_'+term/set email
+
+ SET - 'TrackedNotifications' term/set
+
* DB 3 - Trending
* DB 4 - Sentiment
* DB 5 - TermCred
diff --git a/README.md b/README.md
index fbcb5363..e8af8e5f 100644
--- a/README.md
+++ b/README.md
@@ -70,8 +70,6 @@ Type these command lines for a fully automated installation and start AIL framew
git clone https://github.com/CIRCL/AIL-framework.git
cd AIL-framework
./installing_deps.sh
-cd var/www/
-./update_thirdparty.sh
cd ~/AIL-framework/
. ./AILENV/bin/activate
cd bin/
@@ -105,7 +103,7 @@ curl https://get.docker.com | /bin/bash
2. Type these commands to build the Docker image:
```bash
-git clone https://github.com/CIRCL/ail-framework
+git clone https://github.com/CIRCL/AIL-framework.git
cd AIL-framework
docker build -t ail-framework .
```
diff --git a/bin/Curve.py b/bin/Curve.py
index 8e228039..c7083c54 100755
--- a/bin/Curve.py
+++ b/bin/Curve.py
@@ -48,6 +48,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
+TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
+
# create direct link in mail
full_paste_url = "/showsavedpaste/?paste="
@@ -71,6 +73,11 @@ def check_if_tracked_term(term, path):
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + term):
sendEmailNotification(email, 'Term', mail_body)
+ # tag paste
+ for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + term):
+ msg = '{};{}'.format(tag, path)
+ p.populate_set_out(msg, 'Tags')
+
def getValueOverRange(word, startDate, num_day):
to_return = 0
diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh
index 684af83b..549c0425 100755
--- a/bin/LAUNCH.sh
+++ b/bin/LAUNCH.sh
@@ -9,12 +9,30 @@ WHITE="\\033[0;02m"
YELLOW="\\033[1;33m"
CYAN="\\033[1;36m"
-[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
-[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
-[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
-[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
-[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
+# Getting CWD where bash script resides
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd |sed 's/bin//' )"
+export AIL_HOME="${DIR}"
+cd ${AIL_HOME}
+
+if [ -e "${DIR}/AILENV/bin/python" ]; then
+ echo "AIL-framework virtualenv seems to exist, good"
+ ENV_PY="${DIR}/AILENV/bin/python"
+else
+ echo "Please make sure you have a AIL-framework environment, au revoir"
+ exit 1
+fi
+
+# redis-server is bundled during install
+## [ ! -f "`which redis-server`" ] && echo "'redis-server' is not installed/not on PATH. Please fix and run again." && exit 1
+
+export AIL_BIN=${AIL_HOME}/bin/
+export AIL_FLASK=${AIL_HOME}/var/www/
+export AIL_REDIS=${AIL_HOME}/redis/src/
+export AIL_ARDB=${AIL_HOME}/ardb/src/
+export AIL_VENV=${AIL_HOME}/AILENV/
+
+export PATH=$AIL_VENV/bin:$PATH
export PATH=$AIL_HOME:$PATH
export PATH=$AIL_REDIS:$PATH
export PATH=$AIL_ARDB:$PATH
@@ -91,9 +109,9 @@ function launching_logs {
screen -dmS "Logging_AIL"
sleep 0.1
echo -e $GREEN"\t* Launching logging process"$DEFAULT
- screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c 'cd '${AIL_BIN}'; log_subscriber -p 6380 -c Queuing -l ../logs/; read x'
+ screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Queuing -l ../logs/; read x"
sleep 0.1
- screen -S "Logging_AIL" -X screen -t "LogScript" bash -c 'cd '${AIL_BIN}'; log_subscriber -p 6380 -c Script -l ../logs/; read x'
+ screen -S "Logging_AIL" -X screen -t "LogScript" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Script -l ../logs/; read x"
}
function launching_queues {
@@ -101,16 +119,16 @@ function launching_queues {
sleep 0.1
echo -e $GREEN"\t* Launching all the queues"$DEFAULT
- screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; python3 launch_queues.py; read x'
+ screen -S "Queue_AIL" -X screen -t "Queues" bash -c "cd ${AIL_BIN}; ${ENV_PY} launch_queues.py; read x"
}
function checking_configuration {
bin_dir=${AIL_HOME}/bin
echo -e "\t* Checking configuration"
if [ "$1" == "automatic" ]; then
- bash -c "python3 $bin_dir/Update-conf.py True"
+ bash -c "${ENV_PY} $bin_dir/Update-conf.py True"
else
- bash -c "python3 $bin_dir/Update-conf.py False"
+ bash -c "${ENV_PY} $bin_dir/Update-conf.py False"
fi
exitStatus=$?
@@ -128,75 +146,75 @@ function launching_scripts {
sleep 0.1
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
- screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; ./ModulesInformationV2.py -k 0 -c 1; read x'
+ screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; ./Mixer.py; read x'
+ screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mixer.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; ./Global.py; read x'
+ screen -S "Script_AIL" -X screen -t "Global" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Global.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; ./Duplicates.py; read x'
+ screen -S "Script_AIL" -X screen -t "Duplicates" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Duplicates.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; ./Lines.py; read x'
+ screen -S "Script_AIL" -X screen -t "Lines" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Lines.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; ./DomClassifier.py; read x'
+ screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./DomClassifier.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; ./Categ.py; read x'
+ screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Categ.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; ./Tokenize.py; read x'
+ screen -S "Script_AIL" -X screen -t "Tokenize" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tokenize.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ./CreditCards.py; read x'
+ screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CreditCards.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; ./BankAccount.py; read x'
+ screen -S "Script_AIL" -X screen -t "BankAccount" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./BankAccount.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ./Onion.py; read x'
+ screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Onion.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ./Mail.py; read x'
+ screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mail.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; ./ApiKey.py; read x'
+ screen -S "Script_AIL" -X screen -t "ApiKey" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ApiKey.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; ./Web.py; read x'
+ screen -S "Script_AIL" -X screen -t "Web" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Web.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; ./Credential.py; read x'
+ screen -S "Script_AIL" -X screen -t "Credential" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Credential.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; ./Curve.py; read x'
+ screen -S "Script_AIL" -X screen -t "Curve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Curve.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; ./CurveManageTopSets.py; read x'
+ screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CurveManageTopSets.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./RegexForTermsFrequency.py; read x'
+ screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexForTermsFrequency.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./SetForTermsFrequency.py; read x'
+ screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SetForTermsFrequency.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; ./Indexer.py; read x'
+ screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ./Keys.py; read x'
+ screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ./Decoder.py; read x'
+ screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Decoder.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ./Bitcoin.py; read x'
+ screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Bitcoin.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; ./Phone.py; read x'
+ screen -S "Script_AIL" -X screen -t "Phone" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Phone.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; ./Release.py; read x'
+ screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; ./Cve.py; read x'
+ screen -S "Script_AIL" -X screen -t "Cve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cve.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; ./WebStats.py; read x'
+ screen -S "Script_AIL" -X screen -t "WebStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./WebStats.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; ./ModuleStats.py; read x'
+ screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModuleStats.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; ./SQLInjectionDetection.py; read x'
+ screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SQLInjectionDetection.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; ./LibInjection.py; read x'
+ screen -S "Script_AIL" -X screen -t "LibInjection" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./LibInjection.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; ./alertHandler.py; read x'
+ screen -S "Script_AIL" -X screen -t "alertHandler" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./alertHandler.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; ./MISP_The_Hive_feeder.py; read x'
+ screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; ./Tags.py; read x'
+ screen -S "Script_AIL" -X screen -t "Tags" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tags.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; ./SentimentAnalysis.py; read x'
+ screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SentimentAnalysis.py; read x"
sleep 0.1
- screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; ./submit_paste.py; read x'
+ screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./submit_paste.py; read x"
}
@@ -219,7 +237,7 @@ function launching_crawler {
sleep 0.1
for ((i=first_port;i<=last_port;i++)); do
- screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; ./Crawler.py onion '$i'; read x'
+ screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Crawler.py onion $i; read x"
sleep 0.1
done
@@ -342,7 +360,7 @@ function launch_flask {
screen -dmS "Flask_AIL"
sleep 0.1
echo -e $GREEN"\t* Launching Flask server"$DEFAULT
- screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ./Flask_server.py; read x"
+ screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ${ENV_PY} ./Flask_server.py; read x"
else
echo -e $RED"\t* A Flask screen is already launched"$DEFAULT
fi
@@ -353,9 +371,9 @@ function launch_feeder {
screen -dmS "Feeder_Pystemon"
sleep 0.1
echo -e $GREEN"\t* Launching Pystemon feeder"$DEFAULT
- screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; ./feeder/pystemon-feeder.py; read x'
+ screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./feeder/pystemon-feeder.py; read x"
sleep 0.1
- screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; python2 pystemon.py; read x'
+ screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c "cd ${AIL_HOME}/../pystemon; ${ENV_PY} ./pystemon.py; read x"
else
echo -e $RED"\t* A Feeder screen is already launched"$DEFAULT
fi
diff --git a/bin/MISP_The_Hive_feeder.py b/bin/MISP_The_Hive_feeder.py
index 0a8f1791..07c121c9 100755
--- a/bin/MISP_The_Hive_feeder.py
+++ b/bin/MISP_The_Hive_feeder.py
@@ -54,7 +54,7 @@ from thehive4py.models import Case, CaseTask, CustomFieldHelper
-def create_the_hive_alert(source, path, content, tag):
+def create_the_hive_alert(source, path, tag):
tags = list(r_serv_metadata.smembers('tag:'+path))
artifacts = [
@@ -63,7 +63,6 @@ def create_the_hive_alert(source, path, content, tag):
]
l_tags = tag.split(',')
- print(tag)
# Prepare the sample Alert
sourceRef = str(uuid.uuid4())[0:6]
@@ -91,6 +90,41 @@ def create_the_hive_alert(source, path, content, tag):
except:
print('hive connection error')
+def feeder(message, count=0):
+
+ if flag_the_hive or flag_misp:
+ tag, path = message.split(';')
+ ## FIXME: remove it
+ if PASTES_FOLDER not in path:
+ path = os.path.join(PASTES_FOLDER, path)
+ try:
+ paste = Paste.Paste(path)
+ except FileNotFoundError:
+ if count < 10:
+ r_serv_db.zincrby('mess_not_saved_export', message, 1)
+ return 0
+ else:
+ r_serv_db.zrem('mess_not_saved_export', message)
+ print('Error: {} do not exist, tag= {}'.format(path, tag))
+ return 0
+
+ source = '/'.join(paste.p_path.split('/')[-6:])
+
+ if HiveApi != False:
+ if int(r_serv_db.get('hive:auto-alerts')) == 1:
+ whitelist_hive = r_serv_db.scard('whitelist_hive')
+ if r_serv_db.sismember('whitelist_hive', tag):
+ create_the_hive_alert(source, path, tag)
+ else:
+ print('hive, auto alerts creation disable')
+ if flag_misp:
+ if int(r_serv_db.get('misp:auto-events')) == 1:
+ if r_serv_db.sismember('whitelist_misp', tag):
+ misp_wrapper.pushToMISP(uuid_ail, path, tag)
+ else:
+ print('misp, auto events creation disable')
+
+
if __name__ == "__main__":
publisher.port = 6380
@@ -119,10 +153,18 @@ if __name__ == "__main__":
db=cfg.getint("ARDB_Metadata", "db"),
decode_responses=True)
+ # set sensor uuid
uuid_ail = r_serv_db.get('ail:uuid')
if uuid_ail is None:
uuid_ail = r_serv_db.set('ail:uuid', uuid.uuid4() )
+ # set default
+ if r_serv_db.get('hive:auto-alerts') is None:
+ r_serv_db.set('hive:auto-alerts', 0)
+
+ if r_serv_db.get('misp:auto-events') is None:
+ r_serv_db.set('misp:auto-events', 0)
+
p = Process(config_section)
# create MISP connection
if flag_misp:
@@ -167,36 +209,30 @@ if __name__ == "__main__":
r_serv_db.set('ail:thehive', False)
print('Not connected to The HIVE')
+ refresh_time = 3
+ ## FIXME: remove it
+ PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes"))
+ time_1 = time.time()
+
while True:
# Get one message from the input queue
message = p.get_from_set()
if message is None:
- publisher.debug("{} queue is empty, waiting 1s".format(config_section))
- time.sleep(1)
- continue
+
+ # handle not saved pastes
+ if int(time.time() - time_1) > refresh_time:
+
+ num_queu = r_serv_db.zcard('mess_not_saved_export')
+ list_queu = r_serv_db.zrange('mess_not_saved_export', 0, -1, withscores=True)
+
+ if num_queu and list_queu:
+ for i in range(0, num_queu):
+ feeder(list_queu[i][0],list_queu[i][1])
+
+ time_1 = time.time()
+ else:
+ publisher.debug("{} queue is empty, waiting 1s".format(config_section))
+ time.sleep(1)
else:
-
- if flag_the_hive or flag_misp:
- tag, path = message.split(';')
- paste = Paste.Paste(path)
- source = '/'.join(paste.p_path.split('/')[-6:])
-
- full_path = os.path.join(os.environ['AIL_HOME'],
- p.config.get("Directories", "pastes"), path)
-
-
- if HiveApi != False:
- if int(r_serv_db.get('hive:auto-alerts')) == 1:
- whitelist_hive = r_serv_db.scard('whitelist_hive')
- if r_serv_db.sismember('whitelist_hive', tag):
- create_the_hive_alert(source, path, full_path, tag)
-
- else:
- print('hive, auto alerts creation disable')
- if flag_misp:
- if int(r_serv_db.get('misp:auto-events')) == 1:
- if r_serv_db.sismember('whitelist_misp', tag):
- misp_wrapper.pushToMISP(uuid_ail, path, tag)
- else:
- print('misp, auto events creation disable')
+ feeder(message)
diff --git a/bin/Mixer.py b/bin/Mixer.py
index e1656b8e..e7f9e6de 100755
--- a/bin/Mixer.py
+++ b/bin/Mixer.py
@@ -80,6 +80,7 @@ if __name__ == '__main__':
# OTHER CONFIG #
operation_mode = cfg.getint("Module_Mixer", "operation_mode")
ttl_key = cfg.getint("Module_Mixer", "ttl_duplicate")
+ default_unnamed_feed_name = cfg.get("Module_Mixer", "default_unnamed_feed_name")
# STATS #
processed_paste = 0
@@ -106,7 +107,7 @@ if __name__ == '__main__':
paste_name = complete_paste
except ValueError as e:
- feeder_name = "unnamed_feeder"
+ feeder_name = default_unnamed_feed_name
paste_name = complete_paste
# Processed paste
diff --git a/bin/NotificationHelper.py b/bin/NotificationHelper.py
index d8f7fe92..1bccd314 100755
--- a/bin/NotificationHelper.py
+++ b/bin/NotificationHelper.py
@@ -3,8 +3,10 @@
import argparse
import configparser
+import traceback
import os
import smtplib
+from pubsublogger import publisher
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
@@ -15,6 +17,9 @@ This module allows the global configuration and management of notification setti
# CONFIG #
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
+publisher.port = 6380
+publisher.channel = "Script"
+
# notifications enabled/disabled
TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
@@ -22,7 +27,6 @@ TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
# Keys will be e.g. TrackedNotificationEmails<TERMNAME>
TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_"
-
def sendEmailNotification(recipient, alert_name, content):
if not os.path.exists(configfile):
@@ -33,22 +37,12 @@ def sendEmailNotification(recipient, alert_name, content):
cfg = configparser.ConfigParser()
cfg.read(configfile)
- sender = cfg.get("Notifications", "sender"),
- sender_host = cfg.get("Notifications", "sender_host"),
- sender_port = cfg.getint("Notifications", "sender_port"),
- sender_pw = cfg.get("Notifications", "sender_pw"),
-
- if isinstance(sender, tuple):
- sender = sender[0]
-
- if isinstance(sender_host, tuple):
- sender_host = sender_host[0]
-
- if isinstance(sender_port, tuple):
- sender_port = sender_port[0]
-
- if isinstance(sender_pw, tuple):
- sender_pw = sender_pw[0]
+ sender = cfg.get("Notifications", "sender")
+ sender_host = cfg.get("Notifications", "sender_host")
+ sender_port = cfg.getint("Notifications", "sender_port")
+ sender_pw = cfg.get("Notifications", "sender_pw")
+ if sender_pw == 'None':
+ sender_pw = None
# raise an exception if any of these is None
if (sender is None or
@@ -83,9 +77,9 @@ def sendEmailNotification(recipient, alert_name, content):
smtp_server.quit()
print('Send notification ' + alert_name + ' to '+recipient)
- except Exception as e:
- print(str(e))
- # raise e
+ except Exception as err:
+ traceback.print_tb(err.__traceback__)
+ publisher.warning(err)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Test notification sender.')
diff --git a/bin/RegexForTermsFrequency.py b/bin/RegexForTermsFrequency.py
index ecca8e4d..0db7f2ee 100755
--- a/bin/RegexForTermsFrequency.py
+++ b/bin/RegexForTermsFrequency.py
@@ -9,35 +9,45 @@ supplied in the term webpage.
import redis
import time
from pubsublogger import publisher
-from packages import lib_words
from packages import Paste
-import os
-from os import environ
-import datetime
import calendar
import re
+import signal
+import time
from Helper import Process
-
# Email notifications
from NotificationHelper import *
+
+class TimeoutException(Exception):
+ pass
+
+
+def timeout_handler(signum, frame):
+ raise TimeoutException
+
+signal.signal(signal.SIGALRM, timeout_handler)
+
# Config Variables
-DICO_REFRESH_TIME = 60 #s
+DICO_REFRESH_TIME = 60 # s
BlackListTermsSet_Name = "BlackListSetTermSet"
TrackedTermsSet_Name = "TrackedSetTermSet"
TrackedRegexSet_Name = "TrackedRegexSet"
-top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set
+top_term_freq_max_set_cardinality = 20 # Max cardinality of the terms frequences set
oneDay = 60*60*24
top_termFreq_setName_day = ["TopTermFreq_set_day_", 1]
top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
-top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
+top_termFreq_set_array = [top_termFreq_setName_day, top_termFreq_setName_week, top_termFreq_setName_month]
+
+TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
# create direct link in mail
full_paste_url = "/showsavedpaste/?paste="
+
def refresh_dicos():
dico_regex = {}
dico_regexname_to_redis = {}
@@ -53,6 +63,7 @@ if __name__ == "__main__":
config_section = 'RegexForTermsFrequency'
p = Process(config_section)
+ max_execution_time = p.config.getint(config_section, "max_execution_time")
# REDIS #
server_term = redis.StrictRedis(
@@ -67,7 +78,7 @@ if __name__ == "__main__":
# create direct link in mail
full_paste_url = p.config.get("Notifications", "ail_domain") + full_paste_url
- #compile the regex
+ # compile the regex
dico_refresh_cooldown = time.time()
dico_regex, dico_regexname_to_redis = refresh_dicos()
@@ -87,13 +98,22 @@ if __name__ == "__main__":
timestamp = calendar.timegm((int(temp[-4]), int(temp[-3]), int(temp[-2]), 0, 0, 0))
curr_set = top_termFreq_setName_day[0] + str(timestamp)
- content = Paste.Paste(filename).get_p_content()
+ paste = Paste.Paste(filename)
+ content = paste.get_p_content()
- #iterate the word with the regex
+ # iterate the word with the regex
for regex_str, compiled_regex in dico_regex.items():
- matched = compiled_regex.search(content)
- if matched is not None: #there is a match
+ signal.alarm(max_execution_time)
+ try:
+ matched = compiled_regex.search(content)
+ except TimeoutException:
+ print ("{0} processing timeout".format(paste.p_path))
+ continue
+ else:
+ signal.alarm(0)
+
+ if matched is not None: # there is a match
print('regex matched {}'.format(regex_str))
matched = matched.group(0)
regex_str_complete = "/" + regex_str + "/"
@@ -104,20 +124,25 @@ if __name__ == "__main__":
# create mail body
mail_body = ("AIL Framework,\n"
- "New occurrence for regex: " + regex_str + "\n"
- ''+full_paste_url + filename)
+ "New occurrence for regex: " + regex_str + "\n"
+ ''+full_paste_url + filename)
# Send to every associated email adress
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + regex_str_complete):
sendEmailNotification(email, 'Term', mail_body)
+ # tag paste
+ for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + regex_str_complete):
+ msg = '{};{}'.format(tag, filename)
+ p.populate_set_out(msg, 'Tags')
+
set_name = 'regex_' + dico_regexname_to_redis[regex_str]
new_to_the_set = server_term.sadd(set_name, filename)
new_to_the_set = True if new_to_the_set == 1 else False
- #consider the num of occurence of this term
+ # consider the num of occurence of this term
regex_value = int(server_term.hincrby(timestamp, dico_regexname_to_redis[regex_str], int(1)))
- #1 term per paste
+ # 1 term per paste
if new_to_the_set:
regex_value_perPaste = int(server_term.hincrby("per_paste_" + str(timestamp), dico_regexname_to_redis[regex_str], int(1)))
server_term.zincrby("per_paste_" + curr_set, dico_regexname_to_redis[regex_str], float(1))
diff --git a/bin/SetForTermsFrequency.py b/bin/SetForTermsFrequency.py
index 78de9b08..19ed7210 100755
--- a/bin/SetForTermsFrequency.py
+++ b/bin/SetForTermsFrequency.py
@@ -34,6 +34,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
+TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
+
# create direct link in mail
full_paste_url = "/showsavedpaste/?paste="
@@ -121,6 +123,11 @@ if __name__ == "__main__":
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + dico_setname_to_redis[str(the_set)]):
sendEmailNotification(email, 'Term', mail_body)
+ # tag paste
+ for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + dico_setname_to_redis[str(the_set)]):
+ msg = '{};{}'.format(tag, filename)
+ p.populate_set_out(msg, 'Tags')
+
print(the_set, "matched in", filename)
set_name = 'set_' + dico_setname_to_redis[the_set]
new_to_the_set = server_term.sadd(set_name, filename)
diff --git a/bin/feeder/pystemon-feeder.py b/bin/feeder/pystemon-feeder.py
index b6680ee9..280849ba 100755
--- a/bin/feeder/pystemon-feeder.py
+++ b/bin/feeder/pystemon-feeder.py
@@ -61,10 +61,10 @@ topic = '102'
while True:
time.sleep(base_sleeptime + sleep_inc)
paste = r.lpop("pastes")
- print(paste)
if paste is None:
continue
try:
+ print(paste)
with open(pystemonpath+paste, 'rb') as f: #.read()
messagedata = f.read()
path_to_send = pastes_directory+paste
diff --git a/bin/feeder/test-zmq.py b/bin/feeder/test-zmq.py
index f6f28aa1..110c5de2 100644
--- a/bin/feeder/test-zmq.py
+++ b/bin/feeder/test-zmq.py
@@ -20,7 +20,7 @@ socket.connect ("tcp://crf.circl.lu:%s" % port)
# 102 Full pastes in raw base64(gz)
topicfilter = "102"
-socket.setsockopt(zmq.SUBSCRIBE, topicfilter)
+socket.setsockopt_string(zmq.SUBSCRIBE, topicfilter)
while True:
message = socket.recv()
diff --git a/bin/packages/Paste.py b/bin/packages/Paste.py
index 524a7665..d02a92f5 100755
--- a/bin/packages/Paste.py
+++ b/bin/packages/Paste.py
@@ -82,6 +82,13 @@ class Paste(object):
db=cfg.getint("ARDB_Metadata", "db"),
decode_responses=True)
+ PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes"))
+ if PASTES_FOLDER not in p_path:
+ self.p_rel_path = p_path
+ p_path = os.path.join(PASTES_FOLDER, p_path)
+ else:
+ self.p_rel_path = None
+
self.p_path = p_path
self.p_name = os.path.basename(self.p_path)
self.p_size = round(os.path.getsize(self.p_path)/1024.0, 2)
@@ -280,6 +287,8 @@ class Paste(object):
def _get_p_duplicate(self):
self.p_duplicate = self.store_metadata.smembers('dup:'+self.p_path)
+ if self.p_rel_path is not None:
+ self.p_duplicate.union( self.store_metadata.smembers('dup:'+self.p_rel_path) )
if self.p_duplicate is not None:
return list(self.p_duplicate)
else:
diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample
index b459f011..b5980766 100644
--- a/bin/packages/config.cfg.sample
+++ b/bin/packages/config.cfg.sample
@@ -27,6 +27,7 @@ ail_domain = http://localhost:7000
sender = sender@example.com
sender_host = smtp.example.com
sender_port = 1337
+sender_pw = None
# optional for using with authenticated SMTP over SSL
# sender_pw = securepassword
@@ -98,6 +99,10 @@ threshold_stucked_module=600
operation_mode = 3
#Define the time that a paste will be considerate duplicate. in seconds (1day = 86400)
ttl_duplicate = 86400
+default_unnamed_feed_name = unnamed_feeder
+
+[RegexForTermsFrequency]
+max_execution_time = 60
##### Redis #####
[Redis_Cache]
@@ -201,6 +206,9 @@ dns = 8.8.8.8
[Mail]
dns = 8.8.8.8
+[Web]
+dns = 149.13.33.69
+
# Indexer configuration
[Indexer]
type = whoosh
diff --git a/bin/packages/lib_refine.py b/bin/packages/lib_refine.py
index 5d2af0a9..32f56900 100644
--- a/bin/packages/lib_refine.py
+++ b/bin/packages/lib_refine.py
@@ -1,6 +1,8 @@
#!/usr/bin/python3
import re
+import os
+import configparser
import dns.resolver
from pubsublogger import publisher
@@ -101,11 +103,20 @@ def checking_MX_record(r_serv, adress_set, addr_dns):
def checking_A_record(r_serv, domains_set):
+ configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
+ if not os.path.exists(configfile):
+ raise Exception('Unable to find the configuration file. \
+ Did you set environment variables? \
+ Or activate the virtualenv.')
+ cfg = configparser.ConfigParser()
+ cfg.read(configfile)
+ dns_server = cfg.get("Web", "dns")
+
score = 0
num = len(domains_set)
WalidA = set([])
resolver = dns.resolver.Resolver()
- resolver.nameservers = ['149.13.33.69']
+ resolver.nameservers = [dns_server]
resolver.timeout = 5
resolver.lifetime = 2
diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg
index deb5a069..0dc40448 100644
--- a/bin/packages/modules.cfg
+++ b/bin/packages/modules.cfg
@@ -32,13 +32,15 @@ publish = Redis_Words
[Curve]
subscribe = Redis_Words
-publish = Redis_CurveManageTopSets
+publish = Redis_CurveManageTopSets,Redis_Tags
[RegexForTermsFrequency]
subscribe = Redis_Global
+publish = Redis_Tags
[SetForTermsFrequency]
subscribe = Redis_Global
+publish = Redis_Tags
[CurveManageTopSets]
subscribe = Redis_CurveManageTopSets
diff --git a/pip3_packages_requirement.txt b/pip3_packages_requirement.txt
index dd447d5c..12ef09ff 100644
--- a/pip3_packages_requirement.txt
+++ b/pip3_packages_requirement.txt
@@ -2,7 +2,7 @@ pymisp
thehive4py
-redis
+redis==2.10.6
#filemagic conflict with magic
crcmod
mmh3
diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py
index ea6fd6ed..7cc802f0 100644
--- a/var/www/modules/Flask_config.py
+++ b/var/www/modules/Flask_config.py
@@ -102,7 +102,6 @@ r_serv_onion = redis.StrictRedis(
db=cfg.getint("ARDB_Onion", "db"),
decode_responses=True)
-
sys.path.append('../../configs/keys')
# MISP #
try:
diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py
index 16930ef8..cc38de77 100644
--- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py
+++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py
@@ -506,6 +506,8 @@ def edit_tag_export():
status_misp = []
status_hive = []
+ infoleak_tags = Taxonomies().get('infoleak').machinetags()
+ is_infoleak_tag = []
for tag in list_export_tags:
if r_serv_db.sismember('whitelist_misp', tag):
@@ -519,6 +521,11 @@ def edit_tag_export():
else:
status_hive.append(False)
+ if tag in infoleak_tags:
+ is_infoleak_tag.append(True)
+ else:
+ is_infoleak_tag.append(False)
+
if misp_auto_events is not None:
if int(misp_auto_events) == 1:
misp_active = True
@@ -543,6 +550,7 @@ def edit_tag_export():
misp_active=misp_active,
hive_active=hive_active,
list_export_tags=list_export_tags,
+ is_infoleak_tag=is_infoleak_tag,
status_misp=status_misp,
status_hive=status_hive,
nb_tags_whitelist_misp=nb_tags_whitelist_misp,
@@ -594,5 +602,37 @@ def disable_hive_auto_alert():
r_serv_db.set('hive:auto-alerts', 0)
return edit_tag_export()
+@PasteSubmit.route("/PasteSubmit/add_push_tag")
+def add_push_tag():
+ tag = request.args.get('tag')
+ if tag is not None:
+
+ #limit tag length
+ if len(tag) > 49:
+ tag = tag[0:48]
+
+ r_serv_db.sadd('list_export_tags', tag)
+
+ to_return = {}
+ to_return["tag"] = tag
+ return jsonify(to_return)
+ else:
+ return 'None args', 400
+
+@PasteSubmit.route("/PasteSubmit/delete_push_tag")
+def delete_push_tag():
+ tag = request.args.get('tag')
+
+ infoleak_tags = Taxonomies().get('infoleak').machinetags()
+ if tag not in infoleak_tags and r_serv_db.sismember('list_export_tags', tag):
+ r_serv_db.srem('list_export_tags', tag)
+ r_serv_db.srem('whitelist_misp', tag)
+ r_serv_db.srem('whitelist_hive', tag)
+ to_return = {}
+ to_return["tag"] = tag
+ return jsonify(to_return)
+ else:
+ return 'this tag can\'t be removed', 400
+
# ========= REGISTRATION =========
app.register_blueprint(PasteSubmit, url_prefix=baseUrl)
diff --git a/var/www/modules/PasteSubmit/templates/edit_tag_export.html b/var/www/modules/PasteSubmit/templates/edit_tag_export.html
index 04a506d6..94980787 100644
--- a/var/www/modules/PasteSubmit/templates/edit_tag_export.html
+++ b/var/www/modules/PasteSubmit/templates/edit_tag_export.html
@@ -37,6 +37,9 @@
background: #d91f2d;
color: #fff;
}
+ .mouse_pointer{
+ cursor: pointer;
+ }
</style>
</head>
@@ -169,7 +172,14 @@
<input type="checkbox" value="{{ tag }}" name="tag_enabled_misp" >
{% endif %}
</td>
- <td>{{ tag }}</td>
+ <td>
+ {{ tag }}
+ {% if not is_infoleak_tag[loop.index0] %}
+ <div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" title="Delete this tag" onclick="delete_push_tag('{{ tag }}')">
+ <span class="glyphicon glyphicon-trash"></span>
+ </div>
+ {% endif %}
+ </td>
</tr>
{% endfor %}
@@ -209,7 +219,14 @@
<input type="checkbox" value="{{ tag }}" name="tag_enabled_hive" >
{% endif %}
</td>
- <td>{{ tag }}</td>
+ <td>
+ {{ tag }}
+ {% if not is_infoleak_tag[loop.index0] %}
+ <div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" title="Delete this tag" onclick="delete_push_tag('{{ tag }}')">
+ <span class="glyphicon glyphicon-trash"></span>
+ </div>
+ {% endif %}
+ </td>
</tr>
{% endfor %}
@@ -232,6 +249,42 @@
</form>
+ <div>
+
+ <div id="add_custom_tag_modal" class="modal fade" role="dialog">
+ <div class="modal-dialog">
+
+ <!-- Modal content-->
+ <div id="mymodalcontent" class="modal-content">
+ <div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
+ <h2 class="text-center">Add Custom Tag</h2>
+
+ </div>
+
+ <div class="modal-body">
+ <div class="form-group input-group" style="margin-bottom: 0px;">
+ <span class="input-group-addon"><i class="fa fa-tag fa"></i></span>
+ <input id="new_custom_tag" class="form-control" placeholder="Add a new custom tag to the tag export" type="text">
+ </div>
+ </div>
+
+ <div class="modal-footer">
+ <button class="btn btn-primary btn-tags" onclick="add_custom_tag()">
+ <span class="glyphicon glyphicon-plus"></span>
+ <span class="label-icon">Add Custom Tag</span>
+ </button>
+ <button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
+
+ </div>
+ </div>
+ </div>
+ </div>
+
+ <button type="button" class="btn btn-primary pull-right" data-toggle="modal" data-target="#add_custom_tag_modal" data-url="{{ url_for('Tags.taxonomies') }}">
+ <span class="glyphicon glyphicon-plus "></span> Add Custom Tag
+ </button>
+ </div>
+
</div>
<!-- /#page-wrapper -->
@@ -277,6 +330,25 @@ $(document).ready(function(){
}
);
}
+
+ function delete_push_tag(tag){
+ //var row_tr = $(this).closest("tr");
+ $.get("{{ url_for('PasteSubmit.delete_push_tag') }}", { tag: tag }, function(data, status){
+ if(status == "success") {
+ //row_tr.remove();
+ window.location.reload(false);
+ }
+ });
+ }
+
+ function add_custom_tag(){
+ $.get("{{ url_for('PasteSubmit.add_push_tag') }}", { tag: document.getElementById('new_custom_tag').value }, function(data, status){
+ if(status == "success") {
+ //row_tr.remove();
+ window.location.reload(false);
+ }
+ });
+ }
</script>
</html>
diff --git a/var/www/modules/Tags/Flask_Tags.py b/var/www/modules/Tags/Flask_Tags.py
index e79d56fc..bbc918ed 100644
--- a/var/www/modules/Tags/Flask_Tags.py
+++ b/var/www/modules/Tags/Flask_Tags.py
@@ -28,6 +28,7 @@ r_serv_statistics = Flask_config.r_serv_statistics
max_preview_char = Flask_config.max_preview_char
max_preview_modal = Flask_config.max_preview_modal
bootstrap_label = Flask_config.bootstrap_label
+PASTES_FOLDER = Flask_config.PASTES_FOLDER
Tags = Blueprint('Tags', __name__, template_folder='templates')
diff --git a/var/www/modules/search/Flask_search.py b/var/www/modules/search/Flask_search.py
index 5b458589..7f6cd724 100644
--- a/var/www/modules/search/Flask_search.py
+++ b/var/www/modules/search/Flask_search.py
@@ -16,6 +16,8 @@ from whoosh import index
from whoosh.fields import Schema, TEXT, ID
from whoosh.qparser import QueryParser
+import time
+
# ============ VARIABLES ============
import Flask_config
@@ -55,8 +57,8 @@ def get_index_list(selected_index=""):
if os.path.isdir(os.path.join(baseindexpath, dirs)):
value = dirs
name = to_iso_date(dirs) + " - " + \
- str(get_dir_size(dirs) / (1000*1000)) + " Mb " + \
- "(" + str(get_item_count(dirs)) + " Items" + ")"
+ str(get_dir_size(dirs) / (1000*1000)) + " Mb " #+ \
+ #"(" + str(get_item_count(dirs))''' + " Items" + ")"
flag = dirs==selected_index.split('/')[-1]
if dirs == "old_index":
temp = [value, name, flag]
@@ -66,6 +68,7 @@ def get_index_list(selected_index=""):
index_list.sort(reverse=True, key=lambda x: x[0])
if len(temp) != 0:
index_list.append(temp)
+
return index_list
def get_dir_size(directory):
@@ -108,6 +111,7 @@ def search():
else:
selected_index = os.path.join(baseindexpath, index_name)
+ ''' temporary disabled
# Search filename
for path in r_serv_pasteName.smembers(q[0]):
r.append(path)
@@ -119,13 +123,14 @@ def search():
curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
paste_date.append(curr_date)
paste_size.append(paste._get_p_size())
+ '''
# Search full line
schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT)
ix = index.open_dir(selected_index)
with ix.searcher() as searcher:
- query = QueryParser("content", ix.schema).parse(" ".join(q))
+ query = QueryParser("content", ix.schema).parse("".join(q))
results = searcher.search_page(query, 1, pagelen=num_elem_to_get)
for x in results:
r.append(x.items()[0][1])
@@ -159,15 +164,18 @@ def search():
results = searcher.search(query)
num_res = len(results)
+ index_list = get_index_list()
+
index_min = 1
- index_max = len(get_index_list())
+ index_max = len(index_list)
+
return render_template("search.html", r=r, c=c,
query=request.form['query'], paste_date=paste_date,
paste_size=paste_size, char_to_display=max_preview_modal,
num_res=num_res, index_min=index_min, index_max=index_max,
bootstrap_label=bootstrap_label,
paste_tags=paste_tags,
- index_list=get_index_list(selected_index)
+ index_list=index_list
)
@@ -212,6 +220,7 @@ def get_more_search_result():
p_tags = r_serv_metadata.smembers('tag:'+path)
l_tags = []
for tag in p_tags:
+ complete_tag = tag
tag = tag.split('=')
if len(tag) > 1:
if tag[1] != '':
@@ -223,7 +232,7 @@ def get_more_search_result():
else:
tag = tag[0]
- l_tags.append(tag)
+ l_tags.append( (tag, complete_tag) )
list_tags.append(l_tags)
to_return = {}
diff --git a/var/www/modules/search/templates/search.html b/var/www/modules/search/templates/search.html
index adc1b555..5754a0ee 100644
--- a/var/www/modules/search/templates/search.html
+++ b/var/www/modules/search/templates/search.html
@@ -201,7 +201,9 @@
var curr_preview = data.preview_array[i].replace(/\"/g, "\'");
var tag = ""
for(j=0; j<data.list_tags[i].length; j++) {
- tag = tag + "<span class=\"label label-" + data.bootstrap_label[j % 5] + " pull-left\">" + data.list_tags[j] + "</span>"
+ tag = tag + "<a href=\"{{ url_for('Tags.get_tagged_paste') }}?ltags=" + data.list_tags[i][j][1] + ">"
+ + "<span class=\"label label-" + data.bootstrap_label[j % 5] + " pull-left\">" + data.list_tags[i][j][0]
+ + "</span>" + "</a>"
}
search_table.row.add( [
init_num_of_elements_in_table+((offset))+i+1,
diff --git a/var/www/modules/showpaste/Flask_showpaste.py b/var/www/modules/showpaste/Flask_showpaste.py
index 4912e7b0..188af759 100644
--- a/var/www/modules/showpaste/Flask_showpaste.py
+++ b/var/www/modules/showpaste/Flask_showpaste.py
@@ -41,7 +41,9 @@ showsavedpastes = Blueprint('showsavedpastes', __name__, template_folder='templa
# ============ FUNCTIONS ============
def showpaste(content_range, requested_path):
+ relative_path = None
if PASTES_FOLDER not in requested_path:
+ relative_path = requested_path
requested_path = os.path.join(PASTES_FOLDER, requested_path)
# remove old full path
#requested_path = requested_path.replace(PASTES_FOLDER, '')
@@ -122,6 +124,8 @@ def showpaste(content_range, requested_path):
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
l_tags = r_serv_metadata.smembers('tag:'+requested_path)
+ if relative_path is not None:
+ l_tags.union( r_serv_metadata.smembers('tag:'+relative_path) )
#active galaxies
active_galaxies = r_serv_tags.smembers('active_galaxies')
@@ -224,7 +228,6 @@ def showpaste(content_range, requested_path):
@showsavedpastes.route("/showsavedpaste/") #completely shows the paste in a new tab
def showsavedpaste():
requested_path = request.args.get('paste', '')
- print(requested_path)
return showpaste(0, requested_path)
@showsavedpastes.route("/showsavedrawpaste/") #shows raw
diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py
index b0794593..86d67af8 100644
--- a/var/www/modules/terms/Flask_terms.py
+++ b/var/www/modules/terms/Flask_terms.py
@@ -10,7 +10,7 @@ import redis
import datetime
import calendar
import flask
-from flask import Flask, render_template, jsonify, request, Blueprint
+from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect
import re
import Paste
from pprint import pprint
@@ -24,6 +24,8 @@ cfg = Flask_config.cfg
baseUrl = Flask_config.baseUrl
r_serv_term = Flask_config.r_serv_term
r_serv_cred = Flask_config.r_serv_cred
+r_serv_db = Flask_config.r_serv_db
+bootstrap_label = Flask_config.bootstrap_label
terms = Blueprint('terms', __name__, template_folder='templates')
@@ -51,6 +53,7 @@ TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
# same value as in `bin/NotificationHelper.py`
# Keys will be e.g. TrackedNotificationEmails_<TERMNAME>
TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_"
+TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
'''CRED'''
REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+'
@@ -130,6 +133,12 @@ def mixUserName(supplied, extensive=False):
filtered_usernames.append(usr)
return filtered_usernames
+def save_tag_to_auto_push(list_tag):
+ for tag in set(list_tag):
+ #limit tag length
+ if len(tag) > 49:
+ tag = tag[0:48]
+ r_serv_db.sadd('list_export_tags', tag)
# ============ ROUTES ============
@@ -152,6 +161,7 @@ def terms_management():
# Maps a specific term to the associated email addresses
notificationEMailTermMapping = {}
+ notificationTagsTermMapping = {}
#Regex
trackReg_list = []
@@ -159,7 +169,8 @@ def terms_management():
trackReg_list_num_of_paste = []
for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name):
- notificationEMailTermMapping[tracked_regex] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex)) )
+ notificationEMailTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex)
+ notificationTagsTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_regex)
if tracked_regex not in notificationEnabledDict:
notificationEnabledDict[tracked_regex] = False
@@ -185,8 +196,8 @@ def terms_management():
for tracked_set in r_serv_term.smembers(TrackedSetSet_Name):
tracked_set = tracked_set
- notificationEMailTermMapping[tracked_set] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set)) )
-
+ notificationEMailTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set)
+ notificationTagsTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_set)
if tracked_set not in notificationEnabledDict:
notificationEnabledDict[tracked_set] = False
@@ -211,7 +222,8 @@ def terms_management():
track_list_num_of_paste = []
for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name):
- notificationEMailTermMapping[tracked_term] = "\n".join( r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term))
+ notificationEMailTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term)
+ notificationTagsTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_term)
if tracked_term not in notificationEnabledDict:
notificationEnabledDict[tracked_term] = False
@@ -244,7 +256,8 @@ def terms_management():
track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste,
trackReg_list_values=trackReg_list_values, trackReg_list_num_of_paste=trackReg_list_num_of_paste,
trackSet_list_values=trackSet_list_values, trackSet_list_num_of_paste=trackSet_list_num_of_paste,
- per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, notificationEMailTermMapping=notificationEMailTermMapping)
+ per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, bootstrap_label=bootstrap_label,
+ notificationEMailTermMapping=notificationEMailTermMapping, notificationTagsTermMapping=notificationTagsTermMapping)
@terms.route("/terms_management_query_paste/")
@@ -313,6 +326,7 @@ def terms_management_action():
action = request.args.get('action')
term = request.args.get('term')
notificationEmailsParam = request.args.get('emailAddresses')
+ input_tags = request.args.get('tags')
if action is None or term is None or notificationEmailsParam is None:
return "None"
@@ -320,11 +334,8 @@ def terms_management_action():
if section == "followTerm":
if action == "add":
- # Strip all whitespace
- notificationEmailsParam = "".join(notificationEmailsParam.split())
-
# Make a list of all passed email addresses
- notificationEmails = notificationEmailsParam.split(",")
+ notificationEmails = notificationEmailsParam.split()
validNotificationEmails = []
# check for valid email addresses
@@ -334,6 +345,8 @@ def terms_management_action():
if re.match(r"[^@]+@[^@]+\.[^@]+", email):
validNotificationEmails.append(email)
+ # create tags list
+ list_tags = input_tags.split()
# check if regex/set or simple term
#regex
@@ -345,6 +358,10 @@ def terms_management_action():
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term, email)
# enable notifications by default
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term)
+ # add tags list
+ for tag in list_tags:
+ r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term, tag)
+ save_tag_to_auto_push(list_tags)
#set
elif term.startswith('\\') and term.endswith('\\'):
@@ -363,6 +380,10 @@ def terms_management_action():
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + set_to_add, email)
# enable notifications by default
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, set_to_add)
+ # add tags list
+ for tag in list_tags:
+ r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + set_to_add, tag)
+ save_tag_to_auto_push(list_tags)
#simple term
else:
@@ -373,6 +394,10 @@ def terms_management_action():
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term.lower(), email)
# enable notifications by default
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower())
+ # add tags list
+ for tag in list_tags:
+ r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term.lower(), tag)
+ save_tag_to_auto_push(list_tags)
elif action == "toggleEMailNotification":
# get the current state
@@ -397,6 +422,8 @@ def terms_management_action():
# delete the associated notification emails too
r_serv_term.delete(TrackedTermsNotificationEmailsPrefix_Name + term)
+ # delete the associated tags set
+ r_serv_term.delete(TrackedTermsNotificationTagsPrefix_Name + term)
elif section == "blacklistTerm":
if action == "add":
@@ -413,6 +440,28 @@ def terms_management_action():
to_return["term"] = term
return jsonify(to_return)
+@terms.route("/terms_management/delete_terms_tags", methods=['POST'])
+def delete_terms_tags():
+ term = request.form.get('term')
+ tags_to_delete = request.form.getlist('tags_to_delete')
+
+ if term is not None and tags_to_delete is not None:
+ for tag in tags_to_delete:
+ r_serv_term.srem(TrackedTermsNotificationTagsPrefix_Name + term, tag)
+ return redirect(url_for('terms.terms_management'))
+ else:
+ return 'None args', 400
+
+@terms.route("/terms_management/delete_terms_email", methods=['GET'])
+def delete_terms_email():
+ term = request.args.get('term')
+ email = request.args.get('email')
+
+ if term is not None and email is not None:
+ r_serv_term.srem(TrackedTermsNotificationEmailsPrefix_Name + term, email)
+ return redirect(url_for('terms.terms_management'))
+ else:
+ return 'None args', 400
@terms.route("/terms_plot_tool/")
diff --git a/var/www/modules/terms/templates/terms_management.html b/var/www/modules/terms/templates/terms_management.html
index 0efda575..a2ecd906 100644
--- a/var/www/modules/terms/templates/terms_management.html
+++ b/var/www/modules/terms/templates/terms_management.html
@@ -36,6 +36,12 @@
white-space:pre-wrap;
word-wrap:break-word;
}
+ .mouse_pointer{
+ cursor: pointer;
+ }
+ .lb-md {
+ font-size: 16px;
+ }
</style>
</head>
<body>
@@ -74,6 +80,7 @@
<div class="row">
<div class="col-lg-12">
<div class="row">
+ {% set uniq_id = namespace(modal_id=0)%}
<div class="col-lg-12">
<label class="switch">
<input id="per_paste" class="switch-input" value="per_paste" type="checkbox" onclick="reload_per_paste()">
@@ -98,7 +105,8 @@
<div class="form-group input-group" style="margin-bottom: 30px;">
<span class="input-group-addon"><span class="fa fa-eye"></span></span>
<input id="followTermInput" class="form-control" placeholder="Term to track." type="text" style="max-width: 400px;">
- <input id="followTermEMailNotificationReceiversInput" class="form-control" placeholder="Notification E-Mails (comma separated)" type="text" style="max-width: 400px;">
+ <input id="followTermEMailNotificationReceiversInput" class="form-control" placeholder="Notification E-Mails (optional, space separated)" type="text" style="max-width: 400px;">
+ <input id="followTermTag" class="form-control" placeholder="Tags (optional, space separated)" type="text" style="max-width: 400px;">
<button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Add term</button>
</div>
@@ -119,7 +127,57 @@
<!-- SET -->
{% for set in trackSet_list %}
<tr style="background-color: #cdffca;">
- <td>{{ set }}</td>
+ <td>
+ <span class="term_name">{{ set }}</span>
+ <div>
+ {% for tag in notificationTagsTermMapping[set] %}
+ <a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
+ <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
+ </a>
+ {% endfor %}
+ {% if notificationTagsTermMapping[set] %}
+ <div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
+
+ <div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
+ <div class="modal-dialog">
+
+ <!-- Modal content-->
+ <div id="mymodalcontent" class="modal-content">
+ <div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
+ <h2 class="text-center">Remove Custom Tag</h2>
+
+ </div>
+
+ <div class="modal-body">
+ <form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
+ {% for tag in notificationTagsTermMapping[set] %}
+ <div class="form-check">
+ <input type="hidden" class="form-control" name="term" value="{{ set }}">
+ <input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
+ <label class="form-check-label">
+ <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
+ </label>
+ <br>
+ </div>
+ {% endfor %}
+ </form>
+ </div>
+
+ <div class="modal-footer">
+ <button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
+ <span class="glyphicon glyphicon-trash"></span>
+ <span class="label-icon">Remove Tags</span>
+ </button>
+ <button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
+
+ </div>
+ </div>
+ </div>
+ </div>
+ {% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
+ {% endif %}
+ </div>
+ </td>
<td>{{ trackSet_list_values[loop.index0][3] }}</td>
<td>{{ trackSet_list_values[loop.index0][0] }}</td>
<td>{{ trackSet_list_values[loop.index0][1] }}</td>
@@ -130,13 +188,73 @@
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ set }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ set }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[set] %} checked {% endif %}>
</p></td>
- <td style="white-space:pre">{{ notificationEMailTermMapping[set] }}</td>
+ <td>
+ {% for email in notificationEMailTermMapping[set] %}
+ <a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{set}}">
+ <div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
+ <span class="glyphicon glyphicon-trash" style="color:Red;" ></span>
+ </div>
+ </a>
+ {{ email }}
+ <br>
+ {% endfor %}
+ </td>
</tr>
{% endfor %}
<!-- REGEX -->
{% for regex in trackReg_list %}
<tr style="background-color: #fffdca;">
- <td>{{ regex }}</td>
+ <td>
+ <span class="term_name">{{ regex }}</span>
+ <div>
+ {% for tag in notificationTagsTermMapping[regex] %}
+ <a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
+ <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
+ </a>
+ {% endfor %}
+ {% if notificationTagsTermMapping[regex] %}
+ <div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
+
+ <div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
+ <div class="modal-dialog">
+
+ <!-- Modal content-->
+ <div id="mymodalcontent" class="modal-content">
+ <div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
+ <h2 class="text-center">Remove Custom Tag</h2>
+
+ </div>
+
+ <div class="modal-body">
+ <form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
+ {% for tag in notificationTagsTermMapping[regex] %}
+ <div class="form-check">
+ <input type="hidden" class="form-control" name="term" value="{{ regex }}">
+ <input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
+ <label class="form-check-label">
+ <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
+ </label>
+ <br>
+ </div>
+ {% endfor %}
+ </form>
+ </div>
+
+ <div class="modal-footer">
+ <button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
+ <span class="glyphicon glyphicon-trash"></span>
+ <span class="label-icon">Remove Tags</span>
+ </button>
+ <button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
+
+ </div>
+ </div>
+ </div>
+ </div>
+ {% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
+ {% endif %}
+ </div>
+ </td>
<td>{{ trackReg_list_values[loop.index0][3] }}</td>
<td>{{ trackReg_list_values[loop.index0][0] }}</td>
<td>{{ trackReg_list_values[loop.index0][1] }}</td>
@@ -147,13 +265,73 @@
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ regex }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ regex }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[regex] %} checked {% endif %}>
</p></td>
- <td style="white-space:pre">{{ notificationEMailTermMapping[regex] }}</td>
+ <td>
+ {% for email in notificationEMailTermMapping[regex] %}
+ <a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{regex}}">
+ <div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
+ <span class="glyphicon glyphicon-trash" style="color:Red;"></span>
+ </div>
+ </a>
+ {{ email }}
+ <br>
+ {% endfor %}
+ </td>
</tr>
{% endfor %}
<!-- Normal term -->
{% for term in track_list %}
<tr>
- <td>{{ term }}</td>
+ <td>
+ <span class="term_name">{{ term }}</span>
+ <div>
+ {% for tag in notificationTagsTermMapping[term] %}
+ <a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
+ <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
+ </a>
+ {% endfor %}
+ {% if notificationTagsTermMapping[term] %}
+ <div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
+
+ <div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
+ <div class="modal-dialog">
+
+ <!-- Modal content-->
+ <div id="mymodalcontent" class="modal-content">
+ <div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
+ <h2 class="text-center">Remove Custom Tag</h2>
+
+ </div>
+
+ <div class="modal-body">
+ <form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
+ {% for tag in notificationTagsTermMapping[term] %}
+ <div class="form-check">
+ <input type="hidden" class="form-control" name="term" value="{{ term }}">
+ <input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
+ <label class="form-check-label">
+ <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
+ </label>
+ <br>
+ </div>
+ {% endfor %}
+ </form>
+ </div>
+
+ <div class="modal-footer">
+ <button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
+ <span class="glyphicon glyphicon-trash"></span>
+ <span class="label-icon">Remove Tags</span>
+ </button>
+ <button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
+
+ </div>
+ </div>
+ </div>
+ </div>
+ {% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
+ {% endif %}
+ </div>
+ </td>
<td>{{ track_list_values[loop.index0][3] }}</td>
<td>{{ track_list_values[loop.index0][0] }}</td>
<td>{{ track_list_values[loop.index0][1] }}</td>
@@ -164,7 +342,17 @@
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ term }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ term }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[term] %} checked {% endif %}>
</p></td>
- <td style="white-space:pre">{{ notificationEMailTermMapping[term] }}</td>
+ <td>
+ {% for email in notificationEMailTermMapping[term] %}
+ <a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{term}}">
+ <div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
+ <span class="glyphicon glyphicon-trash" style="color:Red;"></span>
+ </div>
+ </a>
+ {{ email }}
+ <br>
+ {% endfor %}
+ </td>
</tr>
{% endfor %}
</tbody>
@@ -351,17 +539,19 @@ function perform_binding() {
function perform_operation(){
var curr_section = $(this).attr('data-section');
var curr_action = $(this).attr('data-action');
+ var row_tr = $(this).closest("tr");
if (curr_action == "add") {
var curr_term = $('#'+curr_section+'Input').val();
var email_addresses = $('#followTermEMailNotificationReceiversInput').val();
+ var tags = $('#followTermTag').val();
} else {
var curr_term = $(this).attr('data-content');
var email_addresses = "";
}
- var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses};
+ var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses, tags: tags};
if (curr_term != "") {
- console.log(data_to_send);
+ //console.log(data_to_send);
$.get("{{ url_for('terms.terms_management_action') }}", data_to_send, function(data, status){
if(status == "success") {
var json = data;
@@ -372,13 +562,8 @@ function perform_operation(){
$.get("{{ url_for('terms.terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
reload_per_paste();
});
- } else if (json.action == "delete") {
- // Find indexes of row which have the term in the first column
- var index = table_track.rows().eq( 0 ).filter( function (rowIdx) {
- console.log(table_track.cell( rowIdx, 0 ).data())
- return table_track.cell( rowIdx, 0 ).data() === json.term;
- } );
- table_track.rows(index).remove().draw( false );
+ } else if (json.action == "delete") {
+ row_tr.remove()
}
} else if(json.section == "blacklistTerm"){
if(json.action == "add") {
diff --git a/var/www/update_thirdparty.sh b/var/www/update_thirdparty.sh
index 76ae9174..47fd4ecc 100755
--- a/var/www/update_thirdparty.sh
+++ b/var/www/update_thirdparty.sh
@@ -86,11 +86,11 @@ popd
#active virtualenv
source ./../../AILENV/bin/activate
#Update MISP Taxonomies and Galaxies
-python3 -m pip install git+https://github.com/MISP/PyTaxonomies
-python3 -m pip install git+https://github.com/MISP/PyMISPGalaxies
+python3 -m pip install git+https://github.com/MISP/PyTaxonomies --upgrade
+python3 -m pip install git+https://github.com/MISP/PyMISPGalaxies --upgrade
#Update PyMISP
-python3 -m pip install git+https://github.com/MISP/PyMISP
+python3 -m pip install git+https://github.com/MISP/PyMISP --upgrade
#Update the Hive
-python3 -m pip install git+https://github.com/TheHive-Project/TheHive4py
+python3 -m pip install git+https://github.com/TheHive-Project/TheHive4py --upgrade