CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [update] add v2.2 update + fix default update + fix Empty Item.get_tags() + add new LAUNCHER options

pull/389/head
Terrtia 2019-09-12 13:25:29 +02:00
parent 934d631f7a
commit 01ff202ae8
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
7 changed files with 236 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

172
bin/LAUNCH.sh
View File

@ -76,12 +76,15 @@ function helptext {
Usage:
-----
LAUNCH.sh
[-l | --launchAuto]
[-k | --killAll]
[-u | --update]
[-c | --configUpdate]
[-t | --thirdpartyUpdate]
[-h | --help]
[-l | --launchAuto] LAUNCH DB + Scripts
[-k | --killAll] Kill DB + Scripts
[-ks | --killscript] Scripts
[-u | --update] Update AIL
[-c | --crawler] LAUNCH Crawlers
[-f | --launchFeeder] LAUNCH Pystemon feeder
[-t | --thirdpartyUpdate] Update Web
[-m | --menu] Display Advanced Menu
[-h | --help] Help
"
}
@ -398,6 +401,18 @@ function launch_feeder {
fi
}
function killscript {
if [[ $islogged || $isqueued || $isscripted || $isflasked || $isfeeded || $iscrawler ]]; then
echo -e $GREEN"Killing Script"$DEFAULT
kill $islogged $isqueued $isscripted $isflasked $isfeeded $iscrawler
sleep 0.2
echo -e $ROSE`screen -ls`$DEFAULT
echo -e $GREEN"\t* $islogged $isqueued $isscripted $isflasked $isfeeded $iscrawler killed."$DEFAULT
else
echo -e $RED"\t* No script to kill"$DEFAULT
fi
}
function killall {
if [[ $isredis || $isardb || $islogged || $isqueued || $isscripted || $isflasked || $isfeeded || $iscrawler ]]; then
if [[ $isredis ]]; then
@ -457,76 +472,82 @@ function launch_all {
launch_flask;
}
#If no params, display the menu
function menu_display {
options=("Redis" "Ardb" "Logs" "Queues" "Scripts" "Flask" "Killall" "Shutdown" "Update" "Update-config" "Update-thirdparty")
menu() {
echo "What do you want to Launch?:"
for i in ${!options[@]}; do
printf "%3d%s) %s\n" $((i+1)) "${choices[i]:- }" "${options[i]}"
done
[[ "$msg" ]] && echo "$msg"; :
}
prompt="Check an option (again to uncheck, ENTER when done): "
while menu && read -rp "$prompt" numinput && [[ "$numinput" ]]; do
for num in $numinput; do
[[ "$num" != *[![:digit:]]* ]] && (( num > 0 && num <= ${#options[@]} )) || {
msg="Invalid option: $num"; break
}
((num--)); msg="${options[num]} was ${choices[num]:+un}checked"
[[ "${choices[num]}" ]] && choices[num]="" || choices[num]="+"
done
done
for i in ${!options[@]}; do
if [[ "${choices[i]}" ]]; then
case ${options[i]} in
Redis)
launch_redis;
;;
Ardb)
launch_ardb;
;;
Logs)
launch_logs;
;;
Queues)
launch_queues;
;;
Scripts)
launch_scripts;
;;
Flask)
launch_flask;
;;
Crawler)
launching_crawler;
;;
Killall)
killall;
;;
Shutdown)
shutdown;
;;
Update)
update;
;;
Update-config)
checking_configuration;
;;
Update-thirdparty)
update_thirdparty;
;;
esac
fi
done
exit
}
#If no params, display the help
[[ $@ ]] || {
helptext;
options=("Redis" "Ardb" "Logs" "Queues" "Scripts" "Flask" "Killall" "Shutdown" "Update" "Update-config" "Update-thirdparty")
menu() {
echo "What do you want to Launch?:"
for i in ${!options[@]}; do
printf "%3d%s) %s\n" $((i+1)) "${choices[i]:- }" "${options[i]}"
done
[[ "$msg" ]] && echo "$msg"; :
}
prompt="Check an option (again to uncheck, ENTER when done): "
while menu && read -rp "$prompt" numinput && [[ "$numinput" ]]; do
for num in $numinput; do
[[ "$num" != *[![:digit:]]* ]] && (( num > 0 && num <= ${#options[@]} )) || {
msg="Invalid option: $num"; break
}
((num--)); msg="${options[num]} was ${choices[num]:+un}checked"
[[ "${choices[num]}" ]] && choices[num]="" || choices[num]="+"
done
done
for i in ${!options[@]}; do
if [[ "${choices[i]}" ]]; then
case ${options[i]} in
Redis)
launch_redis;
;;
Ardb)
launch_ardb;
;;
Logs)
launch_logs;
;;
Queues)
launch_queues;
;;
Scripts)
launch_scripts;
;;
Flask)
launch_flask;
;;
Crawler)
launching_crawler;
;;
Killall)
killall;
;;
Shutdown)
shutdown;
;;
Update)
update;
;;
Update-config)
checking_configuration;
;;
Update-thirdparty)
update_thirdparty;
;;
esac
fi
done
exit
}
#echo "$@"
@ -547,6 +568,10 @@ while [ "$1" != "" ]; do
;;
-k | --killAll ) killall;
;;
-ks | --killscript ) killscript;
;;
-m | --menu ) menu_display;
;;
-u | --update ) update;
;;
-t | --thirdpartyUpdate ) update_thirdparty;
@ -559,7 +584,6 @@ while [ "$1" != "" ]; do
exit
;;
-kh | --khelp ) helptext;
;;
* ) helptext
exit 1

3
bin/packages/Item.py
View File

@ -22,6 +22,9 @@ def exist_item(item_id):
else:
return False
def get_item_id(full_path):
return full_path.replace(PASTES_FOLDER, '', 1)
def get_item_date(item_id):
l_directory = item_id.split('/')
return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])

2
bin/packages/Tag.py
View File

@ -82,7 +82,7 @@ def get_item_tags(item_id):
if tags:
return list(tags)
else:
return '[]'
return []
# TEMPLATE + API QUERY
def add_items_tag(tags=[], galaxy_tags=[], item_id=None):

5
bin/packages/config.cfg.sample
View File

@ -180,6 +180,11 @@ host = localhost
port = 6382
db = 3
[ARDB_Tracker]
host = localhost
port = 6382
db = 3
[ARDB_Hashs]
host = localhost
db = 1

8
update/default_update/Update.sh
View File

@ -20,8 +20,8 @@ export PATH=$AIL_FLASK:$PATH
GREEN="\\033[1;32m"
DEFAULT="\\033[0;39m"
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
bash ${AIL_BIN}/LAUNCH.sh -k
echo -e $GREEN"Shutting down AIL Script ..."$DEFAULT
bash ${AIL_BIN}/LAUNCH.sh -ks
wait
echo ""
@ -37,8 +37,8 @@ echo ""
echo ""
echo ""
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
bash ${AIL_BIN}/LAUNCH.sh -k
echo -e $GREEN"Killing Script ..."$DEFAULT
bash ${AIL_BIN}/LAUNCH.sh -ks
wait
echo ""

122
update/v2.2/Update.py Executable file
View File

@ -0,0 +1,122 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
import configparser
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
import Item
import Term
def rreplace(s, old, new, occurrence):
li = s.rsplit(old, occurrence)
return new.join(li)
if __name__ == '__main__':
start_deb = time.time()
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg.sample')
if not os.path.exists(configfile):
raise Exception('Unable to find the configuration file. \
Did you set environment variables? \
Or activate the virtualenv.')
cfg = configparser.ConfigParser()
cfg.read(configfile)
r_serv_term_stats = redis.StrictRedis(
host=cfg.get("ARDB_Trending", "host"),
port=cfg.getint("ARDB_Trending", "port"),
db=cfg.getint("ARDB_Trending", "db"),
decode_responses=True)
r_serv_termfreq = redis.StrictRedis(
host=cfg.get("ARDB_TermFreq", "host"),
port=cfg.getint("ARDB_TermFreq", "port"),
db=cfg.getint("ARDB_TermFreq", "db"),
decode_responses=True)
r_serv_term_stats.flushdb()
#convert all regex:
all_regex = r_serv_termfreq.smembers('TrackedRegexSet')
for regex in all_regex:
tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex)) )
mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex)) )
new_term = regex[1:-1]
res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
if res[1] == 200:
term_uuid = res[0]['uuid']
list_items = r_serv_termfreq.smembers('regex_{}'.format(regex))
for paste_item in list_items:
item_id = Item.get_item_id(paste_item)
item_date = Item.get_item_date(item_id)
Term.add_tracked_item(term_uuid, item_id, item_date)
# Invalid Tracker => remove it
else:
print('Invalid Regex Removed: {}'.format(regex))
print(res[0])
# allow reprocess
r_serv_termfreq.srem('TrackedRegexSet', regex)
all_tokens = r_serv_termfreq.smembers('TrackedSetTermSet')
for token in all_tokens:
tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token)) )
mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token)) )
res = Term.parse_json_term_to_add({"term": token, "type": 'word', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
if res[1] == 200:
term_uuid = res[0]['uuid']
list_items = r_serv_termfreq.smembers('tracked_{}'.format(token))
for paste_item in list_items:
item_id = Item.get_item_id(paste_item)
item_date = Item.get_item_date(item_id)
Term.add_tracked_item(term_uuid, item_id, item_date)
# Invalid Tracker => remove it
else:
print('Invalid Token Removed: {}'.format(token))
print(res[0])
# allow reprocess
r_serv_termfreq.srem('TrackedSetTermSet', token)
all_set = r_serv_termfreq.smembers('TrackedSetSet')
for curr_set in all_set:
tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set)) )
mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set)) )
to_remove = ',{}'.format(curr_set.split(',')[-1])
new_set = rreplace(curr_set, to_remove, '', 1)
new_set = new_set[2:]
new_set = new_set.replace(',', '')
res = Term.parse_json_term_to_add({"term": new_set, "type": 'set', "nb_words": 1, "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
if res[1] == 200:
term_uuid = res[0]['uuid']
list_items = r_serv_termfreq.smembers('tracked_{}'.format(curr_set))
for paste_item in list_items:
item_id = Item.get_item_id(paste_item)
item_date = Item.get_item_date(item_id)
Term.add_tracked_item(term_uuid, item_id, item_date)
# Invalid Tracker => remove it
else:
print('Invalid Set Removed: {}'.format(curr_set))
print(res[0])
# allow reprocess
r_serv_termfreq.srem('TrackedSetSet', curr_set)
r_serv_termfreq.flushdb()
#Set current ail version
r_serv.set('ail:version', 'v2.2')
#Set current ail version
r_serv.hset('ail:update_date', 'v2.2', datetime.datetime.now().strftime("%Y%m%d"))

6
var/www/modules/Flask_config.py
View File

@ -61,9 +61,9 @@ r_serv_sentiment = redis.StrictRedis(
decode_responses=True)
r_serv_term = redis.StrictRedis(
host=cfg.get("ARDB_TermFreq", "host"),
port=cfg.getint("ARDB_TermFreq", "port"),
db=cfg.getint("ARDB_TermFreq", "db"),
host=cfg.get("ARDB_Tracker", "host"),
port=cfg.getint("ARDB_Tracker", "port"),
db=cfg.getint("ARDB_Tracker", "db"),
decode_responses=True)
r_serv_cred = redis.StrictRedis(