CIRCL
/
AIL-framework
mirror of https://github.com/CIRCL/AIL-framework
2
1
Fork 0
Code Issues Releases Wiki Activity

chg: [config] Kvrocks config + reset DB

pull/594/head
Terrtia 2023-04-04 11:50:48 +02:00
parent 80efc9cdbb
commit 6e66fc943b
No known key found for this signature in database
GPG Key ID: 1E1B1F50D84613D0
2 changed files with 54 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
configs/core.cfg.sample
View File

@ -97,18 +97,6 @@ max_execution_time = 180
[PgpDump] [PgpDump]
max_execution_time = 60 max_execution_time = 60
[Base64]
path = Base64/
max_execution_time = 60
[Binary]
path = Base64/
max_execution_time = 60
[Hex]
path = Base64/
max_execution_time = 60
[Modules_Duplicates] [Modules_Duplicates]
#Number of month to look back #Number of month to look back
maximum_month_range = 3 maximum_month_range = 3
@ -162,71 +150,49 @@ host = localhost
port = 6381 port = 6381
db = 1 db = 1
##### ARDB ##### ##### KVROCKS #####
[ARDB_DB]
host = localhost
port = 6382
db = 0
[DB_Tracking]
host = localhost
port = 6382
db = 2
[ARDB_Sentiment]
host = localhost
port = 6382
db = 4
[ARDB_TermCred]
host = localhost
port = 6382
db = 5
[ARDB_Trending]
host = localhost
port = 6382
db = 3
[ARDB_Tracker]
host = localhost
port = 6382
db = 3
[ARDB_Hashs]
host = localhost
db = 1
[ARDB_Tags]
host = localhost
port = 6382
db = 6
[ARDB_Metadata]
host = localhost
port = 6382
db = 7
[ARDB_Statistics]
host = localhost
port = 6382
db = 8
[ARDB_Onion]
host = localhost
port = 6382
db = 9
[ARDB_Objects]
host = localhost
port = 6382
db = 10
[Kvrocks_DB] [Kvrocks_DB]
host = localhost host = localhost
port = 6383 port = 6383
db = 0 password = ail
[Kvrocks_Duplicates]
host = localhost
port = 6383
password = ail_dups
[Kvrocks_Correlations]
host = localhost
port = 6383
password = ail_correls
[Kvrocks_Crawler]
host = localhost
port = 6383
password = ail_crawlers
[Kvrocks_Objects]
host = localhost
port = 6383
password = ail_objs
[Kvrocks_Stats]
host = localhost
port = 6383
password = ail_stats
[Kvrocks_Tags]
host = localhost
port = 6383
password = ail_tags
[Kvrocks_Trackers]
host = localhost
port = 6383
password = ail_trackers
##### - #####
[Url] [Url]
cc_critical = DE cc_critical = DE
@ -239,9 +205,6 @@ dns = 8.8.8.8
[Mail] [Mail]
dns = 8.8.8.8 dns = 8.8.8.8
[Web]
dns = 8.8.8.8
# Indexer configuration # Indexer configuration
[Indexer] [Indexer]
type = whoosh type = whoosh
@ -258,16 +221,11 @@ maxDuplicateToPushToMISP=10
# For multiple feed, add them with "," without space # For multiple feed, add them with "," without space
# e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557 # e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557
[ZMQ_Global] [ZMQ_Global]
#address = tcp://crf.circl.lu:5556
# address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556 # address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556
address = tcp://127.0.0.1:5556 address = tcp://127.0.0.1:5556
channel = 102 channel = 102
bind = tcp://127.0.0.1:5556 bind = tcp://127.0.0.1:5556
[ZMQ_FetchedOnion]
address = tcp://127.0.0.1:5005
channel = FetchedOnion
[RedisPubSub] [RedisPubSub]
host = localhost host = localhost
port = 6381 port = 6381

32
reset_AIL.sh
View File

@ -70,7 +70,7 @@ function reset_dir {
if [ -d CRAWLED_SCREESHOT/ ]; then if [ -d CRAWLED_SCREESHOT/ ]; then
pushd CRAWLED_SCREESHOT/ pushd CRAWLED_SCREESHOT/
rm -r * rm -r *
echo 'cleaned CRAWLED_SCREESHOT' echo 'cleaned CRAWLED_SCREENSHOT'
popd popd
fi fi
@ -92,20 +92,18 @@ function reset_dir {
} }
function flush_DB_keep_user { function flush_DB_keep_user {
bash ${AIL_BIN}LAUNCH.sh -lav & bash ${AIL_BIN}LAUNCH.sh -lkv &
wait wait
echo "" echo ""
pushd redis/src pushd redis/src
./redis-cli -p 6382 -n 1 FLUSHDB; ./redis-cli -p 6383 -a ail_correls FLUSHDB;
./redis-cli -p 6382 -n 2 FLUSHDB; ./redis-cli -p 6383 -a ail_crawlers FLUSHDB;
./redis-cli -p 6382 -n 3 FLUSHDB; ./redis-cli -p 6383 -a ail_dups FLUSHDB;
./redis-cli -p 6382 -n 4 FLUSHDB; ./redis-cli -p 6383 -a ail_objs FLUSHDB;
./redis-cli -p 6382 -n 5 FLUSHDB; ./redis-cli -p 6383 -a ail_stats FLUSHDB;
./redis-cli -p 6382 -n 6 FLUSHDB; ./redis-cli -p 6383 -a ail_tags FLUSHDB;
./redis-cli -p 6382 -n 7 FLUSHDB; ./redis-cli -p 6383 -a ail_trackers FLUSHDB;
./redis-cli -p 6382 -n 8 FLUSHDB; echo "KVROCKS FLUSHED"
./redis-cli -p 6382 -n 9 FLUSHDB;
echo "ARDB FLUSHED"
popd popd
bash ${AIL_BIN}LAUNCH.sh -k bash ${AIL_BIN}LAUNCH.sh -k
} }
@ -113,7 +111,7 @@ function flush_DB_keep_user {
function validate_reset { function validate_reset {
echo -e $RED"WARNING: DELETE AIL DATA"$DEFAULT echo -e $RED"WARNING: DELETE AIL DATA"$DEFAULT
# Make sure the reseting is intentional # Make sure the reset is intentional
num=$(( ( RANDOM % 100 ) + 1 )) num=$(( ( RANDOM % 100 ) + 1 ))
echo -e $RED"To reset the platform, enter the following number: "$DEFAULT $num echo -e $RED"To reset the platform, enter the following number: "$DEFAULT $num
@ -121,7 +119,7 @@ function validate_reset {
if [ $userInput -eq $num ] if [ $userInput -eq $num ]
then then
echo "Reseting AIL..." echo "Resetting AIL..."
else else
echo "Wrong number" echo "Wrong number"
exit 1; exit 1;
@ -146,10 +144,10 @@ function soft_reset {
if [ $userInput -eq $num ] if [ $userInput -eq $num ]
then then
if [ -d DATA_ARDB/ ]; then if [ -d DATA_KVROCKS/ ]; then
pushd DATA_ARDB/ pushd DATA_KVROCKS/
rm -r * rm -r *
echo 'cleaned DATA_ARDB' echo 'cleaned DATA_KVROCKS'
popd popd
fi fi
fi fi