mirror of https://github.com/CIRCL/AIL-framework
chg: [ail] cleanup
Terrtia
parent
dea270f914
commit
88d3870230
|
|
@ -6,7 +6,6 @@
|
||||||
|
|
||||||
# Install Dirs
|
# Install Dirs
|
||||||
AILENV
|
AILENV
|
||||||
redis-leveldb
|
|
||||||
redis
|
redis
|
||||||
ardb
|
ardb
|
||||||
kvrocks
|
kvrocks
|
||||||
|
|
@ -42,8 +41,6 @@ var/www/server.key
|
||||||
|
|
||||||
# Local config
|
# Local config
|
||||||
configs/keys
|
configs/keys
|
||||||
bin/packages/core.cfg
|
|
||||||
bin/packages/config.cfg.backup
|
|
||||||
configs/core.cfg
|
configs/core.cfg
|
||||||
configs/core.cfg.backup
|
configs/core.cfg.backup
|
||||||
configs/update.cfg
|
configs/update.cfg
|
||||||
|
|
@ -55,8 +52,8 @@ configs/d4client_passiveDNS_conf/uuid
|
||||||
bin/trackers/yara/custom-rules/*
|
bin/trackers/yara/custom-rules/*
|
||||||
|
|
||||||
# Helper
|
# Helper
|
||||||
bin/helper/gen_cert/rootCA.*
|
tools/gen_cert/rootCA.*
|
||||||
bin/helper/gen_cert/server.*
|
tools/gen_cert/server.*
|
||||||
|
|
||||||
|
|
||||||
# Pystemon archives
|
# Pystemon archives
|
||||||
|
|
|
||||||
|
|
@ -1,852 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
# -*-coding:UTF-8 -*
|
|
||||||
|
|
||||||
from asciimatics.widgets import Frame, ListBox, Layout, Divider, Text, \
|
|
||||||
Button, Label
|
|
||||||
from asciimatics.scene import Scene
|
|
||||||
from asciimatics.screen import Screen
|
|
||||||
from asciimatics.exceptions import ResizeScreenError, NextScene, StopApplication
|
|
||||||
from asciimatics.event import Event
|
|
||||||
from asciimatics.event import KeyboardEvent, MouseEvent
|
|
||||||
import sys, os
|
|
||||||
import time, datetime
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import redis
|
|
||||||
import psutil
|
|
||||||
from subprocess import PIPE, Popen
|
|
||||||
from lib.objects.Items import Item
|
|
||||||
|
|
||||||
sys.path.append(os.environ['AIL_BIN'])
|
|
||||||
from lib import ConfigLoader
|
|
||||||
|
|
||||||
# CONFIG VARIABLES
|
|
||||||
kill_retry_threshold = 60 #1m
|
|
||||||
log_filename = "../logs/moduleInfo.log"
|
|
||||||
command_search_pid = "ps a -o pid,cmd | grep {}"
|
|
||||||
command_search_name = "ps a -o pid,cmd | grep {}"
|
|
||||||
command_restart_module = "screen -S \"Script\" -X screen -t \"{}\" bash -c \"./{}.py; read x\""
|
|
||||||
|
|
||||||
printarrayLog = [None]*14
|
|
||||||
lastTimeKillCommand = {}
|
|
||||||
|
|
||||||
# Used to pass information through scenes
|
|
||||||
current_selected_value = 0
|
|
||||||
current_selected_queue = ""
|
|
||||||
current_selected_action = ""
|
|
||||||
current_selected_amount = 0
|
|
||||||
|
|
||||||
# Map PID to Queue name (For restart and killing)
|
|
||||||
PID_NAME_DICO = {}
|
|
||||||
|
|
||||||
# Tables containing info for the dashboad
|
|
||||||
TABLES = {"running": [], "idle": [], "notRunning": [], "logs": [("No events recorded yet", 0)]}
|
|
||||||
TABLES_TITLES = {"running": "", "idle": "", "notRunning": "", "logs": ""}
|
|
||||||
TABLES_PADDING = {"running": [12, 23, 8, 8, 23, 10, 55, 11, 11, 12], "idle": [9, 23, 8, 12, 50], "notRunning": [9, 23, 35], "logs": [15, 23, 8, 50]}
|
|
||||||
|
|
||||||
# Indicator for the health of a queue (green(0), red(2), yellow(1))
|
|
||||||
QUEUE_STATUS = {}
|
|
||||||
|
|
||||||
# Maintain the state of the CPU objects
|
|
||||||
CPU_TABLE = {}
|
|
||||||
CPU_OBJECT_TABLE = {}
|
|
||||||
|
|
||||||
# Path of the current item for a pid
|
|
||||||
COMPLETE_PASTE_PATH_PER_PID = {}
|
|
||||||
|
|
||||||
'''
|
|
||||||
ASCIIMATICS WIDGETS EXTENSION
|
|
||||||
'''
|
|
||||||
|
|
||||||
# Custom listbox
|
|
||||||
class CListBox(ListBox):
|
|
||||||
|
|
||||||
def __init__(self, queue_name, *args, **kwargs):
|
|
||||||
self.queue_name = queue_name
|
|
||||||
super(CListBox, self).__init__(*args, **kwargs)
|
|
||||||
|
|
||||||
def update(self, frame_no):
|
|
||||||
self._options = TABLES[self.queue_name]
|
|
||||||
|
|
||||||
self._draw_label()
|
|
||||||
|
|
||||||
# Calculate new visible limits if needed.
|
|
||||||
width = self._w - self._offset
|
|
||||||
height = self._h
|
|
||||||
dx = dy = 0
|
|
||||||
|
|
||||||
# Clear out the existing box content
|
|
||||||
(colour, attr, bg) = self._frame.palette["field"]
|
|
||||||
for i in range(height):
|
|
||||||
self._frame.canvas.print_at(
|
|
||||||
" " * width,
|
|
||||||
self._x + self._offset + dx,
|
|
||||||
self._y + i + dy,
|
|
||||||
colour, attr, bg)
|
|
||||||
|
|
||||||
# Don't bother with anything else if there are no options to render.
|
|
||||||
if len(self._options) <= 0:
|
|
||||||
return
|
|
||||||
|
|
||||||
# Render visible portion of the text.
|
|
||||||
self._start_line = max(0, max(self._line - height + 1,
|
|
||||||
min(self._start_line, self._line)))
|
|
||||||
for i, (text, pid) in enumerate(self._options):
|
|
||||||
if self._start_line <= i < self._start_line + height:
|
|
||||||
colour, attr, bg = self._pick_colours("field", i == self._line)
|
|
||||||
self._frame.canvas.print_at(
|
|
||||||
"{:{width}}".format(text, width=width),
|
|
||||||
self._x + self._offset + dx,
|
|
||||||
self._y + i + dy - self._start_line,
|
|
||||||
colour, attr, bg)
|
|
||||||
|
|
||||||
# Pick color depending on queue health
|
|
||||||
if self.queue_name == "running":
|
|
||||||
if QUEUE_STATUS[pid] == 2:
|
|
||||||
queueStatus = Screen.COLOUR_RED
|
|
||||||
elif QUEUE_STATUS[pid] == 1:
|
|
||||||
queueStatus = Screen.COLOUR_YELLOW
|
|
||||||
else:
|
|
||||||
queueStatus = Screen.COLOUR_GREEN
|
|
||||||
|
|
||||||
self._frame.canvas.print_at(" ",
|
|
||||||
self._x + 9 + dx,
|
|
||||||
self._y + i + dy - self._start_line,
|
|
||||||
colour, attr, queueStatus)
|
|
||||||
|
|
||||||
|
|
||||||
def process_event(self, event):
|
|
||||||
if isinstance(event, KeyboardEvent):
|
|
||||||
if len(self._options) > 0 and event.key_code == Screen.KEY_UP:
|
|
||||||
# Move up one line in text - use value to trigger on_select.
|
|
||||||
self._line = max(0, self._line - 1)
|
|
||||||
self._line = min(self._line, len(self._options)-1) #If we move a line cursor from a line that has dissapear
|
|
||||||
self.value = self._options[self._line][1]
|
|
||||||
|
|
||||||
elif len(self._options) > 0 and event.key_code == Screen.KEY_DOWN:
|
|
||||||
# Move down one line in text - use value to trigger on_select.
|
|
||||||
self._line = min(len(self._options) - 1, self._line + 1)
|
|
||||||
self.value = self._options[self._line][1]
|
|
||||||
|
|
||||||
elif len(self._options) > 0 and event.key_code in [ord(' '), ord('\n')] :
|
|
||||||
global current_selected_value, current_selected_queue
|
|
||||||
if self.queue_name == "logs":
|
|
||||||
return event
|
|
||||||
current_selected_value = self.value
|
|
||||||
current_selected_queue = self.queue_name
|
|
||||||
self._frame.save()
|
|
||||||
raise NextScene("action_choice")
|
|
||||||
|
|
||||||
# Quit if press q
|
|
||||||
elif event.key_code == ord('q'):
|
|
||||||
Dashboard._quit()
|
|
||||||
|
|
||||||
else:
|
|
||||||
# Ignore any other key press.
|
|
||||||
return event
|
|
||||||
|
|
||||||
elif isinstance(event, MouseEvent):
|
|
||||||
# Mouse event - rebase coordinates to Frame context.
|
|
||||||
new_event = self._frame.rebase_event(event)
|
|
||||||
if event.buttons != 0:
|
|
||||||
if (len(self._options) > 0 and
|
|
||||||
self.is_mouse_over(new_event, include_label=False)):
|
|
||||||
# Use property to trigger events.
|
|
||||||
self._line = min(new_event.y - self._y,
|
|
||||||
len(self._options) - 1)
|
|
||||||
self.value = self._options[self._line][1]
|
|
||||||
# If clicked on button <k>, kill the queue
|
|
||||||
if self._x+2 <= new_event.x < self._x+4:
|
|
||||||
if self.queue_name in ["running", "idle"]:
|
|
||||||
kill_module(PID_NAME_DICO[int(self.value)], self.value)
|
|
||||||
else:
|
|
||||||
restart_module(self.value)
|
|
||||||
|
|
||||||
return
|
|
||||||
# Ignore other mouse events.
|
|
||||||
return event
|
|
||||||
else:
|
|
||||||
# Ignore other events
|
|
||||||
return event
|
|
||||||
|
|
||||||
|
|
||||||
# Custom label centered in the middle
|
|
||||||
class CLabel(Label):
|
|
||||||
def __init__(self, label, listTitle=False):
|
|
||||||
super(Label, self).__init__(None, tab_stop=False)
|
|
||||||
# Although this is a label, we don't want it to contribute to the layout
|
|
||||||
# tab calculations, so leave internal `_label` value as None.
|
|
||||||
self._text = label
|
|
||||||
self.listTitle = listTitle
|
|
||||||
self._required_height = 1
|
|
||||||
|
|
||||||
def set_layout(self, x, y, offset, w, h):
|
|
||||||
# Do the usual layout work. then recalculate exact x/w values for the
|
|
||||||
# rendered button.
|
|
||||||
super(Label, self).set_layout(x, y, offset, w, h)
|
|
||||||
self._x += max(0, (self._w - self._offset - len(self._text)) // 2) if not self.listTitle else 0
|
|
||||||
self._w = min(self._w, len(self._text))
|
|
||||||
|
|
||||||
def update(self, frame_no):
|
|
||||||
(colour, attr, bg) = self._frame.palette["title"]
|
|
||||||
colour = Screen.COLOUR_YELLOW if not self.listTitle else colour
|
|
||||||
self._frame.canvas.print_at(
|
|
||||||
self._text, self._x, self._y, colour, attr, bg)
|
|
||||||
|
|
||||||
'''
|
|
||||||
END EXTENSION
|
|
||||||
'''
|
|
||||||
|
|
||||||
'''
|
|
||||||
SCENE DEFINITION
|
|
||||||
'''
|
|
||||||
|
|
||||||
class Dashboard(Frame):
|
|
||||||
def __init__(self, screen):
|
|
||||||
super(Dashboard, self).__init__(screen,
|
|
||||||
screen.height,
|
|
||||||
screen.width,
|
|
||||||
hover_focus=True,
|
|
||||||
reduce_cpu=True)
|
|
||||||
|
|
||||||
self._list_view_run_queue = CListBox(
|
|
||||||
"running",
|
|
||||||
screen.height // 2,
|
|
||||||
[], name="LIST")
|
|
||||||
self._list_view_idle_queue = CListBox(
|
|
||||||
"idle",
|
|
||||||
screen.height // 2,
|
|
||||||
[], name="LIST")
|
|
||||||
self._list_view_noRunning = CListBox(
|
|
||||||
"notRunning",
|
|
||||||
screen.height // 5,
|
|
||||||
[], name="LIST")
|
|
||||||
self._list_view_Log = CListBox(
|
|
||||||
"logs",
|
|
||||||
screen.height // 4,
|
|
||||||
[], name="LIST")
|
|
||||||
#self._list_view_Log.disabled = True
|
|
||||||
|
|
||||||
|
|
||||||
#Running Queues
|
|
||||||
layout = Layout([100])
|
|
||||||
self.add_layout(layout)
|
|
||||||
text_rq = CLabel("Running Queues")
|
|
||||||
layout.add_widget(text_rq)
|
|
||||||
layout.add_widget(CLabel(TABLES_TITLES["running"], listTitle=True))
|
|
||||||
layout.add_widget(self._list_view_run_queue)
|
|
||||||
layout.add_widget(Divider())
|
|
||||||
|
|
||||||
#Idling Queues
|
|
||||||
layout2 = Layout([1,1])
|
|
||||||
self.add_layout(layout2)
|
|
||||||
text_iq = CLabel("Idling Queues")
|
|
||||||
layout2.add_widget(text_iq, 0)
|
|
||||||
layout2.add_widget(CLabel(TABLES_TITLES["idle"], listTitle=True), 0)
|
|
||||||
layout2.add_widget(self._list_view_idle_queue, 0)
|
|
||||||
#Non Running Queues
|
|
||||||
text_nq = CLabel("Queues not running")
|
|
||||||
layout2.add_widget(text_nq, 1)
|
|
||||||
layout2.add_widget(CLabel(TABLES_TITLES["notRunning"], listTitle=True), 1)
|
|
||||||
layout2.add_widget(self._list_view_noRunning, 1)
|
|
||||||
layout2.add_widget(Divider(), 1)
|
|
||||||
#Log
|
|
||||||
text_l = CLabel("Logs")
|
|
||||||
layout2.add_widget(text_l, 1)
|
|
||||||
layout2.add_widget(CLabel(TABLES_TITLES["logs"], listTitle=True), 1)
|
|
||||||
layout2.add_widget(self._list_view_Log, 1)
|
|
||||||
|
|
||||||
self.fix()
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _quit():
|
|
||||||
raise StopApplication("User pressed quit")
|
|
||||||
|
|
||||||
class Confirm(Frame):
|
|
||||||
def __init__(self, screen):
|
|
||||||
super(Confirm, self).__init__(screen,
|
|
||||||
screen.height * 1 // 8,
|
|
||||||
screen.width * 1 // 3,
|
|
||||||
hover_focus=True,
|
|
||||||
on_load=self._setValue,
|
|
||||||
title="Confirm action",
|
|
||||||
reduce_cpu=True)
|
|
||||||
|
|
||||||
# Create the form for displaying the list of contacts.
|
|
||||||
layout = Layout([100], fill_frame=True)
|
|
||||||
self.add_layout(layout)
|
|
||||||
self.label = CLabel("{} module {} {}?")
|
|
||||||
layout.add_widget(Label(" "))
|
|
||||||
layout.add_widget(self.label)
|
|
||||||
layout2 = Layout([1,1])
|
|
||||||
self.add_layout(layout2)
|
|
||||||
layout2.add_widget(Button("Ok", self._ok), 0)
|
|
||||||
layout2.add_widget(Button("Cancel", self._cancel), 1)
|
|
||||||
|
|
||||||
self.fix()
|
|
||||||
|
|
||||||
def _ok(self):
|
|
||||||
global current_selected_value, current_selected_queue, current_selected_action, current_selected_amount
|
|
||||||
if current_selected_action == "KILL":
|
|
||||||
kill_module(PID_NAME_DICO[int(current_selected_value)], current_selected_value)
|
|
||||||
else:
|
|
||||||
count = int(current_selected_amount) #Number of queue to start
|
|
||||||
if current_selected_queue in ["running", "idle"]:
|
|
||||||
restart_module(PID_NAME_DICO[int(current_selected_value)], count)
|
|
||||||
else:
|
|
||||||
restart_module(current_selected_value, count)
|
|
||||||
|
|
||||||
current_selected_value = 0
|
|
||||||
current_selected_amount = 0
|
|
||||||
current_selected_action = ""
|
|
||||||
self.label._text = "{} module {} {}?"
|
|
||||||
self.save()
|
|
||||||
raise NextScene("dashboard")
|
|
||||||
|
|
||||||
def _cancel(self):
|
|
||||||
global current_selected_value
|
|
||||||
current_selected_value = 0
|
|
||||||
current_selected_amount = 0
|
|
||||||
current_selected_action = ""
|
|
||||||
self.label._text = "{} module {} {}?"
|
|
||||||
self.save()
|
|
||||||
raise NextScene("dashboard")
|
|
||||||
|
|
||||||
def _setValue(self):
|
|
||||||
global current_selected_value, current_selected_queue, current_selected_action, current_selected_amount
|
|
||||||
if current_selected_queue in ["running", "idle"]:
|
|
||||||
action = current_selected_action if current_selected_action == "KILL" else current_selected_action +" "+ str(current_selected_amount) + "x"
|
|
||||||
modulename = PID_NAME_DICO[int(current_selected_value)]
|
|
||||||
pid = current_selected_value
|
|
||||||
else:
|
|
||||||
action = current_selected_action + " " + str(current_selected_amount) + "x"
|
|
||||||
modulename = current_selected_value
|
|
||||||
pid = ""
|
|
||||||
self.label._text = self.label._text.format(action, modulename, pid)
|
|
||||||
|
|
||||||
class Action_choice(Frame):
|
|
||||||
def __init__(self, screen):
|
|
||||||
super(Action_choice, self).__init__(screen,
|
|
||||||
screen.height * 1 // 8,
|
|
||||||
screen.width * 1 // 2,
|
|
||||||
hover_focus=True,
|
|
||||||
on_load=self._setValue,
|
|
||||||
title="Confirm action",
|
|
||||||
reduce_cpu=True)
|
|
||||||
|
|
||||||
# Create the form for displaying the list of contacts.
|
|
||||||
layout = Layout([100], fill_frame=True)
|
|
||||||
self.add_layout(layout)
|
|
||||||
self.label = CLabel("Choose action on module {} {}")
|
|
||||||
layout.add_widget(self.label)
|
|
||||||
layout2 = Layout([1,1,1,1])
|
|
||||||
self.add_layout(layout2)
|
|
||||||
layout2.add_widget(Button("Cancel", self._cancel), 0)
|
|
||||||
self._ShowPasteBtn = Button("Show current paste", self._showpaste)
|
|
||||||
layout2.add_widget(self._ShowPasteBtn, 1)
|
|
||||||
self._killBtn = Button("KILL", self._kill)
|
|
||||||
layout2.add_widget(self._killBtn, 2)
|
|
||||||
layout2.add_widget(Button("START", self._start), 3)
|
|
||||||
layout3 = Layout([1,1,1,1])
|
|
||||||
self.add_layout(layout3)
|
|
||||||
self.textEdit = Text("Amount", "amount")
|
|
||||||
layout3.add_widget(self.textEdit, 3)
|
|
||||||
|
|
||||||
self.fix()
|
|
||||||
|
|
||||||
def _kill(self):
|
|
||||||
global current_selected_action
|
|
||||||
current_selected_action = "KILL"
|
|
||||||
self.label._text = "Choose action on module {} {}"
|
|
||||||
self.save()
|
|
||||||
raise NextScene("confirm")
|
|
||||||
|
|
||||||
def _start(self):
|
|
||||||
global current_selected_action, current_selected_amount
|
|
||||||
current_selected_action = "START"
|
|
||||||
try:
|
|
||||||
count = int(self.textEdit.value)
|
|
||||||
count = count if count < 20 else 1
|
|
||||||
except Exception:
|
|
||||||
count = 1
|
|
||||||
current_selected_amount = count
|
|
||||||
self.label._text = "Choose action on module {} {}"
|
|
||||||
self.save()
|
|
||||||
raise NextScene("confirm")
|
|
||||||
|
|
||||||
|
|
||||||
def _cancel(self):
|
|
||||||
global current_selected_value
|
|
||||||
current_selected_value = 0
|
|
||||||
self.label._text = "Choose action on module {} {}"
|
|
||||||
self.save()
|
|
||||||
raise NextScene("dashboard")
|
|
||||||
|
|
||||||
def _showpaste(self):
|
|
||||||
self.label._text = "Choose action on module {} {}"
|
|
||||||
self.save()
|
|
||||||
raise NextScene("show_paste")
|
|
||||||
|
|
||||||
def _setValue(self):
|
|
||||||
self._killBtn.disabled = False
|
|
||||||
self._ShowPasteBtn.disabled = False
|
|
||||||
global current_selected_value, current_selected_queue
|
|
||||||
if current_selected_queue in ["running", "idle"]:
|
|
||||||
modulename = PID_NAME_DICO[int(current_selected_value)]
|
|
||||||
pid = current_selected_value
|
|
||||||
else:
|
|
||||||
self._killBtn.disabled = True
|
|
||||||
self._ShowPasteBtn.disabled = True
|
|
||||||
modulename = current_selected_value
|
|
||||||
pid = ""
|
|
||||||
self.label._text = self.label._text.format(modulename, pid)
|
|
||||||
|
|
||||||
class Show_paste(Frame):
|
|
||||||
def __init__(self, screen):
|
|
||||||
super(Show_paste, self).__init__(screen,
|
|
||||||
screen.height,
|
|
||||||
screen.width,
|
|
||||||
hover_focus=True,
|
|
||||||
on_load=self._setValue,
|
|
||||||
title="Show current paste",
|
|
||||||
reduce_cpu=True)
|
|
||||||
|
|
||||||
layout = Layout([100], fill_frame=True)
|
|
||||||
self.layout = layout
|
|
||||||
self.add_layout(layout)
|
|
||||||
|
|
||||||
self.label_list = []
|
|
||||||
self.num_label = 42 # Number of line available for displaying the paste
|
|
||||||
for i in range(self.num_label):
|
|
||||||
self.label_list += [Label("THE PASTE CONTENT " + str(i))]
|
|
||||||
layout.add_widget(self.label_list[i])
|
|
||||||
|
|
||||||
layout2 = Layout([100])
|
|
||||||
self.add_layout(layout2)
|
|
||||||
layout2.add_widget(Button("Ok", self._ok), 0)
|
|
||||||
self.fix()
|
|
||||||
|
|
||||||
def _ok(self):
|
|
||||||
global current_selected_value, current_selected_queue, current_selected_action, current_selected_amount
|
|
||||||
current_selected_value = 0
|
|
||||||
current_selected_amount = 0
|
|
||||||
current_selected_action = ""
|
|
||||||
self.save()
|
|
||||||
raise NextScene("dashboard")
|
|
||||||
|
|
||||||
def _setValue(self):
|
|
||||||
try:
|
|
||||||
#Verify that the module have a paste
|
|
||||||
if COMPLETE_PASTE_PATH_PER_PID[current_selected_value] is None:
|
|
||||||
self.label_list[0]._text = "No paste for this module"
|
|
||||||
for i in range(1,self.num_label):
|
|
||||||
self.label_list[i]._text = ""
|
|
||||||
return
|
|
||||||
|
|
||||||
item = Item(COMPLETE_PASTE_PATH_PER_PID[current_selected_value])
|
|
||||||
old_content = item.get_content()[0:4000] # Limit number of char to be displayed
|
|
||||||
|
|
||||||
# Replace unprintable char by ?
|
|
||||||
content = ""
|
|
||||||
for i, c in enumerate(old_content):
|
|
||||||
if ord(c) > 127: # Used to avoid printing unprintable char
|
|
||||||
content += '?'
|
|
||||||
elif c == "\t": # Replace tab by 4 spaces
|
|
||||||
content += " "
|
|
||||||
else:
|
|
||||||
content += c
|
|
||||||
|
|
||||||
# Print in the correct label, END or more
|
|
||||||
to_print = ""
|
|
||||||
i = 0
|
|
||||||
for line in content.split("\n"):
|
|
||||||
if i > self.num_label - 2:
|
|
||||||
break
|
|
||||||
self.label_list[i]._text = str(i) + ". " + line.replace("\r","")
|
|
||||||
i += 1
|
|
||||||
|
|
||||||
if i > self.num_label - 2:
|
|
||||||
self.label_list[i]._text = "- ALL PASTE NOT DISPLAYED -"
|
|
||||||
i += 1
|
|
||||||
else:
|
|
||||||
self.label_list[i]._text = "- END of PASTE -"
|
|
||||||
i += 1
|
|
||||||
|
|
||||||
while i<self.num_label: # Clear out remaining lines
|
|
||||||
self.label_list[i]._text = ""
|
|
||||||
i += 1
|
|
||||||
|
|
||||||
except OSError as e:
|
|
||||||
self.label_list[0]._text = "Error during parsing the filepath. Please, check manually"
|
|
||||||
self.label_list[1]._text = COMPLETE_PASTE_PATH_PER_PID[current_selected_value]
|
|
||||||
for i in range(2,self.num_label):
|
|
||||||
self.label_list[i]._text = ""
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
if current_selected_value in COMPLETE_PASTE_PATH_PER_PID:
|
|
||||||
self.label_list[0]._text = "Error while displaying the paste: " + COMPLETE_PASTE_PATH_PER_PID[current_selected_value]
|
|
||||||
else:
|
|
||||||
self.label_list[0]._text = "Error Generic exception caught"
|
|
||||||
self.label_list[1]._text = str(e)
|
|
||||||
for i in range(2,self.num_label):
|
|
||||||
self.label_list[i]._text = ""
|
|
||||||
|
|
||||||
|
|
||||||
'''
|
|
||||||
END SCENES DEFINITION
|
|
||||||
'''
|
|
||||||
|
|
||||||
|
|
||||||
'''
|
|
||||||
MANAGE MODULES AND GET INFOS
|
|
||||||
'''
|
|
||||||
|
|
||||||
def getPid(module):
|
|
||||||
p = Popen([command_search_pid.format(module+".py")], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
|
|
||||||
for line in p.stdout:
|
|
||||||
splittedLine = line.split()
|
|
||||||
if 'python3' in splittedLine:
|
|
||||||
return int(splittedLine[0])
|
|
||||||
return None
|
|
||||||
|
|
||||||
def clearRedisModuleInfo():
|
|
||||||
for k in server.keys("MODULE_*"):
|
|
||||||
server.delete(k)
|
|
||||||
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
|
||||||
log(([str(inst_time).split(' ')[1], "*", "-", "Cleared redis module info"], 0))
|
|
||||||
|
|
||||||
def cleanRedis():
|
|
||||||
for k in server.keys("MODULE_TYPE_*"):
|
|
||||||
moduleName = k[12:].split('_')[0]
|
|
||||||
for pid in server.smembers(k):
|
|
||||||
flag_pid_valid = False
|
|
||||||
proc = Popen([command_search_name.format(pid)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
|
|
||||||
try:
|
|
||||||
for line in proc.stdout:
|
|
||||||
line = line.decode('utf8')
|
|
||||||
splittedLine = line.split()
|
|
||||||
if ('python3.5' in splittedLine or 'python3' in splittedLine or 'python' in splittedLine):
|
|
||||||
moduleCommand = "./"+moduleName + ".py"
|
|
||||||
moduleCommand2 = moduleName + ".py"
|
|
||||||
if(moduleCommand in splittedLine or moduleCommand2 in splittedLine):
|
|
||||||
flag_pid_valid = True
|
|
||||||
|
|
||||||
|
|
||||||
if not flag_pid_valid:
|
|
||||||
#print flag_pid_valid, 'cleaning', pid, 'in', k
|
|
||||||
server.srem(k, pid)
|
|
||||||
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
|
||||||
log(([str(inst_time).split(' ')[1], moduleName, pid, "Cleared invalid pid in " + (k)], 0))
|
|
||||||
|
|
||||||
#Error due to resize, interrupted sys call
|
|
||||||
except IOError as e:
|
|
||||||
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
|
||||||
log(([str(inst_time).split(' ')[1], " - ", " - ", "Cleaning fail due to resize."], 0))
|
|
||||||
|
|
||||||
|
|
||||||
def restart_module(module, count=1):
|
|
||||||
for i in range(count):
|
|
||||||
p2 = Popen([command_restart_module.format(module, module)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
|
|
||||||
time.sleep(0.2)
|
|
||||||
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
|
||||||
log(([str(inst_time).split(' ')[1], module, "?", "Restarted " + str(count) + "x"], 0))
|
|
||||||
|
|
||||||
|
|
||||||
def kill_module(module, pid):
|
|
||||||
#print '-> trying to kill module:', module
|
|
||||||
|
|
||||||
if pid is None:
|
|
||||||
#print 'pid was None'
|
|
||||||
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
|
||||||
log(([str(inst_time).split(' ')[1], module, pid, "PID was None"], 0))
|
|
||||||
pid = getPid(module)
|
|
||||||
else: #Verify that the pid is at least in redis
|
|
||||||
if server.exists("MODULE_"+module+"_"+str(pid)) == 0:
|
|
||||||
return
|
|
||||||
|
|
||||||
lastTimeKillCommand[pid] = int(time.time())
|
|
||||||
if pid is not None:
|
|
||||||
try:
|
|
||||||
p = psutil.Process(int(pid))
|
|
||||||
p.terminate()
|
|
||||||
except Exception as e:
|
|
||||||
#print pid, 'already killed'
|
|
||||||
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
|
||||||
log(([str(inst_time).split(' ')[1], module, pid, "Already killed"], 0))
|
|
||||||
return
|
|
||||||
time.sleep(0.2)
|
|
||||||
if not p.is_running():
|
|
||||||
#print module, 'has been killed'
|
|
||||||
#print 'restarting', module, '...'
|
|
||||||
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
|
||||||
log(([str(inst_time).split(' ')[1], module, pid, "Killed"], 0))
|
|
||||||
#restart_module(module)
|
|
||||||
|
|
||||||
else:
|
|
||||||
#print 'killing failed, retrying...'
|
|
||||||
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
|
||||||
log(([str(inst_time).split(' ')[1], module, pid, "Killing #1 failed."], 0))
|
|
||||||
|
|
||||||
p.terminate()
|
|
||||||
if not p.is_running():
|
|
||||||
#print module, 'has been killed'
|
|
||||||
#print 'restarting', module, '...'
|
|
||||||
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
|
||||||
log(([str(inst_time).split(' ')[1], module, pid, "Killed"], 0))
|
|
||||||
#restart_module(module)
|
|
||||||
else:
|
|
||||||
#print 'killing failed!'
|
|
||||||
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
|
||||||
log(([str(inst_time).split(' ')[1], module, pid, "Killing failed!"], 0))
|
|
||||||
else:
|
|
||||||
#print 'Module does not exist'
|
|
||||||
inst_time = datetime.datetime.fromtimestamp(int(time.time()))
|
|
||||||
log(([str(inst_time).split(' ')[1], module, pid, "Killing failed, module not found"], 0))
|
|
||||||
cleanRedis()
|
|
||||||
|
|
||||||
|
|
||||||
# Fetch the data for all queue
|
|
||||||
def fetchQueueData():
|
|
||||||
|
|
||||||
all_queue = set()
|
|
||||||
printarray_running = []
|
|
||||||
printarray_idle = []
|
|
||||||
printarray_notrunning = []
|
|
||||||
for queue, card in iter(server.hgetall("queues").items()):
|
|
||||||
all_queue.add(queue)
|
|
||||||
key = "MODULE_" + queue + "_"
|
|
||||||
keySet = "MODULE_TYPE_" + queue
|
|
||||||
array_module_type = []
|
|
||||||
|
|
||||||
for moduleNum in server.smembers(keySet):
|
|
||||||
value = server.get(key + str(moduleNum))
|
|
||||||
complete_paste_path = ( server.get(key + str(moduleNum) + "_PATH") )
|
|
||||||
if(complete_paste_path is not None):
|
|
||||||
complete_paste_path = complete_paste_path
|
|
||||||
COMPLETE_PASTE_PATH_PER_PID[moduleNum] = complete_paste_path
|
|
||||||
|
|
||||||
if value is not None:
|
|
||||||
timestamp, path = value.split(", ")
|
|
||||||
if timestamp is not None and path is not None:
|
|
||||||
# Queue health
|
|
||||||
startTime_readable = datetime.datetime.fromtimestamp(int(timestamp))
|
|
||||||
processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0]
|
|
||||||
if ((datetime.datetime.now() - startTime_readable).total_seconds()) > args.treshold:
|
|
||||||
QUEUE_STATUS[moduleNum] = 2
|
|
||||||
elif ((datetime.datetime.now() - startTime_readable).total_seconds()) > args.treshold/2:
|
|
||||||
QUEUE_STATUS[moduleNum] = 1
|
|
||||||
else:
|
|
||||||
QUEUE_STATUS[moduleNum] = 0
|
|
||||||
|
|
||||||
# Queue contain elements
|
|
||||||
if int(card) > 0:
|
|
||||||
# Queue need to be killed
|
|
||||||
if int((datetime.datetime.now() - startTime_readable).total_seconds()) > args.treshold:
|
|
||||||
log(([str(time.time()), queue, "-", "ST:"+str(timestamp)+" PT:"+str(time.time()-float(timestamp))], 0), True, show_in_board=False)
|
|
||||||
try:
|
|
||||||
last_kill_try = time.time() - lastTimeKillCommand[moduleNum]
|
|
||||||
except KeyError:
|
|
||||||
last_kill_try = kill_retry_threshold+1
|
|
||||||
if args.autokill == 1 and last_kill_try > kill_retry_threshold :
|
|
||||||
kill_module(queue, int(moduleNum))
|
|
||||||
|
|
||||||
# Create CPU objects
|
|
||||||
try:
|
|
||||||
cpu_percent = CPU_OBJECT_TABLE[int(moduleNum)].cpu_percent()
|
|
||||||
CPU_TABLE[moduleNum].insert(1, cpu_percent)
|
|
||||||
cpu_avg = sum(CPU_TABLE[moduleNum])/len(CPU_TABLE[moduleNum])
|
|
||||||
if len(CPU_TABLE[moduleNum]) > args.refresh*10:
|
|
||||||
CPU_TABLE[moduleNum].pop()
|
|
||||||
|
|
||||||
mem_percent = CPU_OBJECT_TABLE[int(moduleNum)].memory_percent()
|
|
||||||
except psutil.NoSuchProcess:
|
|
||||||
del CPU_OBJECT_TABLE[int(moduleNum)]
|
|
||||||
del CPU_TABLE[moduleNum]
|
|
||||||
cpu_percent = 0
|
|
||||||
cpu_avg = cpu_percent
|
|
||||||
mem_percent = 0
|
|
||||||
except KeyError:
|
|
||||||
#print('key error2')
|
|
||||||
try:
|
|
||||||
CPU_OBJECT_TABLE[int(moduleNum)] = psutil.Process(int(moduleNum))
|
|
||||||
cpu_percent = CPU_OBJECT_TABLE[int(moduleNum)].cpu_percent()
|
|
||||||
CPU_TABLE[moduleNum] = []
|
|
||||||
cpu_avg = cpu_percent
|
|
||||||
mem_percent = CPU_OBJECT_TABLE[int(moduleNum)].memory_percent()
|
|
||||||
except psutil.NoSuchProcess:
|
|
||||||
cpu_percent = 0
|
|
||||||
cpu_avg = cpu_percent
|
|
||||||
mem_percent = 0
|
|
||||||
|
|
||||||
array_module_type.append( ([" <K> [ ]", str(queue), str(moduleNum), str(card), str(startTime_readable),
|
|
||||||
str(processed_time_readable), str(path), "{0:.2f}".format(cpu_percent)+"%",
|
|
||||||
"{0:.2f}".format(mem_percent)+"%", "{0:.2f}".format(cpu_avg)+"%"], moduleNum) )
|
|
||||||
|
|
||||||
else:
|
|
||||||
printarray_idle.append( ([" <K> ", str(queue), str(moduleNum), str(processed_time_readable), str(path)], moduleNum) )
|
|
||||||
|
|
||||||
PID_NAME_DICO[int(moduleNum)] = str(queue)
|
|
||||||
#array_module_type.sort(lambda x,y: cmp(x[0][4], y[0][4]), reverse=True) #Sort by num of pastes
|
|
||||||
for e in array_module_type:
|
|
||||||
printarray_running.append(e)
|
|
||||||
|
|
||||||
for curr_queue in module_file_array:
|
|
||||||
if curr_queue not in all_queue: #Module not running by default
|
|
||||||
printarray_notrunning.append( ([" <S> ", curr_queue, "Not running by default"], curr_queue) )
|
|
||||||
else: #Module did not process anything yet
|
|
||||||
if len(list(server.smembers('MODULE_TYPE_'+curr_queue))) == 0:
|
|
||||||
if curr_queue not in no_info_modules:
|
|
||||||
no_info_modules[curr_queue] = int(time.time())
|
|
||||||
printarray_notrunning.append( ([" <S> ", curr_queue, "No data"], curr_queue) )
|
|
||||||
else:
|
|
||||||
#If no info since long time, try to kill
|
|
||||||
if args.autokill == 1:
|
|
||||||
if int(time.time()) - no_info_modules[curr_queue] > args.treshold:
|
|
||||||
kill_module(curr_queue, None)
|
|
||||||
no_info_modules[curr_queue] = int(time.time())
|
|
||||||
printarray_notrunning.append( ([" <S> ", curr_queue, "Stuck or idle, restarting in " + str(abs(args.treshold - (int(time.time()) - no_info_modules[curr_queue]))) + "s"], curr_queue) )
|
|
||||||
else:
|
|
||||||
printarray_notrunning.append( ([" <S> ", curr_queue, "Stuck or idle, restarting disabled"], curr_queue) )
|
|
||||||
|
|
||||||
|
|
||||||
printarray_running.sort(key=lambda x: x[0], reverse=False)
|
|
||||||
printarray_idle.sort(key=lambda x: x[0], reverse=False)
|
|
||||||
printarray_notrunning.sort(key=lambda x: x[0][1], reverse=False)
|
|
||||||
|
|
||||||
printstring_running = format_string(printarray_running, TABLES_PADDING["running"])
|
|
||||||
printstring_idle = format_string(printarray_idle, TABLES_PADDING["idle"])
|
|
||||||
printstring_notrunning = format_string(printarray_notrunning, TABLES_PADDING["notRunning"])
|
|
||||||
|
|
||||||
return {"running": printstring_running, "idle": printstring_idle, "notRunning": printstring_notrunning}
|
|
||||||
|
|
||||||
# Format the input string with its related padding to have collumn like text in CListBox
|
|
||||||
def format_string(tab, padding_row):
|
|
||||||
printstring = []
|
|
||||||
for row in tab:
|
|
||||||
if row is None:
|
|
||||||
continue
|
|
||||||
the_array = row[0]
|
|
||||||
the_pid = row[1]
|
|
||||||
|
|
||||||
text=""
|
|
||||||
for ite, elem in enumerate(the_array):
|
|
||||||
|
|
||||||
if elem is not None and type(elem) is str:
|
|
||||||
if len(elem) > padding_row[ite]:
|
|
||||||
text += "*" + elem[-padding_row[ite]+6:]
|
|
||||||
padd_off = " "*5
|
|
||||||
else:
|
|
||||||
text += elem
|
|
||||||
padd_off = " "*0
|
|
||||||
text += (padding_row[ite] - len(elem))*" " + padd_off
|
|
||||||
|
|
||||||
printstring.append( (text, the_pid) )
|
|
||||||
return printstring
|
|
||||||
|
|
||||||
def log(data, write_on_disk=False, show_in_board=True):
|
|
||||||
if show_in_board:
|
|
||||||
printarrayLog.insert(0, data)
|
|
||||||
printarrayLog.pop()
|
|
||||||
if write_on_disk:
|
|
||||||
with open(log_filename, 'a') as log:
|
|
||||||
log.write(json.dumps(data[0]) + "\n")
|
|
||||||
|
|
||||||
'''
|
|
||||||
END MANAGE
|
|
||||||
'''
|
|
||||||
|
|
||||||
def demo(screen):
|
|
||||||
dashboard = Dashboard(screen)
|
|
||||||
confirm = Confirm(screen)
|
|
||||||
action_choice = Action_choice(screen)
|
|
||||||
show_paste = Show_paste(screen)
|
|
||||||
scenes = [
|
|
||||||
Scene([dashboard], -1, name="dashboard"),
|
|
||||||
Scene([action_choice], -1, name="action_choice"),
|
|
||||||
Scene([confirm], -1, name="confirm"),
|
|
||||||
Scene([show_paste], -1, name="show_paste"),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
screen.set_scenes(scenes)
|
|
||||||
time_cooldown = time.time() # Cooldown before refresh
|
|
||||||
global TABLES
|
|
||||||
while True:
|
|
||||||
#Stop on resize
|
|
||||||
if screen.has_resized():
|
|
||||||
screen._scenes[screen._scene_index].exit()
|
|
||||||
raise ResizeScreenError("Screen resized", screen._scenes[screen._scene_index])
|
|
||||||
|
|
||||||
if time.time() - time_cooldown > args.refresh:
|
|
||||||
cleanRedis()
|
|
||||||
for key, val in iter(fetchQueueData().items()): #fetch data and put it into the tables
|
|
||||||
TABLES[key] = val
|
|
||||||
TABLES["logs"] = format_string(printarrayLog, TABLES_PADDING["logs"])
|
|
||||||
|
|
||||||
#refresh dashboad only if the scene is active (no value selected)
|
|
||||||
if current_selected_value == 0:
|
|
||||||
dashboard._update(None)
|
|
||||||
screen.refresh()
|
|
||||||
time_cooldown = time.time()
|
|
||||||
screen.draw_next_frame()
|
|
||||||
time.sleep(0.02) #time between screen refresh (For UI navigation, not data actualisation)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description='Show info concerning running modules and log suspected stucked modules. May be use to automatically kill and restart stucked one.')
|
|
||||||
parser.add_argument('-r', '--refresh', type=int, required=False, default=5, help='Refresh rate')
|
|
||||||
parser.add_argument('-t', '--treshold', type=int, required=False, default=60*10*1, help='Refresh rate')
|
|
||||||
parser.add_argument('-k', '--autokill', type=int, required=False, default=0, help='Enable auto kill option (1 for TRUE, anything else for FALSE)')
|
|
||||||
parser.add_argument('-c', '--clear', type=int, required=False, default=0, help='Clear the current module information (Used to clear data from old launched modules)')
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
config_loader = ConfigLoader.ConfigLoader()
|
|
||||||
|
|
||||||
# REDIS #
|
|
||||||
server = config_loader.get_redis_conn("Redis_Queues")
|
|
||||||
config_loader = None
|
|
||||||
|
|
||||||
if args.clear == 1:
|
|
||||||
clearRedisModuleInfo()
|
|
||||||
|
|
||||||
lastTime = datetime.datetime.now()
|
|
||||||
|
|
||||||
module_file_array = set()
|
|
||||||
no_info_modules = {}
|
|
||||||
path_allmod = os.path.join(os.environ['AIL_HOME'], 'doc/all_modules.txt')
|
|
||||||
|
|
||||||
try:
|
|
||||||
with open(path_allmod, 'r') as module_file:
|
|
||||||
for line in module_file:
|
|
||||||
module_file_array.add(line[:-1])
|
|
||||||
except IOError as e:
|
|
||||||
if e.errno == 2: #module_file not found, creating a new one
|
|
||||||
print(path_allmod + " not found.\nCreating a new one.")
|
|
||||||
os.system("./../doc/generate_modules_data_flow_graph.sh")
|
|
||||||
with open(path_allmod, 'r') as module_file:
|
|
||||||
for line in module_file:
|
|
||||||
module_file_array.add(line[:-1])
|
|
||||||
cleanRedis()
|
|
||||||
|
|
||||||
|
|
||||||
TABLES_TITLES["running"] = format_string([([" Action", "Queue name", "PID", "#", "S Time", "R Time", "Processed element", "CPU %", "Mem %", "Avg CPU%"],0)], TABLES_PADDING["running"])[0][0]
|
|
||||||
TABLES_TITLES["idle"] = format_string([([" Action", "Queue", "PID", "Idle Time", "Last paste hash"],0)], TABLES_PADDING["idle"])[0][0]
|
|
||||||
TABLES_TITLES["notRunning"] = format_string([([" Action", "Queue", "State"],0)], TABLES_PADDING["notRunning"])[0][0]
|
|
||||||
TABLES_TITLES["logs"] = format_string([(["Time", "Module", "PID", "Info"],0)], TABLES_PADDING["logs"])[0][0]
|
|
||||||
|
|
||||||
try:
|
|
||||||
input("Press < ENTER > to launch the manager...")
|
|
||||||
except SyntaxError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
last_scene = None
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
Screen.wrapper(demo)
|
|
||||||
sys.exit(0)
|
|
||||||
except ResizeScreenError as e:
|
|
||||||
pass
|
|
||||||
except StopApplication:
|
|
||||||
sys.exit(0)
|
|
||||||
|
|
@ -1,78 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
# -*-coding:UTF-8 -*
|
|
||||||
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import logging.config
|
|
||||||
import traceback
|
|
||||||
import smtplib
|
|
||||||
from email.mime.multipart import MIMEMultipart
|
|
||||||
from email.mime.text import MIMEText
|
|
||||||
|
|
||||||
sys.path.append(os.environ['AIL_BIN'])
|
|
||||||
from lib import ail_logger
|
|
||||||
from lib import ConfigLoader
|
|
||||||
|
|
||||||
"""
|
|
||||||
This module allows the global configuration and management of notification settings and methods.
|
|
||||||
"""
|
|
||||||
|
|
||||||
config_loader = ConfigLoader.ConfigLoader()
|
|
||||||
|
|
||||||
logging.config.dictConfig(ail_logger.get_config())
|
|
||||||
logger = logging.getLogger()
|
|
||||||
|
|
||||||
def sendEmailNotification(recipient, mail_subject, mail_body):
|
|
||||||
|
|
||||||
sender = config_loader.get_config_str("Notifications", "sender")
|
|
||||||
sender_user = config_loader.get_config_str("Notifications", "sender_user")
|
|
||||||
sender_host = config_loader.get_config_str("Notifications", "sender_host")
|
|
||||||
sender_port = config_loader.get_config_int("Notifications", "sender_port")
|
|
||||||
sender_pw = config_loader.get_config_str("Notifications", "sender_pw")
|
|
||||||
if sender_pw == 'None':
|
|
||||||
sender_pw = None
|
|
||||||
|
|
||||||
# raise an exception if any of these is None
|
|
||||||
if sender is None or sender_host is None or sender_port is None:
|
|
||||||
raise Exception('SMTP configuration (host, port, sender) is missing or incomplete!')
|
|
||||||
|
|
||||||
try:
|
|
||||||
if sender_pw is not None:
|
|
||||||
try:
|
|
||||||
smtp_server = smtplib.SMTP(sender_host, sender_port)
|
|
||||||
smtp_server.starttls()
|
|
||||||
except smtplib.SMTPNotSupportedError:
|
|
||||||
print("The server does not support the STARTTLS extension.")
|
|
||||||
smtp_server = smtplib.SMTP_SSL(sender_host, sender_port)
|
|
||||||
|
|
||||||
smtp_server.ehlo()
|
|
||||||
if sender_user is not None:
|
|
||||||
smtp_server.login(sender_user, sender_pw)
|
|
||||||
else:
|
|
||||||
smtp_server.login(sender, sender_pw)
|
|
||||||
else:
|
|
||||||
smtp_server = smtplib.SMTP(sender_host, sender_port)
|
|
||||||
|
|
||||||
mime_msg = MIMEMultipart()
|
|
||||||
mime_msg['From'] = sender
|
|
||||||
mime_msg['To'] = recipient
|
|
||||||
mime_msg['Subject'] = mail_subject
|
|
||||||
|
|
||||||
mime_msg.attach(MIMEText(mail_body, 'plain'))
|
|
||||||
|
|
||||||
smtp_server.sendmail(sender, recipient, mime_msg.as_string())
|
|
||||||
smtp_server.quit()
|
|
||||||
print('Send notification: ' + mail_subject + ' to '+recipient)
|
|
||||||
|
|
||||||
except Exception as err:
|
|
||||||
traceback.print_tb(err.__traceback__)
|
|
||||||
logger.warning(err)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
parser = argparse.ArgumentParser(description='Test notification sender.')
|
|
||||||
parser.add_argument("addr", help="Test mail 'to' address")
|
|
||||||
args = parser.parse_args()
|
|
||||||
sendEmailNotification(args.addr, '_mail test_', 'Success.')
|
|
||||||
|
|
@ -1,45 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
# -*-coding:UTF-8 -*
|
|
||||||
|
|
||||||
# import os
|
|
||||||
# import sys
|
|
||||||
# import uuid
|
|
||||||
#
|
|
||||||
# sys.path.append(os.environ['AIL_BIN'])
|
|
||||||
# ##################################
|
|
||||||
# # Import Project packages
|
|
||||||
# ##################################
|
|
||||||
# from lib.ConfigLoader import ConfigLoader
|
|
||||||
|
|
||||||
## LOAD CONFIG ##
|
|
||||||
# config_loader = ConfigLoader()
|
|
||||||
#
|
|
||||||
# r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") ######################################
|
|
||||||
# config_loader = None
|
|
||||||
## -- ##
|
|
||||||
|
|
||||||
# sys.path.append('../../configs/keys')
|
|
||||||
##################################
|
|
||||||
# Import Keys
|
|
||||||
##################################
|
|
||||||
from thehive4py.api import TheHiveApi
|
|
||||||
from thehive4py.models import Alert, AlertArtifact, Case, CaseObservable
|
|
||||||
import thehive4py.exceptions
|
|
||||||
|
|
||||||
from pymisp import MISPEvent, MISPObject, PyMISP
|
|
||||||
|
|
||||||
###########################################################
|
|
||||||
# # set default
|
|
||||||
# if r_serv_db.get('hive:auto-alerts') is None:
|
|
||||||
# r_serv_db.set('hive:auto-alerts', 0)
|
|
||||||
#
|
|
||||||
# if r_serv_db.get('misp:auto-events') is None:
|
|
||||||
# r_serv_db.set('misp:auto-events', 0)
|
|
||||||
|
|
||||||
# if __name__ == '__main__':
|
|
||||||
# from lib.objects.Cves import Cve
|
|
||||||
# create_misp_event([Item('crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'),
|
|
||||||
# Cve('CVE-2020-16856'), Cve('CVE-2014-6585'), Cve('CVE-2015-0383'),
|
|
||||||
# Cve('CVE-2015-0410')])
|
|
||||||
|
|
||||||
# create_investigation_misp_event('c6bbf8fa9ead4cc698eaeb07835cca5d)
|
|
||||||
|
|
@ -118,15 +118,15 @@ if [ -z "$VIRTUAL_ENV" ]; then
|
||||||
. ./AILENV/bin/activate
|
. ./AILENV/bin/activate
|
||||||
fi
|
fi
|
||||||
|
|
||||||
pushd ${AIL_BIN}/helper/gen_cert
|
pushd ${AIL_HOME}/tools/gen_cert
|
||||||
./gen_root.sh
|
./gen_root.sh
|
||||||
wait
|
wait
|
||||||
./gen_cert.sh
|
./gen_cert.sh
|
||||||
wait
|
wait
|
||||||
popd
|
popd
|
||||||
|
|
||||||
cp ${AIL_BIN}/helper/gen_cert/server.crt ${AIL_FLASK}/server.crt
|
cp ${AIL_HOME}/tools/gen_cert/server.crt ${AIL_FLASK}/server.crt
|
||||||
cp ${AIL_BIN}/helper/gen_cert/server.key ${AIL_FLASK}/server.key
|
cp ${AIL_HOME}/tools/gen_cert/server.key ${AIL_FLASK}/server.key
|
||||||
|
|
||||||
mkdir -p $AIL_HOME/PASTES
|
mkdir -p $AIL_HOME/PASTES
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,89 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
# -*-coding:UTF-8 -*
|
|
||||||
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import json
|
|
||||||
import argparse
|
|
||||||
|
|
||||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
|
||||||
import Cryptocurrency
|
|
||||||
|
|
||||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
|
||||||
import ConfigLoader
|
|
||||||
import Correlate_object
|
|
||||||
|
|
||||||
mode = 'union'
|
|
||||||
|
|
||||||
def sanitise_int(page, default_value):
|
|
||||||
try:
|
|
||||||
page = int(page)
|
|
||||||
except:
|
|
||||||
page = default_value
|
|
||||||
if page < 1:
|
|
||||||
page = default_value
|
|
||||||
return page
|
|
||||||
|
|
||||||
def sanitise_nb_max_nodes(nb_max_nodes):
|
|
||||||
try:
|
|
||||||
nb_max_nodes = int(nb_max_nodes)
|
|
||||||
if nb_max_nodes < 2:
|
|
||||||
nb_max_nodes = 300
|
|
||||||
except:
|
|
||||||
nb_max_nodes = 300
|
|
||||||
return nb_max_nodes
|
|
||||||
|
|
||||||
def get_object_correlation_json(correlation_id, subtype, max_nodes):
|
|
||||||
object_type = 'cryptocurrency'
|
|
||||||
max_nodes = sanitise_nb_max_nodes(max_nodes)
|
|
||||||
|
|
||||||
# FIXME
|
|
||||||
# ALL correlations
|
|
||||||
#correlation_names = Correlate_object.sanitise_correlation_names('')
|
|
||||||
#correlation_objects = Correlate_object.sanitise_correlation_objects('')
|
|
||||||
correlation_objects = ['domain']
|
|
||||||
|
|
||||||
res = Correlate_object.get_graph_node_object_correlation(object_type, correlation_id, mode, correlation_names,
|
|
||||||
correlation_objects, requested_correl_type=subtype,
|
|
||||||
max_nodes=max_nodes, flask_context=False)
|
|
||||||
return res
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description='Trigger backgroud update')
|
|
||||||
parser.add_argument('-t', '--type', help='Cryptocurrency type (bitcoin, bitcoin-cash, ethereum, litecoin, monero, dash, zcash)', type=str, dest='type', required=True, default=None)
|
|
||||||
parser.add_argument('-a', '--address', help='Cryptocurrency addresses', type=str, dest='address', default=None, nargs="*")
|
|
||||||
parser.add_argument('-p', '--page',help='page number, default=1' , type=int, default=1, dest='page')
|
|
||||||
parser.add_argument('-n', '--nb',help='number of addresses by page, default=50' , type=int, default=50, dest='nb_elem')
|
|
||||||
parser.add_argument('-fo', '--filter_objects',help='filter correlation by object : domain, paste/item' , type=str, default=[], dest='objects', nargs="*")
|
|
||||||
parser.add_argument('--node' ,help='correlation graph: max number of nodes, default=50' , type=int, default=50, dest='max_nodes')
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
subtype = args.type
|
|
||||||
if subtype is None:
|
|
||||||
parser.print_help()
|
|
||||||
sys.exit(0)
|
|
||||||
else:
|
|
||||||
res = Cryptocurrency.cryptocurrency.api_check_objs_type([args.type])
|
|
||||||
if res:
|
|
||||||
print(json.dumps(res[0]))
|
|
||||||
sys.exit(0)
|
|
||||||
|
|
||||||
page = sanitise_int(args.page, 1)
|
|
||||||
nb_elem = sanitise_int(args.nb_elem, 50)
|
|
||||||
max_nodes = sanitise_int(args.max_nodes, 300)
|
|
||||||
if args.objects:
|
|
||||||
res = Correlate_object.api_check_correlation_objects(args.objects)
|
|
||||||
if res:
|
|
||||||
print(json.dumps(res[0]))
|
|
||||||
sys.exit(0)
|
|
||||||
|
|
||||||
dict_json = {}
|
|
||||||
if args.address:
|
|
||||||
l_addresse = Cryptocurrency.cryptocurrency.paginate_list(args.address, nb_elem=nb_elem, page=page)
|
|
||||||
else:
|
|
||||||
l_addresse = Cryptocurrency.cryptocurrency.get_all_correlations_by_subtype_pagination(subtype, nb_elem=nb_elem, page=page)
|
|
||||||
for address in l_addresse:
|
|
||||||
dict_json[address] = get_object_correlation_json(address, subtype, max_nodes)
|
|
||||||
|
|
||||||
print(json.dumps(dict_json))
|
|
||||||
|
|
@ -1,31 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
# -*-coding:UTF-8 -*
|
|
||||||
|
|
||||||
'''
|
|
||||||
Flask functions and routes for the trending modules page
|
|
||||||
'''
|
|
||||||
import redis
|
|
||||||
from flask import Flask, render_template, jsonify, request, Blueprint
|
|
||||||
|
|
||||||
# ============ VARIABLES ============
|
|
||||||
import Flask_config
|
|
||||||
|
|
||||||
app = Flask_config.app
|
|
||||||
cfg = Flask_config.cfg
|
|
||||||
baseUrl = Flask_config.baseUrl
|
|
||||||
|
|
||||||
MODULENAME = Blueprint('MODULENAME', __name__, template_folder='templates')
|
|
||||||
|
|
||||||
# ============ FUNCTIONS ============
|
|
||||||
def one():
|
|
||||||
return 1
|
|
||||||
|
|
||||||
# ============= ROUTES ==============
|
|
||||||
|
|
||||||
@MODULENAME.route("/MODULENAME/", methods=['GET'])
|
|
||||||
def MODULENAME_page():
|
|
||||||
return render_template("MODULENAME.html")
|
|
||||||
|
|
||||||
|
|
||||||
# ========= REGISTRATION =========
|
|
||||||
app.register_blueprint(MODULENAME, url_prefix=baseUrl)
|
|
||||||
|
|
@ -1,44 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
# -*-coding:UTF-8 -*
|
|
||||||
|
|
||||||
"Hepler to create a new webpage associated with a module."
|
|
||||||
|
|
||||||
import os
|
|
||||||
|
|
||||||
def createModuleFolder(modulename):
|
|
||||||
path_module = os.path.join('modules', modulename)
|
|
||||||
os.mkdir(path_module)
|
|
||||||
|
|
||||||
# create html template
|
|
||||||
with open('templates/base_template.html', 'r') as templateFile:
|
|
||||||
template = templateFile.read()
|
|
||||||
template = template.replace('MODULENAME', modulename)
|
|
||||||
|
|
||||||
os.mkdir(os.path.join(path_module, 'templates'))
|
|
||||||
with open(os.path.join(os.path.join(path_module, 'templates'), modulename+'.html'), 'w') as toWriteTemplate:
|
|
||||||
toWriteTemplate.write(template)
|
|
||||||
|
|
||||||
# create html header template
|
|
||||||
with open('templates/header_base_template.html', 'r') as header_templateFile:
|
|
||||||
header = header_templateFile.read()
|
|
||||||
header = header.replace('MODULENAME', modulename)
|
|
||||||
|
|
||||||
with open(os.path.join(os.path.join(path_module, 'templates'), 'header_{}.html'.format(modulename) ), 'w') as toWriteHeader:
|
|
||||||
toWriteHeader.write(header)
|
|
||||||
|
|
||||||
|
|
||||||
#create flask template
|
|
||||||
with open('Flask_base_template.py', 'r') as flaskFile:
|
|
||||||
flask = flaskFile.read()
|
|
||||||
flask = flask.replace('MODULENAME', modulename)
|
|
||||||
|
|
||||||
with open(os.path.join(path_module, 'Flask_{}.py'.format(modulename)), 'w') as toWriteFlask:
|
|
||||||
toWriteFlask.write(flask)
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
rep1 = input('New module name: ')
|
|
||||||
createModuleFolder(rep1)
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
Loading…
Reference in New Issue