MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge remote-tracking branch 'origin/main'

pull/700/head
Sami Mokaddem 2024-10-15 12:00:11 +02:00
parent eb55006f15 7e2297aa71
commit edad5580dd
No known key found for this signature in database
GPG Key ID: 164C473F627A06FA
22 changed files with 1497 additions and 1261 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
misp_modules/modules/expansion/circl_passivedns.py
View File

@ -48,14 +48,20 @@ class PassiveDNSParser:
self.result = {'error': 'Not found'}
return
mapping = {'count': 'counter', 'origin': 'text',
'time_first': 'datetime', 'rrtype': 'text',
'rrname': 'text', 'rdata': 'text',
'time_last': 'datetime'}
mapping = {
'count': 'counter', 'origin': 'text', 'rrtype': 'text',
'rrname': 'text', 'rdata': 'text',
}
for result in results:
pdns_object = MISPObject('passive-dns')
for relation, attribute_type in mapping.items():
pdns_object.add_attribute(relation, type=attribute_type, value=result[relation])
pdns_object.add_attribute(relation, result[relation], type=attribute_type)
first_seen = result['time_first']
pdns_object.add_attribute('time_first', first_seen, type='datetime')
pdns_object.first_seen = first_seen
last_seen = result['time_last']
pdns_object.add_attribute('time_last', last_seen, type='datetime')
pdns_object.last_seen = last_seen
pdns_object.add_reference(self.attribute.uuid, 'associated-to')
self.misp_event.add_object(**pdns_object)

2
misp_modules/modules/expansion/dns.py
View File

@ -7,7 +7,7 @@ mispattributes = {'input': ['hostname', 'domain', 'domain|ip'], 'output': ['ip-s
moduleinfo = {
'version': '0.3',
'author': 'Alexandre Dulaunoy',
'description': 'jj',
'description': 'Simple DNS expansion service to resolve IP address from MISP attributes',
'module-type': ['expansion', 'hover'],
'name': 'DNS Resolver',
'logo': '',

2287
poetry.lock generated
View File

File diff suppressed because it is too large Load Diff

3
pyproject.toml
View File

@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
[tool.poetry]
name = "misp-modules"
version = "2.4.197"
version = "2.4.198"
description = "MISP modules are autonomous modules that can be used for expansion and other services in MISP"
authors = ["Alexandre Dulaunoy <alexandre.dulaunoy@circl.lu>"]
license = "AGPL-3.0-only"
@ -36,6 +36,7 @@ psutil = "*"
pyparsing = "*"
redis = "*"
tornado = "*"
urllib3 = ">=1.26,<2"
## module dependencies (if a dependency fails loading with '*', pin it here)
censys = "2.0.9"
socialscan = "<2.0.0"

52
website/app.py
View File

@ -1,52 +0,0 @@
from app import create_app, db
import argparse
from flask import render_template
import os
from app.utils.init_modules import create_modules_db
import signal
import sys
import subprocess
from app.utils.utils import gen_admin_password
def signal_handler(sig, frame):
path = os.path.join(os.getcwd(), "launch.sh")
req = [path, "-ks"]
subprocess.call(req)
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--init_db", help="Initialise the db if it not exist", action="store_true")
parser.add_argument("-r", "--recreate_db", help="Delete and initialise the db", action="store_true")
parser.add_argument("-d", "--delete_db", help="Delete the db", action="store_true")
parser.add_argument("-m", "--create_module", help="Create modules in db", action="store_true")
args = parser.parse_args()
os.environ.setdefault('FLASKENV', 'development')
app = create_app()
@app.errorhandler(404)
def error_page_not_found(e):
return render_template('404.html'), 404
if args.init_db:
with app.app_context():
db.create_all()
elif args.recreate_db:
with app.app_context():
db.drop_all()
db.create_all()
elif args.delete_db:
with app.app_context():
db.drop_all()
elif args.create_module:
with app.app_context():
create_modules_db()
else:
gen_admin_password()
app.run(host=app.config.get("FLASK_URL"), port=app.config.get("FLASK_PORT"))

33
website/app_creation.py Normal file
View File

@ -0,0 +1,33 @@
from app import create_app, db
from flask import render_template
import os
from app.utils.init_modules import create_modules_db
from app.utils.utils import gen_admin_password
os.environ.setdefault('FLASKENV', 'development')
app = create_app()
@app.errorhandler(404)
def error_page_not_found(e):
return render_template('404.html'), 404
def main(init_db=False, recreate_db=False, delete_db=False, create_module=False):
if init_db:
with app.app_context():
db.create_all()
elif recreate_db:
with app.app_context():
db.drop_all()
db.create_all()
elif delete_db:
with app.app_context():
db.drop_all()
elif create_module:
with app.app_context():
create_modules_db()
else:
gen_admin_password()
app.run(host=app.config.get("FLASK_URL"), port=app.config.get("FLASK_PORT") , use_reloader=False)

43
website/launch.py Executable file
View File

@ -0,0 +1,43 @@
import os
import argparse
import subprocess
import time
from app_creation import main
import signal
import sys
def signal_handler(sig, frame):
kill_script()
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--init_db", help="Initialise the db if it not exist", action="store_true")
parser.add_argument("-r", "--reload_db", help="Delete and initialise the db", action="store_true")
parser.add_argument("-l", "--launch", help="Launch the app", action="store_true")
parser.add_argument("-ks", "--killscript", help="Kill screen running background", action="store_true")
args = parser.parse_args()
def kill_script():
r = ["screen", "-ls", "|", "egrep", "[0-9]+.misp_mod", "|", "cut", "-d.", "-f1"]
process = subprocess.Popen(r, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = process.communicate()
if out:
subprocess.call(["screen", "-X", "-S", "misp_mod", "quit"])
if args.init_db:
main(init_db=True)
elif args.reload_db:
main(recreate_db=True)
elif args.launch:
os.environ.setdefault('FLASKENV', 'development')
kill_script()
subprocess.call(["screen", "-dmS", "misp_mod"])
r = ["screen", "-S", "misp_mod", "-X", "screen", "-t", "misp_modules_server", "bash", "-c", "../env/bin/misp-modules", "-l", "127.0.0.1;", "read x"]
subprocess.call(r)
time.sleep(2)
main(create_module=True)
main()
elif args.killscript:
kill_script()

49
website/launch.sh
View File

@ -1,49 +0,0 @@
#!/bin/bash
isscripted=`screen -ls | egrep '[0-9]+.misp_mod' | cut -d. -f1`
function killscript {
if [ $isscripted ]; then
screen -X -S misp_mod quit
fi
}
function launch {
export FLASKENV="development"
killscript
screen -dmS "misp_mod"
screen -S "misp_mod" -X screen -t "misp_modules_server" bash -c "misp-modules -l 127.0.0.1; read x"
sleep 2
python3 app.py -m
python3 app.py
}
function test {
export FLASKENV="testing"
pytest
}
function init_db {
python3 app.py -i
}
function reload_db {
python3 app.py -r
}
if [ "$1" ]; then
case $1 in
-l | --launch ) launch;
;;
-i | --init_db ) init_db;
;;
-r | --reload_db ) reload_db;
;;
-t | --test ) test;
;;
-ks | --killscript ) killscript;
esac
shift
else
launch
fi

19
website/migrate.py Normal file
View File

@ -0,0 +1,19 @@
import os
import argparse
import subprocess
os.environ.setdefault('FLASKENV', 'development')
parser = argparse.ArgumentParser()
parser.add_argument("-m", "--migrate", help="Initialise the db if it not exist", action="store_true")
parser.add_argument("-u", "--upgrade", help="Delete and initialise the db", action="store_true")
parser.add_argument("-d", "--downgrade", help="Launch the app", action="store_true")
args = parser.parse_args()
if args.migrate:
subprocess.call(["flask", "db", "migrate"])
elif args.upgrade:
subprocess.call(["flask", "db", "upgrade"])
elif args.downgrade:
subprocess.call(["flask", "db", "downgrade"])

30
website/migrate.sh
View File

@ -1,30 +0,0 @@
#!/bin/bash
source env/bin/activate
export FLASKENV=development
function migrate {
flask db migrate
}
function upgrade {
flask db upgrade
}
function downgrade {
flask db downgrade
}
if [ "$1" ]; then
case $1 in
-m | --migrate ) migrate;
;;
-u | --upgrade ) upgrade;
;;
-d | --downgrade ) downgrade;
esac
shift
else
echo "need -m or -u or -d"
fi

8
website/migrations/versions/01de07e2686a_.py
View File

@ -7,6 +7,7 @@ Create Date: 2024-02-05 14:38:17.739081
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.exc import OperationalError
# revision identifiers, used by Alembic.
@ -18,8 +19,11 @@ depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('module', schema=None) as batch_op:
batch_op.add_column(sa.Column('is_active', sa.Boolean(), nullable=True, default=True))
try:
with op.batch_alter_table('module', schema=None) as batch_op:
batch_op.add_column(sa.Column('is_active', sa.Boolean(), nullable=True, default=True))
except OperationalError:
print("Column 'is_active' already exist in 'module'")
# ### end Alembic commands ###

40
website/migrations/versions/0b5e7db16af8_.py
View File

@ -7,6 +7,7 @@ Create Date: 2024-02-07 11:37:07.698058
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.exc import OperationalError
# revision identifiers, used by Alembic.
@ -18,19 +19,34 @@ depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('history',
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
sa.Column('session_id', sa.Integer(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
with op.batch_alter_table('history', schema=None) as batch_op:
batch_op.create_index(batch_op.f('ix_history_session_id'), ['session_id'], unique=False)
try:
op.create_table('history',
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
sa.Column('session_id', sa.Integer(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
except OperationalError:
print("Table 'history' already exist")
with op.batch_alter_table('session', schema=None) as batch_op:
batch_op.add_column(sa.Column('nb_errors', sa.Integer(), nullable=True))
batch_op.drop_index('ix_session_uuid')
batch_op.create_index(batch_op.f('ix_session_uuid'), ['uuid'], unique=True)
batch_op.create_index(batch_op.f('ix_session_nb_errors'), ['nb_errors'], unique=False)
try:
with op.batch_alter_table('history', schema=None) as batch_op:
batch_op.create_index(batch_op.f('ix_history_session_id'), ['session_id'], unique=False)
except OperationalError:
print("Index already exist for history")
try:
with op.batch_alter_table('session', schema=None) as batch_op:
batch_op.add_column(sa.Column('nb_errors', sa.Integer(), nullable=True))
batch_op.create_index(batch_op.f('ix_session_uuid'), ['uuid'], unique=True)
batch_op.create_index(batch_op.f('ix_session_nb_errors'), ['nb_errors'], unique=False)
except OperationalError:
print("Column 'nb_errors' already exist in 'session'")
try:
with op.batch_alter_table('session', schema=None) as batch_op:
batch_op.drop_index('ix_session_uuid')
except OperationalError:
print("Index already dropped from session")
# ### end Alembic commands ###

15
website/migrations/versions/0c02b861944b_.py
View File

@ -7,6 +7,7 @@ Create Date: 2024-02-06 09:08:59.802932
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.exc import OperationalError
# revision identifiers, used by Alembic.
@ -18,11 +19,17 @@ depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('module', schema=None) as batch_op:
batch_op.add_column(sa.Column('request_on_query', sa.Boolean(), nullable=True))
try:
with op.batch_alter_table('module', schema=None) as batch_op:
batch_op.add_column(sa.Column('request_on_query', sa.Boolean(), nullable=True))
except OperationalError:
print("Column 'request_on_query' already exist in 'module'")
with op.batch_alter_table('module__config', schema=None) as batch_op:
batch_op.drop_column('request_on_query')
try:
with op.batch_alter_table('module__config', schema=None) as batch_op:
batch_op.drop_column('request_on_query')
except OperationalError:
print("Column 'request_on_query' already dropped from 'module__config'")
# ### end Alembic commands ###

16
website/migrations/versions/0d3b152688fe_.py
View File

@ -7,6 +7,7 @@ Create Date: 2024-02-07 11:08:00.337971
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.exc import OperationalError
# revision identifiers, used by Alembic.
@ -18,12 +19,15 @@ depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('session', schema=None) as batch_op:
batch_op.add_column(sa.Column('glob_query', sa.String(), nullable=True))
batch_op.add_column(sa.Column('query', sa.String(), nullable=True))
batch_op.add_column(sa.Column('input_query', sa.String(), nullable=True))
batch_op.add_column(sa.Column('config_module', sa.String(), nullable=True))
batch_op.add_column(sa.Column('result', sa.String(), nullable=True))
try:
with op.batch_alter_table('session', schema=None) as batch_op:
batch_op.add_column(sa.Column('glob_query', sa.String(), nullable=True))
batch_op.add_column(sa.Column('query', sa.String(), nullable=True))
batch_op.add_column(sa.Column('input_query', sa.String(), nullable=True))
batch_op.add_column(sa.Column('config_module', sa.String(), nullable=True))
batch_op.add_column(sa.Column('result', sa.String(), nullable=True))
except OperationalError:
print("Columns already exist in 'session'")
# ### end Alembic commands ###

8
website/migrations/versions/183bf8fa2b87_.py
View File

@ -7,6 +7,7 @@ Create Date: 2024-02-09 15:21:17.274707
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.exc import OperationalError
# revision identifiers, used by Alembic.
@ -18,8 +19,11 @@ depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('module', schema=None) as batch_op:
batch_op.add_column(sa.Column('input_attr', sa.String(), nullable=True))
try:
with op.batch_alter_table('module', schema=None) as batch_op:
batch_op.add_column(sa.Column('input_attr', sa.String(), nullable=True))
except OperationalError:
print("Column 'input_attr' already exist in 'module'")
# ### end Alembic commands ###

50
website/migrations/versions/1d38ae8361a7_.py
View File

@ -7,6 +7,7 @@ Create Date: 2024-02-07 11:44:30.236490
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.exc import OperationalError
# revision identifiers, used by Alembic.
@ -18,26 +19,39 @@ depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('session_db',
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
sa.Column('uuid', sa.String(length=36), nullable=True),
sa.Column('glob_query', sa.String(), nullable=True),
sa.Column('query', sa.String(), nullable=True),
sa.Column('input_query', sa.String(), nullable=True),
sa.Column('config_module', sa.String(), nullable=True),
sa.Column('result', sa.String(), nullable=True),
sa.Column('nb_errors', sa.Integer(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
with op.batch_alter_table('session_db', schema=None) as batch_op:
batch_op.create_index(batch_op.f('ix_session_db_nb_errors'), ['nb_errors'], unique=False)
batch_op.create_index(batch_op.f('ix_session_db_uuid'), ['uuid'], unique=True)
try:
op.create_table('session_db',
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
sa.Column('uuid', sa.String(length=36), nullable=True),
sa.Column('glob_query', sa.String(), nullable=True),
sa.Column('query', sa.String(), nullable=True),
sa.Column('input_query', sa.String(), nullable=True),
sa.Column('config_module', sa.String(), nullable=True),
sa.Column('result', sa.String(), nullable=True),
sa.Column('nb_errors', sa.Integer(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
except OperationalError:
print("Table 'session_db' already exist")
with op.batch_alter_table('session', schema=None) as batch_op:
batch_op.drop_index('ix_session_nb_errors')
batch_op.drop_index('ix_session_uuid')
try:
with op.batch_alter_table('session_db', schema=None) as batch_op:
batch_op.create_index(batch_op.f('ix_session_db_nb_errors'), ['nb_errors'], unique=False)
batch_op.create_index(batch_op.f('ix_session_db_uuid'), ['uuid'], unique=True)
except OperationalError:
print("Index already exist in 'session_db'")
op.drop_table('session')
try:
with op.batch_alter_table('session', schema=None) as batch_op:
batch_op.drop_index('ix_session_nb_errors')
batch_op.drop_index('ix_session_uuid')
except OperationalError:
print("Index already dropped from 'session'")
try:
op.drop_table('session')
except OperationalError:
print("Table 'session' already dropped")
# ### end Alembic commands ###

10
website/migrations/versions/206e0e808b0c_.py
View File

@ -7,6 +7,7 @@ Create Date: 2024-08-21 09:36:37.801809
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.exc import OperationalError
# revision identifiers, used by Alembic.
@ -18,9 +19,12 @@ depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('external_tools', schema=None) as batch_op:
batch_op.add_column(sa.Column('api_key', sa.String(length=60), nullable=True))
batch_op.create_index(batch_op.f('ix_external_tools_api_key'), ['api_key'], unique=False)
try:
with op.batch_alter_table('external_tools', schema=None) as batch_op:
batch_op.add_column(sa.Column('api_key', sa.String(length=60), nullable=True))
batch_op.create_index(batch_op.f('ix_external_tools_api_key'), ['api_key'], unique=False)
except OperationalError:
print("Column 'api_key' already exist in 'external_tools")
# ### end Alembic commands ###

25
website/migrations/versions/3a631d400f60_.py
View File

@ -7,6 +7,7 @@ Create Date: 2024-06-27 11:15:52.165895
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.exc import OperationalError
# revision identifiers, used by Alembic.
@ -18,15 +19,21 @@ depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('external_tools',
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
sa.Column('name', sa.String(length=64), nullable=True),
sa.Column('url', sa.String(), nullable=True),
sa.Column('is_active', sa.Boolean(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
with op.batch_alter_table('external_tools', schema=None) as batch_op:
batch_op.create_index(batch_op.f('ix_external_tools_name'), ['name'], unique=False)
try:
op.create_table('external_tools',
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
sa.Column('name', sa.String(length=64), nullable=True),
sa.Column('url', sa.String(), nullable=True),
sa.Column('is_active', sa.Boolean(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
except OperationalError:
print("Table 'external_tools' already exist")
try:
with op.batch_alter_table('external_tools', schema=None) as batch_op:
batch_op.create_index(batch_op.f('ix_external_tools_name'), ['name'], unique=False)
except OperationalError:
print("Index already exist for 'external_tools'")
# ### end Alembic commands ###

10
website/migrations/versions/91f830996ff4_.py
View File

@ -7,6 +7,7 @@ Create Date: 2024-02-09 09:51:11.639862
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.exc import OperationalError
# revision identifiers, used by Alembic.
@ -18,9 +19,12 @@ depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('session_db', schema=None) as batch_op:
batch_op.add_column(sa.Column('query_date', sa.DateTime(), nullable=True))
batch_op.create_index(batch_op.f('ix_session_db_query_date'), ['query_date'], unique=False)
try:
with op.batch_alter_table('session_db', schema=None) as batch_op:
batch_op.add_column(sa.Column('query_date', sa.DateTime(), nullable=True))
batch_op.create_index(batch_op.f('ix_session_db_query_date'), ['query_date'], unique=False)
except OperationalError:
print("Column 'query_date' already exist in 'session_db'")
# ### end Alembic commands ###

17
website/migrations/versions/c0e243696052_.py
View File

@ -7,6 +7,7 @@ Create Date: 2024-02-07 11:59:20.615266
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.exc import OperationalError
# revision identifiers, used by Alembic.
@ -18,9 +19,19 @@ depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('session_db', schema=None) as batch_op:
batch_op.add_column(sa.Column('query_enter', sa.String(), nullable=True))
batch_op.drop_column('query')
try:
with op.batch_alter_table('session_db', schema=None) as batch_op:
batch_op.add_column(sa.Column('query_enter', sa.String(), nullable=True))
except OperationalError:
print("Column 'query_entry' already exist in 'session_db'")
try:
with op.batch_alter_table('session_db', schema=None) as batch_op:
batch_op.drop_column('query')
except OperationalError:
print("Column 'query' already dropped from 'session_db'")
except KeyError:
print("Column 'query' already dropped from 'session_db'")
# ### end Alembic commands ###

17
website/migrations/versions/d6f8bfc9d454_.py
View File

@ -7,6 +7,7 @@ Create Date: 2024-02-08 15:23:46.714541
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.exc import OperationalError
# revision identifiers, used by Alembic.
@ -18,9 +19,19 @@ depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('session_db', schema=None) as batch_op:
batch_op.add_column(sa.Column('modules_list', sa.String(), nullable=True))
batch_op.drop_column('glob_query')
try:
with op.batch_alter_table('session_db', schema=None) as batch_op:
batch_op.add_column(sa.Column('modules_list', sa.String(), nullable=True))
except OperationalError:
print("Column 'modules_list' already exist in 'session_db'")
try:
with op.batch_alter_table('session_db', schema=None) as batch_op:
batch_op.drop_column('glob_query')
except OperationalError:
print("Column 'glob_query' already dropped from 'session_db'")
except KeyError:
print("Column 'glob_query' already dropped from 'session_db'")
# ### end Alembic commands ###

8
website/migrations/versions/ef119b41c5f2_.py
View File

@ -7,6 +7,7 @@ Create Date: 2024-02-06 08:10:37.101421
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.exc import OperationalError
# revision identifiers, used by Alembic.
@ -18,8 +19,11 @@ depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('module__config', schema=None) as batch_op:
batch_op.add_column(sa.Column('request_on_query', sa.Boolean(), nullable=True))
try:
with op.batch_alter_table('module__config', schema=None) as batch_op:
batch_op.add_column(sa.Column('request_on_query', sa.Boolean(), nullable=True))
except OperationalError:
print('Coulmn "request_on_query" already exist in "module__config"')
# ### end Alembic commands ###