MISP
/
misp-modules
mirror of https://github.com/MISP/misp-modules
2
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'master' of github.com:MISP/misp-modules

pull/302/head
Alexandre Dulaunoy 2019-04-26 14:06:09 +02:00
parent 980760790f c9281e605d
commit 5efe7c330e
No known key found for this signature in database
GPG Key ID: 09E2CD4944E6CBCD
4 changed files with 173 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Pipfile
View File

@ -45,7 +45,7 @@ backscatter = "*"
pyzbar = "*"
opencv-python = "*"
np = "*"
ODTReader = {editable = true,git = "https://github.com/cartertemm/ODTReader.git"}
ODTReader = {editable = true,git = "https://github.com/cartertemm/ODTReader.git/"}
python-pptx = "*"
python-docx = "*"
ezodf = "*"
@ -54,6 +54,7 @@ pandas_ods_reader = "*"
pdftotext = "*"
lxml = "*"
xlrd = "*"
idna-ssl = {markers="python_version < '3.7'"}
[requires]
python_version = "3"

124
Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "7fee9399d8a7151a79b6f8bbce64564062fd562b0a091fd45a875884d3fb954e"
"sha256": "9aac0a9c45df16b9502c13f9468095cf5ffdb8bc407fe2b55faee3ff53d8eba3"
},
"pipfile-spec": 6,
"requires": {
@ -150,6 +150,13 @@
],
"version": "==0.9"
},
"ezodf": {
"hashes": [
"sha256:000da534f689c6d55297a08f9e2ed7eada9810d194d31d164388162fb391122d"
],
"index": "pypi",
"version": "==0.3.2"
},
"future": {
"hashes": [
"sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8"
@ -170,6 +177,14 @@
],
"version": "==2.8"
},
"idna-ssl": {
"hashes": [
"sha256:a933e3bb13da54383f9e8f35dc4f9cb9eb9b3b78c6b36f311254d6d0d92c6c7c"
],
"index": "pypi",
"markers": "python_version < '3.7'",
"version": "==1.1.0"
},
"isodate": {
"hashes": [
"sha256:2e364a3d5759479cdb2d37cce6b9376ea504db2ff90252a2e5b7cc89cc9ff2d8",
@ -184,6 +199,38 @@
],
"version": "==3.0.1"
},
"lxml": {
"hashes": [
"sha256:03984196d00670b2ab14ae0ea83d5cc0cfa4f5a42558afa9ab5fa745995328f5",
"sha256:0815b0c9f897468de6a386dc15917a0becf48cc92425613aa8bbfc7f0f82951f",
"sha256:175f3825f075cf02d15099eb52658457cf0ff103dcf11512b5d2583e1d40f58b",
"sha256:30e14c62d88d1e01a26936ecd1c6e784d4afc9aa002bba4321c5897937112616",
"sha256:3210da6f36cf4b835ff1be853962b22cc354d506f493b67a4303c88bbb40d57b",
"sha256:40f60819fbd5bad6e191ba1329bfafa09ab7f3f174b3d034d413ef5266963294",
"sha256:43b26a865a61549919f8a42e094dfdb62847113cf776d84bd6b60e4e3fc20ea3",
"sha256:4a03dd682f8e35a10234904e0b9508d705ff98cf962c5851ed052e9340df3d90",
"sha256:62f382cddf3d2e52cf266e161aa522d54fd624b8cc567bc18f573d9d50d40e8e",
"sha256:7b98f0325be8450da70aa4a796c4f06852949fe031878b4aa1d6c417a412f314",
"sha256:846a0739e595871041385d86d12af4b6999f921359b38affb99cdd6b54219a8f",
"sha256:a3080470559938a09a5d0ec558c005282e99ac77bf8211fb7b9a5c66390acd8d",
"sha256:ad841b78a476623955da270ab8d207c3c694aa5eba71f4792f65926dc46c6ee8",
"sha256:afdd75d9735e44c639ffd6258ce04a2de3b208f148072c02478162d0944d9da3",
"sha256:b4fbf9b552faff54742bcd0791ab1da5863363fb19047e68f6592be1ac2dab33",
"sha256:b90c4e32d6ec089d3fa3518436bdf5ce4d902a0787dbd9bb09f37afe8b994317",
"sha256:b91cfe4438c741aeff662d413fd2808ac901cc6229c838236840d11de4586d63",
"sha256:bdb0593a42070b0a5f138b79b872289ee73c8e25b3f0bea6564e795b55b6bcdd",
"sha256:c4e4bca2bb68ce22320297dfa1a7bf070a5b20bcbaec4ee023f83d2f6e76496f",
"sha256:cec4ab14af9eae8501be3266ff50c3c2aecc017ba1e86c160209bb4f0423df6a",
"sha256:e83b4b2bf029f5104bc1227dbb7bf5ace6fd8fabaebffcd4f8106fafc69fc45f",
"sha256:e995b3734a46d41ae60b6097f7c51ba9958648c6d1e0935b7e0ee446ee4abe22",
"sha256:f679d93dec7f7210575c85379a31322df4c46496f184ef650d3aba1484b38a2d",
"sha256:fd213bb5166e46974f113c8228daaef1732abc47cb561ce9c4c8eaed4bd3b09b",
"sha256:fdcb57b906dbc1f80666e6290e794ab8fb959a2e17aa5aee1758a85d1da4533f",
"sha256:ff424b01d090ffe1947ec7432b07f536912e0300458f9a7f48ea217dd8362b86"
],
"index": "pypi",
"version": "==4.3.3"
},
"maclookup": {
"hashes": [
"sha256:33bf8eaebe3b1e4ab4ae9277dd93c78024e0ebf6b3c42f76c37695bc26ce287a",
@ -273,6 +320,11 @@
"index": "pypi",
"version": "==1.9.0.post1"
},
"odtreader": {
"editable": true,
"git": "https://github.com/cartertemm/ODTReader.git/",
"ref": "49d6938693f6faa3ff09998f86dba551ae3a996b"
},
"opencv-python": {
"hashes": [
"sha256:1703a296a96d3d46615e5053f224867977accb4240bcaa0fcabcb0768bf5ac13",
@ -306,6 +358,40 @@
"index": "pypi",
"version": "==4.1.0.25"
},
"pandas": {
"hashes": [
"sha256:071e42b89b57baa17031af8c6b6bbd2e9a5c68c595bc6bf9adabd7a9ed125d3b",
"sha256:17450e25ae69e2e6b303817bdf26b2cd57f69595d8550a77c308be0cd0fd58fa",
"sha256:17916d818592c9ec891cbef2e90f98cc85e0f1e89ed0924c9b5220dc3209c846",
"sha256:2538f099ab0e9f9c9d09bbcd94b47fd889bad06dc7ae96b1ed583f1dc1a7a822",
"sha256:366f30710172cb45a6b4f43b66c220653b1ea50303fbbd94e50571637ffb9167",
"sha256:42e5ad741a0d09232efbc7fc648226ed93306551772fc8aecc6dce9f0e676794",
"sha256:4e718e7f395ba5bfe8b6f6aaf2ff1c65a09bb77a36af6394621434e7cc813204",
"sha256:4f919f409c433577a501e023943e582c57355d50a724c589e78bc1d551a535a2",
"sha256:4fe0d7e6438212e839fc5010c78b822664f1a824c0d263fd858f44131d9166e2",
"sha256:5149a6db3e74f23dc3f5a216c2c9ae2e12920aa2d4a5b77e44e5b804a5f93248",
"sha256:627594338d6dd995cfc0bacd8e654cd9e1252d2a7c959449228df6740d737eb8",
"sha256:83c702615052f2a0a7fb1dd289726e29ec87a27272d775cb77affe749cca28f8",
"sha256:8c872f7fdf3018b7891e1e3e86c55b190e6c5cee70cab771e8f246c855001296",
"sha256:90f116086063934afd51e61a802a943826d2aac572b2f7d55caaac51c13db5b5",
"sha256:a3352bacac12e1fc646213b998bce586f965c9d431773d9e91db27c7c48a1f7d",
"sha256:bcdd06007cca02d51350f96debe51331dec429ac8f93930a43eb8fb5639e3eb5",
"sha256:c1bd07ebc15285535f61ddd8c0c75d0d6293e80e1ee6d9a8d73f3f36954342d0",
"sha256:c9a4b7c55115eb278c19aa14b34fcf5920c8fe7797a09b7b053ddd6195ea89b3",
"sha256:cc8fc0c7a8d5951dc738f1c1447f71c43734244453616f32b8aa0ef6013a5dfb",
"sha256:d7b460bc316064540ce0c41c1438c416a40746fd8a4fb2999668bf18f3c4acf1"
],
"index": "pypi",
"version": "==0.24.2"
},
"pandas-ods-reader": {
"hashes": [
"sha256:0f7d510639c8957a06aa1227b9f84d1be47a437dfd306464ce803b91cf5eeec4",
"sha256:d85ef58fc3aeac1616028d22954b6ef2e8983ab9bae015e1e90ce3979d138553"
],
"index": "pypi",
"version": "==0.0.6"
},
"passivetotal": {
"hashes": [
"sha256:d745a6519ec04e3a354682978ebf07778bf7602beac30307cbad075ff1a4418d"
@ -313,6 +399,13 @@
"index": "pypi",
"version": "==1.0.30"
},
"pdftotext": {
"hashes": [
"sha256:e3ad11efe0aa22cbfc46aa1296b2ea5a52ad208b778288311f2801adef178ccb"
],
"index": "pypi",
"version": "==2.1.1"
},
"pillow": {
"hashes": [
"sha256:15c056bfa284c30a7f265a41ac4cbbc93bdbfc0dfe0613b9cb8a8581b51a9e55",
@ -447,6 +540,27 @@
],
"version": "==2.8.0"
},
"python-docx": {
"hashes": [
"sha256:bc76ecac6b2d00ce6442a69d03a6f35c71cd72293cd8405a7472dfe317920024"
],
"index": "pypi",
"version": "==0.8.10"
},
"python-pptx": {
"hashes": [
"sha256:1f2d5d1d923d91f50a1f0ed794935e7d670993fdcb6c12c81cc83977c1f23e14"
],
"index": "pypi",
"version": "==0.6.17"
},
"pytz": {
"hashes": [
"sha256:303879e36b721603cc54604edcac9d20401bdbe31e1e4fdee5b9f98d5d31dfda",
"sha256:d747dd3d23d77ef44c6a3526e274af6efeb0a6f1afd5a69ba4d5be4098c8e141"
],
"version": "==2019.1"
},
"pyyaml": {
"hashes": [
"sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c",
@ -642,6 +756,14 @@
"index": "pypi",
"version": "==0.5.3"
},
"xlrd": {
"hashes": [
"sha256:546eb36cee8db40c3eaa46c351e67ffee6eeb5fa2650b71bc4c758a29a1b29b2",
"sha256:e551fb498759fa3a5384a94ccd4c3c02eb7c00ea424426e212ac0c57be9dfbde"
],
"index": "pypi",
"version": "==1.2.0"
},
"xlsxwriter": {
"hashes": [
"sha256:2a40b427dac0f640031e5b33abe97e761de6e0f12d4d346e7b2e2b67cf6ee927",

11
REQUIREMENTS
View File

@ -5,6 +5,7 @@
-e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471
-e git+https://github.com/MISP/PyMISP.git@582dda0ce2a8ca8e1dd2cf3842e0491caca51c62#egg=pymisp
-e git+https://github.com/Rafiot/uwhoisd.git@411572840eba4c72dc321c549b36a54ed5cea9de#egg=uwhois&subdirectory=client
-e git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader
-e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
-e git+https://github.com/sebdraven/pyonyphe@cbb0168d5cb28a9f71f7ab3773164a7039ccdb12#egg=pyonyphe
aiohttp==3.4.4
@ -23,18 +24,24 @@ dnspython==1.16.0
domaintools-api==0.3.3
enum-compat==0.0.2
ez-setup==0.9
ezodf==0.3.2
future==0.17.1
httplib2==0.12.3
idna-ssl==1.1.0 ; python_version < '3.7'
idna==2.8
isodate==0.6.0
jsonschema==3.0.1
lxml==4.3.3
maclookup==1.0.3
multidict==4.5.2
np==1.0.2
numpy==1.16.3
oauth2==1.9.0.post1
opencv-python==4.1.0.25
pandas-ods-reader==0.0.6
pandas==0.24.2
passivetotal==1.0.30
pdftotext==2.1.1
pillow==6.0.0
psutil==5.6.2
pyeupi==1.0
@ -45,6 +52,9 @@ pypssl==2.1
pyrsistent==0.15.1
pytesseract==0.2.6
python-dateutil==2.8.0
python-docx==0.8.10
python-pptx==0.6.17
pytz==2019.1
pyyaml==5.1
pyzbar==0.1.8
rdflib==4.2.2
@ -65,6 +75,7 @@ urlarchiver==0.2
urllib3==1.24.2
vulners==1.5.0
wand==0.5.3
xlrd==1.2.0
xlsxwriter==1.1.7
yara-python==3.8.1
yarl==1.3.0

52
misp_modules/__init__.py
View File

@ -210,37 +210,59 @@ class QueryModule(tornado.web.RequestHandler):
self.finish()
def _launch_from_current_dir():
log.info('Launch MISP modules server from current directory.')
os.chdir(os.path.dirname(__file__))
modulesdir = 'modules'
helpersdir = 'helpers'
load_helpers(helpersdir=helpersdir)
return load_modules(modulesdir)
def main():
global mhandlers
global loaded_modules
signal.signal(signal.SIGINT, handle_signal)
signal.signal(signal.SIGTERM, handle_signal)
argParser = argparse.ArgumentParser(description='misp-modules server')
argParser = argparse.ArgumentParser(description='misp-modules server', formatter_class=argparse.RawTextHelpFormatter)
argParser.add_argument('-t', default=False, action='store_true', help='Test mode')
argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)')
argParser.add_argument('-d', default=False, action='store_true', help='Enable debugging')
argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)')
argParser.add_argument('-l', default='localhost', help='misp-modules listen address (default localhost)')
argParser.add_argument('-m', default=[], action='append', help='Register a custom module')
argParser.add_argument('--devel', default=False, action='store_true', help='''Start in development mode, enable debug, start only the module(s) listed in -m.\nExample: -m misp_modules.modules.expansion.bgpranking''')
args = argParser.parse_args()
port = args.p
listen = args.l
log = init_logger(level=args.d)
if args.s:
log.info('Launch MISP modules server from package.')
load_package_helpers()
mhandlers, loaded_modules = load_package_modules()
if args.devel:
log = init_logger(level=True)
log.info('Launch MISP modules server in developement mode. Enable debug, load a list of modules is -m is used.')
if args.m:
mhandlers = {}
modules = []
for module in args.m:
splitted = module.split(".")
modulename = splitted[-1]
moduletype = splitted[2]
mhandlers[modulename] = importlib.import_module(module)
mhandlers['type:' + modulename] = moduletype
modules.append(modulename)
log.info('MISP modules {0} imported'.format(modulename))
else:
mhandlers, loaded_modules = _launch_from_current_dir()
else:
log.info('Launch MISP modules server from current directory.')
os.chdir(os.path.dirname(__file__))
modulesdir = 'modules'
helpersdir = 'helpers'
load_helpers(helpersdir=helpersdir)
mhandlers, loaded_modules = load_modules(modulesdir)
log = init_logger(level=args.d)
if args.s:
log.info('Launch MISP modules server from package.')
load_package_helpers()
mhandlers, loaded_modules = load_package_modules()
else:
mhandlers, loaded_modules = _launch_from_current_dir()
for module in args.m:
mispmod = importlib.import_module(module)
mispmod.register(mhandlers, loaded_modules)
for module in args.m:
mispmod = importlib.import_module(module)
mispmod.register(mhandlers, loaded_modules)
service = [(r'/modules', ListModules), (r'/query', QueryModule)]