From 48c158271b5e3b0285ec8a4b915d1cf9d16784b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Apr 2019 13:48:38 +0200 Subject: [PATCH 1/4] new: Devel mode. Fix #293 --- misp_modules/__init__.py | 52 ++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/misp_modules/__init__.py b/misp_modules/__init__.py index 68a2c69..440ad3f 100644 --- a/misp_modules/__init__.py +++ b/misp_modules/__init__.py @@ -210,37 +210,59 @@ class QueryModule(tornado.web.RequestHandler): self.finish() +def _launch_from_current_dir(): + log.info('Launch MISP modules server from current directory.') + os.chdir(os.path.dirname(__file__)) + modulesdir = 'modules' + helpersdir = 'helpers' + load_helpers(helpersdir=helpersdir) + return load_modules(modulesdir) + + def main(): global mhandlers global loaded_modules signal.signal(signal.SIGINT, handle_signal) signal.signal(signal.SIGTERM, handle_signal) - argParser = argparse.ArgumentParser(description='misp-modules server') + argParser = argparse.ArgumentParser(description='misp-modules server', formatter_class=argparse.RawTextHelpFormatter) argParser.add_argument('-t', default=False, action='store_true', help='Test mode') argParser.add_argument('-s', default=False, action='store_true', help='Run a system install (package installed via pip)') argParser.add_argument('-d', default=False, action='store_true', help='Enable debugging') argParser.add_argument('-p', default=6666, help='misp-modules TCP port (default 6666)') argParser.add_argument('-l', default='localhost', help='misp-modules listen address (default localhost)') argParser.add_argument('-m', default=[], action='append', help='Register a custom module') + argParser.add_argument('--devel', default=False, action='store_true', help='''Start in development mode, enable debug, start only the module(s) listed in -m.\nExample: -m misp_modules.modules.expansion.bgpranking''') args = argParser.parse_args() port = args.p listen = args.l - log = init_logger(level=args.d) - if args.s: - log.info('Launch MISP modules server from package.') - load_package_helpers() - mhandlers, loaded_modules = load_package_modules() + if args.devel: + log = init_logger(level=True) + log.info('Launch MISP modules server in developement mode. Enable debug, load a list of modules is -m is used.') + if args.m: + mhandlers = {} + modules = [] + for module in args.m: + splitted = module.split(".") + modulename = splitted[-1] + moduletype = splitted[2] + mhandlers[modulename] = importlib.import_module(module) + mhandlers['type:' + modulename] = moduletype + modules.append(modulename) + log.info('MISP modules {0} imported'.format(modulename)) + else: + mhandlers, loaded_modules = _launch_from_current_dir() else: - log.info('Launch MISP modules server from current directory.') - os.chdir(os.path.dirname(__file__)) - modulesdir = 'modules' - helpersdir = 'helpers' - load_helpers(helpersdir=helpersdir) - mhandlers, loaded_modules = load_modules(modulesdir) + log = init_logger(level=args.d) + if args.s: + log.info('Launch MISP modules server from package.') + load_package_helpers() + mhandlers, loaded_modules = load_package_modules() + else: + mhandlers, loaded_modules = _launch_from_current_dir() - for module in args.m: - mispmod = importlib.import_module(module) - mispmod.register(mhandlers, loaded_modules) + for module in args.m: + mispmod = importlib.import_module(module) + mispmod.register(mhandlers, loaded_modules) service = [(r'/modules', ListModules), (r'/query', QueryModule)] From 929dbd2463f159f23de7df4b0089409af5670b8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Apr 2019 13:49:16 +0200 Subject: [PATCH 2/4] chg: Bump dependencies. --- Pipfile | 2 +- Pipfile.lock | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 116 insertions(+), 2 deletions(-) diff --git a/Pipfile b/Pipfile index 16ac02e..762e4c5 100644 --- a/Pipfile +++ b/Pipfile @@ -45,7 +45,7 @@ backscatter = "*" pyzbar = "*" opencv-python = "*" np = "*" -ODTReader = {editable = true,git = "https://github.com/cartertemm/ODTReader.git"} +ODTReader = {editable = true,git = "https://github.com/cartertemm/ODTReader.git/"} python-pptx = "*" python-docx = "*" ezodf = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 5d395d5..6ac6493 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "7fee9399d8a7151a79b6f8bbce64564062fd562b0a091fd45a875884d3fb954e" + "sha256": "1d4c69ced012268e1ab20cee76f76652e1acd9f5133636551d5686f15748d3b4" }, "pipfile-spec": 6, "requires": { @@ -150,6 +150,13 @@ ], "version": "==0.9" }, + "ezodf": { + "hashes": [ + "sha256:000da534f689c6d55297a08f9e2ed7eada9810d194d31d164388162fb391122d" + ], + "index": "pypi", + "version": "==0.3.2" + }, "future": { "hashes": [ "sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8" @@ -184,6 +191,38 @@ ], "version": "==3.0.1" }, + "lxml": { + "hashes": [ + "sha256:03984196d00670b2ab14ae0ea83d5cc0cfa4f5a42558afa9ab5fa745995328f5", + "sha256:0815b0c9f897468de6a386dc15917a0becf48cc92425613aa8bbfc7f0f82951f", + "sha256:175f3825f075cf02d15099eb52658457cf0ff103dcf11512b5d2583e1d40f58b", + "sha256:30e14c62d88d1e01a26936ecd1c6e784d4afc9aa002bba4321c5897937112616", + "sha256:3210da6f36cf4b835ff1be853962b22cc354d506f493b67a4303c88bbb40d57b", + "sha256:40f60819fbd5bad6e191ba1329bfafa09ab7f3f174b3d034d413ef5266963294", + "sha256:43b26a865a61549919f8a42e094dfdb62847113cf776d84bd6b60e4e3fc20ea3", + "sha256:4a03dd682f8e35a10234904e0b9508d705ff98cf962c5851ed052e9340df3d90", + "sha256:62f382cddf3d2e52cf266e161aa522d54fd624b8cc567bc18f573d9d50d40e8e", + "sha256:7b98f0325be8450da70aa4a796c4f06852949fe031878b4aa1d6c417a412f314", + "sha256:846a0739e595871041385d86d12af4b6999f921359b38affb99cdd6b54219a8f", + "sha256:a3080470559938a09a5d0ec558c005282e99ac77bf8211fb7b9a5c66390acd8d", + "sha256:ad841b78a476623955da270ab8d207c3c694aa5eba71f4792f65926dc46c6ee8", + "sha256:afdd75d9735e44c639ffd6258ce04a2de3b208f148072c02478162d0944d9da3", + "sha256:b4fbf9b552faff54742bcd0791ab1da5863363fb19047e68f6592be1ac2dab33", + "sha256:b90c4e32d6ec089d3fa3518436bdf5ce4d902a0787dbd9bb09f37afe8b994317", + "sha256:b91cfe4438c741aeff662d413fd2808ac901cc6229c838236840d11de4586d63", + "sha256:bdb0593a42070b0a5f138b79b872289ee73c8e25b3f0bea6564e795b55b6bcdd", + "sha256:c4e4bca2bb68ce22320297dfa1a7bf070a5b20bcbaec4ee023f83d2f6e76496f", + "sha256:cec4ab14af9eae8501be3266ff50c3c2aecc017ba1e86c160209bb4f0423df6a", + "sha256:e83b4b2bf029f5104bc1227dbb7bf5ace6fd8fabaebffcd4f8106fafc69fc45f", + "sha256:e995b3734a46d41ae60b6097f7c51ba9958648c6d1e0935b7e0ee446ee4abe22", + "sha256:f679d93dec7f7210575c85379a31322df4c46496f184ef650d3aba1484b38a2d", + "sha256:fd213bb5166e46974f113c8228daaef1732abc47cb561ce9c4c8eaed4bd3b09b", + "sha256:fdcb57b906dbc1f80666e6290e794ab8fb959a2e17aa5aee1758a85d1da4533f", + "sha256:ff424b01d090ffe1947ec7432b07f536912e0300458f9a7f48ea217dd8362b86" + ], + "index": "pypi", + "version": "==4.3.3" + }, "maclookup": { "hashes": [ "sha256:33bf8eaebe3b1e4ab4ae9277dd93c78024e0ebf6b3c42f76c37695bc26ce287a", @@ -273,6 +312,11 @@ "index": "pypi", "version": "==1.9.0.post1" }, + "odtreader": { + "editable": true, + "git": "https://github.com/cartertemm/ODTReader.git/", + "ref": "49d6938693f6faa3ff09998f86dba551ae3a996b" + }, "opencv-python": { "hashes": [ "sha256:1703a296a96d3d46615e5053f224867977accb4240bcaa0fcabcb0768bf5ac13", @@ -306,6 +350,40 @@ "index": "pypi", "version": "==4.1.0.25" }, + "pandas": { + "hashes": [ + "sha256:071e42b89b57baa17031af8c6b6bbd2e9a5c68c595bc6bf9adabd7a9ed125d3b", + "sha256:17450e25ae69e2e6b303817bdf26b2cd57f69595d8550a77c308be0cd0fd58fa", + "sha256:17916d818592c9ec891cbef2e90f98cc85e0f1e89ed0924c9b5220dc3209c846", + "sha256:2538f099ab0e9f9c9d09bbcd94b47fd889bad06dc7ae96b1ed583f1dc1a7a822", + "sha256:366f30710172cb45a6b4f43b66c220653b1ea50303fbbd94e50571637ffb9167", + "sha256:42e5ad741a0d09232efbc7fc648226ed93306551772fc8aecc6dce9f0e676794", + "sha256:4e718e7f395ba5bfe8b6f6aaf2ff1c65a09bb77a36af6394621434e7cc813204", + "sha256:4f919f409c433577a501e023943e582c57355d50a724c589e78bc1d551a535a2", + "sha256:4fe0d7e6438212e839fc5010c78b822664f1a824c0d263fd858f44131d9166e2", + "sha256:5149a6db3e74f23dc3f5a216c2c9ae2e12920aa2d4a5b77e44e5b804a5f93248", + "sha256:627594338d6dd995cfc0bacd8e654cd9e1252d2a7c959449228df6740d737eb8", + "sha256:83c702615052f2a0a7fb1dd289726e29ec87a27272d775cb77affe749cca28f8", + "sha256:8c872f7fdf3018b7891e1e3e86c55b190e6c5cee70cab771e8f246c855001296", + "sha256:90f116086063934afd51e61a802a943826d2aac572b2f7d55caaac51c13db5b5", + "sha256:a3352bacac12e1fc646213b998bce586f965c9d431773d9e91db27c7c48a1f7d", + "sha256:bcdd06007cca02d51350f96debe51331dec429ac8f93930a43eb8fb5639e3eb5", + "sha256:c1bd07ebc15285535f61ddd8c0c75d0d6293e80e1ee6d9a8d73f3f36954342d0", + "sha256:c9a4b7c55115eb278c19aa14b34fcf5920c8fe7797a09b7b053ddd6195ea89b3", + "sha256:cc8fc0c7a8d5951dc738f1c1447f71c43734244453616f32b8aa0ef6013a5dfb", + "sha256:d7b460bc316064540ce0c41c1438c416a40746fd8a4fb2999668bf18f3c4acf1" + ], + "index": "pypi", + "version": "==0.24.2" + }, + "pandas-ods-reader": { + "hashes": [ + "sha256:0f7d510639c8957a06aa1227b9f84d1be47a437dfd306464ce803b91cf5eeec4", + "sha256:d85ef58fc3aeac1616028d22954b6ef2e8983ab9bae015e1e90ce3979d138553" + ], + "index": "pypi", + "version": "==0.0.6" + }, "passivetotal": { "hashes": [ "sha256:d745a6519ec04e3a354682978ebf07778bf7602beac30307cbad075ff1a4418d" @@ -313,6 +391,13 @@ "index": "pypi", "version": "==1.0.30" }, + "pdftotext": { + "hashes": [ + "sha256:e3ad11efe0aa22cbfc46aa1296b2ea5a52ad208b778288311f2801adef178ccb" + ], + "index": "pypi", + "version": "==2.1.1" + }, "pillow": { "hashes": [ "sha256:15c056bfa284c30a7f265a41ac4cbbc93bdbfc0dfe0613b9cb8a8581b51a9e55", @@ -447,6 +532,27 @@ ], "version": "==2.8.0" }, + "python-docx": { + "hashes": [ + "sha256:bc76ecac6b2d00ce6442a69d03a6f35c71cd72293cd8405a7472dfe317920024" + ], + "index": "pypi", + "version": "==0.8.10" + }, + "python-pptx": { + "hashes": [ + "sha256:1f2d5d1d923d91f50a1f0ed794935e7d670993fdcb6c12c81cc83977c1f23e14" + ], + "index": "pypi", + "version": "==0.6.17" + }, + "pytz": { + "hashes": [ + "sha256:303879e36b721603cc54604edcac9d20401bdbe31e1e4fdee5b9f98d5d31dfda", + "sha256:d747dd3d23d77ef44c6a3526e274af6efeb0a6f1afd5a69ba4d5be4098c8e141" + ], + "version": "==2019.1" + }, "pyyaml": { "hashes": [ "sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c", @@ -642,6 +748,14 @@ "index": "pypi", "version": "==0.5.3" }, + "xlrd": { + "hashes": [ + "sha256:546eb36cee8db40c3eaa46c351e67ffee6eeb5fa2650b71bc4c758a29a1b29b2", + "sha256:e551fb498759fa3a5384a94ccd4c3c02eb7c00ea424426e212ac0c57be9dfbde" + ], + "index": "pypi", + "version": "==1.2.0" + }, "xlsxwriter": { "hashes": [ "sha256:2a40b427dac0f640031e5b33abe97e761de6e0f12d4d346e7b2e2b67cf6ee927", From d77fdabeb2fab2e7cdf010d8023f1678a6dba3f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Apr 2019 13:59:36 +0200 Subject: [PATCH 3/4] fix: Re-enable python 3.6 support --- Pipfile | 1 + Pipfile.lock | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index 762e4c5..36f059e 100644 --- a/Pipfile +++ b/Pipfile @@ -54,6 +54,7 @@ pandas_ods_reader = "*" pdftotext = "*" lxml = "*" xlrd = "*" +idna-ssl = {markers="python_version < '3.7'"} [requires] python_version = "3" diff --git a/Pipfile.lock b/Pipfile.lock index 6ac6493..fa07b4c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "1d4c69ced012268e1ab20cee76f76652e1acd9f5133636551d5686f15748d3b4" + "sha256": "9aac0a9c45df16b9502c13f9468095cf5ffdb8bc407fe2b55faee3ff53d8eba3" }, "pipfile-spec": 6, "requires": { @@ -177,6 +177,14 @@ ], "version": "==2.8" }, + "idna-ssl": { + "hashes": [ + "sha256:a933e3bb13da54383f9e8f35dc4f9cb9eb9b3b78c6b36f311254d6d0d92c6c7c" + ], + "index": "pypi", + "markers": "python_version < '3.7'", + "version": "==1.1.0" + }, "isodate": { "hashes": [ "sha256:2e364a3d5759479cdb2d37cce6b9376ea504db2ff90252a2e5b7cc89cc9ff2d8", From c9281e605da93f7ab9923b798e1e92e4b51fc26a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 26 Apr 2019 14:05:02 +0200 Subject: [PATCH 4/4] chg: Bump REQUIREMENTS --- REQUIREMENTS | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/REQUIREMENTS b/REQUIREMENTS index 37c5a68..9447c47 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -5,6 +5,7 @@ -e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 -e git+https://github.com/MISP/PyMISP.git@582dda0ce2a8ca8e1dd2cf3842e0491caca51c62#egg=pymisp -e git+https://github.com/Rafiot/uwhoisd.git@411572840eba4c72dc321c549b36a54ed5cea9de#egg=uwhois&subdirectory=client +-e git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails -e git+https://github.com/sebdraven/pyonyphe@cbb0168d5cb28a9f71f7ab3773164a7039ccdb12#egg=pyonyphe aiohttp==3.4.4 @@ -23,18 +24,24 @@ dnspython==1.16.0 domaintools-api==0.3.3 enum-compat==0.0.2 ez-setup==0.9 +ezodf==0.3.2 future==0.17.1 httplib2==0.12.3 +idna-ssl==1.1.0 ; python_version < '3.7' idna==2.8 isodate==0.6.0 jsonschema==3.0.1 +lxml==4.3.3 maclookup==1.0.3 multidict==4.5.2 np==1.0.2 numpy==1.16.3 oauth2==1.9.0.post1 opencv-python==4.1.0.25 +pandas-ods-reader==0.0.6 +pandas==0.24.2 passivetotal==1.0.30 +pdftotext==2.1.1 pillow==6.0.0 psutil==5.6.2 pyeupi==1.0 @@ -45,6 +52,9 @@ pypssl==2.1 pyrsistent==0.15.1 pytesseract==0.2.6 python-dateutil==2.8.0 +python-docx==0.8.10 +python-pptx==0.6.17 +pytz==2019.1 pyyaml==5.1 pyzbar==0.1.8 rdflib==4.2.2 @@ -65,6 +75,7 @@ urlarchiver==0.2 urllib3==1.24.2 vulners==1.5.0 wand==0.5.3 +xlrd==1.2.0 xlsxwriter==1.1.7 yara-python==3.8.1 yarl==1.3.0