From 2753f354ab3291985b8beb9aa745fe30d1e6b2a8 Mon Sep 17 00:00:00 2001 From: Vincent-CIRCL Date: Mon, 18 Feb 2019 14:27:16 +0100 Subject: [PATCH 01/16] test update --- misp_modules/modules/export_mod/pdfexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index df7f879..1b4c731 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -12,7 +12,7 @@ from pymisp import MISPEvent misperrors = {'error': 'Error'} -moduleinfo = {'version': '1', +moduleinfo = {'version': '42', 'author': 'Raphaël Vinot', 'description': 'Simple export to PDF', 'module-type': ['export'], From be01d547791f7f9950dad170b3908155e15ff3c8 Mon Sep 17 00:00:00 2001 From: Vincent-CIRCL Date: Mon, 18 Feb 2019 15:23:57 +0100 Subject: [PATCH 02/16] print values --- misp_modules/modules/export_mod/pdfexport.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index 1b4c731..074e473 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -144,6 +144,8 @@ def handler(q=False): return False for evt in request['data']: + print(request['data']) + report = ReportGenerator() report.report_headers() report.from_event(evt) From 2d29ce11bbf8dadd82fe886330476488df32c140 Mon Sep 17 00:00:00 2001 From: Falconieri Date: Thu, 21 Feb 2019 15:42:18 +0100 Subject: [PATCH 03/16] Test 1 - PDF call --- misp_modules/modules/export_mod/pdfexport.py | 68 +++++++------------- 1 file changed, 22 insertions(+), 46 deletions(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index 074e473..23d0edd 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -7,61 +7,26 @@ import shlex import subprocess import base64 -from pymisp import MISPEvent +from pymisp import MISPEvent, reportlab_generator misperrors = {'error': 'Error'} -moduleinfo = {'version': '42', - 'author': 'Raphaël Vinot', +moduleinfo = {'version': '2', + 'author': 'Vincent Falconieri (prev. Raphaël Vinot)', 'description': 'Simple export to PDF', 'module-type': ['export'], 'require_standard_format': True} moduleconfig = [] - mispattributes = {} + outputFileExtension = "pdf" responseType = "application/pdf" types_to_attach = ['ip-dst', 'url', 'domain'] objects_to_attach = ['domain-ip'] -headers = """ -:toc: right -:toclevels: 1 -:toc-title: Daily Report -:icons: font -:sectanchors: -:sectlinks: -= Daily report by {org_name} -{date} - -:icons: font - -""" - -event_level_tags = """ -IMPORTANT: This event is classified TLP:{value}. - -{expanded} - -""" - -attributes = """ -=== Indicator(s) of compromise - -{list_attributes} - -""" - -title = """ -== ({internal_id}) {title} - -{summary} - -""" - class ReportGenerator(): def __init__(self): @@ -79,6 +44,9 @@ class ReportGenerator(): self.misp_event = MISPEvent() self.misp_event.load(event) + ''' + + def attributes(self): if not self.misp_event.attributes: return '' @@ -132,7 +100,7 @@ class ReportGenerator(): self.report += self.title() self.report += self.event_level_tags() self.report += self.attributes() - + ''' def handler(q=False): if q is False: @@ -144,19 +112,27 @@ def handler(q=False): return False for evt in request['data']: + + ''' + print(" DATA ") print(request['data']) + + reportlab_generator. report = ReportGenerator() report.report_headers() report.from_event(evt) report.asciidoc() - command_line = 'asciidoctor-pdf -' - args = shlex.split(command_line) - with subprocess.Popen(args, stdout=subprocess.PIPE, stdin=subprocess.PIPE) as process: - cmd_out, cmd_err = process.communicate( - input=report.report.encode('utf-8')) - return {'response': [], 'data': str(base64.b64encode(cmd_out), 'utf-8')} + print(" REPORT : ") + print(report) + ''' + misp_event = MISPEvent() + misp_event.load(request['data']) + + pdf = reportlab_generator.get_base64_from_buffer(reportlab_generator.convert_event_in_pdf_buffer(misp_event)) + + return {'response': [], 'data': str(pdf, 'utf-8')} def introspection(): From a93b34208f358c76184b725f656f43f39dbd8e18 Mon Sep 17 00:00:00 2001 From: Falconieri Date: Fri, 22 Feb 2019 10:14:22 +0100 Subject: [PATCH 04/16] fix: [pdfexport] Bugfix on PyMisp exportpdf call --- misp_modules/modules/export_mod/pdfexport.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index 23d0edd..cb4e297 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -7,8 +7,10 @@ import shlex import subprocess import base64 -from pymisp import MISPEvent, reportlab_generator +print("test PDF pdf export (reportlab generator import)") +from pymisp import MISPEvent +from pymisp.tools import reportlab_generator misperrors = {'error': 'Error'} @@ -45,8 +47,6 @@ class ReportGenerator(): self.misp_event.load(event) ''' - - def attributes(self): if not self.misp_event.attributes: return '' @@ -127,12 +127,13 @@ def handler(q=False): print(" REPORT : ") print(report) ''' - misp_event = MISPEvent() - misp_event.load(request['data']) - pdf = reportlab_generator.get_base64_from_buffer(reportlab_generator.convert_event_in_pdf_buffer(misp_event)) + misp_event = MISPEvent() + misp_event.load(evt) - return {'response': [], 'data': str(pdf, 'utf-8')} + pdf = reportlab_generator.get_base64_from_value(reportlab_generator.convert_event_in_pdf_buffer(misp_event)) + + return {'response': [], 'data': str(pdf, 'utf-8')} def introspection(): @@ -164,3 +165,8 @@ def introspection(): def version(): moduleinfo['config'] = moduleconfig return moduleinfo + +import pprint + +if __name__ == "__main__": + pprint.pprint("test") \ No newline at end of file From 40cd32f1b8c073caf20d133ca0e9780ec2f602cc Mon Sep 17 00:00:00 2001 From: Falconieri Date: Fri, 22 Feb 2019 10:25:12 +0100 Subject: [PATCH 05/16] tidy: Remove old dead export code --- misp_modules/modules/export_mod/pdfexport.py | 78 -------------------- 1 file changed, 78 deletions(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index cb4e297..ef3d775 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -7,8 +7,6 @@ import shlex import subprocess import base64 -print("test PDF pdf export (reportlab generator import)") - from pymisp import MISPEvent from pymisp.tools import reportlab_generator @@ -46,62 +44,6 @@ class ReportGenerator(): self.misp_event = MISPEvent() self.misp_event.load(event) - ''' - def attributes(self): - if not self.misp_event.attributes: - return '' - list_attributes = [] - for attribute in self.misp_event.attributes: - if attribute.type in types_to_attach: - list_attributes.append("* {}".format(attribute.value)) - for obj in self.misp_event.Object: - if obj.name in objects_to_attach: - for attribute in obj.Attribute: - if attribute.type in types_to_attach: - list_attributes.append("* {}".format(attribute.value)) - return attributes.format(list_attributes="\n".join(list_attributes)) - - def _get_tag_info(self, machinetag): - return self.taxonomies.revert_machinetag(machinetag) - - def report_headers(self): - content = {'org_name': 'name', - 'date': date.today().isoformat()} - self.report += headers.format(**content) - - def event_level_tags(self): - if not self.misp_event.Tag: - return '' - for tag in self.misp_event.Tag: - # Only look for TLP for now - if tag['name'].startswith('tlp'): - tax, predicate = self._get_tag_info(tag['name']) - return self.event_level_tags.format(value=predicate.predicate.upper(), expanded=predicate.expanded) - - def title(self): - internal_id = '' - summary = '' - # Get internal refs for report - if not hasattr(self.misp_event, 'Object'): - return '' - for obj in self.misp_event.Object: - if obj.name != 'report': - continue - for a in obj.Attribute: - if a.object_relation == 'case-number': - internal_id = a.value - if a.object_relation == 'summary': - summary = a.value - - return title.format(internal_id=internal_id, title=self.misp_event.info, - summary=summary) - - def asciidoc(self, lang='en'): - self.report += self.title() - self.report += self.event_level_tags() - self.report += self.attributes() - ''' - def handler(q=False): if q is False: return False @@ -113,21 +55,6 @@ def handler(q=False): for evt in request['data']: - ''' - print(" DATA ") - print(request['data']) - - reportlab_generator. - - report = ReportGenerator() - report.report_headers() - report.from_event(evt) - report.asciidoc() - - print(" REPORT : ") - print(report) - ''' - misp_event = MISPEvent() misp_event.load(evt) @@ -165,8 +92,3 @@ def introspection(): def version(): moduleinfo['config'] = moduleconfig return moduleinfo - -import pprint - -if __name__ == "__main__": - pprint.pprint("test") \ No newline at end of file From 9f0f6e71e87f658ebb4518f9c9142db1cf0efe1e Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Fri, 22 Feb 2019 12:15:28 +0100 Subject: [PATCH 06/16] chg: PyMISP requirement --- REQUIREMENTS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REQUIREMENTS b/REQUIREMENTS index 0720e90..6f3d1b2 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -3,7 +3,7 @@ -e git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client -e git+https://github.com/D4-project/IPASN-History.git/@e846cd36fe1ed6b22f60890bba89f84e61b62e59#egg=pyipasnhistory&subdirectory=client -e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 --e git+https://github.com/MISP/PyMISP.git@2c877f2aec11b7f5d2f23dfc5ce7398b2ce33b48#egg=pymisp +-e git+https://github.com/MISP/PyMISP.git@ccd7565d3ce4693b96ea2352792099b40c53e494#egg=pymisp -e git+https://github.com/Rafiot/uwhoisd.git@f6f035e52213c8abc20f2084d28cfffb399457cb#egg=uwhois&subdirectory=client -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails -e git+https://github.com/sebdraven/pyonyphe@66329baeee7cab844f2203c047c2551828eaf14d#egg=pyonyphe From 66ee78e7af41f7062e3239e7fe676c80dc8a378d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sun, 24 Feb 2019 16:02:13 -0800 Subject: [PATCH 07/16] new: Add systemd launcher --- etc/systemd/system/misp-modules.service | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 etc/systemd/system/misp-modules.service diff --git a/etc/systemd/system/misp-modules.service b/etc/systemd/system/misp-modules.service new file mode 100644 index 0000000..3ff05ae --- /dev/null +++ b/etc/systemd/system/misp-modules.service @@ -0,0 +1,14 @@ +[Unit] +Description=System-wide instance of the MISP Modules +After=network.target + +[Service] +User=www-data +Group=www-data +WorkingDirectory=/usr/local/src/misp-modules +Environment="PATH=/var/www/MISP/venv/bin" +ExecStart=misp-modules -l 127.0.0.1 -s + +[Install] +WantedBy=multi-user.target + From 43d2ae6203a484e4614166a610a3d6bc73c12b03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Sun, 24 Feb 2019 18:20:28 -0800 Subject: [PATCH 08/16] fix: systemd service --- etc/systemd/system/misp-modules.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/systemd/system/misp-modules.service b/etc/systemd/system/misp-modules.service index 3ff05ae..99cd102 100644 --- a/etc/systemd/system/misp-modules.service +++ b/etc/systemd/system/misp-modules.service @@ -7,7 +7,7 @@ User=www-data Group=www-data WorkingDirectory=/usr/local/src/misp-modules Environment="PATH=/var/www/MISP/venv/bin" -ExecStart=misp-modules -l 127.0.0.1 -s +ExecStart=/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s [Install] WantedBy=multi-user.target From a3a871f2faa6b27c79e8c55bd0b128b9edbc7cf3 Mon Sep 17 00:00:00 2001 From: Falconieri Date: Mon, 25 Feb 2019 15:51:33 +0100 Subject: [PATCH 09/16] fix [exportpdf] update parameters for links generation --- misp_modules/modules/export_mod/pdfexport.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index ef3d775..977ee87 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -18,7 +18,9 @@ moduleinfo = {'version': '2', 'module-type': ['export'], 'require_standard_format': True} -moduleconfig = [] +# config fields that your code expects from the site admin +moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata"] + mispattributes = {} outputFileExtension = "pdf" @@ -53,12 +55,19 @@ def handler(q=False): if 'data' not in request: return False + config = {} + + # Construct config object for reportlab_generator + for config_item in moduleconfig : + if (request.get('config')) and (request['config'].get(config_item) is not None): + config[config_item] = request['config'].get(config_item) + for evt in request['data']: misp_event = MISPEvent() misp_event.load(evt) - pdf = reportlab_generator.get_base64_from_value(reportlab_generator.convert_event_in_pdf_buffer(misp_event)) + pdf = reportlab_generator.get_base64_from_value(reportlab_generator.convert_event_in_pdf_buffer(misp_event, config)) return {'response': [], 'data': str(pdf, 'utf-8')} From 0d8ead483e204045eaff9af35bc61836488c30fc Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 25 Feb 2019 16:18:41 +0100 Subject: [PATCH 10/16] chg: [PyMISP] dep updated to the latest version --- REQUIREMENTS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REQUIREMENTS b/REQUIREMENTS index 6f3d1b2..e42481b 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -3,7 +3,7 @@ -e git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client -e git+https://github.com/D4-project/IPASN-History.git/@e846cd36fe1ed6b22f60890bba89f84e61b62e59#egg=pyipasnhistory&subdirectory=client -e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 --e git+https://github.com/MISP/PyMISP.git@ccd7565d3ce4693b96ea2352792099b40c53e494#egg=pymisp +-e git+https://github.com/MISP/PyMISP.git@345f055844fed0acdfb34c52d96d1751728bb82c#egg=pymisp -e git+https://github.com/Rafiot/uwhoisd.git@f6f035e52213c8abc20f2084d28cfffb399457cb#egg=uwhois&subdirectory=client -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails -e git+https://github.com/sebdraven/pyonyphe@66329baeee7cab844f2203c047c2551828eaf14d#egg=pyonyphe From 9e48b3994a70cc9447c279470a3dce9b23a1d278 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 25 Feb 2019 20:29:04 +0100 Subject: [PATCH 11/16] chg: [requirements] updated --- REQUIREMENTS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REQUIREMENTS b/REQUIREMENTS index e42481b..69b0568 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -3,7 +3,7 @@ -e git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client -e git+https://github.com/D4-project/IPASN-History.git/@e846cd36fe1ed6b22f60890bba89f84e61b62e59#egg=pyipasnhistory&subdirectory=client -e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471 --e git+https://github.com/MISP/PyMISP.git@345f055844fed0acdfb34c52d96d1751728bb82c#egg=pymisp +-e git+https://github.com/MISP/PyMISP.git@634ecc3ac308d01ebf5f5fbb9aace7746a2b8707#egg=pymisp -e git+https://github.com/Rafiot/uwhoisd.git@f6f035e52213c8abc20f2084d28cfffb399457cb#egg=uwhois&subdirectory=client -e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails -e git+https://github.com/sebdraven/pyonyphe@66329baeee7cab844f2203c047c2551828eaf14d#egg=pyonyphe From bbe7fe51e70ee7ef24fb9a9573d7ad5a85ddaf9f Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 25 Feb 2019 20:34:48 +0100 Subject: [PATCH 12/16] chg: [pipenv] Pipfile.lock updated --- Pipfile.lock | 174 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 106 insertions(+), 68 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 19f32f0..1c08572 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -150,9 +150,9 @@ }, "httplib2": { "hashes": [ - "sha256:f61fb838a94ce3b349aa32c92fd8430f7e3511afdb18bf9640d647e30c90a6d6" + "sha256:4ba6b8fd77d0038769bf3c33c9a96a6f752bc4cdf739701fdcaf210121f399d4" ], - "version": "==0.12.0" + "version": "==0.12.1" }, "idna": { "hashes": [ @@ -177,10 +177,10 @@ }, "jsonschema": { "hashes": [ - "sha256:000e68abd33c972a5248544925a0cae7d1125f9bf6c58280d37546b946769a08", - "sha256:6ff5f3180870836cae40f06fa10419f557208175f13ad7bc26caa77beb1f6e02" + "sha256:acc8a90c31d11060516cfd0b414b9f8bcf4bc691b21f0f786ea57dd5255c79db", + "sha256:dd3f8ecb1b52d94d45eedb67cb86cac57b94ded562c5d98f63719e55ce58557b" ], - "version": "==2.6.0" + "version": "==3.0.0" }, "maclookup": { "hashes": [ @@ -281,22 +281,22 @@ }, "psutil": { "hashes": [ - "sha256:04d2071100aaad59f9bcbb801be2125d53b2e03b1517d9fed90b45eea51d297e", - "sha256:1aba93430050270750d046a179c5f3d6e1f5f8b96c20399ba38c596b28fc4d37", - "sha256:3ac48568f5b85fee44cd8002a15a7733deca056a191d313dbf24c11519c0c4a8", - "sha256:96f3fdb4ef7467854d46ad5a7e28eb4c6dc6d455d751ddf9640cd6d52bdb03d7", - "sha256:b755be689d6fc8ebc401e1d5ce5bac867e35788f10229e166338484eead51b12", - "sha256:c8ee08ad1b716911c86f12dc753eb1879006224fd51509f077987bb6493be615", - "sha256:d0c4230d60376aee0757d934020b14899f6020cd70ef8d2cb4f228b6ffc43e8f", - "sha256:d23f7025bac9b3e38adc6bd032cdaac648ac0074d18e36950a04af35458342e8", - "sha256:f0fcb7d3006dd4d9ccf3ccd0595d44c6abbfd433ec31b6ca177300ee3f19e54e" + "sha256:5ce6b5eb0267233459f4d3980c205828482f450999b8f5b684d9629fea98782a", + "sha256:72cebfaa422b7978a1d3632b65ff734a34c6b34f4578b68a5c204d633756b810", + "sha256:77c231b4dff8c1c329a4cd1c22b96c8976c597017ff5b09993cd148d6a94500c", + "sha256:8846ab0be0cdccd6cc92ecd1246a16e2f2e49f53bd73e522c3a75ac291e1b51d", + "sha256:a013b4250ccbddc9d22feca0f986a1afc71717ad026c0f2109bbffd007351191", + "sha256:ad43b83119eeea6d5751023298cd331637e542cbd332196464799e25a5519f8f", + "sha256:c177777c787d247d02dae6c855330f9ed3e1abf8ca1744c26dd5ff968949999a", + "sha256:ec1ef313530a9457e48d25e3fdb1723dfa636008bf1b970027462d46f2555d59", + "sha256:ef3e5e02b3c5d1df366abe7b4820400d5c427579668ad4465ff189d28ded5ebd" ], - "version": "==5.5.0" + "version": "==5.5.1" }, "pybgpranking": { "editable": true, "git": "https://github.com/D4-project/BGP-Ranking.git/", - "ref": "7e698f87366e6f99b4d0d11852737db28e3ddc62", + "ref": "37c97ae252ec4bf1d67733a49d4895c8cb009cf9", "subdirectory": "client" }, "pydnstrails": { @@ -333,12 +333,12 @@ "pymisp": { "editable": true, "git": "https://github.com/MISP/PyMISP.git", - "ref": "2c877f2aec11b7f5d2f23dfc5ce7398b2ce33b48" + "ref": "634ecc3ac308d01ebf5f5fbb9aace7746a2b8707" }, "pyonyphe": { "editable": true, "git": "https://github.com/sebdraven/pyonyphe", - "ref": "66329baeee7cab844f2203c047c2551828eaf14d" + "ref": "cbb0168d5cb28a9f71f7ab3773164a7039ccdb12" }, "pyparsing": { "hashes": [ @@ -361,6 +361,12 @@ "index": "pypi", "version": "==2.1" }, + "pyrsistent": { + "hashes": [ + "sha256:3ca82748918eb65e2d89f222b702277099aca77e34843c5eb9d52451173970e2" + ], + "version": "==0.14.11" + }, "pytesseract": { "hashes": [ "sha256:11c20321595b6e2e904b594633edf1a717212b13bac7512986a2d807b8849770" @@ -370,10 +376,10 @@ }, "python-dateutil": { "hashes": [ - "sha256:063df5763652e21de43de7d9e00ccf239f953a832941e37be541614732cdfc93", - "sha256:88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02" + "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", + "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" ], - "version": "==2.7.5" + "version": "==2.8.0" }, "pyyaml": { "hashes": [ @@ -400,10 +406,43 @@ }, "redis": { "hashes": [ - "sha256:74c892041cba46078ae1ef845241548baa3bd3634f9a6f0f952f006eb1619c71", - "sha256:7ba8612bbfd966dea8c62322543fed0095da2834dbd5a7c124afbc617a156aa7" + "sha256:724932360d48e5407e8f82e405ab3650a36ed02c7e460d1e6fddf0f038422b54", + "sha256:9b19425a38fd074eb5795ff2b0d9a55b46a44f91f5347995f27e3ad257a7d775" ], - "version": "==3.1.0" + "version": "==3.2.0" + }, + "reportlab": { + "hashes": [ + "sha256:069f684cd0aaa518a27dc9124aed29cee8998e21ddf19604e53214ec8462bdd7", + "sha256:09b68ec01d86b4b120456b3f3202570ec96f57624e3a4fc36f3829323391daa4", + "sha256:0c32be9a406172c29ea20ff55a709ccac1e7fb09f15aba67cb7b455fd1d3dbe0", + "sha256:233196cf25e97cfe7c452524ea29d9a4909f1cb66599299233be1efaaaa7a7a3", + "sha256:2b5e4533f3e5b962835a5ce44467e66d1ecc822761d1b508077b5087a06be338", + "sha256:2e860bcdace5a558356802a92ae8658d7e5fdaa00ded82e83a3f2987c562cb66", + "sha256:3546029e63a9a9dc24ee38959eb417678c2425b96cd27b31e09e216dafc94666", + "sha256:4452b93f9c73b6b70311e7d69082d64da81b38e91bfb4766397630092e6da6fd", + "sha256:528c74a1c6527d1859c2c7a64a94a1cba485b00175162ea23699ae58a1e94939", + "sha256:6116e750f98018febc08dfee6df20446cf954adbcfa378d2c703d56c8864aff3", + "sha256:6b2b3580c647d75ef129172cb3da648cdb24566987b0b59c5ebb80ab770748d6", + "sha256:727b5f2bed08552d143fc99649b1863c773729f580a416844f9d9967bb0a1ae8", + "sha256:74c24a3ec0a3d4f8acb13a07192f45bdb54a1cc3c2286241677e7e8bcd5011fa", + "sha256:98ccd2f8b4f8636db05f3f14db0b471ad6bb4b66ae0dc9052c4822b3bd5d6a7d", + "sha256:a5905aa567946bc938b489a7249c7890c3fd3c9b7b5680dece5bc551c2ddbe0d", + "sha256:acbb7f676b8586b770719e9683eda951fdb38eb7970d46fcbf3cdda88d912a64", + "sha256:b5e30f865add48cf880f1c363eb505b97f2f7baaa88c155f87a335a76515a3e5", + "sha256:be2a7c33a2c28bbd3f453ffe4f0e5200b88c803a097f4cf52d69c6b53fad7a8f", + "sha256:c356bb600f59ac64955813d6497a08bfd5d0c451cb5829b61e3913d0ac084e26", + "sha256:c7ec4ae2393beab584921b1287a04e94fd98c28315e348362d89b85f4b464546", + "sha256:d476edc831bb3e9ebd04d1403abaf3ea57b3e4c2276c91a54fdfb6efbd3f9d97", + "sha256:db059e1a0691c872784062421ec51848539eb4f5210142682e61059a5ca7cc55", + "sha256:dd423a6753509ab14a0ac1b5be39d219c8f8d3781cce3deb4f45eda31969b5e8", + "sha256:ed9b7c0d71ce6fe2b31c6cde530ad8238632b876a5d599218739bda142a77f7c", + "sha256:f0a2465af4006f97b05e1f1546d67d3a3213d414894bf28be7f87f550a7f4a55", + "sha256:f20bfe26e57e8e1f575a9e0325be04dd3562db9f247ffdd73b5d4df6dec53bc2", + "sha256:f3463f2cb40a1b515ac0133ba859eca58f53b56760da9abb27ed684c565f853c", + "sha256:facc3c9748ab1525fb8401a1223bce4f24f0d6aa1a9db86c55db75777ccf40f9" + ], + "version": "==3.5.13" }, "requests": { "hashes": [ @@ -422,10 +461,10 @@ }, "shodan": { "hashes": [ - "sha256:c40abb6ff2fd66bdee9f773746fb961eefdfaa8e720a07cb12fb70def136268d" + "sha256:f93b7199e89eecf5c84647f66316c2c044c3aebfc1fe4d9caa43dfda07f74c4e" ], "index": "pypi", - "version": "==1.10.4" + "version": "==1.11.1" }, "sigmatools": { "hashes": [ @@ -443,10 +482,10 @@ }, "soupsieve": { "hashes": [ - "sha256:466910df7561796a60748826781ebe9a888f7a1668a636ae86783f44d10aae73", - "sha256:87db12ae79194f0ff9808d2b1641c4f031ae39ffa3cab6b907ea7c1e5e5ed445" + "sha256:afa56bf14907bb09403e5d15fbed6275caa4174d36b975226e3b67a3bb6e2c4b", + "sha256:eaed742b48b1f3e2d45ba6f79401b2ed5dc33b2123dfe216adb90d4bfa0ade26" ], - "version": "==1.7.3" + "version": "==1.8" }, "sparqlwrapper": { "hashes": [ @@ -500,49 +539,48 @@ "uwhois": { "editable": true, "git": "https://github.com/Rafiot/uwhoisd.git", - "ref": "f6f035e52213c8abc20f2084d28cfffb399457cb", + "ref": "411572840eba4c72dc321c549b36a54ed5cea9de", "subdirectory": "client" }, "vulners": { "hashes": [ - "sha256:5f05404041cfaa8e5367bf884fc9ee319ebf34bedc495d7f84c433fa121cdb49", - "sha256:919b24df64ea55b6a8ba13e2a0530578f8a4be6a9cee257bf2214046e81c6f35", - "sha256:d45ecb13f5111947056a2dcc071b3e3fd45f6ad654eda06526245bba3850325e" + "sha256:40041bcf893fa1bfaf29c650369d9a249991911f28b4d8795f7bc06508013e14", + "sha256:6d00709300dcc7e2727499d8a60f51eaced1dc6b63cc19cb8a4b065b658c51aa", + "sha256:de8cef247c9852c39bd54434e63026b46bdb2bd4ca22813bf66626b7d359b0f3" ], "index": "pypi", - "version": "==1.4.0" + "version": "==1.4.4" }, "wand": { "hashes": [ - "sha256:3e59e4bda9ef9d643d90e881cc950c8eee1508ec2cde1c150a1cbd5a12c1c007", - "sha256:52763dbf65d00cf98d7bc910b49329eea15896249c5555d47e169f2b6efbe166" + "sha256:7d6b8dc9d4eaccc430b9c86e6b749013220c994970a3f39e902b397e2fa732c3", + "sha256:cc0b5c9cd50fecd10dc8888b739dd5984c6f8085d2954f34903b83ca39a91236" ], "index": "pypi", - "version": "==0.5.0" + "version": "==0.5.1" }, "xlsxwriter": { "hashes": [ - "sha256:7cc07619760641b67112dbe0df938399d4d915d9b9924bb58eb5c17384d29cc6", - "sha256:ae22658a0fc5b9e875fa97c213d1ffd617d86dc49bf08be99ebdac814db7bf36" + "sha256:de9ef46088489915eaaee00c7088cff93cf613e9990b46b933c98eb46f21b47f", + "sha256:df96eafc3136d9e790e35d6725b473e46ada6f585c1f6519da69b27f5c8873f7" ], - "version": "==1.1.2" + "version": "==1.1.5" }, "yara-python": { "hashes": [ - "sha256:03e5c5e333c8572e7994b0b11964d515d61a393f23c5e272f8d0e4229f368c58", - "sha256:0423e08bd618752a028ac0405ff8e0103f3a8fd607dde7618a64a4c010c3757b", - "sha256:0a0dd632dcdb347d1a9a8b1f6a83b3a77d5e63f691357ea4021fb1cf1d7ff0a4", - "sha256:728b99627a8072a877eaaa4dafb4eff39d1b14ff4fd70d39f18899ce81e29625", - "sha256:7cb0d5724eccfa52e1bcd352a56cb4dc422aa51f5f6d0945d4f830783927513b", - "sha256:8c76531e89806c0309586dd4863a972d12f1d5d63261c6d4b9331a99859fd1d8", - "sha256:9472676583e212bc4e17c2236634e02273d53c872b350f0571b48e06183de233", - "sha256:9735b680a7d95c1d3f255c351bb067edc62cdb3c0999f7064278cb2c85245405", - "sha256:997f104590167220a9af5564c042ec4d6534261e7b8a5b49655d8dffecc6b8a2", - "sha256:a48e071d02a3699363e628ac899b5b7237803bcb4b512c92ebcb4fb9b1488497", - "sha256:b67c0d75a6519ca357b4b85ede9768c96a81fff20fbc169bd805ff009ddee561" + "sha256:0d002170b2f2c56ff75c846ad1e6765f59d4569e81494c76f15243197e4a974c", + "sha256:16be7c7623685b4b2813db33a39553d6faef236ddffa0758c08e2071ab11ed84", + "sha256:2031ac6ac01754dbc82b5a47b69cb91302c6b66ea9d9f2f27cc2eaf771e19c14", + "sha256:228a96efc86c766d968c984bd80f5ebb0bb775afb9045c10fb632e2b7275c9c1", + "sha256:468a9770e6b578f0562a540b6cb5cafd4122bea989404b53440d4eb065d54eda", + "sha256:752d12a795159b806cd74ab7f0fd7c3a14cb6e17c9e4a818511dc7a4932b15df", + "sha256:755406cb5fa944d5e0dd097a4b25c3fcdd5ba244f0367114afed1ba30ccd2a12", + "sha256:7936c10c8802fc279802dcdda8270d3fda5c3d3c8fbe6bb02010934ed30b8929", + "sha256:95c8d39ee5938744dbd8e0153ec6d466f8a4ed11b8ac7b1068f498c26a292b65", + "sha256:cfd00cfb7bcbe862b0793f91b5393bad3fb37da78883af19924059367ba80f51" ], "index": "pypi", - "version": "==3.8.1" + "version": "==3.9.0" }, "yarl": { "hashes": [ @@ -643,11 +681,11 @@ }, "flake8": { "hashes": [ - "sha256:09b9bb539920776da542e67a570a5df96ff933c9a08b62cfae920bcc789e4383", - "sha256:e0f8cd519cfc0072c0ee31add5def09d2b3ef6040b34dc426445c3af9b02163c" + "sha256:859996073f341f2670741b51ec1e67a01da142831aa1fdc6242dbf88dffbe661", + "sha256:a796a115208f5c03b18f332f7c11729812c8c3ded6c46319c59b53efd3819da8" ], "index": "pypi", - "version": "==3.7.4" + "version": "==3.7.7" }, "idna": { "hashes": [ @@ -665,11 +703,11 @@ }, "more-itertools": { "hashes": [ - "sha256:38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4", - "sha256:c0a5785b1109a6bd7fac76d6837fd1feca158e54e521ccd2ae8bfe393cc9d4fc", - "sha256:fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9" + "sha256:0125e8f60e9e031347105eb1682cef932f5e97d7b9a1a28d9bf00c22a5daef40", + "sha256:590044e3942351a1bdb1de960b739ff4ce277960f2425ad4509446dbace8d9d1" ], - "version": "==5.0.0" + "markers": "python_version > '2.7'", + "version": "==6.0.0" }, "nose": { "hashes": [ @@ -682,17 +720,17 @@ }, "pluggy": { "hashes": [ - "sha256:8ddc32f03971bfdf900a81961a48ccf2fb677cf7715108f85295c67405798616", - "sha256:980710797ff6a041e9a73a5787804f848996ecaa6f8a1b1e08224a5894f2074a" + "sha256:19ecf9ce9db2fce065a7a0586e07cfb4ac8614fe96edf628a264b1c70116cf8f", + "sha256:84d306a647cc805219916e62aab89caa97a33a1dd8c342e87a37f91073cd4746" ], - "version": "==0.8.1" + "version": "==0.9.0" }, "py": { "hashes": [ - "sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694", - "sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6" + "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", + "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" ], - "version": "==1.7.0" + "version": "==1.8.0" }, "pycodestyle": { "hashes": [ @@ -710,11 +748,11 @@ }, "pytest": { "hashes": [ - "sha256:65aeaa77ae87c7fc95de56285282546cfa9c886dc8e5dc78313db1c25e21bc07", - "sha256:6ac6d467d9f053e95aaacd79f831dbecfe730f419c6c7022cb316b365cd9199d" + "sha256:067a1d4bf827ffdd56ad21bd46674703fce77c5957f6c1eef731f6146bfcef1c", + "sha256:9687049d53695ad45cf5fdc7bbd51f0c49f1ea3ecfc4b7f3fde7501b541f17f4" ], "index": "pypi", - "version": "==4.2.0" + "version": "==4.3.0" }, "requests": { "hashes": [ From 637d7f25381b33f30a947cc23c50325f088e21cc Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 25 Feb 2019 20:42:45 +0100 Subject: [PATCH 13/16] chg: [requirements] reportlab added --- REQUIREMENTS | 1 + 1 file changed, 1 insertion(+) diff --git a/REQUIREMENTS b/REQUIREMENTS index 69b0568..4891c60 100644 --- a/REQUIREMENTS +++ b/REQUIREMENTS @@ -47,6 +47,7 @@ rdflib==4.2.2 redis==3.1.0 requests-cache==0.4.13 requests==2.21.0 +reportlab shodan==1.10.4 sigmatools==0.7.1 six==1.12.0 From b0ea67e393f91aefbf770123ce6f4cd0699d0e5e Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 25 Feb 2019 21:11:24 +0100 Subject: [PATCH 14/16] chg: [pipenv] fix the temporary issue that python-yara is not officially released --- Pipfile | 3 ++- Pipfile.lock | 26 ++++++++++++++------------ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/Pipfile b/Pipfile index c086e62..45c05f5 100644 --- a/Pipfile +++ b/Pipfile @@ -25,12 +25,13 @@ pytesseract = "*" pygeoip = "*" beautifulsoup4 = "*" oauth2 = "*" -yara-python = ">=3.8.0" +yara-python = "==3.8.1" sigmatools = "*" stix2-patterns = "*" maclookup = "*" vulners = "*" blockchain = "*" +reportlab = "*" pyintel471 = {editable = true,git = "https://github.com/MISP/PyIntel471.git"} shodan = "*" Pillow = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 1c08572..9e6265d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f501a84bdd41ca21a2af020278ce030985cccd5f2f5683cd075797be4523587d" + "sha256": "d0cd64bfe7702365d3ea66d1f51a1ec8592df2490899e7e163fe38f97172561e" }, "pipfile-spec": 6, "requires": { @@ -442,6 +442,7 @@ "sha256:f3463f2cb40a1b515ac0133ba859eca58f53b56760da9abb27ed684c565f853c", "sha256:facc3c9748ab1525fb8401a1223bce4f24f0d6aa1a9db86c55db75777ccf40f9" ], + "index": "pypi", "version": "==3.5.13" }, "requests": { @@ -568,19 +569,20 @@ }, "yara-python": { "hashes": [ - "sha256:0d002170b2f2c56ff75c846ad1e6765f59d4569e81494c76f15243197e4a974c", - "sha256:16be7c7623685b4b2813db33a39553d6faef236ddffa0758c08e2071ab11ed84", - "sha256:2031ac6ac01754dbc82b5a47b69cb91302c6b66ea9d9f2f27cc2eaf771e19c14", - "sha256:228a96efc86c766d968c984bd80f5ebb0bb775afb9045c10fb632e2b7275c9c1", - "sha256:468a9770e6b578f0562a540b6cb5cafd4122bea989404b53440d4eb065d54eda", - "sha256:752d12a795159b806cd74ab7f0fd7c3a14cb6e17c9e4a818511dc7a4932b15df", - "sha256:755406cb5fa944d5e0dd097a4b25c3fcdd5ba244f0367114afed1ba30ccd2a12", - "sha256:7936c10c8802fc279802dcdda8270d3fda5c3d3c8fbe6bb02010934ed30b8929", - "sha256:95c8d39ee5938744dbd8e0153ec6d466f8a4ed11b8ac7b1068f498c26a292b65", - "sha256:cfd00cfb7bcbe862b0793f91b5393bad3fb37da78883af19924059367ba80f51" + "sha256:03e5c5e333c8572e7994b0b11964d515d61a393f23c5e272f8d0e4229f368c58", + "sha256:0423e08bd618752a028ac0405ff8e0103f3a8fd607dde7618a64a4c010c3757b", + "sha256:0a0dd632dcdb347d1a9a8b1f6a83b3a77d5e63f691357ea4021fb1cf1d7ff0a4", + "sha256:728b99627a8072a877eaaa4dafb4eff39d1b14ff4fd70d39f18899ce81e29625", + "sha256:7cb0d5724eccfa52e1bcd352a56cb4dc422aa51f5f6d0945d4f830783927513b", + "sha256:8c76531e89806c0309586dd4863a972d12f1d5d63261c6d4b9331a99859fd1d8", + "sha256:9472676583e212bc4e17c2236634e02273d53c872b350f0571b48e06183de233", + "sha256:9735b680a7d95c1d3f255c351bb067edc62cdb3c0999f7064278cb2c85245405", + "sha256:997f104590167220a9af5564c042ec4d6534261e7b8a5b49655d8dffecc6b8a2", + "sha256:a48e071d02a3699363e628ac899b5b7237803bcb4b512c92ebcb4fb9b1488497", + "sha256:b67c0d75a6519ca357b4b85ede9768c96a81fff20fbc169bd805ff009ddee561" ], "index": "pypi", - "version": "==3.9.0" + "version": "==3.8.1" }, "yarl": { "hashes": [ From e7fd7e8eb20ed92ab5b09e83d7acf004fa366b6f Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 25 Feb 2019 21:18:26 +0100 Subject: [PATCH 15/16] chg: [pdfexport] make flake8 happy --- misp_modules/modules/export_mod/pdfexport.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/misp_modules/modules/export_mod/pdfexport.py b/misp_modules/modules/export_mod/pdfexport.py index 977ee87..6b0c12f 100755 --- a/misp_modules/modules/export_mod/pdfexport.py +++ b/misp_modules/modules/export_mod/pdfexport.py @@ -1,11 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from datetime import date import json -import shlex -import subprocess -import base64 from pymisp import MISPEvent from pymisp.tools import reportlab_generator @@ -46,6 +42,7 @@ class ReportGenerator(): self.misp_event = MISPEvent() self.misp_event.load(event) + def handler(q=False): if q is False: return False @@ -58,12 +55,11 @@ def handler(q=False): config = {} # Construct config object for reportlab_generator - for config_item in moduleconfig : + for config_item in moduleconfig: if (request.get('config')) and (request['config'].get(config_item) is not None): config[config_item] = request['config'].get(config_item) for evt in request['data']: - misp_event = MISPEvent() misp_event.load(evt) From 2a59c6becc3e24c56febe838adbcc965e929d49b Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 25 Feb 2019 21:33:47 +0100 Subject: [PATCH 16/16] chg: [doc] PDF export --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ee4f2f8..501e54f 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [CEF](misp_modules/modules/export_mod/cef_export.py) module to export Common Event Format (CEF). * [GoAML export](misp_modules/modules/export_mod/goamlexport.py) module to export in [GoAML format](http://goaml.unodc.org/goaml/en/index.html). * [Lite Export](misp_modules/modules/export_mod/liteexport.py) module to export a lite event. -* [Simple PDF export](misp_modules/modules/export_mod/pdfexport.py) module to export in PDF (required: asciidoctor-pdf). +* [PDF export](misp_modules/modules/export_mod/pdfexport.py) module to export an event in PDF. * [Nexthink query format](misp_modules/modules/export_mod/nexthinkexport.py) module to export in Nexthink query format. * [osquery](misp_modules/modules/export_mod/osqueryexport.py) module to export in [osquery](https://osquery.io/) query format. * [ThreatConnect](misp_modules/modules/export_mod/threat_connect_export.py) module to export in ThreatConnect CSV format.