From b5f2424f274813e044ba88210628575855263b5e Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Mon, 8 Apr 2019 16:17:22 +0900 Subject: [PATCH 1/7] chg: [doc] Updated README to reflect current virtualenv efforts. TODO: pipenv --- README.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 18b5548..400356e 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ * [ThreatAnalyzer](misp_modules/modules/import_mod/threatanalyzer_import.py) - An import module to process ThreatAnalyzer archive.zip/analysis.json sandbox exports. * [VMRay](misp_modules/modules/import_mod/vmray_import.py) - An import module to process VMRay export. -## How to install and start MISP modules in a Python virtualenv? +## How to install and start MISP modules in a Python virtualenv? (recommended) ~~~~bash sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick virtualenv @@ -97,7 +97,10 @@ sudo git clone https://github.com/MISP/misp-modules.git cd misp-modules sudo -u www-data /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS sudo -u www-data /var/www/MISP/venv/bin/pip install . -sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local +# Start misp-modules as a service +sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now misp-modules /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules ~~~~ @@ -110,7 +113,10 @@ sudo git clone https://github.com/MISP/misp-modules.git cd misp-modules sudo pip3 install -I -r REQUIREMENTS sudo pip3 install -I . -sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local +# Start misp-modules as a service +sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now misp-modules /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules ~~~~ @@ -490,7 +496,7 @@ Download a pre-built virtual image from the [MISP training materials](https://ww - Create a Host-Only adapter in VirtualBox - Set your Misp OVA to that Host-Only adapter - Start the virtual machine -- Get the IP address of the virutal machine +- Get the IP address of the virtual machine - SSH into the machine (Login info on training page) - Go into the misp-modules directory @@ -510,14 +516,16 @@ Remove the contents of the build directory and re-install misp-modules. ~~~bash sudo rm -fr build/* -sudo pip3 install --upgrade . +sudo -u www-data /var/www/MISP/venv/bin/pip install --upgrade . ~~~ SSH in with a different terminal and run `misp-modules` with debugging enabled. ~~~bash -sudo killall misp-modules -misp-modules -d +# In case misp-modules is not a service do: +# sudo killall misp-modules +sudo systemctl disable --now misp-modules +sudo -u www-data /var/www/MISP/venv/bin/misp-modules -d ~~~ From 7171c8ce92432e7850b8f5d6a51420c3543a36b0 Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Wed, 24 Apr 2019 13:54:21 +0200 Subject: [PATCH 2/7] initial version of OCR expansion module --- misp_modules/modules/expansion/__init__.py | 3 +- misp_modules/modules/expansion/ocr.py | 51 ++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 misp_modules/modules/expansion/ocr.py diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index 994b289..ec78e9b 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -8,4 +8,5 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io', - 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors', 'qrcode'] + 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors', + 'qrcode', 'ocr'] diff --git a/misp_modules/modules/expansion/ocr.py b/misp_modules/modules/expansion/ocr.py new file mode 100644 index 0000000..afdf343 --- /dev/null +++ b/misp_modules/modules/expansion/ocr.py @@ -0,0 +1,51 @@ +import json +import re +import binascii +import cv2 +import np +import pytesseract + +misperrors = {'error': 'Error'} +mispattributes = {'input': ['attachment'], + 'output': ['freetext', 'text']} +moduleinfo = {'version': '0.1', 'author': 'Sascha Rommelfangen', + 'description': 'OCR decoder', + 'module-type': ['expansion']} + +moduleconfig = [] + + +def handler(q=False): + if q is False: + return False + q = json.loads(q) + filename = q['attachment'] + try: + img_array = np.frombuffer(binascii.a2b_base64(q['data']), np.uint8) + except Exception as e: + print(e) + err = "Couldn't fetch attachment (JSON 'data' is empty). Are you using the 'Query enrichment' action?" + misperrors['error'] = err + print(err) + return misperrors + + image = img_array + image = cv2.imdecode(img_array, cv2.IMREAD_COLOR) + try: + decoded = pytesseract.image_to_string(image) + return {'results': [{'types': ['freetext'], 'values': decoded, 'comment': "OCR from file " + filename}, + {'types': ['text'], 'values': decoded, 'comment': "ORC from file " + filename}]} + except Exception as e: + print(e) + err = "Couldn't analyze file type. Only images are supported right now." + misperrors['error'] = err + return misperrors + + +def introspection(): + return mispattributes + + +def version(): + moduleinfo['config'] = moduleconfig + return moduleinfo From 614fc1354b49bf281b746d3c6ca0bbbffcc21bff Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Wed, 24 Apr 2019 14:01:08 +0200 Subject: [PATCH 3/7] chg: [ocr] re module not used - removed --- misp_modules/modules/expansion/ocr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/misp_modules/modules/expansion/ocr.py b/misp_modules/modules/expansion/ocr.py index afdf343..cd6baca 100644 --- a/misp_modules/modules/expansion/ocr.py +++ b/misp_modules/modules/expansion/ocr.py @@ -1,5 +1,4 @@ import json -import re import binascii import cv2 import np From 81b0082ae5cb2a42303c4629dd53af954713cb25 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Wed, 24 Apr 2019 14:01:48 +0200 Subject: [PATCH 4/7] chg: [init] removed trailing whitespace --- misp_modules/modules/expansion/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index ec78e9b..03a7899 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -8,5 +8,5 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io', - 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors', + 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors', 'qrcode', 'ocr'] From 5104bce4519d994aa868f9b576b803e393ca1ce7 Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Wed, 24 Apr 2019 14:53:03 +0200 Subject: [PATCH 5/7] renamed module --- misp_modules/modules/expansion/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_modules/modules/expansion/__init__.py b/misp_modules/modules/expansion/__init__.py index ec78e9b..38d59ad 100644 --- a/misp_modules/modules/expansion/__init__.py +++ b/misp_modules/modules/expansion/__init__.py @@ -9,4 +9,4 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries', 'dbl_spamhaus', 'vulners', 'yara_query', 'macaddress_io', 'intel471', 'backscatter_io', 'btc_scam_check', 'hibp', 'greynoise', 'macvendors', - 'qrcode', 'ocr'] + 'qrcode', 'ocr-enrich'] From 07f759b07a5994e2fea4410a4a230e2304159009 Mon Sep 17 00:00:00 2001 From: Sascha Rommelfangen Date: Wed, 24 Apr 2019 14:53:16 +0200 Subject: [PATCH 6/7] renamed file --- misp_modules/modules/expansion/{ocr.py => ocr-enrich.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename misp_modules/modules/expansion/{ocr.py => ocr-enrich.py} (100%) diff --git a/misp_modules/modules/expansion/ocr.py b/misp_modules/modules/expansion/ocr-enrich.py similarity index 100% rename from misp_modules/modules/expansion/ocr.py rename to misp_modules/modules/expansion/ocr-enrich.py From 29e57dfcc6afa59b3d68d5ccb3c65ee649001a09 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 25 Apr 2019 17:36:32 +0900 Subject: [PATCH 7/7] chg: [doc] Added new dependencies and updated RHEL/CentOS howto. --- README.md | 39 ++++++++++++++------------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 169151e..64d0960 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/ ## How to install and start MISP modules in a Python virtualenv? (recommended) ~~~~bash -sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick virtualenv +sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick virtualenv libopencv-dev zbar-tools sudo -u www-data virtualenv -p python3 /var/www/MISP/venv cd /usr/local/src/ sudo git clone https://github.com/MISP/misp-modules.git @@ -107,36 +107,24 @@ sudo systemctl enable --now misp-modules /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules ~~~~ -## How to install and start MISP modules on Debian-based distributions ? - -~~~~bash -sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick -cd /usr/local/src/ -sudo git clone https://github.com/MISP/misp-modules.git -cd misp-modules -sudo pip3 install -I -r REQUIREMENTS -sudo pip3 install -I . -# Start misp-modules as a service -sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/ -sudo systemctl daemon-reload -sudo systemctl enable --now misp-modules -/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules -~~~~ - ## How to install and start MISP modules on RHEL-based distributions ? As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the [SCL](https://access.redhat.com/documentation/en-us/red_hat_software_collections/3/html/3.2_release_notes/chap-installation#sect-Installation-Subscribe) repository. + ~~~~bash -yum install rh-ruby22 +sudo yum install rh-ruby22 +sudo yum install openjpeg-devel +sudo yum install rubygem-rouge rubygem-asciidoctor zbar-devel opencv-devel cd /var/www/MISP git clone https://github.com/MISP/misp-modules.git cd misp-modules -scl enable rh-python36 ‘python3 –m pip install cryptography’ -scl enable rh-python36 ‘python3 –m pip install -I -r REQUIREMENTS’ -scl enable rh-python36 ‘python3 –m pip install –I .’ +sudo -u apache /usr/bin/scl enable rh-python36 "virtualenv -p python3 /var/www/MISP/venv" +sudo -u apache /var/www/MISP/venv/bin/pip install -U -I -r REQUIREMENTS +sudo -u apache /var/www/MISP/venv/bin/pip install -U . ~~~~ + Create the service file /etc/systemd/system/misp-modules.service : ~~~~ -[Unit] +echo "[Unit] Description=MISP's modules After=misp-workers.service @@ -144,15 +132,16 @@ After=misp-workers.service Type=simple User=apache Group=apache -ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 ‘/opt/rh/rh-python36/root/bin/misp-modules –l 127.0.0.1 –s’ +ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 '/opt/rh/rh-python36/root/bin/misp-modules –l 127.0.0.1 –s' Restart=always RestartSec=10 [Install] -WantedBy=multi-user.target +WantedBy=multi-user.target" | sudo tee /etc/systemd/system/misp-modules.service ~~~~ + The `After=misp-workers.service` must be changed or removed if you have not created a misp-workers service. -Then, enable the misp-modules service and start it ; +Then, enable the misp-modules service and start it: ~~~~bash systemctl daemon-reload systemctl enable --now misp-modules