Merge branch 'master' into dnstrails

pull/208/head
sebdraven 2018-07-18 11:07:44 +02:00 committed by GitHub
commit d7fac002af
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 166 additions and 15 deletions

View File

@ -6,11 +6,11 @@ services:
cache: pip
python:
- "3.4"
- "3.5"
- "3.5-dev"
- "3.6"
- "3.6-dev"
- "3.7-dev"
install:
- pip install -U nose codecov pytest

View File

@ -38,8 +38,10 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
* [rbl](misp_modules/modules/expansion/rbl.py) - a module to get RBL (Real-Time Blackhost List) values from an attribute.
* [reversedns](misp_modules/modules/expansion/reversedns.py) - Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes.
* [shodan](misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module.
* [Sigma queries](misp_modules/modules/expansion/sigma_queries.py) - Experimental expansion module querying a sigma rule to convert it into all the available SIEM signatures.
* [Sigma syntax validator](misp_modules/modules/expansion/sigma_syntax_validator.py) - Sigma syntax validator.
* [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.
* [STIX2 pattern syntax validator](misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) - a module to check a STIX2 pattern syntax.
* [ThreatCrowd](misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/).
* [threatminer](misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/).
* [virustotal](misp_modules/modules/expansion/virustotal.py) - an expansion module to pull known resolutions and malware samples related with an IP/Domain from virusTotal (this modules require a VirusTotal private API key)
@ -380,7 +382,7 @@ Recommended Plugin.Import_ocr_enabled true Enable or disable the ocr
In this same menu set any other plugin settings that are required for testing.
## Install misp-module on an offline instance.
First, you need to grab all necessery packages for example like this :
First, you need to grab all necessary packages for example like this :
Use pip wheel to create an archive
~~~

View File

@ -15,6 +15,7 @@ git+https://github.com/sebdraven/pyonyphe#egg=pyonyphe
git+https://github.com/sebdraven/pydnstrails#egg=pydnstrails
pillow
pytesseract
wand
SPARQLWrapper
domaintools_api
pygeoip
@ -22,3 +23,4 @@ bs4
oauth2
yara
sigmatools
stix2-patterns

View File

@ -193,7 +193,7 @@ class QueryModule(tornado.web.RequestHandler):
if dict_payload.get('timeout'):
timeout = datetime.timedelta(seconds=int(dict_payload.get('timeout')))
else:
timeout = datetime.timedelta(seconds=30)
timeout = datetime.timedelta(seconds=300)
response = yield tornado.gen.with_timeout(timeout, self.run_request(jsonpayload))
self.write(response)
except tornado.gen.TimeoutError:

View File

@ -1,3 +1,3 @@
from . import _vmray
__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator']
__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries']

View File

@ -0,0 +1,50 @@
import sys, os, io, json
try:
from sigma.parser import SigmaCollectionParser
from sigma.config import SigmaConfiguration
from sigma.backends import getBackend, BackendOptions
except ModuleNotFoundError:
print("sigma or yaml is missing, use 'pip3 install sigmatools' to install it.")
misperrors = {'error': 'Error'}
mispattributes = {'input': ['sigma'], 'output': ['text']}
moduleinfo = {'version': '0.1', 'author': 'Christian Studer', 'module-type': ['expansion', 'hover'],
'description': 'An expansion hover module to display the result of sigma queries.'}
moduleconfig = []
sigma_targets = ('es-dsl', 'es-qs', 'graylog', 'kibana', 'xpack-watcher', 'logpoint', 'splunk', 'grep', 'wdatp', 'splunkxml', 'arcsight', 'qualys')
def handler(q=False):
if q is False:
return False
request = json.loads(q)
if not request.get('sigma'):
misperrors['error'] = 'Sigma rule missing'
return misperrors
config = SigmaConfiguration()
backend_options = BackendOptions(None)
f = io.TextIOWrapper(io.BytesIO(request.get('sigma').encode()), encoding='utf-8')
parser = SigmaCollectionParser(f, config, None)
targets = []
old_stdout = sys.stdout
result = io.StringIO()
sys.stdout = result
for t in sigma_targets:
backend = getBackend(t)(config, backend_options, None)
try:
parser.generate(backend)
backend.finalize()
print("#NEXT")
targets.append(t)
except:
continue
sys.stdout = old_stdout
results = result.getvalue()[:-5].split('#NEXT')
d_result = {t: r.strip() for t,r in zip(targets, results)}
return {'results': [{'types': mispattributes['output'], 'values': d_result}]}
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

View File

@ -0,0 +1,42 @@
import json
try:
from stix2patterns.validator import run_validator
except ModuleNotFoundError:
print("stix2 patterns python library is missing, use 'pip3 install stix2-patterns' to install it.")
misperrors = {'error': 'Error'}
mispattributes = {'input': ['stix2-pattern'], 'output': ['text']}
moduleinfo = {'version': '0.1', 'author': 'Christian Studer', 'module-type': ['expansion', 'hover'],
'description': 'An expansion hover module to perform a syntax check on stix2 patterns.'}
moduleconfig = []
def handler(q=False):
if q is False:
return False
request = json.loads(q)
if not request.get('stix2-pattern'):
misperrors['error'] = 'STIX2 pattern missing'
return misperrors
pattern = request.get('stix2-pattern')
syntax_errors = []
for p in pattern[2:-2].split(' AND '):
syntax_validator = run_validator("[{}]".format(p))
if syntax_validator:
for error in syntax_validator:
syntax_errors.append(error)
if syntax_errors:
s = 's' if len(syntax_errors) > 1 else ''
s_errors = ""
for error in syntax_errors:
s_errors += "{}\n".format(error[6:])
result = "Syntax error{}: \n{}".format(s, s_errors[:-1])
else:
result = "Syntax valid"
return {'results': [{'types': mispattributes['output'], 'values': result}]}
def introspection():
return mispattributes
def version():
moduleinfo['config'] = moduleconfig
return moduleinfo

View File

@ -1 +1 @@
__all__ = ['testexport','cef_export','liteexport','goamlexport','threat_connect_export','pdfexport','threatStream_misp_export']
__all__ = ['cef_export','liteexport','goamlexport','threat_connect_export','pdfexport','threatStream_misp_export']

View File

@ -1,3 +1,3 @@
from . import _vmray
__all__ = ['vmray_import', 'testimport', 'ocr', 'cuckooimport', 'goamlimport', 'email_import', 'mispjson', 'openiocimport', 'threatanalyzer_import', 'csvimport']
__all__ = ['vmray_import', 'ocr', 'cuckooimport', 'goamlimport', 'email_import', 'mispjson', 'openiocimport', 'threatanalyzer_import', 'csvimport']

View File

@ -1,16 +1,24 @@
import sys
import json
import base64
from PIL import Image
from pytesseract import image_to_string
from io import BytesIO
import logging
log = logging.getLogger('ocr')
log.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
log.addHandler(ch)
misperrors = {'error': 'Error'}
userConfig = {};
inputSource = ['file']
moduleinfo = {'version': '0.1', 'author': 'Alexandre Dulaunoy',
moduleinfo = {'version': '0.2', 'author': 'Alexandre Dulaunoy',
'description': 'Optical Character Recognition (OCR) module for MISP',
'module-type': ['import']}
@ -18,14 +26,61 @@ moduleconfig = []
def handler(q=False):
# try to import modules and return errors if module not found
try:
from PIL import Image
except ImportError:
misperrors['error'] = "Please pip(3) install pillow"
return misperrors
try:
# Official ImageMagick module
from wand.image import Image as WImage
except ImportError:
misperrors['error'] = "Please pip(3) install wand"
return misperrors
try:
from pytesseract import image_to_string
except ImportError:
misperrors['error'] = "Please pip(3) install pytesseract"
return misperrors
if q is False:
return False
r = {'results': []}
request = json.loads(q)
image = base64.b64decode(request["data"])
document = base64.b64decode(request["data"])
document = WImage(blob=document)
if document.format == 'PDF':
with document as pdf:
# Get number of pages
pages=len(pdf.sequence)
log.debug("PDF with {} page(s) detected".format(pages))
# Create new image object where the height will be the number of pages. With huge PDFs this will overflow, break, consume silly memory etc…
img = WImage(width=pdf.width, height=pdf.height * pages)
# Cycle through pages and stitch it together to one big file
for p in range(pages):
log.debug("Stitching page {}".format(p+1))
image = img.composite(pdf.sequence[p], top=pdf.height * p, left=0)
# Create a png blob
image = img.make_blob('png')
log.debug("Final image size is {}x{}".format(pdf.width, pdf.height*(p+1)))
else:
image = document
image_file = BytesIO(image)
image_file.seek(0)
ocrized = image_to_string(Image.open(image_file))
try:
im = Image.open(image_file)
except IOError:
misperrors['error'] = "Corrupt or not an image file."
return misperrors
ocrized = image_to_string(im)
freetext = {}
freetext['values'] = ocrized
freetext['types'] = ['freetext']

View File

@ -69,7 +69,7 @@ def handler(q=False):
results.append({
'values': current_sample_filename,
'data': base64.b64encode(file_data).decode(),
'type': 'malware-sample', 'categories': ['Artifacts dropped', 'Payload delivery'], 'to_ids': True, 'comment': ''})
'type': 'malware-sample', 'categories': ['Payload delivery', 'Artifacts dropped'], 'to_ids': True, 'comment': ''})
if 'Analysis/analysis.json' in zip_file_name:
with zf.open(zip_file_name, mode='r', pwd=None) as fp: