mirror of https://github.com/MISP/misp-modules
Merge branch 'master' of github.com:MISP/misp-modules into new_module
commit
1e27c2de5a
10
.travis.yml
10
.travis.yml
|
@ -9,14 +9,22 @@ python:
|
||||||
- "3.6"
|
- "3.6"
|
||||||
- "3.6-dev"
|
- "3.6-dev"
|
||||||
- "3.7-dev"
|
- "3.7-dev"
|
||||||
|
- "3.8-dev"
|
||||||
|
|
||||||
before_install:
|
before_install:
|
||||||
- docker build -t misp-modules --build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") docker/
|
- docker build -t misp-modules --build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") docker/
|
||||||
|
|
||||||
install:
|
install:
|
||||||
- sudo apt-get install libzbar0 libzbar-dev libpoppler-cpp-dev tesseract-ocr libfuzzy-dev
|
- sudo apt-get install libzbar0 libzbar-dev libpoppler-cpp-dev tesseract-ocr libfuzzy-dev libcaca-dev liblua5.3-dev
|
||||||
- pip install pipenv
|
- pip install pipenv
|
||||||
- pipenv install --dev
|
- pipenv install --dev
|
||||||
|
# install gtcaca
|
||||||
|
- git clone git://github.com/stricaud/gtcaca.git
|
||||||
|
- mkdir -p gtcaca/build
|
||||||
|
- pushd gtcaca/build
|
||||||
|
- cmake .. && make
|
||||||
|
- sudo make install
|
||||||
|
- popd
|
||||||
# install pyfaup
|
# install pyfaup
|
||||||
- git clone https://github.com/stricaud/faup.git
|
- git clone https://github.com/stricaud/faup.git
|
||||||
- pushd faup/build
|
- pushd faup/build
|
||||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -26,6 +26,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj
|
||||||
* [RansomcoinDB check](misp_modules/modules/expansion/ransomcoindb.py) - An expansion hover module to query the [ransomcoinDB](https://ransomcoindb.concinnity-risks.com): it contains mapping between BTC addresses and malware hashes. Enrich MISP by querying for BTC -> hash or hash -> BTC addresses.
|
* [RansomcoinDB check](misp_modules/modules/expansion/ransomcoindb.py) - An expansion hover module to query the [ransomcoinDB](https://ransomcoindb.concinnity-risks.com): it contains mapping between BTC addresses and malware hashes. Enrich MISP by querying for BTC -> hash or hash -> BTC addresses.
|
||||||
* [BTC scam check](misp_modules/modules/expansion/btc_scam_check.py) - An expansion hover module to instantly check if a BTC address has been abused.
|
* [BTC scam check](misp_modules/modules/expansion/btc_scam_check.py) - An expansion hover module to instantly check if a BTC address has been abused.
|
||||||
* [BTC transactions](misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP.
|
* [BTC transactions](misp_modules/modules/expansion/btc_steroids.py) - An expansion hover module to get a blockchain balance and the transactions from a BTC address in MISP.
|
||||||
|
* [Censys-enrich](misp_modules/modules/expansion/censys_enrich.py) - An expansion and module to retrieve information from censys.io about a particular IP or certificate.
|
||||||
* [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
|
* [CIRCL Passive DNS](misp_modules/modules/expansion/circl_passivedns.py) - a hover and expansion module to expand hostname and IP addresses with passive DNS information.
|
||||||
* [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate(s) seen.
|
* [CIRCL Passive SSL](misp_modules/modules/expansion/circl_passivessl.py) - a hover and expansion module to expand IP addresses with the X.509 certificate(s) seen.
|
||||||
* [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to.
|
* [countrycode](misp_modules/modules/expansion/countrycode.py) - a hover module to tell you what country a URL belongs to.
|
||||||
|
|
102
REQUIREMENTS
102
REQUIREMENTS
|
@ -3,29 +3,31 @@
|
||||||
-e git+https://github.com/D4-project/BGP-Ranking.git/@fd9c0e03af9b61d4bf0b67ac73c7208a55178a54#egg=pybgpranking&subdirectory=client
|
-e git+https://github.com/D4-project/BGP-Ranking.git/@fd9c0e03af9b61d4bf0b67ac73c7208a55178a54#egg=pybgpranking&subdirectory=client
|
||||||
-e git+https://github.com/D4-project/IPASN-History.git/@fc5e48608afc113e101ca6421bf693b7b9753f9e#egg=pyipasnhistory&subdirectory=client
|
-e git+https://github.com/D4-project/IPASN-History.git/@fc5e48608afc113e101ca6421bf693b7b9753f9e#egg=pyipasnhistory&subdirectory=client
|
||||||
-e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471
|
-e git+https://github.com/MISP/PyIntel471.git@0df8d51f1c1425de66714b3a5a45edb69b8cc2fc#egg=pyintel471
|
||||||
-e git+https://github.com/MISP/PyMISP.git@a26a8e450b14d48bb0c8ef46b32bff2f1eadc514#egg=pymisp[fileobjects,openioc,virustotal,pdfexport]
|
-e git+https://github.com/MISP/PyMISP.git@b5b40ae2c5225a4b349c26294cfc012309a61352#egg=pymisp[fileobjects,openioc,virustotal,pdfexport]
|
||||||
-e git+https://github.com/Rafiot/uwhoisd.git@411572840eba4c72dc321c549b36a54ed5cea9de#egg=uwhois&subdirectory=client
|
-e git+https://github.com/Rafiot/uwhoisd.git@411572840eba4c72dc321c549b36a54ed5cea9de#egg=uwhois&subdirectory=client
|
||||||
-e git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader
|
-e git+https://github.com/cartertemm/ODTReader.git/@49d6938693f6faa3ff09998f86dba551ae3a996b#egg=odtreader
|
||||||
-e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
|
-e git+https://github.com/sebdraven/pydnstrails@48c1f740025c51289f43a24863d1845ff12fd21a#egg=pydnstrails
|
||||||
-e git+https://github.com/sebdraven/pyonyphe@cbb0168d5cb28a9f71f7ab3773164a7039ccdb12#egg=pyonyphe
|
-e git+https://github.com/sebdraven/pyonyphe@1ce15581beebb13e841193a08a2eb6f967855fcb#egg=pyonyphe
|
||||||
|
-e git+https://github.com/stricaud/faup.git#egg=pyfaup&subdirectory=src/lib/bindings/python
|
||||||
aiohttp==3.4.4
|
aiohttp==3.4.4
|
||||||
antlr4-python3-runtime==4.7.2 ; python_version >= '3'
|
antlr4-python3-runtime==4.8 ; python_version >= '3'
|
||||||
apiosintds==1.8.3
|
apiosintds==1.8.3
|
||||||
argparse==1.4.0
|
argparse==1.4.0
|
||||||
assemblyline-client==3.7.3
|
assemblyline-client==3.7.3
|
||||||
async-timeout==3.0.1
|
async-timeout==3.0.1
|
||||||
attrs==19.3.0
|
attrs==19.3.0
|
||||||
backscatter==0.2.4
|
backscatter==0.2.4
|
||||||
beautifulsoup4==4.8.1
|
beautifulsoup4==4.8.2
|
||||||
blockchain==1.4.4
|
blockchain==1.4.4
|
||||||
|
censys==0.0.8
|
||||||
certifi==2019.11.28
|
certifi==2019.11.28
|
||||||
cffi==1.13.2
|
cffi==1.14.0
|
||||||
chardet==3.0.4
|
chardet==3.0.4
|
||||||
click-plugins==1.1.1
|
click-plugins==1.1.1
|
||||||
click==7.0
|
click==7.1.1
|
||||||
colorama==0.4.3
|
colorama==0.4.3
|
||||||
cryptography==2.8
|
cryptography==2.8
|
||||||
decorator==4.4.1
|
decorator==4.4.2
|
||||||
deprecated==1.2.7
|
deprecated==1.2.7
|
||||||
dnspython==1.16.0
|
dnspython==1.16.0
|
||||||
domaintools-api==0.3.3
|
domaintools-api==0.3.3
|
||||||
|
@ -33,77 +35,77 @@ enum-compat==0.0.3
|
||||||
ez-setup==0.9
|
ez-setup==0.9
|
||||||
ezodf==0.3.2
|
ezodf==0.3.2
|
||||||
future==0.18.2
|
future==0.18.2
|
||||||
geoip2==2.9.0
|
futures==3.1.1
|
||||||
httplib2==0.14.0
|
geoip2==3.0.0
|
||||||
|
httplib2==0.17.0
|
||||||
idna-ssl==1.1.0 ; python_version < '3.7'
|
idna-ssl==1.1.0 ; python_version < '3.7'
|
||||||
idna==2.8
|
idna==2.9
|
||||||
importlib-metadata==1.3.0 ; python_version < '3.8'
|
importlib-metadata==1.6.0 ; python_version < '3.8'
|
||||||
isodate==0.6.0
|
isodate==0.6.0
|
||||||
jbxapi==3.4.0
|
jbxapi==3.4.0
|
||||||
jsonschema==3.2.0
|
jsonschema==3.2.0
|
||||||
lief==0.10.1
|
lief==0.10.1
|
||||||
lxml==4.4.2
|
lxml==4.5.0
|
||||||
maclookup==1.0.3
|
maclookup==1.0.3
|
||||||
maxminddb==1.5.1
|
maxminddb==1.5.2
|
||||||
more-itertools==8.0.2
|
multidict==4.7.5
|
||||||
multidict==4.7.1
|
|
||||||
np==1.0.2
|
np==1.0.2
|
||||||
numpy==1.17.4
|
numpy==1.18.2
|
||||||
oauth2==1.9.0.post1
|
oauth2==1.9.0.post1
|
||||||
opencv-python==4.1.2.30
|
opencv-python==4.2.0.32
|
||||||
pandas-ods-reader==0.0.7
|
pandas-ods-reader==0.0.7
|
||||||
pandas==0.25.3
|
pandas==1.0.3
|
||||||
passivetotal==1.0.31
|
passivetotal==1.0.31
|
||||||
pdftotext==2.1.2
|
pdftotext==2.1.4
|
||||||
pillow==6.2.1
|
pillow==7.0.0
|
||||||
progressbar2==3.47.0
|
progressbar2==3.50.1
|
||||||
psutil==5.6.7
|
psutil==5.7.0
|
||||||
pycparser==2.19
|
pycparser==2.20
|
||||||
pycryptodome==3.9.4
|
pycryptodome==3.9.7
|
||||||
pycryptodomex==3.9.4
|
pycryptodomex==3.9.7
|
||||||
pydeep==0.4
|
pydeep==0.4
|
||||||
pyeupi==1.0
|
pyeupi==1.0
|
||||||
pygeoip==0.3.2
|
pygeoip==0.3.2
|
||||||
pyopenssl==19.1.0
|
pyopenssl==19.1.0
|
||||||
pyparsing==2.4.5
|
pyparsing==2.4.6
|
||||||
pypdns==1.4.1
|
pypdns==1.5.1
|
||||||
pypssl==2.1
|
pypssl==2.1
|
||||||
pyrsistent==0.15.6
|
pyrsistent==0.16.0
|
||||||
pytesseract==0.3.0
|
pytesseract==0.3.3
|
||||||
python-dateutil==2.8.1
|
python-dateutil==2.8.1
|
||||||
python-docx==0.8.10
|
python-docx==0.8.10
|
||||||
python-magic==0.4.15
|
python-magic==0.4.15
|
||||||
python-pptx==0.6.18
|
python-pptx==0.6.18
|
||||||
python-utils==2.3.0
|
python-utils==2.4.0
|
||||||
pytz==2019.3
|
pytz==2019.3
|
||||||
pyyaml==5.2
|
pyyaml==5.3.1
|
||||||
pyzbar==0.1.8
|
pyzbar==0.1.8
|
||||||
pyzipper==0.3.1 ; python_version >= '3.5'
|
pyzipper==0.3.1 ; python_version >= '3.5'
|
||||||
rdflib==4.2.2
|
rdflib==4.2.2
|
||||||
redis==3.3.11
|
redis==3.4.1
|
||||||
reportlab==3.5.32
|
reportlab==3.5.42
|
||||||
requests-cache==0.5.2
|
requests-cache==0.5.2
|
||||||
requests[security]==2.22.0
|
requests[security]==2.23.0
|
||||||
shodan==1.21.0
|
shodan==1.22.0
|
||||||
sigmatools==0.15.0
|
sigmatools==0.16.0
|
||||||
six==1.13.0
|
six==1.14.0
|
||||||
socketio-client==0.5.6
|
socketio-client==0.5.6
|
||||||
soupsieve==1.9.5
|
soupsieve==2.0
|
||||||
sparqlwrapper==1.8.4
|
sparqlwrapper==1.8.5
|
||||||
stix2-patterns==1.2.1
|
stix2-patterns==1.3.0
|
||||||
tabulate==0.8.6
|
tabulate==0.8.7
|
||||||
tornado==6.0.3
|
tornado==6.0.4
|
||||||
url-normalize==1.4.1
|
url-normalize==1.4.1
|
||||||
urlarchiver==0.2
|
urlarchiver==0.2
|
||||||
urllib3==1.25.7
|
urllib3==1.25.8
|
||||||
validators==0.14.0
|
validators==0.14.0
|
||||||
vulners==1.5.4
|
vt-graph-api==1.0.1
|
||||||
wand==0.5.8
|
vulners==1.5.5
|
||||||
websocket-client==0.56.0
|
wand==0.5.9
|
||||||
wrapt==1.11.2
|
websocket-client==0.57.0
|
||||||
|
wrapt==1.12.1
|
||||||
xlrd==1.2.0
|
xlrd==1.2.0
|
||||||
xlsxwriter==1.2.6
|
xlsxwriter==1.2.8
|
||||||
yara-python==3.8.1
|
yara-python==3.8.1
|
||||||
yarl==1.4.2
|
yarl==1.4.2
|
||||||
zipp==0.6.0
|
zipp==3.1.0
|
||||||
vt-graph-api
|
|
||||||
|
|
|
@ -152,6 +152,22 @@ An expansion hover module to get a blockchain balance from a BTC address in MISP
|
||||||
|
|
||||||
-----
|
-----
|
||||||
|
|
||||||
|
#### [censys_enrich](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/censys_enrich.py)
|
||||||
|
|
||||||
|
An expansion module to enrich attributes in MISP by quering the censys.io API
|
||||||
|
- **features**:
|
||||||
|
>This module takes an IP, hostname or a certificate fingerprint and attempts to enrich it by querying the Censys API.
|
||||||
|
- **input**:
|
||||||
|
>IP, domain or certificate fingerprint (md5, sha1 or sha256)
|
||||||
|
- **output**:
|
||||||
|
>MISP objects retrieved from censys, including open ports, ASN, Location of the IP, x509 details
|
||||||
|
- **references**:
|
||||||
|
>https://www.censys.io
|
||||||
|
- **requirements**:
|
||||||
|
>API credentials to censys.io
|
||||||
|
|
||||||
|
-----
|
||||||
|
|
||||||
#### [circl_passivedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivedns.py)
|
#### [circl_passivedns](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/circl_passivedns.py)
|
||||||
|
|
||||||
<img src=logos/passivedns.png height=60>
|
<img src=logos/passivedns.png height=60>
|
||||||
|
@ -295,7 +311,7 @@ An expansion hover module to expand information about CVE id.
|
||||||
|
|
||||||
-----
|
-----
|
||||||
|
|
||||||
#### [cytomic_orion.py](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cytomic_orion.py.py)
|
#### [cytomic_orion](https://github.com/MISP/misp-modules/tree/master/misp_modules/modules/expansion/cytomic_orion.py)
|
||||||
|
|
||||||
<img src=logos/cytomic_orion.png height=60>
|
<img src=logos/cytomic_orion.png height=60>
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,8 @@
|
||||||
|
{
|
||||||
|
"description": "An expansion module to enrich attributes in MISP by quering the censys.io API",
|
||||||
|
"requirements": ["API credentials to censys.io"],
|
||||||
|
"input": "IP, domain or certificate fingerprint (md5, sha1 or sha256)",
|
||||||
|
"output": "MISP objects retrieved from censys, including open ports, ASN, Location of the IP, x509 details",
|
||||||
|
"references": ["https://www.censys.io"],
|
||||||
|
"features": "This module takes an IP, hostname or a certificate fingerprint and attempts to enrich it by querying the Censys API."
|
||||||
|
}
|
|
@ -21,8 +21,28 @@ $SUDO_WWW virtualenv -p python3 /var/www/MISP/venv
|
||||||
# END with virtualenv
|
# END with virtualenv
|
||||||
|
|
||||||
cd /usr/local/src/
|
cd /usr/local/src/
|
||||||
sudo git clone https://github.com/MISP/misp-modules.git
|
# Ideally you add your user to the staff group and make /usr/local/src group writeable, below follows an example with user misp
|
||||||
cd misp-modules
|
sudo adduser misp staff
|
||||||
|
sudo chmod 2775 /usr/local/src
|
||||||
|
sudo chown root:staff /usr/local/src
|
||||||
|
git clone https://github.com/MISP/misp-modules.git
|
||||||
|
git clone git://github.com/stricaud/faup.git faup
|
||||||
|
git clone git://github.com/stricaud/gtcaca.git gtcaca
|
||||||
|
|
||||||
|
# Install gtcaca/faup
|
||||||
|
cd gtcaca
|
||||||
|
mkdir -p build
|
||||||
|
cd build
|
||||||
|
cmake .. && make
|
||||||
|
sudo make install
|
||||||
|
cd ../../faup
|
||||||
|
mkdir -p build
|
||||||
|
cd build
|
||||||
|
cmake .. && make
|
||||||
|
sudo make install
|
||||||
|
sudo ldconfig
|
||||||
|
|
||||||
|
cd ../../misp-modules
|
||||||
|
|
||||||
# BEGIN with virtualenv:
|
# BEGIN with virtualenv:
|
||||||
$SUDO_WWW /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS
|
$SUDO_WWW /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS
|
||||||
|
@ -168,4 +188,4 @@ tar xvf misp-module-bundeled.tar.bz2 -C misp-modules-bundle
|
||||||
cd misp-modules-bundle
|
cd misp-modules-bundle
|
||||||
ls -1|while read line; do sudo pip3 install --force-reinstall --ignore-installed --upgrade --no-index --no-deps ${line};done
|
ls -1|while read line; do sudo pip3 install --force-reinstall --ignore-installed --upgrade --no-index --no-deps ${line};done
|
||||||
~~~
|
~~~
|
||||||
Next you can follow standard install procedure.
|
Next you can follow standard install procedure.
|
||||||
|
|
|
@ -16,4 +16,4 @@ __all__ = ['cuckoo_submit', 'vmray_submit', 'bgpranking', 'circl_passivedns', 'c
|
||||||
'ods_enrich', 'odt_enrich', 'joesandbox_submit', 'joesandbox_query', 'urlhaus',
|
'ods_enrich', 'odt_enrich', 'joesandbox_submit', 'joesandbox_query', 'urlhaus',
|
||||||
'virustotal_public', 'apiosintds', 'urlscan', 'securitytrails', 'apivoid',
|
'virustotal_public', 'apiosintds', 'urlscan', 'securitytrails', 'apivoid',
|
||||||
'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', 'malwarebazaar',
|
'assemblyline_submit', 'assemblyline_query', 'ransomcoindb', 'malwarebazaar',
|
||||||
'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion']
|
'lastline_query', 'lastline_submit', 'sophoslabs_intelix', 'cytomic_orion', 'censys_enrich']
|
||||||
|
|
|
@ -0,0 +1,255 @@
|
||||||
|
# encoding: utf-8
|
||||||
|
import json
|
||||||
|
import base64
|
||||||
|
import codecs
|
||||||
|
from dateutil.parser import isoparse
|
||||||
|
from pymisp import MISPAttribute, MISPEvent, MISPObject
|
||||||
|
try:
|
||||||
|
import censys.base
|
||||||
|
import censys.ipv4
|
||||||
|
import censys.websites
|
||||||
|
import censys.certificates
|
||||||
|
except ImportError:
|
||||||
|
print("Censys module not installed. Try 'pip install censys'")
|
||||||
|
|
||||||
|
misperrors = {'error': 'Error'}
|
||||||
|
moduleconfig = ['api_id', 'api_secret']
|
||||||
|
mispattributes = {'input': ['ip-src', 'ip-dst', 'domain', 'hostname', 'hostname|port', 'domain|ip', 'ip-dst|port', 'ip-src|port',
|
||||||
|
'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256'], 'format': 'misp_standard'}
|
||||||
|
moduleinfo = {'version': '0.1', 'author': 'Loïc Fortemps',
|
||||||
|
'description': 'Censys.io expansion module', 'module-type': ['expansion', 'hover']}
|
||||||
|
|
||||||
|
|
||||||
|
def handler(q=False):
|
||||||
|
if q is False:
|
||||||
|
return False
|
||||||
|
request = json.loads(q)
|
||||||
|
|
||||||
|
if request.get('config'):
|
||||||
|
if (request['config'].get('api_id') is None) or (request['config'].get('api_secret') is None):
|
||||||
|
misperrors['error'] = "Censys API credentials are missing"
|
||||||
|
return misperrors
|
||||||
|
else:
|
||||||
|
misperrors['error'] = "Please provide config options"
|
||||||
|
return misperrors
|
||||||
|
|
||||||
|
api_id = request['config']['api_id']
|
||||||
|
api_secret = request['config']['api_secret']
|
||||||
|
|
||||||
|
if not request.get('attribute'):
|
||||||
|
return {'error': 'Unsupported input.'}
|
||||||
|
attribute = request['attribute']
|
||||||
|
if not any(input_type == attribute['type'] for input_type in mispattributes['input']):
|
||||||
|
return {'error': 'Unsupported attributes type'}
|
||||||
|
|
||||||
|
attribute = MISPAttribute()
|
||||||
|
attribute.from_dict(**request['attribute'])
|
||||||
|
# Lists to accomodate multi-types attribute
|
||||||
|
conn = list()
|
||||||
|
types = list()
|
||||||
|
values = list()
|
||||||
|
results = list()
|
||||||
|
|
||||||
|
if "|" in attribute.type:
|
||||||
|
t_1, t_2 = attribute.type.split('|')
|
||||||
|
v_1, v_2 = attribute.value.split('|')
|
||||||
|
# We cannot use the port information
|
||||||
|
if t_2 == "port":
|
||||||
|
types.append(t_1)
|
||||||
|
values.append(v_1)
|
||||||
|
else:
|
||||||
|
types = [t_1, t_2]
|
||||||
|
values = [v_1, v_2]
|
||||||
|
else:
|
||||||
|
types.append(attribute.type)
|
||||||
|
values.append(attribute.value)
|
||||||
|
|
||||||
|
for t in types:
|
||||||
|
# ip, ip-src or ip-dst
|
||||||
|
if t[:2] == "ip":
|
||||||
|
conn.append(censys.ipv4.CensysIPv4(api_id=api_id, api_secret=api_secret))
|
||||||
|
elif t == 'domain' or t == "hostname":
|
||||||
|
conn.append(censys.websites.CensysWebsites(api_id=api_id, api_secret=api_secret))
|
||||||
|
elif 'x509-fingerprint' in t:
|
||||||
|
conn.append(censys.certificates.CensysCertificates(api_id=api_id, api_secret=api_secret))
|
||||||
|
|
||||||
|
found = True
|
||||||
|
for c in conn:
|
||||||
|
val = values.pop(0)
|
||||||
|
try:
|
||||||
|
r = c.view(val)
|
||||||
|
results.append(parse_response(r, attribute))
|
||||||
|
found = True
|
||||||
|
except censys.base.CensysNotFoundException:
|
||||||
|
found = False
|
||||||
|
except Exception:
|
||||||
|
misperrors['error'] = "Connection issue"
|
||||||
|
return misperrors
|
||||||
|
|
||||||
|
if not found:
|
||||||
|
misperrors['error'] = "Nothing could be found on Censys"
|
||||||
|
return misperrors
|
||||||
|
|
||||||
|
return {'results': remove_duplicates(results)}
|
||||||
|
|
||||||
|
|
||||||
|
def parse_response(censys_output, attribute):
|
||||||
|
misp_event = MISPEvent()
|
||||||
|
misp_event.add_attribute(**attribute)
|
||||||
|
# Generic fields (for IP/Websites)
|
||||||
|
if "autonomous_system" in censys_output:
|
||||||
|
cen_as = censys_output['autonomous_system']
|
||||||
|
asn_object = MISPObject('asn')
|
||||||
|
asn_object.add_attribute('asn', value=cen_as["asn"])
|
||||||
|
asn_object.add_attribute('description', value=cen_as['name'])
|
||||||
|
asn_object.add_attribute('subnet-announced', value=cen_as['routed_prefix'])
|
||||||
|
asn_object.add_attribute('country', value=cen_as['country_code'])
|
||||||
|
asn_object.add_reference(attribute.uuid, 'associated-to')
|
||||||
|
misp_event.add_object(**asn_object)
|
||||||
|
|
||||||
|
if "ip" in censys_output and "ports" in censys_output:
|
||||||
|
ip_object = MISPObject('ip-port')
|
||||||
|
ip_object.add_attribute('ip', value=censys_output['ip'])
|
||||||
|
for p in censys_output['ports']:
|
||||||
|
ip_object.add_attribute('dst-port', value=p)
|
||||||
|
ip_object.add_reference(attribute.uuid, 'associated-to')
|
||||||
|
misp_event.add_object(**ip_object)
|
||||||
|
|
||||||
|
# We explore all ports to find https or ssh services
|
||||||
|
for k in censys_output.keys():
|
||||||
|
if not isinstance(censys_output[k], dict):
|
||||||
|
continue
|
||||||
|
if 'https' in censys_output[k]:
|
||||||
|
try:
|
||||||
|
cert = censys_output[k]['https']['tls']['certificate']
|
||||||
|
cert_obj = get_certificate_object(cert, attribute)
|
||||||
|
misp_event.add_object(**cert_obj)
|
||||||
|
except KeyError:
|
||||||
|
print("Error !")
|
||||||
|
if 'ssh' in censys_output[k]:
|
||||||
|
try:
|
||||||
|
cert = censys_output[k]['ssh']['v2']['server_host_key']
|
||||||
|
# TODO enable once the type is merged
|
||||||
|
# misp_event.add_attribute(type='hasshserver-sha256', value=cert['fingerprint_sha256'])
|
||||||
|
except KeyError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Info from certificate query
|
||||||
|
if "parsed" in censys_output:
|
||||||
|
cert_obj = get_certificate_object(censys_output, attribute)
|
||||||
|
misp_event.add_object(**cert_obj)
|
||||||
|
|
||||||
|
# Location can be present for IP/Websites results
|
||||||
|
if "location" in censys_output:
|
||||||
|
loc_obj = MISPObject('geolocation')
|
||||||
|
loc = censys_output['location']
|
||||||
|
loc_obj.add_attribute('latitude', value=loc['latitude'])
|
||||||
|
loc_obj.add_attribute('longitude', value=loc['longitude'])
|
||||||
|
if 'city' in loc:
|
||||||
|
loc_obj.add_attribute('city', value=loc['city'])
|
||||||
|
loc_obj.add_attribute('country', value=loc['country'])
|
||||||
|
if 'postal_code' in loc:
|
||||||
|
loc_obj.add_attribute('zipcode', value=loc['postal_code'])
|
||||||
|
if 'province' in loc:
|
||||||
|
loc_obj.add_attribute('region', value=loc['province'])
|
||||||
|
loc_obj.add_reference(attribute.uuid, 'associated-to')
|
||||||
|
misp_event.add_object(**loc_obj)
|
||||||
|
|
||||||
|
event = json.loads(misp_event.to_json())
|
||||||
|
return {'Object': event['Object'], 'Attribute': event['Attribute']}
|
||||||
|
|
||||||
|
|
||||||
|
# In case of multiple enrichment (ip and domain), we need to filter out similar objects
|
||||||
|
# TODO: make it more granular
|
||||||
|
def remove_duplicates(results):
|
||||||
|
# Only one enrichment was performed so no duplicate
|
||||||
|
if len(results) == 1:
|
||||||
|
return results[0]
|
||||||
|
elif len(results) == 2:
|
||||||
|
final_result = results[0]
|
||||||
|
obj_l2 = results[1]['Object']
|
||||||
|
for o2 in obj_l2:
|
||||||
|
if o2['name'] == "asn":
|
||||||
|
key = "asn"
|
||||||
|
elif o2['name'] == "ip-port":
|
||||||
|
key = "ip"
|
||||||
|
elif o2['name'] == "x509":
|
||||||
|
key = "x509-fingerprint-sha256"
|
||||||
|
elif o2['name'] == "geolocation":
|
||||||
|
key = "latitude"
|
||||||
|
if not check_if_present(o2, key, final_result['Object']):
|
||||||
|
final_result['Object'].append(o2)
|
||||||
|
|
||||||
|
return final_result
|
||||||
|
else:
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def check_if_present(object, attribute_name, list_objects):
|
||||||
|
"""
|
||||||
|
Assert if a given object is present in the list.
|
||||||
|
|
||||||
|
This function check if object (json format) is present in list_objects
|
||||||
|
using attribute_name for the matching
|
||||||
|
"""
|
||||||
|
for o in list_objects:
|
||||||
|
# We first look for a match on the name
|
||||||
|
if o['name'] == object['name']:
|
||||||
|
for attr in object['Attribute']:
|
||||||
|
# Within the attributes, we look for the one to compare
|
||||||
|
if attr['type'] == attribute_name:
|
||||||
|
# Then we check the attributes of the other object and look for a match
|
||||||
|
for attr2 in o['Attribute']:
|
||||||
|
if attr2['type'] == attribute_name and attr2['value'] == attr['value']:
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def get_certificate_object(cert, attribute):
|
||||||
|
parsed = cert['parsed']
|
||||||
|
cert_object = MISPObject('x509')
|
||||||
|
cert_object.add_attribute('x509-fingerprint-sha256', value=parsed['fingerprint_sha256'])
|
||||||
|
cert_object.add_attribute('x509-fingerprint-sha1', value=parsed['fingerprint_sha1'])
|
||||||
|
cert_object.add_attribute('x509-fingerprint-md5', value=parsed['fingerprint_md5'])
|
||||||
|
cert_object.add_attribute('serial-number', value=parsed['serial_number'])
|
||||||
|
cert_object.add_attribute('version', value=parsed['version'])
|
||||||
|
cert_object.add_attribute('subject', value=parsed['subject_dn'])
|
||||||
|
cert_object.add_attribute('issuer', value=parsed['issuer_dn'])
|
||||||
|
cert_object.add_attribute('validity-not-before', value=isoparse(parsed['validity']['start']))
|
||||||
|
cert_object.add_attribute('validity-not-after', value=isoparse(parsed['validity']['end']))
|
||||||
|
cert_object.add_attribute('self_signed', value=parsed['signature']['self_signed'])
|
||||||
|
cert_object.add_attribute('signature_algorithm', value=parsed['signature']['signature_algorithm']['name'])
|
||||||
|
|
||||||
|
cert_object.add_attribute('pubkey-info-algorithm', value=parsed['subject_key_info']['key_algorithm']['name'])
|
||||||
|
|
||||||
|
if 'rsa_public_key' in parsed['subject_key_info']:
|
||||||
|
pub_key = parsed['subject_key_info']['rsa_public_key']
|
||||||
|
cert_object.add_attribute('pubkey-info-size', value=pub_key['length'])
|
||||||
|
cert_object.add_attribute('pubkey-info-exponent', value=pub_key['exponent'])
|
||||||
|
hex_mod = codecs.encode(base64.b64decode(pub_key['modulus']), 'hex').decode()
|
||||||
|
cert_object.add_attribute('pubkey-info-modulus', value=hex_mod)
|
||||||
|
|
||||||
|
if "extensions" in parsed and "subject_alt_name" in parsed["extensions"]:
|
||||||
|
san = parsed["extensions"]["subject_alt_name"]
|
||||||
|
if "dns_names" in san:
|
||||||
|
for dns in san['dns_names']:
|
||||||
|
cert_object.add_attribute('dns_names', value=dns)
|
||||||
|
if "ip_addresses" in san:
|
||||||
|
for ip in san['ip_addresses']:
|
||||||
|
cert_object.add_attribute('ip', value=ip)
|
||||||
|
|
||||||
|
if "raw" in cert:
|
||||||
|
cert_object.add_attribute('raw-base64', value=cert['raw'])
|
||||||
|
|
||||||
|
cert_object.add_reference(attribute.uuid, 'associated-to')
|
||||||
|
return cert_object
|
||||||
|
|
||||||
|
|
||||||
|
def introspection():
|
||||||
|
return mispattributes
|
||||||
|
|
||||||
|
|
||||||
|
def version():
|
||||||
|
moduleinfo['config'] = moduleconfig
|
||||||
|
return moduleinfo
|
|
@ -1,7 +1,7 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
'''
|
'''
|
||||||
Submit sample to VMRay.
|
Submit sample to VMRay.
|
||||||
|
|
||||||
Requires "vmray_rest_api"
|
Requires "vmray_rest_api"
|
||||||
|
|
||||||
|
@ -14,6 +14,7 @@ as a cron job
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import base64
|
import base64
|
||||||
|
from distutils.util import strtobool
|
||||||
|
|
||||||
import io
|
import io
|
||||||
import zipfile
|
import zipfile
|
||||||
|
@ -22,7 +23,7 @@ from ._vmray.vmray_rest_api import VMRayRESTAPI
|
||||||
|
|
||||||
misperrors = {'error': 'Error'}
|
misperrors = {'error': 'Error'}
|
||||||
mispattributes = {'input': ['attachment', 'malware-sample'], 'output': ['text', 'sha1', 'sha256', 'md5', 'link']}
|
mispattributes = {'input': ['attachment', 'malware-sample'], 'output': ['text', 'sha1', 'sha256', 'md5', 'link']}
|
||||||
moduleinfo = {'version': '0.2', 'author': 'Koen Van Impe',
|
moduleinfo = {'version': '0.3', 'author': 'Koen Van Impe',
|
||||||
'description': 'Submit a sample to VMRay',
|
'description': 'Submit a sample to VMRay',
|
||||||
'module-type': ['expansion']}
|
'module-type': ['expansion']}
|
||||||
moduleconfig = ['apikey', 'url', 'shareable', 'do_not_reanalyze', 'do_not_include_vmrayjobids']
|
moduleconfig = ['apikey', 'url', 'shareable', 'do_not_reanalyze', 'do_not_include_vmrayjobids']
|
||||||
|
@ -71,25 +72,13 @@ def handler(q=False):
|
||||||
do_not_reanalyze = request["config"].get("do_not_reanalyze")
|
do_not_reanalyze = request["config"].get("do_not_reanalyze")
|
||||||
do_not_include_vmrayjobids = request["config"].get("do_not_include_vmrayjobids")
|
do_not_include_vmrayjobids = request["config"].get("do_not_include_vmrayjobids")
|
||||||
|
|
||||||
# Do we want the sample to be shared?
|
try:
|
||||||
if shareable == "True":
|
shareable = bool(strtobool(shareable)) # Do we want the sample to be shared?
|
||||||
shareable = True
|
reanalyze = not bool(strtobool(do_not_reanalyze)) # Always reanalyze the sample?
|
||||||
else:
|
include_vmrayjobids = not bool(strtobool(do_not_include_vmrayjobids)) # Include the references to VMRay job IDs
|
||||||
shareable = False
|
except ValueError:
|
||||||
|
misperrors["error"] = "Error while processing settings. Please double-check your values."
|
||||||
# Always reanalyze the sample?
|
return misperrors
|
||||||
if do_not_reanalyze == "True":
|
|
||||||
do_not_reanalyze = True
|
|
||||||
else:
|
|
||||||
do_not_reanalyze = False
|
|
||||||
reanalyze = not do_not_reanalyze
|
|
||||||
|
|
||||||
# Include the references to VMRay job IDs
|
|
||||||
if do_not_include_vmrayjobids == "True":
|
|
||||||
do_not_include_vmrayjobids = True
|
|
||||||
else:
|
|
||||||
do_not_include_vmrayjobids = False
|
|
||||||
include_vmrayjobids = not do_not_include_vmrayjobids
|
|
||||||
|
|
||||||
if data and sample_filename:
|
if data and sample_filename:
|
||||||
args = {}
|
args = {}
|
||||||
|
@ -99,7 +88,7 @@ def handler(q=False):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
vmraydata = vmraySubmit(api, args)
|
vmraydata = vmraySubmit(api, args)
|
||||||
if vmraydata["errors"]:
|
if vmraydata["errors"] and "Submission not stored" not in vmraydata["errors"][0]["error_msg"]:
|
||||||
misperrors['error'] = "VMRay: %s" % vmraydata["errors"][0]["error_msg"]
|
misperrors['error'] = "VMRay: %s" % vmraydata["errors"][0]["error_msg"]
|
||||||
return misperrors
|
return misperrors
|
||||||
else:
|
else:
|
||||||
|
@ -125,22 +114,20 @@ def vmrayProcess(vmraydata):
|
||||||
''' Process the JSON file returned by vmray'''
|
''' Process the JSON file returned by vmray'''
|
||||||
if vmraydata:
|
if vmraydata:
|
||||||
try:
|
try:
|
||||||
submissions = vmraydata["submissions"][0]
|
sample = vmraydata["samples"][0]
|
||||||
jobs = vmraydata["jobs"]
|
jobs = vmraydata["jobs"]
|
||||||
|
|
||||||
# Result received?
|
# Result received?
|
||||||
if submissions and jobs:
|
if sample:
|
||||||
r = {'results': []}
|
r = {'results': []}
|
||||||
r['results'].append({'types': 'md5', 'values': submissions['submission_sample_md5']})
|
r['results'].append({'types': 'md5', 'values': sample['sample_md5hash']})
|
||||||
r['results'].append({'types': 'sha1', 'values': submissions['submission_sample_sha1']})
|
r['results'].append({'types': 'sha1', 'values': sample['sample_sha1hash']})
|
||||||
r['results'].append({'types': 'sha256', 'values': submissions['submission_sample_sha256']})
|
r['results'].append({'types': 'sha256', 'values': sample['sample_sha256hash']})
|
||||||
r['results'].append({'types': 'text', 'values': 'VMRay Sample ID: %s' % submissions['submission_sample_id'], 'tags': 'workflow:state="incomplete"'})
|
r['results'].append({'types': 'text', 'values': 'VMRay Sample ID: %s' % sample['sample_id'], 'tags': 'workflow:state="incomplete"'})
|
||||||
r['results'].append({'types': 'text', 'values': 'VMRay Submission ID: %s' % submissions['submission_id']})
|
r['results'].append({'types': 'link', 'values': sample['sample_webif_url']})
|
||||||
r['results'].append({'types': 'text', 'values': 'VMRay Submission Sample IP: %s' % submissions['submission_ip_ip']})
|
|
||||||
r['results'].append({'types': 'link', 'values': submissions['submission_webif_url']})
|
|
||||||
|
|
||||||
# Include data from different jobs
|
# Include data from different jobs
|
||||||
if include_vmrayjobids:
|
if include_vmrayjobids and len(jobs) > 0:
|
||||||
for job in jobs:
|
for job in jobs:
|
||||||
job_id = job["job_id"]
|
job_id = job["job_id"]
|
||||||
job_vm_name = job["job_vm_name"]
|
job_vm_name = job["job_vm_name"]
|
||||||
|
|
|
@ -105,9 +105,10 @@ class TestExpansions(unittest.TestCase):
|
||||||
query = {"module": "btc_steroids", "btc": "1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA"}
|
query = {"module": "btc_steroids", "btc": "1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA"}
|
||||||
response = self.misp_modules_post(query)
|
response = self.misp_modules_post(query)
|
||||||
try:
|
try:
|
||||||
self.assertTrue(self.get_values(response).startswith('\n\nAddress:\t1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA\nBalance:\t0.0000000000 BTC (+0.0005355700 BTC / -0.0005355700 BTC)'))
|
self.assertTrue(self.get_values(response).startswith('\n\nAddress:\t1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA\nBalance:\t0.0002126800 BTC (+0.0007482500 BTC / -0.0005355700 BTC)'))
|
||||||
|
|
||||||
except Exception:
|
except Exception:
|
||||||
self.assertEqual(self.get_values(response), 'Not a valid BTC address')
|
self.assertEqual(self.get_values(response), 'Not a valid BTC address, or Balance has changed')
|
||||||
|
|
||||||
def test_btc_scam_check(self):
|
def test_btc_scam_check(self):
|
||||||
query = {"module": "btc_scam_check", "btc": "1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA"}
|
query = {"module": "btc_scam_check", "btc": "1ES14c7qLb5CYhLMUekctxLgc1FV2Ti9DA"}
|
||||||
|
|
Loading…
Reference in New Issue