commit 2c1c2bb50245ff780e4a2ba5710ffe406c67c75a Author: Raphaël Vinot Date: Tue Jul 25 18:04:15 2017 +0200 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0799bd9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,106 @@ +# Local exclude +scraped/ +*.swp +lookyloo/ete3_webserver/webapi.py + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..f59384d --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "pymispgalaxies/data/misp-galaxy"] + path = pymispgalaxies/data/misp-galaxy + url = https://github.com/MISP/misp-galaxy.git diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..e415f38 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,19 @@ +language: python + +cache: pip + +python: + - "3.6" + - "3.6-dev" + - "nightly" + +install: + - pip install coveralls codecov jsonschema + - pip install . + +script: + - nosetests --with-coverage --cover-package=pymispgalaxies -d + +after_success: + - codecov + - coveralls diff --git a/README.md b/README.md new file mode 100644 index 0000000..4a78c74 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +# PyMISPGalaxies + +[![Build Status](https://travis-ci.org/MISP/PyMISPGalaxies.svg?branch=master)](https://travis-ci.org/MISP/PyMISPGalaxies) +[![codecov.io](https://codecov.io/github/MISP/PyMISPGalaxies/coverage.svg?branch=master)](https://codecov.io/github/MISP/PyMISPGalaxies?branch=master) + +Pythonic way to work with the galaxies defined there: https://github.com/MISP/misp-galaxy + +# Usage + +Clusters and Galaxies are represented as immutable Python dictionaries. + diff --git a/pymispgalaxies/__init__.py b/pymispgalaxies/__init__.py new file mode 100644 index 0000000..3c9f1da --- /dev/null +++ b/pymispgalaxies/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from .api import Galaxies, Clusters diff --git a/pymispgalaxies/api.py b/pymispgalaxies/api.py new file mode 100644 index 0000000..13add5e --- /dev/null +++ b/pymispgalaxies/api.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import json +import os +import sys +import collections +from glob import glob + +try: + import jsonschema + HAS_JSONSCHEMA = True +except ImportError: + HAS_JSONSCHEMA = False + + +class Galaxy(): + + def __init__(self, galaxy): + self.galaxy = galaxy + self.type = self.galaxy['type'] + self.name = self.galaxy['name'] + self.description = self.galaxy['description'] + self.version = self.galaxy['version'] + self.uuid = self.galaxy['uuid'] + + def _json(self): + return {'type': self.type, 'name': self.name, 'description': self.description, + 'version': self.version, 'uuid': self.uuid} + + +class Galaxies(collections.Mapping): + + def __init__(self): + self.root_dir_galaxies = os.path.join(os.path.abspath(os.path.dirname(sys.modules['pymispgalaxies'].__file__)), + 'data', 'misp-galaxy', 'galaxies') + self.galaxies = {} + for galaxy_file in glob(os.path.join(self.root_dir_galaxies, '*.json')): + with open(galaxy_file, 'r') as f: + galaxy = json.load(f) + self.galaxies[galaxy['name']] = Galaxy(galaxy) + + def validate_with_schema(self): + if not HAS_JSONSCHEMA: + raise ImportError('jsonschema is required: pip install jsonschema') + schema = os.path.join(os.path.abspath(os.path.dirname(sys.modules['pymispgalaxies'].__file__)), + 'data', 'misp-galaxy', 'schema_galaxies.json') + with open(schema, 'r') as f: + loaded_schema = json.load(f) + for g in self.galaxies.values(): + jsonschema.validate(g.galaxy, loaded_schema) + + def __getitem__(self, name): + return self.galaxies[name] + + def __iter__(self): + return iter(self.galaxies) + + def __len__(self): + return len(self.galaxies) + + def __str__(self): + to_print = '' + for galaxy in self.galaxies.values(): + to_print += '{}\n\n'.format(str(galaxy)) + return to_print + + +class ClusterValueMeta(): + + def __init__(self, m): + self.type = m.pop('type', None) + self.complexity = m.pop('complexity', None) + self.effectiveness = m.pop('effectiveness', None) + self.country = m.pop('country', None) + self.possible_issues = m.pop('possible_issues', None) + self.colour = m.pop('colour', None) + self.motive = m.pop('motive', None) + self.impact = m.pop('impact', None) + self.refs = m.pop('refs', None) + self.synonyms = m.pop('synonyms', None) + self.derivated_from = m.pop('derivated_from', None) + self.status = m.pop('status', None) + self.date = m.pop('date', None) + self.encryption = m.pop('encryption', None) + self.extensions = m.pop('extensions', None) + self.ransomnotes = m.pop('ransomnotes', None) + # NOTE: meta can have aditional properties. We only load the ones + # defined on the schema + self.additional_properties = m + + def _json(self): + to_return = {} + if self.type: + to_return['type'] = self.type + if self.complexity: + to_return['complexity'] = self.complexity + if self.effectiveness: + to_return['effectiveness'] = self.effectiveness + if self.country: + to_return['country'] = self.country + if self.possible_issues: + to_return['possible_issues'] = self.possible_issues + if self.colour: + to_return['colour'] = self.colour + if self.motive: + to_return['motive'] = self.motive + if self.impact: + to_return['impact'] = self.impact + if self.refs: + to_return['refs'] = self.refs + if self.synonyms: + to_return['synonyms'] = self.synonyms + if self.derivated_from: + to_return['derivated_from'] = self.derivated_from + if self.status: + to_return['status'] = self.status + if self.date: + to_return['date'] = self.date + if self.encryption: + to_return['encryption'] = self.encryption + if self.extensions: + to_return['extensions'] = self.extensions + if self.ransomnotes: + to_return['ransomnotes'] = self.ransomnotes + if self.additional_properties: + to_return.update(self.additional_properties) + return to_return + + +class ClusterValue(): + + def __init__(self, v): + self.value = v['value'] + self.description = v.get('description') + self.meta = self.__init_meta(v.get('meta')) + + def __init_meta(self, m): + if not m: + return None + return ClusterValueMeta(m) + + def _json(self): + to_return = {'value': self.value} + if self.description: + to_return['description'] = self.description + if self.meta: + to_return['meta'] = self.meta._json() + return to_return + + +class Cluster(): + + def __init__(self, cluster): + self.cluster = cluster + self.name = self.cluster['name'] + self.type = self.cluster['type'] + self.source = self.cluster['source'] + self.authors = self.cluster['authors'] + self.description = self.cluster['description'] + self.uuid = self.cluster['uuid'] + self.version = self.cluster['version'] + self.values = [] + for value in self.cluster['values']: + self.values.append(ClusterValue(value)) + + def _json(self): + to_return = {'name': self.name, 'type': self.type, 'source': self.source, + 'authors': self.authors, 'description': self.description, + 'uuid': self.uuid, 'version': self.version, 'values': []} + for v in self.values: + to_return['values'].append(v._json()) + return to_return + + +class Clusters(collections.Mapping): + + def __init__(self): + self.root_dir_clusters = os.path.join(os.path.abspath(os.path.dirname(sys.modules['pymispgalaxies'].__file__)), + 'data', 'misp-galaxy', 'clusters') + self.clusters = {} + for cluster_file in glob(os.path.join(self.root_dir_clusters, '*.json')): + with open(cluster_file, 'r') as f: + cluster = json.load(f) + self.clusters[cluster['name']] = Cluster(cluster) + + def validate_with_schema(self): + if not HAS_JSONSCHEMA: + raise ImportError('jsonschema is required: pip install jsonschema') + schema = os.path.join(os.path.abspath(os.path.dirname(sys.modules['pymispgalaxies'].__file__)), + 'data', 'misp-galaxy', 'schema_clusters.json') + with open(schema, 'r') as f: + loaded_schema = json.load(f) + for c in self.clusters.values(): + jsonschema.validate(c.cluster, loaded_schema) + + def __getitem__(self, name): + return self.clusters[name] + + def __iter__(self): + return iter(self.clusters) + + def __len__(self): + return len(self.clusters) + + def __str__(self): + to_print = '' + for cluster in self.clusters.values(): + to_print += '{}\n\n'.format(str(cluster)) + return to_print diff --git a/pymispgalaxies/data/misp-galaxy b/pymispgalaxies/data/misp-galaxy new file mode 160000 index 0000000..3f8b2b4 --- /dev/null +++ b/pymispgalaxies/data/misp-galaxy @@ -0,0 +1 @@ +Subproject commit 3f8b2b4b013f8c8452297dfc8dada045d9462c41 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..27b325a --- /dev/null +++ b/setup.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from setuptools import setup + + +setup( + name='pymispgalaxies', + version='0.1', + author='Raphaël Vinot', + author_email='raphael.vinot@circl.lu', + maintainer='Raphaël Vinot', + url='https://github.com/MISP/PyMISPGalaxies', + description='Python API for the MISP Galaxies.', + packages=['pymispgalaxies'], + classifiers=[ + 'License :: OSI Approved :: BSD License', + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', + 'Intended Audience :: Science/Research', + 'Intended Audience :: Telecommunications Industry', + 'Programming Language :: Python', + 'Topic :: Security', + 'Topic :: Internet', + ], + tests_requires=['nose'], + test_suite='nose.collector', + package_data={'pymispgalaxies': ['data/misp-galaxy/schema_*.json', + 'data/misp-galaxy/clusters/*.json', + 'data/misp-galaxy/galaxies/*.json', + 'data/misp-galaxy/misp/*.json', + 'data/misp-galaxy/vocabularies/common/*.json', + 'data/misp-galaxy/vocabularies/threat-actor/*.json']} +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/tests.py b/tests/tests.py new file mode 100644 index 0000000..ebb05b7 --- /dev/null +++ b/tests/tests.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import unittest +from pymispgalaxies import Galaxies, Clusters +from glob import glob +import os +import json + + +class TestPyMISPGalaxies(unittest.TestCase): + + def setUp(self): + self.galaxies = Galaxies() + self.clusters = Clusters() + self.maxDiff = None + + def test_dump_galaxies(self): + galaxies_from_files = {} + for galaxy_file in glob(os.path.join(self.galaxies.root_dir_galaxies, '*.json')): + with open(galaxy_file, 'r') as f: + galaxy = json.load(f) + galaxies_from_files[galaxy['name']] = galaxy + for name, g in self.galaxies.items(): + out = g._json() + self.assertDictEqual(out, galaxies_from_files[g.name]) + + def test_dump_clusters(self): + clusters_from_files = {} + for cluster_file in glob(os.path.join(self.clusters.root_dir_clusters, '*.json')): + with open(cluster_file, 'r') as f: + cluster = json.load(f) + clusters_from_files[cluster['name']] = cluster + for name, c in self.clusters.items(): + out = c._json() + self.assertDictEqual(out, clusters_from_files[c.name]) + + def test_validate_schema_clusters(self): + self.clusters.validate_with_schema() + + def test_validate_schema_galaxies(self): + self.galaxies.validate_with_schema() + + def test_meta_additional_properties(self): + # All the properties in the meta key of the bundled-in clusters should be known + for c in self.clusters.values(): + for cv in c.values: + if cv.meta: + self.assertIsNot(cv.meta.additional_properties, {})