Initial commit

.gitignore

@@ -0,0 +1,106 @@
+# Local exclude
+scraped/
+*.swp
+lookyloo/ete3_webserver/webapi.py
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "pymispgalaxies/data/misp-galaxy"]
+	path = pymispgalaxies/data/misp-galaxy
+	url = https://github.com/MISP/misp-galaxy.git

.travis.yml

@@ -0,0 +1,19 @@
+language: python
+
+cache: pip
+
+python:
+    - "3.6"
+    - "3.6-dev"
+    - "nightly"
+
+install:
+    - pip install coveralls codecov jsonschema
+    - pip install .
+
+script:
+    - nosetests --with-coverage --cover-package=pymispgalaxies -d
+
+after_success:
+    - codecov
+    - coveralls

README.md

@@ -0,0 +1,11 @@
+# PyMISPGalaxies
+
+[![Build Status](https://travis-ci.org/MISP/PyMISPGalaxies.svg?branch=master)](https://travis-ci.org/MISP/PyMISPGalaxies)
+[![codecov.io](https://codecov.io/github/MISP/PyMISPGalaxies/coverage.svg?branch=master)](https://codecov.io/github/MISP/PyMISPGalaxies?branch=master)
+
+Pythonic way to work with the galaxies defined there: https://github.com/MISP/misp-galaxy
+
+# Usage
+
+Clusters and Galaxies are represented as immutable Python dictionaries.
+

pymispgalaxies/__init__.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from .api import Galaxies, Clusters

pymispgalaxies/api.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import json
+import os
+import sys
+import collections
+from glob import glob
+
+try:
+    import jsonschema
+    HAS_JSONSCHEMA = True
+except ImportError:
+    HAS_JSONSCHEMA = False
+
+
+class Galaxy():
+
+    def __init__(self, galaxy):
+        self.galaxy = galaxy
+        self.type = self.galaxy['type']
+        self.name = self.galaxy['name']
+        self.description = self.galaxy['description']
+        self.version = self.galaxy['version']
+        self.uuid = self.galaxy['uuid']
+
+    def _json(self):
+        return {'type': self.type, 'name': self.name, 'description': self.description,
+                'version': self.version, 'uuid': self.uuid}
+
+
+class Galaxies(collections.Mapping):
+
+    def __init__(self):
+        self.root_dir_galaxies = os.path.join(os.path.abspath(os.path.dirname(sys.modules['pymispgalaxies'].__file__)),
+                                              'data', 'misp-galaxy', 'galaxies')
+        self.galaxies = {}
+        for galaxy_file in glob(os.path.join(self.root_dir_galaxies, '*.json')):
+            with open(galaxy_file, 'r') as f:
+                galaxy = json.load(f)
+            self.galaxies[galaxy['name']] = Galaxy(galaxy)
+
+    def validate_with_schema(self):
+        if not HAS_JSONSCHEMA:
+            raise ImportError('jsonschema is required: pip install jsonschema')
+        schema = os.path.join(os.path.abspath(os.path.dirname(sys.modules['pymispgalaxies'].__file__)),
+                              'data', 'misp-galaxy', 'schema_galaxies.json')
+        with open(schema, 'r') as f:
+            loaded_schema = json.load(f)
+        for g in self.galaxies.values():
+            jsonschema.validate(g.galaxy, loaded_schema)
+
+    def __getitem__(self, name):
+        return self.galaxies[name]
+
+    def __iter__(self):
+        return iter(self.galaxies)
+
+    def __len__(self):
+        return len(self.galaxies)
+
+    def __str__(self):
+        to_print = ''
+        for galaxy in self.galaxies.values():
+            to_print += '{}\n\n'.format(str(galaxy))
+        return to_print
+
+
+class ClusterValueMeta():
+
+    def __init__(self, m):
+        self.type = m.pop('type', None)
+        self.complexity = m.pop('complexity', None)
+        self.effectiveness = m.pop('effectiveness', None)
+        self.country = m.pop('country', None)
+        self.possible_issues = m.pop('possible_issues', None)
+        self.colour = m.pop('colour', None)
+        self.motive = m.pop('motive', None)
+        self.impact = m.pop('impact', None)
+        self.refs = m.pop('refs', None)
+        self.synonyms = m.pop('synonyms', None)
+        self.derivated_from = m.pop('derivated_from', None)
+        self.status = m.pop('status', None)
+        self.date = m.pop('date', None)
+        self.encryption = m.pop('encryption', None)
+        self.extensions = m.pop('extensions', None)
+        self.ransomnotes = m.pop('ransomnotes', None)
+        # NOTE: meta can have aditional properties. We only load the ones
+        # defined on the schema
+        self.additional_properties = m
+
+    def _json(self):
+        to_return = {}
+        if self.type:
+            to_return['type'] = self.type
+        if self.complexity:
+            to_return['complexity'] = self.complexity
+        if self.effectiveness:
+            to_return['effectiveness'] = self.effectiveness
+        if self.country:
+            to_return['country'] = self.country
+        if self.possible_issues:
+            to_return['possible_issues'] = self.possible_issues
+        if self.colour:
+            to_return['colour'] = self.colour
+        if self.motive:
+            to_return['motive'] = self.motive
+        if self.impact:
+            to_return['impact'] = self.impact
+        if self.refs:
+            to_return['refs'] = self.refs
+        if self.synonyms:
+            to_return['synonyms'] = self.synonyms
+        if self.derivated_from:
+            to_return['derivated_from'] = self.derivated_from
+        if self.status:
+            to_return['status'] = self.status
+        if self.date:
+            to_return['date'] = self.date
+        if self.encryption:
+            to_return['encryption'] = self.encryption
+        if self.extensions:
+            to_return['extensions'] = self.extensions
+        if self.ransomnotes:
+            to_return['ransomnotes'] = self.ransomnotes
+        if self.additional_properties:
+            to_return.update(self.additional_properties)
+        return to_return
+
+
+class ClusterValue():
+
+    def __init__(self, v):
+        self.value = v['value']
+        self.description = v.get('description')
+        self.meta = self.__init_meta(v.get('meta'))
+
+    def __init_meta(self, m):
+        if not m:
+            return None
+        return ClusterValueMeta(m)
+
+    def _json(self):
+        to_return = {'value': self.value}
+        if self.description:
+            to_return['description'] = self.description
+        if self.meta:
+            to_return['meta'] = self.meta._json()
+        return to_return
+
+
+class Cluster():
+
+    def __init__(self, cluster):
+        self.cluster = cluster
+        self.name = self.cluster['name']
+        self.type = self.cluster['type']
+        self.source = self.cluster['source']
+        self.authors = self.cluster['authors']
+        self.description = self.cluster['description']
+        self.uuid = self.cluster['uuid']
+        self.version = self.cluster['version']
+        self.values = []
+        for value in self.cluster['values']:
+            self.values.append(ClusterValue(value))
+
+    def _json(self):
+        to_return = {'name': self.name, 'type': self.type, 'source': self.source,
+                     'authors': self.authors, 'description': self.description,
+                     'uuid': self.uuid, 'version': self.version, 'values': []}
+        for v in self.values:
+            to_return['values'].append(v._json())
+        return to_return
+
+
+class Clusters(collections.Mapping):
+
+    def __init__(self):
+        self.root_dir_clusters = os.path.join(os.path.abspath(os.path.dirname(sys.modules['pymispgalaxies'].__file__)),
+                                              'data', 'misp-galaxy', 'clusters')
+        self.clusters = {}
+        for cluster_file in glob(os.path.join(self.root_dir_clusters, '*.json')):
+            with open(cluster_file, 'r') as f:
+                cluster = json.load(f)
+            self.clusters[cluster['name']] = Cluster(cluster)
+
+    def validate_with_schema(self):
+        if not HAS_JSONSCHEMA:
+            raise ImportError('jsonschema is required: pip install jsonschema')
+        schema = os.path.join(os.path.abspath(os.path.dirname(sys.modules['pymispgalaxies'].__file__)),
+                              'data', 'misp-galaxy', 'schema_clusters.json')
+        with open(schema, 'r') as f:
+            loaded_schema = json.load(f)
+        for c in self.clusters.values():
+            jsonschema.validate(c.cluster, loaded_schema)
+
+    def __getitem__(self, name):
+        return self.clusters[name]
+
+    def __iter__(self):
+        return iter(self.clusters)
+
+    def __len__(self):
+        return len(self.clusters)
+
+    def __str__(self):
+        to_print = ''
+        for cluster in self.clusters.values():
+            to_print += '{}\n\n'.format(str(cluster))
+        return to_print

pymispgalaxies/data/misp-galaxy

@@ -0,0 +1 @@
+Subproject commit 3f8b2b4b013f8c8452297dfc8dada045d9462c41

setup.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from setuptools import setup
+
+
+setup(
+    name='pymispgalaxies',
+    version='0.1',
+    author='Raphaël Vinot',
+    author_email='raphael.vinot@circl.lu',
+    maintainer='Raphaël Vinot',
+    url='https://github.com/MISP/PyMISPGalaxies',
+    description='Python API for the MISP Galaxies.',
+    packages=['pymispgalaxies'],
+    classifiers=[
+        'License :: OSI Approved :: BSD License',
+        'Development Status :: 5 - Production/Stable',
+        'Environment :: Console',
+        'Intended Audience :: Science/Research',
+        'Intended Audience :: Telecommunications Industry',
+        'Programming Language :: Python',
+        'Topic :: Security',
+        'Topic :: Internet',
+    ],
+    tests_requires=['nose'],
+    test_suite='nose.collector',
+    package_data={'pymispgalaxies': ['data/misp-galaxy/schema_*.json',
+                                     'data/misp-galaxy/clusters/*.json',
+                                     'data/misp-galaxy/galaxies/*.json',
+                                     'data/misp-galaxy/misp/*.json',
+                                     'data/misp-galaxy/vocabularies/common/*.json',
+                                     'data/misp-galaxy/vocabularies/threat-actor/*.json']}
+)

tests/tests.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import unittest
+from pymispgalaxies import Galaxies, Clusters
+from glob import glob
+import os
+import json
+
+
+class TestPyMISPGalaxies(unittest.TestCase):
+
+    def setUp(self):
+        self.galaxies = Galaxies()
+        self.clusters = Clusters()
+        self.maxDiff = None
+
+    def test_dump_galaxies(self):
+        galaxies_from_files = {}
+        for galaxy_file in glob(os.path.join(self.galaxies.root_dir_galaxies, '*.json')):
+            with open(galaxy_file, 'r') as f:
+                galaxy = json.load(f)
+            galaxies_from_files[galaxy['name']] = galaxy
+        for name, g in self.galaxies.items():
+            out = g._json()
+            self.assertDictEqual(out, galaxies_from_files[g.name])
+
+    def test_dump_clusters(self):
+        clusters_from_files = {}
+        for cluster_file in glob(os.path.join(self.clusters.root_dir_clusters, '*.json')):
+            with open(cluster_file, 'r') as f:
+                cluster = json.load(f)
+            clusters_from_files[cluster['name']] = cluster
+        for name, c in self.clusters.items():
+            out = c._json()
+            self.assertDictEqual(out, clusters_from_files[c.name])
+
+    def test_validate_schema_clusters(self):
+        self.clusters.validate_with_schema()
+
+    def test_validate_schema_galaxies(self):
+        self.galaxies.validate_with_schema()
+
+    def test_meta_additional_properties(self):
+        # All the properties in the meta key of the bundled-in clusters should be known
+        for c in self.clusters.values():
+            for cv in c.values:
+                if cv.meta:
+                    self.assertIsNot(cv.meta.additional_properties, {})