commit b4f1b9dd695b8bf2c791a027df2bb888b2a737a6 Author: Raphaël Vinot Date: Sat Oct 28 19:40:41 2017 -0400 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0799bd9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,106 @@ +# Local exclude +scraped/ +*.swp +lookyloo/ete3_webserver/webapi.py + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..8f207dc --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "pymispwarninglists/data/misp-warninglists"] + path = pymispwarninglists/data/misp-warninglists + url = https://github.com/MISP/misp-warninglists.git diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..d03d276 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,19 @@ +language: python + +cache: pip + +python: + - "3.6" + - "3.6-dev" + - "nightly" + +install: + - pip install coveralls codecov jsonschema + - pip install . + +script: + - nosetests --with-coverage --cover-package=pymispwarninglists -d + +after_success: + - codecov + - coveralls diff --git a/README.md b/README.md new file mode 100644 index 0000000..ad2f29c --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +# PyMISPWarningLists + +[![Build Status](https://travis-ci.org/MISP/PyMISPWarningLists.svg?branch=master)](https://travis-ci.org/MISP/PyMISPWarningLists) +[![codecov.io](https://codecov.io/github/MISP/PyMISParningLists/coverage.svg?branch=master)](https://codecov.io/github/MISP/PyMISPWarningLists?branch=master) + +Pythonic way to work with the warning lists defined there: https://github.com/MISP/misp-warninglists + +# Usage + +Warning lists are represented as immutable Python dictionaries. + diff --git a/pymispwarninglists/__init__.py b/pymispwarninglists/__init__.py new file mode 100644 index 0000000..1ddde4a --- /dev/null +++ b/pymispwarninglists/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from .api import WarningLists, EncodeWarningLists diff --git a/pymispwarninglists/api.py b/pymispwarninglists/api.py new file mode 100644 index 0000000..1f02d2b --- /dev/null +++ b/pymispwarninglists/api.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import json +from json import JSONEncoder +import os +import sys +import collections +from glob import glob + +try: + import jsonschema + HAS_JSONSCHEMA = True +except ImportError: + HAS_JSONSCHEMA = False + + +class EncodeWarningLists(JSONEncoder): + def default(self, obj): + try: + return obj._json() + except AttributeError: + return JSONEncoder.default(self, obj) + + +class PyMISPWarningListsError(Exception): + def __init__(self, message): + super(PyMISPWarningListsError, self).__init__(message) + self.message = message + + +class WarningList(): + + def __init__(self, warninglist): + self.warninglist = warninglist + self.list = self.warninglist['list'] + self.description = self.warninglist['description'] + self.version = int(self.warninglist['version']) + self.name = self.warninglist['name'] + if self.warninglist.get('type'): + self.type = self.warninglist['type'] + if self.warninglist.get('matching_attributes'): + self.matching_attributes = self.warninglist['matching_attributes'] + + def _json(self): + to_return = {'list': self.list, 'name': self.name, 'description': self.description, + 'version': self.version} + if hasattr(self, 'type'): + to_return['type'] = self.type + if hasattr(self, 'matching_attributes'): + to_return['matching_attributes'] = self.matching_attributes + return to_return + + +class WarningLists(collections.Mapping): + + def __init__(self): + self.root_dir_warninglists = os.path.join(os.path.abspath(os.path.dirname(sys.modules['pymispwarninglists'].__file__)), + 'data', 'misp-warninglists', 'lists') + self.warninglists = {} + for warninglist_file in glob(os.path.join(self.root_dir_warninglists, '*', 'list.json')): + with open(warninglist_file, 'r') as f: + warninglist = json.load(f) + self.warninglists[warninglist['name']] = WarningList(warninglist) + + def validate_with_schema(self): + if not HAS_JSONSCHEMA: + raise ImportError('jsonschema is required: pip install jsonschema') + schema = os.path.join(os.path.abspath(os.path.dirname(sys.modules['pymispwarninglists'].__file__)), + 'data', 'misp-warninglists', 'schema.json') + with open(schema, 'r') as f: + loaded_schema = json.load(f) + for w in self.warninglists.values(): + jsonschema.validate(w.warninglist, loaded_schema) + + def __getitem__(self, name): + return self.warninglists[name] + + def __iter__(self): + return iter(self.warninglists) + + def __len__(self): + return len(self.warninglists) diff --git a/pymispwarninglists/data/misp-warninglists b/pymispwarninglists/data/misp-warninglists new file mode 160000 index 0000000..095b506 --- /dev/null +++ b/pymispwarninglists/data/misp-warninglists @@ -0,0 +1 @@ +Subproject commit 095b5063f3bd427d787639e8c092ad204d13b282 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..23471b6 --- /dev/null +++ b/setup.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from setuptools import setup + + +setup( + name='pymispwarninglist', + version='0.1', + author='Raphaël Vinot', + author_email='raphael.vinot@circl.lu', + maintainer='Raphaël Vinot', + url='https://github.com/MISP/PyMISPWarningLists', + description='Python API for the MISP warning lists.', + packages=['pymispwarninglists'], + classifiers=[ + 'License :: OSI Approved :: BSD License', + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', + 'Intended Audience :: Science/Research', + 'Intended Audience :: Telecommunications Industry', + 'Programming Language :: Python', + 'Topic :: Security', + 'Topic :: Internet', + ], + tests_requires=['nose'], + test_suite='nose.collector', + package_data={'pymispwarninglists': ['data/misp-warninglists/schema.json', + 'data/misp-warninglists/*/*.json']} +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/tests.py b/tests/tests.py new file mode 100644 index 0000000..2dec739 --- /dev/null +++ b/tests/tests.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import unittest +from pymispwarninglists import WarningLists, EncodeWarningLists +from glob import glob +import os +import json + + +class TestPyMISPWarningLists(unittest.TestCase): + + def setUp(self): + self.warninglists = WarningLists() + + def test_dump_warninglists(self): + warninglists_from_files = {} + for warninglist_file in glob(os.path.join(self.warninglists.root_dir_warninglists, '*', 'list.json')): + with open(warninglist_file, 'r') as f: + warninglist = json.load(f) + warninglists_from_files[warninglist['name']] = warninglist + for name, w in self.warninglists.items(): + out = w._json() + self.assertDictEqual(out, warninglists_from_files[w.name]) + + def test_validate_schema_warninglists(self): + self.warninglists.validate_with_schema() + + def test_json(self): + for w in self.warninglists.values(): + json.dumps(w, cls=EncodeWarningLists)