Merge branch 'main' of github.com:misp/pymisp

2023-01-23 10:44:07 +01:00 · 2023-01-23 10:44:07 +01:00 · 846490ed18
parent 12e740672d 9a7adb2e0d
commit 846490ed18
12 changed files with 2947 additions and 2362 deletions
--- a/.github/workflows/pytest.yml
+++ b/.github/workflows/pytest.yml
@ -12,7 +12,7 @@ jobs:
    runs-on: ubuntu-latest
    strategy:
      matrix:
-        python-version: [3.7, 3.8, 3.9, '3.10']
+        python-version: [3.8, 3.9, '3.10', '3.11']

    steps:

--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@ -1,7 +1,7 @@
 version: 2

 python:
-  version: 3.7
+  version: 3.8
  install:
      - method: pip
        path: .
--- a/CHANGELOG.txt
+++ b/CHANGELOG.txt
@ -2,6 +2,93 @@ Changelog
 =========


+v2.4.168 (2023-01-23)
+---------------------
+
+Changes
+~~~~~~~
+- Bump version. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+
+
+v2.4.167.2 (2023-01-17)
+-----------------------
+
+Changes
+~~~~~~~
+- Bump changelog. [Raphaël Vinot]
+- Bump deps, version. [Raphaël Vinot]
+
+Fix
+~~~
+- Set relationship_type default in MISPTag to empty string. [Raphaël
+  Vinot]
+- Another typo in readme. [Raphaël Vinot]
+- Typo in readme. [Raphaël Vinot]
+
+
+v2.4.167.1 (2023-01-16)
+-----------------------
+
+New
+~~~
+- Add relationship_type in Tag entries for feeds. [Raphaël Vinot]
+
+Changes
+~~~~~~~
+- Bump changelog. [Raphaël Vinot]
+- Bump version. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+- Bump requests. [Raphaël Vinot]
+- Bump pyzmq. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+- Bump python version used by read the docs. [Raphaël Vinot]
+- Bump warning to inform user that python 3.10 wil be required in 12
+  months. [Raphaël Vinot]
+- Bump minimal PyMISP version to 3.8. [Raphaël Vinot]
+- Re-bump changelog. [Raphaël Vinot]
+
+Fix
+~~~
+- Update whl files. [Raphaël Vinot]
+- Nvm, readthedocs requires python 3.8 at most. [Raphaël Vinot]
+
+
+v2.4.167 (2022-12-22)
+---------------------
+
+Changes
+~~~~~~~
+- Bump objects. [Raphaël Vinot]
+- Bump changelog. [Raphaël Vinot]
+- Bump version. [Raphaël Vinot]
+- Bump objects. [Raphaël Vinot]
+- Bump dependencies, move to poetry 1.3. [Raphaël Vinot]
+- Bump certifi. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+- Bump deps. [Raphaël Vinot]
+- Re-order classes. [Raphaël Vinot]
+
+Other
+~~~~~
+- Creation fo "add_attributes_from_csv.py" [Julien Mongenet]
+
+  The file aims to ingest a formated CSV file containing attributes for MISP ingestion.
+- Graceful handling of tagging when name attribute is missing. [Sura De
+  Silva]
+- Add: Galaxy test sample. [Christian Studer]
+- Add: Added very straight forward tests to make sure the galaxy
+  clusters are properly defined. [Christian Studer]
+- Add: Added the `Galaxy` field to MISPAttribute using the MISPGalaxy
+  class. [Christian Studer]
+
+  - Including an `add_galaxy` method similar to the
+    one used for events
+  - `attribute.galaxies` gives the list of attached
+    galaxy clusters
+
+
 v2.4.166 (2022-11-28)
 ---------------------

@ -13,6 +100,7 @@ New

 Changes
 ~~~~~~~
+- Re-bump changelog. [Raphaël Vinot]
 - Bump changelog. [Raphaël Vinot]
 - Bump deps, version. [Raphaël Vinot]
 - [types] added azure-application-id. [iglocska]
--- a/README.md
+++ b/README.md
@ -1,10 +1,10 @@
-**IMPORTANT NOTE**: This library will require **at least** python 3.8 starting the 1st of January 2022. If you have legacy versions of python, please use the latest PyMISP version that will be released in December 2021, and consider updating your system(s). Anything released within the last 2 years will do, starting with Ubuntu 20.04.
+**IMPORTANT NOTE**: This library will require **at least** Python 3.10 starting the 1st of January 2024. If you have legacy versions of python, please use the latest PyMISP version that will be released in December 2023, and consider updating your system(s). Anything released within the last 2 years will do, starting with Ubuntu 22.04.

 # PyMISP - Python Library to access MISP

 [![Documentation Status](https://readthedocs.org/projects/pymisp/badge/?version=latest)](http://pymisp.readthedocs.io/?badge=latest)
 [![Coverage Status](https://coveralls.io/repos/github/MISP/PyMISP/badge.svg?branch=main)](https://coveralls.io/github/MISP/PyMISP?branch=main)
-[![Python 3.6](https://img.shields.io/badge/python-3.6+-blue.svg)](https://www.python.org/downloads/release/python-360/)
+[![Python 3.8](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/release/python-380/)
 [![PyPi version](https://img.shields.io/pypi/v/pymisp.svg)](https://pypi.python.org/pypi/pymisp/)
 [![Number of PyPI downloads](https://img.shields.io/pypi/dm/pymisp.svg)](https://pypi.python.org/pypi/pymisp/)

@ -33,7 +33,7 @@ And there are a few optional dependencies:
 * email: to generate MISP Email objects
 * brotli: to use the brotli compression when interacting with a MISP instance

-Example: 
+Example:

 ```
 pip3 install pymisp[virustotal,email]
--- a/examples/add_attributes_from_csv.py
+++ b/examples/add_attributes_from_csv.py
@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import csv
+from pymisp import PyMISP
+from pymisp import ExpandedPyMISP, MISPAttribute
+from keys import misp_url, misp_key, misp_verifycert
+from requests.packages.urllib3.exceptions import InsecureRequestWarning
+import argparse
+import urllib3
+import requests
+requests.packages.urllib3.disable_warnings() 
+
+
+"""
+
+Sample usage:
+
+python3 add_filetype_object_from_csv.py -e <Event_UUID> -f <formated_file_with_attributes>.csv
+
+
+Attribute CSV file (aach line is an entry):
+
+value;category;type;comment;to_ids;first_seen;last_seen;tag1;tag2
+test.pdf;Payload delivery;filename;Email attachment;0;1970-01-01;1970-01-01;tlp:green;ransomware
+127.0.0.1;Network activity;ip-dst;C2 server;1;;;tlp:white;
+
+value = IOC's value
+category = its MISP category (https://www.circl.lu/doc/misp/categories-and-types/)
+type = its MISP type (https://www.circl.lu/doc/misp/categories-and-types/)
+comment = IOC's description
+to_ids = Boolean expected (0 = IDS flag not checked // 1 = IDS flag checked)
+first_seen = First seen date, if any (left empty if not)
+last_seen = Last seen date, if any (left empty if not)
+tag = IOC tag, if any 
+
+"""
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Add attributes to a MISP event from a semi-colon formated csv file')
+    parser.add_argument("-e", "--event_uuid", required=True, help="Event UUID to update")
+    parser.add_argument("-f", "--attr_file", required=True, help="Attribute CSV file path")
+    args = parser.parse_args()
+
+    pymisp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
+
+    f = open(args.attr_file, newline='')
+    csv_reader = csv.reader(f, delimiter=";")
+
+    for line in csv_reader:
+       value = line[0]
+       category = line[1]
+       type = line[2]
+       comment = line[3]
+       ids = line[4]
+       fseen = line[5]
+       lseen = line[6]
+       tags = line[7:]
+
+       misp_attribute = MISPAttribute()
+       misp_attribute.value = str(value)
+       misp_attribute.category = str(category)
+       misp_attribute.type = str(type)
+       misp_attribute.comment = str(comment)
+       misp_attribute.to_ids = str(ids)
+       if fseen != '':
+          misp_attribute.first_seen = str(fseen)
+       if lseen != '':
+          misp_attribute.last_seen = str(lseen)
+       for x in tags:
+            misp_attribute.add_tag(x)
+       r = pymisp.add_attribute(args.event_uuid, misp_attribute)
+       print(line)
+    print("\nAttributes successfully saved :)")
--- a/poetry.lock
+++ b/poetry.lock
--- a/pymisp/init.py
+++ b/pymisp/init.py
@ -1,4 +1,4 @@
-__version__ = '2.4.166'
+__version__ = '2.4.168'
 import logging
 import sys
 import warnings
@ -6,11 +6,11 @@ import warnings
 logger = logging.getLogger(__name__)


-def warning_2022():
-    if sys.version_info < (3, 8):
+def warning_2024():
+    if sys.version_info < (3, 10):
        warnings.warn("""
-As our baseline system is the latest Ubuntu LTS, and Ubuntu LTS 20.04 has Python 3.8 available,
-we will officially deprecate python versions below 3.8 on January 1st 2022.
+As our baseline system is the latest Ubuntu LTS, and Ubuntu LTS 22.04 has Python 3.10 available,
+we will officially deprecate python versions below 3.10 on January 1st 2024.
 **Please update your codebase.**""", DeprecationWarning, stacklevel=3)


@ -25,7 +25,7 @@ Response (if any):


 try:
-    warning_2022()
+    warning_2024()
    from .exceptions import (PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, # noqa
                             InvalidMISPObject, UnknownMISPObjectTemplate, PyMISPInvalidFormat, MISPServerError, PyMISPNotImplementedYet, PyMISPUnexpectedResponse, PyMISPEmptyResponse)
    from .abstract import AbstractMISP, MISPEncode, pymisp_json_default, MISPTag, Distribution, ThreatLevel, Analysis # noqa
--- a/pymisp/abstract.py
+++ b/pymisp/abstract.py
@ -367,13 +367,14 @@ class AbstractMISP(MutableMapping, MISPFileCache, metaclass=ABCMeta):

 class MISPTag(AbstractMISP):

-    _fields_for_feed: set = {'name', 'colour'}
+    _fields_for_feed: set = {'name', 'colour', 'relationship_type'}

    def __init__(self, **kwargs: Dict):
        super().__init__(**kwargs)
        self.name: str
        self.exportable: bool
        self.local: bool
+        self.relationship_type: Optional[str]

    def from_dict(self, **kwargs):
        if kwargs.get('Tag'):
@ -381,6 +382,8 @@ class MISPTag(AbstractMISP):
        super().from_dict(**kwargs)

    def _set_default(self):
+        if not hasattr(self, 'relationship_type'):
+            self.relationship_type = ''
        if not hasattr(self, 'colour'):
            self.colour = '#ffffff'

--- a/pymisp/api.py
+++ b/pymisp/api.py
@ -3492,8 +3492,10 @@ class PyMISP:
        """
        uuid = get_uuid_or_id_from_abstract_misp(misp_entity)
        if isinstance(tag, MISPTag):
-            tag = tag.name
-        to_post = {'uuid': uuid, 'tag': tag, 'local': local}
+            tag_name = tag.name if 'name' in tag else ""
+        else:
+            tag_name = tag
+        to_post = {'uuid': uuid, 'tag': tag_name, 'local': local}
        response = self._prepare_request('POST', 'tags/attachTagToObject', data=to_post)
        return self._check_json_response(response)

@ -3505,8 +3507,7 @@ class PyMISP:
        """
        uuid = get_uuid_or_id_from_abstract_misp(misp_entity)
        if isinstance(tag, MISPTag):
-            if 'name' in tag:
-                tag_name = tag.name
+            tag_name = tag.name if 'name' in tag else ""
        else:
            tag_name = tag
        to_post = {'uuid': uuid, 'tag': tag_name}
--- a/pymisp/data/misp-objects
+++ b/pymisp/data/misp-objects
@ -1 +1 @@
-Subproject commit 2787dc45d7efbf32e0fbe81ea95f0af642ae8963
+Subproject commit fd603be3283953b68ed48ede7afd2e19f43577ac
--- a/pymisp/tools/emailobject.py
+++ b/pymisp/tools/emailobject.py
@ -11,8 +11,8 @@ from io import BytesIO
 from pathlib import Path
 from typing import Union, List, Tuple, Dict, cast, Any, Optional

-from extract_msg import openMsg
-from extract_msg.message import Message as MsgObj
+from extract_msg import openMsg  # type: ignore
+from extract_msg.message import Message as MsgObj  # type: ignore
 from RTFDE.exceptions import MalformedEncapsulatedRtf, NotEncapsulatedRtf  # type: ignore
 from RTFDE.deencapsulate import DeEncapsulator  # type: ignore
 from oletools.common.codepages import codepage2codec  # type: ignore
--- a/pyproject.toml
+++ b/pyproject.toml
@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pymisp"
-version = "2.4.166"
+version = "2.4.168"
 description = "Python API for MISP."
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-2-Clause"
@ -18,9 +18,10 @@ classifiers=[
    'Intended Audience :: Science/Research',
    'Intended Audience :: Telecommunications Industry',
    'Intended Audience :: Information Technology',
-    'Programming Language :: Python :: 3.7',
    'Programming Language :: Python :: 3.8',
    'Programming Language :: Python :: 3.9',
+    'Programming Language :: Python :: 3.10',
+    'Programming Language :: Python :: 3.11',
    'Topic :: Security',
    'Topic :: Internet'
 ]
@ -41,12 +42,12 @@ include = [
 "Source" = "https://github.com/MISP/PyMISP"

 [tool.poetry.dependencies]
-python = "^3.7"
-requests = "^2.28.1"
+python = "^3.8"
+requests = "^2.28.2"
 python-dateutil = "^2.8.2"
 jsonschema = "^4.17.3"
 deprecated = "^1.2.13"
-extract_msg = {version = "^0.38.4", optional = true}
+extract_msg = {version = "^0.39.0", optional = true}
 RTFDE = {version = "^0.0.2", optional = true}
 oletools = {version = "^0.60.1", optional = true}
 python-magic = {version = "^0.4.27", optional = true}
@ -54,13 +55,12 @@ pydeep2 = {version = "^0.5.1", optional = true}
 lief = {version = "^0.12.3", optional = true}
 beautifulsoup4 = {version = "^4.11.1", optional = true}
 validators = {version = "^0.20.0", optional = true}
-sphinx-autodoc-typehints = {version = "^1.19.5", optional = true}
+sphinx-autodoc-typehints = {version = "^1.21.7", optional = true}
 recommonmark = {version = "^0.7.1", optional = true}
 reportlab = {version = "^3.6.12", optional = true}
 pyfaup = {version = "^1.2", optional = true}
-publicsuffixlist = {version = "^0.9.1", optional = true}
-chardet = {version = "^5.1.0", optional = true}
-urllib3 = {extras = ["brotli"], version = "^1.26.13", optional = true}
+publicsuffixlist = {version = "^0.9.2", optional = true}
+urllib3 = {extras = ["brotli"], version = "^1.26.14", optional = true}

 [tool.poetry.extras]
 fileobjects = ['python-magic', 'pydeep2', 'lief']
@ -68,18 +68,18 @@ openioc = ['beautifulsoup4']
 virustotal = ['validators']
 docs = ['sphinx-autodoc-typehints', 'recommonmark']
 pdfexport = ['reportlab']
-url = ['pyfaup', 'chardet']
+url = ['pyfaup']
 email = ['extract_msg', "RTFDE", "oletools"]
 brotli = ['urllib3']

 [tool.poetry.group.dev.dependencies]
 requests-mock = "^1.10.0"
 mypy = "^0.991"
-ipython = "^7.34.0"
-jupyterlab = "^3.5.1"
-types-requests = "^2.28.11.5"
-types-python-dateutil = "^2.8.19.4"
-types-redis = "^4.3.21.6"
+ipython = "^8.8.0"
+jupyterlab = "^3.5.2"
+types-requests = "^2.28.11.8"
+types-python-dateutil = "^2.8.19.6"
+types-redis = "^4.4.0.3"
 types-Flask = "^1.1.6"
 pytest-cov = "^4.0.0"