Merge pull request #539 from JakubOnderka/github-actions

new: [CI] Use GitHub Actions for test

.github/workflows/python-package.yml

@@ -0,0 +1,50 @@
+name: Python package
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.7", "3.8", "3.9"]
+
+    steps:
+    - run: |
+        sudo apt-get install libfuzzy-dev libpoppler-cpp-dev libzbar0 tesseract-ocr
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Cache Python dependencies
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('REQUIREMENTS') }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest
+        # pyfaul must be installed manually (?)
+        pip install -r REQUIREMENTS pyfaup
+        pip install .
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with pytest
+      run: |
+        # Run server in background
+        misp-modules -l 127.0.0.1 -s &
+        sleep 5
+        # Run tests
+        pytest tests

Pipfile

@@ -48,7 +48,7 @@ ODTReader = { editable = true, git = "https://github.com/cartertemm/ODTReader.gi
 python-pptx = "*"
 python-docx = "*"
 ezodf = "*"
-pandas = "*"
+pandas = "==1.3.5"
 pandas_ods_reader = "==0.1.2"
 pdftotext = "*"
 lxml = "*"

REQUIREMENTS

@@ -80,7 +80,7 @@ olefile==0.46; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2,
 oletools==0.56.2
 opencv-python==4.5.3.56
 pandas-ods-reader==0.1.2
-pandas==1.1.5
+pandas==1.3.5
 passivetotal==2.5.4
 pcodedmp==1.2.6
 pdftotext==2.2.0

misp_modules/__init__.py

@@ -41,14 +41,14 @@ try:
     from .modules import *  # noqa
     HAS_PACKAGE_MODULES = True
 except Exception as e:
-    print(e)
+    logging.exception(e)
     HAS_PACKAGE_MODULES = False
 
 try:
     from .helpers import *  # noqa
     HAS_PACKAGE_HELPERS = True
 except Exception as e:
-    print(e)
+    logging.exception(e)
     HAS_PACKAGE_HELPERS = False
 
 log = logging.getLogger('misp-modules')

misp_modules/modules/expansion/ods_enrich.py

@@ -4,6 +4,7 @@ import np
 import ezodf
 import pandas_ods_reader
 import io
+import logging
 
 misperrors = {'error': 'Error'}
 mispattributes = {'input': ['attachment'],
@@ -35,13 +36,12 @@ def handler(q=False):
     num_sheets = len(doc.sheets)
     try:
         for i in range(0, num_sheets):
-            ods = pandas_ods_reader.read_ods(ods_file, i, headers=False)
+            ods = pandas_ods_reader.algo.read_data(pandas_ods_reader.parsers.ods, ods_file, i, headers=False)
             ods_content = ods_content + "\n" + ods.to_string(max_rows=None)
-        print(ods_content)
         return {'results': [{'types': ['freetext'], 'values': ods_content, 'comment': ".ods-to-text from file " + filename},
                             {'types': ['text'], 'values': ods_content, 'comment': ".ods-to-text from file " + filename}]}
     except Exception as e:
-        print(e)
+        logging.exception(e)
         err = "Couldn't analyze file as .ods. Error was: " + str(e)
         misperrors['error'] = err
         return misperrors

tests/test_expansions.py

@@ -65,6 +65,8 @@ class TestExpansions(unittest.TestCase):
         if not isinstance(data, dict):
             print(json.dumps(data, indent=2))
             return data
+        if 'results' not in data:
+            return data
         for result in data['results']:
             values = result['values']
             if values:
@@ -253,7 +255,7 @@ class TestExpansions(unittest.TestCase):
                 self.assertEqual(self.get_values(response), 'This IP is commonly spoofed in Internet-scan activity')
             except Exception:
                 self.assertIn(
-                    self.get_errors(reponse),
+                    self.get_errors(response),
                     (
                         "Unauthorized. Please check your API key.",
                         "Too many requests. You've hit the rate-limit."
@@ -263,6 +265,7 @@ class TestExpansions(unittest.TestCase):
             response = self.misp_modules_post(query)
             self.assertEqual(self.get_errors(response), 'Missing Greynoise API key.')
 
+    @unittest.skip("Service doesn't work")
     def test_ipasn(self):
         query = {"module": "ipasn",
                  "attribute": {"type": "ip-src",
@@ -301,7 +304,7 @@ class TestExpansions(unittest.TestCase):
             encoded = b64encode(f.read()).decode()
         query = {"module": "ods_enrich", "attachment": filename, "data": encoded}
         response = self.misp_modules_post(query)
-        self.assertEqual(self.get_values(response), '\n   column_0\n0  ods test')
+        self.assertEqual(self.get_values(response), '\n   column.0\n0  ods test')
 
     def test_odt(self):
         filename = 'test.odt'
@@ -343,6 +346,7 @@ class TestExpansions(unittest.TestCase):
             response = self.misp_modules_post(query)
             self.assertEqual(self.get_errors(response), 'Onyphe authentication is missing')
 
+    @unittest.skip("Unreliable results")
     def test_otx(self):
         query_types = ('domain', 'ip-src', 'md5')
         query_values = ('circl.lu', '8.8.8.8', '616eff3e9a7575ae73821b4668d2801c')