Merge pull request #539 from JakubOnderka/github-actions

new: [CI] Use GitHub Actions for test
pull/541/head
Alexandre Dulaunoy 2021-12-25 08:10:42 +01:00 committed by GitHub
commit 3f127faa50
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 63 additions and 9 deletions

50
.github/workflows/python-package.yml vendored Normal file
View File

@ -0,0 +1,50 @@
name: Python package
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
jobs:
build:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: ["3.7", "3.8", "3.9"]
steps:
- run: |
sudo apt-get install libfuzzy-dev libpoppler-cpp-dev libzbar0 tesseract-ocr
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Cache Python dependencies
uses: actions/cache@v2
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('REQUIREMENTS') }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
python -m pip install flake8 pytest
# pyfaul must be installed manually (?)
pip install -r REQUIREMENTS pyfaup
pip install .
- name: Lint with flake8
run: |
# stop the build if there are Python syntax errors or undefined names
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
- name: Test with pytest
run: |
# Run server in background
misp-modules -l 127.0.0.1 -s &
sleep 5
# Run tests
pytest tests

View File

@ -48,7 +48,7 @@ ODTReader = { editable = true, git = "https://github.com/cartertemm/ODTReader.gi
python-pptx = "*" python-pptx = "*"
python-docx = "*" python-docx = "*"
ezodf = "*" ezodf = "*"
pandas = "*" pandas = "==1.3.5"
pandas_ods_reader = "==0.1.2" pandas_ods_reader = "==0.1.2"
pdftotext = "*" pdftotext = "*"
lxml = "*" lxml = "*"

View File

@ -80,7 +80,7 @@ olefile==0.46; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2,
oletools==0.56.2 oletools==0.56.2
opencv-python==4.5.3.56 opencv-python==4.5.3.56
pandas-ods-reader==0.1.2 pandas-ods-reader==0.1.2
pandas==1.1.5 pandas==1.3.5
passivetotal==2.5.4 passivetotal==2.5.4
pcodedmp==1.2.6 pcodedmp==1.2.6
pdftotext==2.2.0 pdftotext==2.2.0

View File

@ -41,14 +41,14 @@ try:
from .modules import * # noqa from .modules import * # noqa
HAS_PACKAGE_MODULES = True HAS_PACKAGE_MODULES = True
except Exception as e: except Exception as e:
print(e) logging.exception(e)
HAS_PACKAGE_MODULES = False HAS_PACKAGE_MODULES = False
try: try:
from .helpers import * # noqa from .helpers import * # noqa
HAS_PACKAGE_HELPERS = True HAS_PACKAGE_HELPERS = True
except Exception as e: except Exception as e:
print(e) logging.exception(e)
HAS_PACKAGE_HELPERS = False HAS_PACKAGE_HELPERS = False
log = logging.getLogger('misp-modules') log = logging.getLogger('misp-modules')

View File

@ -4,6 +4,7 @@ import np
import ezodf import ezodf
import pandas_ods_reader import pandas_ods_reader
import io import io
import logging
misperrors = {'error': 'Error'} misperrors = {'error': 'Error'}
mispattributes = {'input': ['attachment'], mispattributes = {'input': ['attachment'],
@ -35,13 +36,12 @@ def handler(q=False):
num_sheets = len(doc.sheets) num_sheets = len(doc.sheets)
try: try:
for i in range(0, num_sheets): for i in range(0, num_sheets):
ods = pandas_ods_reader.read_ods(ods_file, i, headers=False) ods = pandas_ods_reader.algo.read_data(pandas_ods_reader.parsers.ods, ods_file, i, headers=False)
ods_content = ods_content + "\n" + ods.to_string(max_rows=None) ods_content = ods_content + "\n" + ods.to_string(max_rows=None)
print(ods_content)
return {'results': [{'types': ['freetext'], 'values': ods_content, 'comment': ".ods-to-text from file " + filename}, return {'results': [{'types': ['freetext'], 'values': ods_content, 'comment': ".ods-to-text from file " + filename},
{'types': ['text'], 'values': ods_content, 'comment': ".ods-to-text from file " + filename}]} {'types': ['text'], 'values': ods_content, 'comment': ".ods-to-text from file " + filename}]}
except Exception as e: except Exception as e:
print(e) logging.exception(e)
err = "Couldn't analyze file as .ods. Error was: " + str(e) err = "Couldn't analyze file as .ods. Error was: " + str(e)
misperrors['error'] = err misperrors['error'] = err
return misperrors return misperrors

View File

@ -65,6 +65,8 @@ class TestExpansions(unittest.TestCase):
if not isinstance(data, dict): if not isinstance(data, dict):
print(json.dumps(data, indent=2)) print(json.dumps(data, indent=2))
return data return data
if 'results' not in data:
return data
for result in data['results']: for result in data['results']:
values = result['values'] values = result['values']
if values: if values:
@ -253,7 +255,7 @@ class TestExpansions(unittest.TestCase):
self.assertEqual(self.get_values(response), 'This IP is commonly spoofed in Internet-scan activity') self.assertEqual(self.get_values(response), 'This IP is commonly spoofed in Internet-scan activity')
except Exception: except Exception:
self.assertIn( self.assertIn(
self.get_errors(reponse), self.get_errors(response),
( (
"Unauthorized. Please check your API key.", "Unauthorized. Please check your API key.",
"Too many requests. You've hit the rate-limit." "Too many requests. You've hit the rate-limit."
@ -263,6 +265,7 @@ class TestExpansions(unittest.TestCase):
response = self.misp_modules_post(query) response = self.misp_modules_post(query)
self.assertEqual(self.get_errors(response), 'Missing Greynoise API key.') self.assertEqual(self.get_errors(response), 'Missing Greynoise API key.')
@unittest.skip("Service doesn't work")
def test_ipasn(self): def test_ipasn(self):
query = {"module": "ipasn", query = {"module": "ipasn",
"attribute": {"type": "ip-src", "attribute": {"type": "ip-src",
@ -301,7 +304,7 @@ class TestExpansions(unittest.TestCase):
encoded = b64encode(f.read()).decode() encoded = b64encode(f.read()).decode()
query = {"module": "ods_enrich", "attachment": filename, "data": encoded} query = {"module": "ods_enrich", "attachment": filename, "data": encoded}
response = self.misp_modules_post(query) response = self.misp_modules_post(query)
self.assertEqual(self.get_values(response), '\n column_0\n0 ods test') self.assertEqual(self.get_values(response), '\n column.0\n0 ods test')
def test_odt(self): def test_odt(self):
filename = 'test.odt' filename = 'test.odt'
@ -343,6 +346,7 @@ class TestExpansions(unittest.TestCase):
response = self.misp_modules_post(query) response = self.misp_modules_post(query)
self.assertEqual(self.get_errors(response), 'Onyphe authentication is missing') self.assertEqual(self.get_errors(response), 'Onyphe authentication is missing')
@unittest.skip("Unreliable results")
def test_otx(self): def test_otx(self):
query_types = ('domain', 'ip-src', 'md5') query_types = ('domain', 'ip-src', 'md5')
query_values = ('circl.lu', '8.8.8.8', '616eff3e9a7575ae73821b4668d2801c') query_values = ('circl.lu', '8.8.8.8', '616eff3e9a7575ae73821b4668d2801c')