diff --git a/.travis.yml b/.travis.yml index 3978062..8866a29 100644 --- a/.travis.yml +++ b/.travis.yml @@ -61,7 +61,7 @@ install: - pushd theZoo/malwares/Binaries - python unpackall.py - popd - - mv theZoo/malwares/Binaries/out tests/src_complex/ + - mv theZoo/malwares/Binaries/out tests/src_invalid/ # Path traversal - git clone https://github.com/jwilk/path-traversal-samples - pushd path-traversal-samples @@ -72,12 +72,12 @@ install: - make - popd - popd - - mv path-traversal-samples/zip/*.zip tests/src_complex/ - - mv path-traversal-samples/rar/*.rar tests/src_complex/ + - mv path-traversal-samples/zip/*.zip tests/src_invalid/ + - mv path-traversal-samples/rar/*.rar tests/src_invalid/ # Office docs - git clone https://github.com/eea/odfpy.git - - mv odfpy/tests/examples/* tests/src_complex/ - - pushd tests/src_complex/ + - mv odfpy/tests/examples/* tests/src_invalid/ + - pushd tests/src_invalid/ - wget https://bitbucket.org/decalage/olefileio_pl/raw/3073963b640935134ed0da34906fea8e506460be/Tests/images/test-ole-file.doc - wget --no-check-certificate https://www.officedissector.com/corpus/fraunhoferlibrary.zip - unzip -o fraunhoferlibrary.zip diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c27f429..e514393 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,5 +29,13 @@ or if you have an example you'd like to contribute. Running the tests ================= -* Running the tests is easy. First, make sure you've installed the project and testing dependencies. -Then, run `python -m pytest` or just `pytest` in the top level or /tests directory. +* Running the tests is fairly straightforward. +* First, make sure you've installed the project and testing dependencies. +* Then, run `python -m pytest` or just `pytest` in the top level directory of the module. +* Each integration test that runs will generate a timestamped copy of the log for that run +in the tests/testlogs directory. +* If you'd like to get information about code coverage, run the tests using +`pytest --cov=kittengroomer`. +* You can test with multiple versions of Python if you have them installed +by running `pip install tox` and then `tox`. Make sure you modify "envlist" +in tox.ini for the Python versions you plan to use. diff --git a/bin/README.md b/bin/README.md index 28edb2e..b910162 100644 --- a/bin/README.md +++ b/bin/README.md @@ -15,6 +15,10 @@ Requirements per script filecheck.py ------------ +This is the script used by the [CIRCLean](https://github.com/CIRCL/Circlean) +USB key sanitizer. It is designed to handle a range of file types, and will +mark them as dangerous if they meet certain criteria. + Requirements by type of document: * Microsoft office: oletools, olefile * OOXML: officedissector @@ -23,12 +27,14 @@ Requirements by type of document: * Metadata: exifread * Images: pillow +Note: pdfid is a not installable with pip. It must be downloaded and installed +manually in the directory where filecheck will be run. ``` sudo apt-get install p7zip-full p7zip-rar libxml2-dev libxslt1-dev pip install lxml oletools olefile pillow exifread pip install git+https://github.com/Rafiot/officedissector.git - # pdfid is not a package, installing manually + # installing pdfid manually wget https://didierstevens.com/files/software/pdfid_v0_2_1.zip unzip pdfid_v0_2_1.zip ``` @@ -36,6 +42,9 @@ Requirements by type of document: generic.py ---------- +This is a script used by an older version of CIRCLean. It has more dependencies +than filecheck.py and they are more complicated to install. + Requirements by type of document: * Office and all text files: unoconv, libreoffice * PDF: ghostscript, pdf2htmlEX @@ -60,9 +69,16 @@ Requirements by type of document: pier9.py -------- +This script has a list of file formats for various brands of industrial +manufacturing equipment, such as 3d printers, CNC machines, etc. It only +copies files that match these file formats. + No external dependencies required. specific.py ----------- +As the name suggests, this script copies only specific file formats according +to the configuration provided by the user. + No external dependencies required. diff --git a/bin/filecheck.py b/bin/filecheck.py index 610de21..e9022bb 100644 --- a/bin/filecheck.py +++ b/bin/filecheck.py @@ -197,6 +197,7 @@ class KittenGroomerFileCheck(KittenGroomerBase): def _print_log(self): """Print the logs related to the current file being processed.""" + # TODO: change name to _write_log tmp_log = self.log_name.fields(**self.cur_file.log_details) if self.cur_file.is_dangerous(): tmp_log.warning(self.cur_file.log_string) diff --git a/tests/logging.py b/tests/logging.py new file mode 100644 index 0000000..e625137 --- /dev/null +++ b/tests/logging.py @@ -0,0 +1,22 @@ +import os + + +def save_logs(groomer, test_description): + divider = ('=' * 10 + '{}' + '=' * 10 + '\n') + test_log_path = 'tests/test_logs/{}.log'.format(test_description) + with open(test_log_path, 'w+') as test_log: + test_log.write(divider.format('TEST LOG')) + with open(groomer.log_processing, 'r') as logfile: + log = logfile.read() + test_log.write(log) + if groomer.debug: + if os.path.exists(groomer.log_debug_err): + test_log.write(divider.format('ERR LOG')) + with open(groomer.log_debug_err, 'r') as debug_err: + err = debug_err.read() + test_log.write(err) + if os.path.exists(groomer.log_debug_out): + test_log.write(divider.format('OUT LOG')) + with open(groomer.log_debug_out, 'r') as debug_out: + out = debug_out.read() + test_log.write(out) diff --git a/tests/src_complex/42.zip b/tests/src_complex/42.zip deleted file mode 100644 index e768153..0000000 Binary files a/tests/src_complex/42.zip and /dev/null differ diff --git a/tests/src_complex/blah.zip b/tests/src_complex/blah.zip deleted file mode 100644 index 3e809f4..0000000 Binary files a/tests/src_complex/blah.zip and /dev/null differ diff --git a/tests/src_complex/blah.conf b/tests/src_invalid/blah.conf similarity index 100% rename from tests/src_complex/blah.conf rename to tests/src_invalid/blah.conf diff --git a/tests/src_complex/blah.tar.bz2 b/tests/src_invalid/blah.tar.bz2 similarity index 100% rename from tests/src_complex/blah.tar.bz2 rename to tests/src_invalid/blah.tar.bz2 diff --git a/tests/src_complex/blah.txt b/tests/src_invalid/blah.txt similarity index 100% rename from tests/src_complex/blah.txt rename to tests/src_invalid/blah.txt diff --git a/tests/src_complex/foobar.dat b/tests/src_invalid/foobar.dat similarity index 100% rename from tests/src_complex/foobar.dat rename to tests/src_invalid/foobar.dat diff --git a/tests/src_complex/geneve_1564.pdf b/tests/src_invalid/geneve_1564.pdf similarity index 100% rename from tests/src_complex/geneve_1564.pdf rename to tests/src_invalid/geneve_1564.pdf diff --git a/tests/src_complex/geneve_1564_wrong_mime.conf b/tests/src_invalid/geneve_1564_wrong_mime.conf similarity index 100% rename from tests/src_complex/geneve_1564_wrong_mime.conf rename to tests/src_invalid/geneve_1564_wrong_mime.conf diff --git a/tests/src_complex/message.msg b/tests/src_invalid/message.msg similarity index 100% rename from tests/src_complex/message.msg rename to tests/src_invalid/message.msg diff --git a/tests/src_complex/ntree.wrl b/tests/src_invalid/ntree.wrl similarity index 100% rename from tests/src_complex/ntree.wrl rename to tests/src_invalid/ntree.wrl diff --git a/tests/src_simple/blah.conf b/tests/src_valid/blah.conf similarity index 100% rename from tests/src_simple/blah.conf rename to tests/src_valid/blah.conf diff --git a/tests/test_binaries.py b/tests/test_binaries.py deleted file mode 100644 index 38d6784..0000000 --- a/tests/test_binaries.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import os -import sys - -import pytest - -from bin.specific import KittenGroomerSpec -from bin.pier9 import KittenGroomerPier9 -from bin.generic import KittenGroomer -from bin.filecheck import KittenGroomerFileCheck - - -skip = pytest.mark.skip -py2_only = pytest.mark.skipif(sys.version_info.major == 3, - reason="filecheck.py only runs on python 2") - - -@pytest.fixture -def src_simple(): - return os.path.join(os.getcwd(), 'tests/src_simple') - - -@pytest.fixture -def src_complex(): - return os.path.join(os.getcwd(), 'tests/src_complex') - - -@pytest.fixture -def dst(): - return os.path.join(os.getcwd(), 'tests/dst') - - -def test_specific_valid(src_simple, dst): - spec = KittenGroomerSpec(src_simple, dst, debug=True) - spec.processdir() - dump_logs(spec) - - -def test_specific_invalid(src_complex, dst): - spec = KittenGroomerSpec(src_complex, dst, debug=True) - spec.processdir() - dump_logs(spec) - - -def test_pier9(src_complex, dst): - spec = KittenGroomerPier9(src_complex, dst, debug=True) - spec.processdir() - dump_logs(spec) - - -def test_generic(src_simple, dst): - spec = KittenGroomer(src_simple, dst, debug=True) - spec.processdir() - dump_logs(spec) - - -def test_generic_2(src_complex, dst): - spec = KittenGroomer(src_complex, dst, debug=True) - spec.processdir() - dump_logs(spec) - - -def test_filecheck(src_complex, dst): - spec = KittenGroomerFileCheck(src_complex, dst, debug=True) - spec.processdir() - dump_logs(spec) - - -def test_filecheck_2(src_simple, dst): - spec = KittenGroomerFileCheck(src_simple, dst, debug=True) - spec.processdir() - dump_logs(spec) - -## Helper functions - -def dump_logs(spec): - print(open(spec.log_processing, 'rb').read()) - if spec.debug: - if os.path.exists(spec.log_debug_err): - print(open(spec.log_debug_err, 'rb').read()) - if os.path.exists(spec.log_debug_out): - print(open(spec.log_debug_out, 'rb').read()) diff --git a/tests/test_filecheck.py b/tests/test_filecheck.py index a4dc481..ac7cf42 100644 --- a/tests/test_filecheck.py +++ b/tests/test_filecheck.py @@ -1,16 +1,48 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import os + import pytest -from bin.filecheck import KittenGroomerFileCheck, File, main +from tests.logging import save_logs +try: + from bin.filecheck import KittenGroomerFileCheck, File, main + NODEPS = False +except ImportError: + NODEPS = True + +skipif_nodeps = pytest.mark.skipif(NODEPS, + reason="Dependencies aren't installed") + + +@skipif_nodeps +class TestIntegration: + + @pytest.fixture + def src_valid(self): + return os.path.join(os.getcwd(), 'tests/src_valid') + + @pytest.fixture + def src_invalid(self): + return os.path.join(os.getcwd(), 'tests/src_invalid') + + @pytest.fixture + def dst(self): + return os.path.join(os.getcwd(), 'tests/dst') + + def test_filecheck(self, src_invalid, dst): + groomer = KittenGroomerFileCheck(src_invalid, dst, debug=True) + groomer.processdir() + test_description = "filecheck_invalid" + save_logs(groomer, test_description) + + def test_filecheck_2(self, src_valid, dst): + groomer = KittenGroomerFileCheck(src_valid, dst, debug=True) + groomer.processdir() + test_description = "filecheck_valid" + save_logs(groomer, test_description) class TestFileHandling: pass - - # We're going to give KittenGroomer a bunch of files, and it's going to process them - # Maybe we want to make a function that processdir delegates to? Or is it just the File Object that's responsible? - # Ideally we should be able to pass a path to a function and have it do stuff? And then we can test that function? - # So we have a function that takes a path and returns...log info? That makes sense actually. Or some sort of meta data - # The function could maybe be called processfile diff --git a/tests/test_generic.py b/tests/test_generic.py new file mode 100644 index 0000000..a17fb5e --- /dev/null +++ b/tests/test_generic.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os + +import pytest + +from bin.generic import KittenGroomer, File, main +from tests.logging import save_logs + +skipif_nodeps = pytest.mark.skipif(os.path.exists('/usr/bin/unoconv') is False, + reason="Dependencies aren't installed") + + +@skipif_nodeps +class TestIntegration: + + @pytest.fixture + def src_valid(self): + return os.path.join(os.getcwd(), 'tests/src_valid') + + @pytest.fixture + def src_invalid(self): + return os.path.join(os.getcwd(), 'tests/src_invalid') + + @pytest.fixture + def dst(self): + return os.path.join(os.getcwd(), 'tests/dst') + + def test_generic(self, src_valid, dst): + groomer = KittenGroomer(src_valid, dst, debug=True) + groomer.processdir() + test_description = 'generic_valid' + save_logs(groomer, test_description) + + def test_generic_2(self, src_invalid, dst): + groomer = KittenGroomer(src_invalid, dst, debug=True) + groomer.processdir() + test_description = 'generic_invalid' + save_logs(groomer, test_description) + + +class TestFileHandling: + pass + + # We're going to give KittenGroomer a bunch of files, and it's going to process them + # Maybe we want to make a function that processdir delegates to? Or is it just the File Object that's responsible? + # Ideally we should be able to pass a path to a function and have it do stuff? And then we can test that function? + # So we have a function that takes a path and returns...log info? That makes sense actually. Or some sort of meta data + # The function could maybe be called processfile diff --git a/tests/test_helpers.py b/tests/test_helpers.py index a14510a..f59c862 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -21,7 +21,7 @@ class TestFileBase: @fixture def source_file(self): - return 'tests/src_simple/blah.conf' + return 'tests/src_valid/blah.conf' @fixture def dest_file(self): @@ -84,7 +84,7 @@ class TestFileBase: # We should probably catch everytime that happens and tell the user explicitly happened (and maybe put it in the log) def test_create(self): - file = FileBase('tests/src_simple/blah.conf', '/tests/dst/blah.conf') + file = FileBase('tests/src_valid/blah.conf', '/tests/dst/blah.conf') def test_create_broken(self, tmpdir): with pytest.raises(TypeError): @@ -221,7 +221,7 @@ class TestKittenGroomerBase: @fixture def source_directory(self): - return 'tests/src_complex' + return 'tests/src_invalid' @fixture def dest_directory(self): diff --git a/tests/test_logs/.keepdir b/tests/test_logs/.keepdir new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_specific_and_pier9.py b/tests/test_specific_and_pier9.py new file mode 100644 index 0000000..d411aa8 --- /dev/null +++ b/tests/test_specific_and_pier9.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os + +import pytest + +from bin.specific import KittenGroomerSpec +from bin.pier9 import KittenGroomerPier9 +from tests.logging import save_logs + + +@pytest.fixture +def src_valid(): + return os.path.join(os.getcwd(), 'tests/src_valid') + + +@pytest.fixture +def src_invalid(): + return os.path.join(os.getcwd(), 'tests/src_invalid') + + +@pytest.fixture +def dst(): + return os.path.join(os.getcwd(), 'tests/dst') + + +def test_specific_valid(src_valid, dst): + groomer = KittenGroomerSpec(src_valid, dst, debug=True) + groomer.processdir() + test_description = 'specific_valid' + save_logs(groomer, test_description) + + +def test_specific_invalid(src_invalid, dst): + groomer = KittenGroomerSpec(src_invalid, dst, debug=True) + groomer.processdir() + test_description = 'specific_invalid' + save_logs(groomer, test_description) + + +def test_pier9_valid(src_invalid, dst): + groomer = KittenGroomerPier9(src_invalid, dst, debug=True) + groomer.processdir() + test_description = 'pier9_valid' + save_logs(groomer, test_description) + + +def test_pier9_invalid(src_invalid, dst): + groomer = KittenGroomerPier9(src_invalid, dst, debug=True) + groomer.processdir() + test_description = 'pier9_invalid' + save_logs(groomer, test_description) diff --git a/tox.ini b/tox.ini index da1ed31..40b9ad6 100644 --- a/tox.ini +++ b/tox.ini @@ -2,4 +2,4 @@ envlist=py27,py35 [testenv] deps=-rdev-requirements.txt -commands= pytest tests/test_helpers.py --cov=kittengroomer +commands= pytest --cov=kittengroomer --cov=bin