From 5d848f4787e37ef9e2accf18a182e173cce97767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 26 Oct 2015 17:11:36 +0100 Subject: [PATCH] Add script for specific purposes, add testcase --- .travis.yml | 21 ++++++++++ bin/specific.py | 90 ++++++++++++++++++++++++++++++++++++++++ kittengroomer/helpers.py | 1 + setup.py | 3 +- tests/__init__.py | 0 tests/dst/.keepdir | 0 tests/src/blah.conf | 1 + tests/test.py | 22 ++++++++++ 8 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 .travis.yml create mode 100644 bin/specific.py create mode 100644 tests/__init__.py create mode 100644 tests/dst/.keepdir create mode 100644 tests/src/blah.conf create mode 100644 tests/test.py diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..c66ef5b --- /dev/null +++ b/.travis.yml @@ -0,0 +1,21 @@ +language: python + +python: + - "2.7" + - "3.3" + - "3.4" + - "3.5" + - "nightly" + +install: + - pip install --user python-magic + - pip install --user coveralls + - pip install --user codecov + - python setup.py -q install + +script: + - coverage run setup.py test + +after_success: + - coveralls + - codecov diff --git a/bin/specific.py b/bin/specific.py new file mode 100644 index 0000000..a467164 --- /dev/null +++ b/bin/specific.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import os +import magic + +from kittengroomer import FileBase, KittenGroomerBase, main + + +# Extension +configfiles = {'.conf': 'text/plain'} + + +class FileSpec(FileBase): + + def __init__(self, src_path, dst_path): + ''' Init file object, set the extension ''' + super(FileSpec, self).__init__(src_path, dst_path) + a, self.extension = os.path.splitext(self.src_path) + self.mimetype = magic.from_file(self.src_path, mime=True).decode("utf-8") + + +class KittenGroomerSpec(KittenGroomerBase): + + def __init__(self, root_src=None, root_dst=None): + ''' + Initialize the basics of the copy + ''' + if root_src is None: + root_src = os.path.join(os.sep, 'media', 'src') + if root_dst is None: + root_dst = os.path.join(os.sep, 'media', 'dst') + super(KittenGroomerSpec, self).__init__(root_src, root_dst) + self.valid_files = {} + + # The initial version will only accept the file extensions/mimetypes listed here. + self.valid_files.update(configfiles) + + def _print_log(self): + ''' + Print the logs related to the current file being processed + ''' + tmp_log = self.log_name.fields(**self.cur_file.log_details) + if not self.cur_file.log_details.get('valid'): + tmp_log.warning(self.cur_file.log_string) + else: + tmp_log.debug(self.cur_file.log_string) + + def processdir(self): + ''' + Main function doing the processing + ''' + to_copy = [] + error = [] + for srcpath in self._list_all_files(self.src_root_dir): + valid = True + self.log_name.info('Processing {}', srcpath.replace(self.src_root_dir + '/', '')) + self.cur_file = FileSpec(srcpath, srcpath.replace(self.src_root_dir, self.dst_root_dir)) + expected_mime = self.valid_files.get(self.cur_file.extension) + if expected_mime is None: + # Unexpected extension => disallowed + valid = False + compare_ext = 'Extension: {} - Expected: {}'.format(self.cur_file.extension, ', '.join(self.valid_files.keys())) + elif self.cur_file.mimetype != expected_mime: + # Unexpected mimetype => dissalowed + valid = False + compare_mime = 'Mime: {} - Expected: {}'.format(self.cur_file.mimetype, expected_mime) + self.cur_file.add_log_details('valid', valid) + if valid: + to_copy.append(self.cur_file) + self.cur_file.log_string = 'Extension: {} - MimeType: {}'.format(self.cur_file.extension, self.cur_file.mimetype) + else: + error.append(self.cur_file) + if compare_ext is not None: + self.cur_file.log_string = compare_ext + else: + self.cur_file.log_string = compare_mime + if len(error) > 0: + for f in error + to_copy: + self.cur_file = f + self._print_log() + else: + for f in to_copy: + self.cur_file = f + self._safe_copy() + self._print_log() + + +if __name__ == '__main__': + main(KittenGroomerSpec, ' Only copy some files, returns an error is anything else is found') + exit(0) diff --git a/kittengroomer/helpers.py b/kittengroomer/helpers.py index 85aec0e..5f9e5ae 100644 --- a/kittengroomer/helpers.py +++ b/kittengroomer/helpers.py @@ -89,6 +89,7 @@ class KittenGroomerBase(object): self.src_root_dir = root_src self.dst_root_dir = root_dst self.log_root_dir = os.path.join(self.dst_root_dir, 'logs') + self._safe_rmtree(self.log_root_dir) self._safe_mkdir(self.log_root_dir) self.log_processing = os.path.join(self.log_root_dir, 'processing.log') diff --git a/setup.py b/setup.py index 010a374..741efb0 100644 --- a/setup.py +++ b/setup.py @@ -11,9 +11,10 @@ setup( url='https://github.com/CIRCL/CIRCLean', description='Standalone CIRCLean/KittenGroomer code.', packages=['kittengroomer'], - scripts=['bin/generic.py', 'bin/pier9.py'], + scripts=['bin/generic.py', 'bin/pier9.py', 'bin/specific.py'], include_package_data = True, package_data = {'data': ['PDFA_def.ps','srgb.icc']}, + test_suite="tests", classifiers=[ 'License :: OSI Approved :: BSD License', 'Development Status :: 5 - Production/Stable', diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/dst/.keepdir b/tests/dst/.keepdir new file mode 100644 index 0000000..e69de29 diff --git a/tests/src/blah.conf b/tests/src/blah.conf new file mode 100644 index 0000000..484ba93 --- /dev/null +++ b/tests/src/blah.conf @@ -0,0 +1 @@ +This is a test. diff --git a/tests/test.py b/tests/test.py new file mode 100644 index 0000000..ed101de --- /dev/null +++ b/tests/test.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import unittest +import subprocess +import time + + +class TestBasic(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + + def test_specific(self): + p = subprocess.Popen(['specific.py', '-s', 'tests/src', '-d', 'tests/dst']) + while True: + p.poll() + print(p.returncode) + if p.returncode is None: + time.sleep(1) + else: + return