mirror of https://github.com/CIRCL/PyCIRCLean
Move default log setup back into filecheck
* Realized that the API consumer might want to write their own logging tool. * FileBase and KittenGroomerBase will have no logging code. * If the API consumer likes, they can import GroomerLogger and use it in their implementation.pull/14/head
parent
18857c8cf7
commit
51760ebbb1
|
@ -16,7 +16,7 @@ from PIL import Image
|
||||||
# from PIL import PngImagePlugin
|
# from PIL import PngImagePlugin
|
||||||
from pdfid import PDFiD, cPDFiD
|
from pdfid import PDFiD, cPDFiD
|
||||||
|
|
||||||
from kittengroomer import FileBase, KittenGroomerBase
|
from kittengroomer import FileBase, KittenGroomerBase, GroomerLogger
|
||||||
|
|
||||||
|
|
||||||
SEVENZ_PATH = '/usr/bin/7z'
|
SEVENZ_PATH = '/usr/bin/7z'
|
||||||
|
@ -86,8 +86,9 @@ class Config:
|
||||||
class File(FileBase):
|
class File(FileBase):
|
||||||
|
|
||||||
def __init__(self, src_path, dst_path, logger):
|
def __init__(self, src_path, dst_path, logger):
|
||||||
super(File, self).__init__(src_path, dst_path, logger)
|
super(File, self).__init__(src_path, dst_path)
|
||||||
self.is_recursive = False
|
self.is_recursive = False
|
||||||
|
self.logger = logger
|
||||||
|
|
||||||
subtypes_apps = [
|
subtypes_apps = [
|
||||||
(Config.mimes_office, self._winoffice),
|
(Config.mimes_office, self._winoffice),
|
||||||
|
@ -170,7 +171,7 @@ class File(FileBase):
|
||||||
|
|
||||||
def _check_filename(self):
|
def _check_filename(self):
|
||||||
if self.filename[0] is '.':
|
if self.filename[0] is '.':
|
||||||
# handle dotfiles
|
# TODO: handle dotfiles?
|
||||||
pass
|
pass
|
||||||
right_to_left_override = u"\u202E"
|
right_to_left_override = u"\u202E"
|
||||||
if right_to_left_override in self.filename:
|
if right_to_left_override in self.filename:
|
||||||
|
@ -188,6 +189,9 @@ class File(FileBase):
|
||||||
if not self.is_dangerous:
|
if not self.is_dangerous:
|
||||||
self.mime_processing_options.get(self.main_type, self.unknown)()
|
self.mime_processing_options.get(self.main_type, self.unknown)()
|
||||||
|
|
||||||
|
def write_log(self):
|
||||||
|
pass
|
||||||
|
|
||||||
# ##### Helper functions #####
|
# ##### Helper functions #####
|
||||||
def _make_method_dict(self, list_of_tuples):
|
def _make_method_dict(self, list_of_tuples):
|
||||||
"""Returns a dictionary with mimetype: method pairs."""
|
"""Returns a dictionary with mimetype: method pairs."""
|
||||||
|
@ -500,9 +504,11 @@ class File(FileBase):
|
||||||
class KittenGroomerFileCheck(KittenGroomerBase):
|
class KittenGroomerFileCheck(KittenGroomerBase):
|
||||||
|
|
||||||
def __init__(self, root_src, root_dst, max_recursive_depth=2, debug=False):
|
def __init__(self, root_src, root_dst, max_recursive_depth=2, debug=False):
|
||||||
super(KittenGroomerFileCheck, self).__init__(root_src, root_dst, debug)
|
super(KittenGroomerFileCheck, self).__init__(root_src, root_dst)
|
||||||
self.recursive_archive_depth = 0
|
self.recursive_archive_depth = 0
|
||||||
self.max_recursive_depth = max_recursive_depth
|
self.max_recursive_depth = max_recursive_depth
|
||||||
|
self.cur_file = None
|
||||||
|
self.logger = GroomerLogger(self.dst_root_dir, debug)
|
||||||
|
|
||||||
def process_dir(self, src_dir, dst_dir):
|
def process_dir(self, src_dir, dst_dir):
|
||||||
"""Process a directory on the source key."""
|
"""Process a directory on the source key."""
|
||||||
|
|
|
@ -37,7 +37,7 @@ class FileBase(object):
|
||||||
Contains file attributes and various helper methods.
|
Contains file attributes and various helper methods.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, src_path, dst_path, logger=None):
|
def __init__(self, src_path, dst_path):
|
||||||
"""
|
"""
|
||||||
Initialized with the source path and expected destination path.
|
Initialized with the source path and expected destination path.
|
||||||
|
|
||||||
|
@ -47,7 +47,6 @@ class FileBase(object):
|
||||||
self.src_path = src_path
|
self.src_path = src_path
|
||||||
self.dst_path = dst_path
|
self.dst_path = dst_path
|
||||||
self.filename = os.path.basename(self.src_path)
|
self.filename = os.path.basename(self.src_path)
|
||||||
self.logger = logger
|
|
||||||
self._file_props = {
|
self._file_props = {
|
||||||
'filepath': self.src_path,
|
'filepath': self.src_path,
|
||||||
'filename': self.filename,
|
'filename': self.filename,
|
||||||
|
@ -261,11 +260,6 @@ class FileBase(object):
|
||||||
self.add_error(e, '')
|
self.add_error(e, '')
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def write_log(self):
|
|
||||||
"""Write logs from file to self.logger."""
|
|
||||||
file_log = self.logger.add_file(self)
|
|
||||||
file_log.fields(**self._file_props)
|
|
||||||
|
|
||||||
|
|
||||||
class GroomerLogger(object):
|
class GroomerLogger(object):
|
||||||
"""Groomer logging interface."""
|
"""Groomer logging interface."""
|
||||||
|
@ -320,19 +314,16 @@ class GroomerLogger(object):
|
||||||
|
|
||||||
def add_file(self, file):
|
def add_file(self, file):
|
||||||
"""Add a file to the log."""
|
"""Add a file to the log."""
|
||||||
return self.log.name('file.src_path')
|
pass
|
||||||
|
|
||||||
|
|
||||||
class KittenGroomerBase(object):
|
class KittenGroomerBase(object):
|
||||||
"""Base object responsible for copy/sanitization process."""
|
"""Base object responsible for copy/sanitization process."""
|
||||||
|
|
||||||
def __init__(self, src_root_dir, dst_root_dir, debug=False):
|
def __init__(self, src_root_dir, dst_root_dir):
|
||||||
"""Initialized with path to source and dest directories."""
|
"""Initialized with path to source and dest directories."""
|
||||||
self.src_root_dir = src_root_dir
|
self.src_root_dir = src_root_dir
|
||||||
self.dst_root_dir = dst_root_dir
|
self.dst_root_dir = dst_root_dir
|
||||||
self.debug = debug
|
|
||||||
self.cur_file = None
|
|
||||||
self.logger = GroomerLogger(self.dst_root_dir, debug)
|
|
||||||
|
|
||||||
def safe_rmtree(self, directory):
|
def safe_rmtree(self, directory):
|
||||||
"""Remove a directory tree if it exists."""
|
"""Remove a directory tree if it exists."""
|
||||||
|
|
|
@ -9,14 +9,13 @@ def save_logs(groomer, test_description):
|
||||||
with open(groomer.logger.log_processing, 'r') as logfile:
|
with open(groomer.logger.log_processing, 'r') as logfile:
|
||||||
log = logfile.read()
|
log = logfile.read()
|
||||||
test_log.write(log)
|
test_log.write(log)
|
||||||
if groomer.debug:
|
if os.path.exists(groomer.logger.log_debug_err):
|
||||||
if os.path.exists(groomer.logger.log_debug_err):
|
test_log.write(divider.format('ERR LOG'))
|
||||||
test_log.write(divider.format('ERR LOG'))
|
with open(groomer.logger.log_debug_err, 'r') as debug_err:
|
||||||
with open(groomer.logger.log_debug_err, 'r') as debug_err:
|
err = debug_err.read()
|
||||||
err = debug_err.read()
|
test_log.write(err)
|
||||||
test_log.write(err)
|
if os.path.exists(groomer.logger.log_debug_out):
|
||||||
if os.path.exists(groomer.logger.log_debug_out):
|
test_log.write(divider.format('OUT LOG'))
|
||||||
test_log.write(divider.format('OUT LOG'))
|
with open(groomer.logger.log_debug_out, 'r') as debug_out:
|
||||||
with open(groomer.logger.log_debug_out, 'r') as debug_out:
|
out = debug_out.read()
|
||||||
out = debug_out.read()
|
test_log.write(out)
|
||||||
test_log.write(out)
|
|
||||||
|
|
|
@ -6,7 +6,6 @@ import os
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from kittengroomer import FileBase, KittenGroomerBase, GroomerLogger
|
from kittengroomer import FileBase, KittenGroomerBase, GroomerLogger
|
||||||
from kittengroomer.helpers import ImplementationRequired
|
|
||||||
|
|
||||||
skip = pytest.mark.skip
|
skip = pytest.mark.skip
|
||||||
xfail = pytest.mark.xfail
|
xfail = pytest.mark.xfail
|
||||||
|
@ -245,10 +244,7 @@ class TestKittenGroomerBase:
|
||||||
assert generic_groomer
|
assert generic_groomer
|
||||||
|
|
||||||
def test_instantiation(self, source_directory, dest_directory):
|
def test_instantiation(self, source_directory, dest_directory):
|
||||||
groomer = KittenGroomerBase(source_directory, dest_directory)
|
KittenGroomerBase(source_directory, dest_directory)
|
||||||
debug_groomer = KittenGroomerBase(source_directory,
|
|
||||||
dest_directory,
|
|
||||||
debug=True)
|
|
||||||
|
|
||||||
def test_list_all_files(self, tmpdir):
|
def test_list_all_files(self, tmpdir):
|
||||||
file = tmpdir.join('test.txt')
|
file = tmpdir.join('test.txt')
|
||||||
|
|
Loading…
Reference in New Issue