2015-11-02 18:00:33 +01:00
|
|
|
#!/usr/bin/env python
|
|
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import os
|
|
|
|
import mimetypes
|
|
|
|
import shlex
|
|
|
|
import subprocess
|
2015-11-03 15:30:59 +01:00
|
|
|
import zipfile
|
2017-02-23 23:39:16 +01:00
|
|
|
import argparse
|
2017-12-20 16:15:30 +01:00
|
|
|
import random
|
2017-04-10 13:07:21 +02:00
|
|
|
import shutil
|
2017-12-20 16:15:30 +01:00
|
|
|
import time
|
|
|
|
import hashlib
|
2015-11-02 18:00:33 +01:00
|
|
|
|
|
|
|
import oletools.oleid
|
|
|
|
import olefile
|
|
|
|
import officedissector
|
2015-12-10 02:26:26 +01:00
|
|
|
import warnings
|
2015-12-15 23:40:25 +01:00
|
|
|
import exifread
|
2015-12-10 02:26:26 +01:00
|
|
|
from PIL import Image
|
2015-11-02 18:00:33 +01:00
|
|
|
from pdfid import PDFiD, cPDFiD
|
|
|
|
|
2017-04-10 13:07:21 +02:00
|
|
|
from kittengroomer import FileBase, KittenGroomerBase, Logging
|
2017-08-26 02:19:36 +02:00
|
|
|
|
|
|
|
|
|
|
|
class Config:
|
|
|
|
"""Configuration information for filecheck.py."""
|
|
|
|
# MIMES
|
|
|
|
# Application subtypes (mimetype: 'application/<subtype>')
|
|
|
|
mimes_ooxml = ('vnd.openxmlformats-officedocument.',)
|
|
|
|
mimes_office = ('msword', 'vnd.ms-',)
|
|
|
|
mimes_libreoffice = ('vnd.oasis.opendocument',)
|
|
|
|
mimes_rtf = ('rtf', 'richtext',)
|
|
|
|
mimes_pdf = ('pdf', 'postscript',)
|
|
|
|
mimes_xml = ('xml',)
|
|
|
|
mimes_ms = ('dosexec',)
|
|
|
|
mimes_compressed = ('zip', 'rar', 'x-rar', 'bzip2', 'lzip', 'lzma', 'lzop',
|
|
|
|
'xz', 'compress', 'gzip', 'tar',)
|
|
|
|
mimes_data = ('octet-stream',)
|
|
|
|
mimes_audio = ('ogg',)
|
|
|
|
|
|
|
|
# Image subtypes
|
|
|
|
mimes_exif = ('image/jpeg', 'image/tiff',)
|
|
|
|
mimes_png = ('image/png',)
|
|
|
|
|
|
|
|
# Mimetypes with metadata
|
|
|
|
mimes_metadata = ('image/jpeg', 'image/tiff', 'image/png',)
|
|
|
|
|
|
|
|
# Mimetype aliases
|
|
|
|
aliases = {
|
|
|
|
# Win executables
|
|
|
|
'application/x-msdos-program': 'application/x-dosexec',
|
|
|
|
'application/x-dosexec': 'application/x-msdos-program',
|
|
|
|
# Other apps with confusing mimetypes
|
|
|
|
'application/rtf': 'text/rtf',
|
|
|
|
'application/rar': 'application/x-rar',
|
|
|
|
'application/ogg': 'audio/ogg',
|
|
|
|
'audio/ogg': 'application/ogg'
|
|
|
|
}
|
|
|
|
|
|
|
|
# EXTS
|
|
|
|
# Commonly used malicious extensions
|
|
|
|
# Sources: http://www.howtogeek.com/137270/50-file-extensions-that-are-potentially-dangerous-on-windows/
|
|
|
|
# https://github.com/wiregit/wirecode/blob/master/components/core-settings/src/main/java/org/limewire/core/settings/FilterSettings.java
|
|
|
|
malicious_exts = (
|
|
|
|
# Applications
|
|
|
|
".exe", ".pif", ".application", ".gadget", ".msi", ".msp", ".com", ".scr",
|
|
|
|
".hta", ".cpl", ".msc", ".jar",
|
|
|
|
# Scripts
|
|
|
|
".bat", ".cmd", ".vb", ".vbs", ".vbe", ".js", ".jse", ".ws", ".wsf",
|
|
|
|
".wsc", ".wsh", ".ps1", ".ps1xml", ".ps2", ".ps2xml", ".psc1", ".psc2",
|
|
|
|
".msh", ".msh1", ".msh2", ".mshxml", ".msh1xml", ".msh2xml",
|
|
|
|
# Shortcuts
|
|
|
|
".scf", ".lnk", ".inf",
|
|
|
|
# Other
|
|
|
|
".reg", ".dll",
|
|
|
|
# Office macro (OOXML with macro enabled)
|
|
|
|
".docm", ".dotm", ".xlsm", ".xltm", ".xlam", ".pptm", ".potm", ".ppam",
|
|
|
|
".ppsm", ".sldm",
|
|
|
|
# banned from wirecode
|
|
|
|
".asf", ".asx", ".au", ".htm", ".html", ".mht", ".vbs",
|
|
|
|
".wax", ".wm", ".wma", ".wmd", ".wmv", ".wmx", ".wmz", ".wvx",
|
|
|
|
# Google chrome malicious extensions
|
|
|
|
".ad", ".ade", ".adp", ".ah", ".apk", ".app", ".application", ".asp",
|
|
|
|
".asx", ".bas", ".bash", ".bat", ".cfg", ".chi", ".chm", ".class",
|
|
|
|
".cmd", ".com", ".command", ".crt", ".crx", ".csh", ".deb", ".dex",
|
|
|
|
".dll", ".drv", ".exe", ".fxp", ".grp", ".hlp", ".hta", ".htm", ".html",
|
|
|
|
".htt", ".inf", ".ini", ".ins", ".isp", ".jar", ".jnlp", ".user.js",
|
|
|
|
".js", ".jse", ".ksh", ".lnk", ".local", ".mad", ".maf", ".mag", ".mam",
|
|
|
|
".manifest", ".maq", ".mar", ".mas", ".mat", ".mau", ".mav", ".maw",
|
|
|
|
".mda", ".mdb", ".mde", ".mdt", ".mdw", ".mdz", ".mht", ".mhtml", ".mmc",
|
|
|
|
".mof", ".msc", ".msh", ".mshxml", ".msi", ".msp", ".mst", ".ocx", ".ops",
|
|
|
|
".pcd", ".pif", ".pkg", ".pl", ".plg", ".prf", ".prg", ".pst", ".py",
|
|
|
|
".pyc", ".pyw", ".rb", ".reg", ".rpm", ".scf", ".scr", ".sct", ".sh",
|
|
|
|
".shar", ".shb", ".shs", ".shtm", ".shtml", ".spl", ".svg", ".swf", ".sys",
|
|
|
|
".tcsh", ".url", ".vb", ".vbe", ".vbs", ".vsd", ".vsmacros", ".vss",
|
|
|
|
".vst", ".vsw", ".ws", ".wsc", ".wsf", ".wsh", ".xbap", ".xht", ".xhtm",
|
|
|
|
".xhtml", ".xml", ".xsl", ".xslt", ".website", ".msh1", ".msh2", ".msh1xml",
|
|
|
|
".msh2xml", ".ps1", ".ps1xml", ".ps2", ".ps2xml", ".psc1", ".psc2", ".xnk",
|
|
|
|
".appref-ms", ".gadget", ".efi", ".fon", ".partial", ".svg", ".xml",
|
|
|
|
".xrm_ms", ".xsl", ".action", ".bin", ".inx", ".ipa", ".isu", ".job",
|
|
|
|
".out", ".pad", ".paf", ".rgs", ".u3p", ".vbscript", ".workflow", ".001",
|
|
|
|
".ace", ".arc", ".arj", ".b64", ".balz", ".bhx", ".cab", ".cpio", ".fat",
|
|
|
|
".hfs", ".hqx", ".iso", ".lha", ".lpaq1", ".lpaq5", ".lpaq8", ".lzh",
|
|
|
|
".mim", ".ntfs", ".paq8f", ".paq8jd", ".paq8l", ".paq8o", ".pea", ".quad",
|
|
|
|
".r00", ".r01", ".r02", ".r03", ".r04", ".r05", ".r06", ".r07", ".r08",
|
|
|
|
".r09", ".r10", ".r11", ".r12", ".r13", ".r14", ".r15", ".r16", ".r17",
|
|
|
|
".r18", ".r19", ".r20", ".r21", ".r22", ".r23", ".r24", ".r25", ".r26",
|
|
|
|
".r27", ".r28", ".r29", ".squashfs", ".swm", ".tpz", ".txz", ".tz", ".udf",
|
|
|
|
".uu", ".uue", ".vhd", ".vmdk", ".wim", ".wrc", ".xar", ".xxe", ".z",
|
|
|
|
".zipx", ".zpaq", ".cdr", ".dart", ".dc42", ".diskcopy42", ".dmg",
|
|
|
|
".dmgpart", ".dvdr", ".img", ".imgpart", ".ndif", ".smi", ".sparsebundle",
|
|
|
|
".sparseimage", ".toast", ".udif",
|
|
|
|
)
|
|
|
|
|
|
|
|
# Sometimes, mimetypes.guess_type gives unexpected results, such as for .tar.gz files:
|
|
|
|
# In [12]: mimetypes.guess_type('toot.tar.gz', strict=False)
|
|
|
|
# Out[12]: ('application/x-tar', 'gzip')
|
|
|
|
# It works as expected if you do mimetypes.guess_type('application/gzip', strict=False)
|
|
|
|
override_ext = {'.gz': 'application/gzip'}
|
2015-11-02 18:00:33 +01:00
|
|
|
|
2017-02-22 17:29:20 +01:00
|
|
|
|
2016-12-16 23:18:53 +01:00
|
|
|
SEVENZ_PATH = '/usr/bin/7z'
|
2015-11-02 18:00:33 +01:00
|
|
|
|
|
|
|
|
|
|
|
class File(FileBase):
|
2017-04-18 00:22:14 +02:00
|
|
|
"""
|
|
|
|
Main file object
|
|
|
|
|
|
|
|
Created for each file that is processed by KittenGroomer. Contains all
|
|
|
|
filetype-specific processing methods.
|
|
|
|
"""
|
2015-11-02 18:00:33 +01:00
|
|
|
|
2017-07-27 19:27:12 +02:00
|
|
|
def __init__(self, src_path, dst_path):
|
2017-03-18 03:19:56 +01:00
|
|
|
super(File, self).__init__(src_path, dst_path)
|
2017-04-17 22:08:55 +02:00
|
|
|
self.is_archive = False
|
2017-04-10 13:07:21 +02:00
|
|
|
self.tempdir_path = self.dst_path + '_temp'
|
2016-12-16 23:18:53 +01:00
|
|
|
|
2017-07-12 23:58:39 +02:00
|
|
|
subtypes_apps = (
|
2017-02-27 20:22:24 +01:00
|
|
|
(Config.mimes_office, self._winoffice),
|
|
|
|
(Config.mimes_ooxml, self._ooxml),
|
|
|
|
(Config.mimes_rtf, self.text),
|
|
|
|
(Config.mimes_libreoffice, self._libreoffice),
|
|
|
|
(Config.mimes_pdf, self._pdf),
|
|
|
|
(Config.mimes_xml, self.text),
|
|
|
|
(Config.mimes_ms, self._executables),
|
|
|
|
(Config.mimes_compressed, self._archive),
|
|
|
|
(Config.mimes_data, self._binary_app),
|
2017-08-22 23:25:41 +02:00
|
|
|
(Config.mimes_audio, self.audio)
|
2017-07-12 23:58:39 +02:00
|
|
|
)
|
2017-02-27 20:22:24 +01:00
|
|
|
self.app_subtype_methods = self._make_method_dict(subtypes_apps)
|
|
|
|
|
2017-07-12 23:58:39 +02:00
|
|
|
types_metadata = (
|
2017-02-27 20:22:24 +01:00
|
|
|
(Config.mimes_exif, self._metadata_exif),
|
|
|
|
(Config.mimes_png, self._metadata_png),
|
2017-07-12 23:58:39 +02:00
|
|
|
)
|
2017-02-27 20:22:24 +01:00
|
|
|
self.metadata_mimetype_methods = self._make_method_dict(types_metadata)
|
|
|
|
|
|
|
|
self.mime_processing_options = {
|
|
|
|
'text': self.text,
|
|
|
|
'audio': self.audio,
|
|
|
|
'image': self.image,
|
|
|
|
'video': self.video,
|
|
|
|
'application': self.application,
|
|
|
|
'example': self.example,
|
|
|
|
'message': self.message,
|
|
|
|
'model': self.model,
|
|
|
|
'multipart': self.multipart,
|
|
|
|
'inode': self.inode,
|
|
|
|
}
|
|
|
|
|
2017-07-20 23:47:23 +02:00
|
|
|
def __repr__(self):
|
|
|
|
return "<filecheck.File object: {{{}}}>".format(self.filename)
|
|
|
|
|
2016-12-16 23:18:53 +01:00
|
|
|
def _check_extension(self):
|
2017-03-16 17:22:26 +01:00
|
|
|
"""
|
|
|
|
Guess the file's mimetype based on its extension.
|
|
|
|
|
|
|
|
If the file's mimetype (as determined by libmagic) is contained in
|
|
|
|
the `mimetype` module's list of valid mimetypes and the expected
|
|
|
|
mimetype based on its extension differs from the mimetype determined
|
|
|
|
by libmagic, then mark the file as dangerous.
|
|
|
|
"""
|
2017-07-25 20:32:40 +02:00
|
|
|
if not self.has_extension:
|
|
|
|
self.make_dangerous('File has no extension')
|
2015-11-02 18:00:33 +01:00
|
|
|
else:
|
2017-07-25 20:32:40 +02:00
|
|
|
if self.extension in Config.override_ext:
|
|
|
|
expected_mimetype = Config.override_ext[self.extension]
|
|
|
|
else:
|
|
|
|
expected_mimetype, encoding = mimetypes.guess_type(self.src_path,
|
|
|
|
strict=False)
|
|
|
|
if expected_mimetype in Config.aliases:
|
|
|
|
expected_mimetype = Config.aliases[expected_mimetype]
|
|
|
|
is_known_extension = self.extension in mimetypes.types_map.keys()
|
|
|
|
if is_known_extension and expected_mimetype != self.mimetype:
|
2017-08-12 00:41:00 +02:00
|
|
|
self.make_dangerous('Mimetype does not match expected mimetype ({}) for this extension'.format(expected_mimetype))
|
2015-11-02 18:00:33 +01:00
|
|
|
|
2017-03-01 21:24:48 +01:00
|
|
|
def _check_mimetype(self):
|
2017-03-16 17:22:26 +01:00
|
|
|
"""
|
|
|
|
Compare mimetype (as determined by libmagic) to extension.
|
|
|
|
|
|
|
|
Determine whether the extension that are normally associated with
|
|
|
|
the mimetype include the file's actual extension.
|
|
|
|
"""
|
2017-07-25 20:32:40 +02:00
|
|
|
if not self.has_mimetype:
|
|
|
|
self.make_dangerous('File has no mimetype')
|
2015-11-05 14:43:54 +01:00
|
|
|
else:
|
2017-07-25 20:32:40 +02:00
|
|
|
if self.mimetype in Config.aliases:
|
|
|
|
mimetype = Config.aliases[self.mimetype]
|
|
|
|
else:
|
|
|
|
mimetype = self.mimetype
|
|
|
|
expected_extensions = mimetypes.guess_all_extensions(mimetype,
|
|
|
|
strict=False)
|
|
|
|
if expected_extensions:
|
|
|
|
if self.has_extension and self.extension not in expected_extensions:
|
2017-08-22 22:41:06 +02:00
|
|
|
self.make_dangerous('Extension does not match expected extensions ({}) for this mimetype'.format(expected_extensions))
|
2015-11-02 18:00:33 +01:00
|
|
|
|
2017-03-16 17:22:26 +01:00
|
|
|
def _check_filename(self):
|
2017-04-18 00:22:14 +02:00
|
|
|
"""
|
|
|
|
Verify the filename
|
|
|
|
|
|
|
|
If the filename contains any dangerous or specific characters, handle
|
|
|
|
them appropriately.
|
|
|
|
"""
|
2017-07-17 16:22:46 +02:00
|
|
|
if self.filename.startswith('.'):
|
|
|
|
macos_hidden_files = set(
|
|
|
|
'.Trashes', '._.Trashes', '.DS_Store', '.fseventsd', '.Spotlight-V100'
|
|
|
|
)
|
|
|
|
if self.filename in macos_hidden_files:
|
2017-08-07 23:17:59 +02:00
|
|
|
self.add_description('MacOS metadata file, added by MacOS to USB drives and some directories')
|
2017-07-17 16:22:46 +02:00
|
|
|
self.should_copy = False
|
2017-03-16 17:22:26 +01:00
|
|
|
right_to_left_override = u"\u202E"
|
|
|
|
if right_to_left_override in self.filename:
|
|
|
|
self.make_dangerous('Filename contains dangerous character')
|
2017-07-17 16:22:46 +02:00
|
|
|
new_filename = self.filename.replace(right_to_left_override, '')
|
|
|
|
self.set_property('filename', new_filename)
|
2017-03-16 17:22:26 +01:00
|
|
|
|
2017-07-25 20:32:40 +02:00
|
|
|
def _check_malicious_exts(self):
|
|
|
|
"""Check that the file's extension isn't contained in a blacklist"""
|
|
|
|
if self.extension in Config.malicious_exts:
|
|
|
|
self.make_dangerous('Extension identifies file as potentially dangerous')
|
|
|
|
|
2017-12-20 16:15:30 +01:00
|
|
|
def _compute_random_hashes(self):
|
2017-12-21 13:29:09 +01:00
|
|
|
"""Compute a random amount of hashes at random positions in the file to ensure integrity after the copy (mitigate TOCTOU attacks)"""
|
2017-12-21 13:47:46 +01:00
|
|
|
if not os.path.exists(self.src_path) or os.path.isdir(self.src_path) or self.maintype == 'image':
|
2017-12-21 13:29:09 +01:00
|
|
|
# Images are converted, no need to compute the hashes
|
|
|
|
return
|
2017-12-20 16:15:30 +01:00
|
|
|
self.random_hashes = []
|
|
|
|
if self.size < 64:
|
|
|
|
# hash the whole file
|
|
|
|
self.block_length = self.size
|
|
|
|
else:
|
|
|
|
if self.size < 128:
|
|
|
|
# Get a random length between 16 and the size of the file
|
|
|
|
self.block_length = random.randint(16, self.size)
|
|
|
|
else:
|
|
|
|
# Get a random length between 16 and 128
|
|
|
|
self.block_length = random.randint(16, 128)
|
|
|
|
|
|
|
|
for i in range(random.randint(3, 6)): # Do a random amound of read on the file (between 5 and 10)
|
|
|
|
start_pos = random.randint(0, self.size - self.block_length) # Pick a random length for the hash to compute
|
|
|
|
with open(self.src_path, 'rb') as f:
|
|
|
|
f.seek(start_pos)
|
|
|
|
hashed = hashlib.sha256(f.read(self.block_length)).hexdigest()
|
|
|
|
self.random_hashes.append((start_pos, hashed))
|
|
|
|
time.sleep(random.uniform(0.1, 0.5)) # Add a random sleep length
|
|
|
|
|
|
|
|
def _validate_random_hashes(self):
|
2017-12-21 13:29:09 +01:00
|
|
|
"""Validate hashes computed by _compute_random_hashes"""
|
2017-12-21 13:47:46 +01:00
|
|
|
if not os.path.exists(self.src_path) or os.path.isdir(self.src_path) or self.maintype == 'image':
|
2017-12-21 13:29:09 +01:00
|
|
|
# Images are converted, we don't have to fear TOCTOU
|
|
|
|
return True
|
2017-12-20 16:15:30 +01:00
|
|
|
for start_pos, hashed_src in self.random_hashes:
|
|
|
|
with open(self.dst_path, 'rb') as f:
|
|
|
|
f.seek(start_pos)
|
|
|
|
hashed = hashlib.sha256(f.read(self.block_length)).hexdigest()
|
|
|
|
if hashed != hashed_src:
|
|
|
|
# Something fucked up happened
|
|
|
|
return False
|
|
|
|
return True
|
|
|
|
|
2017-02-27 20:22:24 +01:00
|
|
|
def check(self):
|
2017-04-18 00:22:14 +02:00
|
|
|
"""
|
2017-07-25 20:32:40 +02:00
|
|
|
Main file processing method.
|
2017-04-18 00:22:14 +02:00
|
|
|
|
2017-07-25 20:32:40 +02:00
|
|
|
First, checks for basic properties that might indicate a dangerous file.
|
|
|
|
If the file isn't dangerous, then delegates to various helper methods
|
|
|
|
for filetype-specific checks based on the file's mimetype.
|
2017-04-18 00:22:14 +02:00
|
|
|
"""
|
2017-07-25 20:32:40 +02:00
|
|
|
# Any of these methods can call make_dangerous():
|
|
|
|
self._check_malicious_exts()
|
|
|
|
self._check_mimetype()
|
|
|
|
self._check_extension()
|
|
|
|
self._check_filename() # can mutate self.filename
|
2017-12-20 16:15:30 +01:00
|
|
|
self._compute_random_hashes()
|
2017-07-25 17:51:23 +02:00
|
|
|
|
|
|
|
if not self.is_dangerous:
|
|
|
|
self.mime_processing_options.get(self.maintype, self.unknown)()
|
2015-11-02 18:00:33 +01:00
|
|
|
|
2016-12-16 23:18:53 +01:00
|
|
|
# ##### Helper functions #####
|
2017-02-22 17:29:20 +01:00
|
|
|
def _make_method_dict(self, list_of_tuples):
|
|
|
|
"""Returns a dictionary with mimetype: method pairs."""
|
|
|
|
dict_to_return = {}
|
|
|
|
for list_of_subtypes, method in list_of_tuples:
|
|
|
|
for subtype in list_of_subtypes:
|
|
|
|
dict_to_return[subtype] = method
|
|
|
|
return dict_to_return
|
2015-11-02 18:00:33 +01:00
|
|
|
|
2017-02-27 17:58:10 +01:00
|
|
|
@property
|
2017-02-27 20:22:24 +01:00
|
|
|
def has_metadata(self):
|
2017-03-16 03:29:51 +01:00
|
|
|
"""True if filetype typically contains metadata, else False."""
|
2017-02-27 20:22:24 +01:00
|
|
|
if self.mimetype in Config.mimes_metadata:
|
|
|
|
return True
|
|
|
|
return False
|
|
|
|
|
2017-03-01 23:43:43 +01:00
|
|
|
def make_tempdir(self):
|
2017-03-16 03:29:51 +01:00
|
|
|
"""Make a temporary directory at self.tempdir_path."""
|
2017-03-01 21:24:48 +01:00
|
|
|
if not os.path.exists(self.tempdir_path):
|
|
|
|
os.makedirs(self.tempdir_path)
|
|
|
|
return self.tempdir_path
|
|
|
|
|
2015-11-02 18:00:33 +01:00
|
|
|
#######################
|
2016-12-16 23:18:53 +01:00
|
|
|
# ##### Discarded mimetypes, reason in the docstring ######
|
2015-11-02 18:00:33 +01:00
|
|
|
def inode(self):
|
2016-12-16 23:18:53 +01:00
|
|
|
"""Empty file or symlink."""
|
2017-03-10 19:13:38 +01:00
|
|
|
if self.is_symlink:
|
2017-03-09 03:30:06 +01:00
|
|
|
symlink_path = self.get_property('symlink')
|
2017-04-10 13:00:34 +02:00
|
|
|
self.add_description('File is a symlink to {}'.format(symlink_path))
|
2015-11-24 17:45:06 +01:00
|
|
|
else:
|
2017-04-10 13:00:34 +02:00
|
|
|
self.add_description('File is an inode (empty file)')
|
2017-07-25 20:41:56 +02:00
|
|
|
self.should_copy = False
|
2015-11-02 18:00:33 +01:00
|
|
|
|
|
|
|
def unknown(self):
|
2016-12-16 23:18:53 +01:00
|
|
|
"""Main type should never be unknown."""
|
2017-04-10 13:00:34 +02:00
|
|
|
self.add_description('Unknown mimetype')
|
2017-07-25 20:41:56 +02:00
|
|
|
self.should_copy = False
|
2015-11-02 18:00:33 +01:00
|
|
|
|
|
|
|
def example(self):
|
2016-12-16 23:18:53 +01:00
|
|
|
"""Used in examples, should never be returned by libmagic."""
|
2017-03-22 17:04:22 +01:00
|
|
|
self.add_description('Example file')
|
2017-07-25 20:41:56 +02:00
|
|
|
self.should_copy = False
|
2015-11-02 18:00:33 +01:00
|
|
|
|
2016-05-17 11:50:34 +02:00
|
|
|
def multipart(self):
|
2016-12-16 23:18:53 +01:00
|
|
|
"""Used in web apps, should never be returned by libmagic"""
|
2017-04-10 13:00:34 +02:00
|
|
|
self.add_description('Multipart file - usually found in web apps')
|
2017-07-25 20:41:56 +02:00
|
|
|
self.should_copy = False
|
2016-05-17 11:50:34 +02:00
|
|
|
|
2016-12-16 23:18:53 +01:00
|
|
|
# ##### Treated as malicious, no reason to have it on a USB key ######
|
2015-11-02 18:00:33 +01:00
|
|
|
def message(self):
|
2016-12-16 23:18:53 +01:00
|
|
|
"""Process a message file."""
|
2017-04-10 13:00:34 +02:00
|
|
|
self.make_dangerous('Message file - should not be found on USB key')
|
2015-11-02 18:00:33 +01:00
|
|
|
|
|
|
|
def model(self):
|
2016-12-16 23:18:53 +01:00
|
|
|
"""Process a model file."""
|
2017-04-10 13:00:34 +02:00
|
|
|
self.make_dangerous('Model file - should not be found on USB key')
|
2015-11-02 18:00:33 +01:00
|
|
|
|
2016-12-16 23:18:53 +01:00
|
|
|
# ##### Files that will be converted ######
|
2015-11-02 18:00:33 +01:00
|
|
|
def text(self):
|
2016-12-16 23:18:53 +01:00
|
|
|
"""Process an rtf, ooxml, or plaintext file."""
|
2017-02-22 17:29:20 +01:00
|
|
|
for mt in Config.mimes_rtf:
|
2017-07-12 23:58:39 +02:00
|
|
|
if mt in self.subtype:
|
2017-04-10 13:00:34 +02:00
|
|
|
self.add_description('Rich Text (rtf) file')
|
2017-02-27 21:10:56 +01:00
|
|
|
self.force_ext('.txt')
|
2016-02-01 12:34:47 +01:00
|
|
|
return
|
2017-02-22 17:29:20 +01:00
|
|
|
for mt in Config.mimes_ooxml:
|
2017-07-12 23:58:39 +02:00
|
|
|
if mt in self.subtype:
|
2016-02-01 12:34:47 +01:00
|
|
|
self._ooxml()
|
|
|
|
return
|
2017-04-10 13:00:34 +02:00
|
|
|
self.add_description('Plain text file')
|
2017-02-27 21:10:56 +01:00
|
|
|
self.force_ext('.txt')
|
2015-11-02 18:00:33 +01:00
|
|
|
|
|
|
|
def application(self):
|
2017-03-16 03:29:51 +01:00
|
|
|
"""Process an application specific file according to its subtype."""
|
2017-07-27 17:30:26 +02:00
|
|
|
for subtype, method in self.app_subtype_methods.items():
|
|
|
|
if subtype in self.subtype: # checking for partial matches
|
|
|
|
method()
|
|
|
|
return
|
|
|
|
self._unknown_app() # if none of the methods match
|
2015-11-02 18:00:33 +01:00
|
|
|
|
|
|
|
def _executables(self):
|
2017-03-16 03:29:51 +01:00
|
|
|
"""Process an executable file."""
|
2017-04-10 13:00:34 +02:00
|
|
|
self.make_dangerous('Executable file')
|
2015-11-02 18:00:33 +01:00
|
|
|
|
|
|
|
def _winoffice(self):
|
2017-03-16 03:29:51 +01:00
|
|
|
"""Process a winoffice file using olefile/oletools."""
|
2017-03-14 15:41:31 +01:00
|
|
|
oid = oletools.oleid.OleID(self.src_path) # First assume a valid file
|
2017-02-27 21:10:56 +01:00
|
|
|
if not olefile.isOleFile(self.src_path):
|
2015-11-02 18:00:33 +01:00
|
|
|
# Manual processing, may already count as suspicious
|
|
|
|
try:
|
2017-02-27 21:10:56 +01:00
|
|
|
ole = olefile.OleFileIO(self.src_path, raise_defects=olefile.DEFECT_INCORRECT)
|
2017-12-20 16:15:30 +01:00
|
|
|
except Exception:
|
2017-04-10 13:00:34 +02:00
|
|
|
self.make_dangerous('Unparsable WinOffice file')
|
2015-11-02 18:00:33 +01:00
|
|
|
if ole.parsing_issues:
|
2017-04-10 13:00:34 +02:00
|
|
|
self.make_dangerous('Parsing issues with WinOffice file')
|
2015-11-02 18:00:33 +01:00
|
|
|
else:
|
|
|
|
if ole.exists('macros/vba') or ole.exists('Macros') \
|
|
|
|
or ole.exists('_VBA_PROJECT_CUR') or ole.exists('VBA'):
|
2017-04-10 13:00:34 +02:00
|
|
|
self.make_dangerous('WinOffice file containing a macro')
|
2015-11-02 18:00:33 +01:00
|
|
|
else:
|
|
|
|
indicators = oid.check()
|
2017-03-09 03:30:06 +01:00
|
|
|
# Encrypted can be set by multiple checks on the script
|
2015-11-02 18:00:33 +01:00
|
|
|
if oid.encrypted.value:
|
2017-04-10 13:00:34 +02:00
|
|
|
self.make_dangerous('Encrypted WinOffice file')
|
2015-11-02 18:00:33 +01:00
|
|
|
if oid.macros.value or oid.ole.exists('macros/vba') or oid.ole.exists('Macros') \
|
|
|
|
or oid.ole.exists('_VBA_PROJECT_CUR') or oid.ole.exists('VBA'):
|
2017-04-10 13:00:34 +02:00
|
|
|
self.make_dangerous('WinOffice file containing a macro')
|
2015-11-02 18:00:33 +01:00
|
|
|
for i in indicators:
|
2016-05-14 20:44:16 +02:00
|
|
|
if i.id == 'ObjectPool' and i.value:
|
2017-07-17 20:52:22 +02:00
|
|
|
self.make_dangerous('WinOffice file containing an object pool')
|
2016-05-14 20:44:16 +02:00
|
|
|
elif i.id == 'flash' and i.value:
|
2017-04-10 13:00:34 +02:00
|
|
|
self.make_dangerous('WinOffice file with embedded flash')
|
|
|
|
self.add_description('WinOffice file')
|
2015-11-02 18:00:33 +01:00
|
|
|
|
|
|
|
def _ooxml(self):
|
2017-03-16 03:29:51 +01:00
|
|
|
"""Process an ooxml file."""
|
2017-08-04 22:31:52 +02:00
|
|
|
self.add_description('OOXML (openoffice) file')
|
2016-02-01 14:19:51 +01:00
|
|
|
try:
|
2017-02-27 21:10:56 +01:00
|
|
|
doc = officedissector.doc.Document(self.src_path)
|
2016-02-01 14:19:51 +01:00
|
|
|
except Exception:
|
2017-04-10 13:00:34 +02:00
|
|
|
self.make_dangerous('Invalid ooxml file')
|
2016-02-01 14:19:51 +01:00
|
|
|
return
|
2015-11-02 18:00:33 +01:00
|
|
|
# There are probably other potentially malicious features:
|
|
|
|
# fonts, custom props, custom XML
|
|
|
|
if doc.is_macro_enabled or len(doc.features.macros) > 0:
|
2017-04-10 13:00:34 +02:00
|
|
|
self.make_dangerous('Ooxml file containing macro')
|
2015-11-02 18:00:33 +01:00
|
|
|
if len(doc.features.embedded_controls) > 0:
|
2017-04-10 13:00:34 +02:00
|
|
|
self.make_dangerous('Ooxml file with activex')
|
2015-11-02 18:00:33 +01:00
|
|
|
if len(doc.features.embedded_objects) > 0:
|
|
|
|
# Exploited by CVE-2014-4114 (OLE)
|
2017-04-10 13:00:34 +02:00
|
|
|
self.make_dangerous('Ooxml file with embedded objects')
|
2015-11-02 18:00:33 +01:00
|
|
|
if len(doc.features.embedded_packages) > 0:
|
2017-04-10 13:00:34 +02:00
|
|
|
self.make_dangerous('Ooxml file with embedded packages')
|
2015-11-02 18:00:33 +01:00
|
|
|
|
|
|
|
def _libreoffice(self):
|
2017-03-16 03:29:51 +01:00
|
|
|
"""Process a libreoffice file."""
|
2017-03-14 15:41:31 +01:00
|
|
|
# As long as there is no way to do a sanity check on the files => dangerous
|
2015-11-03 15:30:59 +01:00
|
|
|
try:
|
2017-02-27 21:10:56 +01:00
|
|
|
lodoc = zipfile.ZipFile(self.src_path, 'r')
|
2017-12-20 16:15:30 +01:00
|
|
|
except Exception:
|
2017-04-10 13:00:34 +02:00
|
|
|
# TODO: are there specific exceptions we should catch here? Or should it be everything
|
|
|
|
self.make_dangerous('Invalid libreoffice file')
|
2015-11-03 15:30:59 +01:00
|
|
|
for f in lodoc.infolist():
|
|
|
|
fname = f.filename.lower()
|
|
|
|
if fname.startswith('script') or fname.startswith('basic') or \
|
|
|
|
fname.startswith('object') or fname.endswith('.bin'):
|
2017-04-10 13:00:34 +02:00
|
|
|
self.make_dangerous('Libreoffice file containing executable code')
|
|
|
|
if not self.is_dangerous:
|
|
|
|
self.add_description('Libreoffice file')
|
2015-11-02 18:00:33 +01:00
|
|
|
|
|
|
|
def _pdf(self):
|
2017-03-16 03:29:51 +01:00
|
|
|
"""Process a PDF file."""
|
2017-02-27 21:10:56 +01:00
|
|
|
xmlDoc = PDFiD(self.src_path)
|
2015-11-03 13:04:14 +01:00
|
|
|
oPDFiD = cPDFiD(xmlDoc, True)
|
2016-12-16 19:55:15 +01:00
|
|
|
if oPDFiD.encrypt.count > 0:
|
2017-04-10 13:00:34 +02:00
|
|
|
self.make_dangerous('Encrypted pdf')
|
2016-12-16 19:55:15 +01:00
|
|
|
if oPDFiD.js.count > 0 or oPDFiD.javascript.count > 0:
|
2017-04-10 13:00:34 +02:00
|
|
|
self.make_dangerous('Pdf with embedded javascript')
|
2016-12-16 19:55:15 +01:00
|
|
|
if oPDFiD.aa.count > 0 or oPDFiD.openaction.count > 0:
|
2017-04-10 13:00:34 +02:00
|
|
|
self.make_dangerous('Pdf with openaction(s)')
|
2016-12-16 19:55:15 +01:00
|
|
|
if oPDFiD.richmedia.count > 0:
|
2017-04-10 13:00:34 +02:00
|
|
|
self.make_dangerous('Pdf containing flash')
|
2016-12-16 19:55:15 +01:00
|
|
|
if oPDFiD.launch.count > 0:
|
2017-04-10 13:00:34 +02:00
|
|
|
self.make_dangerous('Pdf with launch action(s)')
|
2017-06-16 14:45:44 +02:00
|
|
|
if oPDFiD.xfa.count > 0:
|
|
|
|
self.make_dangerous('Pdf with XFA structures')
|
2017-06-19 11:24:47 +02:00
|
|
|
if oPDFiD.objstm.count > 0:
|
|
|
|
self.make_dangerous('Pdf with ObjectStream structures')
|
2017-04-10 13:00:34 +02:00
|
|
|
if not self.is_dangerous:
|
|
|
|
self.add_description('Pdf file')
|
2015-11-02 18:00:33 +01:00
|
|
|
|
|
|
|
def _archive(self):
|
2017-03-16 03:29:51 +01:00
|
|
|
"""
|
|
|
|
Process an archive using 7zip.
|
|
|
|
|
|
|
|
The archive is extracted to a temporary directory and self.process_dir
|
|
|
|
is called on that directory. The recursive archive depth is increased
|
|
|
|
to protect against archive bombs.
|
|
|
|
"""
|
2017-04-10 13:00:34 +02:00
|
|
|
# TODO: change this to something archive type specific instead of generic 'Archive'
|
|
|
|
self.add_description('Archive')
|
2017-03-14 15:41:31 +01:00
|
|
|
self.should_copy = False
|
2017-04-17 22:08:55 +02:00
|
|
|
self.is_archive = True
|
2016-12-16 23:18:53 +01:00
|
|
|
|
2015-11-02 18:00:33 +01:00
|
|
|
def _unknown_app(self):
|
2017-03-16 03:29:51 +01:00
|
|
|
"""Process an unknown file."""
|
2017-07-15 03:57:29 +02:00
|
|
|
self.make_dangerous('Unknown application file')
|
2015-11-02 18:00:33 +01:00
|
|
|
|
|
|
|
def _binary_app(self):
|
2017-03-16 03:29:51 +01:00
|
|
|
"""Process an unknown binary file."""
|
2017-07-15 03:57:29 +02:00
|
|
|
self.make_dangerous('Unknown binary file')
|
2015-11-02 18:00:33 +01:00
|
|
|
|
|
|
|
#######################
|
2015-12-15 23:13:26 +01:00
|
|
|
# Metadata extractors
|
2017-02-16 23:27:00 +01:00
|
|
|
def _metadata_exif(self, metadata_file_path):
|
2017-03-16 03:29:51 +01:00
|
|
|
"""Read exif metadata from a jpg or tiff file using exifread."""
|
2017-04-10 13:00:34 +02:00
|
|
|
# TODO: can we shorten this method somehow?
|
2017-12-21 15:12:57 +01:00
|
|
|
with open(self.src_path, 'rb') as img:
|
|
|
|
tags = None
|
2015-12-15 23:40:25 +01:00
|
|
|
try:
|
|
|
|
tags = exifread.process_file(img, debug=True)
|
|
|
|
except Exception as e:
|
2017-12-21 15:12:57 +01:00
|
|
|
self.add_error(e, "Error while trying to grab full metadata for file {}; retrying for partial data.".format(self.src_path))
|
|
|
|
if tags is None:
|
|
|
|
try:
|
|
|
|
tags = exifread.process_file(img, debug=True)
|
|
|
|
except Exception as e:
|
|
|
|
self.add_error(e, "Failed to get any metadata for file {}.".format(self.src_path))
|
|
|
|
return False
|
|
|
|
for tag in sorted(tags.keys()):
|
|
|
|
# These tags are long and obnoxious/binary so we don't add them
|
|
|
|
if tag not in ('JPEGThumbnail', 'TIFFThumbnail'):
|
|
|
|
tag_string = str(tags[tag])
|
|
|
|
# Exifreader truncates data.
|
|
|
|
if len(tag_string) > 25 and tag_string.endswith(", ... ]"):
|
|
|
|
tag_value = tags[tag].values
|
|
|
|
tag_string = str(tag_value)
|
|
|
|
with open(metadata_file_path, 'w+') as metadata_file:
|
|
|
|
metadata_file.write("Key: {}\tValue: {}\n".format(tag, tag_string))
|
|
|
|
# TODO: how do we want to log metadata?
|
|
|
|
self.set_property('metadata', 'exif')
|
2015-12-15 23:40:25 +01:00
|
|
|
return True
|
2015-12-15 23:13:26 +01:00
|
|
|
|
2017-02-16 23:27:00 +01:00
|
|
|
def _metadata_png(self, metadata_file_path):
|
2017-03-16 03:29:51 +01:00
|
|
|
"""Extract metadata from a png file using PIL/Pillow."""
|
2015-12-15 23:40:25 +01:00
|
|
|
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
|
|
|
try:
|
2017-12-21 15:12:57 +01:00
|
|
|
with Image.open(self.src_path) as img:
|
|
|
|
for tag in sorted(img.info.keys()):
|
|
|
|
# These are long and obnoxious/binary
|
|
|
|
if tag not in ('icc_profile'):
|
|
|
|
with open(metadata_file_path, 'w+') as metadata_file:
|
|
|
|
metadata_file.write("Key: {}\tValue: {}\n".format(tag, img.info[tag]))
|
|
|
|
# LOG: handle metadata
|
|
|
|
self.set_property('metadata', 'png')
|
2017-03-18 05:10:17 +01:00
|
|
|
except Exception as e: # Catch decompression bombs
|
2017-03-14 15:41:31 +01:00
|
|
|
# TODO: only catch DecompressionBombWarnings here?
|
2017-03-09 03:30:06 +01:00
|
|
|
self.add_error(e, "Caught exception processing metadata for {}".format(self.src_path))
|
|
|
|
self.make_dangerous('exception processing metadata')
|
2015-12-15 23:40:25 +01:00
|
|
|
return False
|
2015-12-15 23:13:26 +01:00
|
|
|
|
|
|
|
def extract_metadata(self):
|
2017-03-16 03:29:51 +01:00
|
|
|
"""Create metadata file and call correct metadata extraction method."""
|
2017-02-27 21:10:56 +01:00
|
|
|
metadata_file_path = self.create_metadata_file(".metadata.txt")
|
|
|
|
mt = self.mimetype
|
2017-02-22 17:29:20 +01:00
|
|
|
metadata_processing_method = self.metadata_mimetype_methods.get(mt)
|
2017-02-16 23:27:00 +01:00
|
|
|
if metadata_processing_method:
|
2017-02-22 17:29:20 +01:00
|
|
|
# TODO: should we return metadata and write it here instead of in processing method?
|
2017-02-16 23:27:00 +01:00
|
|
|
metadata_processing_method(metadata_file_path)
|
2015-11-02 18:00:33 +01:00
|
|
|
|
2015-12-15 23:13:26 +01:00
|
|
|
#######################
|
2016-12-16 23:18:53 +01:00
|
|
|
# ##### Media - audio and video aren't converted ######
|
2015-11-02 18:00:33 +01:00
|
|
|
def audio(self):
|
2017-03-16 03:29:51 +01:00
|
|
|
"""Process an audio file."""
|
2017-04-10 13:00:34 +02:00
|
|
|
self.add_description('Audio file')
|
2015-11-02 18:00:33 +01:00
|
|
|
self._media_processing()
|
|
|
|
|
2016-12-16 23:18:53 +01:00
|
|
|
def video(self):
|
2017-03-16 03:29:51 +01:00
|
|
|
"""Process a video."""
|
2017-04-10 13:00:34 +02:00
|
|
|
self.add_description('Video file')
|
2016-12-16 23:18:53 +01:00
|
|
|
self._media_processing()
|
|
|
|
|
|
|
|
def _media_processing(self):
|
|
|
|
"""Generic way to process all media files."""
|
2017-04-10 13:00:34 +02:00
|
|
|
self.add_description('Media file')
|
2016-12-16 23:18:53 +01:00
|
|
|
|
2015-11-02 18:00:33 +01:00
|
|
|
def image(self):
|
2017-03-16 03:29:51 +01:00
|
|
|
"""
|
|
|
|
Process an image.
|
|
|
|
|
|
|
|
Extracts metadata to dest key using self.extract_metada() if metadata
|
|
|
|
is present. Creates a temporary directory on dest key, opens the image
|
|
|
|
using PIL.Image, saves it to the temporary directory, and copies it to
|
|
|
|
the destination.
|
|
|
|
"""
|
2017-02-27 17:58:10 +01:00
|
|
|
if self.has_metadata:
|
2015-12-15 23:13:26 +01:00
|
|
|
self.extract_metadata()
|
2017-03-01 23:43:43 +01:00
|
|
|
tempdir_path = self.make_tempdir()
|
2017-03-01 21:24:48 +01:00
|
|
|
tempfile_path = os.path.join(tempdir_path, self.filename)
|
2015-12-10 02:26:26 +01:00
|
|
|
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
2017-03-01 21:24:48 +01:00
|
|
|
try: # Do image conversions
|
2017-12-21 15:12:57 +01:00
|
|
|
with Image.open(self.src_path) as img_in:
|
|
|
|
with Image.frombytes(img_in.mode, img_in.size, img_in.tobytes()) as img_out:
|
|
|
|
img_out.save(tempfile_path)
|
|
|
|
self.src_path = tempfile_path
|
2017-03-01 21:24:48 +01:00
|
|
|
except Exception as e: # Catch decompression bombs
|
2017-03-01 23:43:43 +01:00
|
|
|
# TODO: change this from all Exceptions to specific DecompressionBombWarning
|
2017-03-09 03:30:06 +01:00
|
|
|
self.add_error(e, "Caught exception (possible decompression bomb?) while translating file {}.".format(self.src_path))
|
2017-04-10 13:07:21 +02:00
|
|
|
self.make_dangerous('Image file containing decompression bomb')
|
|
|
|
if not self.is_dangerous:
|
|
|
|
self.add_description('Image file')
|
|
|
|
|
|
|
|
|
|
|
|
class GroomerLogger(object):
|
|
|
|
"""Groomer logging interface."""
|
|
|
|
|
|
|
|
def __init__(self, src_root_path, dst_root_path, debug=False):
|
|
|
|
self._src_root_path = src_root_path
|
|
|
|
self._dst_root_path = dst_root_path
|
|
|
|
self._log_dir_path = self._make_log_dir(dst_root_path)
|
|
|
|
self.log_path = os.path.join(self._log_dir_path, 'circlean_log.txt')
|
|
|
|
self._add_root_dir(src_root_path)
|
|
|
|
if debug:
|
|
|
|
self.log_debug_err = os.path.join(self._log_dir_path, 'debug_stderr.log')
|
|
|
|
self.log_debug_out = os.path.join(self._log_dir_path, 'debug_stdout.log')
|
|
|
|
else:
|
|
|
|
self.log_debug_err = os.devnull
|
|
|
|
self.log_debug_out = os.devnull
|
|
|
|
|
|
|
|
def _make_log_dir(self, root_dir_path):
|
2017-04-18 00:22:14 +02:00
|
|
|
"""Create the directory in the dest dir that will hold the logs"""
|
2017-04-10 13:07:21 +02:00
|
|
|
log_dir_path = os.path.join(root_dir_path, 'logs')
|
|
|
|
if os.path.exists(log_dir_path):
|
|
|
|
shutil.rmtree(log_dir_path)
|
|
|
|
os.makedirs(log_dir_path)
|
|
|
|
return log_dir_path
|
|
|
|
|
|
|
|
def _add_root_dir(self, root_path):
|
2017-04-18 00:22:14 +02:00
|
|
|
"""Add the root directory to the log"""
|
2017-04-10 13:07:21 +02:00
|
|
|
dirname = os.path.split(root_path)[1] + '/'
|
|
|
|
with open(self.log_path, mode='ab') as lf:
|
|
|
|
lf.write(bytes(dirname, 'utf-8'))
|
|
|
|
lf.write(b'\n')
|
|
|
|
|
|
|
|
def add_file(self, file_path, file_props, in_tempdir=False):
|
2017-07-20 19:38:54 +02:00
|
|
|
"""Add a file to the log. Takes a path and a dict of file properties."""
|
2017-04-10 13:07:21 +02:00
|
|
|
depth = self._get_path_depth(file_path)
|
2017-07-20 19:56:32 +02:00
|
|
|
try:
|
|
|
|
file_hash = Logging.computehash(file_path)[:6]
|
|
|
|
except IsADirectoryError:
|
|
|
|
file_hash = 'directory'
|
2017-08-07 23:57:42 +02:00
|
|
|
except FileNotFoundError:
|
|
|
|
file_hash = '------'
|
2017-07-20 19:38:54 +02:00
|
|
|
if file_props['is_symlink']:
|
|
|
|
symlink_template = "+- NOT COPIED: symbolic link to {name} ({sha_hash})"
|
|
|
|
log_string = symlink_template.format(
|
|
|
|
name=file_props['symlink_path'],
|
|
|
|
sha_hash=file_hash
|
|
|
|
)
|
2017-04-10 13:07:21 +02:00
|
|
|
else:
|
2017-07-20 19:38:54 +02:00
|
|
|
if file_props['is_dangerous']:
|
|
|
|
category = "Dangerous"
|
|
|
|
else:
|
|
|
|
category = "Normal"
|
|
|
|
size_string = self._format_file_size(file_props['file_size'])
|
|
|
|
if not file_props['copied']:
|
|
|
|
copied_string = 'NOT COPIED: '
|
|
|
|
else:
|
|
|
|
copied_string = ''
|
|
|
|
file_template = "+- {copied}{name} ({sha_hash}): {size}, type: {mt}/{st}. {cat}: {desc_str}"
|
|
|
|
log_string = file_template.format(
|
|
|
|
copied=copied_string,
|
|
|
|
name=file_props['filename'],
|
|
|
|
sha_hash=file_hash,
|
|
|
|
size=size_string,
|
|
|
|
mt=file_props['maintype'],
|
|
|
|
st=file_props['subtype'],
|
|
|
|
cat=category,
|
|
|
|
desc_str=file_props['description_string'],
|
|
|
|
)
|
|
|
|
if file_props['errors']:
|
|
|
|
error_string = ', '.join([str(key) for key in file_props['errors']])
|
2017-08-07 20:13:02 +02:00
|
|
|
log_string += (' Errors: ' + error_string)
|
2017-04-10 13:07:21 +02:00
|
|
|
if in_tempdir:
|
|
|
|
depth -= 1
|
2017-07-20 19:38:54 +02:00
|
|
|
self._write_line_to_log(log_string, depth)
|
2017-04-10 13:07:21 +02:00
|
|
|
|
|
|
|
def add_dir(self, dir_path):
|
2017-04-18 00:22:14 +02:00
|
|
|
"""Add a directory to the log"""
|
2017-04-10 13:07:21 +02:00
|
|
|
path_depth = self._get_path_depth(dir_path)
|
|
|
|
dirname = os.path.split(dir_path)[1] + '/'
|
|
|
|
log_line = '+- ' + dirname
|
|
|
|
self._write_line_to_log(log_line, path_depth)
|
|
|
|
|
2017-04-17 23:06:56 +02:00
|
|
|
def _format_file_size(self, size):
|
2017-04-18 00:22:14 +02:00
|
|
|
"""Returns a string with the file size and appropriate unit"""
|
2017-04-17 23:06:56 +02:00
|
|
|
file_size = size
|
|
|
|
for unit in ('B', 'KB', 'MB', 'GB'):
|
|
|
|
if file_size < 1024:
|
|
|
|
return str(int(file_size)) + unit
|
|
|
|
else:
|
|
|
|
file_size = file_size / 1024
|
|
|
|
return str(int(file_size)) + 'GB'
|
|
|
|
|
2017-04-10 13:07:21 +02:00
|
|
|
def _get_path_depth(self, path):
|
2017-04-18 00:22:14 +02:00
|
|
|
"""Returns the relative path depth compared to root directory"""
|
2017-04-10 13:07:21 +02:00
|
|
|
if self._dst_root_path in path:
|
|
|
|
base_path = self._dst_root_path
|
|
|
|
elif self._src_root_path in path:
|
|
|
|
base_path = self._src_root_path
|
|
|
|
relpath = os.path.relpath(path, base_path)
|
|
|
|
path_depth = relpath.count(os.path.sep)
|
|
|
|
return path_depth
|
|
|
|
|
|
|
|
def _write_line_to_log(self, line, indentation_depth):
|
2017-04-18 00:22:14 +02:00
|
|
|
"""
|
|
|
|
Write a line to the log
|
|
|
|
|
|
|
|
Pad the line according to the `indentation_depth`.
|
|
|
|
"""
|
2017-04-10 13:07:21 +02:00
|
|
|
padding = b' '
|
|
|
|
padding += b'| ' * indentation_depth
|
2017-04-10 13:39:28 +02:00
|
|
|
line_bytes = os.fsencode(line)
|
2017-04-10 13:07:21 +02:00
|
|
|
with open(self.log_path, mode='ab') as lf:
|
|
|
|
lf.write(padding)
|
2017-04-10 13:39:28 +02:00
|
|
|
lf.write(line_bytes)
|
2017-04-10 13:07:21 +02:00
|
|
|
lf.write(b'\n')
|
2015-12-10 02:26:26 +01:00
|
|
|
|
2017-02-27 20:22:24 +01:00
|
|
|
|
|
|
|
class KittenGroomerFileCheck(KittenGroomerBase):
|
|
|
|
|
|
|
|
def __init__(self, root_src, root_dst, max_recursive_depth=2, debug=False):
|
2017-03-18 03:19:56 +01:00
|
|
|
super(KittenGroomerFileCheck, self).__init__(root_src, root_dst)
|
2017-02-27 20:22:24 +01:00
|
|
|
self.recursive_archive_depth = 0
|
|
|
|
self.max_recursive_depth = max_recursive_depth
|
2017-04-10 13:07:21 +02:00
|
|
|
self.logger = GroomerLogger(root_src, root_dst, debug)
|
2016-12-22 03:24:29 +01:00
|
|
|
|
2017-07-20 23:47:23 +02:00
|
|
|
def __repr__(self):
|
|
|
|
return "filecheck.KittenGroomerFileCheck object: {{{}}}".format(
|
|
|
|
os.path.basename(self.src_root_path)
|
|
|
|
)
|
|
|
|
|
2017-02-23 23:39:16 +01:00
|
|
|
def process_dir(self, src_dir, dst_dir):
|
2017-03-16 03:29:51 +01:00
|
|
|
"""Process a directory on the source key."""
|
2017-04-10 13:07:21 +02:00
|
|
|
for srcpath in self.list_files_dirs(src_dir):
|
2017-07-17 00:36:22 +02:00
|
|
|
if not os.path.islink(srcpath) and os.path.isdir(srcpath):
|
2017-04-10 13:07:21 +02:00
|
|
|
self.logger.add_dir(srcpath)
|
|
|
|
else:
|
|
|
|
dstpath = os.path.join(dst_dir, os.path.basename(srcpath))
|
2017-07-27 19:27:12 +02:00
|
|
|
cur_file = File(srcpath, dstpath)
|
2017-07-17 00:36:22 +02:00
|
|
|
self.process_file(cur_file)
|
2017-03-01 21:24:48 +01:00
|
|
|
|
|
|
|
def process_file(self, file):
|
2017-03-16 03:29:51 +01:00
|
|
|
"""
|
|
|
|
Process an individual file.
|
|
|
|
|
|
|
|
Check the file, handle archives using self.process_archive, copy
|
|
|
|
the file to the destionation key, and clean up temporary directory.
|
|
|
|
"""
|
2017-03-01 21:24:48 +01:00
|
|
|
file.check()
|
2017-04-17 22:08:55 +02:00
|
|
|
if file.is_archive:
|
2017-04-10 13:07:21 +02:00
|
|
|
self.process_archive(file)
|
2017-07-20 19:38:54 +02:00
|
|
|
else:
|
|
|
|
if file.should_copy:
|
|
|
|
file.safe_copy()
|
|
|
|
file.set_property('copied', True)
|
2017-12-20 16:15:30 +01:00
|
|
|
if not file._validate_random_hashes():
|
|
|
|
# Something's fucked up.
|
|
|
|
file.make_dangerous('The copied file is different from the one checked, removing.')
|
|
|
|
os.remove(file.dst_path)
|
2017-07-27 19:27:12 +02:00
|
|
|
self.write_file_to_log(file)
|
2017-04-10 13:07:21 +02:00
|
|
|
# TODO: Can probably handle cleaning up the tempdir better
|
2017-03-09 03:30:06 +01:00
|
|
|
if hasattr(file, 'tempdir_path'):
|
2017-03-07 21:31:32 +01:00
|
|
|
self.safe_rmtree(file.tempdir_path)
|
2017-03-01 21:24:48 +01:00
|
|
|
|
|
|
|
def process_archive(self, file):
|
2017-03-16 03:29:51 +01:00
|
|
|
"""
|
|
|
|
Unpack an archive using 7zip and process contents using process_dir.
|
2017-03-01 21:24:48 +01:00
|
|
|
|
|
|
|
Should be given a Kittengroomer file object whose src_path points
|
2017-03-16 03:29:51 +01:00
|
|
|
to an archive.
|
|
|
|
"""
|
2017-03-01 21:24:48 +01:00
|
|
|
self.recursive_archive_depth += 1
|
|
|
|
if self.recursive_archive_depth >= self.max_recursive_depth:
|
2017-03-14 15:41:31 +01:00
|
|
|
file.make_dangerous('Archive bomb')
|
2017-03-01 21:24:48 +01:00
|
|
|
else:
|
2017-03-01 23:43:43 +01:00
|
|
|
tempdir_path = file.make_tempdir()
|
|
|
|
command_str = '{} -p1 x "{}" -o"{}" -bd -aoa'
|
2017-08-12 02:02:05 +02:00
|
|
|
# -p1=password, x=extract, -o=output location, -bd=no % indicator, -aoa=overwrite existing files
|
2017-03-01 23:43:43 +01:00
|
|
|
unpack_command = command_str.format(SEVENZ_PATH,
|
|
|
|
file.src_path, tempdir_path)
|
|
|
|
self._run_process(unpack_command)
|
2017-07-27 19:27:12 +02:00
|
|
|
self.write_file_to_log(file)
|
2017-03-01 21:24:48 +01:00
|
|
|
self.process_dir(tempdir_path, file.dst_path)
|
2017-03-07 21:31:32 +01:00
|
|
|
self.safe_rmtree(tempdir_path)
|
2017-03-01 21:24:48 +01:00
|
|
|
self.recursive_archive_depth -= 1
|
|
|
|
|
2017-03-01 23:43:43 +01:00
|
|
|
def _run_process(self, command_string, timeout=None):
|
|
|
|
"""Run command_string in a subprocess, wait until it finishes."""
|
|
|
|
args = shlex.split(command_string)
|
|
|
|
with open(self.logger.log_debug_err, 'ab') as stderr, open(self.logger.log_debug_out, 'ab') as stdout:
|
|
|
|
try:
|
|
|
|
subprocess.check_call(args, stdout=stdout, stderr=stderr, timeout=timeout)
|
|
|
|
except (subprocess.TimeoutExpired, subprocess.CalledProcessError):
|
|
|
|
return
|
|
|
|
return True
|
2015-11-02 18:00:33 +01:00
|
|
|
|
2017-07-27 19:27:12 +02:00
|
|
|
def write_file_to_log(self, file):
|
|
|
|
"""Pass information about `file` to self.logger."""
|
|
|
|
props = file.get_all_props()
|
|
|
|
if not file.is_archive:
|
|
|
|
# FIXME: in_tempdir is a hack to make image files appear at the correct tree depth in log
|
|
|
|
in_tempdir = os.path.exists(file.tempdir_path)
|
|
|
|
self.logger.add_file(file.src_path, props, in_tempdir)
|
|
|
|
|
2017-04-10 13:07:21 +02:00
|
|
|
def list_files_dirs(self, root_dir_path):
|
2017-04-18 00:22:14 +02:00
|
|
|
"""
|
|
|
|
Returns a list of all files and directories
|
|
|
|
|
|
|
|
Performs a depth-first traversal of the file tree.
|
|
|
|
"""
|
2017-04-10 13:07:21 +02:00
|
|
|
queue = []
|
|
|
|
for path in sorted(os.listdir(root_dir_path), key=lambda x: str.lower(x)):
|
|
|
|
full_path = os.path.join(root_dir_path, path)
|
2017-07-16 20:05:48 +02:00
|
|
|
# check for symlinks first to prevent getting trapped in infinite symlink recursion
|
|
|
|
if os.path.islink(full_path):
|
2017-04-10 13:07:21 +02:00
|
|
|
queue.append(full_path)
|
2017-07-16 20:05:48 +02:00
|
|
|
elif os.path.isdir(full_path):
|
2017-07-14 23:52:21 +02:00
|
|
|
queue.append(full_path)
|
2017-07-16 20:05:48 +02:00
|
|
|
queue += self.list_files_dirs(full_path)
|
|
|
|
elif os.path.isfile(full_path):
|
2017-04-10 13:07:21 +02:00
|
|
|
queue.append(full_path)
|
|
|
|
return queue
|
|
|
|
|
2017-02-23 23:39:16 +01:00
|
|
|
def run(self):
|
2017-03-21 00:39:37 +01:00
|
|
|
self.process_dir(self.src_root_path, self.dst_root_path)
|
2017-02-23 23:39:16 +01:00
|
|
|
|
|
|
|
|
|
|
|
def main(kg_implementation, description):
|
|
|
|
parser = argparse.ArgumentParser(prog='KittenGroomer', description=description)
|
|
|
|
parser.add_argument('-s', '--source', type=str, help='Source directory')
|
|
|
|
parser.add_argument('-d', '--destination', type=str, help='Destination directory')
|
|
|
|
args = parser.parse_args()
|
|
|
|
kg = kg_implementation(args.source, args.destination)
|
|
|
|
kg.run()
|
2017-02-23 22:49:29 +01:00
|
|
|
|
2016-12-16 23:18:53 +01:00
|
|
|
|
2015-11-02 18:00:33 +01:00
|
|
|
if __name__ == '__main__':
|
2016-12-16 23:18:53 +01:00
|
|
|
main(KittenGroomerFileCheck, 'File sanitizer used in CIRCLean. Renames potentially dangerous files.')
|