MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #5865 from JakubOnderka/attachment_tool 

chg: [internal] Move attachment handling to one place
pull/6198/head
Jakub Onderka 2020-08-14 13:53:51 +02:00 committed by GitHub
parent d5d2b50e5f 778d39e175
commit c397375634
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 675 additions and 525 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
app/Controller/AttributesController.php
View File

@ -2,6 +2,7 @@
App::uses('AppController', 'Controller');
App::uses('Folder', 'Utility');
App::uses('File', 'Utility');
App::uses('AttachmentTool', 'Tools');
/**
* @property Attribute $Attribute
@ -333,36 +334,7 @@ class AttributesController extends AppController
private function __downloadAttachment($attribute)
{
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$attachments_dir = $this->Attribute->getDefaultAttachments_dir();
}
$is_s3 = substr($attachments_dir, 0, 2) === "s3";
if ($is_s3) {
// We have to download it!
App::uses('AWSS3Client', 'Tools');
$client = new AWSS3Client();
$client->initTool();
// Use tmpdir as opposed to attachments dir since we can't write to s3://
$attachments_dir = Configure::read('MISP.tmpdir');
if (empty($attachments_dir)) {
$this->loadModel('Server');
$attachments_dir = $this->Server->getDefaultTmp_dir();
}
// Now download the file
$resp = $client->download($attribute['event_id'] . DS . $attribute['id']);
// Save to a tmpfile
$tmpFile = new File($attachments_dir . DS . $attribute['uuid'], true, 0600);
$tmpFile->write($resp);
$tmpFile->close();
$path = $attachments_dir . DS;
$file = $attribute['uuid'];
} else {
$path = $attachments_dir . DS . $attribute['event_id'] . DS;
$file = $attribute['id'];
}
$file = $this->Attribute->getAttachmentFile($attribute);
if ('attachment' == $attribute['type']) {
$filename = $attribute['value'];
@ -378,7 +350,7 @@ class AttributesController extends AppController
$this->autoRender = false;
$this->response->type($fileExt);
$download_attachments_on_load = Configure::check('MISP.download_attachments_on_load') ? Configure::read('MISP.download_attachments_on_load') : true;
$this->response->file($path . $file, array('download' => $download_attachments_on_load, 'name' => $filename . '.' . $fileExt));
$this->response->file($file->path, array('download' => $download_attachments_on_load, 'name' => $filename . '.' . $fileExt));
}
public function add_attachment($eventId = null)
@ -2958,19 +2930,15 @@ class AttributesController extends AppController
'recursive' => -1)
);
$counter = 0;
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$this->loadModel('Server');
$attachments_dir = $this->Server->getDefaultAttachments_dir();
}
$attachmentTool = new AttachmentTool();
foreach ($attributes as $attribute) {
$path = $attachments_dir . DS . $attribute['Attribute']['event_id'] . DS;
$file = $attribute['Attribute']['id'];
if (!file_exists($path . $file)) {
$exists = $attachmentTool->exists($attribute['Attribute']['event_id'], $attribute['Attribute']['id']);
if (!$exists) {
$counter++;
}
}
return new CakeResponse(array('body'=>$counter, 'status'=>200));
return new CakeResponse(array('body' => $counter, 'status' => 200));
}
public function exportSearch($type = false)

5
app/Controller/ServersController.php
View File

@ -1,6 +1,7 @@
<?php
App::uses('AppController', 'Controller');
App::uses('Xml', 'Utility');
App::uses('AttachmentTool', 'Tools');
class ServersController extends AppController
{
@ -1022,9 +1023,9 @@ class ServersController extends AppController
$php_ini = php_ini_loaded_file();
$this->set('php_ini', $php_ini);
$malwareTool = new MalwareTool();
$attachmentTool = new AttachmentTool();
try {
$advanced_attachments = $malwareTool->checkAdvancedExtractionStatus($this->Server->getPythonVersion());
$advanced_attachments = $attachmentTool->checkAdvancedExtractionStatus($this->Server->getPythonVersion());
} catch (Exception $e) {
$this->log($e->getMessage(), LOG_NOTICE);
$advanced_attachments = false;

19
app/Controller/ShadowAttributesController.php
View File

@ -2,6 +2,7 @@
App::uses('AppController', 'Controller');
App::uses('Folder', 'Utility');
App::uses('File', 'Utility');
App::uses('AttachmentTool', 'Tools');
/**
* @property ShadowAttribute $ShadowAttribute
@ -57,7 +58,7 @@ class ShadowAttributesController extends AppController
}
$this->ShadowAttribute->publishKafkaNotification('shadow_attribute', $shadow, 'accept');
$shadow = $shadow['ShadowAttribute'];
if ($this->ShadowAttribute->typeIsAttachment($shadow['type'])) {
if ($this->ShadowAttribute->typeIsAttachment($shadow['type']) && !$shadow['proposal_to_delete']) {
$encodedFile = $this->ShadowAttribute->base64EncodeAttachment($shadow);
$shadow['data'] = $encodedFile;
}
@ -449,19 +450,15 @@ class ShadowAttributesController extends AppController
$this->__downloadAttachment($sa['ShadowAttribute']);
}
private function __downloadAttachment($shadowAttribute)
private function __downloadAttachment(array $shadowAttribute)
{
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$attachments_dir = $this->ShadowAttribute->getDefaultAttachments_dir();
}
$path = $attachments_dir . DS . 'shadow' . DS . $shadowAttribute['event_id'] . DS;
$file = $shadowAttribute['id'];
if ('attachment' == $shadowAttribute['type']) {
$file = $this->ShadowAttribute->getAttachmentFile($shadowAttribute);
if ('attachment' === $shadowAttribute['type']) {
$filename = $shadowAttribute['value'];
$fileExt = pathinfo($filename, PATHINFO_EXTENSION);
$filename = substr($filename, 0, strlen($filename) - strlen($fileExt) - 1);
} elseif ('malware-sample' == $shadowAttribute['type']) {
} elseif ('malware-sample' === $shadowAttribute['type']) {
$filenameHash = explode('|', $shadowAttribute['value']);
$filename = substr($filenameHash[0], strrpos($filenameHash[0], '\\'));
$fileExt = "zip";
@ -470,7 +467,7 @@ class ShadowAttributesController extends AppController
}
$this->autoRender = false;
$this->response->type($fileExt);
$this->response->file($path . $file, array('download' => true, 'name' => $filename . '.' . $fileExt));
$this->response->file($file->path, array('download' => true, 'name' => $filename . '.' . $fileExt));
}
public function add_attachment($eventId = null)

8
app/Lib/Tools/AWSS3Client.php
View File

@ -49,6 +49,14 @@ class AWSS3Client
return $s3;
}
public function exist($key)
{
return $this->__client->doesObjectExist([
'Bucket' => $this->__settings['bucket_name'],
'Key' => $key,
]);
}
public function upload($key, $data)
{
$this->__client->putObject([

564
app/Lib/Tools/AttachmentTool.php Normal file
View File

@ -0,0 +1,564 @@
<?php
App::uses('AWSS3Client', 'Tools');
class AttachmentTool
{
const ZIP_PASSWORD = 'infected';
const ADVANCED_EXTRACTION_SCRIPT_PATH = APP . 'files/scripts/generate_file_objects.py';
/** @var AWSS3Client */
private $s3client;
/**
* @param int $eventId
* @param int $attributeId
* @param string $path_suffix
* @return bool
* @throws Exception
*/
public function exists($eventId, $attributeId, $path_suffix = '')
{
return $this->_exists(false, $eventId, $attributeId, $path_suffix);
}
/**
* @param int $eventId
* @param int $attributeId
* @param string $path_suffix
* @return bool
* @throws Exception
*/
public function shadowExists($eventId, $attributeId, $path_suffix = '')
{
return $this->_exists(true, $eventId, $attributeId, $path_suffix);
}
/**
* @param bool $shadow
* @param int $eventId
* @param int $attributeId
* @param string $path_suffix
* @return bool
* @throws Exception
*/
protected function _exists($shadow, $eventId, $attributeId, $path_suffix = '')
{
if ($this->attachmentDirIsS3()) {
$s3 = $this->loadS3Client();
$path = $this->getPath($shadow, $eventId, $attributeId, $path_suffix);
return $s3->exist($path);
} else {
try {
$this->_getFile($shadow, $eventId, $attributeId, $path_suffix);
} catch (NotFoundException $e) {
return false;
}
}
return true;
}
/**
* @param int $eventId
* @param int $attributeId
* @param string $path_suffix
* @return string
* @throws Exception
*/
public function getContent($eventId, $attributeId, $path_suffix = '')
{
return $this->_getContent(false, $eventId, $attributeId, $path_suffix);
}
/**
* @param int $eventId
* @param int $attributeId
* @param string $path_suffix
* @return string
* @throws Exception
*/
public function getShadowContent($eventId, $attributeId, $path_suffix = '')
{
return $this->_getContent(true, $eventId, $attributeId, $path_suffix);
}
/**
* @param bool $shadow
* @param int $eventId
* @param int $attributeId
* @param string $path_suffix
* @return string
* @throws Exception
*/
protected function _getContent($shadow, $eventId, $attributeId, $path_suffix = '')
{
if ($this->attachmentDirIsS3()) {
$s3 = $this->loadS3Client();
$path = $this->getPath($shadow, $eventId, $attributeId, $path_suffix);
return $s3->download($path);
} else {
$file = $this->_getFile($shadow, $eventId, $attributeId, $path_suffix);
$result = $file->read();
if ($result === false) {
throw new Exception("Could not read file '{$file->path}'.");
}
return $result;
}
}
/**
* @param int $eventId
* @param int $attributeId
* @param string $pathSuffix
* @return File
* @throws Exception
*/
public function getFile($eventId, $attributeId, $pathSuffix = '')
{
return $this->_getFile(false, $eventId, $attributeId, $pathSuffix);
}
/**
* @param int $eventId
* @param int $attributeId
* @param string $pathSuffix
* @return File
* @throws Exception
*/
public function getShadowFile($eventId, $attributeId, $pathSuffix = '')
{
return $this->_getFile(true, $eventId, $attributeId, $pathSuffix);
}
/**
* @param bool $shadow
* @param int $eventId
* @param int $attributeId
* @param string $pathSuffix
* @return File
* @throws Exception
*/
protected function _getFile($shadow, $eventId, $attributeId, $pathSuffix = '')
{
$path = $this->getPath($shadow, $eventId, $attributeId, $pathSuffix);
if ($this->attachmentDirIsS3()) {
$s3 = $this->loadS3Client();
$content = $s3->download($path);
$file = new File($this->tempFileName());
if (!$file->write($content)) {
throw new Exception("Could not write temporary file '{$file->path}'.");
}
} else {
$filepath = $this->attachmentDir() . DS . $path;
$file = new File($filepath);
if (!$file->exists()) {
throw new NotFoundException("File '$filepath' does not exists.");
}
}
return $file;
}
/**
* @param int $eventId
* @param int $attributeId
* @param string $data
* @param string $pathSuffix
* @return bool
* @throws Exception
*/
public function save($eventId, $attributeId, $data, $pathSuffix = '')
{
return $this->_save(false, $eventId, $attributeId, $data, $pathSuffix);
}
/**
* @param int $eventId
* @param int $attributeId
* @param string $data
* @param string $pathSuffix
* @return bool
* @throws Exception
*/
public function saveShadow($eventId, $attributeId, $data, $pathSuffix = '')
{
return $this->_save(true, $eventId, $attributeId, $data, $pathSuffix);
}
/**
* @param bool $shadow
* @param int $eventId
* @param int $attributeId
* @param string $data
* @param string $pathSuffix
* @return bool
* @throws Exception
*/
protected function _save($shadow, $eventId, $attributeId, $data, $pathSuffix = '')
{
$path = $this->getPath($shadow, $eventId, $attributeId, $pathSuffix);
if ($this->attachmentDirIsS3()) {
$s3 = $this->loadS3Client();
$s3->upload($path, $data);
} else {
$path = $this->attachmentDir() . DS . $path;
$file = new File($path, true);
if (!$file->write($data)) {
throw new Exception("Could not save attachment to file '$path'.");
}
}
return true;
}
/**
* @param int $eventId
* @param int $attributeId
* @param string $pathSuffix
* @return bool
* @throws Exception
*/
public function delete($eventId, $attributeId, $pathSuffix = '')
{
return $this->_delete(false, $eventId, $attributeId, $pathSuffix);
}
/**
* @param int $eventId
* @param int $attributeId
* @param string $pathSuffix
* @return bool
* @throws Exception
*/
public function deleteShadow($eventId, $attributeId, $pathSuffix = '')
{
return $this->_delete(true, $eventId, $attributeId, $pathSuffix);
}
/**
* @param bool $shadow
* @param int $eventId
* @param int $attributeId
* @param string $pathSuffix
* @return bool Return true if file was deleted, `false` if file doesn't exists.
* @throws Exception
*/
protected function _delete($shadow, $eventId, $attributeId, $pathSuffix = '')
{
if ($this->attachmentDirIsS3()) {
$s3 = $this->loadS3Client();
$path = $this->getPath($shadow, $eventId, $attributeId, $pathSuffix);
$s3->delete($path);
} else {
try {
$file = $this->_getFile($shadow, $eventId, $attributeId, $pathSuffix);
} catch (NotFoundException $e) {
return false;
}
if (!$file->delete()) {
throw new Exception(__('Delete of file attachment failed. Please report to administrator.'));
}
}
return true;
}
/**
* Deletes all attributes and shadow attributes files.
*
* @param int $eventId
* @return bool
* @throws Exception
*/
public function deleteAll($eventId)
{
if ($this->attachmentDirIsS3()) {
$s3 = $this->loadS3Client();
$s3->deleteDirectory($eventId);
} else {
$dirPath = $this->attachmentDir();
foreach (array($dirPath, $dirPath . DS . 'shadow') as $dirPath) {
$folder = new Folder($dirPath . DS . $eventId);
if ($folder->pwd() && !$folder->delete()) {
throw new Exception("Delete of directory '{$folder->pwd()}' failed: " . implode(', ', $folder->errors()));
}
}
}
return true;
}
/**
* @param string $originalFilename
* @param string $content
* @param string $md5
* @return string Content of zipped file
* @throws Exception
*/
public function encrypt($originalFilename, $content, $md5)
{
if (method_exists("ZipArchive", "setEncryptionName")) {
// When PHP zip extension is installed and supports creating encrypted archives.
return $this->encryptByExtension($originalFilename, $content, $md5);
} else {
return $this->encryptByCommand($originalFilename, $content, $md5);
}
}
/**
* @param string $originalFilename
* @param string $content
* @param string $md5
* @return string Content of zipped file
* @throws Exception
*/
private function encryptByCommand($originalFilename, $content, $md5)
{
$tempDir = $this->tempDir();
$contentsFile = new File($tempDir . DS . $md5, true);
if (!$contentsFile->write($content)) {
throw new Exception("Could not write content to file '{$contentsFile->path}'.");
}
$contentsFile->close();
$fileNameFile = new File($tempDir . DS . $md5 . '.filename.txt', true);
if (!$fileNameFile->write($originalFilename)) {
throw new Exception("Could not write original file name to file '{$fileNameFile->path}'.");
}
$fileNameFile->close();
$zipFile = new File($tempDir . DS . $md5 . '.zip');
$exec = [
'zip',
'-j', // junk (don't record) directory names
'-P', // use standard encryption
self::ZIP_PASSWORD,
escapeshellarg($zipFile->path),
escapeshellarg($contentsFile->path),
escapeshellarg($fileNameFile->path),
];
try {
$this->execute($exec);
$zipContent = $zipFile->read();
if ($zipContent === false) {
throw new Exception("Could not read content of newly created ZIP file.");
}
return $zipContent;
} catch (Exception $e) {
throw new Exception("Could not create encrypted ZIP file '{$zipFile->path}'.", 0, $e);
} finally {
$fileNameFile->delete();
$contentsFile->delete();
$zipFile->delete();
}
}
/**
* @param string $originalFilename
* @param string $content
* @param string $md5
* @return string Content of zipped file
* @throws Exception
*/
private function encryptByExtension($originalFilename, $content, $md5)
{
$zipFilePath = $this->tempFileName();
$zip = new ZipArchive();
$result = $zip->open($zipFilePath, ZipArchive::CREATE);
if ($result === true) {
$zip->setPassword(self::ZIP_PASSWORD);
$zip->addFromString($md5, $content);
$zip->setEncryptionName($md5, ZipArchive::EM_AES_128);
$zip->addFromString("$md5.filename.txt", $originalFilename);
$zip->setEncryptionName("$md5.filename.txt", ZipArchive::EM_AES_128);
$zip->close();
} else {
throw new Exception("Could not create encrypted ZIP file '$zipFilePath'. Error code: $result");
}
$zipFile = new File($zipFilePath);
$zipContent = $zipFile->read();
if ($zipContent === false) {
throw new Exception("Could not read content of newly created ZIP file.");
}
$zipFile->delete();
return $zipContent;
}
/**
* @param string $content
* @param array $hashTypes
* @return array
* @throws InvalidArgumentException
*/
public function computeHashes($content, array $hashTypes = array())
{
$validHashes = array('md5', 'sha1', 'sha256');
$hashes = [];
foreach ($hashTypes as $hashType) {
if (!in_array($hashType, $validHashes)) {
throw new InvalidArgumentException("Hash type '$hashType' is not valid hash type.");
}
$hashes[$hashType] = hash($hashType, $content);
}
return $hashes;
}
/**
* @param string $pythonBin
* @param string $filePath
* @return array
* @throws Exception
*/
public function advancedExtraction($pythonBin, $filePath)
{
return $this->executeAndParseJsonOutput([
$pythonBin,
self::ADVANCED_EXTRACTION_SCRIPT_PATH,
'-p',
escapeshellarg($filePath),
]);
}
/**
* @param string $pythonBin
* @return array
* @throws Exception
*/
public function checkAdvancedExtractionStatus($pythonBin)
{
return $this->executeAndParseJsonOutput([$pythonBin, self::ADVANCED_EXTRACTION_SCRIPT_PATH, '-c']);
}
private function tempFileName()
{
$randomName = (new RandomTool())->random_str(false, 12);
return $this->tempDir() . DS . $randomName;
}
/**
* @return string
*/
private function tempDir()
{
return Configure::read('MISP.tmpdir') ?: sys_get_temp_dir();
}
/**
* @return string
*/
private function attachmentDir()
{
return Configure::read('MISP.attachments_dir') ?: (APP . 'files');
}
/**
* Naive way to detect if we're working in S3
* @return bool
*/
private function attachmentDirIsS3()
{
return substr(Configure::read('MISP.attachments_dir'), 0, 2) === "s3";
}
/**
* @return AWSS3Client
*/
private function loadS3Client()
{
if ($this->s3client) {
return $this->s3client;
}
$client = new AWSS3Client();
$client->initTool();
$this->s3client = $client;
return $client;
}
/**
* @param bool $shadow
* @param int $eventId
* @param int $attributeId
* @param string $pathSuffix
* @return string
*/
private function getPath($shadow, $eventId, $attributeId, $pathSuffix)
{
$path = $shadow ? ('shadow' . DS) : '';
return $path . $eventId . DS . $attributeId . $pathSuffix;
}
/**
* @param array $command
* @return array
* @throws Exception
*/
private function executeAndParseJsonOutput(array $command)
{
$output = $this->execute($command);
$json = json_decode($output, true);
if ($json === null) {
throw new Exception("Command output is not valid JSON: " . json_last_error_msg());
}
return $json;
}
/**
* This method is much more complicated than just `exec`, but it also provide stderr output, so Exceptions
* can be much more specific.
*
* @param array $command
* @return string
* @throws Exception
*/
private function execute(array $command)
{
$descriptorspec = [
1 => ["pipe", "w"], // stdout
2 => ["pipe", "w"], // stderr
];
$command = implode(' ', $command);
$process = proc_open($command, $descriptorspec, $pipes);
if (!$process) {
throw new Exception("Command '$command' could be started.");
}
$stdout = stream_get_contents($pipes[1]);
if ($stdout === false) {
throw new Exception("Could not get STDOUT of command.");
}
fclose($pipes[1]);
$stderr = stream_get_contents($pipes[2]);
fclose($pipes[2]);
$returnCode = proc_close($process);
if ($returnCode !== 0) {
throw new Exception("Command '$command' return error code $returnCode. STDERR: '$stderr', STDOUT: '$stdout'");
}
return $stdout;
}
}

227
app/Lib/Tools/MalwareTool.php
View File

@ -1,227 +0,0 @@
<?php
class MalwareTool
{
const ZIP_PASSWORD = 'infected';
const ADVANCED_EXTRACTION_SCRIPT_PATH = APP . 'files/scripts/generate_file_objects.py';
/**
* @param string $originalFilename
* @param string $content
* @param string $md5
* @return string Content of zipped file
* @throws Exception
*/
public function encrypt($originalFilename, $content, $md5)
{
if (method_exists("ZipArchive", "setEncryptionName")) {
// When PHP zip extension is installed and supports creating encrypted archives.
return $this->encryptByExtension($originalFilename, $content, $md5);
} else {
return $this->encryptByCommand($originalFilename, $content, $md5);
}
}
/**
* @param string $originalFilename
* @param string $content
* @param string $md5
* @return string Content of zipped file
* @throws Exception
*/
private function encryptByCommand($originalFilename, $content, $md5)
{
$tempDir = $this->tempDir();
$contentsFile = new File($tempDir . DS . $md5, true);
if (!$contentsFile->write($content)) {
throw new Exception("Could not write content to file '{$contentsFile->path}'.");
}
$contentsFile->close();
$fileNameFile = new File($tempDir . DS . $md5 . '.filename.txt', true);
if (!$fileNameFile->write($originalFilename)) {
throw new Exception("Could not write original file name to file '{$fileNameFile->path}'.");
}
$fileNameFile->close();
$zipFile = new File($tempDir . DS . $md5 . '.zip');
$exec = [
'zip',
'-j', // junk (don't record) directory names
'-P', // use standard encryption
self::ZIP_PASSWORD,
escapeshellarg($zipFile->path),
escapeshellarg($contentsFile->path),
escapeshellarg($fileNameFile->path),
];
try {
$this->execute($exec);
$zipContent = $zipFile->read();
if ($zipContent === false) {
throw new Exception("Could not read content of newly created ZIP file.");
}
return $zipContent;
} catch (Exception $e) {
throw new Exception("Could not create encrypted ZIP file '{$zipFile->path}'.", 0, $e);
} finally {
$fileNameFile->delete();
$contentsFile->delete();
$zipFile->delete();
}
}
/**
* @param string $originalFilename
* @param string $content
* @param string $md5
* @return string Content of zipped file
* @throws Exception
*/
private function encryptByExtension($originalFilename, $content, $md5)
{
$zipFilePath = $this->tempFileName();
$zip = new ZipArchive();
$result = $zip->open($zipFilePath, ZipArchive::CREATE);
if ($result === true) {
$zip->setPassword(self::ZIP_PASSWORD);
$zip->addFromString($md5, $content);
$zip->setEncryptionName($md5, ZipArchive::EM_AES_128);
$zip->addFromString("$md5.filename.txt", $originalFilename);
$zip->setEncryptionName("$md5.filename.txt", ZipArchive::EM_AES_128);
$zip->close();
} else {
throw new Exception("Could not create encrypted ZIP file '$zipFilePath'. Error code: $result");
}
$zipFile = new File($zipFilePath);
$zipContent = $zipFile->read();
if ($zipContent === false) {
throw new Exception("Could not read content of newly created ZIP file.");
}
$zipFile->delete();
return $zipContent;
}
/**
* @param string $content
* @param array $hashTypes
* @return array
* @throws InvalidArgumentException
*/
public function computeHashes($content, array $hashTypes = array())
{
$validHashes = array('md5', 'sha1', 'sha256');
$hashes = [];
foreach ($hashTypes as $hashType) {
if (!in_array($hashType, $validHashes)) {
throw new InvalidArgumentException("Hash type '$hashType' is not valid hash type.");
}
$hashes[$hashType] = hash($hashType, $content);
}
return $hashes;
}
/**
* @param string $pythonBin
* @param string $filePath
* @return array
* @throws Exception
*/
public function advancedExtraction($pythonBin, $filePath)
{
return $this->executeAndParseJsonOutput([
$pythonBin,
self::ADVANCED_EXTRACTION_SCRIPT_PATH,
'-p',
escapeshellarg($filePath),
]);
}
/**
* @param string $pythonBin
* @return array
* @throws Exception
*/
public function checkAdvancedExtractionStatus($pythonBin)
{
return $this->executeAndParseJsonOutput([$pythonBin, self::ADVANCED_EXTRACTION_SCRIPT_PATH, '-c']);
}
private function tempFileName()
{
$randomName = (new RandomTool())->random_str(false, 12);
return $this->tempDir() . DS . $randomName;
}
/**
* @return string
*/
private function tempDir()
{
return Configure::read('MISP.tmpdir') ?: sys_get_temp_dir();
}
/**
* @param array $command
* @return array
* @throws Exception
*/
private function executeAndParseJsonOutput(array $command)
{
$output = $this->execute($command);
$json = json_decode($output, true);
if ($json === null) {
throw new Exception("Command output is not valid JSON: " . json_last_error_msg());
}
return $json;
}
/**
* This method is much more complicated than just `exec`, but it also provide stderr output, so Exceptions
* can be much more specific.
*
* @param array $command
* @return string
* @throws Exception
*/
private function execute(array $command)
{
$descriptorspec = [
1 => ["pipe", "w"], // stdout
2 => ["pipe", "w"], // stderr
];
$command = implode(' ', $command);
$process = proc_open($command, $descriptorspec, $pipes);
if (!$process) {
throw new Exception("Command '$command' could be started.");
}
$stdout = stream_get_contents($pipes[1]);
if ($stdout === false) {
throw new Exception("Could not get STDOUT of command.");
}
fclose($pipes[1]);
$stderr = stream_get_contents($pipes[2]);
fclose($pipes[2]);
$returnCode = proc_close($process);
if ($returnCode !== 0) {
throw new Exception("Command '$command' return error code $returnCode. STDERR: '$stderr', STDOUT: '$stdout'");
}
return $stdout;
}
}

39
app/Model/AppModel.php
View File

@ -44,7 +44,9 @@ class AppModel extends Model
private $__profiler = array();
public $elasticSearchClient = false;
public $s3Client = false;
/** @var AttachmentTool|null */
private $attachmentTool;
public function __construct($id = false, $table = null, $ds = null)
{
@ -2402,29 +2404,6 @@ class AppModel extends Model
$this->elasticSearchClient = $client;
}
public function getS3Client()
{
if (!$this->s3Client) {
$this->s3Client = $this->loadS3Client();
}
return $this->s3Client;
}
public function loadS3Client()
{
App::uses('AWSS3Client', 'Tools');
$client = new AWSS3Client();
$client->initTool();
return $client;
}
public function attachmentDirIsS3()
{
// Naive way to detect if we're working in S3
return substr(Configure::read('MISP.attachments_dir'), 0, 2) === "s3";
}
public function checkVersionRequirements($versionString, $minVersion)
{
$version = explode('.', $versionString);
@ -2935,4 +2914,16 @@ class AppModel extends Model
$input
);
}
/**
* @return AttachmentTool
*/
protected function loadAttachmentTool()
{
if ($this->attachmentTool === null) {
$this->attachmentTool = new AttachmentTool();
}
return $this->attachmentTool;
}
}

129
app/Model/Attribute.php
View File

@ -5,7 +5,7 @@ App::uses('Folder', 'Utility');
App::uses('File', 'Utility');
App::uses('FinancialTool', 'Tools');
App::uses('RandomTool', 'Tools');
App::uses('MalwareTool', 'Tools');
App::uses('AttachmentTool', 'Tools');
App::uses('TmpFileTool', 'Tools');
/**
@ -773,27 +773,7 @@ class Attribute extends AppModel
// delete attachments from the disk
$this->read(); // first read the attribute from the db
if ($this->typeIsAttachment($this->data['Attribute']['type'])) {
// only delete the file if it exists
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$attachments_dir = $this->getDefaultAttachments_dir();
}
// Special case - If using S3, we have to delete from there
if ($this->attachmentDirIsS3()) {
// We're working in S3
$s3 = $this->getS3Client();
$s3->delete($this->data['Attribute']['event_id'] . DS . $this->data['Attribute']['id']);
} else {
// Standard delete
$filepath = $attachments_dir . DS . $this->data['Attribute']['event_id'] . DS . $this->data['Attribute']['id'];
$file = new File($filepath);
if ($file->exists()) {
if (!$file->delete()) {
throw new InternalErrorException(__('Delete of file attachment failed. Please report to administrator.'));
}
}
}
$this->loadAttachmentTool()->delete($this->data['Attribute']['event_id'], $this->data['Attribute']['id']);
}
// update correlation..
$this->__beforeDeleteCorrelation($this->data['Attribute']['id']);
@ -1753,81 +1733,49 @@ class Attribute extends AppModel
public function typeIsMalware($type)
{
if (in_array($type, $this->zippedDefinitions)) {
return true;
} else {
return false;
}
return in_array($type, $this->zippedDefinitions);
}
public function typeIsAttachment($type)
{
if ((in_array($type, $this->zippedDefinitions)) || (in_array($type, $this->uploadDefinitions))) {
return true;
} else {
return false;
}
return in_array($type, $this->zippedDefinitions) || in_array($type, $this->uploadDefinitions);
}
public function getAttachment($attribute, $path_suffix='')
{
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$attachments_dir = $this->getDefaultAttachments_dir();
}
return $this->loadAttachmentTool()->getContent($attribute['event_id'], $attribute['id'], $path_suffix);
}
if ($this->attachmentDirIsS3()) {
// S3 - we have to first get the object then we can encode it
$s3 = $this->getS3Client();
// This will return the content of the object
$content = $s3->download($attribute['event_id'] . DS . $attribute['id'] . $path_suffix);
} else {
// Standard filesystem
$filepath = $attachments_dir . DS . $attribute['event_id'] . DS . $attribute['id'] . $path_suffix;
$file = new File($filepath);
if (!$file->readable()) {
return '';
}
$content = $file->read();
}
return $content;
/**
* @param array $attribute
* @param string $path_suffix
* @return File
* @throws Exception
*/
public function getAttachmentFile(array $attribute, $path_suffix='')
{
return $this->loadAttachmentTool()->getFile($attribute['event_id'], $attribute['id'], $path_suffix);
}
public function saveAttachment($attribute, $path_suffix='')
{
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$attachments_dir = $this->getDefaultAttachments_dir();
}
if ($this->attachmentDirIsS3()) {
// This is the cloud!
// We don't need your fancy directory structures and
// PEE AICH PEE meddling
$s3 = $this->getS3Client();
$data = $attribute['data'];
$key = $attribute['event_id'] . DS . $attribute['id'] . $path_suffix;
$s3->upload($key, $data);
return true;
} else {
// Plebian filesystem operations
$rootDir = $attachments_dir . DS . $attribute['event_id'];
$dir = new Folder($rootDir, true); // create directory structure
$destpath = $rootDir . DS . $attribute['id'] . $path_suffix;
$file = new File($destpath, true); // create the file
$decodedData = $attribute['data']; // decode
if ($file->write($decodedData)) { // save the data
return true;
} else {
// error
return false;
}
}
return $this->loadAttachmentTool()->save($attribute['event_id'], $attribute['id'], $attribute['data'], $path_suffix);
}
public function base64EncodeAttachment($attribute)
/**
* Returns attribute attachment content as base64 encoded string. If file doesn't exists, empty string is returned.
*
* @param array $attribute
* @return string
*/
public function base64EncodeAttachment(array $attribute)
{
return base64_encode($this->getAttachment($attribute));
try {
return base64_encode($this->getAttachment($attribute));
} catch (NotFoundException $e) {
$this->log($e->getMessage(), LOG_NOTICE);
return '';
}
}
public function saveBase64EncodedAttachment($attribute)
@ -1860,9 +1808,10 @@ class Attribute extends AppModel
if ($thumbnail && extension_loaded('gd')) {
if ($maxWidth == 200 && $maxHeight == 200) {
// Return thumbnail directly if already exists
$imageData = $this->getAttachment($attribute['Attribute'], $path_suffix='_thumbnail');
if ($imageData !== '') {
return $imageData;
try {
return $this->getAttachment($attribute['Attribute'], $path_suffix = '_thumbnail');
} catch (NotFoundException $e) {
// pass
}
}
@ -3626,10 +3575,10 @@ class Attribute extends AppModel
$content = base64_decode($base64);
$malwareTool = new MalwareTool();
$hashes = $malwareTool->computeHashes($content, $hash_types);
$attachmentTool = $this->loadAttachmentTool();
$hashes = $attachmentTool->computeHashes($content, $hash_types);
try {
$encrypted = $malwareTool->encrypt($original_filename, $content, $hashes['md5']);
$encrypted = $attachmentTool->encrypt($original_filename, $content, $hashes['md5']);
} catch (Exception $e) {
$this->logException("Could not create encrypted malware sample.", $e);
return array('success' => false);
@ -3644,9 +3593,8 @@ class Attribute extends AppModel
*/
public function isAdvancedExtractionAvailable()
{
$malwareTool = new MalwareTool();
try {
$types = $malwareTool->checkAdvancedExtractionStatus($this->getPythonVersion());
$types = $this->loadAttachmentTool()->checkAdvancedExtractionStatus($this->getPythonVersion());
} catch (Exception $e) {
return false;
}
@ -4043,9 +3991,8 @@ class Attribute extends AppModel
public function advancedAddMalwareSample($event_id, $attribute_settings, $filename, $tmpfile)
{
$malwareTool = new MalwareTool();
try {
$result = $malwareTool->advancedExtraction($this->getPythonVersion(), $tmpfile->path);
$result = $this->loadAttachmentTool()->advancedExtraction($this->getPythonVersion(), $tmpfile->path);
} catch (Exception $e) {
$this->logException("Could not finish advanced extraction", $e);
return $this->simpleAddMalwareSample($event_id, $attribute_settings, $filename, $tmpfile);

45
app/Model/Event.php
View File

@ -2,6 +2,7 @@
App::uses('AppModel', 'Model');
App::uses('CakeEmail', 'Network/Email');
App::uses('RandomTool', 'Tools');
App::uses('AttachmentTool', 'Tools');
App::uses('TmpFileTool', 'Tools');
class Event extends AppModel
@ -532,46 +533,12 @@ class Event extends AppModel
// delete all of the event->tag combinations that involve the deleted event
$this->EventTag->deleteAll(array('event_id' => $this->id));
// only delete the file if it exists
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$attachments_dir = $this->getDefaultAttachments_dir();
try {
$this->loadAttachmentTool()->deleteAll($this->id);
} catch (Exception $e) {
$this->logException('Delete of event file directory failed.', $e);
throw new InternalErrorException('Delete of event file directory failed. Please report to administrator.');
}
// Things get a little funky here
if ($this->attachmentDirIsS3()) {
// S3 doesn't have folders
// So we have to basically `ls` them to look for a prefix
$s3 = $this->getS3Client();
$s3->deleteDirectory($this->id);
} else {
$filepath = $attachments_dir . DS . $this->id;
App::uses('Folder', 'Utility');
if (is_dir($filepath)) {
if (!$this->destroyDir($filepath)) {
throw new InternalErrorException('Delete of event file directory failed. Please report to administrator.');
}
}
}
}
public function destroyDir($dir)
{
if (!is_dir($dir) || is_link($dir)) {
return unlink($dir);
}
foreach (scandir($dir) as $file) {
if ($file == '.' || $file == '..') {
continue;
}
if (!$this->destroyDir($dir . DS . $file)) {
chmod($dir . DS . $file, 0777);
if (!$this->destroyDir($dir . DS . $file)) {
return false;
}
}
}
return rmdir($dir);
}
public function beforeValidate($options = array())

107
app/Model/ShadowAttribute.php
View File

@ -3,6 +3,7 @@
App::uses('AppModel', 'Model');
App::uses('Folder', 'Utility');
App::uses('File', 'Utility');
App::uses('AttachmentTool', 'Tools');
/**
* @property Event $Event
@ -269,24 +270,7 @@ class ShadowAttribute extends AppModel
if (isset($this->data['ShadowAttribute']['deleted']) && $this->data['ShadowAttribute']['deleted']) {
$sa = $this->find('first', array('conditions' => array('ShadowAttribute.id' => $this->data['ShadowAttribute']['id']), 'recursive' => -1, 'fields' => array('ShadowAttribute.id', 'ShadowAttribute.event_id', 'ShadowAttribute.type')));
if ($this->typeIsAttachment($sa['ShadowAttribute']['type'])) {
// only delete the file if it exists
if ($this->attachmentDirIsS3()) {
$s3 = $this->getS3Client();
$s3->delete('shadow' . DS . $sa['ShadowAttribute']['event_id'] . DS . $sa['ShadowAttribute']['id']);
} else {
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . 'shadow' . DS . $sa['ShadowAttribute']['event_id'] . DS . $sa['ShadowAttribute']['id'];
$file = new File($filepath);
if ($file->exists()) {
if (!$file->delete()) {
throw new InternalErrorException('Delete of file attachment failed. Please report to administrator.');
}
}
}
$this->loadAttachmentTool()->deleteShadow($sa['ShadowAttribute']['event_id'], $sa['ShadowAttribute']['id']);
}
} else {
if (isset($this->data['ShadowAttribute']['type']) && $this->typeIsAttachment($this->data['ShadowAttribute']['type']) && !empty($this->data['ShadowAttribute']['data'])) {
@ -311,24 +295,7 @@ class ShadowAttribute extends AppModel
// delete attachments from the disk
$this->read(); // first read the attribute from the db
if ($this->typeIsAttachment($this->data['ShadowAttribute']['type'])) {
// only delete the file if it exists
if ($this->attachmentDirIsS3()) {
$s3 = $this->getS3Client();
$s3->delete('shadow' . DS . $this->data['ShadowAttribute']['event_id'] . DS . $this->data['ShadowAttribute']['id']);
} else {
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . 'shadow' . DS . $this->data['ShadowAttribute']['event_id'] . DS . $this->data['ShadowAttribute']['id'];
$file = new File($filepath);
if ($file->exists()) {
if (!$file->delete()) {
throw new InternalErrorException('Delete of file attachment failed. Please report to administrator.');
}
}
}
$this->loadAttachmentTool()->deleteShadow($this->data['ShadowAttribute']['event_id'], $this->data['ShadowAttribute']['id']);
}
}
@ -419,69 +386,35 @@ class ShadowAttribute extends AppModel
public function typeIsMalware($type)
{
if (in_array($type, $this->zippedDefinitions)) {
return true;
} else {
return false;
}
return in_array($type, $this->zippedDefinitions);
}
public function typeIsAttachment($type)
{
if ((in_array($type, $this->zippedDefinitions)) || (in_array($type, $this->uploadDefinitions))) {
return true;
} else {
return false;
}
return in_array($type, $this->zippedDefinitions) || in_array($type, $this->uploadDefinitions);
}
public function base64EncodeAttachment($attribute)
{
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
if ($this->attachmentDirIsS3()) {
$s3 = $this->getS3Client();
$content = $s3->download('shadow' . DS . $attribute['event_id'] . DS. $attribute['id']);
} else {
$filepath = $attachments_dir . DS . 'shadow' . DS . $attribute['event_id'] . DS. $attribute['id'];
$file = new File($filepath);
if (!$file->exists()) {
return '';
}
$content = $file->read();
}
$content = $this->loadAttachmentTool()->getShadowContent($attribute['event_id'], $attribute['id']);
return base64_encode($content);
}
public function saveBase64EncodedAttachment($attribute)
{
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
if ($this->attachmentDirIsS3()) {
$s3 = $this->getS3Client();
$decodedData = base64_decode($attribute['data']);
$s3->upload('shadow' . DS . $attribute['event_id'], $decodedData);
return true;
} else {
$rootDir = $attachments_dir . DS . 'shadow' . DS . $attribute['event_id'];
$dir = new Folder($rootDir, true); // create directory structure
$destpath = $rootDir . DS . $attribute['id'];
$file = new File($destpath, true); // create the file
$decodedData = base64_decode($attribute['data']); // decode
if ($file->write($decodedData)) { // save the data
return true;
} else {
// error
return false;
}
}
$data = base64_decode($attribute['data']);
return $this->loadAttachmentTool()->saveShadow($attribute['event_id'], $attribute['id'], $data);
}
/**
* @param array $shadowAttribute
* @param string $path_suffix
* @return File
* @throws Exception
*/
public function getAttachmentFile(array $shadowAttribute, $path_suffix='')
{
return $this->loadAttachmentTool()->getShadowFile($shadowAttribute['event_id'], $shadowAttribute['id'], $path_suffix);
}
public function checkComposites()
@ -587,7 +520,7 @@ class ShadowAttribute extends AppModel
'contactalert' => 1,
'disabled' => 0
),
'fields' => array('email', 'gpgkey', 'certif_public', 'contactalert', 'id')
'fields' => array('email', 'gpgkey', 'certif_public', 'contactalert', 'id', 'disabled'),
));
$body = "Hello, \n\n";

9
app/webroot/js/misp.js
View File

@ -335,18 +335,19 @@ function submitGenericForm(url, form, target) {
}
function acceptObject(type, id, event) {
name = '#ShadowAttribute_' + id + '_accept';
var name = '#ShadowAttribute_' + id + '_accept';
var formData = $(name).serialize();
$.ajax({
data: formData,
success:function (data, textStatus) {
success: function (data, textStatus) {
updateIndex(event, 'event');
eventUnpublish();
handleGenericAjaxResponse(data);
},
type:"post",
error: xhrFailCallback,
type: "post",
cache: false,
url:"/shadow_attributes/accept/" + id,
url: "/shadow_attributes/accept/" + id,
});
}