MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [internal] Refactoring malware handling

pull/5272/head
Jakub Onderka 2019-10-05 15:40:34 +02:00
parent c628375e96
commit 73b9513a38
3 changed files with 268 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/Controller/ServersController.php
View File

@ -985,13 +985,15 @@ class ServersController extends AppController
if ($tab == 'diagnostics' || $tab == 'download' || $this->_isRest()) {
$php_ini = php_ini_loaded_file();
$this->set('php_ini', $php_ini);
$advanced_attachments = shell_exec($this->Server->getPythonVersion() . ' ' . APP . 'files/scripts/generate_file_objects.py -c');
$malwareTool = new MalwareTool();
try {
$advanced_attachments = json_decode($advanced_attachments, true);
$advanced_attachments = $malwareTool->checkAdvancedExtractionStatus($this->Server->getPythonVersion());
} catch (Exception $e) {
$this->log($e->getMessage(), LOG_NOTICE);
$advanced_attachments = false;
}
$this->set('advanced_attachments', $advanced_attachments);
// check if the current version of MISP is outdated or not
$version = $this->__checkVersion();

227
app/Lib/Tools/MalwareTool.php Normal file
View File

@ -0,0 +1,227 @@
<?php
class MalwareTool
{
const ZIP_PASSWORD = 'infected';
const ADVANCED_EXTRACTION_SCRIPT_PATH = APP . 'files/scripts/generate_file_objects.py';
/**
* @param string $originalFilename
* @param string $content
* @param string $md5
* @return string Content of zipped file
* @throws Exception
*/
public function encrypt($originalFilename, $content, $md5)
{
if (method_exists("ZipArchive", "setEncryptionName")) {
// When PHP zip extension is installed and supports creating encrypted archives.
return $this->encryptByExtension($originalFilename, $content, $md5);
} else {
return $this->encryptByCommand($originalFilename, $content, $md5);
}
}
/**
* @param string $originalFilename
* @param string $content
* @param string $md5
* @return string Content of zipped file
* @throws Exception
*/
private function encryptByCommand($originalFilename, $content, $md5)
{
$tempDir = $this->tempDir();
$contentsFile = new File($tempDir . DS . $md5, true);
if (!$contentsFile->write($content)) {
throw new Exception("Could not write content to file '{$contentsFile->path}'.");
}
$contentsFile->close();
$fileNameFile = new File($tempDir . DS . $md5 . '.filename.txt', true);
if (!$fileNameFile->write($originalFilename)) {
throw new Exception("Could not write original file name to file '{$fileNameFile->path}'.");
}
$fileNameFile->close();
$zipFile = new File($tempDir . DS . $md5 . '.zip');
$exec = [
'zip',
'-j', // junk (don't record) directory names
'-P', // use standard encryption
self::ZIP_PASSWORD,
escapeshellarg($zipFile->path),
escapeshellarg($contentsFile->path),
escapeshellarg($fileNameFile->path),
];
try {
$this->execute($exec);
$zipContent = $zipFile->read();
if ($zipContent === false) {
throw new Exception("Could not read content of newly created ZIP file.");
}
return $zipContent;
} catch (Exception $e) {
throw new Exception("Could not create encrypted ZIP file '{$zipFile->path}'.", 0, $e);
} finally {
$fileNameFile->delete();
$contentsFile->delete();
$zipFile->delete();
}
}
/**
* @param string $originalFilename
* @param string $content
* @param string $md5
* @return string Content of zipped file
* @throws Exception
*/
private function encryptByExtension($originalFilename, $content, $md5)
{
$zipFilePath = $this->tempFileName();
$zip = new ZipArchive();
$result = $zip->open($zipFilePath, ZipArchive::CREATE);
if ($result === true) {
$zip->setPassword(self::ZIP_PASSWORD);
$zip->addFromString($md5, $content);
$zip->setEncryptionName($md5, ZipArchive::EM_AES_128);
$zip->addFromString("$md5.filename.txt", $originalFilename);
$zip->setEncryptionName("$md5.filename.txt", ZipArchive::EM_AES_128);
$zip->close();
} else {
throw new Exception("Could not create encrypted ZIP file '$zipFilePath'. Error code: $result");
}
$zipFile = new File($zipFilePath);
$zipContent = $zipFile->read();
if ($zipContent === false) {
throw new Exception("Could not read content of newly created ZIP file.");
}
$zipFile->delete();
return $zipContent;
}
/**
* @param string $content
* @param array $hashTypes
* @return array
* @throws InvalidArgumentException
*/
public function computeHashes($content, array $hashTypes = array())
{
$validHashes = array('md5', 'sha1', 'sha256');
$hashes = [];
foreach ($hashTypes as $hashType) {
if (!in_array($hashType, $validHashes)) {
throw new InvalidArgumentException("Hash type '$hashType' is not valid hash type.");
}
$hashes[$hashType] = hash($hashType, $content);
}
return $hashes;
}
/**
* @param string $pythonBin
* @param string $filePath
* @return array
* @throws Exception
*/
public function advancedExtraction($pythonBin, $filePath)
{
return $this->executeAndParseJsonOutput([
$pythonBin,
self::ADVANCED_EXTRACTION_SCRIPT_PATH,
'-p',
escapeshellarg($filePath),
]);
}
/**
* @param string $pythonBin
* @return array
* @throws Exception
*/
public function checkAdvancedExtractionStatus($pythonBin)
{
return $this->executeAndParseJsonOutput([$pythonBin, self::ADVANCED_EXTRACTION_SCRIPT_PATH, '-c']);
}
private function tempFileName()
{
$randomName = (new RandomTool())->random_str(false, 12);
return $this->tempDir() . DS . $randomName;
}
/**
* @return string
*/
private function tempDir()
{
return Configure::read('MISP.tmpdir') ?: sys_get_temp_dir();
}
/**
* @param array $command
* @return array
* @throws Exception
*/
private function executeAndParseJsonOutput(array $command)
{
$output = $this->execute($command);
$json = json_decode($output, true);
if ($json === null) {
throw new Exception("Command output is not valid JSON: " . json_last_error_msg());
}
return $json;
}
/**
* This method is much more complicated than just `exec`, but it also provide stderr output, so Exceptions
* can be much more specific.
*
* @param array $command
* @return string
* @throws Exception
*/
private function execute(array $command)
{
$descriptorspec = [
1 => ["pipe", "w"], // stdout
2 => ["pipe", "w"], // stderr
];
$command = implode(' ', $command);
$process = proc_open($command, $descriptorspec, $pipes);
if (!$process) {
throw new Exception("Command '$command' could be started.");
}
$stdout = stream_get_contents($pipes[1]);
if ($stdout === false) {
throw new Exception("Could not get STDOUT of command.");
}
fclose($pipes[1]);
$stderr = stream_get_contents($pipes[2]);
fclose($pipes[2]);
$returnCode = proc_close($process);
if ($returnCode !== 0) {
throw new Exception("Command '$command' return error code $returnCode. STDERR: '$stderr', STDOUT: '$stdout'");
}
return $stdout;
}
}

116
app/Model/Attribute.php
View File

@ -5,6 +5,7 @@ App::uses('Folder', 'Utility');
App::uses('File', 'Utility');
App::uses('FinancialTool', 'Tools');
App::uses('RandomTool', 'Tools');
App::uses('MalwareTool', 'Tools');
class Attribute extends AppModel
{
@ -3494,66 +3495,22 @@ class Attribute extends AppModel
if (!is_numeric($event_id)) {
throw new Exception(__('Something went wrong. Received a non-numeric event ID while trying to create a zip archive of an uploaded malware sample.'));
}
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$attachments_dir = $this->getDefaultAttachments_dir();
$content = base64_decode($base64);
$malwareTool = new MalwareTool();
$hashes = $malwareTool->computeHashes($content, $hash_types);
try {
$encrypted = $malwareTool->encrypt($original_filename, $content, $hashes['md5']);
} catch (Exception $e) {
$this->logException("Could not create encrypted malware sample.", $e);
return array('success' => false);
}
// If we've set attachments to S3, we can't write there
if ($this->attachmentDirIsS3()) {
$attachments_dir = Configure::read('MISP.tmpdir');
// Sometimes it's not set?
if (empty($attachments_dir)) {
// Get a default tmpdir
$attachments_dir = $this->getDefaultTmp_dir();
}
}
if ($proposal) {
$dir = new Folder($attachments_dir . DS . $event_id . DS . 'shadow', true);
} else {
$dir = new Folder($attachments_dir . DS . $event_id, true);
}
$tmpFile = new File($dir->path . DS . $this->generateRandomFileName(), true, 0600);
$tmpFile->write(base64_decode($base64));
$hashes = array();
foreach ($hash_types as $hash) {
$hashes[$hash] = $this->__hashRouter($hash, $tmpFile->path);
}
$contentsFile = new File($dir->path . DS . $hashes['md5']);
rename($tmpFile->path, $contentsFile->path);
$fileNameFile = new File($dir->path . DS . $hashes['md5'] . '.filename.txt');
$fileNameFile->write($original_filename);
$fileNameFile->close();
$zipFile = new File($dir->path . DS . $hashes['md5'] . '.zip');
exec('zip -j -P infected ' . escapeshellarg($zipFile->path) . ' ' . escapeshellarg($contentsFile->path) . ' ' . escapeshellarg($fileNameFile->path), $execOutput, $execRetval);
if ($execRetval != 0) {
$result = array('success' => false);
} else {
$result = array_merge(array('data' => base64_encode($zipFile->read()), 'success' => true), $hashes);
}
$fileNameFile->delete();
$zipFile->delete();
$contentsFile->delete();
$result = array_merge(array('data' => base64_encode($encrypted), 'success' => true), $hashes);
return $result;
}
private function __hashRouter($hashType, $file)
{
$validHashes = array('md5', 'sha1', 'sha256');
if (!in_array($hashType, $validHashes)) {
return false;
}
switch ($hashType) {
case 'md5':
case 'sha1':
case 'sha256':
return hash_file($hashType, $file);
break;
}
return false;
}
public function resolveHashType($hash)
{
$hashTypes = $this->hashTypes;
@ -3897,7 +3854,7 @@ class Attribute extends AppModel
'event_id' => $event_id,
'comment' => !empty($attribute_settings['comment']) ? $attribute_settings['comment'] : ''
);
$result = $this->Event->Attribute->handleMaliciousBase64($event_id, $filename, base64_encode($tmpfile->read()), $hashes);
$result = $this->handleMaliciousBase64($event_id, $filename, base64_encode($tmpfile->read()), $hashes);
foreach ($attributes as $k => $v) {
$attribute = array(
'distribution' => 5,
@ -3928,33 +3885,34 @@ class Attribute extends AppModel
public function advancedAddMalwareSample($event_id, $attribute_settings, $filename, $tmpfile)
{
$execRetval = '';
$execOutput = array();
$result = shell_exec($this->getPythonVersion() . ' ' . APP . 'files/scripts/generate_file_objects.py -p ' . $tmpfile->path);
if (!empty($result)) {
$result = json_decode($result, true);
if (isset($result['objects'])) {
$result['Object'] = $result['objects'];
unset($result['objects']);
}
if (isset($result['references'])) {
$result['ObjectReference'] = $result['references'];
unset($result['references']);
}
foreach ($result['Object'] as $k => $object) {
$result['Object'][$k]['distribution'] = $attribute_settings['distribution'];
$result['Object'][$k]['sharing_group_id'] = isset($attribute_settings['distribution']) ? $attribute_settings['distribution'] : 0;
if (!empty($result['Object'][$k]['Attribute'])) {
foreach ($result['Object'][$k]['Attribute'] as $k2 => $attribute) {
if ($attribute['value'] == $tmpfile->name) {
$result['Object'][$k]['Attribute'][$k2]['value'] = $filename;
}
$malwareTool = new MalwareTool();
try {
$result = $malwareTool->advancedExtraction($this->getPythonVersion(), $tmpfile->path);
} catch (Exception $e) {
$this->logException("Could not finish advanced extraction", $e);
return $this->simpleAddMalwareSample($event_id, $attribute_settings, $filename, $tmpfile);
}
if (isset($result['objects'])) {
$result['Object'] = $result['objects'];
unset($result['objects']);
}
if (isset($result['references'])) {
$result['ObjectReference'] = $result['references'];
unset($result['references']);
}
foreach ($result['Object'] as $k => $object) {
$result['Object'][$k]['distribution'] = $attribute_settings['distribution'];
$result['Object'][$k]['sharing_group_id'] = isset($attribute_settings['distribution']) ? $attribute_settings['distribution'] : 0;
if (!empty($result['Object'][$k]['Attribute'])) {
foreach ($result['Object'][$k]['Attribute'] as $k2 => $attribute) {
if ($attribute['value'] == $tmpfile->name) {
$result['Object'][$k]['Attribute'][$k2]['value'] = $filename;
}
}
}
} else {
$result = $this->simpleAddMalwareSample($event_id, $attribute_settings, $filename, $tmpfile);
}
return $result;
}