MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

new: [av] Malware protection for uploaded files

pull/6411/head
Jakub Onderka 2020-10-08 19:27:49 +02:00
parent 976f6f5357
commit 49660255fe
11 changed files with 612 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
app/Console/Command/AdminShell.php
View File

@ -583,7 +583,7 @@ class AdminShell extends AppShell
'db_version' => $dbVersion
);
$file = new File(ROOT . DS . 'db_schema.json', true);
$file->write(json_encode($data, JSON_PRETTY_PRINT));
$file->write(json_encode($data, JSON_PRETTY_PRINT) . "\n");
$file->close();
echo __("> Database schema dumped on disk") . PHP_EOL;
} else {
@ -640,4 +640,19 @@ class AdminShell extends AppShell
PHP_EOL, PHP_EOL, $ip, PHP_EOL, PHP_EOL, $user['User']['id'], $user['User']['email'], PHP_EOL, PHP_EOL
);
}
public function scanAttachment()
{
$input = $this->args[0];
$attributeId = isset($this->args[1]) ? $this->args[1] : null;
$jobId = isset($this->args[2]) ? $this->args[2] : null;
$this->loadModel('AttachmentScan');
$result = $this->AttachmentScan->scan($input, $attributeId, $jobId);
if ($result === false) {
echo 'Job failed' . PHP_EOL;
} else {
echo $result . PHP_EOL;
}
}
}

5
app/Controller/AttributesController.php
View File

@ -1623,6 +1623,7 @@ class AttributesController extends AppController
$this->Feed = ClassRegistry::init('Feed');
$this->loadModel('Sighting');
$this->loadModel('AttachmentScan');
$user = $this->Auth->user();
foreach ($attributes as $k => $attribute) {
$attributeId = $attribute['Attribute']['id'];
@ -1635,6 +1636,10 @@ class AttributesController extends AppController
}
$attributes[$k] = $attribute;
}
if ($attribute['Attribute']['type'] === 'attachment' && $this->AttachmentScan->isEnabled()) {
$infected = $this->AttachmentScan->isInfected(AttachmentScan::TYPE_ATTRIBUTE, $attribute['Attribute']['id']);
$attributes[$k]['Attribute']['infected'] = $infected;
}
if ($attribute['Attribute']['distribution'] == 4) {
$attributes[$k]['Attribute']['SharingGroup'] = $attribute['SharingGroup'];

3
app/Lib/Tools/AttachmentTool.php
View File

@ -33,7 +33,6 @@ class AttachmentTool
return $this->_exists(true, $eventId, $attributeId, $path_suffix);
}
/**
* @param bool $shadow
* @param int $eventId
@ -424,7 +423,7 @@ class AttachmentTool
* Naive way to detect if we're working in S3
* @return bool
*/
private function attachmentDirIsS3()
public function attachmentDirIsS3()
{
return substr(Configure::read('MISP.attachments_dir'), 0, 2) === "s3";
}

159
app/Lib/Tools/ClamAvTool.php Normal file
View File

@ -0,0 +1,159 @@
<?php
// TODO: Connection timeout
class ClamAvTool
{
/** @var resource */
private $socket;
/** @var string */
private $connectionString;
/**
* @param $connectionString
*/
public function __construct($connectionString)
{
$this->connectionString = $connectionString;
}
/**
* @throws Exception
*/
protected function connect()
{
if (is_resource($this->socket)) {
return;
}
if (strpos($this->connectionString, 'unix://') === 0) {
$socket = @socket_create(AF_UNIX, SOCK_STREAM, 0);
if ($socket === false) {
$this->socketException();
}
$path = substr($this->connectionString, 7);
$hasError = @socket_connect($socket, $path);
if ($hasError === false) {
$this->socketException($socket);
}
} else {
if (strpos(':', $this->connectionString) !== false) {
throw new InvalidArgumentException("Connection string must be in IP:PORT format.");
}
list ($address, $port) = explode(':', $this->connectionString);
$socket = @socket_create(AF_INET, SOCK_STREAM, 0);
if ($socket === false) {
$this->socketException();
}
$hasError = @socket_connect($socket, $address, $port);
if ($hasError === false) {
$this->socketException($socket);
}
}
$this->socket = $socket;
}
/**
* Returns version of ClamAV.
* @return array
* @throws Exception
*/
public function version()
{
$this->connect();
$this->send("zVERSION\0");
$result = $this->read();
list($version, $databaseVersion, $databaseDate) = explode("/", $result);
return array(
'version' => $version,
'databaseVersion' => $databaseVersion,
'databaseDate' => DateTime::createFromFormat('D M d H:i:s Y', $databaseDate),
);
}
/**
* @param resource $resource
* @return array
* @throws Exception
*/
public function scanResource($resource)
{
if (!is_resource($resource)) {
throw new InvalidArgumentException("Invalid resource");
}
$this->connect();
$this->send("zINSTREAM\0");
$this->streamResource($resource);
$result = $this->read();
list($type, $scanResult) = explode(': ', $result, 2);
if ($scanResult === 'OK') {
return array('found' => false);
} else {
$pos = strpos($scanResult, 'FOUND');
return array('found' => true, 'name' => trim(substr($scanResult, 0, $pos)));
}
}
/**
* @param resource $resource
* @return int Number of bytes written
* @throws Exception
*/
private function streamResource($resource)
{
$result = 0;
while ($chunk = fread($resource, 1024 * 1024)) {
$size = pack('N', strlen($chunk));
$result += $this->send($size . $chunk);
}
$result += $this->send(pack('N', 0));
return $result;
}
/**
* @param string $buf
* @param int $flags
* @return int
* @throws Exception
*/
private function send($buf, $flags = 0)
{
$len = strlen($buf);
if ($len !== socket_send($this->socket, $buf, $len, $flags)) {
throw new Exception("Not all data send to stream.");
}
return $len;
}
/**
* @param int $flags
* @return string
*/
private function read($flags = MSG_WAITALL)
{
$data = '';
while (socket_recv($this->socket, $chunk, 8192, $flags)) {
$data .= $chunk;
}
socket_close($this->socket);
$this->socket = null;
return rtrim($data);
}
/**
* @param resource|null $socket
* @throws Exception
*/
private function socketException($socket = null)
{
$code = socket_last_error($socket);
throw new Exception(socket_strerror($code), $code);
}
}

26
app/Model/AppModel.php
View File

@ -86,7 +86,7 @@ class AppModel extends Model
39 => false, 40 => false, 41 => false, 42 => false, 43 => false, 44 => false,
45 => false, 46 => false, 47 => false, 48 => false, 49 => false, 50 => false,
51 => false, 52 => false, 53 => false, 54 => false, 55 => false, 56 => false,
57 => false, 58 => false, 59 => false
57 => false, 58 => false, 59 => false, 60 => false,
);
public $advanced_updates_description = array(
@ -1436,6 +1436,18 @@ class AppModel extends Model
INDEX `name` (`name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;";
break;
case 60:
$sqlArray[] = "CREATE TABLE IF NOT EXISTS `attachment_scans` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`type` varchar(40) COLLATE utf8_bin NOT NULL,
`attribute_id` int(11) NOT NULL,
`infected` tinyint(1) NOT NULL,
`malware_name` varchar(191) NULL,
`timestamp` int(11) NOT NULL,
PRIMARY KEY (`id`),
INDEX `index` (`type`, `attribute_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;";
break;
case 'fixNonEmptySharingGroupID':
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
$sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
@ -2980,6 +2992,18 @@ class AppModel extends Model
return $this->attachmentTool;
}
/**
* @return AttachmentScan
*/
protected function loadAttachmentScan()
{
if ($this->AttachmentScan === null) {
$this->AttachmentScan = ClassRegistry::init('AttachmentScan');
}
return $this->AttachmentScan;
}
/**
* @return Log
*/

303
app/Model/AttachmentScan.php Normal file
View File

@ -0,0 +1,303 @@
<?php
App::uses('AppModel', 'Model');
App::uses('ClamAvTool', 'Tools');
class AttachmentScan extends AppModel
{
const TYPE_ATTRIBUTE = 'Attribute',
TYPE_SHADOW_ATTRIBUTE = 'ShadowAttribute';
/** @var AttachmentTool */
private $attachmentTool;
/** @var mixed|null */
private $clamAvConfig;
public function __construct($id = false, $table = null, $ds = null)
{
parent::__construct($id, $table, $ds);
$this->clamAvConfig = Configure::read('MISP.clam_av');
}
/**
* @return bool
*/
public function isEnabled()
{
return !empty($this->clamAvConfig);
}
/**
* @param string $type
* @param int $attributeId Attribute or Shadow Attribute ID
* @param bool $infected
* @param string|null $malwareName
* @return bool
* @throws Exception
*/
public function insertScan($type, $attributeId, $infected, $malwareName = null)
{
$this->checkType($type);
$this->create();
$result = $this->save(array(
'type' => $type,
'attribute_id' => $attributeId,
'infected' => $infected,
'malware_name' => $malwareName,
'timestamp' => time(),
));
if (!$result) {
throw new Exception("Could not save scan result for attribute $attributeId: " . json_encode($this->validationErrors));
}
}
/**
* @param string $type
* @param int $attributeId Attribute or Shadow Attribute ID
* @return array|null
*/
public function getLatestScan($type, $attributeId)
{
$this->checkType($type);
return $this->find('first', array(
'conditions' => array(
'type' => $type,
'attribute_id' => $attributeId,
),
'order_by' => 'timestamp DESC', // newest first
));
}
/**
* Checks if file is infected according to latest scan. Return values:
* - null - file was never checked
* - false - file is not infected according to latest scan
* - string - file is infected, string contains malware name
*
* @param string $type
* @param int $attributeId Attribute or Shadow Attribute ID
* @return bool|null|string
*/
public function isInfected($type, $attributeId)
{
$latest = $this->getLatestScan($type, $attributeId);
if (empty($latest)) {
return null;
}
if ($latest['AttachmentScan']['infected']) {
return $latest['AttachmentScan']['malware_name'];
} else {
return false;
}
}
/**
* @param string $type
* @param array $attribute
* @return bool|null Return true if attachment is infected.
* @throws Exception
*/
public function scanAttachment($type, array $attribute)
{
$this->checkType($type);
if (!isset($attribute['type'])) {
throw new InvalidArgumentException("Invalid attribute provided.");
}
if ($attribute['type'] !== 'attachment') {
throw new InvalidArgumentException("Just attachment attributes can be scanned, attribute with type '{$attribute['type']}' provided.");
}
if (!$this->isEnabled()) {
throw new Exception("ClamAV is not configured.");
}
if ($this->attachmentTool()->attachmentDirIsS3()) {
throw new Exception("S3 attachment storage is not supported now for AV scanning.");
}
if ($type === self::TYPE_ATTRIBUTE) {
$file = $this->attachmentTool()->getFile($attribute['event_id'], $attribute['id']);
} else {
$file = $this->attachmentTool()->getShadowFile($attribute['event_id'], $attribute['id']);
}
/* if ($file->size() > 50 * 1024 * 1024) {
$this->log("File '$file->path' is bigger than 50 MB, will be not scanned.", LOG_NOTICE);
return false;
}*/
if (!$file->open()) {
throw new Exception("Could not open file '$file->path' for reading.");
}
$clamAv = new ClamAvTool($this->clamAvConfig);
$output = $clamAv->scanResource($file->handle);
if ($output['found']) {
$this->insertScan($type, $attribute['id'], true, $output['name']);
return true;
} else {
$this->insertScan($type, $attribute['id'], false);
return false;
}
}
/**
* @param string $type
* @param int $attributeId Attribute or ShadowAttribute ID
* @param null $jobId
* @return bool|string
*/
public function scan($type, $attributeId = null, $jobId = null)
{
/** @var Job $job */
$job = ClassRegistry::init('Job');
if ($jobId && !$job->exists($jobId)) {
$jobId = null;
}
if ($type === 'all') {
$attributes = ClassRegistry::init('Attribute')->find('all', array(
'recursive' => -1,
'conditions' => ['type' => 'attachment'],
'fields' => ['id', 'type', 'event_id'],
));
$shadowAttributes = ClassRegistry::init('ShadowAttribute')->find('all', array(
'recursive' => -1,
'conditions' => ['type' => 'attachment'],
'fields' => ['id', 'type', 'event_id'],
));
$attributes = array_merge($attributes, $shadowAttributes);
} else if ($type === self::TYPE_ATTRIBUTE) {
$attributes = ClassRegistry::init('Attribute')->find('all', array(
'recursive' => -1,
'conditions' => ['type' => 'attachment', 'id' => $attributeId],
'fields' => ['id', 'type', 'event_id'],
));
} else if ($type === self::TYPE_SHADOW_ATTRIBUTE) {
$attributes = ClassRegistry::init('ShadowAttribute')->find('all', array(
'recursive' => -1,
'conditions' => ['type' => 'attachment', 'id' => $attributeId],
'fields' => ['id', 'type', 'event_id'],
));
} else {
throw new InvalidArgumentException("Input must be 'all', 'Attribute' or 'ShadowAttribute', '$type' provided.");
}
if (empty($attributes) && $type !== 'all') {
$message = "$type not found";
$job->saveStatus($jobId, false, $message);
return $message;
}
try {
// Try to connect to ClamAV before we will scan all files
$clamAv = new ClamAvTool($this->clamAvConfig);
$clamAvVersion = $clamAv->version();
$clamAvVersion = "{$clamAvVersion['version']}/{$clamAvVersion['databaseVersion']}";
} catch (Exception $e) {
$job->saveStatus($jobId, false, 'Could not get ClamAV version');
$this->logException('Could not get ClamAV version', $e);
return false;
}
$scanned = 0;
$fails = 0;
$virusFound = 0;
foreach ($attributes as $attribute) {
$type = isset($attribute['Attribute']) ? self::TYPE_ATTRIBUTE : self::TYPE_SHADOW_ATTRIBUTE;
try {
$infected = $this->scanAttachment($type, $attribute[$type]);
if ($infected === true) {
$virusFound++;
}
$scanned++;
} catch (NotFoundException $e) {
// skip
} catch (Exception $e) {
$this->logException("Could not scan attachment for $type {$attribute['Attribute']['id']}", $e);
$fails++;
}
$message = "$scanned files scanned, $virusFound malware files found (by ClamAV $clamAvVersion).";
$job->saveProgress($jobId, $message, ($scanned + $fails) / count($attributes) * 100);
}
if ($scanned === 0 && $fails > 0) {
$job->saveStatus($jobId, false);
return false;
} else {
$message = "$scanned files scanned, $virusFound malware files found (by ClamAV $clamAvVersion).";
if ($fails) {
$message .= " $fails files failed to scan (see error log for more details).";
}
$job->saveStatus($jobId, true, "Job done, $message");
return $message;
}
}
/**
* @param string $type
* @param array $attribute Attribute or Shadow Attribute
* @throws Exception
*/
public function backgroundScan($type, array $attribute)
{
$this->checkType($type);
$canScan = $attribute['type'] === 'attachment' &&
$this->isEnabled() &&
Configure::read('MISP.background_jobs') &&
!$this->attachmentTool()->attachmentDirIsS3();
if ($canScan) {
$job = ClassRegistry::init('Job');
$job->create();
$job->save(array(
'worker' => 'default',
'job_type' => 'virus_scan',
'job_input' => ($type === self::TYPE_ATTRIBUTE ? 'Attribute: ' : 'Shadow attribute: ') . $attribute['id'],
'status' => 0,
'retries' => 0,
'org' => 'SYSTEM',
'message' => 'Scanning...',
));
$jobId = $job->id;
$processId = CakeResque::enqueue(
'default',
'AdminShell',
array('scanAttachment', $type, $attribute['id'], $jobId),
true
);
$job->saveField('process_id', $processId);
}
}
/**
* @return AttachmentTool
*/
private function attachmentTool()
{
if (!$this->attachmentTool) {
$this->attachmentTool = new AttachmentTool();
}
return $this->attachmentTool;
}
/**
* @param string $type
* @raise InvalidArgumentException
*/
private function checkType($type)
{
if (!in_array($type, [self::TYPE_ATTRIBUTE, self::TYPE_SHADOW_ATTRIBUTE])) {
throw new InvalidArgumentException("Type must be 'Attribute' or 'ShadowAttribute', '$type' provided.");
}
}
}

6
app/Model/Attribute.php
View File

@ -1820,7 +1820,11 @@ class Attribute extends AppModel
public function saveAttachment($attribute, $path_suffix='')
{
return $this->loadAttachmentTool()->save($attribute['event_id'], $attribute['id'], $attribute['data'], $path_suffix);
$result = $this->loadAttachmentTool()->save($attribute['event_id'], $attribute['id'], $attribute['data'], $path_suffix);
if ($result) {
$this->loadAttachmentScan()->backgroundScan(AttachmentScan::TYPE_ATTRIBUTE, $attribute);
}
return $result;
}
/**

4
app/Model/Event.php
View File

@ -5371,6 +5371,10 @@ class Event extends AppModel
}
}
}
if ($object['type'] === 'attachment' && $this->loadAttachmentScan()->isEnabled()) {
$type = $object['objectType'] === 'attribute' ? AttachmentScan::TYPE_ATTRIBUTE : AttachmentScan::TYPE_SHADOW_ATTRIBUTE;
$object['infected'] = $this->loadAttachmentScan()->isInfected($type, $object['id']);;
}
return $object;
}

6
app/Model/ShadowAttribute.php
View File

@ -403,7 +403,11 @@ class ShadowAttribute extends AppModel
public function saveBase64EncodedAttachment($attribute)
{
$data = base64_decode($attribute['data']);
return $this->loadAttachmentTool()->saveShadow($attribute['event_id'], $attribute['id'], $data);
$result = $this->loadAttachmentTool()->saveShadow($attribute['event_id'], $attribute['id'], $data);
if ($result) {
$this->loadAttachmentScan()->backgroundScan(AttachmentScan::TYPE_SHADOW_ATTRIBUTE, $attribute);
}
return $result;
}
/**

15
app/View/Elements/Events/View/value_field.ctp
View File

@ -46,15 +46,28 @@ switch ($object['type']) {
}
if (isset($object['objectType'])) {
if (array_key_exists('infected', $object) && $object['infected'] !== false) { // it is not possible to use isset
if ($object['infected'] === null) {
$confirm = __('This file was not checked by AV scan. Do you really want to download it?');
} else {
$confirm = __('According to AV scan, this file contains %s malware. Do you really want to download it?', $object['infected']);
}
} else {
$confirm = null;
}
$controller = $object['objectType'] === 'proposal' ? 'shadow_attributes' : 'attributes';
$url = array('controller' => $controller, 'action' => 'download', $object['id']);
echo $this->Html->link($filename, $url, array('class' => $linkClass));
echo $this->Html->link($filename, $url, array('class' => $linkClass), $confirm);
} else {
echo $filename;
}
if (isset($filenameHash[1])) {
echo '<br>' . $filenameHash[1];
}
if (isset($object['infected']) && $object['infected'] !== false) {
echo ' <i class="fas fa-virus" title="' . __('This file contains malware %s', $object['infected']) . '"></i>';
}
}
break;

77
db_schema.json
View File

@ -59,6 +59,74 @@
"extra": ""
}
],
"attachment_scans": [
{
"column_name": "id",
"is_nullable": "NO",
"data_type": "int",
"character_maximum_length": null,
"numeric_precision": "10",
"collation_name": null,
"column_type": "int(11)",
"column_default": null,
"extra": "auto_increment"
},
{
"column_name": "type",
"is_nullable": "NO",
"data_type": "varchar",
"character_maximum_length": "40",
"numeric_precision": null,
"collation_name": "utf8_bin",
"column_type": "varchar(40)",
"column_default": null,
"extra": ""
},
{
"column_name": "attribute_id",
"is_nullable": "NO",
"data_type": "int",
"character_maximum_length": null,
"numeric_precision": "10",
"collation_name": null,
"column_type": "int(11)",
"column_default": null,
"extra": ""
},
{
"column_name": "infected",
"is_nullable": "NO",
"data_type": "tinyint",
"character_maximum_length": null,
"numeric_precision": "3",
"collation_name": null,
"column_type": "tinyint(1)",
"column_default": null,
"extra": ""
},
{
"column_name": "malware_name",
"is_nullable": "YES",
"data_type": "varchar",
"character_maximum_length": "191",
"numeric_precision": null,
"collation_name": "utf8mb4_general_ci",
"column_type": "varchar(191)",
"column_default": "NULL",
"extra": ""
},
{
"column_name": "timestamp",
"is_nullable": "NO",
"data_type": "int",
"character_maximum_length": null,
"numeric_precision": "10",
"collation_name": null,
"column_type": "int(11)",
"column_default": null,
"extra": ""
}
],
"attributes": [
{
"column_name": "id",
@ -6882,6 +6950,11 @@
"allowedlist": {
"id": true
},
"attachment_scans": {
"id": true,
"type": false,
"attribute_id": false
},
"attributes": {
"id": true,
"uuid": false,
@ -7275,5 +7348,5 @@
"id": true
}
},
"db_version": "59"
}
"db_version": "60"
}