MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

new: Add upload/download for attachments

pull/3560/head
Hannah Ward 2018-08-15 14:07:44 +01:00
parent 597802501f
commit c883a7b6d6
No known key found for this signature in database
GPG Key ID: 6F3BAD60DE190290
6 changed files with 229 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
app/Lib/Tools/S3Client.php → app/Lib/Tools/AWSS3Client.php
View File

@ -46,25 +46,49 @@ class AWSS3Client
$this->__client = $s3;
$this->__settings = $settings;
return $client;
return $s3;
}
public function upload($key, $data)
{
$this->__client.putObject([
'Bucket' => $this.__settings['bucket_name'],
$this->__client->putObject([
'Bucket' => $this->__settings['bucket_name'],
'Key' => $key,
'Body' => $data
]);
]);
}
public function download($key)
{
$result = $this->__client.getObject([
'Bucket' => $this.__settings['bucket_name'],
$result = $this->__client->getObject([
'Bucket' => $this->__settings['bucket_name'],
'Key' => $key
]);
return $result['Body'];
}
public function delete($key)
{
$this->__client->deleteObject([
'Bucket' => $this->__settings['bucket_name'],
'Key' => $key
]);
}
public function deleteDirectory($prefix) {
$keys = $s3->listObjects([
'Bucket' => $this->__settings['bucket_name'],
'Prefix' => $prefix
]) ->getPath('Contents/*/Key');
$s3->deleteObjects([
'Bucket' => $bucket,
'Delete' => [
'Objects' => array_map(function ($key) {
return ['Key' => $key];
}, $keys)
],
]);
}
}

21
app/Model/AppModel.php
View File

@ -38,6 +38,7 @@ class AppModel extends Model
private $__profiler = array();
public $elasticSearchClient = false;
public $s3Client = false;
public function __construct($id = false, $table = null, $ds = null)
{
@ -1457,6 +1458,26 @@ class AppModel extends Model
$this->elasticSearchClient = $client;
}
public function getS3Client() {
if (!$this->s3Client) {
$this->s3Client = $this->loadS3Client();
}
return $this->s3Client;
}
public function loadS3Client() {
App::uses('AWSS3Client', 'Tools');
$client = new AWSS3Client();
$client->initTool();
return $client;
}
public function attachmentDirIsS3() {
// Naive way to detect if we're working in S3
return substr(Configure::read('MISP.attachments_dir'), 0, 2) === "s3";
}
public function checkVersionRequirements($versionString, $minVersion)
{
$version = explode('.', $versionString);

81
app/Model/Attribute.php
View File

@ -661,11 +661,21 @@ class Attribute extends AppModel
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . $this->data['Attribute']['event_id'] . DS . $this->data['Attribute']['id'];
$file = new File($filepath);
if ($file->exists()) {
if (!$file->delete()) {
throw new InternalErrorException(__('Delete of file attachment failed. Please report to administrator.'));
// Special case - If using S3, we have to delete from there
if ($this->attachmentDirIsS3()) {
// We're working in S3
$s3 = $this->getS3Client();
$s3->delete($this->data['Attribute']['event_id'] . DS . $this->data['Attribute']['id']);
}
else {
// Standard delete
$filepath = $attachments_dir . DS . $this->data['Attribute']['event_id'] . DS . $this->data['Attribute']['id'];
$file = new File($filepath);
if ($file->exists()) {
if (!$file->delete()) {
throw new InternalErrorException(__('Delete of file attachment failed. Please report to administrator.'));
}
}
}
}
@ -1507,12 +1517,22 @@ class Attribute extends AppModel
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . $attribute['event_id'] . DS . $attribute['id'];
$file = new File($filepath);
if (!$file->readable()) {
return '';
if ($this->attachmentDirIsS3()) {
// S3 - we have to first get the object then we can encode it
$s3 = $this->getS3Client();
// This will return the content of the object
$content = $s3->download($attribute['event_id'] . DS . $attribute['id']);
} else {
// Standard filesystem
$filepath = $attachments_dir . DS . $attribute['event_id'] . DS . $attribute['id'];
$file = new File($filepath);
if (!$file->readable()) {
return '';
}
$content = $file->read();
}
$content = $file->read();
return base64_encode($content);
}
@ -1523,16 +1543,29 @@ class Attribute extends AppModel
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$rootDir = $attachments_dir . DS . $attribute['event_id'];
$dir = new Folder($rootDir, true); // create directory structure
$destpath = $rootDir . DS . $attribute['id'];
$file = new File($destpath, true); // create the file
$decodedData = base64_decode($attribute['data']); // decode
if ($file->write($decodedData)) { // save the data
if ($this->attachmentDirIsS3()) {
// This is the cloud!
// We don't need your fancy directory structures and
// PEE AICH PEE meddling
$s3 = $this->getS3Client();
$data = base64_decode($attribute['data']);
$key = $attribute['event_id'] . DS . $attribute['id'];
$s3->upload($key, $data);
return true;
} else {
// error
return false;
// Plebian filesystem operations
$rootDir = $attachments_dir . DS . $attribute['event_id'];
$dir = new Folder($rootDir, true); // create directory structure
$destpath = $rootDir . DS . $attribute['id'];
$file = new File($destpath, true); // create the file
$decodedData = base64_decode($attribute['data']); // decode
if ($file->write($decodedData)) { // save the data
return true;
} else {
// error
return false;
}
}
}
@ -2856,6 +2889,18 @@ class Attribute extends AppModel
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
// If we've set attachments to S3, we can't write there
if ($this->attachmentDirIsS3()) {
$attachments_dir = Configure::read('MISP.tmpdir');
// Sometimes it's not set?
if (empty($attachments_dir)) {
// Get a default tmpdir
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultTmp_dir();
}
}
if ($proposal) {
$dir = new Folder($attachments_dir . DS . $event_id . DS . 'shadow', true);
} else {

19
app/Model/Event.php
View File

@ -359,11 +359,20 @@ class Event extends AppModel
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . $this->id;
App::uses('Folder', 'Utility');
if (is_dir($filepath)) {
if (!$this->destroyDir($filepath)) {
throw new InternalErrorException('Delete of event file directory failed. Please report to administrator.');
// Things get a little funky here
if ($this->attachmentDirIsS3()) {
// S3 doesn't have folders
// So we have to basically `ls` them to look for a prefix
$s3 = $this->getS3Client();
$s3.deleteDirectory($this->id);
} else {
$filepath = $attachments_dir . DS . $this->id;
App::uses('Folder', 'Utility');
if (is_dir($filepath)) {
if (!$this->destroyDir($filepath)) {
throw new InternalErrorException('Delete of event file directory failed. Please report to administrator.');
}
}
}
}

45
app/Model/Server.php
View File

@ -1386,6 +1386,46 @@ class Server extends AppModel
'test' => 'testForEmpty',
'type' => 'string'
),
'S3_enable' => array(
'level' => 2,
'description' => __('Enables or disables uploading of malware samples to S3 rather than to disk (WARNING: Get permission from amazon first!)'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean'
),
'S3_bucket_name' => array(
'level' => 2,
'description' => __('Bucket name to upload to'),
'value' => '',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'S3_region' => array(
'level' => 2,
'description' => __('Region in which your S3 bucket resides'),
'value' => '',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'S3_aws_access_key' => array(
'level' => 2,
'description' => __('AWS key to use when uploading samples (WARNING: It\' highly recommended that you use EC2 IAM roles if at all possible)'),
'value' => '',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'S3_aws_secret_key' => array(
'level' => 2,
'description' => __('AWS secret key to use when uploading samples'),
'value' => '',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'Sightings_policy' => array(
'level' => 1,
'description' => __('This setting defines who will have access to seeing the reported sightings. The default setting is the event owner alone (in addition to everyone seeing their own contribution) with the other options being Sighting reporters (meaning the event owner and anyone that provided sighting data about the event) and Everyone (meaning anyone that has access to seeing the event / attribute).'),
@ -3981,6 +4021,11 @@ class Server extends AppModel
return APP . 'files';
}
public function getDefaultTmp_dir()
{
return sys_get_temp_dir();
}
public function fetchServer($id)
{
if (empty($id)) {

89
app/Model/ShadowAttribute.php
View File

@ -248,16 +248,21 @@ class ShadowAttribute extends AppModel
$sa = $this->find('first', array('conditions' => array('ShadowAttribute.id' => $this->data['ShadowAttribute']['id']), 'recursive' => -1, 'fields' => array('ShadowAttribute.id', 'ShadowAttribute.event_id', 'ShadowAttribute.type')));
if ($this->typeIsAttachment($sa['ShadowAttribute']['type'])) {
// only delete the file if it exists
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . 'shadow' . DS . $sa['ShadowAttribute']['event_id'] . DS . $sa['ShadowAttribute']['id'];
$file = new File($filepath);
if ($file->exists()) {
if (!$file->delete()) {
throw new InternalErrorException('Delete of file attachment failed. Please report to administrator.');
if ($this->attachmentDirIsS3()) {
$s3 = $this->getS3Client();
$s3->delete('shadow' . DS . $sa['ShadowAttribute']['event_id'] . DS . $sa['ShadowAttribute']['id']);
} else {
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . 'shadow' . DS . $sa['ShadowAttribute']['event_id'] . DS . $sa['ShadowAttribute']['id'];
$file = new File($filepath);
if ($file->exists()) {
if (!$file->delete()) {
throw new InternalErrorException('Delete of file attachment failed. Please report to administrator.');
}
}
}
}
@ -281,16 +286,21 @@ class ShadowAttribute extends AppModel
$this->read(); // first read the attribute from the db
if ($this->typeIsAttachment($this->data['ShadowAttribute']['type'])) {
// only delete the file if it exists
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . 'shadow' . DS . $this->data['ShadowAttribute']['event_id'] . DS . $this->data['ShadowAttribute']['id'];
$file = new File($filepath);
if ($file->exists()) {
if (!$file->delete()) {
throw new InternalErrorException('Delete of file attachment failed. Please report to administrator.');
if ($this->attachmentDirIsS3()) {
$s3 = $this->getS3Client();
$s3->delete('shadow' . DS . $this->data['ShadowAttribute']['event_id'] . DS . $this->data['ShadowAttribute']['id']);
} else {
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . 'shadow' . DS . $this->data['ShadowAttribute']['event_id'] . DS . $this->data['ShadowAttribute']['id'];
$file = new File($filepath);
if ($file->exists()) {
if (!$file->delete()) {
throw new InternalErrorException('Delete of file attachment failed. Please report to administrator.');
}
}
}
}
@ -394,12 +404,18 @@ class ShadowAttribute extends AppModel
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . 'shadow' . DS . $attribute['event_id'] . DS. $attribute['id'];
$file = new File($filepath);
if (!$file->exists()) {
return '';
if ($this->attachmentDirIsS3()) {
$s3 = $this->getS3Client();
$content = $s3->download('shadow' . DS . $attribute['event_id'] . DS. $attribute['id']);
} else {
$filepath = $attachments_dir . DS . 'shadow' . DS . $attribute['event_id'] . DS. $attribute['id'];
$file = new File($filepath);
if (!$file->exists()) {
return '';
}
$content = $file->read();
}
$content = $file->read();
return base64_encode($content);
}
@ -410,16 +426,23 @@ class ShadowAttribute extends AppModel
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$rootDir = $attachments_dir . DS . 'shadow' . DS . $attribute['event_id'];
$dir = new Folder($rootDir, true); // create directory structure
$destpath = $rootDir . DS . $attribute['id'];
$file = new File($destpath, true); // create the file
$decodedData = base64_decode($attribute['data']); // decode
if ($file->write($decodedData)) { // save the data
if ($this->attachmentDirIsS3()) {
$s3 = $this->getS3Client();
$decodedData = base64_decode($attribute['data']);
$s3->upload('shadow' . DS . $attribute['event_id'], $decodedData);
return true;
} else {
// error
return false;
$rootDir = $attachments_dir . DS . 'shadow' . DS . $attribute['event_id'];
$dir = new Folder($rootDir, true); // create directory structure
$destpath = $rootDir . DS . $attribute['id'];
$file = new File($destpath, true); // create the file
$decodedData = base64_decode($attribute['data']); // decode
if ($file->write($decodedData)) { // save the data
return true;
} else {
// error
return false;
}
}
}