MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch '2.4' of github.com:MISP/MISP into 2.4

pull/3570/head
iglocska 2018-08-17 14:47:42 +02:00
parent 1d7af1bfac fce069ffc3
commit 40f4ea4b75
10 changed files with 420 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
INSTALL/INSTALL.s3-attachments.txt Normal file
View File

@ -0,0 +1,97 @@
Using S3 as an attachment store
===============================
It is possible to use Amazon's Simple Storage Service (S3) to store event attachments
to allow for a stateless MISP setup (i.e for containerisation)
There's a massive caveat here so let me make this incredibly clear
##############################################
# WARNING WARNING WARNING #
# #
# Storing malware is against amazon's #
# terms of service. #
# #
# DO NOT USE THIS UNLESS YOU HAVE #
# THEIR EXPLICIT PERMISSION #
##############################################
0. Installing Dependencies
--------------------------
Install the AWS PHP SDK
```bash
cd /var/www/MISP/app
sudo -u www-data php composer.phar config vendor-dir Vendor
sudo -u www-data php composer.phar require aws/aws-sdk-php
```
1. Creating an S3 bucket
-------------------------
Go to https://s3.console.aws.amazon.com/s3/home
And create a bucket. It has to have a globally unique name, and
this cannot be changed later on.
2a. Using an EC2 instance for MISP
-----------------------------------
If you run MISP on EC2, this will be super duper easy peasy.
Simply create an IAM role with the following permissions and assign it to the instance
by right-clicking and selecting "Instance Settings -> Attach/Replace IAM role"
```json
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "PermitMISPAttachmentsToS3",
"Effect": "Allow",
"Action": [
"s3:*"
],
"Resource": [
"arn:aws:s3:::your-bucket-name"
]
}
]
}
```
2b. Using AWS access keys
-------------------------
This is not recommended, but it works I think.
Create a new programmatic access user via IAM and apply the same
policy outlined above.
Copy the access keys and save them for the next step
3. Setting up MISP
------------------
In Administration -> Server Settings & Maintenance -> MISP settings
Set MISP.attachments_dir to "s3://"
In Administration -> Server Settings & Maintenance -> Plugin Settings -> S3
Set S3_enable to True
Set S3_bucket-name to the bucket you created earlier
Set S3_region to your region
ONLY IF YOU DID NOT USE THE EC2 METHOD
Set aws_access_key and aws_secret_key to the ones you created in 2b
Now theoretically it should work.
Addendum
========
If you are migrating a server currently in use, simply copy the directory structure from
the attachments folder (usually /var/www/MISP/app/files) to S3 and everything should
continue to work.

29
app/Controller/AttributesController.php
View File

@ -433,8 +433,33 @@ class AttributesController extends AppController
$this->loadModel('Server');
$attachments_dir = $this->Server->getDefaultAttachments_dir();
}
$path = $attachments_dir . DS . $attribute['event_id'] . DS;
$file = $attribute['id'];
$is_s3 = substr($attachments_dir, 0, 2) === "s3";
if ($is_s3) {
// We have to download it!
App::uses('AWSS3Client', 'Tools');
$client = new AWSS3Client();
$client->initTool();
// Use tmpdir as opposed to attachments dir since we can't write to s3://
$attachments_dir = Configure::read('MISP.tmpdir');
if (empty($attachments_dir)) {
$this->loadModel('Server');
$attachments_dir = $this->Server->getDefaultTmp_dir();
}
// Now download the file
$resp = $client->download($attribute['event_id'] . DS . $attribute['id']);
// Save to a tmpfile
$tmpFile = new File($attachments_dir . DS . $attribute['uuid'], true, 0600);
$tmpFile->write($resp);
$tmpFile->close();
$path = $attachments_dir . DS;
$file = $attribute['uuid'];
} else {
$path = $attachments_dir . DS . $attribute['event_id'] . DS;
$file = $attribute['id'];
}
if ('attachment' == $attribute['type']) {
$filename = $attribute['value'];
$fileExt = pathinfo($filename, PATHINFO_EXTENSION);

94
app/Lib/Tools/AWSS3Client.php Normal file
View File

@ -0,0 +1,94 @@
<?php
use Aws\S3\S3Client;
class AWSS3Client
{
private $__settings = false;
private $__client = false;
private function __getSetSettings()
{
$settings = array(
'enabled' => false,
'bucket_name' => 'my-malware-bucket',
'region' => 'eu-west-1',
'aws_access_key' => '',
'aws_secret_key' => ''
);
// We have 2 situations
// Either we're running on EC2 and we can assume an IAM role
// Or we're not and need explicitly set AWS key
if (strlen($settings['aws_access_key']) > 0) {
putenv('AWS_ACCESS_KEY_ID='.$settings['aws_access_key']);
}
if (strlen($settings['aws_secret_key']) > 0) {
putenv('AWS_SECRET_ACCESS_KEY='.$settings['aws_secret_key']);
}
foreach ($settings as $key => $setting) {
$temp = Configure::read('Plugin.S3_' . $key);
if ($temp) {
$settings[$key] = $temp;
}
}
return $settings;
}
public function initTool()
{
$settings = $this->__getSetSettings();
$s3 = new Aws\S3\S3Client([
'version' => 'latest',
'region' => $settings['region']
]);
$this->__client = $s3;
$this->__settings = $settings;
return $s3;
}
public function upload($key, $data)
{
$this->__client->putObject([
'Bucket' => $this->__settings['bucket_name'],
'Key' => $key,
'Body' => $data
]);
}
public function download($key)
{
$result = $this->__client->getObject([
'Bucket' => $this->__settings['bucket_name'],
'Key' => $key
]);
return $result['Body'];
}
public function delete($key)
{
$this->__client->deleteObject([
'Bucket' => $this->__settings['bucket_name'],
'Key' => $key
]);
}
public function deleteDirectory($prefix) {
$keys = $s3->listObjects([
'Bucket' => $this->__settings['bucket_name'],
'Prefix' => $prefix
]) ->getPath('Contents/*/Key');
$s3->deleteObjects([
'Bucket' => $bucket,
'Delete' => [
'Objects' => array_map(function ($key) {
return ['Key' => $key];
}, $keys)
],
]);
}
}

21
app/Model/AppModel.php
View File

@ -38,6 +38,7 @@ class AppModel extends Model
private $__profiler = array();
public $elasticSearchClient = false;
public $s3Client = false;
public function __construct($id = false, $table = null, $ds = null)
{
@ -1457,6 +1458,26 @@ class AppModel extends Model
$this->elasticSearchClient = $client;
}
public function getS3Client() {
if (!$this->s3Client) {
$this->s3Client = $this->loadS3Client();
}
return $this->s3Client;
}
public function loadS3Client() {
App::uses('AWSS3Client', 'Tools');
$client = new AWSS3Client();
$client->initTool();
return $client;
}
public function attachmentDirIsS3() {
// Naive way to detect if we're working in S3
return substr(Configure::read('MISP.attachments_dir'), 0, 2) === "s3";
}
public function checkVersionRequirements($versionString, $minVersion)
{
$version = explode('.', $versionString);

81
app/Model/Attribute.php
View File

@ -661,11 +661,21 @@ class Attribute extends AppModel
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . $this->data['Attribute']['event_id'] . DS . $this->data['Attribute']['id'];
$file = new File($filepath);
if ($file->exists()) {
if (!$file->delete()) {
throw new InternalErrorException(__('Delete of file attachment failed. Please report to administrator.'));
// Special case - If using S3, we have to delete from there
if ($this->attachmentDirIsS3()) {
// We're working in S3
$s3 = $this->getS3Client();
$s3->delete($this->data['Attribute']['event_id'] . DS . $this->data['Attribute']['id']);
}
else {
// Standard delete
$filepath = $attachments_dir . DS . $this->data['Attribute']['event_id'] . DS . $this->data['Attribute']['id'];
$file = new File($filepath);
if ($file->exists()) {
if (!$file->delete()) {
throw new InternalErrorException(__('Delete of file attachment failed. Please report to administrator.'));
}
}
}
}
@ -1507,12 +1517,22 @@ class Attribute extends AppModel
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . $attribute['event_id'] . DS . $attribute['id'];
$file = new File($filepath);
if (!$file->readable()) {
return '';
if ($this->attachmentDirIsS3()) {
// S3 - we have to first get the object then we can encode it
$s3 = $this->getS3Client();
// This will return the content of the object
$content = $s3->download($attribute['event_id'] . DS . $attribute['id']);
} else {
// Standard filesystem
$filepath = $attachments_dir . DS . $attribute['event_id'] . DS . $attribute['id'];
$file = new File($filepath);
if (!$file->readable()) {
return '';
}
$content = $file->read();
}
$content = $file->read();
return base64_encode($content);
}
@ -1523,16 +1543,29 @@ class Attribute extends AppModel
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$rootDir = $attachments_dir . DS . $attribute['event_id'];
$dir = new Folder($rootDir, true); // create directory structure
$destpath = $rootDir . DS . $attribute['id'];
$file = new File($destpath, true); // create the file
$decodedData = base64_decode($attribute['data']); // decode
if ($file->write($decodedData)) { // save the data
if ($this->attachmentDirIsS3()) {
// This is the cloud!
// We don't need your fancy directory structures and
// PEE AICH PEE meddling
$s3 = $this->getS3Client();
$data = base64_decode($attribute['data']);
$key = $attribute['event_id'] . DS . $attribute['id'];
$s3->upload($key, $data);
return true;
} else {
// error
return false;
// Plebian filesystem operations
$rootDir = $attachments_dir . DS . $attribute['event_id'];
$dir = new Folder($rootDir, true); // create directory structure
$destpath = $rootDir . DS . $attribute['id'];
$file = new File($destpath, true); // create the file
$decodedData = base64_decode($attribute['data']); // decode
if ($file->write($decodedData)) { // save the data
return true;
} else {
// error
return false;
}
}
}
@ -2856,6 +2889,18 @@ class Attribute extends AppModel
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
// If we've set attachments to S3, we can't write there
if ($this->attachmentDirIsS3()) {
$attachments_dir = Configure::read('MISP.tmpdir');
// Sometimes it's not set?
if (empty($attachments_dir)) {
// Get a default tmpdir
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultTmp_dir();
}
}
if ($proposal) {
$dir = new Folder($attachments_dir . DS . $event_id . DS . 'shadow', true);
} else {

19
app/Model/Event.php
View File

@ -359,11 +359,20 @@ class Event extends AppModel
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . $this->id;
App::uses('Folder', 'Utility');
if (is_dir($filepath)) {
if (!$this->destroyDir($filepath)) {
throw new InternalErrorException('Delete of event file directory failed. Please report to administrator.');
// Things get a little funky here
if ($this->attachmentDirIsS3()) {
// S3 doesn't have folders
// So we have to basically `ls` them to look for a prefix
$s3 = $this->getS3Client();
$s3->deleteDirectory($this->id);
} else {
$filepath = $attachments_dir . DS . $this->id;
App::uses('Folder', 'Utility');
if (is_dir($filepath)) {
if (!$this->destroyDir($filepath)) {
throw new InternalErrorException('Delete of event file directory failed. Please report to administrator.');
}
}
}
}

45
app/Model/Server.php
View File

@ -1386,6 +1386,46 @@ class Server extends AppModel
'test' => 'testForEmpty',
'type' => 'string'
),
'S3_enable' => array(
'level' => 2,
'description' => __('Enables or disables uploading of malware samples to S3 rather than to disk (WARNING: Get permission from amazon first!)'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean'
),
'S3_bucket_name' => array(
'level' => 2,
'description' => __('Bucket name to upload to'),
'value' => '',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'S3_region' => array(
'level' => 2,
'description' => __('Region in which your S3 bucket resides'),
'value' => '',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'S3_aws_access_key' => array(
'level' => 2,
'description' => __('AWS key to use when uploading samples (WARNING: It\' highly recommended that you use EC2 IAM roles if at all possible)'),
'value' => '',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'S3_aws_secret_key' => array(
'level' => 2,
'description' => __('AWS secret key to use when uploading samples'),
'value' => '',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'Sightings_policy' => array(
'level' => 1,
'description' => __('This setting defines who will have access to seeing the reported sightings. The default setting is the event owner alone (in addition to everyone seeing their own contribution) with the other options being Sighting reporters (meaning the event owner and anyone that provided sighting data about the event) and Everyone (meaning anyone that has access to seeing the event / attribute).'),
@ -3981,6 +4021,11 @@ class Server extends AppModel
return APP . 'files';
}
public function getDefaultTmp_dir()
{
return sys_get_temp_dir();
}
public function fetchServer($id)
{
if (empty($id)) {

89
app/Model/ShadowAttribute.php
View File

@ -248,16 +248,21 @@ class ShadowAttribute extends AppModel
$sa = $this->find('first', array('conditions' => array('ShadowAttribute.id' => $this->data['ShadowAttribute']['id']), 'recursive' => -1, 'fields' => array('ShadowAttribute.id', 'ShadowAttribute.event_id', 'ShadowAttribute.type')));
if ($this->typeIsAttachment($sa['ShadowAttribute']['type'])) {
// only delete the file if it exists
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . 'shadow' . DS . $sa['ShadowAttribute']['event_id'] . DS . $sa['ShadowAttribute']['id'];
$file = new File($filepath);
if ($file->exists()) {
if (!$file->delete()) {
throw new InternalErrorException('Delete of file attachment failed. Please report to administrator.');
if ($this->attachmentDirIsS3()) {
$s3 = $this->getS3Client();
$s3->delete('shadow' . DS . $sa['ShadowAttribute']['event_id'] . DS . $sa['ShadowAttribute']['id']);
} else {
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . 'shadow' . DS . $sa['ShadowAttribute']['event_id'] . DS . $sa['ShadowAttribute']['id'];
$file = new File($filepath);
if ($file->exists()) {
if (!$file->delete()) {
throw new InternalErrorException('Delete of file attachment failed. Please report to administrator.');
}
}
}
}
@ -281,16 +286,21 @@ class ShadowAttribute extends AppModel
$this->read(); // first read the attribute from the db
if ($this->typeIsAttachment($this->data['ShadowAttribute']['type'])) {
// only delete the file if it exists
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . 'shadow' . DS . $this->data['ShadowAttribute']['event_id'] . DS . $this->data['ShadowAttribute']['id'];
$file = new File($filepath);
if ($file->exists()) {
if (!$file->delete()) {
throw new InternalErrorException('Delete of file attachment failed. Please report to administrator.');
if ($this->attachmentDirIsS3()) {
$s3 = $this->getS3Client();
$s3->delete('shadow' . DS . $this->data['ShadowAttribute']['event_id'] . DS . $this->data['ShadowAttribute']['id']);
} else {
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . 'shadow' . DS . $this->data['ShadowAttribute']['event_id'] . DS . $this->data['ShadowAttribute']['id'];
$file = new File($filepath);
if ($file->exists()) {
if (!$file->delete()) {
throw new InternalErrorException('Delete of file attachment failed. Please report to administrator.');
}
}
}
}
@ -394,12 +404,18 @@ class ShadowAttribute extends AppModel
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . 'shadow' . DS . $attribute['event_id'] . DS. $attribute['id'];
$file = new File($filepath);
if (!$file->exists()) {
return '';
if ($this->attachmentDirIsS3()) {
$s3 = $this->getS3Client();
$content = $s3->download('shadow' . DS . $attribute['event_id'] . DS. $attribute['id']);
} else {
$filepath = $attachments_dir . DS . 'shadow' . DS . $attribute['event_id'] . DS. $attribute['id'];
$file = new File($filepath);
if (!$file->exists()) {
return '';
}
$content = $file->read();
}
$content = $file->read();
return base64_encode($content);
}
@ -410,16 +426,23 @@ class ShadowAttribute extends AppModel
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$rootDir = $attachments_dir . DS . 'shadow' . DS . $attribute['event_id'];
$dir = new Folder($rootDir, true); // create directory structure
$destpath = $rootDir . DS . $attribute['id'];
$file = new File($destpath, true); // create the file
$decodedData = base64_decode($attribute['data']); // decode
if ($file->write($decodedData)) { // save the data
if ($this->attachmentDirIsS3()) {
$s3 = $this->getS3Client();
$decodedData = base64_decode($attribute['data']);
$s3->upload('shadow' . DS . $attribute['event_id'], $decodedData);
return true;
} else {
// error
return false;
$rootDir = $attachments_dir . DS . 'shadow' . DS . $attribute['event_id'];
$dir = new Folder($rootDir, true); // create directory structure
$destpath = $rootDir . DS . $attribute['id'];
$file = new File($destpath, true); // create the file
$decodedData = base64_decode($attribute['data']); // decode
if ($file->write($decodedData)) { // save the data
return true;
} else {
// error
return false;
}
}
}

3
app/composer.json
View File

@ -7,6 +7,7 @@
"pear/net_geoip": "@dev"
},
"suggest": {
"elasticsearch/elasticsearch": "For logging to elasticsearch"
"elasticsearch/elasticsearch": "For logging to elasticsearch",
"aws/aws-sdk-php": "To upload samples to S3"
}
}

2
app/files/misp-galaxy

@ -1 +1 @@
Subproject commit b9fc7e7552ee01139a41edcc981104e10381e92f
Subproject commit cd76f19f52e94a61f0d500fa3cdbf89a758e1c19