mirror of https://github.com/MISP/MISP
Merge branch '2.4' into feature/api_rework
commit
f675fb8b29
|
@ -0,0 +1,97 @@
|
|||
Using S3 as an attachment store
|
||||
===============================
|
||||
|
||||
It is possible to use Amazon's Simple Storage Service (S3) to store event attachments
|
||||
to allow for a stateless MISP setup (i.e for containerisation)
|
||||
|
||||
There's a massive caveat here so let me make this incredibly clear
|
||||
|
||||
##############################################
|
||||
# WARNING WARNING WARNING #
|
||||
# #
|
||||
# Storing malware is against amazon's #
|
||||
# terms of service. #
|
||||
# #
|
||||
# DO NOT USE THIS UNLESS YOU HAVE #
|
||||
# THEIR EXPLICIT PERMISSION #
|
||||
##############################################
|
||||
|
||||
0. Installing Dependencies
|
||||
--------------------------
|
||||
|
||||
Install the AWS PHP SDK
|
||||
|
||||
```bash
|
||||
cd /var/www/MISP/app
|
||||
sudo -u www-data php composer.phar config vendor-dir Vendor
|
||||
sudo -u www-data php composer.phar require aws/aws-sdk-php
|
||||
```
|
||||
|
||||
1. Creating an S3 bucket
|
||||
-------------------------
|
||||
|
||||
Go to https://s3.console.aws.amazon.com/s3/home
|
||||
|
||||
And create a bucket. It has to have a globally unique name, and
|
||||
this cannot be changed later on.
|
||||
|
||||
2a. Using an EC2 instance for MISP
|
||||
-----------------------------------
|
||||
|
||||
If you run MISP on EC2, this will be super duper easy peasy.
|
||||
|
||||
Simply create an IAM role with the following permissions and assign it to the instance
|
||||
by right-clicking and selecting "Instance Settings -> Attach/Replace IAM role"
|
||||
|
||||
```json
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "PermitMISPAttachmentsToS3",
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:*"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::your-bucket-name"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
2b. Using AWS access keys
|
||||
-------------------------
|
||||
|
||||
This is not recommended, but it works I think.
|
||||
|
||||
Create a new programmatic access user via IAM and apply the same
|
||||
policy outlined above.
|
||||
|
||||
Copy the access keys and save them for the next step
|
||||
|
||||
3. Setting up MISP
|
||||
------------------
|
||||
|
||||
In Administration -> Server Settings & Maintenance -> MISP settings
|
||||
|
||||
Set MISP.attachments_dir to "s3://"
|
||||
|
||||
In Administration -> Server Settings & Maintenance -> Plugin Settings -> S3
|
||||
|
||||
Set S3_enable to True
|
||||
Set S3_bucket-name to the bucket you created earlier
|
||||
Set S3_region to your region
|
||||
|
||||
ONLY IF YOU DID NOT USE THE EC2 METHOD
|
||||
Set aws_access_key and aws_secret_key to the ones you created in 2b
|
||||
|
||||
Now theoretically it should work.
|
||||
|
||||
Addendum
|
||||
========
|
||||
|
||||
If you are migrating a server currently in use, simply copy the directory structure from
|
||||
the attachments folder (usually /var/www/MISP/app/files) to S3 and everything should
|
||||
continue to work.
|
|
@ -128,7 +128,7 @@ cp -fa /home/misp/public_html/MISP/INSTALL/setup/config.php /home/misp/public_ht
|
|||
----------------------
|
||||
|
||||
# Check if the permissions are set correctly using the following commands:
|
||||
sudo chmod -R 750 /home/misp/public_html/MISP
|
||||
sudo chmod -R 770 /home/misp/public_html/MISP
|
||||
sudo chmod -R g+ws /home/misp/public_html/MISP/app/tmp
|
||||
sudo chmod -R g+ws /home/misp/public_html/MISP/app/files
|
||||
sudo chmod -R g+ws /home/misp/public_html/MISP/app/files/scripts/tmp
|
||||
|
@ -193,7 +193,7 @@ cp -a /home/misp/public_html/MISP/app/Config/core.default.php /home/misp/public_
|
|||
cp -a /home/misp/public_html/MISP/app/Config/config.default.php /home/misp/public_html/MISP/app/Config/config.php
|
||||
|
||||
# Configure the fields in the newly created files:
|
||||
vim /home/misp/public_html/MISP/app/Config/database.php
|
||||
vi /home/misp/public_html/MISP/app/Config/database.php
|
||||
# DATABASE_CONFIG has to be filled
|
||||
# With the default values provided in section 6, this would look like:
|
||||
# class DATABASE_CONFIG {
|
||||
|
@ -219,7 +219,7 @@ vim /home/misp/public_html/MISP/app/Config/database.php
|
|||
# delete the user from mysql and log in again using the default admin credentials (admin@admin.test / admin)
|
||||
|
||||
# Change base url in config.php
|
||||
vim /home/misp/public_html/MISP/app/Config/config.php
|
||||
vi /home/misp/public_html/MISP/app/Config/config.php
|
||||
# example: 'baseurl' => 'https://<your.FQDN.here>',
|
||||
# alternatively, you can leave this field empty if you would like to use relative pathing in MISP
|
||||
# 'baseurl' => '',
|
||||
|
@ -253,9 +253,49 @@ gpg --homedir /home/misp/public_html/MISP/.gnupg --export --armor YOUR-KEYS-EMAI
|
|||
|
||||
# To make the background workers start on boot
|
||||
chmod +x /home/misp/public_html/MISP/app/Console/worker/start.sh
|
||||
sudo vim /etc/rc.local
|
||||
# Add the following line before the last line (exit 0). Make sure that you replace www-data with your apache user:
|
||||
sudo -u www-data bash /home/misp/public_html/MISP/app/Console/worker/start.sh
|
||||
|
||||
# Activate rc.local in systemd
|
||||
# Systemd developers, in their wisdom, decided to complicate things a bit so you'll have to
|
||||
# create the rc-local.service
|
||||
sudo vi /etc/systemd/system/rc-local.service
|
||||
# and paste the following in it
|
||||
[Unit]
|
||||
Description=/etc/rc.local Compatibility
|
||||
ConditionPathExists=/etc/rc.local
|
||||
|
||||
[Service]
|
||||
Type=forking
|
||||
ExecStart=/etc/rc.local start
|
||||
TimeoutSec=0
|
||||
StandardOutput=tty
|
||||
RemainAfterExit=yes
|
||||
SysVStartPriority=99
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
# Hit the "esc" button then type :wq! to write the file and exit vi
|
||||
|
||||
# Create/edit /etc/rc.local
|
||||
sudo vi /etc/rc.local
|
||||
# If the file is empty add the following including the #
|
||||
#!/bin/bash
|
||||
|
||||
# Then add this
|
||||
sudo -u misp bash /home/misp/public_html/MISP/app/Console/worker/start.sh
|
||||
|
||||
# If the file was empty add this as the last line
|
||||
exit 0
|
||||
|
||||
# save, quit vi and set permissions
|
||||
sudo chmod +x /etc/rc.local
|
||||
|
||||
# Enable it in systemd
|
||||
sudo systemctl enable rc-local
|
||||
|
||||
#Start the rc-local compatibility layer and check if AOK
|
||||
sudo systemctl start rc-local.service
|
||||
sudo systemctl status rc-local.service
|
||||
|
||||
# Now log in using the webinterface:
|
||||
# The default user/pass = admin@admin.test/admin
|
||||
|
@ -271,8 +311,8 @@ sudo -u www-data bash /home/misp/public_html/MISP/app/Console/worker/start.sh
|
|||
# If any of the directories that MISP uses to store files is not writeable to the apache user, change the permissions
|
||||
# you can do this by running the following commands:
|
||||
|
||||
sudo chmod -R 750 /home/misp/public_html/MISP/<directory path with an indicated issue>
|
||||
sudo chown -R misp:misp /home/misp/public_html/MISP/<directory path with an indicated issue>
|
||||
sudo chmod -R 770 /home/misp/public_html/MISP/<directory path with an indicated issue>
|
||||
sudo chown -R misp:www-data /home/misp/public_html/MISP/<directory path with an indicated issue>
|
||||
|
||||
# Make sure that the STIX libraries and GnuPG work as intended, if not, refer to INSTALL.txt's paragraphs dealing with these two items
|
||||
|
||||
|
|
|
@ -433,8 +433,33 @@ class AttributesController extends AppController
|
|||
$this->loadModel('Server');
|
||||
$attachments_dir = $this->Server->getDefaultAttachments_dir();
|
||||
}
|
||||
$path = $attachments_dir . DS . $attribute['event_id'] . DS;
|
||||
$file = $attribute['id'];
|
||||
|
||||
$is_s3 = substr($attachments_dir, 0, 2) === "s3";
|
||||
|
||||
if ($is_s3) {
|
||||
// We have to download it!
|
||||
App::uses('AWSS3Client', 'Tools');
|
||||
$client = new AWSS3Client();
|
||||
$client->initTool();
|
||||
// Use tmpdir as opposed to attachments dir since we can't write to s3://
|
||||
$attachments_dir = Configure::read('MISP.tmpdir');
|
||||
if (empty($attachments_dir)) {
|
||||
$this->loadModel('Server');
|
||||
$attachments_dir = $this->Server->getDefaultTmp_dir();
|
||||
}
|
||||
// Now download the file
|
||||
$resp = $client->download($attribute['event_id'] . DS . $attribute['id']);
|
||||
// Save to a tmpfile
|
||||
$tmpFile = new File($attachments_dir . DS . $attribute['uuid'], true, 0600);
|
||||
$tmpFile->write($resp);
|
||||
$tmpFile->close();
|
||||
$path = $attachments_dir . DS;
|
||||
$file = $attribute['uuid'];
|
||||
} else {
|
||||
$path = $attachments_dir . DS . $attribute['event_id'] . DS;
|
||||
$file = $attribute['id'];
|
||||
}
|
||||
|
||||
if ('attachment' == $attribute['type']) {
|
||||
$filename = $attribute['value'];
|
||||
$fileExt = pathinfo($filename, PATHINFO_EXTENSION);
|
||||
|
@ -877,10 +902,12 @@ class AttributesController extends AppController
|
|||
if (count($existingAttribute) && !$existingAttribute['Attribute']['deleted']) {
|
||||
$this->request->data['Attribute']['id'] = $existingAttribute['Attribute']['id'];
|
||||
$dateObj = new DateTime();
|
||||
$skipTimeCheck = false;
|
||||
if (!isset($this->request->data['Attribute']['timestamp'])) {
|
||||
$this->request->data['Attribute']['timestamp'] = $dateObj->getTimestamp();
|
||||
$skipTimeCheck = true;
|
||||
}
|
||||
if ($this->request->data['Attribute']['timestamp'] > $existingAttribute['Attribute']['timestamp']) {
|
||||
if ($skipTimeCheck || $this->request->data['Attribute']['timestamp'] > $existingAttribute['Attribute']['timestamp']) {
|
||||
$recoverFields = array('value', 'to_ids', 'distribution', 'category', 'type', 'comment');
|
||||
foreach ($recoverFields as $rF) {
|
||||
if (!isset($this->request->data['Attribute'][$rF])) {
|
||||
|
@ -2102,7 +2129,7 @@ class AttributesController extends AppController
|
|||
}
|
||||
$data = $this->request->data;
|
||||
} else {
|
||||
throw new BadRequestException(__('Either specify the search terms in the url, or POST a json array / xml (with the root element being "request" and specify the correct accept and content type headers.'));
|
||||
throw new BadRequestException(__('Either specify the search terms in the url, or POST a json array / xml (with the root element being "request" and specify the correct accept and content type headers).'));
|
||||
}
|
||||
if (!isset($data['request'])) {
|
||||
$data['request'] = $data;
|
||||
|
@ -2260,7 +2287,7 @@ class AttributesController extends AppController
|
|||
} elseif ($this->response->type() === 'application/xml' && !empty($this->request->data)) {
|
||||
$data = $this->request->data;
|
||||
} else {
|
||||
throw new BadRequestException(__('Either specify the search terms in the url, or POST a json array / xml (with the root element being "request" and specify the correct accept and content type headers.'));
|
||||
throw new BadRequestException(__('Either specify the search terms in the url, or POST a json array / xml (with the root element being "request" and specify the correct accept and content type headers).'));
|
||||
}
|
||||
$paramArray = array('type', 'sigOnly');
|
||||
foreach ($paramArray as $p) {
|
||||
|
|
|
@ -164,6 +164,9 @@ class UsersController extends AppController
|
|||
|
||||
public function change_pw()
|
||||
{
|
||||
if (!$this->_isAdmin() && Configure::read('MISP.disableUserSelfManagement')) {
|
||||
throw new MethodNotAllowedException('User self-management has been disabled on this instance.');
|
||||
}
|
||||
$id = $this->Auth->user('id');
|
||||
$user = $this->User->find('first', array(
|
||||
'conditions' => array('User.id' => $id),
|
||||
|
@ -1724,8 +1727,12 @@ class UsersController extends AppController
|
|||
$statistics[$scope]['data'][$range] = $this->{$scope_data['model']}->find('count', $params);
|
||||
}
|
||||
}
|
||||
$this->set('statistics', $statistics);
|
||||
$this->render('statistics_users');
|
||||
if ($this->_isRest()) {
|
||||
return $this->RestResponse->viewData($statistics, $this->response->type());
|
||||
} else {
|
||||
$this->set('statistics', $statistics);
|
||||
$this->render('statistics_users');
|
||||
}
|
||||
}
|
||||
|
||||
public function tagStatisticsGraph()
|
||||
|
|
|
@ -0,0 +1,94 @@
|
|||
<?php
|
||||
|
||||
use Aws\S3\S3Client;
|
||||
|
||||
class AWSS3Client
|
||||
{
|
||||
private $__settings = false;
|
||||
private $__client = false;
|
||||
|
||||
private function __getSetSettings()
|
||||
{
|
||||
$settings = array(
|
||||
'enabled' => false,
|
||||
'bucket_name' => 'my-malware-bucket',
|
||||
'region' => 'eu-west-1',
|
||||
'aws_access_key' => '',
|
||||
'aws_secret_key' => ''
|
||||
);
|
||||
|
||||
// We have 2 situations
|
||||
// Either we're running on EC2 and we can assume an IAM role
|
||||
// Or we're not and need explicitly set AWS key
|
||||
if (strlen($settings['aws_access_key']) > 0) {
|
||||
putenv('AWS_ACCESS_KEY_ID='.$settings['aws_access_key']);
|
||||
}
|
||||
if (strlen($settings['aws_secret_key']) > 0) {
|
||||
putenv('AWS_SECRET_ACCESS_KEY='.$settings['aws_secret_key']);
|
||||
}
|
||||
|
||||
foreach ($settings as $key => $setting) {
|
||||
$temp = Configure::read('Plugin.S3_' . $key);
|
||||
if ($temp) {
|
||||
$settings[$key] = $temp;
|
||||
}
|
||||
}
|
||||
return $settings;
|
||||
}
|
||||
|
||||
public function initTool()
|
||||
{
|
||||
$settings = $this->__getSetSettings();
|
||||
$s3 = new Aws\S3\S3Client([
|
||||
'version' => 'latest',
|
||||
'region' => $settings['region']
|
||||
]);
|
||||
|
||||
$this->__client = $s3;
|
||||
$this->__settings = $settings;
|
||||
return $s3;
|
||||
}
|
||||
|
||||
public function upload($key, $data)
|
||||
{
|
||||
$this->__client->putObject([
|
||||
'Bucket' => $this->__settings['bucket_name'],
|
||||
'Key' => $key,
|
||||
'Body' => $data
|
||||
]);
|
||||
}
|
||||
|
||||
public function download($key)
|
||||
{
|
||||
$result = $this->__client->getObject([
|
||||
'Bucket' => $this->__settings['bucket_name'],
|
||||
'Key' => $key
|
||||
]);
|
||||
|
||||
return $result['Body'];
|
||||
}
|
||||
|
||||
public function delete($key)
|
||||
{
|
||||
$this->__client->deleteObject([
|
||||
'Bucket' => $this->__settings['bucket_name'],
|
||||
'Key' => $key
|
||||
]);
|
||||
}
|
||||
|
||||
public function deleteDirectory($prefix) {
|
||||
$keys = $s3->listObjects([
|
||||
'Bucket' => $this->__settings['bucket_name'],
|
||||
'Prefix' => $prefix
|
||||
]) ->getPath('Contents/*/Key');
|
||||
|
||||
$s3->deleteObjects([
|
||||
'Bucket' => $bucket,
|
||||
'Delete' => [
|
||||
'Objects' => array_map(function ($key) {
|
||||
return ['Key' => $key];
|
||||
}, $keys)
|
||||
],
|
||||
]);
|
||||
}
|
||||
}
|
|
@ -38,6 +38,7 @@ class AppModel extends Model
|
|||
private $__profiler = array();
|
||||
|
||||
public $elasticSearchClient = false;
|
||||
public $s3Client = false;
|
||||
|
||||
public function __construct($id = false, $table = null, $ds = null)
|
||||
{
|
||||
|
@ -1457,6 +1458,26 @@ class AppModel extends Model
|
|||
$this->elasticSearchClient = $client;
|
||||
}
|
||||
|
||||
public function getS3Client() {
|
||||
if (!$this->s3Client) {
|
||||
$this->s3Client = $this->loadS3Client();
|
||||
}
|
||||
|
||||
return $this->s3Client;
|
||||
}
|
||||
|
||||
public function loadS3Client() {
|
||||
App::uses('AWSS3Client', 'Tools');
|
||||
$client = new AWSS3Client();
|
||||
$client->initTool();
|
||||
return $client;
|
||||
}
|
||||
|
||||
public function attachmentDirIsS3() {
|
||||
// Naive way to detect if we're working in S3
|
||||
return substr(Configure::read('MISP.attachments_dir'), 0, 2) === "s3";
|
||||
}
|
||||
|
||||
public function checkVersionRequirements($versionString, $minVersion)
|
||||
{
|
||||
$version = explode('.', $versionString);
|
||||
|
|
|
@ -661,11 +661,21 @@ class Attribute extends AppModel
|
|||
$my_server = ClassRegistry::init('Server');
|
||||
$attachments_dir = $my_server->getDefaultAttachments_dir();
|
||||
}
|
||||
$filepath = $attachments_dir . DS . $this->data['Attribute']['event_id'] . DS . $this->data['Attribute']['id'];
|
||||
$file = new File($filepath);
|
||||
if ($file->exists()) {
|
||||
if (!$file->delete()) {
|
||||
throw new InternalErrorException(__('Delete of file attachment failed. Please report to administrator.'));
|
||||
|
||||
// Special case - If using S3, we have to delete from there
|
||||
if ($this->attachmentDirIsS3()) {
|
||||
// We're working in S3
|
||||
$s3 = $this->getS3Client();
|
||||
$s3->delete($this->data['Attribute']['event_id'] . DS . $this->data['Attribute']['id']);
|
||||
}
|
||||
else {
|
||||
// Standard delete
|
||||
$filepath = $attachments_dir . DS . $this->data['Attribute']['event_id'] . DS . $this->data['Attribute']['id'];
|
||||
$file = new File($filepath);
|
||||
if ($file->exists()) {
|
||||
if (!$file->delete()) {
|
||||
throw new InternalErrorException(__('Delete of file attachment failed. Please report to administrator.'));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -1507,12 +1517,22 @@ class Attribute extends AppModel
|
|||
$my_server = ClassRegistry::init('Server');
|
||||
$attachments_dir = $my_server->getDefaultAttachments_dir();
|
||||
}
|
||||
$filepath = $attachments_dir . DS . $attribute['event_id'] . DS . $attribute['id'];
|
||||
$file = new File($filepath);
|
||||
if (!$file->readable()) {
|
||||
return '';
|
||||
|
||||
if ($this->attachmentDirIsS3()) {
|
||||
// S3 - we have to first get the object then we can encode it
|
||||
$s3 = $this->getS3Client();
|
||||
// This will return the content of the object
|
||||
$content = $s3->download($attribute['event_id'] . DS . $attribute['id']);
|
||||
} else {
|
||||
// Standard filesystem
|
||||
$filepath = $attachments_dir . DS . $attribute['event_id'] . DS . $attribute['id'];
|
||||
$file = new File($filepath);
|
||||
if (!$file->readable()) {
|
||||
return '';
|
||||
}
|
||||
$content = $file->read();
|
||||
}
|
||||
$content = $file->read();
|
||||
|
||||
return base64_encode($content);
|
||||
}
|
||||
|
||||
|
@ -1523,16 +1543,29 @@ class Attribute extends AppModel
|
|||
$my_server = ClassRegistry::init('Server');
|
||||
$attachments_dir = $my_server->getDefaultAttachments_dir();
|
||||
}
|
||||
$rootDir = $attachments_dir . DS . $attribute['event_id'];
|
||||
$dir = new Folder($rootDir, true); // create directory structure
|
||||
$destpath = $rootDir . DS . $attribute['id'];
|
||||
$file = new File($destpath, true); // create the file
|
||||
$decodedData = base64_decode($attribute['data']); // decode
|
||||
if ($file->write($decodedData)) { // save the data
|
||||
|
||||
if ($this->attachmentDirIsS3()) {
|
||||
// This is the cloud!
|
||||
// We don't need your fancy directory structures and
|
||||
// PEE AICH PEE meddling
|
||||
$s3 = $this->getS3Client();
|
||||
$data = base64_decode($attribute['data']);
|
||||
$key = $attribute['event_id'] . DS . $attribute['id'];
|
||||
$s3->upload($key, $data);
|
||||
return true;
|
||||
} else {
|
||||
// error
|
||||
return false;
|
||||
// Plebian filesystem operations
|
||||
$rootDir = $attachments_dir . DS . $attribute['event_id'];
|
||||
$dir = new Folder($rootDir, true); // create directory structure
|
||||
$destpath = $rootDir . DS . $attribute['id'];
|
||||
$file = new File($destpath, true); // create the file
|
||||
$decodedData = base64_decode($attribute['data']); // decode
|
||||
if ($file->write($decodedData)) { // save the data
|
||||
return true;
|
||||
} else {
|
||||
// error
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -2894,6 +2927,18 @@ class Attribute extends AppModel
|
|||
$my_server = ClassRegistry::init('Server');
|
||||
$attachments_dir = $my_server->getDefaultAttachments_dir();
|
||||
}
|
||||
|
||||
// If we've set attachments to S3, we can't write there
|
||||
if ($this->attachmentDirIsS3()) {
|
||||
$attachments_dir = Configure::read('MISP.tmpdir');
|
||||
// Sometimes it's not set?
|
||||
if (empty($attachments_dir)) {
|
||||
// Get a default tmpdir
|
||||
$my_server = ClassRegistry::init('Server');
|
||||
$attachments_dir = $my_server->getDefaultTmp_dir();
|
||||
}
|
||||
}
|
||||
|
||||
if ($proposal) {
|
||||
$dir = new Folder($attachments_dir . DS . $event_id . DS . 'shadow', true);
|
||||
} else {
|
||||
|
|
|
@ -359,11 +359,20 @@ class Event extends AppModel
|
|||
$my_server = ClassRegistry::init('Server');
|
||||
$attachments_dir = $my_server->getDefaultAttachments_dir();
|
||||
}
|
||||
$filepath = $attachments_dir . DS . $this->id;
|
||||
App::uses('Folder', 'Utility');
|
||||
if (is_dir($filepath)) {
|
||||
if (!$this->destroyDir($filepath)) {
|
||||
throw new InternalErrorException('Delete of event file directory failed. Please report to administrator.');
|
||||
|
||||
// Things get a little funky here
|
||||
if ($this->attachmentDirIsS3()) {
|
||||
// S3 doesn't have folders
|
||||
// So we have to basically `ls` them to look for a prefix
|
||||
$s3 = $this->getS3Client();
|
||||
$s3->deleteDirectory($this->id);
|
||||
} else {
|
||||
$filepath = $attachments_dir . DS . $this->id;
|
||||
App::uses('Folder', 'Utility');
|
||||
if (is_dir($filepath)) {
|
||||
if (!$this->destroyDir($filepath)) {
|
||||
throw new InternalErrorException('Delete of event file directory failed. Please report to administrator.');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1386,6 +1386,46 @@ class Server extends AppModel
|
|||
'test' => 'testForEmpty',
|
||||
'type' => 'string'
|
||||
),
|
||||
'S3_enable' => array(
|
||||
'level' => 2,
|
||||
'description' => __('Enables or disables uploading of malware samples to S3 rather than to disk (WARNING: Get permission from amazon first!)'),
|
||||
'value' => false,
|
||||
'errorMessage' => '',
|
||||
'test' => 'testBool',
|
||||
'type' => 'boolean'
|
||||
),
|
||||
'S3_bucket_name' => array(
|
||||
'level' => 2,
|
||||
'description' => __('Bucket name to upload to'),
|
||||
'value' => '',
|
||||
'errorMessage' => '',
|
||||
'test' => 'testForEmpty',
|
||||
'type' => 'string'
|
||||
),
|
||||
'S3_region' => array(
|
||||
'level' => 2,
|
||||
'description' => __('Region in which your S3 bucket resides'),
|
||||
'value' => '',
|
||||
'errorMessage' => '',
|
||||
'test' => 'testForEmpty',
|
||||
'type' => 'string'
|
||||
),
|
||||
'S3_aws_access_key' => array(
|
||||
'level' => 2,
|
||||
'description' => __('AWS key to use when uploading samples (WARNING: It\' highly recommended that you use EC2 IAM roles if at all possible)'),
|
||||
'value' => '',
|
||||
'errorMessage' => '',
|
||||
'test' => 'testForEmpty',
|
||||
'type' => 'string'
|
||||
),
|
||||
'S3_aws_secret_key' => array(
|
||||
'level' => 2,
|
||||
'description' => __('AWS secret key to use when uploading samples'),
|
||||
'value' => '',
|
||||
'errorMessage' => '',
|
||||
'test' => 'testForEmpty',
|
||||
'type' => 'string'
|
||||
),
|
||||
'Sightings_policy' => array(
|
||||
'level' => 1,
|
||||
'description' => __('This setting defines who will have access to seeing the reported sightings. The default setting is the event owner alone (in addition to everyone seeing their own contribution) with the other options being Sighting reporters (meaning the event owner and anyone that provided sighting data about the event) and Everyone (meaning anyone that has access to seeing the event / attribute).'),
|
||||
|
@ -4001,6 +4041,11 @@ class Server extends AppModel
|
|||
return APP . 'files';
|
||||
}
|
||||
|
||||
public function getDefaultTmp_dir()
|
||||
{
|
||||
return sys_get_temp_dir();
|
||||
}
|
||||
|
||||
public function fetchServer($id)
|
||||
{
|
||||
if (empty($id)) {
|
||||
|
|
|
@ -248,16 +248,21 @@ class ShadowAttribute extends AppModel
|
|||
$sa = $this->find('first', array('conditions' => array('ShadowAttribute.id' => $this->data['ShadowAttribute']['id']), 'recursive' => -1, 'fields' => array('ShadowAttribute.id', 'ShadowAttribute.event_id', 'ShadowAttribute.type')));
|
||||
if ($this->typeIsAttachment($sa['ShadowAttribute']['type'])) {
|
||||
// only delete the file if it exists
|
||||
$attachments_dir = Configure::read('MISP.attachments_dir');
|
||||
if (empty($attachments_dir)) {
|
||||
$my_server = ClassRegistry::init('Server');
|
||||
$attachments_dir = $my_server->getDefaultAttachments_dir();
|
||||
}
|
||||
$filepath = $attachments_dir . DS . 'shadow' . DS . $sa['ShadowAttribute']['event_id'] . DS . $sa['ShadowAttribute']['id'];
|
||||
$file = new File($filepath);
|
||||
if ($file->exists()) {
|
||||
if (!$file->delete()) {
|
||||
throw new InternalErrorException('Delete of file attachment failed. Please report to administrator.');
|
||||
if ($this->attachmentDirIsS3()) {
|
||||
$s3 = $this->getS3Client();
|
||||
$s3->delete('shadow' . DS . $sa['ShadowAttribute']['event_id'] . DS . $sa['ShadowAttribute']['id']);
|
||||
} else {
|
||||
$attachments_dir = Configure::read('MISP.attachments_dir');
|
||||
if (empty($attachments_dir)) {
|
||||
$my_server = ClassRegistry::init('Server');
|
||||
$attachments_dir = $my_server->getDefaultAttachments_dir();
|
||||
}
|
||||
$filepath = $attachments_dir . DS . 'shadow' . DS . $sa['ShadowAttribute']['event_id'] . DS . $sa['ShadowAttribute']['id'];
|
||||
$file = new File($filepath);
|
||||
if ($file->exists()) {
|
||||
if (!$file->delete()) {
|
||||
throw new InternalErrorException('Delete of file attachment failed. Please report to administrator.');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -281,16 +286,21 @@ class ShadowAttribute extends AppModel
|
|||
$this->read(); // first read the attribute from the db
|
||||
if ($this->typeIsAttachment($this->data['ShadowAttribute']['type'])) {
|
||||
// only delete the file if it exists
|
||||
$attachments_dir = Configure::read('MISP.attachments_dir');
|
||||
if (empty($attachments_dir)) {
|
||||
$my_server = ClassRegistry::init('Server');
|
||||
$attachments_dir = $my_server->getDefaultAttachments_dir();
|
||||
}
|
||||
$filepath = $attachments_dir . DS . 'shadow' . DS . $this->data['ShadowAttribute']['event_id'] . DS . $this->data['ShadowAttribute']['id'];
|
||||
$file = new File($filepath);
|
||||
if ($file->exists()) {
|
||||
if (!$file->delete()) {
|
||||
throw new InternalErrorException('Delete of file attachment failed. Please report to administrator.');
|
||||
if ($this->attachmentDirIsS3()) {
|
||||
$s3 = $this->getS3Client();
|
||||
$s3->delete('shadow' . DS . $this->data['ShadowAttribute']['event_id'] . DS . $this->data['ShadowAttribute']['id']);
|
||||
} else {
|
||||
$attachments_dir = Configure::read('MISP.attachments_dir');
|
||||
if (empty($attachments_dir)) {
|
||||
$my_server = ClassRegistry::init('Server');
|
||||
$attachments_dir = $my_server->getDefaultAttachments_dir();
|
||||
}
|
||||
$filepath = $attachments_dir . DS . 'shadow' . DS . $this->data['ShadowAttribute']['event_id'] . DS . $this->data['ShadowAttribute']['id'];
|
||||
$file = new File($filepath);
|
||||
if ($file->exists()) {
|
||||
if (!$file->delete()) {
|
||||
throw new InternalErrorException('Delete of file attachment failed. Please report to administrator.');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -394,12 +404,18 @@ class ShadowAttribute extends AppModel
|
|||
$my_server = ClassRegistry::init('Server');
|
||||
$attachments_dir = $my_server->getDefaultAttachments_dir();
|
||||
}
|
||||
$filepath = $attachments_dir . DS . 'shadow' . DS . $attribute['event_id'] . DS. $attribute['id'];
|
||||
$file = new File($filepath);
|
||||
if (!$file->exists()) {
|
||||
return '';
|
||||
|
||||
if ($this->attachmentDirIsS3()) {
|
||||
$s3 = $this->getS3Client();
|
||||
$content = $s3->download('shadow' . DS . $attribute['event_id'] . DS. $attribute['id']);
|
||||
} else {
|
||||
$filepath = $attachments_dir . DS . 'shadow' . DS . $attribute['event_id'] . DS. $attribute['id'];
|
||||
$file = new File($filepath);
|
||||
if (!$file->exists()) {
|
||||
return '';
|
||||
}
|
||||
$content = $file->read();
|
||||
}
|
||||
$content = $file->read();
|
||||
return base64_encode($content);
|
||||
}
|
||||
|
||||
|
@ -410,16 +426,23 @@ class ShadowAttribute extends AppModel
|
|||
$my_server = ClassRegistry::init('Server');
|
||||
$attachments_dir = $my_server->getDefaultAttachments_dir();
|
||||
}
|
||||
$rootDir = $attachments_dir . DS . 'shadow' . DS . $attribute['event_id'];
|
||||
$dir = new Folder($rootDir, true); // create directory structure
|
||||
$destpath = $rootDir . DS . $attribute['id'];
|
||||
$file = new File($destpath, true); // create the file
|
||||
$decodedData = base64_decode($attribute['data']); // decode
|
||||
if ($file->write($decodedData)) { // save the data
|
||||
if ($this->attachmentDirIsS3()) {
|
||||
$s3 = $this->getS3Client();
|
||||
$decodedData = base64_decode($attribute['data']);
|
||||
$s3->upload('shadow' . DS . $attribute['event_id'], $decodedData);
|
||||
return true;
|
||||
} else {
|
||||
// error
|
||||
return false;
|
||||
$rootDir = $attachments_dir . DS . 'shadow' . DS . $attribute['event_id'];
|
||||
$dir = new Folder($rootDir, true); // create directory structure
|
||||
$destpath = $rootDir . DS . $attribute['id'];
|
||||
$file = new File($destpath, true); // create the file
|
||||
$decodedData = base64_decode($attribute['data']); // decode
|
||||
if ($file->write($decodedData)) { // save the data
|
||||
return true;
|
||||
} else {
|
||||
// error
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -779,7 +779,7 @@ class User extends AppModel
|
|||
}
|
||||
// SMIME if not GPG key
|
||||
if (!$failed && !$canEncryptGPG && $canEncryptSMIME) {
|
||||
$encryptionResult = $this->__encryptUsingSmime($Email, $body, $subject);
|
||||
$encryptionResult = $this->__encryptUsingSmime($Email, $body, $subject, $user);
|
||||
if (isset($encryptionResult['failed'])) {
|
||||
$failed = true;
|
||||
}
|
||||
|
@ -893,7 +893,7 @@ class User extends AppModel
|
|||
return true;
|
||||
}
|
||||
|
||||
private function __encryptUsingSmime(&$Email, &$body, $subject)
|
||||
private function __encryptUsingSmime(&$Email, &$body, $subject, $user)
|
||||
{
|
||||
try {
|
||||
$prependedBody = 'Content-Transfer-Encoding: 7bit' . PHP_EOL . 'Content-Type: text/plain;' . PHP_EOL . ' charset=us-ascii' . PHP_EOL . PHP_EOL . $body;
|
||||
|
|
|
@ -4,7 +4,7 @@
|
|||
<?php
|
||||
foreach ($result as $r) {
|
||||
?>
|
||||
<h3>V<?php echo __('alidation errors for event: %s', h($r['id']));?></h3>
|
||||
<h3><?php echo __('Validation errors for event: %s', h($r['id']));?></h3>
|
||||
<?php print_r($r['error']); ?><br />
|
||||
<?php echo __('Attribute details');?>:<br />
|
||||
<?php print_r(h($r['details'])); ?>
|
||||
|
|
|
@ -36,7 +36,7 @@
|
|||
<tr>
|
||||
<?php if ($isSiteAdmin): ?>
|
||||
<th>
|
||||
<input class="select_all select" type="checkbox" title="<?php echo __('Select all');?>" role="button" tabindex="0" aria-label="<?php echo __('Select all eventson current page');?>" onClick="toggleAllCheckboxes();" />
|
||||
<input class="select_all select" type="checkbox" title="<?php echo __('Select all');?>" role="button" tabindex="0" aria-label="<?php echo __('Select all events on current page');?>" onClick="toggleAllCheckboxes();" />
|
||||
</th>
|
||||
<?php else: ?>
|
||||
<th style="padding-left:0px;padding-right:0px;"> </th>
|
||||
|
|
|
@ -7,6 +7,7 @@
|
|||
"pear/net_geoip": "@dev"
|
||||
},
|
||||
"suggest": {
|
||||
"elasticsearch/elasticsearch": "For logging to elasticsearch"
|
||||
"elasticsearch/elasticsearch": "For logging to elasticsearch",
|
||||
"aws/aws-sdk-php": "To upload samples to S3"
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1 +1 @@
|
|||
Subproject commit 37396ed6ca7023a65981e95aa5842951a9c6a985
|
||||
Subproject commit cd76f19f52e94a61f0d500fa3cdbf89a758e1c19
|
|
@ -15,25 +15,17 @@
|
|||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import sys, json, os, time, uuid
|
||||
import mixbox.namespaces as mixbox_ns
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
import stix2misp_mapping
|
||||
from operator import attrgetter
|
||||
from pymisp import MISPEvent, MISPObject, MISPAttribute, __path__
|
||||
from stix.core import STIXPackage
|
||||
from collections import defaultdict
|
||||
|
||||
file_object_type = {"type": "filename", "relation": "filename"}
|
||||
|
||||
eventTypes = {"ArtifactObjectType": {"type": "attachment", "relation": "attachment"},
|
||||
"DomainNameObjectType": {"type": "domain", "relation": "domain"},
|
||||
"FileObjectType": file_object_type,
|
||||
"HostnameObjectType": {"type": "hostname", "relation": "host"},
|
||||
"MutexObjectType": {"type": "mutex", "relation": "mutex"},
|
||||
"PDFFileObjectType": file_object_type,
|
||||
"PortObjectType": {"type": "port", "relation": "port"},
|
||||
"URIObjectType": {"type": "url", "relation": "url"},
|
||||
"WindowsExecutableFileObjectType": file_object_type,
|
||||
"WindowsRegistryKeyObjectType": {"type": "regkey", "relation": ""}}
|
||||
|
||||
cybox_to_misp_object = {"Account": "credential", "AutonomousSystem": "asn",
|
||||
"EmailMessage": "email", "NetworkConnection": "network-connection",
|
||||
"NetworkSocket": "network-socket", "Process": "process",
|
||||
|
@ -62,7 +54,7 @@ class StixParser():
|
|||
try:
|
||||
import maec
|
||||
print(2)
|
||||
except:
|
||||
except ModuleNotFoundError:
|
||||
print(3)
|
||||
sys.exit(0)
|
||||
title = event.stix_header.title
|
||||
|
@ -204,10 +196,9 @@ class StixParser():
|
|||
|
||||
# Parse a DNS object
|
||||
def resolve_dns_objects(self):
|
||||
for domain in self.dns_objects['domain']:
|
||||
domain_object = self.dns_objects['domain'][domain]
|
||||
ip_reference = domain_object['related']
|
||||
domain_attribute = domain_object['data']
|
||||
for domain, domain_dict in self.dns_objects['domain'].items():
|
||||
ip_reference = domain_dict['related']
|
||||
domain_attribute = domain_dict['data']
|
||||
if ip_reference in self.dns_objects['ip']:
|
||||
misp_object = MISPObject('passive-dns')
|
||||
domain_attribute['object_relation'] = "rrname"
|
||||
|
@ -221,10 +212,9 @@ class StixParser():
|
|||
self.misp_event.add_object(**misp_object)
|
||||
else:
|
||||
self.misp_event.add_attribute(**domain_attribute)
|
||||
for ip in self.dns_objects['ip']:
|
||||
for ip, ip_dict in self.dns_objects['ip'].items():
|
||||
if ip not in self.dns_ips:
|
||||
# print(ip)
|
||||
self.misp_event.add_attribute(**self.dns_objects['ip'][ip])
|
||||
self.misp_event.add_attribute(**ip_dict)
|
||||
|
||||
def set_distribution(self):
|
||||
for attribute in self.misp_event.attributes:
|
||||
|
@ -347,17 +337,17 @@ class StixParser():
|
|||
# Define type & value of an attribute or object in MISP
|
||||
def handle_attribute_type(self, properties, is_object=False, title=None, observable_id=None):
|
||||
xsi_type = properties._XSI_TYPE
|
||||
try:
|
||||
args = [properties]
|
||||
if xsi_type in ("FileObjectType", "PDFFileObjectType"):
|
||||
args.append(is_object)
|
||||
elif xsi_type == "ArtifactObjectType":
|
||||
args.append(title)
|
||||
return self.attribute_types_mapping[xsi_type](*args)
|
||||
except AttributeError:
|
||||
# ATM USED TO TEST TYPES
|
||||
print("Unparsed type: {}".format(xsi_type))
|
||||
sys.exit(1)
|
||||
# try:
|
||||
args = [properties]
|
||||
if xsi_type in ("FileObjectType", "PDFFileObjectType"):
|
||||
args.append(is_object)
|
||||
elif xsi_type == "ArtifactObjectType":
|
||||
args.append(title)
|
||||
return self.attribute_types_mapping[xsi_type](*args)
|
||||
# except AttributeError:
|
||||
# # ATM USED TO TEST TYPES
|
||||
# print("Unparsed type: {}".format(xsi_type))
|
||||
# sys.exit(1)
|
||||
|
||||
# Return type & value of an ip address attribute
|
||||
@staticmethod
|
||||
|
@ -369,23 +359,15 @@ class StixParser():
|
|||
return ip_type, properties.address_value.value, "ip"
|
||||
|
||||
def handle_as(self, properties):
|
||||
attributes = []
|
||||
if properties.number:
|
||||
attributes.append(['AS', properties.number.value, 'asn'])
|
||||
if properties.handle:
|
||||
attributes.append(['AS', properties.handle.value, 'asn'])
|
||||
if properties.name:
|
||||
attributes.append(['text', properties.name.value, 'description'])
|
||||
if len(attributes) == 1:
|
||||
return attributes[0]
|
||||
return 'asn', self.return_attributes(attributes), ''
|
||||
attributes = self.fetch_attributes_with_partial_key_parsing(properties, stix2misp_mapping._as_mapping)
|
||||
return attributes[0] if len(attributes) == 1 else ('asn', self.return_attributes(attributes), '')
|
||||
|
||||
# Return type & value of an attachment attribute
|
||||
@staticmethod
|
||||
def handle_attachment(properties, title):
|
||||
if properties.hashes:
|
||||
return "malware-sample", "{}|{}".format(title, properties.hashes[0], properties.raw_artifact.value)
|
||||
return eventTypes[properties._XSI_TYPE]['type'], title, properties.raw_artifact.value
|
||||
return stix2misp_mapping.eventTypes[properties._XSI_TYPE]['type'], title, properties.raw_artifact.value
|
||||
|
||||
# Return type & attributes of a credential object
|
||||
def handle_credential(self, properties):
|
||||
|
@ -393,38 +375,13 @@ class StixParser():
|
|||
if properties.description:
|
||||
attributes.append(["text", properties.description.value, "text"])
|
||||
if properties.authentication:
|
||||
l_password, l_type, l_format = [], [], []
|
||||
for authentication in properties.authentication:
|
||||
if authentication.authentication_type:
|
||||
auth_type = authentication.authentication_type.value
|
||||
if auth_type not in l_type:
|
||||
l_type.append(auth_type)
|
||||
if authentication.authentication_data:
|
||||
auth_data = authentication.authentication_data.value
|
||||
if auth_data not in l_password:
|
||||
l_password.append(auth_data)
|
||||
if authentication.structured_authentication_mechanism:
|
||||
auth_format = authentication.structured_authentication_mechanism.description.value
|
||||
if auth_format not in l_format:
|
||||
l_format.append(auth_format)
|
||||
if l_type:
|
||||
# auth_type = l_type[0] if len(l_type) == 1 else "".join(["{}, ".format(t) for t in l_type])[:-2]
|
||||
for auth_type in l_type:
|
||||
attributes.append(["text", auth_type, "type"])
|
||||
if l_format:
|
||||
# auth_format = l_format[0] if len(l_format) == 1 else "".join(["{}, ".format(f) for f in l_format])[:-2]
|
||||
for auth_format in l_format:
|
||||
attributes.append(["text", auth_format, "format"])
|
||||
if l_password:
|
||||
for password in l_password:
|
||||
attributes.append(["text", password, "password"])
|
||||
attributes += self.fetch_attributes_with_key_parsing(authentication, stix2misp_mapping._credential_authentication_mapping)
|
||||
if properties.custom_properties:
|
||||
for prop in properties.custom_properties:
|
||||
if prop.name in ("username", "origin", "notification"):
|
||||
attributes.append(["text", prop.value, prop.name])
|
||||
if len(attributes) == 1:
|
||||
return attributes[0]
|
||||
return "credential", self.return_attributes(attributes), ""
|
||||
if prop.name in stix2misp_mapping._credential_custom_types:
|
||||
attributes.append(['text', prop.value, prop.name])
|
||||
return attributes[0] if len(attributes) == 1 else ("credential", self.return_attributes(attributes), "")
|
||||
|
||||
# Return type & attributes (or value) of a Custom Object
|
||||
def handle_custom(self, properties):
|
||||
|
@ -467,43 +424,31 @@ class StixParser():
|
|||
# Return type & value of a domain or url attribute
|
||||
@staticmethod
|
||||
def handle_domain_or_url(properties):
|
||||
event_types = eventTypes[properties._XSI_TYPE]
|
||||
event_types = stix2misp_mapping.eventTypes[properties._XSI_TYPE]
|
||||
return event_types['type'], properties.value.value, event_types['relation']
|
||||
|
||||
# Return type & value of an email attribute
|
||||
def handle_email_attribute(self, properties):
|
||||
attributes = []
|
||||
if properties.header:
|
||||
header = properties.header
|
||||
if header.from_:
|
||||
attributes.append(["email-src", header.from_.address_value.value, "from"])
|
||||
attributes = self.fetch_attributes_with_key_parsing(header, stix2misp_mapping._email_mapping)
|
||||
if header.to:
|
||||
for to in header.to:
|
||||
attributes.append(["email-dst", to.address_value.value, "to"])
|
||||
if header.cc:
|
||||
for cc in header.cc:
|
||||
attributes.append(["email-dst", cc.address_value.value, "cc"])
|
||||
if header.reply_to:
|
||||
attributes.append(["email-reply-to", header.reply_to.address_value.value, "reply-to"])
|
||||
if header.subject:
|
||||
attributes.append(["email-subject", header.subject.value, "subject"])
|
||||
if header.x_mailer:
|
||||
attributes.append(["email-x-mailer", header.x_mailer.value, "x-mailer"])
|
||||
if header.boundary:
|
||||
attributes.append(["email-mime-boundary", header.boundary.value, "mime-boundary"])
|
||||
if header.user_agent:
|
||||
attributes.append(["text", header.user_agent.value, "user-agent"])
|
||||
else:
|
||||
attributes = []
|
||||
if properties.attachments:
|
||||
attributes.append([self.handle_email_attachment(properties.parent)])
|
||||
if len(attributes) == 1:
|
||||
return attributes[0]
|
||||
return "email", self.return_attributes(attributes), ""
|
||||
attributes.append(self.handle_email_attachment(properties.parent))
|
||||
return attributes[0] if len(attributes) == 1 else ("email", self.return_attributes(attributes), "")
|
||||
|
||||
# Return type & value of an email attachment
|
||||
@staticmethod
|
||||
def handle_email_attachment(indicator_object):
|
||||
properties = indicator_object.related_objects[0].properties
|
||||
return "email-attachment", properties.file_name.value, "attachment"
|
||||
return ["email-attachment", properties.file_name.value, "attachment"]
|
||||
|
||||
# Return type & attributes of a file object
|
||||
def handle_file(self, properties, is_object):
|
||||
|
@ -513,26 +458,13 @@ class StixParser():
|
|||
b_hash = True
|
||||
for h in properties.hashes:
|
||||
attributes.append(self.handle_hashes_attribute(h))
|
||||
if properties.file_format and properties.file_format.value:
|
||||
attributes.append(["mime-type", properties.file_format.value, "mimetype"])
|
||||
if properties.file_name:
|
||||
value = properties.file_name.value
|
||||
if value:
|
||||
b_file = True
|
||||
event_types = eventTypes[properties._XSI_TYPE]
|
||||
attributes.append([event_types['type'], value, event_types['relation']])
|
||||
if properties.file_path:
|
||||
value = properties.file_path.value
|
||||
if value:
|
||||
attributes.append(['text', value, 'path'])
|
||||
if properties.byte_runs:
|
||||
attribute_type = "pattern-in-file"
|
||||
attributes.append([attribute_type, properties.byte_runs[0].byte_run_data, attribute_type])
|
||||
if properties.size_in_bytes and properties.size_in_bytes.value:
|
||||
attribute_type = "size-in-bytes"
|
||||
attributes.append([attribute_type, properties.size_in_bytes.value, attribute_type])
|
||||
if properties.peak_entropy and properties.peak_entropy.value:
|
||||
attributes.append(["float", properties.peak_entropy.value, "entropy"])
|
||||
attribute_type, relation = stix2misp_mapping.eventTypes[properties._XSI_TYPE]
|
||||
attributes.append([attribute_type, value, relation])
|
||||
self.fetch_attributes_with_keys(properties, stix2misp_mapping._file_mapping, attributes)
|
||||
if len(attributes) == 1:
|
||||
return attributes[0]
|
||||
if len(attributes) == 2:
|
||||
|
@ -588,7 +520,7 @@ class StixParser():
|
|||
# Return type & value of a hostname attribute
|
||||
@staticmethod
|
||||
def handle_hostname(properties):
|
||||
event_types = eventTypes[properties._XSI_TYPE]
|
||||
event_types = stix2misp_mapping.eventTypes[properties._XSI_TYPE]
|
||||
return event_types['type'], properties.hostname_value.value, event_types['relation']
|
||||
|
||||
# Return type & value of a http request attribute
|
||||
|
@ -610,42 +542,25 @@ class StixParser():
|
|||
# Return type & value of a mutex attribute
|
||||
@staticmethod
|
||||
def handle_mutex(properties):
|
||||
event_types = eventTypes[properties._XSI_TYPE]
|
||||
event_types = stix2misp_mapping.eventTypes[properties._XSI_TYPE]
|
||||
return event_types['type'], properties.name.value, event_types['relation']
|
||||
|
||||
# Return type & attributes of a network connection object
|
||||
def handle_network_connection(self, properties):
|
||||
attributes = []
|
||||
if properties.source_socket_address:
|
||||
self.handle_socket(attributes, properties.source_socket_address, "src")
|
||||
if properties.destination_socket_address:
|
||||
self.handle_socket(attributes, properties.destination_socket_address, "dst")
|
||||
if properties.layer3_protocol:
|
||||
attributes.append(["text", properties.layer3_protocol.value, "layer3-protocol"])
|
||||
if properties.layer4_protocol:
|
||||
attributes.append(["text", properties.layer4_protocol.value, "layer4-protocol"])
|
||||
if properties.layer7_protocol:
|
||||
attributes.append(["text", properties.layer7_protocol.value, "layer7-protocol"])
|
||||
attributes = self.fetch_attributes_from_sockets(properties, stix2misp_mapping._network_connection_addresses)
|
||||
for prop in ('layer3_protocol', 'layer4_protocol', 'layer7_protocol'):
|
||||
if getattr(properties, prop):
|
||||
attributes.append(['text', attrgetter("{}.value".format(prop))(properties), prop.replace('_', '-')])
|
||||
if attributes:
|
||||
return "network-connection", self.return_attributes(attributes), ""
|
||||
|
||||
# Return type & attributes of a network socket objet
|
||||
def handle_network_socket(self, properties):
|
||||
attributes = []
|
||||
if properties.local_address:
|
||||
self.handle_socket(attributes, properties.local_address, "src")
|
||||
if properties.remote_address:
|
||||
self.handle_socket(attributes, properties.remote_address, "dst")
|
||||
if properties.protocol:
|
||||
attributes.append(["text", properties.protocol.value, "protocol"])
|
||||
if properties.is_listening:
|
||||
attributes.append(["text", "listening", "state"])
|
||||
if properties.is_blocking:
|
||||
attributes.append(["text", "blocking", "state"])
|
||||
if properties.address_family:
|
||||
attributes.append(["text", properties.address_family.value, "address-family"])
|
||||
if properties.domain:
|
||||
attributes.append(["text", properties.domain.value, "domain-family"])
|
||||
attributes = self.fetch_attributes_from_sockets(properties, stix2misp_mapping._network_socket_addresses)
|
||||
self.fetch_attributes_with_keys(properties, stix2misp_mapping._network_socket_mapping, attributes)
|
||||
for prop in ('is_listening', 'is_blocking'):
|
||||
if getattr(properties, prop):
|
||||
attributes.append(["text", prop.split('_')[1], "state"])
|
||||
if attributes:
|
||||
return "network-socket", self.return_attributes(attributes), ""
|
||||
|
||||
|
@ -653,7 +568,7 @@ class StixParser():
|
|||
@staticmethod
|
||||
def handle_port(*kwargs):
|
||||
properties = kwargs[0]
|
||||
event_types = eventTypes[properties._XSI_TYPE]
|
||||
event_types = stix2misp_mapping.eventTypes[properties._XSI_TYPE]
|
||||
relation = event_types['relation']
|
||||
if len(kwargs) > 1:
|
||||
observable_id = kwargs[1]
|
||||
|
@ -665,21 +580,10 @@ class StixParser():
|
|||
|
||||
# Return type & attributes of a process object
|
||||
def handle_process(self, properties):
|
||||
attributes = []
|
||||
if properties.creation_time:
|
||||
attributes.append(["datetime", properties.creation_time.value, "creation-time"])
|
||||
if properties.start_time:
|
||||
attributes.append(["datetime", properties.creation_time.value, "start-time"])
|
||||
attribute_type = "text"
|
||||
if properties.name:
|
||||
attributes.append([attribute_type, properties.name.value, "name"])
|
||||
if properties.pid:
|
||||
attributes.append([attribute_type, properties.pid.value, "pid"])
|
||||
if properties.parent_pid:
|
||||
attributes.append([attribute_type, properties.parent_pid.value, "parent-pid"])
|
||||
attributes = self.fetch_attributes_with_partial_key_parsing(properties, stix2misp_mapping._process_mapping)
|
||||
if properties.child_pid_list:
|
||||
for child in properties.child_pid_list:
|
||||
attributes.append([attribute_type, child.value, "child-pid"])
|
||||
attributes.append(["text", child.value, "child-pid"])
|
||||
# if properties.port_list:
|
||||
# for port in properties.port_list:
|
||||
# attributes.append(["src-port", port.port_value.value, "port"])
|
||||
|
@ -698,33 +602,20 @@ class StixParser():
|
|||
|
||||
# Return type & value of a regkey attribute
|
||||
def handle_regkey(self, properties):
|
||||
attributes = []
|
||||
if properties.hive:
|
||||
attributes.append(["text", properties.hive.value, "hive"])
|
||||
if properties.key:
|
||||
attributes.append(["regkey", properties.key.value, "key"])
|
||||
attributes = self.fetch_attributes_with_partial_key_parsing(properties, stix2misp_mapping._regkey_mapping)
|
||||
if properties.values:
|
||||
values = properties.values
|
||||
value = values[0]
|
||||
if value.data:
|
||||
attributes.append(["text", value.data.value, "data"])
|
||||
if value.datatype:
|
||||
attributes.append(["text", value.datatype.value, "data-type"])
|
||||
if value.name:
|
||||
attributes.append(["text", value.name.value, "name"])
|
||||
attributes += self.fetch_attributes_with_partial_key_parsing(value, stix2misp_mapping._regkey_value_mapping)
|
||||
return "registry-key", self.return_attributes(attributes), ""
|
||||
|
||||
# Parse a socket address object into a network connection or socket object,
|
||||
# in order to add its attributes
|
||||
@staticmethod
|
||||
def handle_socket(attributes, socket, s_type):
|
||||
if socket.ip_address:
|
||||
ip_type = "ip-{}".format(s_type)
|
||||
attributes.append([ip_type, socket.ip_address.address_value.value, ip_type])
|
||||
if socket.port:
|
||||
attributes.append(["port", socket.port.port_value.value, "{}-port".format(s_type)])
|
||||
if socket.hostname:
|
||||
attributes.append(["hostname", socket.hostname.hostname_value.value, "hostname-{}".format(s_type)])
|
||||
for prop, mapping in stix2misp_mapping._socket_mapping.items():
|
||||
if getattr(socket, prop):
|
||||
attribute_type, properties_key, relation = mapping
|
||||
attribute_type, relation = [elem.format(s_type) for elem in (attribute_type, relation)]
|
||||
attributes.append([attribute_type, attrgetter('{}.{}.value'.format(prop, properties_key))(socket), relation])
|
||||
|
||||
# Parse a socket address object in order to return type & value
|
||||
# of a composite attribute ip|port or hostname|port
|
||||
|
@ -746,22 +637,11 @@ class StixParser():
|
|||
# Return type & attributes of a whois object if we have the required fields
|
||||
# Otherwise create attributes and return type & value of the last attribute to avoid crashing the parent function
|
||||
def handle_whois(self, properties):
|
||||
required_one_of = False
|
||||
attributes = []
|
||||
if properties.registrar_info:
|
||||
attribute_type = "whois-registrar"
|
||||
attributes.append([attribute_type, properties.registrar_info.value, attribute_type])
|
||||
required_one_of = True
|
||||
attributes = self.fetch_attributes_with_key_parsing(properties, stix2misp_mapping._whois_mapping)
|
||||
required_one_of = True if attributes else False
|
||||
if properties.registrants:
|
||||
registrant = properties.registrants[0]
|
||||
if registrant.email_address:
|
||||
attributes.append(["whois-registrant-email", registrant.email_address.address_value.value, "registrant-email"])
|
||||
if registrant.name:
|
||||
attributes.append(["whois-registrant-name", registrant.name.value, "registrant-name"])
|
||||
if registrant.phone_number:
|
||||
attributes.append(["whois-registrant-phone", registrant.phone_number.value, "registrant-phone"])
|
||||
if registrant.organization:
|
||||
attributes.append(["whois-registrant-org", registrant.organization.value, "registrant-org"])
|
||||
attributes += self.fetch_attributes_with_key_parsing(registrant, stix2misp_mapping._whois_registrant_mapping)
|
||||
if properties.creation_date:
|
||||
attributes.append(["datetime", properties.creation_date.value.strftime('%Y-%m-%d'), "creation-date"])
|
||||
required_one_of = True
|
||||
|
@ -772,13 +652,6 @@ class StixParser():
|
|||
if properties.nameservers:
|
||||
for nameserver in properties.nameservers:
|
||||
attributes.append(["hostname", nameserver.value.value, "nameserver"])
|
||||
if properties.ip_address:
|
||||
attributes.append(["ip-src", properties.ip_address.address_value.value, "ip-address"])
|
||||
required_one_of = True
|
||||
if properties.domain_name:
|
||||
attribute_type = "domain"
|
||||
attributes.append([attribute_type, properties.domain_name.value.value, attribute_type])
|
||||
required_one_of = True
|
||||
if properties.remarks:
|
||||
attribute_type = "text"
|
||||
relation = "comment" if attributes else attribute_type
|
||||
|
@ -788,16 +661,15 @@ class StixParser():
|
|||
if required_one_of:
|
||||
# if yes, we return the object type and the attributes
|
||||
return "whois", self.return_attributes(attributes), ""
|
||||
else:
|
||||
# otherwise, attributes are added in the event, and one attribute is returned to not make the function crash
|
||||
if len(attributes) == 1:
|
||||
return attributes[0]
|
||||
last_attribute = attributes.pop(-1)
|
||||
for attribute in attributes:
|
||||
attribute_type, attribute_value, attribute_relation = attribute
|
||||
misp_attributes = {"comment": "Whois {}".format(attribute_relation)}
|
||||
self.misp_event.add_attribute(attribute_type, attribute_value, **misp_attributes)
|
||||
return last_attribute
|
||||
# otherwise, attributes are added in the event, and one attribute is returned to not make the function crash
|
||||
if len(attributes) == 1:
|
||||
return attributes[0]
|
||||
last_attribute = attributes.pop(-1)
|
||||
for attribute in attributes:
|
||||
attribute_type, attribute_value, attribute_relation = attribute
|
||||
misp_attributes = {"comment": "Whois {}".format(attribute_relation)}
|
||||
self.misp_event.add_attribute(attribute_type, attribute_value, **misp_attributes)
|
||||
return last_attribute
|
||||
|
||||
# Return type & value of a windows service object
|
||||
@staticmethod
|
||||
|
@ -806,33 +678,7 @@ class StixParser():
|
|||
return "windows-service-name", properties.name.value, ""
|
||||
|
||||
def handle_x509(self, properties):
|
||||
attributes = []
|
||||
if properties.certificate:
|
||||
certificate = properties.certificate
|
||||
if certificate.validity:
|
||||
validity = certificate.validity
|
||||
if validity.not_before:
|
||||
attributes.append(["datetime", validity.not_before.value, "validity-not-before"])
|
||||
if validity.not_after:
|
||||
attributes.append(["datetime", validity.not_after.value, "validity-not-after"])
|
||||
if certificate.subject_public_key:
|
||||
subject_pubkey = certificate.subject_public_key
|
||||
if subject_pubkey.rsa_public_key:
|
||||
rsa_pubkey = subject_pubkey.rsa_public_key
|
||||
if rsa_pubkey.exponent:
|
||||
attributes.append(["text", rsa_pubkey.exponent.value, "pubkey-info-exponent"])
|
||||
if rsa_pubkey.modulus:
|
||||
attributes.append(["text", rsa_pubkey.modulus.value, "pubkey-info-modulus"])
|
||||
if subject_pubkey.public_key_algorithm:
|
||||
attributes.append(["text", subject_pubkey.public_key_algorithm.value, "pubkey-info-algorithm"])
|
||||
if certificate.version:
|
||||
attributes.append(["text", certificate.version.value, "version"])
|
||||
if certificate.serial_number:
|
||||
attributes.append(["text", certificate.serial_number.value, "serial-number"])
|
||||
if certificate.issuer:
|
||||
attributes.append(["text", certificate.issuer.value, "issuer"])
|
||||
if certificate.subject:
|
||||
attributes.append(["text", certificate.subject.value, "subject"])
|
||||
attributes = self.handle_x509_certificate(properties.certificate) if properties.certificate else []
|
||||
if properties.raw_certificate:
|
||||
raw = properties.raw_certificate.value
|
||||
try:
|
||||
|
@ -849,6 +695,28 @@ class StixParser():
|
|||
attributes.append([attribute_type, signature.signature.value, attribute_type])
|
||||
return "x509", self.return_attributes(attributes), ""
|
||||
|
||||
@staticmethod
|
||||
def handle_x509_certificate(certificate):
|
||||
attributes = []
|
||||
if certificate.validity:
|
||||
validity = certificate.validity
|
||||
for prop in stix2misp_mapping._x509_datetime_types:
|
||||
if getattr(validity, prop):
|
||||
attributes.append(['datetime', attrgetter('{}.value'.format(prop))(validity), 'validity-{}'.format(prop.replace('_', '-'))])
|
||||
if certificate.subject_public_key:
|
||||
subject_pubkey = certificate.subject_public_key
|
||||
if subject_pubkey.rsa_public_key:
|
||||
rsa_pubkey = subject_pubkey.rsa_public_key
|
||||
for prop in stix2misp_mapping._x509__x509_pubkey_types:
|
||||
if getattr(rsa_pubkey, prop):
|
||||
attributes.append(['text', attrgetter('{}.value'.format(prop))(rsa_pubkey), 'pubkey-info-{}'.format(prop)])
|
||||
if subject_pubkey.public_key_algorithm:
|
||||
attributes.append(["text", subject_pubkey.public_key_algorithm.value, "pubkey-info-algorithm"])
|
||||
for prop in stix2misp_mapping._x509_certificate_types:
|
||||
if getattr(certificate, prop):
|
||||
attributes.append(['text', attrgetter('{}.value'.format(prop))(certificate), prop.replace('_', '-')])
|
||||
return attributes
|
||||
|
||||
# Return type & attributes of the file defining a portable executable object
|
||||
def handle_pe(self, properties):
|
||||
pe_uuid = self.parse_pe(properties)
|
||||
|
@ -976,7 +844,7 @@ class StixParser():
|
|||
continue
|
||||
if properties:
|
||||
attribute_type, attribute_value, compl_data = self.handle_attribute_type(properties)
|
||||
if type(attribute_value) in (str, int):
|
||||
if isinstance(attribute_value, (str, int)):
|
||||
# if the returned value is a simple value, we build an attribute
|
||||
attribute = {'to_ids': True}
|
||||
if indicator.timestamp:
|
||||
|
@ -1003,8 +871,7 @@ class StixParser():
|
|||
# print("Error with an object of type: {}\n{}".format(properties._XSI_TYPE, observable.to_json()))
|
||||
continue
|
||||
object_uuid = self.fetch_uuid(observable_object.id_)
|
||||
attr_type = type(attribute_value)
|
||||
if attr_type in (str, int):
|
||||
if isinstance(attribute_value, (str, int)):
|
||||
# if the returned value is a simple value, we build an attribute
|
||||
attribute = {'to_ids': False, 'uuid': object_uuid}
|
||||
if observable_object.related_objects:
|
||||
|
@ -1017,7 +884,9 @@ class StixParser():
|
|||
self.dns_ips.append(related_ip)
|
||||
continue
|
||||
if attribute_type in ('ip-src', 'ip-dst'):
|
||||
self.dns_objects['ip'][object_uuid] = {"type": attribute_type, "value": attribute_value}
|
||||
attribute['type'] = attribute_type
|
||||
attribute['value'] = attribute_value
|
||||
self.dns_objects['ip'][object_uuid] = attribute
|
||||
continue
|
||||
self.handle_attribute_case(attribute_type, attribute_value, compl_data, attribute)
|
||||
else:
|
||||
|
@ -1064,6 +933,39 @@ class StixParser():
|
|||
misp_object.add_reference(uuid, 'connected-to')
|
||||
self.misp_event.add_object(**misp_object)
|
||||
|
||||
def fetch_attributes_from_sockets(self, properties, mapping_dict):
|
||||
attributes = []
|
||||
for prop, s_type in zip(mapping_dict, stix2misp_mapping._s_types):
|
||||
address_property = getattr(properties, prop)
|
||||
if address_property:
|
||||
self.handle_socket(attributes, address_property, s_type)
|
||||
return attributes
|
||||
|
||||
@staticmethod
|
||||
def fetch_attributes_with_keys(properties, mapping_dict, attributes):
|
||||
for prop, mapping in mapping_dict.items():
|
||||
if getattr(properties,prop):
|
||||
attribute_type, properties_key, relation = mapping
|
||||
attributes.append([attribute_type, attrgetter(properties_key)(properties), relation])
|
||||
|
||||
@staticmethod
|
||||
def fetch_attributes_with_key_parsing(properties, mapping_dict):
|
||||
attributes = []
|
||||
for prop, mapping in mapping_dict.items():
|
||||
if getattr(properties, prop):
|
||||
attribute_type, properties_key, relation = mapping
|
||||
attributes.append([attribute_type, attrgetter('{}.{}'.format(prop, properties_key))(properties), relation])
|
||||
return attributes
|
||||
|
||||
@staticmethod
|
||||
def fetch_attributes_with_partial_key_parsing(properties, mapping_dict):
|
||||
attributes = []
|
||||
for prop, mapping in mapping_dict.items():
|
||||
if getattr(properties, prop):
|
||||
attribute_type, relation = mapping
|
||||
attributes.append([attribute_type, attrgetter('{}.value'.format(prop))(properties), relation])
|
||||
return attributes
|
||||
|
||||
# Extract the uuid from a stix id
|
||||
@staticmethod
|
||||
def fetch_uuid(object_id):
|
||||
|
@ -1105,6 +1007,7 @@ class StixParser():
|
|||
attribute = {'type': 'text', 'object_relation': 'name',
|
||||
'value': coa.title}
|
||||
misp_object.add_attribute(**attribute)
|
||||
# for prop in stix2misp_mapping.
|
||||
if coa.type_:
|
||||
attribute = {'type': 'text', 'object_relation': 'type',
|
||||
'value': coa.type_.value}
|
||||
|
|
|
@ -0,0 +1,57 @@
|
|||
_file_attribute_type = ('filename', 'filename')
|
||||
_network_socket_addresses = ['local_address', 'remote_address']
|
||||
_network_connection_addresses = ['source_socket_address', 'destination_socket_address']
|
||||
_s_types = ['src', 'dst']
|
||||
|
||||
eventTypes = {"ArtifactObjectType": {"type": "attachment", "relation": "attachment"},
|
||||
"DomainNameObjectType": {"type": "domain", "relation": "domain"},
|
||||
"FileObjectType": _file_attribute_type,
|
||||
"HostnameObjectType": {"type": "hostname", "relation": "host"},
|
||||
"MutexObjectType": {"type": "mutex", "relation": "mutex"},
|
||||
"PDFFileObjectType": _file_attribute_type,
|
||||
"PortObjectType": {"type": "port", "relation": "port"},
|
||||
"URIObjectType": {"type": "url", "relation": "url"},
|
||||
"WindowsExecutableFileObjectType": _file_attribute_type,
|
||||
"WindowsRegistryKeyObjectType": {"type": "regkey", "relation": ""}}
|
||||
|
||||
_AS_attribute = ('AS', 'asn')
|
||||
_as_mapping = {'number': _AS_attribute, 'handle': _AS_attribute, 'name': ('text', 'description')}
|
||||
_credential_authentication_mapping = {'authentication_type': ('text', 'value', 'type'),
|
||||
'authentication_data': ('text', 'value', 'password'),
|
||||
'structured_authentication_mechanism': ('text', 'description.value', 'format')}
|
||||
_credential_custom_types = ("username", "origin", "notification")
|
||||
_email_mapping = {'from_': ("email-src", "address_value.value", "from"),
|
||||
'reply_to': ("email-reply-to", 'address_value.value', "reply-to"),
|
||||
'subject': ("email-subject", 'value', "subject"),
|
||||
'x_mailer': ("email-x-mailer", 'value', "x-mailer"),
|
||||
'boundary': ("email-mime-boundary", 'value', "mime-boundary"),
|
||||
'user_agent': ("text", 'value', "user-agent")}
|
||||
_file_mapping = {'file_path': ('text', 'file_path.value', 'path'),
|
||||
'file_format': ('mime-type', 'file_format.value', 'mimetype'),
|
||||
'byte_runs': ('pattern-in-file', 'byte_runs[0].byte_run_data', 'pattern-in-file'),
|
||||
'size_in_bytes': ('size-in-bytes', 'size_in_bytes.value', 'size-in-bytes'),
|
||||
'peak_entropy': ('float', 'peak_entropy.value', 'entropy')}
|
||||
_network_socket_mapping = {'protocol': ('text', 'protocol.value', 'protocol'),
|
||||
'address_family': ('text', 'address_family.value', 'address-family'),
|
||||
'domain': ('text', 'domain.value', 'domain-family')}
|
||||
_process_mapping = {'creation_time': ('datetime', 'creation-time'),
|
||||
'start_time': ('datetime', 'start-time'),
|
||||
'name': ('text', 'name'),
|
||||
'pid': ('text', 'pid'),
|
||||
'parent_pid': ('text', 'parent-pid')}
|
||||
_regkey_mapping = {'hive': ('text', 'hive'), 'key': ('regkey', 'key')}
|
||||
_regkey_value_mapping = {'data': ('text', 'data'), 'datatype': ('text', 'data-type'), 'name': ('text', 'name')}
|
||||
_socket_mapping = {'ip_address': ('ip-{}', 'address_value', 'ip-{}'),
|
||||
'port': ('port', 'port_value', '{}-port'),
|
||||
'hostname': ('hostname', 'hostname_value', 'hostname-{}')}
|
||||
_whois_registrant_mapping = {'email_address': ('whois-registrant-email', 'address_value.value', 'registrant-email'),
|
||||
'name': ('whois-registrant-name', 'value', 'registrant-name'),
|
||||
'phone_number': ('whois-registrant-phone', 'value', 'registrant-phone'),
|
||||
'organization': ('whois-registrant-org', 'value', 'registrant-org')}
|
||||
|
||||
_whois_mapping = {'registrar_info': ('whois-registrar', 'value', 'whois-registrar'),
|
||||
'ip_address': ('ip-src', 'address_value.value', 'ip-address'),
|
||||
'domain_name': ('domain', 'value.value', 'domain')}
|
||||
_x509_datetime_types = ('not_before', 'not_after')
|
||||
_x509_pubkey_types = ('exponent', 'modulus')
|
||||
_x509_certificate_types = ('version', 'serial_number', 'issuer', 'subject')
|
|
@ -1 +1 @@
|
|||
Subproject commit e597831a25c2a2f8bf08320420efee4dbb386389
|
||||
Subproject commit 0ebe53940cb31fe674ea0caf35cfe81269b7b32a
|
Loading…
Reference in New Issue