Merge branch '2.4' into feature/api_rework

pull/3608/head
iglocska 2018-08-17 14:49:09 +02:00
commit f675fb8b29
18 changed files with 680 additions and 311 deletions

View File

@ -0,0 +1,97 @@
Using S3 as an attachment store
===============================
It is possible to use Amazon's Simple Storage Service (S3) to store event attachments
to allow for a stateless MISP setup (i.e for containerisation)
There's a massive caveat here so let me make this incredibly clear
##############################################
# WARNING WARNING WARNING #
# #
# Storing malware is against amazon's #
# terms of service. #
# #
# DO NOT USE THIS UNLESS YOU HAVE #
# THEIR EXPLICIT PERMISSION #
##############################################
0. Installing Dependencies
--------------------------
Install the AWS PHP SDK
```bash
cd /var/www/MISP/app
sudo -u www-data php composer.phar config vendor-dir Vendor
sudo -u www-data php composer.phar require aws/aws-sdk-php
```
1. Creating an S3 bucket
-------------------------
Go to https://s3.console.aws.amazon.com/s3/home
And create a bucket. It has to have a globally unique name, and
this cannot be changed later on.
2a. Using an EC2 instance for MISP
-----------------------------------
If you run MISP on EC2, this will be super duper easy peasy.
Simply create an IAM role with the following permissions and assign it to the instance
by right-clicking and selecting "Instance Settings -> Attach/Replace IAM role"
```json
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "PermitMISPAttachmentsToS3",
"Effect": "Allow",
"Action": [
"s3:*"
],
"Resource": [
"arn:aws:s3:::your-bucket-name"
]
}
]
}
```
2b. Using AWS access keys
-------------------------
This is not recommended, but it works I think.
Create a new programmatic access user via IAM and apply the same
policy outlined above.
Copy the access keys and save them for the next step
3. Setting up MISP
------------------
In Administration -> Server Settings & Maintenance -> MISP settings
Set MISP.attachments_dir to "s3://"
In Administration -> Server Settings & Maintenance -> Plugin Settings -> S3
Set S3_enable to True
Set S3_bucket-name to the bucket you created earlier
Set S3_region to your region
ONLY IF YOU DID NOT USE THE EC2 METHOD
Set aws_access_key and aws_secret_key to the ones you created in 2b
Now theoretically it should work.
Addendum
========
If you are migrating a server currently in use, simply copy the directory structure from
the attachments folder (usually /var/www/MISP/app/files) to S3 and everything should
continue to work.

View File

@ -128,7 +128,7 @@ cp -fa /home/misp/public_html/MISP/INSTALL/setup/config.php /home/misp/public_ht
----------------------
# Check if the permissions are set correctly using the following commands:
sudo chmod -R 750 /home/misp/public_html/MISP
sudo chmod -R 770 /home/misp/public_html/MISP
sudo chmod -R g+ws /home/misp/public_html/MISP/app/tmp
sudo chmod -R g+ws /home/misp/public_html/MISP/app/files
sudo chmod -R g+ws /home/misp/public_html/MISP/app/files/scripts/tmp
@ -193,7 +193,7 @@ cp -a /home/misp/public_html/MISP/app/Config/core.default.php /home/misp/public_
cp -a /home/misp/public_html/MISP/app/Config/config.default.php /home/misp/public_html/MISP/app/Config/config.php
# Configure the fields in the newly created files:
vim /home/misp/public_html/MISP/app/Config/database.php
vi /home/misp/public_html/MISP/app/Config/database.php
# DATABASE_CONFIG has to be filled
# With the default values provided in section 6, this would look like:
# class DATABASE_CONFIG {
@ -219,7 +219,7 @@ vim /home/misp/public_html/MISP/app/Config/database.php
# delete the user from mysql and log in again using the default admin credentials (admin@admin.test / admin)
# Change base url in config.php
vim /home/misp/public_html/MISP/app/Config/config.php
vi /home/misp/public_html/MISP/app/Config/config.php
# example: 'baseurl' => 'https://<your.FQDN.here>',
# alternatively, you can leave this field empty if you would like to use relative pathing in MISP
# 'baseurl' => '',
@ -253,9 +253,49 @@ gpg --homedir /home/misp/public_html/MISP/.gnupg --export --armor YOUR-KEYS-EMAI
# To make the background workers start on boot
chmod +x /home/misp/public_html/MISP/app/Console/worker/start.sh
sudo vim /etc/rc.local
# Add the following line before the last line (exit 0). Make sure that you replace www-data with your apache user:
sudo -u www-data bash /home/misp/public_html/MISP/app/Console/worker/start.sh
# Activate rc.local in systemd
# Systemd developers, in their wisdom, decided to complicate things a bit so you'll have to
# create the rc-local.service
sudo vi /etc/systemd/system/rc-local.service
# and paste the following in it
[Unit]
Description=/etc/rc.local Compatibility
ConditionPathExists=/etc/rc.local
[Service]
Type=forking
ExecStart=/etc/rc.local start
TimeoutSec=0
StandardOutput=tty
RemainAfterExit=yes
SysVStartPriority=99
[Install]
WantedBy=multi-user.target
# Hit the "esc" button then type :wq! to write the file and exit vi
# Create/edit /etc/rc.local
sudo vi /etc/rc.local
# If the file is empty add the following including the #
#!/bin/bash
# Then add this
sudo -u misp bash /home/misp/public_html/MISP/app/Console/worker/start.sh
# If the file was empty add this as the last line
exit 0
# save, quit vi and set permissions
sudo chmod +x /etc/rc.local
# Enable it in systemd
sudo systemctl enable rc-local
#Start the rc-local compatibility layer and check if AOK
sudo systemctl start rc-local.service
sudo systemctl status rc-local.service
# Now log in using the webinterface:
# The default user/pass = admin@admin.test/admin
@ -271,8 +311,8 @@ sudo -u www-data bash /home/misp/public_html/MISP/app/Console/worker/start.sh
# If any of the directories that MISP uses to store files is not writeable to the apache user, change the permissions
# you can do this by running the following commands:
sudo chmod -R 750 /home/misp/public_html/MISP/<directory path with an indicated issue>
sudo chown -R misp:misp /home/misp/public_html/MISP/<directory path with an indicated issue>
sudo chmod -R 770 /home/misp/public_html/MISP/<directory path with an indicated issue>
sudo chown -R misp:www-data /home/misp/public_html/MISP/<directory path with an indicated issue>
# Make sure that the STIX libraries and GnuPG work as intended, if not, refer to INSTALL.txt's paragraphs dealing with these two items

View File

@ -433,8 +433,33 @@ class AttributesController extends AppController
$this->loadModel('Server');
$attachments_dir = $this->Server->getDefaultAttachments_dir();
}
$path = $attachments_dir . DS . $attribute['event_id'] . DS;
$file = $attribute['id'];
$is_s3 = substr($attachments_dir, 0, 2) === "s3";
if ($is_s3) {
// We have to download it!
App::uses('AWSS3Client', 'Tools');
$client = new AWSS3Client();
$client->initTool();
// Use tmpdir as opposed to attachments dir since we can't write to s3://
$attachments_dir = Configure::read('MISP.tmpdir');
if (empty($attachments_dir)) {
$this->loadModel('Server');
$attachments_dir = $this->Server->getDefaultTmp_dir();
}
// Now download the file
$resp = $client->download($attribute['event_id'] . DS . $attribute['id']);
// Save to a tmpfile
$tmpFile = new File($attachments_dir . DS . $attribute['uuid'], true, 0600);
$tmpFile->write($resp);
$tmpFile->close();
$path = $attachments_dir . DS;
$file = $attribute['uuid'];
} else {
$path = $attachments_dir . DS . $attribute['event_id'] . DS;
$file = $attribute['id'];
}
if ('attachment' == $attribute['type']) {
$filename = $attribute['value'];
$fileExt = pathinfo($filename, PATHINFO_EXTENSION);
@ -877,10 +902,12 @@ class AttributesController extends AppController
if (count($existingAttribute) && !$existingAttribute['Attribute']['deleted']) {
$this->request->data['Attribute']['id'] = $existingAttribute['Attribute']['id'];
$dateObj = new DateTime();
$skipTimeCheck = false;
if (!isset($this->request->data['Attribute']['timestamp'])) {
$this->request->data['Attribute']['timestamp'] = $dateObj->getTimestamp();
$skipTimeCheck = true;
}
if ($this->request->data['Attribute']['timestamp'] > $existingAttribute['Attribute']['timestamp']) {
if ($skipTimeCheck || $this->request->data['Attribute']['timestamp'] > $existingAttribute['Attribute']['timestamp']) {
$recoverFields = array('value', 'to_ids', 'distribution', 'category', 'type', 'comment');
foreach ($recoverFields as $rF) {
if (!isset($this->request->data['Attribute'][$rF])) {
@ -2102,7 +2129,7 @@ class AttributesController extends AppController
}
$data = $this->request->data;
} else {
throw new BadRequestException(__('Either specify the search terms in the url, or POST a json array / xml (with the root element being "request" and specify the correct accept and content type headers.'));
throw new BadRequestException(__('Either specify the search terms in the url, or POST a json array / xml (with the root element being "request" and specify the correct accept and content type headers).'));
}
if (!isset($data['request'])) {
$data['request'] = $data;
@ -2260,7 +2287,7 @@ class AttributesController extends AppController
} elseif ($this->response->type() === 'application/xml' && !empty($this->request->data)) {
$data = $this->request->data;
} else {
throw new BadRequestException(__('Either specify the search terms in the url, or POST a json array / xml (with the root element being "request" and specify the correct accept and content type headers.'));
throw new BadRequestException(__('Either specify the search terms in the url, or POST a json array / xml (with the root element being "request" and specify the correct accept and content type headers).'));
}
$paramArray = array('type', 'sigOnly');
foreach ($paramArray as $p) {

View File

@ -164,6 +164,9 @@ class UsersController extends AppController
public function change_pw()
{
if (!$this->_isAdmin() && Configure::read('MISP.disableUserSelfManagement')) {
throw new MethodNotAllowedException('User self-management has been disabled on this instance.');
}
$id = $this->Auth->user('id');
$user = $this->User->find('first', array(
'conditions' => array('User.id' => $id),
@ -1724,8 +1727,12 @@ class UsersController extends AppController
$statistics[$scope]['data'][$range] = $this->{$scope_data['model']}->find('count', $params);
}
}
$this->set('statistics', $statistics);
$this->render('statistics_users');
if ($this->_isRest()) {
return $this->RestResponse->viewData($statistics, $this->response->type());
} else {
$this->set('statistics', $statistics);
$this->render('statistics_users');
}
}
public function tagStatisticsGraph()

View File

@ -0,0 +1,94 @@
<?php
use Aws\S3\S3Client;
class AWSS3Client
{
private $__settings = false;
private $__client = false;
private function __getSetSettings()
{
$settings = array(
'enabled' => false,
'bucket_name' => 'my-malware-bucket',
'region' => 'eu-west-1',
'aws_access_key' => '',
'aws_secret_key' => ''
);
// We have 2 situations
// Either we're running on EC2 and we can assume an IAM role
// Or we're not and need explicitly set AWS key
if (strlen($settings['aws_access_key']) > 0) {
putenv('AWS_ACCESS_KEY_ID='.$settings['aws_access_key']);
}
if (strlen($settings['aws_secret_key']) > 0) {
putenv('AWS_SECRET_ACCESS_KEY='.$settings['aws_secret_key']);
}
foreach ($settings as $key => $setting) {
$temp = Configure::read('Plugin.S3_' . $key);
if ($temp) {
$settings[$key] = $temp;
}
}
return $settings;
}
public function initTool()
{
$settings = $this->__getSetSettings();
$s3 = new Aws\S3\S3Client([
'version' => 'latest',
'region' => $settings['region']
]);
$this->__client = $s3;
$this->__settings = $settings;
return $s3;
}
public function upload($key, $data)
{
$this->__client->putObject([
'Bucket' => $this->__settings['bucket_name'],
'Key' => $key,
'Body' => $data
]);
}
public function download($key)
{
$result = $this->__client->getObject([
'Bucket' => $this->__settings['bucket_name'],
'Key' => $key
]);
return $result['Body'];
}
public function delete($key)
{
$this->__client->deleteObject([
'Bucket' => $this->__settings['bucket_name'],
'Key' => $key
]);
}
public function deleteDirectory($prefix) {
$keys = $s3->listObjects([
'Bucket' => $this->__settings['bucket_name'],
'Prefix' => $prefix
]) ->getPath('Contents/*/Key');
$s3->deleteObjects([
'Bucket' => $bucket,
'Delete' => [
'Objects' => array_map(function ($key) {
return ['Key' => $key];
}, $keys)
],
]);
}
}

View File

@ -38,6 +38,7 @@ class AppModel extends Model
private $__profiler = array();
public $elasticSearchClient = false;
public $s3Client = false;
public function __construct($id = false, $table = null, $ds = null)
{
@ -1457,6 +1458,26 @@ class AppModel extends Model
$this->elasticSearchClient = $client;
}
public function getS3Client() {
if (!$this->s3Client) {
$this->s3Client = $this->loadS3Client();
}
return $this->s3Client;
}
public function loadS3Client() {
App::uses('AWSS3Client', 'Tools');
$client = new AWSS3Client();
$client->initTool();
return $client;
}
public function attachmentDirIsS3() {
// Naive way to detect if we're working in S3
return substr(Configure::read('MISP.attachments_dir'), 0, 2) === "s3";
}
public function checkVersionRequirements($versionString, $minVersion)
{
$version = explode('.', $versionString);

View File

@ -661,11 +661,21 @@ class Attribute extends AppModel
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . $this->data['Attribute']['event_id'] . DS . $this->data['Attribute']['id'];
$file = new File($filepath);
if ($file->exists()) {
if (!$file->delete()) {
throw new InternalErrorException(__('Delete of file attachment failed. Please report to administrator.'));
// Special case - If using S3, we have to delete from there
if ($this->attachmentDirIsS3()) {
// We're working in S3
$s3 = $this->getS3Client();
$s3->delete($this->data['Attribute']['event_id'] . DS . $this->data['Attribute']['id']);
}
else {
// Standard delete
$filepath = $attachments_dir . DS . $this->data['Attribute']['event_id'] . DS . $this->data['Attribute']['id'];
$file = new File($filepath);
if ($file->exists()) {
if (!$file->delete()) {
throw new InternalErrorException(__('Delete of file attachment failed. Please report to administrator.'));
}
}
}
}
@ -1507,12 +1517,22 @@ class Attribute extends AppModel
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . $attribute['event_id'] . DS . $attribute['id'];
$file = new File($filepath);
if (!$file->readable()) {
return '';
if ($this->attachmentDirIsS3()) {
// S3 - we have to first get the object then we can encode it
$s3 = $this->getS3Client();
// This will return the content of the object
$content = $s3->download($attribute['event_id'] . DS . $attribute['id']);
} else {
// Standard filesystem
$filepath = $attachments_dir . DS . $attribute['event_id'] . DS . $attribute['id'];
$file = new File($filepath);
if (!$file->readable()) {
return '';
}
$content = $file->read();
}
$content = $file->read();
return base64_encode($content);
}
@ -1523,16 +1543,29 @@ class Attribute extends AppModel
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$rootDir = $attachments_dir . DS . $attribute['event_id'];
$dir = new Folder($rootDir, true); // create directory structure
$destpath = $rootDir . DS . $attribute['id'];
$file = new File($destpath, true); // create the file
$decodedData = base64_decode($attribute['data']); // decode
if ($file->write($decodedData)) { // save the data
if ($this->attachmentDirIsS3()) {
// This is the cloud!
// We don't need your fancy directory structures and
// PEE AICH PEE meddling
$s3 = $this->getS3Client();
$data = base64_decode($attribute['data']);
$key = $attribute['event_id'] . DS . $attribute['id'];
$s3->upload($key, $data);
return true;
} else {
// error
return false;
// Plebian filesystem operations
$rootDir = $attachments_dir . DS . $attribute['event_id'];
$dir = new Folder($rootDir, true); // create directory structure
$destpath = $rootDir . DS . $attribute['id'];
$file = new File($destpath, true); // create the file
$decodedData = base64_decode($attribute['data']); // decode
if ($file->write($decodedData)) { // save the data
return true;
} else {
// error
return false;
}
}
}
@ -2894,6 +2927,18 @@ class Attribute extends AppModel
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
// If we've set attachments to S3, we can't write there
if ($this->attachmentDirIsS3()) {
$attachments_dir = Configure::read('MISP.tmpdir');
// Sometimes it's not set?
if (empty($attachments_dir)) {
// Get a default tmpdir
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultTmp_dir();
}
}
if ($proposal) {
$dir = new Folder($attachments_dir . DS . $event_id . DS . 'shadow', true);
} else {

View File

@ -359,11 +359,20 @@ class Event extends AppModel
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . $this->id;
App::uses('Folder', 'Utility');
if (is_dir($filepath)) {
if (!$this->destroyDir($filepath)) {
throw new InternalErrorException('Delete of event file directory failed. Please report to administrator.');
// Things get a little funky here
if ($this->attachmentDirIsS3()) {
// S3 doesn't have folders
// So we have to basically `ls` them to look for a prefix
$s3 = $this->getS3Client();
$s3->deleteDirectory($this->id);
} else {
$filepath = $attachments_dir . DS . $this->id;
App::uses('Folder', 'Utility');
if (is_dir($filepath)) {
if (!$this->destroyDir($filepath)) {
throw new InternalErrorException('Delete of event file directory failed. Please report to administrator.');
}
}
}
}

View File

@ -1386,6 +1386,46 @@ class Server extends AppModel
'test' => 'testForEmpty',
'type' => 'string'
),
'S3_enable' => array(
'level' => 2,
'description' => __('Enables or disables uploading of malware samples to S3 rather than to disk (WARNING: Get permission from amazon first!)'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean'
),
'S3_bucket_name' => array(
'level' => 2,
'description' => __('Bucket name to upload to'),
'value' => '',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'S3_region' => array(
'level' => 2,
'description' => __('Region in which your S3 bucket resides'),
'value' => '',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'S3_aws_access_key' => array(
'level' => 2,
'description' => __('AWS key to use when uploading samples (WARNING: It\' highly recommended that you use EC2 IAM roles if at all possible)'),
'value' => '',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'S3_aws_secret_key' => array(
'level' => 2,
'description' => __('AWS secret key to use when uploading samples'),
'value' => '',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string'
),
'Sightings_policy' => array(
'level' => 1,
'description' => __('This setting defines who will have access to seeing the reported sightings. The default setting is the event owner alone (in addition to everyone seeing their own contribution) with the other options being Sighting reporters (meaning the event owner and anyone that provided sighting data about the event) and Everyone (meaning anyone that has access to seeing the event / attribute).'),
@ -4001,6 +4041,11 @@ class Server extends AppModel
return APP . 'files';
}
public function getDefaultTmp_dir()
{
return sys_get_temp_dir();
}
public function fetchServer($id)
{
if (empty($id)) {

View File

@ -248,16 +248,21 @@ class ShadowAttribute extends AppModel
$sa = $this->find('first', array('conditions' => array('ShadowAttribute.id' => $this->data['ShadowAttribute']['id']), 'recursive' => -1, 'fields' => array('ShadowAttribute.id', 'ShadowAttribute.event_id', 'ShadowAttribute.type')));
if ($this->typeIsAttachment($sa['ShadowAttribute']['type'])) {
// only delete the file if it exists
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . 'shadow' . DS . $sa['ShadowAttribute']['event_id'] . DS . $sa['ShadowAttribute']['id'];
$file = new File($filepath);
if ($file->exists()) {
if (!$file->delete()) {
throw new InternalErrorException('Delete of file attachment failed. Please report to administrator.');
if ($this->attachmentDirIsS3()) {
$s3 = $this->getS3Client();
$s3->delete('shadow' . DS . $sa['ShadowAttribute']['event_id'] . DS . $sa['ShadowAttribute']['id']);
} else {
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . 'shadow' . DS . $sa['ShadowAttribute']['event_id'] . DS . $sa['ShadowAttribute']['id'];
$file = new File($filepath);
if ($file->exists()) {
if (!$file->delete()) {
throw new InternalErrorException('Delete of file attachment failed. Please report to administrator.');
}
}
}
}
@ -281,16 +286,21 @@ class ShadowAttribute extends AppModel
$this->read(); // first read the attribute from the db
if ($this->typeIsAttachment($this->data['ShadowAttribute']['type'])) {
// only delete the file if it exists
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . 'shadow' . DS . $this->data['ShadowAttribute']['event_id'] . DS . $this->data['ShadowAttribute']['id'];
$file = new File($filepath);
if ($file->exists()) {
if (!$file->delete()) {
throw new InternalErrorException('Delete of file attachment failed. Please report to administrator.');
if ($this->attachmentDirIsS3()) {
$s3 = $this->getS3Client();
$s3->delete('shadow' . DS . $this->data['ShadowAttribute']['event_id'] . DS . $this->data['ShadowAttribute']['id']);
} else {
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . 'shadow' . DS . $this->data['ShadowAttribute']['event_id'] . DS . $this->data['ShadowAttribute']['id'];
$file = new File($filepath);
if ($file->exists()) {
if (!$file->delete()) {
throw new InternalErrorException('Delete of file attachment failed. Please report to administrator.');
}
}
}
}
@ -394,12 +404,18 @@ class ShadowAttribute extends AppModel
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$filepath = $attachments_dir . DS . 'shadow' . DS . $attribute['event_id'] . DS. $attribute['id'];
$file = new File($filepath);
if (!$file->exists()) {
return '';
if ($this->attachmentDirIsS3()) {
$s3 = $this->getS3Client();
$content = $s3->download('shadow' . DS . $attribute['event_id'] . DS. $attribute['id']);
} else {
$filepath = $attachments_dir . DS . 'shadow' . DS . $attribute['event_id'] . DS. $attribute['id'];
$file = new File($filepath);
if (!$file->exists()) {
return '';
}
$content = $file->read();
}
$content = $file->read();
return base64_encode($content);
}
@ -410,16 +426,23 @@ class ShadowAttribute extends AppModel
$my_server = ClassRegistry::init('Server');
$attachments_dir = $my_server->getDefaultAttachments_dir();
}
$rootDir = $attachments_dir . DS . 'shadow' . DS . $attribute['event_id'];
$dir = new Folder($rootDir, true); // create directory structure
$destpath = $rootDir . DS . $attribute['id'];
$file = new File($destpath, true); // create the file
$decodedData = base64_decode($attribute['data']); // decode
if ($file->write($decodedData)) { // save the data
if ($this->attachmentDirIsS3()) {
$s3 = $this->getS3Client();
$decodedData = base64_decode($attribute['data']);
$s3->upload('shadow' . DS . $attribute['event_id'], $decodedData);
return true;
} else {
// error
return false;
$rootDir = $attachments_dir . DS . 'shadow' . DS . $attribute['event_id'];
$dir = new Folder($rootDir, true); // create directory structure
$destpath = $rootDir . DS . $attribute['id'];
$file = new File($destpath, true); // create the file
$decodedData = base64_decode($attribute['data']); // decode
if ($file->write($decodedData)) { // save the data
return true;
} else {
// error
return false;
}
}
}

View File

@ -779,7 +779,7 @@ class User extends AppModel
}
// SMIME if not GPG key
if (!$failed && !$canEncryptGPG && $canEncryptSMIME) {
$encryptionResult = $this->__encryptUsingSmime($Email, $body, $subject);
$encryptionResult = $this->__encryptUsingSmime($Email, $body, $subject, $user);
if (isset($encryptionResult['failed'])) {
$failed = true;
}
@ -893,7 +893,7 @@ class User extends AppModel
return true;
}
private function __encryptUsingSmime(&$Email, &$body, $subject)
private function __encryptUsingSmime(&$Email, &$body, $subject, $user)
{
try {
$prependedBody = 'Content-Transfer-Encoding: 7bit' . PHP_EOL . 'Content-Type: text/plain;' . PHP_EOL . ' charset=us-ascii' . PHP_EOL . PHP_EOL . $body;

View File

@ -4,7 +4,7 @@
<?php
foreach ($result as $r) {
?>
<h3>V<?php echo __('alidation errors for event: %s', h($r['id']));?></h3>
<h3><?php echo __('Validation errors for event: %s', h($r['id']));?></h3>
<?php print_r($r['error']); ?><br />
<?php echo __('Attribute details');?>:<br />
<?php print_r(h($r['details'])); ?>

View File

@ -36,7 +36,7 @@
<tr>
<?php if ($isSiteAdmin): ?>
<th>
<input class="select_all select" type="checkbox" title="<?php echo __('Select all');?>" role="button" tabindex="0" aria-label="<?php echo __('Select all eventson current page');?>" onClick="toggleAllCheckboxes();" />&nbsp;
<input class="select_all select" type="checkbox" title="<?php echo __('Select all');?>" role="button" tabindex="0" aria-label="<?php echo __('Select all events on current page');?>" onClick="toggleAllCheckboxes();" />&nbsp;
</th>
<?php else: ?>
<th style="padding-left:0px;padding-right:0px;">&nbsp;</th>

View File

@ -7,6 +7,7 @@
"pear/net_geoip": "@dev"
},
"suggest": {
"elasticsearch/elasticsearch": "For logging to elasticsearch"
"elasticsearch/elasticsearch": "For logging to elasticsearch",
"aws/aws-sdk-php": "To upload samples to S3"
}
}

@ -1 +1 @@
Subproject commit 37396ed6ca7023a65981e95aa5842951a9c6a985
Subproject commit cd76f19f52e94a61f0d500fa3cdbf89a758e1c19

View File

@ -15,25 +15,17 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import sys, json, os, time, uuid
import mixbox.namespaces as mixbox_ns
import sys
import json
import os
import time
import uuid
import stix2misp_mapping
from operator import attrgetter
from pymisp import MISPEvent, MISPObject, MISPAttribute, __path__
from stix.core import STIXPackage
from collections import defaultdict
file_object_type = {"type": "filename", "relation": "filename"}
eventTypes = {"ArtifactObjectType": {"type": "attachment", "relation": "attachment"},
"DomainNameObjectType": {"type": "domain", "relation": "domain"},
"FileObjectType": file_object_type,
"HostnameObjectType": {"type": "hostname", "relation": "host"},
"MutexObjectType": {"type": "mutex", "relation": "mutex"},
"PDFFileObjectType": file_object_type,
"PortObjectType": {"type": "port", "relation": "port"},
"URIObjectType": {"type": "url", "relation": "url"},
"WindowsExecutableFileObjectType": file_object_type,
"WindowsRegistryKeyObjectType": {"type": "regkey", "relation": ""}}
cybox_to_misp_object = {"Account": "credential", "AutonomousSystem": "asn",
"EmailMessage": "email", "NetworkConnection": "network-connection",
"NetworkSocket": "network-socket", "Process": "process",
@ -62,7 +54,7 @@ class StixParser():
try:
import maec
print(2)
except:
except ModuleNotFoundError:
print(3)
sys.exit(0)
title = event.stix_header.title
@ -204,10 +196,9 @@ class StixParser():
# Parse a DNS object
def resolve_dns_objects(self):
for domain in self.dns_objects['domain']:
domain_object = self.dns_objects['domain'][domain]
ip_reference = domain_object['related']
domain_attribute = domain_object['data']
for domain, domain_dict in self.dns_objects['domain'].items():
ip_reference = domain_dict['related']
domain_attribute = domain_dict['data']
if ip_reference in self.dns_objects['ip']:
misp_object = MISPObject('passive-dns')
domain_attribute['object_relation'] = "rrname"
@ -221,10 +212,9 @@ class StixParser():
self.misp_event.add_object(**misp_object)
else:
self.misp_event.add_attribute(**domain_attribute)
for ip in self.dns_objects['ip']:
for ip, ip_dict in self.dns_objects['ip'].items():
if ip not in self.dns_ips:
# print(ip)
self.misp_event.add_attribute(**self.dns_objects['ip'][ip])
self.misp_event.add_attribute(**ip_dict)
def set_distribution(self):
for attribute in self.misp_event.attributes:
@ -347,17 +337,17 @@ class StixParser():
# Define type & value of an attribute or object in MISP
def handle_attribute_type(self, properties, is_object=False, title=None, observable_id=None):
xsi_type = properties._XSI_TYPE
try:
args = [properties]
if xsi_type in ("FileObjectType", "PDFFileObjectType"):
args.append(is_object)
elif xsi_type == "ArtifactObjectType":
args.append(title)
return self.attribute_types_mapping[xsi_type](*args)
except AttributeError:
# ATM USED TO TEST TYPES
print("Unparsed type: {}".format(xsi_type))
sys.exit(1)
# try:
args = [properties]
if xsi_type in ("FileObjectType", "PDFFileObjectType"):
args.append(is_object)
elif xsi_type == "ArtifactObjectType":
args.append(title)
return self.attribute_types_mapping[xsi_type](*args)
# except AttributeError:
# # ATM USED TO TEST TYPES
# print("Unparsed type: {}".format(xsi_type))
# sys.exit(1)
# Return type & value of an ip address attribute
@staticmethod
@ -369,23 +359,15 @@ class StixParser():
return ip_type, properties.address_value.value, "ip"
def handle_as(self, properties):
attributes = []
if properties.number:
attributes.append(['AS', properties.number.value, 'asn'])
if properties.handle:
attributes.append(['AS', properties.handle.value, 'asn'])
if properties.name:
attributes.append(['text', properties.name.value, 'description'])
if len(attributes) == 1:
return attributes[0]
return 'asn', self.return_attributes(attributes), ''
attributes = self.fetch_attributes_with_partial_key_parsing(properties, stix2misp_mapping._as_mapping)
return attributes[0] if len(attributes) == 1 else ('asn', self.return_attributes(attributes), '')
# Return type & value of an attachment attribute
@staticmethod
def handle_attachment(properties, title):
if properties.hashes:
return "malware-sample", "{}|{}".format(title, properties.hashes[0], properties.raw_artifact.value)
return eventTypes[properties._XSI_TYPE]['type'], title, properties.raw_artifact.value
return stix2misp_mapping.eventTypes[properties._XSI_TYPE]['type'], title, properties.raw_artifact.value
# Return type & attributes of a credential object
def handle_credential(self, properties):
@ -393,38 +375,13 @@ class StixParser():
if properties.description:
attributes.append(["text", properties.description.value, "text"])
if properties.authentication:
l_password, l_type, l_format = [], [], []
for authentication in properties.authentication:
if authentication.authentication_type:
auth_type = authentication.authentication_type.value
if auth_type not in l_type:
l_type.append(auth_type)
if authentication.authentication_data:
auth_data = authentication.authentication_data.value
if auth_data not in l_password:
l_password.append(auth_data)
if authentication.structured_authentication_mechanism:
auth_format = authentication.structured_authentication_mechanism.description.value
if auth_format not in l_format:
l_format.append(auth_format)
if l_type:
# auth_type = l_type[0] if len(l_type) == 1 else "".join(["{}, ".format(t) for t in l_type])[:-2]
for auth_type in l_type:
attributes.append(["text", auth_type, "type"])
if l_format:
# auth_format = l_format[0] if len(l_format) == 1 else "".join(["{}, ".format(f) for f in l_format])[:-2]
for auth_format in l_format:
attributes.append(["text", auth_format, "format"])
if l_password:
for password in l_password:
attributes.append(["text", password, "password"])
attributes += self.fetch_attributes_with_key_parsing(authentication, stix2misp_mapping._credential_authentication_mapping)
if properties.custom_properties:
for prop in properties.custom_properties:
if prop.name in ("username", "origin", "notification"):
attributes.append(["text", prop.value, prop.name])
if len(attributes) == 1:
return attributes[0]
return "credential", self.return_attributes(attributes), ""
if prop.name in stix2misp_mapping._credential_custom_types:
attributes.append(['text', prop.value, prop.name])
return attributes[0] if len(attributes) == 1 else ("credential", self.return_attributes(attributes), "")
# Return type & attributes (or value) of a Custom Object
def handle_custom(self, properties):
@ -467,43 +424,31 @@ class StixParser():
# Return type & value of a domain or url attribute
@staticmethod
def handle_domain_or_url(properties):
event_types = eventTypes[properties._XSI_TYPE]
event_types = stix2misp_mapping.eventTypes[properties._XSI_TYPE]
return event_types['type'], properties.value.value, event_types['relation']
# Return type & value of an email attribute
def handle_email_attribute(self, properties):
attributes = []
if properties.header:
header = properties.header
if header.from_:
attributes.append(["email-src", header.from_.address_value.value, "from"])
attributes = self.fetch_attributes_with_key_parsing(header, stix2misp_mapping._email_mapping)
if header.to:
for to in header.to:
attributes.append(["email-dst", to.address_value.value, "to"])
if header.cc:
for cc in header.cc:
attributes.append(["email-dst", cc.address_value.value, "cc"])
if header.reply_to:
attributes.append(["email-reply-to", header.reply_to.address_value.value, "reply-to"])
if header.subject:
attributes.append(["email-subject", header.subject.value, "subject"])
if header.x_mailer:
attributes.append(["email-x-mailer", header.x_mailer.value, "x-mailer"])
if header.boundary:
attributes.append(["email-mime-boundary", header.boundary.value, "mime-boundary"])
if header.user_agent:
attributes.append(["text", header.user_agent.value, "user-agent"])
else:
attributes = []
if properties.attachments:
attributes.append([self.handle_email_attachment(properties.parent)])
if len(attributes) == 1:
return attributes[0]
return "email", self.return_attributes(attributes), ""
attributes.append(self.handle_email_attachment(properties.parent))
return attributes[0] if len(attributes) == 1 else ("email", self.return_attributes(attributes), "")
# Return type & value of an email attachment
@staticmethod
def handle_email_attachment(indicator_object):
properties = indicator_object.related_objects[0].properties
return "email-attachment", properties.file_name.value, "attachment"
return ["email-attachment", properties.file_name.value, "attachment"]
# Return type & attributes of a file object
def handle_file(self, properties, is_object):
@ -513,26 +458,13 @@ class StixParser():
b_hash = True
for h in properties.hashes:
attributes.append(self.handle_hashes_attribute(h))
if properties.file_format and properties.file_format.value:
attributes.append(["mime-type", properties.file_format.value, "mimetype"])
if properties.file_name:
value = properties.file_name.value
if value:
b_file = True
event_types = eventTypes[properties._XSI_TYPE]
attributes.append([event_types['type'], value, event_types['relation']])
if properties.file_path:
value = properties.file_path.value
if value:
attributes.append(['text', value, 'path'])
if properties.byte_runs:
attribute_type = "pattern-in-file"
attributes.append([attribute_type, properties.byte_runs[0].byte_run_data, attribute_type])
if properties.size_in_bytes and properties.size_in_bytes.value:
attribute_type = "size-in-bytes"
attributes.append([attribute_type, properties.size_in_bytes.value, attribute_type])
if properties.peak_entropy and properties.peak_entropy.value:
attributes.append(["float", properties.peak_entropy.value, "entropy"])
attribute_type, relation = stix2misp_mapping.eventTypes[properties._XSI_TYPE]
attributes.append([attribute_type, value, relation])
self.fetch_attributes_with_keys(properties, stix2misp_mapping._file_mapping, attributes)
if len(attributes) == 1:
return attributes[0]
if len(attributes) == 2:
@ -588,7 +520,7 @@ class StixParser():
# Return type & value of a hostname attribute
@staticmethod
def handle_hostname(properties):
event_types = eventTypes[properties._XSI_TYPE]
event_types = stix2misp_mapping.eventTypes[properties._XSI_TYPE]
return event_types['type'], properties.hostname_value.value, event_types['relation']
# Return type & value of a http request attribute
@ -610,42 +542,25 @@ class StixParser():
# Return type & value of a mutex attribute
@staticmethod
def handle_mutex(properties):
event_types = eventTypes[properties._XSI_TYPE]
event_types = stix2misp_mapping.eventTypes[properties._XSI_TYPE]
return event_types['type'], properties.name.value, event_types['relation']
# Return type & attributes of a network connection object
def handle_network_connection(self, properties):
attributes = []
if properties.source_socket_address:
self.handle_socket(attributes, properties.source_socket_address, "src")
if properties.destination_socket_address:
self.handle_socket(attributes, properties.destination_socket_address, "dst")
if properties.layer3_protocol:
attributes.append(["text", properties.layer3_protocol.value, "layer3-protocol"])
if properties.layer4_protocol:
attributes.append(["text", properties.layer4_protocol.value, "layer4-protocol"])
if properties.layer7_protocol:
attributes.append(["text", properties.layer7_protocol.value, "layer7-protocol"])
attributes = self.fetch_attributes_from_sockets(properties, stix2misp_mapping._network_connection_addresses)
for prop in ('layer3_protocol', 'layer4_protocol', 'layer7_protocol'):
if getattr(properties, prop):
attributes.append(['text', attrgetter("{}.value".format(prop))(properties), prop.replace('_', '-')])
if attributes:
return "network-connection", self.return_attributes(attributes), ""
# Return type & attributes of a network socket objet
def handle_network_socket(self, properties):
attributes = []
if properties.local_address:
self.handle_socket(attributes, properties.local_address, "src")
if properties.remote_address:
self.handle_socket(attributes, properties.remote_address, "dst")
if properties.protocol:
attributes.append(["text", properties.protocol.value, "protocol"])
if properties.is_listening:
attributes.append(["text", "listening", "state"])
if properties.is_blocking:
attributes.append(["text", "blocking", "state"])
if properties.address_family:
attributes.append(["text", properties.address_family.value, "address-family"])
if properties.domain:
attributes.append(["text", properties.domain.value, "domain-family"])
attributes = self.fetch_attributes_from_sockets(properties, stix2misp_mapping._network_socket_addresses)
self.fetch_attributes_with_keys(properties, stix2misp_mapping._network_socket_mapping, attributes)
for prop in ('is_listening', 'is_blocking'):
if getattr(properties, prop):
attributes.append(["text", prop.split('_')[1], "state"])
if attributes:
return "network-socket", self.return_attributes(attributes), ""
@ -653,7 +568,7 @@ class StixParser():
@staticmethod
def handle_port(*kwargs):
properties = kwargs[0]
event_types = eventTypes[properties._XSI_TYPE]
event_types = stix2misp_mapping.eventTypes[properties._XSI_TYPE]
relation = event_types['relation']
if len(kwargs) > 1:
observable_id = kwargs[1]
@ -665,21 +580,10 @@ class StixParser():
# Return type & attributes of a process object
def handle_process(self, properties):
attributes = []
if properties.creation_time:
attributes.append(["datetime", properties.creation_time.value, "creation-time"])
if properties.start_time:
attributes.append(["datetime", properties.creation_time.value, "start-time"])
attribute_type = "text"
if properties.name:
attributes.append([attribute_type, properties.name.value, "name"])
if properties.pid:
attributes.append([attribute_type, properties.pid.value, "pid"])
if properties.parent_pid:
attributes.append([attribute_type, properties.parent_pid.value, "parent-pid"])
attributes = self.fetch_attributes_with_partial_key_parsing(properties, stix2misp_mapping._process_mapping)
if properties.child_pid_list:
for child in properties.child_pid_list:
attributes.append([attribute_type, child.value, "child-pid"])
attributes.append(["text", child.value, "child-pid"])
# if properties.port_list:
# for port in properties.port_list:
# attributes.append(["src-port", port.port_value.value, "port"])
@ -698,33 +602,20 @@ class StixParser():
# Return type & value of a regkey attribute
def handle_regkey(self, properties):
attributes = []
if properties.hive:
attributes.append(["text", properties.hive.value, "hive"])
if properties.key:
attributes.append(["regkey", properties.key.value, "key"])
attributes = self.fetch_attributes_with_partial_key_parsing(properties, stix2misp_mapping._regkey_mapping)
if properties.values:
values = properties.values
value = values[0]
if value.data:
attributes.append(["text", value.data.value, "data"])
if value.datatype:
attributes.append(["text", value.datatype.value, "data-type"])
if value.name:
attributes.append(["text", value.name.value, "name"])
attributes += self.fetch_attributes_with_partial_key_parsing(value, stix2misp_mapping._regkey_value_mapping)
return "registry-key", self.return_attributes(attributes), ""
# Parse a socket address object into a network connection or socket object,
# in order to add its attributes
@staticmethod
def handle_socket(attributes, socket, s_type):
if socket.ip_address:
ip_type = "ip-{}".format(s_type)
attributes.append([ip_type, socket.ip_address.address_value.value, ip_type])
if socket.port:
attributes.append(["port", socket.port.port_value.value, "{}-port".format(s_type)])
if socket.hostname:
attributes.append(["hostname", socket.hostname.hostname_value.value, "hostname-{}".format(s_type)])
for prop, mapping in stix2misp_mapping._socket_mapping.items():
if getattr(socket, prop):
attribute_type, properties_key, relation = mapping
attribute_type, relation = [elem.format(s_type) for elem in (attribute_type, relation)]
attributes.append([attribute_type, attrgetter('{}.{}.value'.format(prop, properties_key))(socket), relation])
# Parse a socket address object in order to return type & value
# of a composite attribute ip|port or hostname|port
@ -746,22 +637,11 @@ class StixParser():
# Return type & attributes of a whois object if we have the required fields
# Otherwise create attributes and return type & value of the last attribute to avoid crashing the parent function
def handle_whois(self, properties):
required_one_of = False
attributes = []
if properties.registrar_info:
attribute_type = "whois-registrar"
attributes.append([attribute_type, properties.registrar_info.value, attribute_type])
required_one_of = True
attributes = self.fetch_attributes_with_key_parsing(properties, stix2misp_mapping._whois_mapping)
required_one_of = True if attributes else False
if properties.registrants:
registrant = properties.registrants[0]
if registrant.email_address:
attributes.append(["whois-registrant-email", registrant.email_address.address_value.value, "registrant-email"])
if registrant.name:
attributes.append(["whois-registrant-name", registrant.name.value, "registrant-name"])
if registrant.phone_number:
attributes.append(["whois-registrant-phone", registrant.phone_number.value, "registrant-phone"])
if registrant.organization:
attributes.append(["whois-registrant-org", registrant.organization.value, "registrant-org"])
attributes += self.fetch_attributes_with_key_parsing(registrant, stix2misp_mapping._whois_registrant_mapping)
if properties.creation_date:
attributes.append(["datetime", properties.creation_date.value.strftime('%Y-%m-%d'), "creation-date"])
required_one_of = True
@ -772,13 +652,6 @@ class StixParser():
if properties.nameservers:
for nameserver in properties.nameservers:
attributes.append(["hostname", nameserver.value.value, "nameserver"])
if properties.ip_address:
attributes.append(["ip-src", properties.ip_address.address_value.value, "ip-address"])
required_one_of = True
if properties.domain_name:
attribute_type = "domain"
attributes.append([attribute_type, properties.domain_name.value.value, attribute_type])
required_one_of = True
if properties.remarks:
attribute_type = "text"
relation = "comment" if attributes else attribute_type
@ -788,16 +661,15 @@ class StixParser():
if required_one_of:
# if yes, we return the object type and the attributes
return "whois", self.return_attributes(attributes), ""
else:
# otherwise, attributes are added in the event, and one attribute is returned to not make the function crash
if len(attributes) == 1:
return attributes[0]
last_attribute = attributes.pop(-1)
for attribute in attributes:
attribute_type, attribute_value, attribute_relation = attribute
misp_attributes = {"comment": "Whois {}".format(attribute_relation)}
self.misp_event.add_attribute(attribute_type, attribute_value, **misp_attributes)
return last_attribute
# otherwise, attributes are added in the event, and one attribute is returned to not make the function crash
if len(attributes) == 1:
return attributes[0]
last_attribute = attributes.pop(-1)
for attribute in attributes:
attribute_type, attribute_value, attribute_relation = attribute
misp_attributes = {"comment": "Whois {}".format(attribute_relation)}
self.misp_event.add_attribute(attribute_type, attribute_value, **misp_attributes)
return last_attribute
# Return type & value of a windows service object
@staticmethod
@ -806,33 +678,7 @@ class StixParser():
return "windows-service-name", properties.name.value, ""
def handle_x509(self, properties):
attributes = []
if properties.certificate:
certificate = properties.certificate
if certificate.validity:
validity = certificate.validity
if validity.not_before:
attributes.append(["datetime", validity.not_before.value, "validity-not-before"])
if validity.not_after:
attributes.append(["datetime", validity.not_after.value, "validity-not-after"])
if certificate.subject_public_key:
subject_pubkey = certificate.subject_public_key
if subject_pubkey.rsa_public_key:
rsa_pubkey = subject_pubkey.rsa_public_key
if rsa_pubkey.exponent:
attributes.append(["text", rsa_pubkey.exponent.value, "pubkey-info-exponent"])
if rsa_pubkey.modulus:
attributes.append(["text", rsa_pubkey.modulus.value, "pubkey-info-modulus"])
if subject_pubkey.public_key_algorithm:
attributes.append(["text", subject_pubkey.public_key_algorithm.value, "pubkey-info-algorithm"])
if certificate.version:
attributes.append(["text", certificate.version.value, "version"])
if certificate.serial_number:
attributes.append(["text", certificate.serial_number.value, "serial-number"])
if certificate.issuer:
attributes.append(["text", certificate.issuer.value, "issuer"])
if certificate.subject:
attributes.append(["text", certificate.subject.value, "subject"])
attributes = self.handle_x509_certificate(properties.certificate) if properties.certificate else []
if properties.raw_certificate:
raw = properties.raw_certificate.value
try:
@ -849,6 +695,28 @@ class StixParser():
attributes.append([attribute_type, signature.signature.value, attribute_type])
return "x509", self.return_attributes(attributes), ""
@staticmethod
def handle_x509_certificate(certificate):
attributes = []
if certificate.validity:
validity = certificate.validity
for prop in stix2misp_mapping._x509_datetime_types:
if getattr(validity, prop):
attributes.append(['datetime', attrgetter('{}.value'.format(prop))(validity), 'validity-{}'.format(prop.replace('_', '-'))])
if certificate.subject_public_key:
subject_pubkey = certificate.subject_public_key
if subject_pubkey.rsa_public_key:
rsa_pubkey = subject_pubkey.rsa_public_key
for prop in stix2misp_mapping._x509__x509_pubkey_types:
if getattr(rsa_pubkey, prop):
attributes.append(['text', attrgetter('{}.value'.format(prop))(rsa_pubkey), 'pubkey-info-{}'.format(prop)])
if subject_pubkey.public_key_algorithm:
attributes.append(["text", subject_pubkey.public_key_algorithm.value, "pubkey-info-algorithm"])
for prop in stix2misp_mapping._x509_certificate_types:
if getattr(certificate, prop):
attributes.append(['text', attrgetter('{}.value'.format(prop))(certificate), prop.replace('_', '-')])
return attributes
# Return type & attributes of the file defining a portable executable object
def handle_pe(self, properties):
pe_uuid = self.parse_pe(properties)
@ -976,7 +844,7 @@ class StixParser():
continue
if properties:
attribute_type, attribute_value, compl_data = self.handle_attribute_type(properties)
if type(attribute_value) in (str, int):
if isinstance(attribute_value, (str, int)):
# if the returned value is a simple value, we build an attribute
attribute = {'to_ids': True}
if indicator.timestamp:
@ -1003,8 +871,7 @@ class StixParser():
# print("Error with an object of type: {}\n{}".format(properties._XSI_TYPE, observable.to_json()))
continue
object_uuid = self.fetch_uuid(observable_object.id_)
attr_type = type(attribute_value)
if attr_type in (str, int):
if isinstance(attribute_value, (str, int)):
# if the returned value is a simple value, we build an attribute
attribute = {'to_ids': False, 'uuid': object_uuid}
if observable_object.related_objects:
@ -1017,7 +884,9 @@ class StixParser():
self.dns_ips.append(related_ip)
continue
if attribute_type in ('ip-src', 'ip-dst'):
self.dns_objects['ip'][object_uuid] = {"type": attribute_type, "value": attribute_value}
attribute['type'] = attribute_type
attribute['value'] = attribute_value
self.dns_objects['ip'][object_uuid] = attribute
continue
self.handle_attribute_case(attribute_type, attribute_value, compl_data, attribute)
else:
@ -1064,6 +933,39 @@ class StixParser():
misp_object.add_reference(uuid, 'connected-to')
self.misp_event.add_object(**misp_object)
def fetch_attributes_from_sockets(self, properties, mapping_dict):
attributes = []
for prop, s_type in zip(mapping_dict, stix2misp_mapping._s_types):
address_property = getattr(properties, prop)
if address_property:
self.handle_socket(attributes, address_property, s_type)
return attributes
@staticmethod
def fetch_attributes_with_keys(properties, mapping_dict, attributes):
for prop, mapping in mapping_dict.items():
if getattr(properties,prop):
attribute_type, properties_key, relation = mapping
attributes.append([attribute_type, attrgetter(properties_key)(properties), relation])
@staticmethod
def fetch_attributes_with_key_parsing(properties, mapping_dict):
attributes = []
for prop, mapping in mapping_dict.items():
if getattr(properties, prop):
attribute_type, properties_key, relation = mapping
attributes.append([attribute_type, attrgetter('{}.{}'.format(prop, properties_key))(properties), relation])
return attributes
@staticmethod
def fetch_attributes_with_partial_key_parsing(properties, mapping_dict):
attributes = []
for prop, mapping in mapping_dict.items():
if getattr(properties, prop):
attribute_type, relation = mapping
attributes.append([attribute_type, attrgetter('{}.value'.format(prop))(properties), relation])
return attributes
# Extract the uuid from a stix id
@staticmethod
def fetch_uuid(object_id):
@ -1105,6 +1007,7 @@ class StixParser():
attribute = {'type': 'text', 'object_relation': 'name',
'value': coa.title}
misp_object.add_attribute(**attribute)
# for prop in stix2misp_mapping.
if coa.type_:
attribute = {'type': 'text', 'object_relation': 'type',
'value': coa.type_.value}

View File

@ -0,0 +1,57 @@
_file_attribute_type = ('filename', 'filename')
_network_socket_addresses = ['local_address', 'remote_address']
_network_connection_addresses = ['source_socket_address', 'destination_socket_address']
_s_types = ['src', 'dst']
eventTypes = {"ArtifactObjectType": {"type": "attachment", "relation": "attachment"},
"DomainNameObjectType": {"type": "domain", "relation": "domain"},
"FileObjectType": _file_attribute_type,
"HostnameObjectType": {"type": "hostname", "relation": "host"},
"MutexObjectType": {"type": "mutex", "relation": "mutex"},
"PDFFileObjectType": _file_attribute_type,
"PortObjectType": {"type": "port", "relation": "port"},
"URIObjectType": {"type": "url", "relation": "url"},
"WindowsExecutableFileObjectType": _file_attribute_type,
"WindowsRegistryKeyObjectType": {"type": "regkey", "relation": ""}}
_AS_attribute = ('AS', 'asn')
_as_mapping = {'number': _AS_attribute, 'handle': _AS_attribute, 'name': ('text', 'description')}
_credential_authentication_mapping = {'authentication_type': ('text', 'value', 'type'),
'authentication_data': ('text', 'value', 'password'),
'structured_authentication_mechanism': ('text', 'description.value', 'format')}
_credential_custom_types = ("username", "origin", "notification")
_email_mapping = {'from_': ("email-src", "address_value.value", "from"),
'reply_to': ("email-reply-to", 'address_value.value', "reply-to"),
'subject': ("email-subject", 'value', "subject"),
'x_mailer': ("email-x-mailer", 'value', "x-mailer"),
'boundary': ("email-mime-boundary", 'value', "mime-boundary"),
'user_agent': ("text", 'value', "user-agent")}
_file_mapping = {'file_path': ('text', 'file_path.value', 'path'),
'file_format': ('mime-type', 'file_format.value', 'mimetype'),
'byte_runs': ('pattern-in-file', 'byte_runs[0].byte_run_data', 'pattern-in-file'),
'size_in_bytes': ('size-in-bytes', 'size_in_bytes.value', 'size-in-bytes'),
'peak_entropy': ('float', 'peak_entropy.value', 'entropy')}
_network_socket_mapping = {'protocol': ('text', 'protocol.value', 'protocol'),
'address_family': ('text', 'address_family.value', 'address-family'),
'domain': ('text', 'domain.value', 'domain-family')}
_process_mapping = {'creation_time': ('datetime', 'creation-time'),
'start_time': ('datetime', 'start-time'),
'name': ('text', 'name'),
'pid': ('text', 'pid'),
'parent_pid': ('text', 'parent-pid')}
_regkey_mapping = {'hive': ('text', 'hive'), 'key': ('regkey', 'key')}
_regkey_value_mapping = {'data': ('text', 'data'), 'datatype': ('text', 'data-type'), 'name': ('text', 'name')}
_socket_mapping = {'ip_address': ('ip-{}', 'address_value', 'ip-{}'),
'port': ('port', 'port_value', '{}-port'),
'hostname': ('hostname', 'hostname_value', 'hostname-{}')}
_whois_registrant_mapping = {'email_address': ('whois-registrant-email', 'address_value.value', 'registrant-email'),
'name': ('whois-registrant-name', 'value', 'registrant-name'),
'phone_number': ('whois-registrant-phone', 'value', 'registrant-phone'),
'organization': ('whois-registrant-org', 'value', 'registrant-org')}
_whois_mapping = {'registrar_info': ('whois-registrar', 'value', 'whois-registrar'),
'ip_address': ('ip-src', 'address_value.value', 'ip-address'),
'domain_name': ('domain', 'value.value', 'domain')}
_x509_datetime_types = ('not_before', 'not_after')
_x509_pubkey_types = ('exponent', 'modulus')
_x509_certificate_types = ('version', 'serial_number', 'issuer', 'subject')

@ -1 +1 @@
Subproject commit e597831a25c2a2f8bf08320420efee4dbb386389
Subproject commit 0ebe53940cb31fe674ea0caf35cfe81269b7b32a