MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'develop' into 2.4

pull/9082/head
iglocska 2023-05-16 15:19:12 +02:00
parent 305b456477 bb502323ea
commit a36398aa03
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
61 changed files with 5091 additions and 2791 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.github/workflows/main.yml vendored
View File

@ -229,12 +229,11 @@ jobs:
echo 'key = "'${AUTH}'"' >> tests/keys.py
cat tests/keys.py
popd
- name: Build test
run: |
. ./venv/bin/activate
pushd tests
./build-test.sh
bash ./build-test.sh
popd
deactivate
- name: Run PHP tests
run: |

2
PyMISP

@ -1 +1 @@
Subproject commit 6b28d8e606747cade1638f651ea531850027ddf4
Subproject commit 4ef26538eb88f3323eeb8773af89b480b7dfb2be

2
VERSION.json
View File

@ -1 +1 @@
{"major":2, "minor":4, "hotfix":170}
{"major":2, "minor":4, "hotfix":171}

45
app/Console/Command/Ls22Shell.php
View File

@ -428,7 +428,7 @@ class Ls22Shell extends AppShell
}
$HttpSocket = $this->Server->setupHttpSocket($server, null);
$request = $this->Server->setupSyncRequest($server);
$response = $HttpSocket->get($server['Server']['url'] . '/organisations/index/scope:all', false, $request);
$response = $HttpSocket->get($server['Server']['url'] . '/organisations/index/scope:local', false, $request);
$orgs = json_decode($response->body(), true);
$this->out(__('Organisations fetched. %d found.', count($orgs)), 1, Shell::VERBOSE);
$org_mapping = [];
@ -439,23 +439,31 @@ class Ls22Shell extends AppShell
if ($org['Organisation']['name'] === 'YT') {
continue;
}
if ($org['Organisation']['name'] === 'ORGNAME') {
continue;
}
$org_mapping[$org['Organisation']['name']] = $org['Organisation']['id'];
}
if (!empty($this->param['from'])) {
$time_range[] = $this->param['from'];
$time_range = [];
if (!empty($this->param('from'))) {
$time_range[] = $this->param('from');
}
if (!empty($this->param['to'])) {
if (!empty($this->param('to'))) {
if (empty($time_range)) {
$time_range[] = '365d';
}
$time_range[] = $this->param['to'];
$time_range[] = $this->param('to');
} else {
if (!empty($time_range)) {
$time_range[] = '0h';
}
}
$event_extended_uuids = [];
$event_uuid_per_org = [];
foreach ($org_mapping as $org_name => $org_id) {
$time_range = [];
$params = [
'org' => $org_id
'org' => $org_id,
'includeWarninglistHits' => true,
];
if (!empty($time_range)) {
$params['publish_timestamp'] = $time_range;
@ -480,9 +488,9 @@ class Ls22Shell extends AppShell
'extending_events' => 0,
];
foreach ($events['response'] as $event) {
$event_uuid_per_org[$event['Event']['uuid']] = $org_name;
$event_uuid_per_org[$event['Event']['uuid']] = $event['Event']['Orgc']['name'];
if (!empty($event['Event']['extends_uuid'])) {
$event_extended_uuids[$org_name] = $event['Event']['extends_uuid'];
$event_extended_uuids[$event['Event']['Orgc']['name']][] = $event['Event']['extends_uuid'];
}
if (!empty($event['Event']['Tag'])) {
@ -545,13 +553,17 @@ class Ls22Shell extends AppShell
}
}
foreach ($event_extended_uuids as $orgc => $uuid) {
$org_name = $event_uuid_per_org[$uuid];
if ($orgc != $org_name) {
// Add point for org extending another event
$results[$orgc]['extending_events'] += 1;
// Add point for org getting their event extended
$results[$org_name]['events_extended'] += 1;
foreach ($event_extended_uuids as $orgc => $uuids) {
foreach ($uuids as $uuid) {
if (!empty($event_uuid_per_org[$uuid])) {
$org_name = $event_uuid_per_org[$uuid];
if ($orgc != $org_name) {
// Add point for org extending another event
$results[$orgc]['extending_events'] += 1;
// Add point for org getting their event extended
$results[$org_name]['events_extended'] += 1;
}
}
}
}
@ -571,6 +583,7 @@ class Ls22Shell extends AppShell
$results[$k]['metrics']['attack_weight'] = 100 * (2*($result['attack']) + $result['attribute_attack']) / ($result['attribute_count'] + $result['object_count']);
$results[$k]['metrics']['other_weight'] = 100 * (2*($result['other']) + $result['attribute_other']) / ($result['attribute_count'] + $result['object_count']);
$results[$k]['metrics']['collaboration'] = 100 * ((2*$result['events_extended'] + $result['extending_events']) / $result['event_count']);
$results[$k]['metrics']['collaboration'] = 100 * (2*(2*$result['events_extended'] + $result['extending_events']) / $result['event_count']);
}
foreach (['connectedness', 'attack_weight', 'other_weight', 'warnings', 'collaboration'] as $metric) {
if (empty($results[$k]['metrics'][$metric])) {

22
app/Controller/AppController.php
View File

@ -33,8 +33,8 @@ class AppController extends Controller
public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
private $__queryVersion = '148';
public $pyMispVersion = '2.4.170';
private $__queryVersion = '150';
public $pyMispVersion = '2.4.171';
public $phpmin = '7.2';
public $phprec = '7.4';
public $phptoonew = '8.0';
@ -417,9 +417,12 @@ class AppController extends Controller
}
}
if ($foundMispAuthKey) {
$authKeyToStore = substr($authKey, 0, 4)
$start = substr($authKey, 0, 4);
$end = substr($authKey, -4);
$authKeyToStore = $start
. str_repeat('*', 32)
. substr($authKey, -4);
. $end;
$this->__logApiKeyUse($start . $end);
if ($user) {
// User found in the db, add the user info to the session
if (Configure::read('MISP.log_auth')) {
@ -642,6 +645,15 @@ class AppController extends Controller
return in_array($this->request->params['action'], $actionsToCheck[$controller], true);
}
private function __logApiKeyUse($apikey)
{
$redis = $this->User->setupRedis();
if (!$redis) {
return;
}
$redis->zIncrBy('misp:authkey_log:' . date("Ymd"), 1, $apikey);
}
/**
* User access monitoring
* @param array $user
@ -1491,7 +1503,7 @@ class AppController extends Controller
protected function __setPagingParams(int $page, int $limit, int $current, string $type = 'named')
{
$this->request->params['paging'] = [
'Correlation' => [
$this->modelClass => [
'page' => $page,
'limit' => $limit,
'current' => $current,

17
app/Controller/AttributesController.php
View File

@ -73,6 +73,9 @@ class AttributesController extends AppController
{
$user = $this->Auth->user();
$this->paginate['conditions']['AND'][] = $this->Attribute->buildConditions($user);
$this->__setIndexFilterConditions();
$attributes = $this->paginate();
if ($this->_isRest()) {
@ -3017,4 +3020,18 @@ class AttributesController extends AppController
$sg = $this->Attribute->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', true, $sharingGroupId);
return !empty($sg);
}
private function __setIndexFilterConditions()
{
// search by attribute value
if (isset($this->request->params['named']['searchvalue'])) {
$v = $this->request->params['named']['searchvalue'];
$this->paginate['conditions']['AND'][] = [
'OR' => [
['Attribute.value1' => $v],
['Attribute.value2' => $v],
]
];
}
}
}

11
app/Controller/DashboardsController.php
View File

@ -316,6 +316,8 @@ class DashboardsController extends AppController
public function listTemplates()
{
$conditions = array();
// load all widgets for internal use, won't be displayed to the user. Thus we circumvent the ACL on it.
$accessible_widgets = array_keys($this->Dashboard->loadAllWidgets($this->Auth->user()));
if (!$this->_isSiteAdmin()) {
$permission_flags = array();
foreach ($this->Auth->user('Role') as $perm => $value) {
@ -394,6 +396,15 @@ class DashboardsController extends AppController
}
$element['Dashboard']['widgets'] = array_keys($widgets);
sort($element['Dashboard']['widgets']);
$temp = [];
foreach ($element['Dashboard']['widgets'] as $widget) {
if (in_array($widget, $accessible_widgets)) {
$temp['allow'][] = $widget;
} else {
$temp['deny'][] = $widget;
}
}
$element['Dashboard']['widgets'] = $temp;
if ($element['Dashboard']['user_id'] != $this->Auth->user('id')) {
$element['User']['email'] = '';
}

86
app/Controller/EventsController.php
View File

@ -669,6 +669,25 @@ class EventsController extends AppController
[$eventReportQuery]
]
];
break;
case 'value':
if ($v == "") {
continue 2;
}
$conditions['OR'] = [
['Attribute.value1' => $v],
['Attribute.value2' => $v],
];
$eventIds = $this->Event->Attribute->fetchAttributes($this->Auth->user(), array(
'conditions' => $conditions,
'flatten' => true,
'event_ids' => true,
'list' => true,
));
$this->paginate['conditions']['AND'][] = array('Event.id' => $eventIds);
break;
default:
continue 2;
@ -682,7 +701,7 @@ class EventsController extends AppController
{
// list the events
$urlparams = "";
$overrideAbleParams = array('all', 'attribute', 'published', 'eventid', 'datefrom', 'dateuntil', 'org', 'eventinfo', 'tag', 'tags', 'distribution', 'sharinggroup', 'analysis', 'threatlevel', 'email', 'hasproposal', 'timestamp', 'publishtimestamp', 'publish_timestamp', 'minimal');
$overrideAbleParams = array('all', 'attribute', 'published', 'eventid', 'datefrom', 'dateuntil', 'org', 'eventinfo', 'tag', 'tags', 'distribution', 'sharinggroup', 'analysis', 'threatlevel', 'email', 'hasproposal', 'timestamp', 'publishtimestamp', 'publish_timestamp', 'minimal', 'value');
$paginationParams = array('limit', 'page', 'sort', 'direction', 'order');
$passedArgs = $this->passedArgs;
if (!empty($this->request->data)) {
@ -2374,20 +2393,54 @@ class EventsController extends AppController
$this->set('title_for_layout', __('Import from MISP Export File'));
}
public function upload_stix($stix_version = '1', $publish = false)
public function upload_stix($stix_version = '1', $publish = false, $galaxies_as_tags = true, $debug = false)
{
$sgs = $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1);
$initialDistribution = 0;
if (Configure::read('MISP.default_event_distribution') != null) {
$initialDistribution = Configure::read('MISP.default_event_distribution');
}
$distributionLevels = $this->Event->distributionLevels;
if ($this->request->is('post')) {
if ($this->_isRest()) {
if (isset($this->params['named']['publish'])) {
$publish = $this->params['named']['publish'];
}
if (isset($this->params['named']['distribution'])) {
$distribution = intval($this->params['named']['distribution']);
if (array_key_exists($distribution, $distributionLevels)) {
$initialDistribution = $distribution;
} else {
throw new MethodNotAllowedException(__('Wrong distribution level'));
}
}
$sharingGroupId = null;
if ($initialDistribution == 4) {
if (!isset($this->params['named']['sharing_group_id'])) {
throw new MethodNotAllowedException(__('The sharing group id is needed when the distribution is set to 4 ("Sharing group").'));
}
$sharingGroupId = intval($this->params['named']['sharing_group_id']);
if (!array_key_exists($sharingGroupId, $sgs)) {
throw new MethodNotAllowedException(__('Please select a valid sharing group id.'));
}
}
if (isset($this->params['named']['galaxies_as_tags'])) {
$galaxies_as_tags = $this->params['named']['galaxies_as_tags'];
}
if (isset($this->params['named']['debugging'])) {
$debug = $this->params['named']['debugging'];
}
$filePath = FileAccessTool::writeToTempFile($this->request->input());
$result = $this->Event->upload_stix(
$this->Auth->user(),
$filePath,
$stix_version,
'uploaded_stix_file.' . ($stix_version == '1' ? 'xml' : 'json'),
$publish
$publish,
$initialDistribution,
$sharingGroupId,
$galaxies_as_tags,
$debug
);
if (is_numeric($result)) {
$event = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $result));
@ -2406,12 +2459,19 @@ class EventsController extends AppController
if (!move_uploaded_file($this->data['Event']['stix']['tmp_name'], $filePath)) {
throw new Exception("Could not move uploaded STIX file.");
}
if (isset($this->data['Event']['debug'])) {
$debug = $this->data['Event']['debug'];
}
$result = $this->Event->upload_stix(
$this->Auth->user(),
$filePath,
$stix_version,
$original_file,
$this->data['Event']['publish']
$this->data['Event']['publish'],
$this->data['Event']['distribution'],
$this->data['Event']['sharing_group_id'],
!boolval($this->data['Event']['galaxies_parsing']),
$debug
);
if (is_numeric($result)) {
$this->Flash->success(__('STIX document imported.'));
@ -2429,6 +2489,20 @@ class EventsController extends AppController
}
}
$this->set('stix_version', $stix_version == 2 ? '2.x JSON' : '1.x XML');
$this->set('initialDistribution', $initialDistribution);
$distributions = array_keys($this->Event->distributionDescriptions);
$distributions = $this->_arrayToValuesIndexArray($distributions);
$this->set('distributions', $distributions);
$fieldDesc = array();
if (empty($sgs)) {
unset($distributionLevels[4]);
}
$this->set('distributionLevels', $distributionLevels);
foreach ($distributionLevels as $key => $value) {
$fieldDesc['distribution'][$key] = $this->Event->distributionDescriptions[$key]['formdesc'];
}
$this->set('sharingGroups', $sgs);
$this->set('fieldDesc', $fieldDesc);
}
public function merge($target_id=null, $source_id=null)
@ -4438,12 +4512,12 @@ class EventsController extends AppController
),
'STIX' => array(
'url' => $this->baseurl . '/events/upload_stix',
'text' => __('STIX 1.1.1 format (lossy)'),
'text' => __('STIX 1.x format (lossy)'),
'ajax' => false,
),
'STIX2' => array(
'url' => $this->baseurl . '/events/upload_stix/2',
'text' => __('STIX 2.0 format (lossy)'),
'text' => __('STIX 2.x format (lossy)'),
'ajax' => false,
)
);

2
app/Controller/LogsController.php
View File

@ -377,7 +377,7 @@ class LogsController extends AppController
$this->set('list', $list);
// set the same view as the index page
$this->render('admin_index');
$this->render('index');
}
} else {
// no search keyword is given, show the search form

10
app/Controller/ServersController.php
View File

@ -57,6 +57,14 @@ class ServersController extends AppController
unset($fields['authkey']);
$fields = array_keys($fields);
$filters = $this->IndexFilter->harvestParameters(['search']);
$conditions = [];
if (!empty($filters['search'])) {
$strSearch = '%' . trim(strtolower($filters['search'])) . '%';
$conditions['OR'][]['LOWER(Server.name) LIKE'] = $strSearch;
$conditions['OR'][]['LOWER(Server.url) LIKE'] = $strSearch;
}
if ($this->_isRest()) {
$params = array(
'fields' => $fields,
@ -72,12 +80,14 @@ class ServersController extends AppController
'fields' => array('RemoteOrg.id', 'RemoteOrg.name', 'RemoteOrg.uuid', 'RemoteOrg.nationality', 'RemoteOrg.sector', 'RemoteOrg.type'),
),
),
'conditions' => $conditions,
);
$servers = $this->Server->find('all', $params);
$servers = $this->Server->attachServerCacheTimestamps($servers);
return $this->RestResponse->viewData($servers, $this->response->type());
} else {
$this->paginate['fields'] = $fields;
$this->paginate['conditions'] = $conditions;
$servers = $this->paginate();
$servers = $this->Server->attachServerCacheTimestamps($servers);
$this->set('servers', $servers);

5
app/Controller/WorkflowsController.php
View File

@ -67,6 +67,7 @@ class WorkflowsController extends AppController
} else {
$successMessage = __('Workflow saved.');
$savedWorkflow = $result['saved'];
$savedWorkflow = $this->Workflow->attachLabelToConnections($savedWorkflow);
return $this->__getSuccessResponseBasedOnContext($successMessage, $savedWorkflow, 'edit', false, $redirectTarget);
}
} else {
@ -101,6 +102,9 @@ class WorkflowsController extends AppController
}
}
$this->CRUD->view($id, [
'afterFind' => function($workflow) {
return $this->Workflow->attachLabelToConnections($workflow);
}
]);
if ($this->IndexFilter->isRest()) {
return $this->restResponsePayload;
@ -151,6 +155,7 @@ class WorkflowsController extends AppController
} else {
$workflow = $this->Workflow->fetchWorkflow($workflow_id);
}
$workflow = $this->Workflow->attachLabelToConnections($workflow, $trigger_id);
$modules = $this->Workflow->attachNotificationToModules($modules, $workflow);
$this->loadModel('WorkflowBlueprint');
$workflowBlueprints = $this->WorkflowBlueprint->find('all');

131
app/Lib/Dashboard/APIActivityWidget.php Normal file
View File

@ -0,0 +1,131 @@
<?php
class APIActivityWidget
{
public $title = 'API Activity';
public $render = 'SimpleList';
public $width = 2;
public $height = 2;
public $params = [
'filter' => 'A list of filters by organisation meta information (sector, type, nationality, id, uuid) to include. (dictionary, prepending values with ! uses them as a negation)',
'limit' => 'Limits the number of displayed APIkeys. (-1 will list all) Default: -1',
'days' => 'How many days back should the list go - for example, setting 7 will only show contributions in the past 7 days. (integer)',
'month' => 'Who contributed most this month? (boolean)',
'year' => 'Which contributed most this year? (boolean)',
];
public $description = 'Basic widget showing some server statistics in regards to MISP.';
public $cacheLifetime = 10;
public $autoRefreshDelay = null;
private $User = null;
private $AuthKey = null;
private function getDates($options)
{
if (!empty($options['days'])) {
$begin = new DateTime(date('Y-m-d', strtotime(sprintf("-%s days", $options['days']))));
} else if (!empty($options['month'])) {
$begin = new DateTime(date('Y-m-d', strtotime('first day of this month 00:00:00', time())));
} else if (!empty($options['year'])) {
$begin = new DateTime(date('Y-m-d', strtotime('first day of this year 00:00:00', time())));
} else {
$begin = new DateTime(date('Y-m-d', strtotime('-7 days', time())));;
}
$now = new DateTime();
$dates = new DatePeriod(
$begin,
new DateInterval('P1D'),
$now
);
$results = [];
foreach ($dates as $date) {
$results[] = $date->format('Ymd');
}
return $results;
}
public function handler($user, $options = array())
{
$this->User = ClassRegistry::init('User');
$this->AuthKey = ClassRegistry::init('AuthKey');
$redis = $this->User->setupRedis();
if (!$redis) {
throw new NotFoundException(__('No redis connection found.'));
}
$params = ['conditions' => []];
$dates = $this->getDates($options);
$pipe = $redis->pipeline();
foreach ($dates as $date) {
$pipe->zrange('misp:authkey_log:' . $date, 0, -1, true);
}
$temp = $pipe->exec();
$raw_results = [];
$counts = [];
foreach ($dates as $k => $date) {
$raw_results[$date] = $temp[$k];
if (!empty($temp[$k])) {
foreach ($temp[$k] as $key => $count) {
if (isset($counts[$key])) {
$counts[$key] += (int)$count;
} else {
$counts[$key] = (int)$count;
}
}
}
}
arsort($counts);
$this->AuthKey->Behaviors->load('Containable');
$temp_apikeys = array_flip(array_keys($counts));
foreach ($temp_apikeys as $apikey => $value) {
$temp_apikeys[$apikey] = $this->AuthKey->find('first', [
'conditions' => [
'AuthKey.authkey_start' => substr($apikey, 0, 4),
'AuthKey.authkey_end' => substr($apikey, 4)
],
'fields' => ['AuthKey.authkey_start', 'AuthKey.authkey_end', 'AuthKey.id', 'User.id', 'User.email'],
'recursive' => 1
]);
}
$baseurl = empty(Configure::read('MISP.external_baseurl')) ? h(Configure::read('MISP.baseurl')) : Configure::read('MISP.external_baseurl');
foreach ($counts as $key => $junk) {
$data = $temp_apikeys[$key];
if (!empty($data)) {
$results[] = [
'html_title' => sprintf(
'<a href="%s/auth_keys/view/%s">%s</a>',
h($baseurl),
h($data['AuthKey']['id']),
$key
),
'html' => sprintf(
'%s (<a href="%s/admin/users/view/%s">%s</a>)',
h($counts[$key]),
h($baseurl),
h($data['User']['id']),
h($data['User']['email'])
)
];
} else {
$results[] = [
'title' => $key,
'html' => sprintf(
'%s (<span class="red" title="%s">%s</span>)',
h($counts[$key]),
__('An unknown key can be caused by the given key having been permanently deleted or falsely mis-identified (for the purposes of this widget) on instances using legacy API key authentication.'),
__('Unknown key')
)
];
}
}
return $results;
}
public function checkPermissions($user)
{
if (empty($user['Role']['perm_site_admin'])) {
return false;
}
return true;
}
}

88
app/Lib/Dashboard/LoginsWidget.php Normal file
View File

@ -0,0 +1,88 @@
<?php
class LoginsWidget
{
public $title = 'Logins';
public $render = 'SimpleList';
public $width = 2;
public $height = 2;
public $params = [
'filter' => 'A list of filters by organisation meta information (sector, type, nationality, id, uuid) to include. (dictionary, prepending values with ! uses them as a negation)',
'limit' => 'Limits the number of displayed APIkeys. (-1 will list all) Default: -1',
'days' => 'How many days back should the list go - for example, setting 7 will only show contributions in the past 7 days. (integer)',
'month' => 'Who contributed most this month? (boolean)',
'year' => 'Which contributed most this year? (boolean)',
];
public $description = 'Basic widget showing some server statistics in regards to MISP.';
public $cacheLifetime = 10;
public $autoRefreshDelay = null;
private $User = null;
private $Log = null;
private function getDates($options)
{
if (!empty($options['days'])) {
$begin = date('Y-m-d H:i:s', strtotime(sprintf("-%s days", $options['days'])));
} else if (!empty($options['month'])) {
$begin = date('Y-m-d H:i:s', strtotime('first day of this month 00:00:00', time()));
} else if (!empty($options['year'])) {
$begin = date('Y-m-d', strtotime('first day of this year 00:00:00', time()));
} else {
$begin = date('Y-m-d H:i:s', strtotime('-7 days', time()));
}
return $begin ? ['Log.created >=' => $begin] : [];
}
public function handler($user, $options = array())
{
$this->User = ClassRegistry::init('User');
$this->Log = ClassRegistry::init('Log');
$conditions = $this->getDates($options);
$conditions['Log.action'] = 'login';
$this->Log->Behaviors->load('Containable');
$this->Log->bindModel([
'belongsTo' => [
'User'
]
]);
$this->Log->virtualFields['count'] = 0;
$this->Log->virtualFields['email'] = '';
$logs = $this->Log->find('all', [
'recursive' => -1,
'conditions' => $conditions,
'fields' => ['Log.user_id', 'COUNT(Log.id) AS Log__count', 'User.email AS Log__email'],
'contain' => ['User'],
'group' => ['Log.user_id']
]);
$counts = [];
$emails = [];
foreach ($logs as $log) {
$counts[$log['Log']['user_id']] = $log['Log']['count'];
$emails[$log['Log']['user_id']] = $log['Log']['email'];
}
$results = [];
arsort($counts);
$baseurl = empty(Configure::read('MISP.external_baseurl')) ? h(Configure::read('MISP.baseurl')) : Configure::read('MISP.external_baseurl');
foreach ($counts as $user_id => $count) {
$results[] = [
'html_title' => sprintf(
'<a href="%s/admin/users/view/%s">%s</a>',
h($baseurl),
h($user_id),
h($emails[$user_id])
),
'value' => $count
];
}
return $results;
}
public function checkPermissions($user)
{
if (empty($user['Role']['perm_site_admin'])) {
return false;
}
return true;
}
}

157
app/Lib/Dashboard/NewOrgsWidget.php Normal file
View File

@ -0,0 +1,157 @@
<?php
class NewOrgsWidget
{
public $title = 'New organisations';
public $render = 'Index';
public $width = 7;
public $height = 6;
public $description = 'A list of the latest new member organisations.';
private $tableDescription = null;
public $cacheLifetime = null;
public $autoRefreshDelay = false;
public $params = [
'limit' => 'Maximum number of joining organisations shown. (integer, defaults to 10 if not set)',
'filter' => 'A list of filters by organisation meta information (nationality, sector, type, name, uuid) to include. (dictionary, prepending values with ! uses them as a negation)',
'days' => 'How many days back should the list go - for example, setting 7 will only show the organisations that were added in the past 7 days. (integer)',
'month' => 'Which organisations have been added this month? (boolean)',
'year' => 'Which organisations have been added this year? (boolean)',
'local' => 'Should the list only show local organisations? (boolean or list of booleans, defaults to 1. To get both sets, use [0,1])',
'fields' => 'Which fields should be displayed, by default all are selected. Pass a list with the following options: [id, uuid, name, sector, type, nationality, creation_date]'
];
private $validFilterKeys = [
'nationality',
'sector',
'type',
'name',
'uuid'
];
public $placeholder =
'{
"limit": 5,
"filter": {
"nationality": [
"Hungary",
"Russia",
"North Korea"
]
},
"month": true
}';
private $Organisation = null;
private function timeConditions($options)
{
$limit = empty($options['limit']) ? 10 : $options['limit'];
if (!empty($options['days'])) {
$condition = strtotime(sprintf("-%s days", $options['days']));
$this->tableDescription = __('The %d newest organisations created in the past %d days', $limit, (int)$options['days']);
} else if (!empty($options['month'])) {
$condition = strtotime('first day of this month 00:00:00', time());
$this->tableDescription = __('The %d newest organisations created during the current month', $limit);
} else if (!empty($options['year'])) {
$condition = strtotime('first day of this year 00:00:00', time());
$this->tableDescription = __('The %d newest organisations created during the current year', $limit);
} else {
$this->tableDescription = __('The %d newest organisations created', $limit);
return null;
}
$datetime = new DateTime();
$datetime->setTimestamp($condition);
return $datetime->format('Y-m-d H:i:s');
}
public function handler($user, $options = array())
{
$this->Organisation = ClassRegistry::init('Organisation');
$field_options = [
'id' => [
'name' => '#',
'url' => Configure::read('MISP.baseurl') . '/organisations/view',
'element' => 'links',
'data_path' => 'Organisation.id',
'url_params_data_paths' => 'Organisation.id'
],
'date_created' => [
'name' => 'Creation date',
'data_path' => 'Organisation.date_created'
],
'name' => [
'name' => 'Name',
'data_path' => 'Organisation.name',
],
'uuid' => [
'name' => 'UUID',
'data_path' => 'Organisation.uuid',
],
'sector' => [
'name' => 'Sector',
'data_path' => 'Organisation.sector',
],
'nationality' => [
'name' => 'Nationality',
'data_path' => 'Organisation.nationality',
],
'type' => [
'name' => 'Type',
'data_path' => 'Organisation.type',
]
];
$params = [
'conditions' => [
'AND' => ['Organisation.local' => !isset($options['local']) ? 1 : $options['local']]
],
'limit' => 10,
'recursive' => -1
];
if (!empty($options['filter']) && is_array($options['filter'])) {
foreach ($this->validFilterKeys as $filterKey) {
if (!empty($options['filter'][$filterKey])) {
if (!is_array($options['filter'][$filterKey])) {
$options['filter'][$filterKey] = [$options['filter'][$filterKey]];
}
$tempConditionBucket = [];
foreach ($options['filter'][$filterKey] as $value) {
if ($value[0] === '!') {
$tempConditionBucket['Organisation.' . $filterKey . ' NOT IN'][] = mb_substr($value, 1);
} else {
$tempConditionBucket['Organisation.' . $filterKey . ' IN'][] = $value;
}
}
if (!empty($tempConditionBucket)) {
$params['conditions']['AND'][] = $tempConditionBucket;
}
}
}
}
$timeConditions = $this->timeConditions($options);
if ($timeConditions) {
$params['conditions']['AND'][] = ['Organisation.date_created >=' => $timeConditions];
}
if (isset($options['fields'])) {
$fields = [];
foreach ($options['fields'] as $field) {
if (isset($field_options[$field])) {
$fields[$field] = $field_options[$field];
}
}
} else {
$fields = $field_options;
}
$data = $this->Organisation->find('all', [
'recursive' => -1,
'conditions' => $params['conditions'],
'limit' => isset($options['limit']) ? (int)$options['limit'] : 10,
'fields' => array_keys($fields),
'order' => 'Organisation.date_created DESC'
]);
return [
'data' => $data,
'fields' => $fields,
'description' => $this->tableDescription
];
}
}

171
app/Lib/Dashboard/NewUsersWidget.php Normal file
View File

@ -0,0 +1,171 @@
<?php
class NewUsersWidget
{
public $title = 'New users';
public $render = 'Index';
public $width = 7;
public $height = 6;
public $description = 'A list of the latest new users.';
private $tableDescription = null;
public $cacheLifetime = null;
public $autoRefreshDelay = false;
public $params = [
'limit' => 'Maximum number of joining users shown. (integer, defaults to 10 if not set)',
'filter' => 'A list of filters for the organisations (nationality, sector, type, name, uuid) to include. (dictionary, prepending values with ! uses them as a negation)',
'days' => 'How many days back should the list go - for example, setting 7 will only show the organisations that were added in the past 7 days. (integer)',
'month' => 'Which organisations have been added this month? (boolean)',
'year' => 'Which organisations have been added this year? (boolean)',
'fields' => 'Which fields should be displayed, by default all are selected. Pass a list with the following options: [id, email, Organisation.name, Role.name, date_created]'
];
private $validFilterKeys = [
'id',
'email',
'Organisation.name',
'Role.name',
'date_created'
];
public $placeholder =
'{
"limit": 10,
"filter": {
"Organisation.name": [
"!FSB",
"!GRU",
"!Kaspersky"
],
"email": [
"!andras.iklody@circl.lu"
],
"Role.name": [
"Publisher",
"User"
]
},
"year": true
}';
private $User = null;
private function timeConditions($options)
{
$limit = empty($options['limit']) ? 10 : $options['limit'];
if (!empty($options['days'])) {
$condition = strtotime(sprintf("-%s days", $options['days']));
$this->tableDescription = __('The %d newest users created in the past %d days', $limit, (int)$options['days']);
} else if (!empty($options['month'])) {
$condition = strtotime('first day of this month 00:00:00', time());
$this->tableDescription = __('The %d newest users created during the current month', $limit);
} else if (!empty($options['year'])) {
$condition = strtotime('first day of this year 00:00:00', time());
$this->tableDescription = __('The %d newest users created during the current year', $limit);
} else {
$this->tableDescription = __('The %d newest users created', $limit);
return null;
}
return $condition;
}
public function handler($user, $options = array())
{
$this->User = ClassRegistry::init('User');
$field_options = [
'id' => [
'name' => '#',
'url' => empty($user['Role']['perm_site_admin']) ? null : Configure::read('MISP.baseurl') . '/admin/users/view',
'element' => 'links',
'data_path' => 'User.id',
'url_params_data_paths' => 'User.id'
],
'date_created' => [
'name' => 'Creation date',
'data_path' => 'User.date_created'
],
'email' => [
'name' => 'E-mail',
'data_path' => 'User.email',
],
'Organisation.name' => [
'name' => 'Organisation',
'data_path' => 'Organisation.name',
],
'Role.name' => [
'name' => 'Role',
'data_path' => 'Role.name',
]
];
$params = [
'conditions' => [],
'limit' => 10,
'recursive' => -1
];
if (!empty($options['filter']) && is_array($options['filter'])) {
foreach ($this->validFilterKeys as $filterKey) {
if (!empty($options['filter'][$filterKey])) {
if (!is_array($options['filter'][$filterKey])) {
$options['filter'][$filterKey] = [$options['filter'][$filterKey]];
}
$tempConditionBucket = [];
foreach ($options['filter'][$filterKey] as $value) {
$filterName = strpos($filterKey, '.') ? $filterKey : 'User.' . $filterKey;
if ($value[0] === '!') {
$tempConditionBucket[$filterName . ' NOT IN'][] = mb_substr($value, 1);
} else {
$tempConditionBucket[$filterName . ' IN'][] = $value;
}
}
if (!empty($tempConditionBucket)) {
$params['conditions']['AND'][] = $tempConditionBucket;
}
}
}
}
$timeConditions = $this->timeConditions($options);
if ($timeConditions) {
$params['conditions']['AND'][] = ['User.date_created >=' => $timeConditions];
}
if (isset($options['fields'])) {
$fields = [];
foreach ($options['fields'] as $field) {
if (isset($field_options[$field])) {
$fields[$field] = $field_options[$field];
}
}
} else {
$fields = $field_options;
}
// redact e-mails for non site admins unless specifically allowed
if (
empty($user['Role']['perm_site_admin']) &&
!Configure::read('Security.disclose_user_emails') &&
isset($fields['email'])
) {
unset($fields['email']);
}
$data = $this->User->find('all', [
'recursive' => -1,
'contain' => ['Organisation.name', 'Role.name'],
'conditions' => $params['conditions'],
'limit' => isset($options['limit']) ? $options['limit'] : 10,
'fields' => array_keys($fields),
'order' => 'User.date_created DESC'
]);
foreach ($data as &$u) {
if (empty($u['User']['date_created'])) {
continue;
}
$tempDate = new DateTime();
$tempDate->setTimestamp($u['User']['date_created']);
$u['User']['date_created'] = $tempDate->format('Y-m-d H:i:s');
}
return [
'data' => $data,
'fields' => $fields,
'description' => $this->tableDescription
];
}
}

107
app/Lib/Dashboard/OrgContributionToplistWidget.php Normal file
View File

@ -0,0 +1,107 @@
<?php
class OrgContributionToplistWidget
{
public $title = 'Contributor Top List (Orgs)';
public $render = 'BarChart';
public $description = 'The top contributors (orgs) in a selected time frame.';
public $width = 3;
public $height = 4;
public $params = [
'days' => 'How many days back should the list go - for example, setting 7 will only show contributions in the past 7 days. (integer)',
'month' => 'Who contributed most this month? (boolean)',
'year' => 'Which contributed most this year? (boolean)',
'filter' => 'A list of filters by organisation meta information (nationality, sector, type, name, uuid, local (- expects a boolean or a list of boolean values)) to include. (dictionary, prepending values with ! uses them as a negation)',
'limit' => 'Limits the number of displayed tags. Default: 10'
];
public $cacheLifetime = null;
public $autoRefreshDelay = false;
private $validFilterKeys = [
'nationality',
'sector',
'type',
'name',
'uuid'
];
public $placeholder =
'{
"days": "7d",
"threshold": 15,
"filter": {
"sector": "Financial"
}
}';
private $Org = null;
private $Event = null;
private function timeConditions($options)
{
$limit = empty($options['limit']) ? 10 : $options['limit'];
if (!empty($options['days'])) {
$condition = strtotime(sprintf("-%s days", $options['days']));
} else if (!empty($options['month'])) {
$condition = strtotime('first day of this month 00:00:00', time());
} else if (!empty($options['year'])) {
$condition = strtotime('first day of this year 00:00:00', time());
} else {
return null;
}
return $condition;
}
public function handler($user, $options = array())
{
$params = ['conditions' => []];
$timeConditions = $this->timeConditions($options);
if ($timeConditions) {
$params['conditions']['AND'][] = ['Event.timestamp >=' => $timeConditions];
}
if (!empty($options['filter']) && is_array($options['filter'])) {
foreach ($this->validFilterKeys as $filterKey) {
if (!empty($options['filter'][$filterKey])) {
if (!is_array($options['filter'][$filterKey])) {
$options['filter'][$filterKey] = [$options['filter'][$filterKey]];
}
$tempConditionBucket = [];
foreach ($options['filter'][$filterKey] as $value) {
if ($value[0] === '!') {
$tempConditionBucket['Organisation.' . $filterKey . ' NOT IN'][] = mb_substr($value, 1);
} else {
$tempConditionBucket['Organisation.' . $filterKey . ' IN'][] = $value;
}
}
if (!empty($tempConditionBucket)) {
$params['conditions']['AND'][] = $tempConditionBucket;
}
}
}
}
if (isset($options['filter']['local'])) {
$params['conditions']['AND']['local'] = $options['filter']['local'];
}
$this->Org = ClassRegistry::init('Organisation');
$org_ids = $this->Org->find('list', [
'fields' => ['Organisation.id', 'Organisation.name'],
'conditions' => $params['conditions']
]);
$conditions = ['Event.orgc_id IN' => array_keys($org_ids)];
$this->Event = ClassRegistry::init('Event');
$this->Event->virtualFields['frequency'] = 0;
$orgs = $this->Event->find('all', [
'recursive' => -1,
'fields' => ['orgc_id', 'count(Event.orgc_id) as Event__frequency'],
'group' => ['orgc_id'],
'conditions' => $conditions,
'order' => 'count(Event.orgc_id) desc',
'limit' => empty($options['limit']) ? 10 : $options['limit']
]);
$results = [];
foreach($orgs as $org) {
$results[$org_ids[$org['Event']['orgc_id']]] = $org['Event']['frequency'];
}
return ['data' => $results];
}
}
?>

250
app/Lib/Dashboard/OrganisationMapWidget.php Normal file
View File

@ -0,0 +1,250 @@
<?php
class OrganisationMapWidget
{
public $title = 'Organisation world map';
public $render = 'WorldMap';
public $description = 'The countries represented via organisations on the current instance.';
public $width = 3;
public $height = 4;
public $params = [
'filter' => 'A list of filters by organisation meta information (sector, type, local (- expects a boolean or a list of boolean values)) to include. (dictionary, prepending values with ! uses them as a negation)',
'limit' => 'Limits the number of displayed tags. Default: 10'
];
public $cacheLifetime = null;
public $autoRefreshDelay = false;
private $validFilterKeys = [
'sector',
'type',
'local'
];
public $placeholder =
'{
"type": "Member",
"local": [0,1]
}';
private $Organisation = null;
public $countryCodes = array(
'Afghanistan' => 'AF',
'Albania' => 'AL',
'Algeria' => 'DZ',
'Angola' => 'AO',
'Argentina' => 'AR',
'Armenia' => 'AM',
'Australia' => 'AU',
'Austria' => 'AT',
'Azerbaijan' => 'AZ',
'Bahamas' => 'BS',
'Bangladesh' => 'BD',
'Belarus' => 'BY',
'Belgium' => 'BE',
'Belize' => 'BZ',
'Benin' => 'BJ',
'Bhutan' => 'BT',
'Bolivia' => 'BO',
'Bosnia and Herz.' => 'BA',
'Botswana' => 'BW',
'Brazil' => 'BR',
'Brunei' => 'BN',
'Bulgaria' => 'BG',
'Burkina Faso' => 'BF',
'Burundi' => 'BI',
'Cambodia' => 'KH',
'Cameroon' => 'CM',
'Canada' => 'CA',
'Central African Rep.' => 'CF',
'Chad' => 'TD',
'Chile' => 'CL',
'China' => 'CN',
'Colombia' => 'CO',
'Congo' => 'CG',
'Costa Rica' => 'CR',
'Croatia' => 'HR',
'Cuba' => 'CU',
'Cyprus' => 'CY',
'Czech Rep.' => 'CZ',
'Côte d\'Ivoire' => 'CI',
'Dem. Rep. Congo' => 'CD',
'Dem. Rep. Korea' => 'KP',
'Denmark' => 'DK',
'Djibouti' => 'DJ',
'Dominican Rep.' => 'DO',
'Ecuador' => 'EC',
'Egypt' => 'EG',
'El Salvador' => 'SV',
'Eq. Guinea' => 'GQ',
'Eritrea' => 'ER',
'Estonia' => 'EE',
'Ethiopia' => 'ET',
'Falkland Is.' => 'FK',
'Fiji' => 'FJ',
'Finland' => 'FI',
'Fr. S. Antarctic Lands' => 'TF',
'France' => 'FR',
'Gabon' => 'GA',
'Gambia' => 'GM',
'Georgia' => 'GE',
'Germany' => 'DE',
'Ghana' => 'GH',
'Greece' => 'GR',
'Greenland' => 'GL',
'Guatemala' => 'GT',
'Guinea' => 'GN',
'Guinea-Bissau' => 'GW',
'Guyana' => 'GY',
'Haiti' => 'HT',
'Honduras' => 'HN',
'Hungary' => 'HU',
'Iceland' => 'IS',
'India' => 'IN',
'Indonesia' => 'ID',
'Iran' => 'IR',
'Iraq' => 'IQ',
'Ireland' => 'IE',
'Israel' => 'IL',
'Italy' => 'IT',
'Jamaica' => 'JM',
'Japan' => 'JP',
'Jordan' => 'JO',
'Kazakhstan' => 'KZ',
'Kenya' => 'KE',
'Korea' => 'KR',
'Kuwait' => 'KW',
'Kyrgyzstan' => 'KG',
'Lao PDR' => 'LA',
'Latvia' => 'LV',
'Lebanon' => 'LB',
'Lesotho' => 'LS',
'Liberia' => 'LR',
'Libya' => 'LY',
'Lithuania' => 'LT',
'Luxembourg' => 'LU',
'Macedonia' => 'MK',
'Madagascar' => 'MG',
'Mainland China' => 'CN',
'Malawi' => 'MW',
'Malaysia' => 'MY',
'Mali' => 'ML',
'Mauritania' => 'MR',
'Mexico' => 'MX',
'Moldova' => 'MD',
'Mongolia' => 'MN',
'Montenegro' => 'ME',
'Morocco' => 'MA',
'Mozamb' => 'MZ',
'Myanmar' => 'MM',
'Namibia' => 'NA',
'Nepal' => 'NP',
'Netherlands' => 'NL',
'New Caledonia' => 'NC',
'New Zealand' => 'NZ',
'Nicaragua' => 'NI',
'Niger' => 'NE',
'Nigeria' => 'NG',
'Norway' => 'NO',
'Oman' => 'OM',
'Pakistan' => 'PK',
'Palestine' => 'PS',
'Panama' => 'PA',
'Papua New Guinea' => 'PG',
'Paraguay' => 'PY',
'Peru' => 'PE',
'Philippines' => 'PH',
'Poland' => 'PL',
'Portugal' => 'PT',
'Puerto Rico' => 'PR',
'Qatar' => 'QA',
'Romania' => 'RO',
'Russia' => 'RU',
'Rwanda' => 'RW',
'S. Sudan' => 'SS',
'Saudi Arabia' => 'SA',
'Senegal' => 'SN',
'Serbia' => 'RS',
'Sierra Leone' => 'SL',
'Slovakia' => 'SK',
'Slovenia' => 'SI',
'Solomon Is.' => 'SB',
'Somalia' => 'SO',
'South Africa' => 'ZA',
'Spain' => 'ES',
'Sri Lanka' => 'LK',
'Sudan' => 'SD',
'Suriname' => 'SR',
'Swaziland' => 'SZ',
'Sweden' => 'SE',
'Switzerland' => 'CH',
'Syria' => 'SY',
'Taiwan' => 'TW',
'Tajikistan' => 'TJ',
'Tanzania' => 'TZ',
'Thailand' => 'TH',
'Timor-Leste' => 'TL',
'Togo' => 'TG',
'Trinidad and Tobago' => 'TT',
'Tunisia' => 'TN',
'Turkey' => 'TR',
'Turkmenistan' => 'TM',
'Uganda' => 'UG',
'Ukraine' => 'UA',
'United Arab Emirates' => 'AE',
'United Kingdom' => 'GB',
'United States' => 'US',
'Uruguay' => 'UY',
'Uzbekistan' => 'UZ',
'Vanuatu' => 'VU',
'Venezuela' => 'VE',
'Vietnam' => 'VN',
'W. Sahara' => 'EH',
'Yemen' => 'YE',
'Zambia' => 'ZM',
'Zimbabwe' => 'ZW'
);
public function handler($user, $options = array())
{
$params = [
'conditions' => [
'Nationality !=' => ''
]
];
if (!empty($options['filter']) && is_array($options['filter'])) {
foreach ($this->validFilterKeys as $filterKey) {
if (!empty($options['filter'][$filterKey])) {
if (!is_array($options['filter'][$filterKey])) {
$options['filter'][$filterKey] = [$options['filter'][$filterKey]];
}
$tempConditionBucket = [];
foreach ($options['filter'][$filterKey] as $value) {
if ($value[0] === '!') {
$tempConditionBucket['Organisation.' . $filterKey . ' NOT IN'][] = mb_substr($value, 1);
} else {
$tempConditionBucket['Organisation.' . $filterKey . ' IN'][] = $value;
}
}
if (!empty($tempConditionBucket)) {
$params['conditions']['AND'][] = $tempConditionBucket;
}
}
}
}
$this->Organisation = ClassRegistry::init('Organisation');
$orgs = $this->Organisation->find('all', [
'recursive' => -1,
'fields' => ['Organisation.nationality', 'COUNT(Organisation.nationality) AS frequency'],
'conditions' => $params['conditions'],
'group' => ['Organisation.nationality']
]);
$results = ['data' => [], 'scope' => 'Organisations'];
foreach($orgs as $org) {
$country = $org['Organisation']['nationality'];
$count = $org['0']['frequency'];
if (isset($this->countryCodes[$country])) {
$countryCode = $this->countryCodes[$country];
$results['data'][$countryCode] = $count;
}
}
return $results;
}
}
?>

136
app/Lib/Dashboard/TrendingAttributesWidget.php Normal file
View File

@ -0,0 +1,136 @@
<?php
class TrendingAttributesWidget
{
public $title = 'Trending Attribute values';
public $render = 'BarChart';
public $width = 3;
public $height = 4;
public $params = array(
'time_window' => 'The time window, going back in seconds, that should be included. (allows for filtering by days - example: 5d. -1 Will fetch all historic data)',
'exclude' => 'List of values to exclude - for example "8.8.8.8".',
'threshold' => 'Limits the number of displayed attribute values. Default: 10',
'type' => 'List of Attribute types to include',
'category' => 'List of Attribute categories to exclude',
'to_ids' => 'A list of to_ids settings accepted for the data displayed ([0], [1], [0,1])',
'org_filter' => 'List of organisation filters to exclude events by, based on organisation meta-data (Organisation.sector, Organisation.type, Organisation.nationality). Pre-pending a value with a "!" negates it.'
);
private $validOrgFilters = [
'sector',
'type',
'national',
'uuid',
'local'
];
public $placeholder =
'{
"time_window": "7d",
"threshold": 15,
"org_filter": {
"sector": ["Financial"]
}
}';
public $description = 'Widget showing the trending tags over the past x seconds, along with the possibility to include/exclude tags.';
public $cacheLifetime = 3;
private function getOrgList($options)
{
$organisationModel = ClassRegistry::init('Organisation');
if (!empty($options['org_filter']) && is_array($options['org_filter'])) {
foreach ($this->validOrgFilters as $filterKey) {
if (!empty($options['org_filter'][$filterKey])) {
if ($filterKey === 'local') {
$tempConditionBucket['Organisation.local'] = $options['org_filter']['local'];
} else {
if (!is_array($options['org_filter'][$filterKey])) {
$options['org_filter'][$filterKey] = [$options['org_filter'][$filterKey]];
}
$tempConditionBucket = [];
foreach ($options['org_filter'][$filterKey] as $value) {
if ($value[0] === '!') {
$tempConditionBucket['Organisation.' . $filterKey . ' NOT IN'][] = mb_substr($value, 1);
} else {
$tempConditionBucket['Organisation.' . $filterKey . ' IN'][] = $value;
}
}
}
if (!empty($tempConditionBucket)) {
$orgConditions[] = $tempConditionBucket;
}
}
}
return $organisationModel->find('column', [
'recursive' => -1,
'conditions' => $orgConditions,
'fields' => ['Organisation.id']
]);
}
}
public function handler($user, $options = array())
{
/** @var Event $eventModel */
$attributeModel = ClassRegistry::init('Attribute');
$threshold = empty($options['threshold']) ? 10 : $options['threshold'];
$time_window = empty($options['time_window']) ? (7 * 24 * 60 * 60) : (int)$options['time_window'];
if (is_string($time_window) && substr($time_window, -1) === 'd') {
$time_window = ((int)substr($time_window, 0, -1)) * 24 * 60 * 60;
}
$conditions = $time_window === -1 ? [] : ['timestamp >=' => time() - $time_window];
$conditions['deleted'] = 0;
$conditionsToParse = ['type', 'category', 'to_ids'];
foreach ($conditionsToParse as $parsedCondition) {
if (!empty($options[$parsedCondition])) {
$conditions[$parsedCondition] = $options[$parsedCondition];
}
}
if (!empty($options['exclude'])) {
$conditions['value1 NOT IN'] = $options['exclude'];
}
if (!empty($options['org_filter'])) {
$conditions['Event.orgc_id IN'] = $this->getOrgList($options);
if (empty($conditions['Event.orgc_id IN'])) {
$conditions['Event.orgc_id IN'] = [-1];
}
}
$attributeModel->virtualFields['frequency'] = 0;
if (!empty($user['Role']['perm_site_admin'])) {
$values = $attributeModel->find('all', [
'recursive' => -1,
'fields' => ['value1', 'count(Attribute.value1) as Attribute__frequency'],
'group' => ['value1'],
'conditions' => $conditions,
'contain' => ['Event.orgc_id'],
'order' => 'count(Attribute.value1) desc',
'limit' => empty($options['threshold']) ? 10 : $options['threshold']
]);
} else {
$conditions['AND'][] = [
'OR' => [
'Event.orgc_id' => $user['org_id'],
]
];
$values = $attributeModel->find('all', [
'recursive' => -1,
'fields' => ['value1', 'count(Attribute.value1) as Attribute__frequency', 'distribution', 'sharing_group_id'],
'group' => 'value1',
'contain' => [
'Event.org_id',
'Event.distribution',
'Event.sharing_group_id',
'Object.distribution',
'Object.sharing_group_id'
],
'conditions' => $conditions,
'order' => 'count(Attribute.value1) desc',
'limit' => empty($options['threshold']) ? 10 : $options['threshold']
]);
}
$data = [];
foreach ($values as $value) {
$data[$value['Attribute']['value1']] = $value['Attribute']['frequency'];
}
return ['data' => $data];
}
}

17
app/Lib/Dashboard/TrendingTagsWidget.php
View File

@ -7,7 +7,7 @@ class TrendingTagsWidget
public $width = 3;
public $height = 4;
public $params = array(
'time_window' => 'The time window, going back in seconds, that should be included.',
'time_window' => 'The time window, going back in seconds, that should be included. (allows for filtering by days - example: 5d. -1 Will fetch all historic data)',
'exclude' => 'List of substrings to exclude tags by - for example "sofacy" would exclude any tag containing sofacy.',
'include' => 'List of substrings to include tags by - for example "sofacy" would include any tag containing sofacy.',
'threshold' => 'Limits the number of displayed tags. Default: 10',
@ -16,23 +16,26 @@ class TrendingTagsWidget
);
public $placeholder =
'{
"time_window": "86400",
"time_window": "7d",
"threshold": 15,
"exclude": ["tlp:", "pap:"],
"include": ["misp-galaxy:", "my-internal-taxonomy"],
"filter_event_tags": ["misp-galaxy:threat-actor="APT 29"],
}';
public $description = 'Widget showing the trending tags over the past x seconds, along with the possibility to include/exclude tags.';
public $cacheLifetime = 600;
public $cacheLifetime = 3;
public function handler($user, $options = array())
{
/** @var Event $eventModel */
$eventModel = ClassRegistry::init('Event');
$threshold = empty($options['threshold']) ? 10 : $options['threshold'];
$params = [
'timestamp' => time() - (empty($options['time_window']) ? 8640000 : $options['time_window']),
];
$time_window = empty($options['time_window']) ? (7 * 24 * 60 * 60) : $options['time_window'];
if (is_string($time_window) && substr($time_window, -1) === 'd') {
$time_window = ((int)substr($time_window, 0, -1)) * 24 * 60 * 60;
}
$params = $time_window === -1 ? [] : ['timestamp' => time() - $time_window];
if (!empty($options['filter_event_tags'])) {
$params['event_tags'] = $options['filter_event_tags'];
}
@ -48,6 +51,7 @@ class TrendingTagsWidget
$events = $eventModel->fetchEvent($user, [
'eventid' => $eventIds,
'order' => 'Event.timestamp',
'metadata' => 1
]);
foreach ($events as $event) {
@ -111,7 +115,6 @@ class TrendingTagsWidget
}
}
return $data;
}

402
app/Lib/Dashboard/UsageDataWidget.php
View File

@ -5,40 +5,82 @@ class UsageDataWidget
public $render = 'SimpleList';
public $width = 2;
public $height = 5;
public $params = array();
public $description = 'Shows usage data / statistics.';
public $cacheLifetime = false;
public $autoRefreshDelay = 3;
public $autoRefreshDelay = false;
public $params = [
'filter' => 'A list of filters by organisation meta information (nationality, sector, type, name, uuid) to include. (dictionary, prepending values with ! uses them as a negation)',
];
private $User = null;
private $Event = null;
private $Correlation = null;
private $Thread = null;
private $AuthKey = null;
private $validFilterKeys = [
'nationality',
'sector',
'type',
'name',
'uuid'
];
private $validFields = [
'Events',
'Attributes',
'Attributes / event',
'Correlations',
'Active proposals',
'Users',
'Users with PGP keys',
'Organisations',
'Local organisations',
'Event creator orgs',
'Average users / org',
'Discussion threads',
'Discussion posts'
];
public function handler($user, $options = array()){
$this->User = ClassRegistry::init('User');
$orgsCount = $this->User->Organisation->find('count');
$localOrgsParams['conditions']['Organisation.local'] = 1;
$localOrgsCount = $this->User->Organisation->find('count', $localOrgsParams);
$thisMonth = strtotime('first day of this month');
$this->Event = ClassRegistry::init('Event');
$eventsCount = $this->Event->find('count', array('recursive' => -1));
$eventsCountMonth = $this->Event->find('count', array('conditions' => array('Event.timestamp >' => $thisMonth), 'recursive' => -1));
$this->Attribute = ClassRegistry::init('Attribute');
$attributesCount = $this->Attribute->find('count', array('conditions' => array('Attribute.deleted' => 0), 'recursive' => -1));
$attributesCountMonth = $this->Attribute->find('count', array('conditions' => array('Attribute.timestamp >' => $thisMonth, 'Attribute.deleted' => 0), 'recursive' => -1));
$attributesPerEvent = round($attributesCount / $eventsCount);
$this->Correlation = ClassRegistry::init('Correlation');
$correlationsCount = $this->Correlation->find('count', array('recursive' => -1)) / 2;
$proposalsCount = $this->Event->ShadowAttribute->find('count', array('recursive' => -1, 'conditions' => array('deleted' => 0)));
$usersCount = $this->User->find('count', array('recursive' => -1));
$usersCountPgp = $this->User->find('count', array('recursive' => -1, 'conditions' => array('User.gpgkey !=' => '')));
$usersCountPgpPercentage = round(100* ($usersCountPgp / $usersCount), 1);
$contributingOrgsCount = $this->Event->find('count', array('recursive' => -1, 'group' => array('Event.orgc_id')));
$averageUsersPerOrg = round($usersCount / $localOrgsCount, 1);
$this->Thread = ClassRegistry::init('Thread');
$this->Correlation = ClassRegistry::init('Correlation');
$thisMonth = strtotime('first day of this month');
$orgConditions = [];
$orgIdList = null;
if (!empty($options['filter']) && is_array($options['filter'])) {
foreach ($this->validFilterKeys as $filterKey) {
if (!empty($options['filter'][$filterKey])) {
if (!is_array($options['filter'][$filterKey])) {
$options['filter'][$filterKey] = [$options['filter'][$filterKey]];
}
$tempConditionBucket = [];
foreach ($options['filter'][$filterKey] as $value) {
if ($value[0] === '!') {
$tempConditionBucket['Organisation.' . $filterKey . ' NOT IN'][] = mb_substr($value, 1);
} else {
$tempConditionBucket['Organisation.' . $filterKey . ' IN'][] = $value;
}
}
if (!empty($tempConditionBucket)) {
$orgConditions[] = $tempConditionBucket;
}
}
}
$orgIdList = $this->User->Organisation->find('column', [
'recursive' => -1,
'conditions' => $orgConditions,
'fields' => ['Organisation.id']
]);
}
$eventsCount = $this->getEventsCount($orgConditions, $orgIdList, $thisMonth);
$attributesCount = $this->getAttributesCount($orgConditions, $orgIdList, $thisMonth);
$usersCount = $this->getUsersCount($orgConditions, $orgIdList, $thisMonth);
$usersCountPgp = $this->getUsersCountPgp($orgConditions, $orgIdList, $thisMonth);
$localOrgsCount = $this->getLocalOrgsCount($orgConditions, $orgIdList, $thisMonth);
$threadCount = $this->Thread->find('count', array('conditions' => array('Thread.post_count >' => 0), 'recursive' => -1));
$threadCountMonth = $this->Thread->find('count', array('conditions' => array('Thread.date_created >' => date("Y-m-d H:i:s", $thisMonth), 'Thread.post_count >' => 0), 'recursive' => -1));
@ -47,21 +89,69 @@ class UsageDataWidget
//Monhtly data is not added to the widget at the moment, could optionally add these later and give user choice?
$statistics = array(
array('title' => 'Events', 'value' => $eventsCount),
array('title' => 'Attributes', 'value' => $attributesCount),
array('title' => 'Attributes / event', 'value' => $attributesPerEvent),
array('title' => 'Correlations', 'value' => $correlationsCount),
array('title' => 'Active proposals', 'value' => $proposalsCount),
array('title' => 'Users', 'value' => $usersCount),
array('title' => 'Users with PGP keys', 'value' => $usersCountPgp . ' (' . $usersCountPgpPercentage . '%)'),
array('title' => 'Organisations', 'value' => $orgsCount),
array('title' => 'Local organisations', 'value' => $localOrgsCount),
array('title' => 'Event creator orgs', 'value' => $contributingOrgsCount),
array('title' => 'Average users / org', 'value' => $averageUsersPerOrg),
array('title' => 'Discussions threads', 'value' => $threadCount),
array('title' => 'Discussion posts', 'value' => $postCount)
);
$statistics = [
'Events' => [
'title' => 'Events',
'value' => $eventsCount,
'change' => $this->getEventsCountMonth($orgConditions, $orgIdList, $thisMonth)
],
'Attributes' => [
'title' => 'Attributes',
'value' => $attributesCount,
'change' => $this->getAttributesCountMonth($orgConditions, $orgIdList, $thisMonth)
],
'Attributes / event' => [
'title' => 'Attributes / event',
'value' => $eventsCount ? round($attributesCount / $eventsCount) : 0
],
'Correlations' => [
'title' => 'Correlations',
'value' => $this->getCorrelationsCount($orgConditions, $orgIdList, $thisMonth)
],
'Active proposals' => [
'title' => 'Active proposals',
'value' => $this->getProposalsCount($orgConditions, $orgIdList, $thisMonth)
],
'Users' => [
'title' => 'Users',
'value' => $usersCount,
'change' => $this->getUsersCountMonth($orgConditions, $orgIdList, $thisMonth)
],
'Users with PGP keys' => [
'title' => 'Users with PGP keys',
'value' => sprintf(
'%s (%s %%)',
$usersCountPgp,
$usersCount ? round(100* ($usersCountPgp / $usersCount), 1) : 0
)
],
'Organisations' => [
'title' => 'Organisations',
'value' => $this->getOrgsCount($orgConditions, $orgIdList, $thisMonth),
'change' => $this->getOrgsCountMonth($orgConditions, $orgIdList, $thisMonth)
],
'Local organisations' => [
'title' => 'Local organisations',
'value' => $localOrgsCount,
'change' => $this->getLocalOrgsCountMonth($orgConditions, $orgIdList, $thisMonth)
],
'Event creator orgs' => [
'title' => 'Event creator orgs', 'value' => $this->getContributingOrgsCount($orgConditions, $orgIdList, $thisMonth)
],
'Average users / org' => [
'title' => 'Average users / org', 'value' => round($usersCount / $localOrgsCount, 1)
],
'Discussion threads' => [
'title' => 'Discussions threads',
'value' => $this->getThreadsCount($orgConditions, $orgIdList, $thisMonth),
'change' => $this->getThreadsCountMonth($orgConditions, $orgIdList, $thisMonth)
],
'Discussion posts' => [
'title' => 'Discussion posts',
'value' => $this->getPostsCount($orgConditions, $orgIdList, $thisMonth),
'change' => $this->getPostsCountMonth($orgConditions, $orgIdList, $thisMonth)
]
];
if(!empty(Configure::read('Security.advanced_authkeys'))){
$this->AuthKey = ClassRegistry::init('AuthKey');
$authkeysCount = $this->AuthKey->find('count', array('recursive' => -1));
@ -70,6 +160,233 @@ class UsageDataWidget
return $statistics;
}
private function getEventsCount($orgConditions, $orgIdList, $thisMonth)
{
$conditions = [];
if (!empty($orgIdList)) {
$conditions['AND'][] = ['Event.orgc_id IN' => $orgIdList];
}
return $this->Event->find('count', [
'recursive' => -1,
'conditions' => $conditions
]);
}
private function getCorrelationsCount($orgConditions, $orgIdList, $thisMonth)
{
$conditions = [];
if (!empty($orgIdList)) {
$conditions['AND']['OR'][] = ['Correlation.org_id IN' => $orgIdList];
$conditions['AND']['OR'][] = ['Correlation.1_org_id IN' => $orgIdList];
}
return $this->Correlation->find('count', [
'recursive' => -1,
'conditions' => $conditions
]);
}
private function getEventsCountMonth($orgConditions, $orgIdList, $thisMonth)
{
$conditions = ['Event.timestamp >' => $thisMonth];
if (!empty($orgIdList)) {
$conditions['AND'][] = ['Event.orgc_id IN' => $orgIdList];
}
return $this->Event->find('count', [
'conditions' => $conditions,
'recursive' => -1
]);
}
private function getAttributesCount($orgConditions, $orgIdList, $thisMonth)
{
$conditions = ['Attribute.deleted' => 0];
if (!empty($orgIdList)) {
$conditions['AND'][] = ['Event.orgc_id IN' => $orgIdList];
}
return $this->Event->Attribute->find('count', [
'conditions' => $conditions,
'contain' => ['Event'],
'recursive' => -1
]);
}
private function getAttributesCountMonth($orgConditions, $orgIdList, $thisMonth)
{
$conditions = ['Attribute.timestamp >' => $thisMonth, 'Attribute.deleted' => 0];
if (!empty($orgIdList)) {
$conditions['AND'][] = ['Event.orgc_id IN' => $orgIdList];
}
return $this->Event->Attribute->find('count', [
'conditions' => $conditions,
'contain' => 'Event.orgc_id',
'recursive' => -1
]);
}
private function getOrgsCount($orgConditions, $orgIdList, $thisMonth)
{
return $this->User->Organisation->find('count', [
'conditions' => [
'AND' => $orgConditions
]
]);
}
private function getOrgsCountMonth($orgConditions, $orgIdList, $thisMonth)
{
$datetime = new DateTime();
$datetime->setTimestamp($thisMonth);
$thisMonth = $datetime->format('Y-m-d H:i:s');
return $this->User->Organisation->find('count', [
'conditions' => [
'AND' => $orgConditions,
'Organisation.date_created >' => $thisMonth
]
]);
}
private function getLocalOrgsCount($orgConditions, $orgIdList, $thisMonth)
{
return $this->User->Organisation->find('count', [
'conditions' => [
'Organisation.local' => 1,
'AND' => $orgConditions
]
]);
}
private function getLocalOrgsCountMonth($orgConditions, $orgIdList, $thisMonth)
{
$datetime = new DateTime();
$datetime->setTimestamp($thisMonth);
$thisMonth = $datetime->format('Y-m-d H:i:s');
return $this->User->Organisation->find('count', [
'conditions' => [
'Organisation.local' => 1,
'AND' => $orgConditions,
'Organisation.date_created >' => $thisMonth
]
]);
}
private function getProposalsCount($orgConditions, $orgIdList, $thisMonth)
{
$conditions = ['deleted' => 0];
if (!empty($orgIdList)) {
$conditions['ShadowAttribute.org_id IN'] = $orgIdList;
}
return $this->Event->ShadowAttribute->find('count', [
'recursive' => -1,
'conditions' => $conditions
]);
}
private function getUsersCount($orgConditions, $orgIdList, $thisMonth)
{
$conditions = [];
if (!empty($orgIdList)) {
$conditions['User.org_id IN'] = $orgIdList;
}
return $this->User->find('count', [
'recursive' => -1,
'conditions' => $conditions
]);
}
private function getUsersCountMonth($orgConditions, $orgIdList, $thisMonth)
{
$conditions = ['User.date_created >' => $thisMonth];
if (!empty($orgIdList)) {
$conditions['User.org_id IN'] = $orgIdList;
}
return $this->User->find('count', [
'recursive' => -1,
'conditions' => $conditions
]);
}
private function getUsersCountPgp($orgConditions, $orgIdList, $thisMonth)
{
$conditions = ['User.gpgkey !=' => ''];
if (!empty($orgIdList)) {
$conditions['User.org_id IN'] = $orgIdList;
}
return $this->User->find('count', [
'recursive' => -1,
'conditions' => $conditions
]);
}
private function getContributingOrgsCount($orgConditions, $orgIdList, $thisMonth)
{
$conditions = [];
if ($orgConditions) {
$conditions['AND'][] = ['Event.orgc_id IN' => $orgIdList];
}
return $this->Event->find('count', [
'recursive' => -1,
'group' => ['Event.orgc_id'],
'conditions' => $conditions
]);
}
private function getThreadsCount($orgConditions, $orgIdList, $thisMonth)
{
$conditions = ['Thread.post_count >' => 0];
if ($orgConditions) {
$conditions['AND'][] = ['Thread.org_id IN' => $orgIdList];
}
return $this->Thread->find('count', [
'conditions' => $conditions,
'recursive' => -1
]);
}
private function getThreadsCountMonth($orgConditions, $orgIdList, $thisMonth)
{
$conditions = [
'Thread.post_count >' => 0,
'Thread.date_created >=' => $thisMonth
];
if ($orgConditions) {
$conditions['AND'][] = ['Thread.org_id IN' => $orgIdList];
}
return $this->Thread->find('count', [
'conditions' => $conditions,
'recursive' => -1
]);
}
private function getPostsCount($orgConditions, $orgIdList, $thisMonth)
{
$conditions = [];
if ($orgConditions) {
$conditions['AND'][] = ['User.org_id IN' => $orgIdList];
}
return $this->Thread->Post->find('count', [
'conditions' => $conditions,
'contain' => ['User.org_id'],
'recursive' => -1
]);
}
private function getPostsCountMonth($orgConditions, $orgIdList, $thisMonth)
{
$conditions = [
'Post.date_created >=' => $thisMonth
];
if ($orgConditions) {
$conditions['AND'][] = ['User.org_id IN' => $orgIdList];
}
return $this->Thread->Post->find('count', [
'conditions' => $conditions,
'contain' => ['User.org_id'],
'recursive' => -1
]);
}
/* There is nothing sensitive in here.
public function checkPermissions($user)
{
if (empty($user['Role']['perm_site_admin'])) {
@ -77,4 +394,5 @@ class UsageDataWidget
}
return true;
}
*/
}

123
app/Lib/Dashboard/UserContributionToplistWidget.php Normal file
View File

@ -0,0 +1,123 @@
<?php
class UserContributionToplistWidget
{
public $title = 'Contributor Top List (Users)';
public $render = 'BarChart';
public $description = 'The top contributors (users) in a selected time frame.';
public $width = 3;
public $height = 4;
public $params = [
'days' => 'How many days back should the list go - for example, setting 7 will only show contributions in the past 7 days. (integer)',
'month' => 'Who contributed most this month? (boolean)',
'year' => 'Which contributed most this year? (boolean)',
'filter' => 'A list of filters by organisation meta information (nationality, sector, type, name, uuid, local (- expects a boolean or a list of boolean values)) to include. (dictionary, prepending values with ! uses them as a negation)',
'limit' => 'Limits the number of displayed tags. Default: 10'
];
public $cacheLifetime = null;
public $autoRefreshDelay = false;
private $validFilterKeys = [
'nationality',
'sector',
'type',
'name',
'uuid'
];
public $placeholder =
'{
"days": "7d",
"threshold": 15,
"filter": {
"sector": "Financial"
}
}';
private $Org = null;
private $Event = null;
private function timeConditions($options)
{
$limit = empty($options['limit']) ? 10 : $options['limit'];
if (!empty($options['days'])) {
$condition = strtotime(sprintf("-%s days", $options['days']));
} else if (!empty($options['month'])) {
$condition = strtotime('first day of this month 00:00:00', time());
} else if (!empty($options['year'])) {
$condition = strtotime('first day of this year 00:00:00', time());
} else {
return null;
}
return $condition;
}
public function handler($user, $options = array())
{
$params = ['conditions' => []];
$timeConditions = $this->timeConditions($options);
if ($timeConditions) {
$params['conditions']['AND'][] = ['Event.timestamp >=' => $timeConditions];
}
if (!empty($options['filter']) && is_array($options['filter'])) {
foreach ($this->validFilterKeys as $filterKey) {
if (!empty($options['filter'][$filterKey])) {
if (!is_array($options['filter'][$filterKey])) {
$options['filter'][$filterKey] = [$options['filter'][$filterKey]];
}
$tempConditionBucket = [];
foreach ($options['filter'][$filterKey] as $value) {
if ($value[0] === '!') {
$tempConditionBucket['Organisation.' . $filterKey . ' NOT IN'][] = mb_substr($value, 1);
} else {
$tempConditionBucket['Organisation.' . $filterKey . ' IN'][] = $value;
}
}
if (!empty($tempConditionBucket)) {
$params['conditions']['AND'][] = $tempConditionBucket;
}
}
}
}
if (isset($options['filter']['local'])) {
$params['conditions']['AND']['local'] = $options['filter']['local'];
}
$this->Org = ClassRegistry::init('Organisation');
$org_ids = $this->Org->find('list', [
'fields' => ['Organisation.id', 'Organisation.name'],
'conditions' => $params['conditions']
]);
$userConditions = [];
if (!empty($org_ids)) {
$userConditions = ['User.org_id IN' => array_keys($org_ids)];
}
$user_ids = $this->Org->User->find('list', [
'fields' => ['User.id', 'User.email'],
'conditions' => $userConditions
]);
$conditions = empty($user_ids) ? [] : ['Event.user_id IN' => array_keys($user_ids)];
$this->Event = ClassRegistry::init('Event');
$this->Event->virtualFields['frequency'] = 0;
$users = $this->Event->find('all', [
'recursive' => -1,
'fields' => ['user_id', 'count(Event.user_id) as Event__frequency'],
'group' => ['user_id'],
'conditions' => $conditions,
'order' => 'count(Event.user_id) desc',
'limit' => empty($options['limit']) ? 10 : $options['limit']
]);
$results = [];
foreach($users as $user) {
$results[$user_ids[$user['Event']['user_id']]] = $user['Event']['frequency'];
}
return ['data' => $results];
}
public function checkPermissions($user)
{
if (empty(Configure::read('Security.disclose_user_emails')) && empty($user['Role']['perm_site_admin'])) {
return false;
}
return true;
}
}
?>

2
app/Lib/Export/Stix2Export.php
View File

@ -4,7 +4,7 @@ App::uses('StixExport', 'Export');
class Stix2Export extends StixExport
{
protected $__attributes_limit = 15000;
protected $__default_version = '2.0';
protected $__default_version = '2.1';
protected $__sane_versions = array('2.0', '2.1');
protected function __initiate_framing_params()

21
app/Lib/Tools/WorkflowFormatConverterTool.php
View File

@ -44,12 +44,16 @@ class WorkflowFormatConverterTool
{
$converted = [];
$converted = JSONConverterTool::convert($event, false, true);
$eventTags = $converted['Event']['Tag'];
foreach ($converted['Event']['Attribute'] as $i => $attribute) {
$converted['Event']['Attribute'][$i] = self::__propagateTagToAttributes($attribute, $eventTags);
$eventTags = !empty($converted['Event']['Tag']) ? $converted['Event']['Tag'] : [];
if (!empty($converted['Event']['Attribute'])) {
foreach ($converted['Event']['Attribute'] as $i => $attribute) {
$converted['Event']['Attribute'][$i] = self::__propagateTagToAttributes($attribute, $eventTags);
}
}
foreach ($converted['Event']['Object'] as $i => $object) {
$converted['Event']['Object'][$i] = self::__propagateTagToObjectAttributes($object, $eventTags);
if (!empty($converted['Event']['Object'])) {
foreach ($converted['Event']['Object'] as $i => $object) {
$converted['Event']['Object'][$i] = self::__propagateTagToObjectAttributes($object, $eventTags);
}
}
return $converted;
}
@ -146,12 +150,11 @@ class WorkflowFormatConverterTool
if (empty($event)) {
return [];
}
$event = self::__convertEvent($event);
$event = $event['Event'];
reset($data);
$entityType = key($data);
$event[$entityType][] = $data[$entityType];
return ['Event' => $event];
$event['Event'][$entityType][] = $data[$entityType];
$event = self::__convertEvent($event);
return $event;
}
private static function __includeFlattenedAttributes(array $event): array

101
app/Lib/Tools/WorkflowGraphTool.php
View File

@ -162,6 +162,12 @@ class GraphWalker
} else if ($node['data']['id'] == 'concurrent-task') {
$this->_evaluateConcurrentTask($node, $roamingData, $outputs['output_1']);
return ['output_1' => []];
} else if ($node['data']['id'] == 'generic-filter-data') {
$this->_evaluateFilterAddLogic($node, $roamingData, $outputs['output_1']);
return ['output_1' => $outputs['output_1']];
} else if ($node['data']['id'] == 'generic-filter-reset') {
$this->_evaluateFilterRemoveLogic($node, $roamingData, $outputs['output_1']);
return ['output_1' => $outputs['output_1']];
} else {
$useFirstOutput = $this->_evaluateCustomLogicCondition($node, $roamingData);
return $useFirstOutput ? ['output_1' => $outputs['output_1']] : ['output_2' => $outputs['output_2']];
@ -175,6 +181,18 @@ class GraphWalker
return $result;
}
private function _evaluateFilterAddLogic($node, WorkflowRoamingData $roamingData): bool
{
$result = $this->WorkflowModel->executeNode($node, $roamingData);
return $result;
}
private function _evaluateFilterRemoveLogic($node, WorkflowRoamingData $roamingData): bool
{
$result = $this->WorkflowModel->executeNode($node, $roamingData);
return $result;
}
private function _evaluateCustomLogicCondition($node, WorkflowRoamingData $roamingData): bool
{
$result = $this->WorkflowModel->executeNode($node, $roamingData);
@ -254,6 +272,7 @@ class WorkflowRoamingData
private $data;
private $workflow;
private $current_node;
private $workflowModel;
public function __construct(array $workflow_user, array $data, array $workflow, int $current_node)
{
@ -270,9 +289,47 @@ class WorkflowRoamingData
public function getData(): array
{
if (!empty($this->getEnabledFilters())) {
return $this->filterDataIfNeeded();
}
return $this->data;
}
public function filterDataIfNeeded(): array
{
$filteredData = $this->data;
$filters = $this->getEnabledFilters();
foreach ($filters as $filteringLabel => $filteringOptions) {
$filteredData = $this->applyFilter($filteredData, $filteringOptions);
}
return $filteredData;
}
private function applyFilter(array $data, array $filteringOptions): array
{
$baseModule = $this->getFilteringModule();
$extracted = $baseModule->extractData($data, $filteringOptions['selector']);
if ($extracted === false) {
$filteredData = false;
}
$filteredData = $baseModule->getItemsMatchingCondition($extracted, $filteringOptions['value'], $filteringOptions['operator'], $filteringOptions['path']);
$newData = Hash::remove($data, $filteringOptions['selector']);
$newData = Hash::insert($data, $filteringOptions['selector'], $filteredData);
return $newData;
}
private function getFilteringModule()
{
$this->workflowModel = ClassRegistry::init('Workflow');
$moduleClass = $this->workflowModel->getModuleClassByType('logic', 'generic-filter-data');
return $moduleClass;
}
public function getEnabledFilters(): array
{
return !empty($this->data['enabledFilters']) ? $this->data['enabledFilters'] : [];
}
public function getWorkflow(): array
{
return $this->workflow;
@ -357,6 +414,50 @@ class WorkflowGraphTool
return $nodes;
}
/**
* extractFilterNodesFromWorkflow Return the list of generic-filter-data's id (or full module) that are included in the workflow
*
* @param array $workflow
* @param bool $fullNode
* @return array
*/
public static function extractFilterNodesFromWorkflow(array $graphData, bool $fullNode = false): array
{
$nodes = [];
foreach ($graphData as $node) {
if ($node['data']['module_type'] == 'logic' && $node['data']['id'] == 'generic-filter-data') {
if (!empty($fullNode)) {
$nodes[] = $node;
} else {
$nodes[] = $node['data']['id'];
}
}
}
return $nodes;
}
/**
* extractResetFilterFromWorkflow Return the list of generic-filter-reset's id (or full module) that are included in the workflow
*
* @param array $workflow
* @param bool $fullNode
* @return array
*/
public static function extractResetFilterFromWorkflow(array $graphData, bool $fullNode = false): array
{
$nodes = [];
foreach ($graphData as $node) {
if ($node['data']['module_type'] == 'logic' && $node['data']['id'] == 'generic-filter-reset') {
if (!empty($fullNode)) {
$nodes[] = $node;
} else {
$nodes[] = $node['data']['id'];
}
}
}
return $nodes;
}
/**
* isAcyclic Return if the graph contains a cycle
*

87
app/Model/Event.php
View File

@ -3641,6 +3641,7 @@ class Event extends AppModel
$this->OrgBlocklist = ClassRegistry::init('OrgBlocklist');
}
if ($this->OrgBlocklist->isBlocked($orgc)) {
$this->OrgBlocklist->saveEventBlocked($orgc);
return 'blocked';
}
}
@ -5837,41 +5838,79 @@ class Event extends AppModel
* @throws InvalidArgumentException
* @throws Exception
*/
public function upload_stix(array $user, $file, $stix_version, $original_file, $publish)
public function upload_stix(array $user, $file, $stix_version, $original_file, $publish, $distribution, $sharingGroupId, $galaxiesAsTags, $debug = false)
{
$scriptDir = APP . 'files' . DS . 'scripts';
if ($stix_version == '2') {
if ($stix_version == '2' || $stix_version == '2.0' || $stix_version == '2.1') {
$scriptFile = $scriptDir . DS . 'stix2' . DS . 'stix2misp.py';
$output_path = $file . '.stix2';
$stix_version = "STIX 2.0";
$output_path = $file . '.out';
$shell_command = [
ProcessTool::pythonBin(),
$scriptFile,
'-i', $file,
'--distribution', $distribution
];
if ($distribution == 4) {
array_push($shell_command, '--sharing_group_id', $sharingGroupId);
}
if ($galaxiesAsTags) {
$shell_command[] = '--galaxies_as_tags';
}
if ($debug) {
$shell_command[] = '--debug';
}
$stix_version = "STIX 2.1";
} elseif ($stix_version == '1' || $stix_version == '1.1' || $stix_version == '1.2') {
$scriptFile = $scriptDir . DS . 'stix2misp.py';
$output_path = $file . '.json';
$shell_command = [
ProcessTool::pythonBin(),
$scriptFile,
$file,
Configure::read('MISP.default_event_distribution'),
Configure::read('MISP.default_attribute_distribution'),
$this->__getTagNamesFromSynonyms($scriptDir)
];
$stix_version = "STIX 1.1";
} else {
throw new InvalidArgumentException('Invalid STIX version');
}
$shell_command = [
ProcessTool::pythonBin(),
$scriptFile,
$file,
Configure::read('MISP.default_event_distribution'),
Configure::read('MISP.default_attribute_distribution'),
$this->__getTagNamesFromSynonyms($scriptDir),
];
$result = ProcessTool::execute($shell_command, null, true);
$result = preg_split("/\r\n|\n|\r/", trim($result));
$result = trim(end($result));
$tempFile = file_get_contents($file);
unlink($file);
if ($result === '1') {
$decoded = JsonTool::decode($result);
if (!empty($decoded['success'])) {
$data = FileAccessTool::readAndDelete($output_path);
$data = $this->jsonDecode($data);
if (empty($data['Event'])) {
$data = array('Event' => $data);
}
if (!$galaxiesAsTags) {
if (!isset($this->GalaxyCluster)) {
$this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
}
$this->__handleGalaxiesAndClusters($user, $data['Event']);
if (!empty($data['Event']['Attribute'])) {
foreach ($data['Event']['Attribute'] as &$attribute) {
$this->__handleGalaxiesAndClusters($user, $attribute);
}
}
if (!empty($data['Event']['Object'])) {
foreach ($data['Event']['Object'] as &$misp_object) {
if (!empty($misp_object['Attribute'])) {
foreach ($misp_object['Attribute'] as &$attribute) {
$this->__handleGalaxiesAndClusters($user, $attribute);
}
}
}
}
}
if (!empty($decoded['stix_version'])) {
$stix_version = 'STIX ' . $decoded['stix_version'];
}
$created_id = false;
$validationIssues = false;
$result = $this->_add($data, true, $user, '', null, false, null, $created_id, $validationIssues);
@ -5889,13 +5928,8 @@ class Event extends AppModel
return $result;
}
return $validationIssues;
} else if ($result === '2') {
$response = __('Issues while loading the stix file.');
} elseif ($result === '3') {
$response = __('Issues with the maec library.');
} else {
$response = __('Issues executing the ingestion script or invalid input.');
}
$response = __($decoded['error']);
if (!$user['Role']['perm_site_admin']) {
$response .= ' ' . __('Please ask your administrator to');
} else {
@ -5905,6 +5939,19 @@ class Event extends AppModel
return $response;
}
private function __handleGalaxiesAndClusters($user, &$data)
{
if (!empty($data['Galaxy'])) {
$tag_names = $this->GalaxyCluster->convertGalaxyClustersToTags($user, $data['Galaxy']);
if (empty($data['Tag'])) {
$data['Tag'] = [];
}
foreach ($tag_names as $tag_name) {
$data['Tag'][] = array('name' => $tag_name);
}
}
}
/**
* @param string $scriptDir
* @return string

27
app/Model/GalaxyCluster.php
View File

@ -2071,4 +2071,31 @@ class GalaxyCluster extends AppModel
}
return $CyCatRelations;
}
/**
* convertGalaxyClustersToTags
*
* @param array $user
* @param array $galaxies
* @return array The tag names extracted from galaxy clusters
*/
public function convertGalaxyClustersToTags($user, $galaxies)
{
$galaxyClusters = [];
$tag_names = [];
foreach ($galaxies as $galaxy) {
if (empty($galaxy['GalaxyCluster'])) {
continue;
}
$clusters = $galaxy['GalaxyCluster'];
unset($galaxy['GalaxyCluster']);
foreach ($clusters as $cluster) {
$cluster['Galaxy'] = $galaxy;
$galaxyClusters[] = array('GalaxyCluster' => $cluster);
$tag_names[] = !empty($cluster['tag_name']) ? $cluster['tag_name'] : 'misp-galaxy:' . $cluster['type'] . '="' . $cluster['uuid'] . '"';
}
}
$this->Galaxy->importGalaxyAndClusters($user, $galaxyClusters);
return $tag_names;
}
}

92
app/Model/OrgBlocklist.php
View File

@ -43,6 +43,24 @@ class OrgBlocklist extends AppModel
return true;
}
public function afterDelete()
{
parent::afterDelete();
if (!empty($this->data['OrgBlocklist']['org_uuid'])) {
$this->cleanupBlockedCount($this->data['OrgBlocklist']['org_uuid']);
}
}
public function afterFind($results, $primary = false)
{
foreach ($results as $k => $result) {
if (isset($result['OrgBlocklist']['org_uuid'])) {
$results[$k]['OrgBlocklist']['blocked_data'] = $this->getBlockedData($result['OrgBlocklist']['org_uuid']);
}
}
return $results;
}
/**
* @param array $eventArray
*/
@ -74,16 +92,7 @@ class OrgBlocklist extends AppModel
}
if (is_numeric($orgIdOrUuid)) {
$this->Organisation = ClassRegistry::init('Organisation');
$orgUuid = $this->Organisation->find('first', [
'conditions' => ['Organisation.id' => $orgIdOrUuid],
'fields' => ['Organisation.uuid'],
'recursive' => -1,
]);
if (empty($orgUuid)) {
return false; // org not found by ID, so it is not blocked
}
$orgUuid = $orgUuid['Organisation']['uuid'];
$orgUuid = $this->getUUIDFromID($orgIdOrUuid);
} else {
$orgUuid = $orgIdOrUuid;
}
@ -92,4 +101,67 @@ class OrgBlocklist extends AppModel
$this->blockedCache[$orgIdOrUuid] = $isBlocked;
return $isBlocked;
}
private function getUUIDFromID($orgID)
{
$this->Organisation = ClassRegistry::init('Organisation');
$orgUuid = $this->Organisation->find('first', [
'conditions' => ['Organisation.id' => $orgID],
'fields' => ['Organisation.uuid'],
'recursive' => -1,
]);
if (empty($orgUuid)) {
return false; // org not found by ID, so it is not blocked
}
$orgUuid = $orgUuid['Organisation']['uuid'];
return $orgUuid;
}
public function saveEventBlocked($orgIdOrUUID)
{
if (is_numeric($orgIdOrUUID)) {
$orgcUUID = $this->getUUIDFromID($orgIdOrUUID);
} else {
$orgcUUID = $orgIdOrUUID;
}
$lastBlockTime = time();
$redisKeyBlockAmount = "misp:blocklist_blocked_amount:{$orgcUUID}";
$redisKeyBlockLastTime = "misp:blocklist_blocked_last_time:{$orgcUUID}";
$redis = RedisTool::init();
if ($redis !== false) {
$pipe = $redis->multi(Redis::PIPELINE)
->incr($redisKeyBlockAmount)
->set($redisKeyBlockLastTime, $lastBlockTime);
$pipe->exec();
}
}
private function cleanupBlockedCount($orgcUUID)
{
$redisKeyBlockAmount = "misp:blocklist_blocked_amount:{$orgcUUID}";
$redisKeyBlockLastTime = "misp:blocklist_blocked_last_time:{$orgcUUID}";
$redis = RedisTool::init();
if ($redis !== false) {
$pipe = $redis->multi(Redis::PIPELINE)
->del($redisKeyBlockAmount)
->del($redisKeyBlockLastTime);
$pipe->exec();
}
}
public function getBlockedData($orgcUUID)
{
$redisKeyBlockAmount = "misp:blocklist_blocked_amount:{$orgcUUID}";
$redisKeyBlockLastTime = "misp:blocklist_blocked_last_time:{$orgcUUID}";
$blockData = [
'blocked_amount' => false,
'blocked_last_time' => false,
];
$redis = RedisTool::init();
if ($redis !== false) {
$blockData['blocked_amount'] = $redis->get($redisKeyBlockAmount);
$blockData['blocked_last_time'] = $redis->get($redisKeyBlockLastTime);
}
return $blockData;
}
}

10
app/Model/Server.php
View File

@ -6584,7 +6584,15 @@ class Server extends AppModel
'type' => 'boolean',
'null' => true,
'cli_only' => true
]
],
'disclose_user_emails' => array(
'level' => 0,
'description' => __('Enable this setting to allow for the user e-mail addresses to be shown to non site-admin users. Keep in mind that in broad communities this can be abused.'),
'value' => false,
'test' => 'testBool',
'type' => 'boolean',
'null' => true
),
),
'SecureAuth' => array(
'branch' => 1,

63
app/Model/Workflow.php
View File

@ -1026,7 +1026,7 @@ class Workflow extends AppModel
$className = explode('/', $filepath);
$className = str_replace('.php', '', $className[count($className)-1]);
try {
if (!@include_once($filepath)) {
if (!include_once($filepath)) {
$message = __('Could not load module for path %s. File does not exist.', $filepath);
$this->log($message, LOG_ERR);
return $message;
@ -1301,6 +1301,67 @@ class Workflow extends AppModel
return $data;
}
public function getLabelsForConnections($workflow, $trigger_id): array
{
$graphData = !empty($workflow['Workflow']) ? $workflow['Workflow']['data'] : $workflow['data'];
$startNodeID = $this->workflowGraphTool->getNodeIdForTrigger($graphData, $trigger_id);
if ($startNodeID == -1) {
return [];
}
$connections = [];
$filterNodes = $this->workflowGraphTool->extractFilterNodesFromWorkflow($graphData, true);
$filterNodeIDToLabel = Hash::combine($filterNodes, '{n}.id', '{n}.data.indexed_params.filtering-label');
$resetFilterNodes = $this->workflowGraphTool->extractResetFilterFromWorkflow($graphData, true);
$resetFilterNodeIDToLabel = Hash::combine($resetFilterNodes, '{n}.id', '{n}.data.indexed_params.filtering-label');
$roamingData = $this->workflowGraphTool->getRoamingData();
$graphWalker = $this->workflowGraphTool->getWalkerIterator($graphData, $this, $startNodeID, GraphWalker::PATH_TYPE_INCLUDE_LOGIC, $roamingData);
foreach ($graphWalker as $graphNode) {
$node = $graphNode['node'];
$nodeID = $node['id'];
$parsedPathList = GraphWalker::parsePathList($graphNode['path_list']);
foreach ($parsedPathList as $pathEntry) {
if (!empty($filterNodeIDToLabel[$pathEntry['source_id']])) {
$connections[$nodeID][] = $filterNodeIDToLabel[$pathEntry['source_id']];
}
if (!empty($resetFilterNodeIDToLabel[$pathEntry['source_id']])) {
if ($resetFilterNodeIDToLabel[$pathEntry['source_id']] == 'all') {
$connections[$nodeID] = [];
} else {
$connections[$nodeID] = array_values(array_diff($connections[$nodeID], [$resetFilterNodeIDToLabel[$pathEntry['source_id']]]));
}
}
}
}
return $connections;
}
public function attachLabelToConnections($workflow, $trigger_id=null): array
{
$graphData = !empty($workflow['Workflow']) ? $workflow['Workflow']['data'] : $workflow['data'];
if (is_null($trigger_id)) {
$startNode = $this->workflowGraphTool->extractTriggerFromWorkflow($graphData, true);
$trigger_id = $startNode['data']['id'];
}
$labelsByNodes = $this->getLabelsForConnections($workflow, $trigger_id);
foreach ($graphData as $i => $node) {
if (!empty($labelsByNodes[$node['id']])) {
foreach ($node['inputs'] as $inputName => $inputs) {
foreach ($inputs['connections'] as $j => $connection) {
$workflow['Workflow']['data'][$i]['inputs'][$inputName]['connections'][$j]['labels'] = array_map(function($label) {
return [
'id' => Inflector::variable($label),
'name' => $label,
'variant' => 'info',
];
}, $labelsByNodes[$node['id']]);
}
}
}
}
return $workflow;
}
/**
* moduleSattelesExecution Executes a module using the provided configuration and returns back the result
*

2
app/Model/WorkflowModules/Module_misp_module.php
View File

@ -46,7 +46,7 @@ class Module_misp_module extends WorkflowBaseActionModule
if (!empty($misp_module_config['meta']['config']['support_filters'])) {
$this->support_filters = !empty($misp_module_config['meta']['config']['support_filters']);
}
if (!empty($misp_module_config['meta']['config'])) {
if (!empty($misp_module_config['meta']['config']['params'])) {
foreach ($misp_module_config['meta']['config']['params'] as $paramName => $moduleParam) {
$this->params[] = $this->translateParams($paramName, $moduleParam);
}

21
app/Model/WorkflowModules/WorkflowBaseModule.php
View File

@ -134,7 +134,7 @@ class WorkflowBaseModule
return 'The Factory Must Grow';
}
protected function extractData($data, $path)
public function extractData($data, $path)
{
$extracted = $data;
if (!empty($path)) {
@ -210,7 +210,7 @@ class WorkflowBaseModule
return false;
}
protected function getItemsMatchingCondition($items, $value, $operator, $path)
public function getItemsMatchingCondition($items, $value, $operator, $path)
{
foreach ($items as $i => $item) {
$subItem = $this->extractData($item, $path, $operator);
@ -297,3 +297,20 @@ class WorkflowBaseLogicModule extends WorkflowBaseModule
class WorkflowBaseActionModule extends WorkflowBaseModule
{
}
class WorkflowFilteringLogicModule extends WorkflowBaseLogicModule
{
public $blocking = false;
public $inputs = 1;
public $outputs = 2;
protected function _genFilteringLabels(): array
{
$names = ['A', 'B', 'C', 'D', 'E', 'F'];
$labels = [];
foreach ($names as $name) {
$labels[$name] = __('Label %s', $name);
}
return $labels;
}
}

2
app/Model/WorkflowModules/action/Module_attribute_ids_flag_operation.php
View File

@ -1,5 +1,5 @@
<?php
include_once APP . 'Model/WorkflowModules/Module_attribute_edition_operation.php';
include_once APP . 'Model/WorkflowModules/action/Module_attribute_edition_operation.php';
class Module_attribute_ids_flag_operation extends Module_attribute_edition_operation
{

4
app/Model/WorkflowModules/action/Module_ms_teams_webhook.php
View File

@ -5,7 +5,7 @@ class Module_ms_teams_webhook extends Module_webhook
{
public $id = 'ms-teams-webhook';
public $name = 'MS Teams Webhook';
public $version = '0.2';
public $version = '0.3';
public $description = 'Perform callbacks to the MS Teams webhook provided by the "Incoming Webhook" connector';
public $icon_path = 'MS_Teams.png';
@ -40,7 +40,7 @@ class Module_ms_teams_webhook extends Module_webhook
protected function doRequest($url, $contentType, $data, $headers = [], $serverConfig = null)
{
$data = '{"text":"' . implode($data) . '"}';
$data = ['text' => JsonTool::encode($data)];
return parent::doRequest($url, $contentType, $data);
}
}

84
app/Model/WorkflowModules/logic/Module_generic_filter_data.php Normal file
View File

@ -0,0 +1,84 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_generic_filter_data extends WorkflowFilteringLogicModule
{
public $id = 'generic-filter-data';
public $name = 'Filter :: Generic';
public $description = 'Generic data filtering block. The module filters incoming data and forward the matching data to its output.';
public $icon = 'filter';
public $inputs = 1;
public $outputs = 1;
public $params = [];
private $operators = [
'in' => 'In',
'not_in' => 'Not in',
'equals' => 'Equals',
'not_equals' => 'Not equals',
];
public function __construct()
{
parent::__construct();
$this->params = [
[
'id' => 'filtering-label',
'label' => __('Filtering Label'),
'type' => 'select',
'options' => $this->_genFilteringLabels(),
],
[
'id' => 'selector',
'label' => __('Data selector'),
'type' => 'input',
'placeholder' => 'Event._AttributeFlattened.{n}',
],
[
'id' => 'value',
'label' => __('Value'),
'type' => 'input',
'placeholder' => 'tlp:red',
],
[
'id' => 'operator',
'label' => __('Operator'),
'type' => 'select',
'default' => 'in',
'options' => $this->operators,
],
[
'id' => 'hash_path',
'label' => __('Hash path'),
'type' => 'input',
'placeholder' => 'Tag.name',
],
];
}
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool
{
parent::exec($node, $roamingData, $errors);
$params = $this->getParamsWithValues($node);
$selector = $params['selector']['value'];
$path = $params['hash_path']['value'];
$operator = $params['operator']['value'];
$value = $params['value']['value'];
$filteringLabel = $params['filtering-label']['value'];
$rData = $roamingData->getData();
$newRData = $rData;
if (empty($newRData['_unfilteredData'])) {
$newRData['_unfilteredData'] = $rData;
}
$newRData['enabledFilters'][$filteringLabel] = [
'selector' => $selector,
'path' => $path,
'operator' => $operator,
'value' => $value,
];
$roamingData->setData($newRData);
return true;
}
}

44
app/Model/WorkflowModules/logic/Module_generic_filter_reset.php Normal file
View File

@ -0,0 +1,44 @@
<?php
include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
class Module_generic_filter_reset extends WorkflowFilteringLogicModule
{
public $id = 'generic-filter-reset';
public $name = 'Filter :: Remove filter';
public $description = 'Reset filtering';
public $icon = 'redo-alt';
public $inputs = 1;
public $outputs = 1;
public $params = [];
public function __construct()
{
parent::__construct();
$this->params = [
[
'id' => 'filtering-label',
'label' => __('Filtering Label to remove'),
'type' => 'select',
'default' => 'all',
'options' => ['all' => __('All filters')] + $this->_genFilteringLabels(),
],
];
}
public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors=[]): bool
{
parent::exec($node, $roamingData, $errors);
$params = $this->getParamsWithValues($node);
$filteringLabel = $params['filtering-label']['value'];
$rData = $roamingData->getData();
$newRData = $rData['_unfilteredData'];
if (in_array($filteringLabel, array_keys($this->_genFilteringLabels()))) {
unset($newRData['enabledFilters'][$filteringLabel]);
} else if ($filteringLabel === 'all') {
$newRData['enabledFilters'] = [];
}
$roamingData->setData($newRData);
return true;
}
}

2
app/View/Dashboards/list_templates.ctp
View File

@ -46,7 +46,7 @@
array(
'name' => __('Widgets Used'),
'data_path' => 'Dashboard.widgets',
'element' => 'list'
'element' => 'allow_deny_list'
),
array(
'name' => __('Selectable'),

538
app/View/Elements/Workflows/infoModal.ctp
View File

@ -7,6 +7,7 @@
<ul class="nav nav-tabs">
<li class="active"><a href=" #modal-info-concept" data-toggle="tab"><?= __('Terminology & Concepts') ?></a></li>
<li class=""><a href=" #modal-hash-path" data-toggle="tab"><?= __('Hash Path') ?></a></li>
<li class=""><a href=" #modal-core-format" data-toggle="tab"><?= __('MISP Core Format') ?></a></li>
<li class=""><a href=" #modal-blueprint" data-toggle="tab"><?= __('Blueprints') ?></a></li>
<li class=""><a href=" #modal-debugging" data-toggle="tab"><?= __('Debugging') ?></a></li>
<li><a href="#modal-info-usage" data-toggle="tab"><?= __('Usage & Shortcuts') ?></a></li>
@ -103,6 +104,543 @@ $data_passed_to_if_module = [
// Then `then` branch will be used by the execution path</pre>
</div>
<div class="tab-pane" id="modal-core-format">
<h2><?= __('MISP Core Format') ?></h2>
<h4><?= __('Accessing Attributes') ?></h4>
<p><?= __('There are two ways to access attributes') ?></p>
<ul>
<li><?= sprintf('%s <code>%s</code> %s', __('Use the'), 'Attribute', __('key to access only attributes')) ?></li>
<li><?= sprintf('%s <code>%s</code> %s', __('Use the'), '_AttributeFlattened', __('key to access all attributes including Object Attributes')) ?></li>
</ul>
<p><strong><?= __('Example:') ?></strong></p>
<pre>Event._AttributeFlattened.{n}</pre>
<h4><?= __('Getting all tag names attached an Attribute only') ?></h4>
<ul>
<li><?= __('First, we access the Attributes with ') ?> <code>Event.Attribute.{n}</code></li>
<li><?= __('Then, we access all tags with ') ?> <code>Tag.{n}.name</code></li>
</ul>
<p><strong><?= __('Full example:') ?></strong></p>
<pre>Event._AttributeFlattened.{n}.Tag.{n}.name</pre>
<h4><?= __('MISP Core Format Sample') ?></h4>
<ul>
<li><?= __('Attributes are always encapsulated in the Event or Object') ?></li>
<li><?= __('Additional key') ?> <code>_AttributeFlattened</code> containing all Attributes</li>
<li><?= __('Additional key') ?> <code>_allTags</code> containing all tags</li>
<ul>
<li><?= __('Additional key %s for Tags', '<code>inherited</code>') ?></li>
</ul>
</ul>
<p><strong><?= __('Sample:') ?></strong></p>
<pre>
{
"Event": {
"id": "64",
"orgc_id": "1",
"org_id": "1",
"date": "2023-05-03",
"threat_level_id": "1",
"info": "Core format sample",
"published": false,
"uuid": "b9557473-bb46-4c65-b69e-974b3c93c1f4",
"attribute_count": "2",
"analysis": "0",
"timestamp": "1683117902",
"distribution": "1",
"proposal_email_lock": false,
"locked": false,
"publish_timestamp": "0",
"sharing_group_id": "0",
"disable_correlation": false,
"extends_uuid": "",
"protected": null,
"event_creator_email": "admin@admin.test",
"Org": {
"id": "1",
"name": "ORGNAME",
"uuid": "c5de83b4-36ba-49d6-9530-2a315caeece6",
"local": true
},
"Orgc": {
"id": "1",
"name": "ORGNAME",
"uuid": "c5de83b4-36ba-49d6-9530-2a315caeece6",
"local": true
},
"Attribute": [
{
"id": "1695",
"type": "ip-src",
"category": "Network activity",
"to_ids": true,
"uuid": "9ac36927-d874-4094-bf4c-f922c1e9cc35",
"event_id": "64",
"distribution": "5",
"timestamp": "1683117902",
"comment": "",
"sharing_group_id": "0",
"deleted": false,
"disable_correlation": false,
"object_id": "0",
"object_relation": null,
"first_seen": null,
"last_seen": null,
"value": "8.8.8.8",
"Galaxy": [],
"ShadowAttribute": [],
"Tag": [
{
"id": "137",
"name": "PAP:AMBER",
"colour": "#ffa800",
"exportable": true,
"user_id": "0",
"hide_tag": false,
"numerical_value": null,
"is_galaxy": false,
"is_custom_galaxy": false,
"local_only": false,
"local": 0,
"relationship_type": null
}
],
"_allTags": [
{
"id": "299",
"name": "misp-galaxy:country=\"belgium\"",
"colour": "#0088cc",
"exportable": true,
"user_id": "0",
"hide_tag": false,
"numerical_value": null,
"is_galaxy": true,
"is_custom_galaxy": false,
"local_only": false,
"local": 0,
"relationship_type": null,
"inherited": true
},
{
"id": "79",
"name": "tlp:green",
"colour": "#339900",
"exportable": true,
"user_id": "0",
"hide_tag": false,
"numerical_value": null,
"is_galaxy": false,
"is_custom_galaxy": false,
"local_only": false,
"local": 0,
"relationship_type": null,
"inherited": true
},
{
"id": "137",
"name": "PAP:AMBER",
"colour": "#ffa800",
"exportable": true,
"user_id": "0",
"hide_tag": false,
"numerical_value": null,
"is_galaxy": false,
"is_custom_galaxy": false,
"local_only": false,
"local": 0,
"relationship_type": null,
"inherited": false
}
]
}
],
"ShadowAttribute": [],
"RelatedEvent": [
{
"Event": {
"id": "43",
"date": "2022-12-15",
"threat_level_id": "1",
"info": "Fake event WF",
"published": false,
"uuid": "3a928b41-a9ec-4252-ab9f-d1859dbfc14a",
"analysis": "0",
"timestamp": "1683116434",
"distribution": "1",
"org_id": "1",
"orgc_id": "1",
"Org": {
"id": "1",
"name": "ORGNAME",
"uuid": "c5de83b4-36ba-49d6-9530-2a315caeece6"
},
"Orgc": {
"id": "1",
"name": "ORGNAME",
"uuid": "c5de83b4-36ba-49d6-9530-2a315caeece6"
}
}
}
],
"Galaxy": [
{
"id": "4",
"uuid": "84668357-5a8c-4bdd-9f0f-6b50b2aee4c1",
"name": "Country",
"type": "country",
"description": "Country meta information based on the database provided by geonames.org.",
"version": "1",
"icon": "globe",
"namespace": "misp",
"enabled": true,
"local_only": false,
"GalaxyCluster": [
{
"id": "1703",
"uuid": "84668357-5a8c-4bdd-9f0f-6b50b242454c",
"collection_uuid": "84668357-5a8c-4bdd-9f0f-6b50b2aee4c1",
"type": "country",
"value": "belgium",
"tag_name": "misp-galaxy:country=\"belgium\"",
"description": "Belgium",
"galaxy_id": "4",
"source": "MISP Project",
"authors": [
"geonames.org"
],
"version": "1",
"distribution": "3",
"sharing_group_id": null,
"org_id": "0",
"orgc_id": "0",
"default": true,
"locked": false,
"extends_uuid": "",
"extends_version": "0",
"published": false,
"deleted": false,
"GalaxyClusterRelation": [],
"Org": {
"id": "0",
"name": "MISP",
"date_created": "",
"date_modified": "",
"description": "Automatically generated MISP organisation",
"type": "",
"nationality": "Not specified",
"sector": "",
"created_by": "0",
"uuid": "0",
"contacts": "",
"local": true,
"restricted_to_domain": [],
"landingpage": null
},
"Orgc": {
"id": "0",
"name": "MISP",
"date_created": "",
"date_modified": "",
"description": "Automatically generated MISP organisation",
"type": "",
"nationality": "Not specified",
"sector": "",
"created_by": "0",
"uuid": "0",
"contacts": "",
"local": true,
"restricted_to_domain": [],
"landingpage": null
},
"meta": {
"Capital": [
"Brussels"
],
"Continent": [
"EU"
],
"CurrencyCode": [
"EUR"
],
"CurrencyName": [
"Euro"
],
"ISO": [
"BE"
],
"ISO3": [
"BEL"
],
"Languages": [
"nl-BE,fr-BE,de-BE"
],
"Population": [
"10403000"
],
"tld": [
".be"
]
},
"tag_id": 299,
"event_tag_id": "380",
"local": false,
"relationship_type": false
}
]
}
],
"Object": [
{
"id": "111",
"name": "url",
"meta-category": "network",
"description": "url object describes an url along with its normalized field (like extracted using faup parsing library) and its metadata.",
"template_uuid": "60efb77b-40b5-4c46-871b-ed1ed999fce5",
"template_version": "9",
"event_id": "64",
"uuid": "384ebae1-c97d-48a4-9efb-94a945c4860f",
"timestamp": "1683117902",
"distribution": "5",
"sharing_group_id": "0",
"comment": "",
"deleted": false,
"first_seen": null,
"last_seen": null,
"ObjectReference": [],
"Attribute": [
{
"id": "1696",
"type": "url",
"category": "Network activity",
"to_ids": true,
"uuid": "a915bbbe-2639-4d9b-83a2-abd58b8e5498",
"event_id": "64",
"distribution": "5",
"timestamp": "1683117902",
"comment": "",
"sharing_group_id": "0",
"deleted": false,
"disable_correlation": false,
"object_id": "111",
"object_relation": "url",
"first_seen": null,
"last_seen": null,
"value": "https://www.misp-project.org/",
"Galaxy": [],
"ShadowAttribute": [],
"_allTags": [
{
"id": "299",
"name": "misp-galaxy:country=\"belgium\"",
"colour": "#0088cc",
"exportable": true,
"user_id": "0",
"hide_tag": false,
"numerical_value": null,
"is_galaxy": true,
"is_custom_galaxy": false,
"local_only": false,
"local": 0,
"relationship_type": null,
"inherited": true
},
{
"id": "79",
"name": "tlp:green",
"colour": "#339900",
"exportable": true,
"user_id": "0",
"hide_tag": false,
"numerical_value": null,
"is_galaxy": false,
"is_custom_galaxy": false,
"local_only": false,
"local": 0,
"relationship_type": null,
"inherited": true
}
]
}
]
}
],
"EventReport": [],
"CryptographicKey": [],
"Tag": [
{
"id": "299",
"name": "misp-galaxy:country=\"belgium\"",
"colour": "#0088cc",
"exportable": true,
"user_id": "0",
"hide_tag": false,
"numerical_value": null,
"is_galaxy": true,
"is_custom_galaxy": false,
"local_only": false,
"local": 0,
"relationship_type": null
},
{
"id": "79",
"name": "tlp:green",
"colour": "#339900",
"exportable": true,
"user_id": "0",
"hide_tag": false,
"numerical_value": null,
"is_galaxy": false,
"is_custom_galaxy": false,
"local_only": false,
"local": 0,
"relationship_type": null
}
],
"_AttributeFlattened": [
{
"id": "1695",
"type": "ip-src",
"category": "Network activity",
"to_ids": true,
"uuid": "9ac36927-d874-4094-bf4c-f922c1e9cc35",
"event_id": "64",
"distribution": "5",
"timestamp": "1683117902",
"comment": "",
"sharing_group_id": "0",
"deleted": false,
"disable_correlation": false,
"object_id": "0",
"object_relation": null,
"first_seen": null,
"last_seen": null,
"value": "8.8.8.8",
"Galaxy": [],
"ShadowAttribute": [],
"Tag": [
{
"id": "137",
"name": "PAP:AMBER",
"colour": "#ffa800",
"exportable": true,
"user_id": "0",
"hide_tag": false,
"numerical_value": null,
"is_galaxy": false,
"is_custom_galaxy": false,
"local_only": false,
"local": 0,
"relationship_type": null
}
],
"_allTags": [
{
"id": "299",
"name": "misp-galaxy:country=\"belgium\"",
"colour": "#0088cc",
"exportable": true,
"user_id": "0",
"hide_tag": false,
"numerical_value": null,
"is_galaxy": true,
"is_custom_galaxy": false,
"local_only": false,
"local": 0,
"relationship_type": null,
"inherited": true
},
{
"id": "79",
"name": "tlp:green",
"colour": "#339900",
"exportable": true,
"user_id": "0",
"hide_tag": false,
"numerical_value": null,
"is_galaxy": false,
"is_custom_galaxy": false,
"local_only": false,
"local": 0,
"relationship_type": null,
"inherited": true
},
{
"id": "137",
"name": "PAP:AMBER",
"colour": "#ffa800",
"exportable": true,
"user_id": "0",
"hide_tag": false,
"numerical_value": null,
"is_galaxy": false,
"is_custom_galaxy": false,
"local_only": false,
"local": 0,
"relationship_type": null,
"inherited": false
}
]
},
{
"id": "1696",
"type": "url",
"category": "Network activity",
"to_ids": true,
"uuid": "a915bbbe-2639-4d9b-83a2-abd58b8e5498",
"event_id": "64",
"distribution": "5",
"timestamp": "1683117902",
"comment": "",
"sharing_group_id": "0",
"deleted": false,
"disable_correlation": false,
"object_id": "111",
"object_relation": "url",
"first_seen": null,
"last_seen": null,
"value": "https://www.misp-project.org/",
"Galaxy": [],
"ShadowAttribute": [],
"_allTags": [
{
"id": "299",
"name": "misp-galaxy:country=\"belgium\"",
"colour": "#0088cc",
"exportable": true,
"user_id": "0",
"hide_tag": false,
"numerical_value": null,
"is_galaxy": true,
"is_custom_galaxy": false,
"local_only": false,
"local": 0,
"relationship_type": null,
"inherited": true
},
{
"id": "79",
"name": "tlp:green",
"colour": "#339900",
"exportable": true,
"user_id": "0",
"hide_tag": false,
"numerical_value": null,
"is_galaxy": false,
"is_custom_galaxy": false,
"local_only": false,
"local": 0,
"relationship_type": null,
"inherited": true
}
]
}
]
}
}
</pre>
</div>
<div class="tab-pane" id="modal-blueprint">
<h3><?= __('Blueprints') ?></h3>
<ul>

9
app/View/Elements/dashboard/Widgets/BarChart.ctp
View File

@ -15,10 +15,15 @@
if (!empty($data['logarithmic'])) {
$value = $data['logarithmic'][$entry];
}
$shortlabel = $entry;
if (mb_strlen($shortlabel) > 30) {
$shortlabel = mb_substr($shortlabel, 0, 30) . '...';
}
echo sprintf(
'<tr><td style="%s">%s</td><td style="%s">%s</td></tr>',
'text-align:right;width:33%;white-space:nowrap;',
'<tr><td style="%s" title="%s">%s</td><td style="%s">%s</td></tr>',
'text-align:right;width:35em;white-space:nowrap;',
h($entry),
h($shortlabel),
'width:100%',
sprintf(
'<div title="%s" style="%s">%s%s</div>',

2
app/View/Elements/dashboard/Widgets/Index.ctp
View File

@ -2,10 +2,10 @@
echo $this->element('genericElements/IndexTable/index_table', [
'data' => [
'data' => $data['data'],
'description' => empty($data['description']) ? false : $data['description'],
'top_bar' => [],
'fields' => $data['fields'],
'title' => false,
'description' => false,
'pull' => 'right',
'skip_pagination' => true,
'actions' => []

21
app/View/Elements/dashboard/Widgets/SimpleList.ctp
View File

@ -18,12 +18,27 @@
$element['value'] = h($element['value']);
}
}
$change = '';
if (!empty($element['change'])) {
$change = (int)$element['change'];
if ($change > 0) {
$change = '<span class="green bold"> (+' . $change . ')</span>';
} else {
$change = '<span class="red bold"> (-' . $change . ')</span>';
}
}
if (!empty($element['html_title'])) {
$title = $element['html_title'];
} else {
$title = h($element['title']);
}
echo sprintf(
'<div><span class="bold">%s</span>: <span class="%s">%s</span>%s</div>',
h($element['title']),
'<div><span class="bold">%s</span>: <span class="%s">%s</span>%s%s</div>',
$title,
empty($element['class']) ? 'blue' : h($element['class']),
!isset($element['value']) ? '' : $element['value'],
empty($element['html']) ? '' : $element['html']
empty($element['html']) ? '' : $element['html'],
$change
);
}
}

1
app/View/Elements/dashboard/Widgets/WorldMap.ctp
View File

@ -20,7 +20,6 @@
), true);
}
?>
<div id="world-map-<?= $randomNumber ?>" style="width: 600px; height: 400px"></div>
<script>
(function() { // variables and functions have their own scope (no override)

2
app/View/Elements/eventdiscussion.ctp
View File

@ -149,7 +149,7 @@
));
?>
</fieldset>
<button class="btn btn-primary" onclick="submitMessageForm('<?php echo $url;?>', 'PostViewForm', 'top'); return false;"><?php echo __('Send comment');?></button>
<button class="btn btn-primary" onclick="submitMessageForm('<?php echo $url;?>'); return false;"><?php echo __('Send comment');?></button>
<?php
echo $this->Form->end();
?>

31
app/View/Elements/genericElements/IndexTable/Fields/allow_deny_list.ctp Normal file
View File

@ -0,0 +1,31 @@
<?php
$data = Hash::extract($row, $field['data_path']);
$setup = [
'allow' => [
'name' => __('Allowed'),
'color' => 'green'
],
'deny' => [
'name' => __('Denied'),
'color' => 'red'
]
];
foreach ($setup as $state => $settings) {
if (!empty($data[$state])) {
echo sprintf(
'<div class="bold %s">%s</div>',
$settings['color'],
$settings['name']
);
foreach ($data[$state] as $k => $element) {
$data[$state][$k] = sprintf(
'<span class="%s">%s</span>',
$settings['color'],
h($element)
);
}
echo implode('<br />', $data[$state]);
}
}
?>

1
app/View/Events/automation.ctp
View File

@ -372,6 +372,7 @@
<b>searchdistribution</b>: <?php echo __('Filters on the distribution level [0,1,2,3] - negatable');?><br />
<b>searchanalysis</b>: <?php echo __('Filters on the given analysis phase of the event [0,1,2] - negatable');?><br />
<b>searchattribute</b>: <?php echo __('Filters on a contained attribute value - negatable');?><br />
<b>searchvalue</b>: <?php echo __('Filter exact matches on the attribute value');?><br />
<b>searchorg</b>: <?php echo __('Filters on the creator organisation - negatable');?><br />
<b>searchemail</b>: <?php echo __('Filters on the creator user\'s email address (admin only) - negatable');?><br />
<b>searchDatefrom</b>: <?php echo __('Filters on the date, anything newer than the given date in YYYY-MM-DD format is taken - non-negatable');?><br />

56
app/View/Events/upload_stix.ctp
View File

@ -1,6 +1,6 @@
<div class="events form">
<?php
echo $this->Form->create('Event', array('type' => 'file'));
echo $this->Form->create('Event', array('type' => 'file'));
?>
<fieldset>
<legend><?= __('Import STIX %s file', $stix_version); ?></legend>
@ -11,6 +11,34 @@
));
?>
<div class="input clear"></div>
<?php
$distributionFormInfo = $this->element(
'genericElements/Form/formInfo',
[
'field' => [
'field' => 'distribution'
],
'modelForForm' => 'Event',
'fieldDesc' => $fieldDesc['distribution'],
]
);
echo $this->Form->input('distribution', array(
'options' => $distributionLevels,
'label' => __('Distribution ') . $distributionFormInfo,
'selected' => $initialDistribution,
));
?>
<div id="SGContainer" style="display:none;">
<?php
if (!empty($sharingGroups)) {
echo $this->Form->input('sharing_group_id', array(
'options' => array($sharingGroups),
'label' => __('Sharing Group'),
));
}
?>
</div>
<div class="input clear"></div>
<?php
echo $this->Form->input('publish', array(
'checked' => false,
@ -23,6 +51,20 @@
'checked' => true,
'label' => __('Include the original imported file as attachment')
));
if ($me['Role']['perm_site_admin'] || $me['Role']['perm_galaxy_editor']) {
echo '<div class="input clear"></div>';
echo $this->Form->input('galaxies_parsing', array(
'checked' => false,
'label' => __('Use Galaxies 2.0')
));
}
if ($me['Role']['perm_site_admin']) {
echo '<div class="input clear"></div>';
echo $this->Form->input('debug', array(
'checked' => true,
'label' => __('Advanced conversion debugging')
));
}
?>
</fieldset>
<?php
@ -33,3 +75,15 @@
<?php
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event-collection', 'menuItem' => 'import_from'));
?>
<script>
$(function(){
$('#EventDistribution').change(function() {
if ($(this).val() == 4) {
$('#SGContainer').show();
} else {
$('#SGContainer').hide();
}
}).change();
});
</script>

3
app/View/Logs/index.ctp
View File

@ -22,6 +22,9 @@
}
}
echo ":</h4>";
$this->LightPaginator->options([
'url' => ['search']
]);
}
?>
<div class="pagination">

11
app/View/OrgBlocklists/index.ctp
View File

@ -32,6 +32,17 @@ echo $this->element('genericElements/IndexTable/scaffold', [
'data_path' => 'OrgBlocklist.comment',
'class' => 'bitwider'
],
[
'name' => 'Blocked amount',
'sort' => 'OrgBlocklist.blocked_data.blocked_amount',
'data_path' => 'OrgBlocklist.blocked_data.blocked_amount',
],
[
'name' => 'Blocked last time ',
'sort' => 'OrgBlocklist.blocked_data.blocked_last_time',
'data_path' => 'OrgBlocklist.blocked_data.blocked_last_time',
'element' => 'datetime'
],
],
'title' => empty($ajax) ? __('Organisation Blocklists') : false,

6
app/View/TaxiiServers/index.ctp
View File

@ -11,7 +11,7 @@
'children' => [
'data' => [
'type' => 'simple',
'text' => __('Add TaxiiServer'),
'text' => __('Add TAXII Server'),
'class' => 'btn btn-primary',
'onClick' => 'openGenericModal',
'onClickParams' => [
@ -70,8 +70,8 @@
'data_path' => 'TaxiiServer.description'
]
],
'title' => empty($ajax) ? __('Linked Taxii Servers') : false,
'description' => empty($ajax) ? __('You can connect your MISP to one or several Taxii servers to push data to using a set of filters.') : false,
'title' => empty($ajax) ? __('Linked TAXII Servers') : false,
'description' => empty($ajax) ? __('You can connect your MISP to one or several TAXII servers to push data to using a set of filters.') : false,
'actions' => [
[
'onclick' => sprintf(

3
app/View/Workflows/editor.ctp
View File

@ -272,7 +272,8 @@ $debugEnabled = !empty($selectedWorkflow['Workflow']['debug_enabled']);
<?php
echo $this->element('genericElements/assetLoader', [
'css' => ['drawflow.min', 'drawflow-default'],
'js' => ['jquery-ui.min', 'drawflow.min', 'doT', 'moment.min', 'viselect.cjs'],
'js' => ['jquery-ui.min', 'drawflow', 'doT', 'moment.min', 'viselect.cjs'],
// 'js' => ['jquery-ui.min', 'drawflow.min', 'doT', 'moment.min', 'viselect.cjs'],
]);
echo $this->element('genericElements/assetLoader', [
'css' => ['workflows-editor'],

2
app/files/scripts/cti-python-stix2

@ -1 +1 @@
Subproject commit 625576622f3e7f2541d2e788d70ed4997e134837
Subproject commit 344454e399853c0d969e4efb048e919e04b21140

2
app/files/scripts/misp-stix

@ -1 +1 @@
Subproject commit 32b57c7e75509feed1b7507b5366a13885e68543
Subproject commit 1a1dd4819f6d80492d0871ea7aa6f8fd57ea1b7e

47
app/files/scripts/stix2/misp2stix2.py
View File

@ -34,15 +34,19 @@ from stix2.base import STIXJSONEncoder
from misp_stix_converter import MISPtoSTIX20Parser, MISPtoSTIX21Parser
def _handle_errors(errors: dict):
for identifier, values in errors.items():
def _handle_messages(field: str, feature: dict):
for identifier, values in feature.items():
values = '\n - '.join(values)
if identifier != 'attributes collection':
identifier = f'MISP event {identifier}'
print(f'Errors encountered while parsing {identifier}:\n - {values}', file=sys.stderr)
print(
f'{field} encountered while parsing {identifier}:\n - {values}',
file=sys.stderr
)
def _process_misp_files(version: str, input_names: Union[list, None], debug: bool):
def _process_misp_files(
version: str, input_names: Union[list, None], debug: bool):
if input_names is None:
print(json.dumps({'error': 'No input file provided.'}))
return
@ -51,10 +55,13 @@ def _process_misp_files(version: str, input_names: Union[list, None], debug: boo
for name in input_names:
parser.parse_json_content(name)
with open(f'{name}.out', 'wt', encoding='utf-8') as f:
f.write(f'{json.dumps(parser.stix_objects, cls=STIXJSONEncoder)}')
errors = parser.errors
if errors:
_handle_errors(errors)
f.write(
f'{json.dumps(parser.stix_objects, cls=STIXJSONEncoder)}'
)
if parser.errors:
_handle_messages('Errors', parser.errors)
if debug and parser.warnings:
_handle_messages('Warnings', parser.warnings)
print(json.dumps({'success': 1}))
except Exception as e:
print(json.dumps({'error': e.__str__()}))
@ -63,11 +70,27 @@ def _process_misp_files(version: str, input_names: Union[list, None], debug: boo
if __name__ == "__main__":
argparser = argparse.ArgumentParser(description='Export MISP into STIX2.')
argparser.add_argument('-v', '--version', default='2.0', choices=['2.0', '2.1'], help='STIX version (2.0 or 2.1).')
argparser.add_argument('-i', '--input', nargs='+', help='Input file(s) containing MISP standard format.')
argparser.add_argument('-d', '--debug', action='store_true', help='Allow debug mode with warnings.')
argparser.add_argument(
'-v', '--version', default='2.1', choices=['2.0', '2.1'],
help='STIX version (2.0 or 2.1).'
)
argparser.add_argument(
'-i', '--input', nargs='+', required=True,
help='Input file(s) containing MISP standard format.'
)
argparser.add_argument(
'-d', '--debug', action='store_true',
help='Allow debug mode with warnings.'
)
try:
args = argparser.parse_args()
_process_misp_files(args.version, args.input, args.debug)
except SystemExit:
print(json.dumps({'error': 'Arguments error, please check you entered a valid version and provided input file names.'}))
print(
json.dumps(
{
'error': 'Arguments error, please check you entered a valid'
' version and provided input file names.'
}
)
)

2204
app/files/scripts/stix2/stix2misp.py
View File

File diff suppressed because it is too large Load Diff

463
app/files/scripts/stix2/stix2misp_mapping.py
View File

@ -1,463 +0,0 @@
################################################################################
# ATTRIBUTES AND OBJECTS MAPPING #
################################################################################
attributes_mapping = {
'filename': '_parse_name',
'ip-src': '_parse_ip_value',
'ip-dst': '_parse_ip_value',
'hostname': '_parse_domain_value',
'domain': '_parse_domain_value',
'domain|ip': '_parse_domain_ip_attribute',
'email-src': '_parse_email_value',
'email-dst': '_parse_email_value',
'email-attachment': '_parse_name',
'url': '_parse_url_value',
'regkey': '_parse_regkey_attribute',
'regkey|value': '_parse_regkey_value',
'malware-sample': '_parse_malware_sample',
'mutex': '_parse_name',
'uri': '_parse_url_value',
'port': '_parse_port',
'ip-dst|port': '_parse_network_attribute',
'ip-src|port': '_parse_network_attribute',
'hostname|port': '_parse_network_attribute',
'email-reply-to': '_parse_email_reply_to',
'attachment': '_parse_attachment',
'mac-address': '_parse_mac_value',
'AS': '_parse_number',
'link': '_parse_url_value'
}
attributes_type_mapping = {
'md5': '_parse_hash',
'sha1': '_parse_hash',
'sha256': '_parse_hash',
'filename|md5': '_parse_filename_hash',
'filename|sha1': '_parse_filename_hash',
'filename|sha256': '_parse_filename_hash',
'email-subject': '_parse_email_message',
'email-body': '_parse_email_message',
'authentihash': '_parse_hash',
'ssdeep': '_parse_hash',
'imphash': '_parse_hash',
'pehash': '_parse_hash',
'impfuzzy': '_parse_hash',
'sha224': '_parse_hash',
'sha384': '_parse_hash',
'sha512': '_parse_hash',
'sha512/224': '_parse_hash',
'sha512/256': '_parse_hash',
'tlsh': '_parse_hash',
'cdhash': '_parse_hash',
'filename|authentihash': '_parse_filename_hash',
'filename|ssdeep': '_parse_filename_hash',
'filename|imphash': '_parse_filename_hash',
'filename|impfuzzy': '_parse_filename_hash',
'filename|pehash': '_parse_filename_hash',
'filename|sha224': '_parse_filename_hash',
'filename|sha384': '_parse_filename_hash',
'filename|sha512': '_parse_filename_hash',
'filename|sha512/224': '_parse_filename_hash',
'filename|sha512/256': '_parse_filename_hash',
'filename|tlsh': '_parse_filename_hash',
'x509-fingerprint-md5': '_parse_x509_attribute',
'x509-fingerprint-sha1': '_parse_x509_attribute',
'x509-fingerprint-sha256': '_parse_x509_attribute'
}
objects_mapping = {
'asn': {
'observable': 'parse_asn_observable',
'pattern': 'parse_asn_pattern'},
'credential': {
'observable': 'parse_credential_observable',
'pattern': 'parse_credential_pattern'},
'domain-ip': {
'observable': 'parse_domain_ip_observable',
'pattern': 'parse_domain_ip_pattern'},
'email': {
'observable': 'parse_email_observable',
'pattern': 'parse_email_pattern'},
'file': {
'observable': 'parse_file_observable',
'pattern': 'parse_file_pattern'},
'ip-port': {
'observable': 'parse_ip_port_observable',
'pattern': 'parse_ip_port_pattern'},
'network-connection': {
'observable': 'parse_network_connection_observable',
'pattern': 'parse_network_connection_pattern'},
'network-socket': {
'observable': 'parse_network_socket_observable',
'pattern': 'parse_network_socket_pattern'},
'process': {
'observable': 'parse_process_observable',
'pattern': 'parse_process_pattern'},
'registry-key': {
'observable': 'parse_regkey_observable',
'pattern': 'parse_regkey_pattern'},
'url': {
'observable': 'parse_url_observable',
'pattern': 'parse_url_pattern'},
'user-account': {
'observable': 'parse_user_account_observable',
'pattern': 'parse_user_account_pattern'},
'WindowsPEBinaryFile': {
'observable': 'parse_pe_observable',
'pattern': 'parse_pe_pattern'},
'x509': {
'observable': 'parse_x509_observable',
'pattern': 'parse_x509_pattern'}
}
observable_mapping = {
('artifact', 'file'): 'parse_file_observable',
('artifact', 'directory', 'file'): 'parse_file_observable',
('artifact', 'email-addr', 'email-message', 'file'): 'parse_email_observable',
('autonomous-system',): 'parse_asn_observable',
('autonomous-system', 'ipv4-addr'): 'parse_asn_observable',
('autonomous-system', 'ipv6-addr'): 'parse_asn_observable',
('autonomous-system', 'ipv4-addr', 'ipv6-addr'): 'parse_asn_observable',
('directory', 'file'): 'parse_file_observable',
('domain-name',): 'parse_domain_ip_observable',
('domain-name', 'ipv4-addr'): 'parse_domain_ip_observable',
('domain-name', 'ipv6-addr'): 'parse_domain_ip_observable',
('domain-name', 'ipv4-addr', 'ipv6-addr'): 'parse_domain_ip_observable',
('domain-name', 'ipv4-addr', 'network-traffic'): 'parse_domain_ip_network_traffic_observable',
('domain-name', 'ipv6-addr', 'network-traffic'): 'parse_domain_ip_network_traffic_observable',
('domain-name', 'ipv4-addr', 'ipv6-addr', 'network-traffic'): 'parse_domain_ip_network_traffic_observable',
('domain-name', 'network-traffic'): 'parse_domain_network_traffic_observable',
('domain-name', 'network-traffic', 'url'): 'parse_url_observable',
('email-addr',): 'parse_email_address_observable',
('email-addr', 'email-message'): 'parse_email_observable',
('email-addr', 'email-message', 'file'): 'parse_email_observable',
('email-message',): 'parse_email_observable',
('file',): 'parse_file_observable',
('file', 'process'): 'parse_process_observable',
('ipv4-addr',): 'parse_ip_address_observable',
('ipv6-addr',): 'parse_ip_address_observable',
('ipv4-addr', 'network-traffic'): 'parse_ip_network_traffic_observable',
('ipv6-addr', 'network-traffic'): 'parse_ip_network_traffic_observable',
('ipv4-addr', 'ipv6-addr', 'network-traffic'): 'parse_ip_network_traffic_observable',
('mac-addr',): 'parse_mac_address_observable',
('mutex',): 'parse_mutex_observable',
('process',): 'parse_process_observable',
('x509-certificate',): 'parse_x509_observable',
('url',): 'parse_url_observable',
('user-account',): 'parse_user_account_observable',
('windows-registry-key',): 'parse_regkey_observable'
}
pattern_mapping = {
('artifact', 'file'): 'parse_file_pattern',
('artifact', 'directory', 'file'): 'parse_file_pattern',
('autonomous-system', ): 'parse_as_pattern',
('autonomous-system', 'ipv4-addr'): 'parse_as_pattern',
('autonomous-system', 'ipv6-addr'): 'parse_as_pattern',
('autonomous-system', 'ipv4-addr', 'ipv6-addr'): 'parse_as_pattern',
('directory',): 'parse_file_pattern',
('directory', 'file'): 'parse_file_pattern',
('domain-name',): 'parse_domain_ip_port_pattern',
('domain-name', 'ipv4-addr'): 'parse_domain_ip_port_pattern',
('domain-name', 'ipv6-addr'): 'parse_domain_ip_port_pattern',
('domain-name', 'ipv4-addr', 'ipv6-addr'): 'parse_domain_ip_port_pattern',
('domain-name', 'ipv4-addr', 'url'): 'parse_url_pattern',
('domain-name', 'ipv6-addr', 'url'): 'parse_url_pattern',
('domain-name', 'ipv4-addr', 'ipv6-addr', 'url'): 'parse_url_pattern',
('domain-name', 'network-traffic'): 'parse_domain_ip_port_pattern',
('domain-name', 'network-traffic', 'url'): 'parse_url_pattern',
('email-addr',): 'parse_email_address_pattern',
('email-message',): 'parse_email_message_pattern',
('file',): 'parse_file_pattern',
('ipv4-addr',): 'parse_ip_address_pattern',
('ipv6-addr',): 'parse_ip_address_pattern',
('ipv4-addr', 'ipv6-addr'): 'parse_ip_address_pattern',
('mac-addr',): 'parse_mac_address_pattern',
('mutex',): 'parse_mutex_pattern',
('network-traffic',): 'parse_network_traffic_pattern',
('process',): 'parse_process_pattern',
('url',): 'parse_url_pattern',
('user-account',): 'parse_user_account_pattern',
('windows-registry-key',): 'parse_regkey_pattern',
('x509-certificate',): 'parse_x509_pattern'
}
pattern_forbidden_relations = (' LIKE ', ' FOLLOWEDBY ', ' MATCHES ', ' ISSUBSET ', ' ISSUPERSET ', ' REPEATS ')
single_attribute_fields = ('type', 'value', 'to_ids')
################################################################################
# OBSERVABLE OBJECTS AND PATTERNS MAPPING. #
################################################################################
address_family_attribute_mapping = {'type': 'text','object_relation': 'address-family'}
as_number_attribute_mapping = {'type': 'AS', 'object_relation': 'asn'}
attack_pattern_id_attribute = {'type': 'text', 'object_relation': 'id'}
attack_pattern_references_attribute = {'type': 'link', 'object_relation': 'references'}
description_attribute_mapping = {'type': 'text', 'object_relation': 'description'}
asn_subnet_attribute_mapping = {'type': 'ip-src', 'object_relation': 'subnet-announced'}
cc_attribute_mapping = {'type': 'email-dst', 'object_relation': 'cc'}
credential_attribute_mapping = {'type': 'text', 'object_relation': 'password'}
data_attribute_mapping = {'type': 'text', 'object_relation': 'data'}
data_type_attribute_mapping = {'type': 'text', 'object_relation': 'data-type'}
domain_attribute_mapping = {'type': 'domain', 'object_relation': 'domain'}
domain_family_attribute_mapping = {'type': 'text', 'object_relation': 'domain-family'}
dst_port_attribute_mapping = {'type': 'port', 'object_relation': 'dst-port'}
email_attachment_attribute_mapping = {'type': 'email-attachment', 'object_relation': 'attachment'}
email_date_attribute_mapping = {'type': 'datetime', 'object_relation': 'send-date'}
email_subject_attribute_mapping = {'type': 'email-subject', 'object_relation': 'subject'}
encoding_attribute_mapping = {'type': 'text', 'object_relation': 'file-encoding'}
end_datetime_attribute_mapping = {'type': 'datetime', 'object_relation': 'last-seen'}
entropy_mapping = {'type': 'float', 'object_relation': 'entropy'}
filename_attribute_mapping = {'type': 'filename', 'object_relation': 'filename'}
from_attribute_mapping = {'type': 'email-src', 'object_relation': 'from'}
imphash_mapping = {'type': 'imphash', 'object_relation': 'imphash'}
id_attribute_mapping = {'type': 'text', 'object_relation': 'id'}
ip_attribute_mapping = {'type': 'ip-dst', 'object_relation': 'ip'}
issuer_attribute_mapping = {'type': 'text', 'object_relation': 'issuer'}
key_attribute_mapping = {'type': 'regkey', 'object_relation': 'key'}
malware_sample_attribute_mapping = {'type': 'malware-sample', 'object_relation': 'malware-sample'}
mime_type_attribute_mapping = {'type': 'mime-type', 'object_relation': 'mimetype'}
modified_attribute_mapping = {'type': 'datetime', 'object_relation': 'last-modified'}
name_attribute_mapping = {'type': 'text', 'object_relation': 'name'}
network_traffic_ip = {'type': 'ip-{}', 'object_relation': 'ip-{}'}
number_sections_mapping = {'type': 'counter', 'object_relation': 'number-sections'}
password_mapping = {'type': 'text', 'object_relation': 'password'}
path_attribute_mapping = {'type': 'text', 'object_relation': 'path'}
pe_type_mapping = {'type': 'text', 'object_relation': 'type'}
pid_attribute_mapping = {'type': 'text', 'object_relation': 'pid'}
process_command_line_mapping = {'type': 'text', 'object_relation': 'command-line'}
process_creation_time_mapping = {'type': 'datetime', 'object_relation': 'creation-time'}
process_image_mapping = {'type': 'filename', 'object_relation': 'image'}
process_name_mapping = {'type': 'text', 'object_relation': 'name'}
regkey_name_attribute_mapping = {'type': 'text', 'object_relation': 'name'}
references_attribute_mapping = {'type': 'link', 'object_relation': 'references'}
reply_to_attribute_mapping = {'type': 'email-reply-to', 'object_relation': 'reply-to'}
screenshot_attribute_mapping = {'type': 'attachment', 'object_relation': 'screenshot'}
section_name_mapping = {'type': 'text', 'object_relation': 'name'}
serial_number_attribute_mapping = {'type': 'text', 'object_relation': 'serial-number'}
size_attribute_mapping = {'type': 'size-in-bytes', 'object_relation': 'size-in-bytes'}
src_port_attribute_mapping = {'type': 'port', 'object_relation': 'src-port'}
start_datetime_attribute_mapping = {'type': 'datetime', 'object_relation': 'first-seen'}
state_attribute_mapping = {'type': 'text', 'object_relation': 'state'}
summary_attribute_mapping = {'type': 'text', 'object_relation': 'summary'}
to_attribute_mapping = {'type': 'email-dst', 'object_relation': 'to'}
url_attribute_mapping = {'type': 'url', 'object_relation': 'url'}
url_port_attribute_mapping = {'type': 'port', 'object_relation': 'port'}
user_id_mapping = {'type': 'text', 'object_relation': 'username'}
x_mailer_attribute_mapping = {'type': 'email-x-mailer', 'object_relation': 'x-mailer'}
x509_md5_attribute_mapping = {'type': 'x509-fingerprint-md5', 'object_relation': 'x509-fingerprint-md5'}
x509_sha1_attribute_mapping = {'type': 'x509-fingerprint-sha1', 'object_relation': 'x509-fingerprint-sha1'}
x509_sha256_attribute_mapping = {'type': 'x509-fingerprint-sha256', 'object_relation': 'x509-fingerprint-sha256'}
x509_spka_attribute_mapping = {'type': 'text', 'object_relation': 'pubkey-info-algorithm'} # x509 subject public key algorithm
x509_spke_attribute_mapping = {'type': 'text', 'object_relation': 'pubkey-info-exponent'} # x509 subject public key exponent
x509_spkm_attribute_mapping = {'type': 'text', 'object_relation': 'pubkey-info-modulus'} # x509 subject public key modulus
x509_subject_attribute_mapping = {'type': 'text', 'object_relation': 'subject'}
x509_version_attribute_mapping = {'type': 'text', 'object_relation': 'version'}
x509_vna_attribute_mapping = {'type': 'datetime', 'object_relation': 'validity-not-after'} # x509 validity not after
x509_vnb_attribute_mapping = {'type': 'datetime', 'object_relation': 'validity-not-before'} # x509 validity not before
asn_mapping = {'number': as_number_attribute_mapping,
'autonomous-system:number': as_number_attribute_mapping,
'name': description_attribute_mapping,
'autonomous-system:name': description_attribute_mapping,
'ipv4-addr': asn_subnet_attribute_mapping,
'ipv6-addr': asn_subnet_attribute_mapping,
'ipv4-addr:value': asn_subnet_attribute_mapping,
'ipv6-addr:value': asn_subnet_attribute_mapping}
attack_pattern_mapping = {'name': name_attribute_mapping,
'description': summary_attribute_mapping}
attack_pattern_references_mapping = {'mitre-attack': references_attribute_mapping,
'capec': id_attribute_mapping}
course_of_action_mapping = {'description': description_attribute_mapping,
'name': name_attribute_mapping}
credential_mapping = {'credential': credential_attribute_mapping,
'user-account:credential': credential_attribute_mapping,
'user_id': user_id_mapping,
'user-account:user_id': user_id_mapping}
domain_ip_mapping = {'domain-name': domain_attribute_mapping,
'domain-name:value': domain_attribute_mapping,
'ipv4-addr': ip_attribute_mapping,
'ipv6-addr': ip_attribute_mapping,
'ipv4-addr:value': ip_attribute_mapping,
'ipv6-addr:value': ip_attribute_mapping,
'domain-name:resolves_to_refs[*].value': ip_attribute_mapping,
'network-traffic:dst_port': dst_port_attribute_mapping,
'network-traffic:src_port': src_port_attribute_mapping}
email_mapping = {'date': email_date_attribute_mapping,
'email-message:date': email_date_attribute_mapping,
'email-message:to_refs[*].value': to_attribute_mapping,
'email-message:cc_refs[*].value': cc_attribute_mapping,
'subject': email_subject_attribute_mapping,
'email-message:subject': email_subject_attribute_mapping,
'X-Mailer': x_mailer_attribute_mapping,
'email-message:additional_header_fields.x_mailer': x_mailer_attribute_mapping,
'Reply-To': reply_to_attribute_mapping,
'email-message:additional_header_fields.reply_to': reply_to_attribute_mapping,
'email-message:from_ref.value': from_attribute_mapping,
'email-addr:value': to_attribute_mapping}
email_references_mapping = {'attachment': email_attachment_attribute_mapping,
'cc_refs': cc_attribute_mapping,
'from_ref': from_attribute_mapping,
'screenshot': screenshot_attribute_mapping,
'to_refs': to_attribute_mapping}
file_mapping = {'artifact:mime_type': mime_type_attribute_mapping,
'file:content_ref.mime_type': mime_type_attribute_mapping,
'mime_type': mime_type_attribute_mapping,
'file:mime_type': mime_type_attribute_mapping,
'name': filename_attribute_mapping,
'file:name': filename_attribute_mapping,
'name_enc': encoding_attribute_mapping,
'file:name_enc': encoding_attribute_mapping,
'file:parent_directory_ref.path': path_attribute_mapping,
'directory:path': path_attribute_mapping,
'size': size_attribute_mapping,
'file:size': size_attribute_mapping}
network_traffic_mapping = {'dst_port':dst_port_attribute_mapping,
'src_port': src_port_attribute_mapping,
'network-traffic:dst_port': dst_port_attribute_mapping,
'network-traffic:src_port': src_port_attribute_mapping}
ip_port_mapping = {'value': domain_attribute_mapping,
'domain-name:value': domain_attribute_mapping,
'network-traffic:dst_ref.value': {'type': 'ip-dst', 'object_relation': 'ip-dst'},
'network-traffic:src_ref.value': {'type': 'ip-src', 'object_relation': 'ip-src'}}
ip_port_mapping.update(network_traffic_mapping)
ip_port_references_mapping = {'domain-name': domain_attribute_mapping,
'ipv4-addr': network_traffic_ip,
'ipv6-addr': network_traffic_ip}
network_socket_extension_mapping = {'address_family': address_family_attribute_mapping,
"network-traffic:extensions.'socket-ext'.address_family": address_family_attribute_mapping,
'protocol_family': domain_family_attribute_mapping,
"network-traffic:extensions.'socket-ext'.protocol_family": domain_family_attribute_mapping,
'is_blocking': state_attribute_mapping,
"network-traffic:extensions.'socket-ext'.is_blocking": state_attribute_mapping,
'is_listening': state_attribute_mapping,
"network-traffic:extensions.'socket-ext'.is_listening": state_attribute_mapping}
network_traffic_references_mapping = {'domain-name': {'type': 'hostname', 'object_relation': 'hostname-{}'},
'ipv4-addr': network_traffic_ip,
'ipv6-addr': network_traffic_ip}
pe_mapping = {'pe_type': pe_type_mapping, 'number_of_sections': number_sections_mapping, 'imphash': imphash_mapping}
pe_section_mapping = {'name': section_name_mapping, 'size': size_attribute_mapping, 'entropy': entropy_mapping}
hash_types = ('MD5', 'SHA-1', 'SHA-256', 'SHA-224', 'SHA-384', 'SHA-512', 'ssdeep', 'tlsh')
for hash_type in hash_types:
misp_hash_type = hash_type.replace('-', '').lower()
attribute = {'type': misp_hash_type, 'object_relation': misp_hash_type}
file_mapping[hash_type] = attribute
file_mapping.update({f"file:hashes.'{feature}'": attribute for feature in (hash_type, misp_hash_type)})
file_mapping.update({f"file:hashes.{feature}": attribute for feature in (hash_type, misp_hash_type)})
pe_section_mapping[hash_type] = attribute
pe_section_mapping[misp_hash_type] = attribute
process_mapping = {'name': process_name_mapping,
'process:name': process_name_mapping,
'pid': pid_attribute_mapping,
'process:pid': pid_attribute_mapping,
'created': process_creation_time_mapping,
'process:created': process_creation_time_mapping,
'command_line': process_command_line_mapping,
'process:command_line': process_command_line_mapping,
'process:parent_ref.pid': {'type': 'text', 'object_relation': 'parent-pid'},
'process:child_refs[*].pid': {'type': 'text', 'object_relation': 'child-pid'},
'process:binary_ref.name': process_image_mapping}
child_process_reference_mapping = {'pid': {'type': 'text', 'object_relation': 'child-pid'}}
parent_process_reference_mapping = {'command_line': {'type': 'text', 'object_relation': 'parent-command-line'},
'pid': {'type': 'text', 'object_relation': 'parent-pid'},
'process-name': {'type': 'text', 'object_relation': 'parent-process-name'}}
regkey_mapping = {'data': data_attribute_mapping,
'windows-registry-key:values.data': data_attribute_mapping,
'data_type': data_type_attribute_mapping,
'windows-registry-key:values.data_type': data_type_attribute_mapping,
'modified': modified_attribute_mapping,
'windows-registry-key:modified': modified_attribute_mapping,
'name': regkey_name_attribute_mapping,
'windows-registry-key:values.name': regkey_name_attribute_mapping,
'key': key_attribute_mapping,
'windows-registry-key:key': key_attribute_mapping,
'windows-registry-key:value': {'type': 'text', 'object_relation': 'hive'}
}
url_mapping = {'url': url_attribute_mapping,
'url:value': url_attribute_mapping,
'domain-name': domain_attribute_mapping,
'domain-name:value': domain_attribute_mapping,
'network-traffic': url_port_attribute_mapping,
'network-traffic:dst_port': url_port_attribute_mapping,
'ipv4-addr:value': ip_attribute_mapping,
'ipv6-addr:value': ip_attribute_mapping
}
user_account_mapping = {'account_created': {'type': 'datetime', 'object_relation': 'created'},
'account_expires': {'type': 'datetime', 'object_relation': 'expires'},
'account_first_login': {'type': 'datetime', 'object_relation': 'first_login'},
'account_last_login': {'type': 'datetime', 'object_relation': 'last_login'},
'account_login': user_id_mapping,
'account_type': {'type': 'text', 'object_relation': 'account-type'},
'can_escalate_privs': {'type': 'boolean', 'object_relation': 'can_escalate_privs'},
'credential': credential_attribute_mapping,
'credential_last_changed': {'type': 'datetime', 'object_relation': 'password_last_changed'},
'display_name': {'type': 'text', 'object_relation': 'display-name'},
'gid': {'type': 'text', 'object_relation': 'group-id'},
'home_dir': {'type': 'text', 'object_relation': 'home_dir'},
'is_disabled': {'type': 'boolean', 'object_relation': 'disabled'},
'is_privileged': {'type': 'boolean', 'object_relation': 'privileged'},
'is_service_account': {'type': 'boolean', 'object_relation': 'is_service_account'},
'shell': {'type': 'text', 'object_relation': 'shell'},
'user_id': {'type': 'text', 'object_relation': 'user-id'}}
vulnerability_mapping = {'name': id_attribute_mapping,
'description': summary_attribute_mapping}
x509_mapping = {'issuer': issuer_attribute_mapping,
'x509-certificate:issuer': issuer_attribute_mapping,
'serial_number': serial_number_attribute_mapping,
'x509-certificate:serial_number': serial_number_attribute_mapping,
'subject': x509_subject_attribute_mapping,
'x509-certificate:subject': x509_subject_attribute_mapping,
'subject_public_key_algorithm': x509_spka_attribute_mapping,
'x509-certificate:subject_public_key_algorithm': x509_spka_attribute_mapping,
'subject_public_key_exponent': x509_spke_attribute_mapping,
'x509-certificate:subject_public_key_exponent': x509_spke_attribute_mapping,
'subject_public_key_modulus': x509_spkm_attribute_mapping,
'x509-certificate:subject_public_key_modulus': x509_spkm_attribute_mapping,
'validity_not_before': x509_vnb_attribute_mapping,
'x509-certificate:validity_not_before': x509_vnb_attribute_mapping,
'validity_not_after': x509_vna_attribute_mapping,
'x509-certificate:validity_not_after': x509_vna_attribute_mapping,
'version': x509_version_attribute_mapping,
'x509-certificate:version': x509_version_attribute_mapping,
'SHA-1': x509_sha1_attribute_mapping,
"x509-certificate:hashes.'sha1'": x509_sha1_attribute_mapping,
'SHA-256': x509_sha256_attribute_mapping,
"x509-certificate:hashes.'sha256'": x509_sha256_attribute_mapping,
'MD5': x509_md5_attribute_mapping,
"x509-certificate:hashes.'md5'": x509_md5_attribute_mapping,
}
attachment_types = ('file:content_ref.name', 'file:content_ref.payload_bin',
'artifact:x_misp_text_name', 'artifact:payload_bin',
"file:hashes.'MD5'", "file:content_ref.hashes.'MD5'",
'file:name')
connection_protocols = {"IP": "3", "ICMP": "3", "ARP": "3",
"TCP": "4", "UDP": "4",
"HTTP": "7", "HTTPS": "7", "FTP": "7"}

28
app/files/scripts/stix2misp.py
View File

@ -22,6 +22,7 @@ import time
import uuid
import base64
import pymisp
import traceback
import stix2misp_mapping
from operator import attrgetter
from collections import defaultdict
@ -1432,7 +1433,7 @@ class ExternalStixParser(StixParser):
if isinstance(attribute_value, list):
attributes.extend([{'type': attribute_type, 'value': value, 'to_ids': False} for value in attribute_value])
else:
attribute.append({'type': attribute_type, 'value': attribute_value, 'to_ids': False})
attributes.append({'type': attribute_type, 'value': attribute_value, 'to_ids': False})
if ttp.exploit_targets and ttp.exploit_targets.exploit_target:
for exploit_target in ttp.exploit_targets.exploit_target:
if exploit_target.item.vulnerabilities:
@ -1544,19 +1545,18 @@ def generate_event(filename, tries=0):
return STIXPackage.from_xml(filename)
except NamespaceNotFoundError:
if tries == 1:
print(4)
print(json.dump({'error': 'Cannot handle STIX namespace'}))
sys.exit()
_update_namespaces()
return generate_event(filename, 1)
except NotImplementedError:
print('ERROR - Missing python library: stix_edh', file=sys.stderr)
except Exception:
print(json.dumps({'error': 'Missing python library: stix_edh'}))
except Exception as e:
try:
import maec
print(2)
print(json.dumps({'error': f'Error while loading the STIX file: {e.__str__()}'}))
except ImportError:
print('ERROR - Missing python library: maec', file=sys.stderr)
print(3)
print(json.dumps({'error': 'Missing python library: maec'}))
sys.exit(0)
@ -1572,11 +1572,15 @@ def main(args):
filename = args[1] if args[1][0] == '/' else '{}/tmp/{}'.format(os.path.dirname(args[0]), args[1])
event = generate_event(filename)
from_misp = is_from_misp(event)
stix_parser = StixFromMISPParser() if from_misp else ExternalStixParser()
stix_parser.load_event(args[2:], filename, from_misp, event.version)
stix_parser.build_misp_event(event)
stix_parser.saveFile()
print(1)
try:
stix_parser = StixFromMISPParser() if from_misp else ExternalStixParser()
stix_parser.load_event(args[2:], filename, from_misp, event.version)
stix_parser.build_misp_event(event)
stix_parser.saveFile()
print(json.dumps({'success': 1}))
except Exception as e:
print(json.dumps({'error': e.__str__()}))
traceback.print_tb(e.__traceback__)
if __name__ == "__main__":
main(sys.argv)

9
app/files/scripts/taxii/taxii_push.py
View File

@ -5,14 +5,17 @@ push the content to a TAXII server.
import argparse
import logging
import logging.config
import misp_stix_converter
import pathlib
import sys
import taxii2client
import urllib.parse
from base64 import b64decode
from pathlib import Path
from requests.auth import HTTPBasicAuth
_script_path = Path(__file__).resolve.parents[1]
sys.path.insert(0, str(_script_path / 'misp-stix'))
import misp_stix_converter
# Name of the logger to use for this application
_LOGGER_NAME = "taxii_push"
@ -95,7 +98,7 @@ def parse_args():
parser.add_argument(
"--dir",
help="A directory with files containing JSON MISP events.",
type=pathlib.Path,
type=Path,
required=True
)

11
app/webroot/css/workflows-editor.css
View File

@ -665,4 +665,15 @@
.drawflow .drawflow-node.block-type-concurrent > .outputs > .output_1::after {
content: "\f074";
}
.drawflow svg.connection .connection-label-container {
display: flex;
flex-direction: column;
row-gap: 3px;
width: fit-content;
transform: translate(-50%, -50%);
background-color: #ffffffaa;
padding: 3px;
border-radius: 5px;
}

1999
app/webroot/js/drawflow.js Normal file
View File

File diff suppressed because it is too large Load Diff

38
app/webroot/js/misp.js
View File

@ -347,26 +347,36 @@ function submitPasswordReset(id) {
});
}
function submitMessageForm(url, form, target) {
function submitMessageForm(url) {
if (!$('#PostMessage').val()) {
showMessage("fail", "Cannot submit empty message.");
} else {
submitGenericForm(url, form, target);
var message = $('#PostMessage').val()
fetchFormDataAjax(url, function (formData) {
var $formData = $(formData);
$formData.find('#PostMessage').val(message);
$.ajax({
data: $formData.find('form').serialize(),
beforeSend: function () {
$(".loading").show();
},
success: function (data) {
showMessage("success", "Message added.");
$('#top').html(data);
},
error: function () {
showMessage('fail', 'Could not add message.');
},
complete: function () {
$(".loading").hide();
},
type: "post",
url: $formData.find('form').attr('action')
});
});
}
}
function submitGenericForm(url, form, target) {
xhr({
data: $('#' + form).serialize(),
success:function (data, textStatus) {
$('#top').html(data);
showMessage("success", "Message added.");
},
type: "post",
url: url,
});
}
function acceptObject(type, id) {
var name = '#ShadowAttribute_' + id + '_accept';
var formData = $(name).serialize();

29
app/webroot/js/workflows-editor/workflows-editor.js
View File

@ -129,6 +129,9 @@ var dotBlock_error = doT.template(' \
</div> \
</div>')
var dotBlock_connectionLabel = doT.template(' \
<span class="label label-{{=it.variant}}" id="{{=it.id}}">{{=it.name}}</span>')
var classBySeverity = {
'info': 'info',
'warning': 'warning',
@ -672,7 +675,7 @@ function getEditorData(cleanNodes) {
node.data.params = deleteInvalidParams(node.data.params)
cleanedIndexedParams = {}
node.data.params.forEach(function(param) {
cleanedIndexedParams[param.id] = param.value
cleanedIndexedParams[param.id] = param.value !== undefined ? param.value : param.default
})
node.data.indexed_params = cleanedIndexedParams
}
@ -771,7 +774,11 @@ function loadWorkflow(workflow) {
Object.values(workflow.data).forEach(function (node) {
for (var input_name in node.inputs) {
node.inputs[input_name].connections.forEach(function (connection) {
editor.addConnection(connection.node, node.id, connection.input, input_name)
connection.labels = connection.labels === undefined ? [] : connection.labels;
var labels = connection.labels.map(function(labelConf) {
return dotBlock_connectionLabel(labelConf)
})
editor.addConnection(connection.node, node.id, connection.input, input_name, labels)
})
}
})
@ -868,7 +875,7 @@ function addNodesFromBlueprint(workflowBlueprint, cursorPosition) {
left: (node.pos_x - minX) * editor.zoom + cursorPosition.left,
}
if (all_modules_by_id[node.data.id] === undefined) {
var errorMessage = 'Invalid ' + node.data.module_data.module_type + ' module id `' + node.data.module_data.id + '` (' + node.id + ')'
var errorMessage = 'Invalid ' + node.data.module_type + ' module id `' + node.data.id + '` (' + node.id + ')'
var html = window['dotBlock_error']({
error: errorMessage,
data: JSON.stringify(node.data.indexed_params, null, 2)
@ -883,20 +890,20 @@ function addNodesFromBlueprint(workflowBlueprint, cursorPosition) {
node.data,
html
)
return
} else {
additionalData = {
indexed_params: node.data.indexed_params,
saved_filters: node.data.saved_filters,
}
addNode(all_modules_by_id[node.data.id], position, additionalData)
}
additionalData = {
indexed_params: node.data.indexed_params,
saved_filters: node.data.saved_filters,
}
addNode(all_modules_by_id[node.data.id], position, additionalData)
oldNewIDMapping[node.id] = editor.nodeId - 1
newNodes.push(getNodeHtmlByID(editor.nodeId - 1)) // nodeId is incremented as soon as a new node is created
})
workflowBlueprint.data.forEach(function (node) {
Object.keys(node.outputs).forEach(function (outputName) {
var newNode = Object.assign({}, all_modules_by_id[node.data.id])
if (newNode.outputs > 0) { // make sure the module configuration didn't change in regards of the outputs
var outputCount = all_modules_by_id[node.data.id] !== undefined ? all_modules_by_id[node.data.id].outputs : Object.keys(node.outputs).length
if (outputCount > 0) { // make sure the module configuration didn't change in regards of the outputs
node.outputs[outputName].connections.forEach(function (connection) {
if (oldNewIDMapping[connection.node] !== undefined) {
editor.addConnection(

13
tests/build-test.sh
View File

@ -3,25 +3,26 @@ set -e
set -o xtrace
# Check if dependencies for zmq are properly installed
python3 ./../app/files/scripts/mispzmq/mispzmqtest.py
python ./../app/files/scripts/mispzmq/mispzmqtest.py
# Check if all attachments handlers dependencies are correctly installed
python3 ./../app/files/scripts/generate_file_objects.py -c | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if len([i for i in data.values() if i == True]) == 0 else 1)'
python ./../app/files/scripts/generate_file_objects.py -c | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if len([i for i in data.values() if i == True]) == 0 else 1)'
# Try to extract data from file
python3 ./../app/files/scripts/generate_file_objects.py -p /bin/ls | python3 -c 'import sys, json; data = json.load(sys.stdin); sys.exit(0 if "objects" in data else 1)'
python ./../app/files/scripts/generate_file_objects.py -p /bin/ls
python ./../app/files/scripts/generate_file_objects.py -p /bin/ls | python3 -c 'import sys, json; data = json.load(sys.stdin); sys.exit(0 if "objects" in data else 1)'
# Test converting stix1 to MISP format
curl https://stixproject.github.io/documentation/idioms/c2-indicator/indicator-for-c2-ip-address.xml > ./../app/files/scripts/tmp/test-stix1.xml
python3 ./../app/files/scripts/stix2misp.py test-stix1.xml 1 1 ./../app/files/scripts/synonymsToTagNames.json | python3 -c 'import sys; data = sys.stdin.read().strip(); print(data); sys.exit(0 if data == "1" else 1)'
python ./../app/files/scripts/stix2misp.py test-stix1.xml 1 1 ./../app/files/scripts/synonymsToTagNames.json | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if data["success"] == 1 else 1)'
rm -f ./../app/files/scripts/tmp/{test-stix1.xml,test-stix1.xml.json}
# Test converting stix2 to MISP format
curl https://raw.githubusercontent.com/oasis-open/cti-stix2-json-schemas/master/examples/indicator-for-c2-ip-address.json > ./../app/files/scripts/tmp/test-stix2.json
python3 ./../app/files/scripts/stix2/stix2misp.py ./../tmp/test-stix2.json 1 1 ./../app/files/scripts/synonymsToTagNames.json | python3 -c 'import sys; data = sys.stdin.read().strip(); print(data); sys.exit(0 if data == "1" else 1)'
python ./../app/files/scripts/stix2/stix2misp.py -i ./../app/files/scripts/tmp/test-stix2.json --distribution 1 | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if data["success"] == 1 else 1)'
rm -f ./../app/files/scripts/tmp/{test-stix2.json,test-stix2.json.stix2}
# Test converting MISP to STIX2
cp event.json /tmp/
python3 ./../app/files/scripts/stix2/misp2stix2.py -i /tmp/event.json | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if data["success"] == 1 else 1)'
python ./../app/files/scripts/stix2/misp2stix2.py -i /tmp/event.json | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if data["success"] == 1 else 1)'
rm -f /tmp/{event.json,event.json.out}