MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'develop' of github.com:MISP/MISP into develop

pull/8692/head
Sami Mokaddem 2022-10-21 08:48:57 +02:00
parent 4c215a86b4 09c0c50868
commit a09721292e
No known key found for this signature in database
GPG Key ID: 164C473F627A06FA
45 changed files with 701 additions and 467 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/Console/Command/AdminShell.php
View File

@ -13,6 +13,8 @@ App::uses('JsonTool', 'Tools');
* @property Warninglist $Warninglist
* @property Attribute $Attribute
* @property Job $Job
* @property Correlation $Correlation
* @property OverCorrelatingValue $OverCorrelatingValue
*/
class AdminShell extends AppShell
{
@ -108,7 +110,7 @@ class AdminShell extends AppShell
}
$jobId = $this->args[0];
$this->Attribute->generateCorrelation($jobId);
$this->Correlation->generateCorrelation($jobId);
}
public function jobGenerateOccurrences()
@ -128,7 +130,7 @@ class AdminShell extends AppShell
}
$jobId = $this->args[0];
$this->Attribute->purgeCorrelations();
$this->Correlation->purgeCorrelations();
$this->Job->saveStatus($jobId);
}

3
app/Console/Command/ServerShell.php
View File

@ -607,6 +607,9 @@ class ServerShell extends AppShell
foreach ($users as $user) {
echo __('Sending `%s` report to `%s`', $period, $user['User']['email']) . PHP_EOL;
$emailTemplate = $this->User->generatePeriodicSummary($user['User']['id'], $period, false);
if ($emailTemplate === null) {
continue; // no new event for this user
}
$this->User->sendEmail($user, $emailTemplate, false, null);
}
}

2
app/Controller/AppController.php
View File

@ -1262,7 +1262,7 @@ class AppController extends Controller
$final = $model->restSearch($user, $returnFormat, $filters, false, false, $elementCounter, $renderView);
if ($renderView) {
$this->layout = false;
$final = json_decode($final->intoString(), true);
$final = JsonTool::decode($final->intoString());
$this->set($final);
$this->render('/Events/module_views/' . $renderView);
} else {

26
app/Controller/AttributesController.php
View File

@ -1156,7 +1156,10 @@ class AttributesController extends AppController
$eventId = $this->request->data['Attribute']['event_id'];
}
} else {
$ids = json_decode($this->request->data['Attribute']['ids_delete']);
$ids = $this->_jsonDecode($this->request->data['Attribute']['ids_delete']);
}
if (empty($ids)) {
throw new NotFoundException(__('No matching attributes found.'));
}
if (empty($eventId)) {
throw new MethodNotAllowedException(__('No event ID set.'));
@ -1174,9 +1177,6 @@ class AttributesController extends AppController
throw new ForbiddenException(__('You do not have permission to do that.'));
}
}
if (empty($ids)) {
$ids = -1;
}
$conditions = ['id' => $ids, 'event_id' => $eventId];
if ($ids === 'all') {
unset($conditions['id']);
@ -1189,15 +1189,16 @@ class AttributesController extends AppController
'conditions' => $conditions,
'fields' => ['id', 'deleted'],
]);
if ($ids === 'all') {
$ids = array_keys($attributes);
}
if (empty($attributes)) {
throw new NotFoundException(__('No matching attributes found.'));
}
if ($ids === 'all') {
$ids = array_keys($attributes);
}
$user = $this->_closeSession();
$successes = [];
foreach ($attributes as $attributeId => $deleted) {
if ($this->Attribute->deleteAttribute($attributeId, $this->Auth->user(), $hard || $deleted == 1)) {
if ($this->Attribute->deleteAttribute($attributeId, $user, $hard || $deleted == 1)) {
$successes[] = $attributeId;
}
}
@ -1918,8 +1919,13 @@ class AttributesController extends AppController
{
if ($this->request->is('post')) {
if (!Configure::read('MISP.background_jobs')) {
$k = $this->Attribute->generateCorrelation();
$this->Flash->success(__('All done. %s attributes processed.', $k));
$k = $this->Attribute->Correlation->generateCorrelation();
$message = __('All done. %s attributes processed.', $k);
if ($this->_isRest()) {
return $this->RestResponse->successResponse(0, $message);
}
$this->Flash->success($message);
$this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
} else {
/** @var Job $job */

1
app/Controller/AuditLogsController.php
View File

@ -53,6 +53,7 @@ class AuditLogsController extends AppController
'News',
'Warninglist',
'Workflow',
'WorkflowBlueprint',
];
public $paginate = [

48
app/Controller/Component/RestResponseComponent.php
View File

@ -1315,7 +1315,7 @@ class RestResponseComponent extends Component
'input' => 'text',
'type' => 'string',
'operators' => array('equal', 'not_equal'),
'help' => __('Events published within the last x amount of time, where x can be defined in days, hours, minutes (for example 5d or 12h or 30m)')
'help' => __('Events published within the last x amount of time, where x can be defined in days, hours, minutes (for example 5d or 12h or 30m), ISO 8601 datetime format or timestamp.')
),
'last_seen' => array(
'input' => 'text',
@ -1333,7 +1333,7 @@ class RestResponseComponent extends Component
'local' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' ),
'values' => array(1 => 'True', 0 => 'False'),
'help' => __('If the organisation should have access to this instance, make sure that the Local organisation setting is checked. If you would only like to add a known external organisation for inclusion in sharing groups, uncheck the Local organisation setting.')
),
'lookup_visible' => array(
@ -1455,7 +1455,7 @@ class RestResponseComponent extends Component
'override_ids' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' ),
'values' => array(1 => 'True', 0 => 'False'),
'help' => __('The IDS flags will be set to off for this feed')
),
'page' => array(
@ -1474,73 +1474,73 @@ class RestResponseComponent extends Component
'perm_admin' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' )
'values' => array(1 => 'True', 0 => 'False')
),
'perm_audit' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' )
'values' => array(1 => 'True', 0 => 'False')
),
'perm_auth' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' )
'values' => array(1 => 'True', 0 => 'False')
),
'perm_delegate' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' )
'values' => array(1 => 'True', 0 => 'False')
),
'perm_regexp_access' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' )
'values' => array(1 => 'True', 0 => 'False')
),
'perm_sharing_group' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' )
'values' => array(1 => 'True', 0 => 'False')
),
'perm_sighting' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' )
'values' => array(1 => 'True', 0 => 'False')
),
'perm_site_admin' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' )
'values' => array(1 => 'True', 0 => 'False')
),
'perm_sync' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' )
'values' => array(1 => 'True', 0 => 'False')
),
'perm_tag_editor' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' )
'values' => array(1 => 'True', 0 => 'False')
),
'perm_tagger' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' )
'values' => array(1 => 'True', 0 => 'False')
),
'perm_galaxy_editor' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' )
'values' => array(1 => 'True', 0 => 'False')
),
'perm_template' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' )
'values' => array(1 => 'True', 0 => 'False')
),
'permission' => array(
'input' => 'select',
'type' => 'string',
'operators' => array('equal'),
'values' => array(0 =>'Read Only', 1 => 'Manage Own Events', 2 => 'Manage Organisation Events', 3 => 'Manage and Publish Organisation Events'),
'values' => array(0 => 'Read Only', 1 => 'Manage Own Events', 2 => 'Manage Organisation Events', 3 => 'Manage and Publish Organisation Events'),
),
'provider' => array(
'input' => 'text',
@ -1551,7 +1551,7 @@ class RestResponseComponent extends Component
'publish' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' ),
'values' => array(1 => 'True', 0 => 'False'),
'help' => __('The event will be published')
),
'publish_timestamp' => array(
@ -1563,7 +1563,7 @@ class RestResponseComponent extends Component
'published' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' )
'values' => array(1 => 'True', 0 => 'False')
),
'publishtimestamp' => array(
'input' => 'number',
@ -1574,19 +1574,19 @@ class RestResponseComponent extends Component
'pull' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' ),
'values' => array(1 => 'True', 0 => 'False'),
'help' => __('Allow the download of events and their attribute from the server')
),
'push' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' ),
'values' => array(1 => 'True', 0 => 'False'),
'help' => __('Allow the upload of events and their attribute to the server')
),
'push_sightings' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' ),
'values' => array(1 => 'True', 0 => 'False'),
'help' => __('Allow the upload of sightings to the server')
),
'referenced_galaxy_cluster_uuid' => array(
@ -1779,7 +1779,7 @@ class RestResponseComponent extends Component
),
'to' => array(
'type' => 'date',
'validation' => array( 'format' => 'YYYY-MM-DD' ),
'validation' => array('format' => 'YYYY-MM-DD'),
'plugin' => 'datepicker',
'plugin_config' => array(
'format' => 'yyyy/mm/dd',
@ -1832,7 +1832,7 @@ class RestResponseComponent extends Component
'withAttachments' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' )
'values' => array(1 => 'True', 0 => 'False')
),
// Not supported yet

2
app/Controller/DashboardsController.php
View File

@ -163,7 +163,7 @@ class DashboardsController extends AppController
throw new MethodNotAllowedException(__('You need to specify the widget to use along with the configuration.'));
}
$value = $this->request->data['data'];
$valueConfig = json_decode($value['config'], true);
$valueConfig = $this->_jsonDecode($value['config']);
$dashboardWidget = $this->Dashboard->loadWidget($user, $value['widget']);
$cacheLifetime = $dashboardWidget->cacheLifetime ?? false;

15
app/Controller/EventsController.php
View File

@ -2589,7 +2589,7 @@ class EventsController extends AppController
$this->request->data = $this->request->data['Event'];
}
if (isset($this->request->data['json'])) {
$this->request->data = json_decode($this->request->data['json'], true);
$this->request->data = $this->_jsonDecode($this->request->data['json']);
}
$eventToSave = $event;
$capturedObjects = ['Attribute', 'Object', 'Tag', 'Galaxy', 'EventReport'];
@ -2599,8 +2599,7 @@ class EventsController extends AppController
}
}
$eventToSave['Event']['published'] = 0;
$date = new DateTime();
$eventToSave['Event']['timestamp'] = $date->getTimestamp();
$eventToSave['Event']['timestamp'] = time();
$result = $this->Event->_edit($eventToSave, $this->Auth->user(), $id);
if ($this->_isRest()) {
if ($result === true) {
@ -3379,7 +3378,7 @@ class EventsController extends AppController
$responseType = $this->Event->validFormats[$returnFormat][0];
$final = $this->Event->restSearch($this->Auth->user(), $returnFormat, $filters, false, false, $elementCounter, $renderView);
if ($renderView) {
$final = json_decode($final->intoString(), true);
$final = JsonTool::decode($final->intoString());
$this->set($final);
$this->set('responseType', $responseType);
$this->set('returnFormat', $returnFormat);
@ -3958,6 +3957,7 @@ class EventsController extends AppController
$complexTypeTool = new ComplexTypeTool();
$this->loadModel('Warninglist');
$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
$complexTypeTool->setSecurityVendorDomains($this->Warninglist->fetchSecurityVendorDomains());
if (!isset($this->request->data['Attribute'])) {
$this->request->data = array('Attribute' => $this->request->data);
}
@ -3968,6 +3968,9 @@ class EventsController extends AppController
$adhereToWarninglists = $this->request->data['Attribute']['adhereToWarninglists'];
}
$resultArray = $complexTypeTool->checkFreeText($this->request->data['Attribute']['value']);
foreach ($resultArray as &$attribute) {
$attribute['to_ids'] = $this->Event->Attribute->typeDefinitions[$attribute['default_type']]['to_ids'];
}
if ($this->_isRest()) {
// Keep this 'types' format for rest response, but it is not necessary for UI
foreach ($resultArray as $key => $r) {
@ -5508,11 +5511,11 @@ class EventsController extends AppController
if ($event['Event']['disable_correlation']) {
$event['Event']['disable_correlation'] = 0;
$this->Event->save($event);
$this->Event->Attribute->generateCorrelation(false, $event['Event']['id']);
$this->Event->Attribute->Correlation->generateCorrelation(false, $event['Event']['id']);
} else {
$event['Event']['disable_correlation'] = 1;
$this->Event->save($event);
$this->Event->Attribute->purgeCorrelations($event['Event']['id']);
$this->Event->Attribute->Correlation->purgeCorrelations($event['Event']['id']);
}
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('events', 'toggleCorrelation', $event['Event']['id'], false, 'Correlation ' . ($event['Event']['disable_correlation'] ? 'disabled' : 'enabled') . '.');

8
app/Controller/ServersController.php
View File

@ -289,7 +289,7 @@ class ServersController extends AppController
$this->request->data['Server']['internal'] = 0;
}
// test the filter fields
if (!empty($this->request->data['Server']['pull_rules']) && !$this->Server->isJson($this->request->data['Server']['pull_rules'])) {
if (!empty($this->request->data['Server']['pull_rules']) && !JsonTool::isValid($this->request->data['Server']['pull_rules'])) {
$fail = true;
$error_msg = __('The pull filter rules must be in valid JSON format.');
if ($this->_isRest()) {
@ -299,7 +299,7 @@ class ServersController extends AppController
}
}
if (!$fail && !empty($this->request->data['Server']['push_rules']) && !$this->Server->isJson($this->request->data['Server']['push_rules'])) {
if (!$fail && !empty($this->request->data['Server']['push_rules']) && !JsonTool::isValid($this->request->data['Server']['push_rules'])) {
$fail = true;
$error_msg = __('The push filter rules must be in valid JSON format.');
if ($this->_isRest()) {
@ -484,7 +484,7 @@ class ServersController extends AppController
$fail = false;
// test the filter fields
if (!empty($this->request->data['Server']['pull_rules']) && !$this->Server->isJson($this->request->data['Server']['pull_rules'])) {
if (!empty($this->request->data['Server']['pull_rules']) && !JsonTool::isValid($this->request->data['Server']['pull_rules'])) {
$fail = true;
$error_msg = __('The pull filter rules must be in valid JSON format.');
if ($this->_isRest()) {
@ -494,7 +494,7 @@ class ServersController extends AppController
}
}
if (!$fail && !empty($this->request->data['Server']['push_rules']) && !$this->Server->isJson($this->request->data['Server']['push_rules'])) {
if (!$fail && !empty($this->request->data['Server']['push_rules']) && !JsonTool::isValid($this->request->data['Server']['push_rules'])) {
$fail = true;
$error_msg = __('The push filter rules must be in valid JSON format.');
if ($this->_isRest()) {

6
app/Controller/UsersController.php
View File

@ -2195,7 +2195,6 @@ class UsersController extends AppController
private function __statisticsGalaxyMatrix(array $user, $params = array())
{
$this->loadModel('Event');
$this->loadModel('Galaxy');
$mitre_galaxy_id = $this->Galaxy->getMitreAttackGalaxyId();
if (isset($params['galaxy_id'])) {
@ -2248,9 +2247,8 @@ class UsersController extends AppController
}
$elementCounter = 0;
$renderView = '';
$final = $this->Event->restSearch($user, 'attack', $filters, false, false, $elementCounter, $renderView);
$final = json_decode($final, true);
$final = $this->User->Event->restSearch($user, 'attack', $filters, false, false, $elementCounter, $renderView);
$final = JsonTool::decode($final);
if (!empty($final)) {
$rest_response_empty = false;
foreach ($final as $key => $data) {

32
app/Controller/WorkflowBlueprintsController.php
View File

@ -1,6 +1,9 @@
<?php
App::uses('AppController', 'Controller');
/**
* @property WorkflowBlueprint $WorkflowBlueprint
*/
class WorkflowBlueprintsController extends AppController
{
public $components = array(
@ -14,10 +17,9 @@ class WorkflowBlueprintsController extends AppController
$message = __('Default workflow blueprints updated');
if ($this->_isRest()) {
return $this->RestResponse->saveSuccessResponse('WorkflowBlueprint', 'update', false, $this->response->type(), $message);
} else {
$this->Flash->success($message);
$this->redirect(array('controller' => 'workflowBlueprints', 'action' => 'index'));
}
$this->Flash->success($message);
$this->redirect(array('controller' => 'workflowBlueprints', 'action' => 'index'));
}
public function index()
@ -69,9 +71,7 @@ class WorkflowBlueprintsController extends AppController
public function delete($id)
{
$params = [
];
$this->CRUD->delete($id, $params);
$this->CRUD->delete($id);
if ($this->IndexFilter->isRest()) {
return $this->restResponsePayload;
}
@ -82,16 +82,15 @@ class WorkflowBlueprintsController extends AppController
{
$filters = $this->IndexFilter->harvestParameters(['format']);
if (!empty($filters['format'])) {
if ($filters['format'] == 'dot') {
if ($filters['format'] === 'dot') {
$dot = $this->WorkflowBlueprint->getDotNotation($id);
return $this->RestResponse->viewData($dot, $this->response->type());
} else if ($filters['format'] == 'mermaid') {
} else if ($filters['format'] === 'mermaid') {
$mermaid = $this->WorkflowBlueprint->getMermaid($id);
return $this->RestResponse->viewData($mermaid, $this->response->type());
}
}
$this->CRUD->view($id, [
]);
$this->CRUD->view($id);
if ($this->IndexFilter->isRest()) {
return $this->restResponsePayload;
}
@ -102,11 +101,8 @@ class WorkflowBlueprintsController extends AppController
public function import()
{
if ($this->request->is('post') || $this->request->is('put')) {
$workflowBlueprintData = JsonTool::decode($this->request->data['WorkflowBlueprint']['data']);
if ($workflowBlueprintData === null) {
throw new MethodNotAllowedException(__('Error while decoding JSON'));
}
$this->request->data['WorkflowBlueprint']['data'] = JsonTool::encode($workflowBlueprintData);
$workflowBlueprintData = $this->_jsonDecode($this->request->data['WorkflowBlueprint']['json']);
$this->request->data = $workflowBlueprintData;
$this->add();
}
}
@ -118,7 +114,11 @@ class WorkflowBlueprintsController extends AppController
'id' => $id,
]
]);
$content = JsonTool::encode($workflowBlueprint, JSON_PRETTY_PRINT);
if (empty($workflowBlueprint)) {
throw new NotFoundException(__('Invalid workflow blueprint'));
}
$content = JsonTool::encode($workflowBlueprint, true);
$this->response->body($content);
$this->response->type('json');
$this->response->download(sprintf('blueprint_%s_%s.json', str_replace(' ', '-', strtolower($workflowBlueprint['WorkflowBlueprint']['name'])), time()));

10
app/Lib/Dashboard/RecentSightingsWidget.php
View File

@ -8,7 +8,7 @@ class RecentSightingsWidget
public $height = 6;
public $params = array(
'limit' => 'Maximum amount of sightings to return',
'last' => 'Limit sightins to last 1d, 12h, ...'
'last' => 'Limit sightings to last 1d, 12h, ...'
);
public $description = 'Widget showing information on recent sightings';
public $cacheLifetime = false;
@ -28,12 +28,14 @@ class RecentSightingsWidget
$last = $params['last'];
$limit = $params['limit'];
$this->Sighting = ClassRegistry::init('Sighting');
$filters = array( 'last' => $last, 'includeAttribute' => 'true', 'includeEvent' => 'true');
/** @var Sighting $Sighting */
$Sighting = ClassRegistry::init('Sighting');
$filters = array('last' => $last, 'includeAttribute' => 'true', 'includeEvent' => 'true');
$data = array();
$count = 0;
foreach(json_decode($this->Sighting->restSearch($user, 'json', $filters))->{'response'} as $el) {
foreach (JsonTool::decode($Sighting->restSearch($user, 'json', $filters)->intoString())->{'response'} as $el) {
$sighting = $el->{'Sighting'};
$event = $sighting->{'Event'};
$attribute = $sighting->{'Attribute'};

15
app/Lib/Dashboard/TresholdSightingsWidget.php
View File

@ -2,14 +2,14 @@
class TresholdSightingsWidget
{
public $title = 'Treshold Sightings';
public $title = 'Threshold Sightings';
public $render = 'SimpleList';
public $width = 8;
public $height = 4;
public $params = array(
'treshold' => 'Treshold for sightings'
'treshold' => 'Threshold for sightings'
);
public $description = 'Widget showing information on sightings above certain treshold';
public $description = 'Widget showing information on sightings above certain threshold';
public $cacheLifetime = false;
public $autoRefreshDelay = 30;
public $placeholder =
@ -24,15 +24,16 @@ class TresholdSightingsWidget
);
$treshold = $params['treshold'];
$this->Sighting = ClassRegistry::init('Sighting');
/** @var Sighting $Sighting */
$Sighting = ClassRegistry::init('Sighting');
$filters = array( 'includeAttribute' => 'true', 'includeEvent' => 'true');
$data = array();
$sightings_score = array();
$restSearch = json_decode($this->Sighting->restSearch($user, 'json', $filters))->{'response'};
$restSearch = JsonTool::decode($Sighting->restSearch($user, 'json', $filters)->intoString())->{'response'};
foreach($restSearch as $el) {
foreach ($restSearch as $el) {
$sighting = $el->{'Sighting'};
$attribute = $sighting->{'Attribute'};
$event = $sighting->{'Event'};
@ -47,7 +48,7 @@ class TresholdSightingsWidget
elseif ($sighting->{'type'} == 1) $sightings_score[$attribute->{'id'}]['score'] = $sightings_score[$attribute->{'id'}]['score'] + 1;
}
foreach($sightings_score as $attribute_id => $s) {
foreach ($sightings_score as $attribute_id => $s) {
if ((int)$s['score'] >= (int)$treshold ) {
$output = "Score: " . $s['score'] . ": " . $s['value'] . " (id: " . $attribute_id . ") in " . $s['event_title'] . " (id: " . $s['event_id'] . ")";
$data[] = array( 'title' => __("False positive above threshold"), 'value' => $output,

4
app/Lib/Export/StixExport.php
View File

@ -92,7 +92,7 @@ abstract class StixExport
$this->__filenames[] = $this->__tmp_file->path;
}
$result = $this->__parse_misp_data();
$decoded = json_decode($result, true);
$decoded = JsonTool::decode($result);
if (!isset($decoded['success']) || !$decoded['success']) {
if (!empty($decoded['filenames'])) {
$this->__delete_temporary_files(false, $decoded['filename']);
@ -382,7 +382,7 @@ abstract class StixExport
}
/**
* @return array
* @return string
*/
abstract protected function __parse_misp_data();

12
app/Lib/Tools/AttributeValidationTool.php
View File

@ -570,8 +570,7 @@ class AttributeValidationTool
case 'float':
return is_numeric($value);
case 'cortex':
json_decode($value);
return json_last_error() === JSON_ERROR_NONE;
return JsonTool::isValid($value);
case 'boolean':
return $value == 1 || $value == 0;
case 'AS':
@ -640,14 +639,7 @@ class AttributeValidationTool
*/
private static function isSsdeep($value)
{
if (strpos($value, "\n") !== false) {
return false;
}
$parts = explode(':', $value);
if (count($parts) !== 3) {
return false;
}
return self::isPositiveInteger($parts[0]);
return preg_match('#^([0-9]+):([0-9a-zA-Z/+]*):([0-9a-zA-Z/+]*)$#', $value);
}
/**

168
app/Lib/Tools/ComplexTypeTool.php
View File

@ -40,7 +40,13 @@ class ComplexTypeTool
128 => ['single' => ['sha512'], 'composite' => ['filename|sha512']],
];
private $__tlds = null;
private $__tlds;
/**
* Hardcoded list if properly warninglist is not available
* @var string[]
*/
private $securityVendorDomains = ['virustotal.com', 'hybrid-analysis.com'];
public static function refangValue($value, $type)
{
@ -60,6 +66,14 @@ class ComplexTypeTool
}
}
public function setSecurityVendorDomains(array $securityVendorDomains)
{
if (empty($securityVendorDomains)) {
return; // if provided warninglist is empty, keep hardcoded domains
}
$this->securityVendorDomains = $securityVendorDomains;
}
public function checkComplexRouter($input, $type, $settings = array())
{
switch ($type) {
@ -188,6 +202,7 @@ class ComplexTypeTool
continue;
}
$resolvedResult = $this->__resolveType($element);
// Do not extract datetime from CSV
if ($resolvedResult) {
$iocArray[] = $resolvedResult;
}
@ -205,17 +220,18 @@ class ComplexTypeTool
*/
public function checkFreeText($input, array $settings = [])
{
$input = str_replace("\xc2\xa0", ' ', $input); // non breaking space to normal space
$input = preg_replace('/\p{C}+/u', ' ', $input);
$iocArray = preg_split("/\r\n|\n|\r|\s|\s+|,|\<|\>|;/", $input);
preg_match_all('/\"([^\"]*)\"/', $input, $matches);
foreach ($matches[1] as $match) {
if ($match !== '') {
$iocArray[] = $match;
}
if (empty($input)) {
return [];
}
unset($matches);
if ($input[0] === '{') {
// If input looks like JSON, try to parse it as JSON
try {
return $this->parseJson($input, $settings);
} catch (Exception $e) {}
}
$iocArray = $this->parseFreetext($input);
$resultArray = [];
foreach ($iocArray as $ioc) {
@ -240,6 +256,69 @@ class ComplexTypeTool
return array_values($resultArray);
}
/**
* @param string $input
* @throws JsonException
*/
private function parseJson($input, array $settings)
{
$parsed = JsonTool::decode($input);
$values = [];
array_walk_recursive($parsed, function ($value) use (&$values) {
if (is_bool($value) || is_int($value) || empty($value)) {
return; // skip boolean, integer or empty values
}
$values[] = $value;
foreach ($this->parseFreetext($value) as $v) {
if ($v !== $value) {
$values[] = $v;
}
}
});
unset($parsed);
$resultArray = [];
foreach ($values as $ioc) {
$ioc = trim($ioc, '\'".,() ' . "\t\n\r\0\x0B"); // custom + default PHP trim
if (empty($ioc)) {
continue;
}
if (!empty($settings['excluderegex']) && preg_match($settings['excluderegex'], $ioc)) {
continue;
}
$typeArray = $this->__resolveType($ioc);
if ($typeArray === false) {
continue;
}
// Remove duplicates
if (isset($resultArray[$typeArray['value']])) {
continue;
}
$typeArray['original_value'] = $ioc;
$resultArray[$typeArray['value']] = $typeArray;
}
return array_values($resultArray);
}
/**
* @param string $input
* @return array|string[]
*/
private function parseFreetext($input)
{
$input = str_replace("\xc2\xa0", ' ', $input); // non breaking space to normal space
$input = preg_replace('/\p{C}+/u', ' ', $input);
$iocArray = preg_split("/\r\n|\n|\r|\s|\s+|,|\<|\>|;/", $input);
preg_match_all('/\"([^\"]+)\"/', $input, $matches);
foreach ($matches[1] as $match) {
$iocArray[] = $match;
}
return $iocArray;
}
/**
* @param string $raw_input Trimmed value
* @return array|false
@ -250,7 +329,6 @@ class ComplexTypeTool
if (filter_var($raw_input, FILTER_VALIDATE_IP)) {
return [
'types' => ['ip-dst', 'ip-src', 'ip-src/ip-dst'],
'to_ids' => true,
'default_type' => 'ip-dst',
'value' => $raw_input,
];
@ -296,7 +374,6 @@ class ComplexTypeTool
if (preg_match("#^([13][a-km-zA-HJ-NP-Z1-9]{25,34})|(bc|tb)1([023456789acdefghjklmnpqrstuvwxyz]{11,71})$#i", $input['raw'])) {
return [
'types' => ['btc'],
'to_ids' => true,
'default_type' => 'btc',
'value' => $input['raw'],
];
@ -311,7 +388,6 @@ class ComplexTypeTool
if (filter_var($input['refanged'], FILTER_VALIDATE_EMAIL)) {
return [
'types' => array('email', 'email-src', 'email-dst', 'target-email', 'whois-registrant-email'),
'to_ids' => true,
'default_type' => 'email-src',
'value' => $input['refanged'],
];
@ -324,7 +400,7 @@ class ComplexTypeTool
{
if (preg_match('#^as[0-9]+$#i', $input['raw'])) {
$input['raw'] = strtoupper($input['raw']);
return array('types' => array('AS'), 'to_ids' => false, 'default_type' => 'AS', 'value' => $input['raw']);
return array('types' => array('AS'), 'default_type' => 'AS', 'value' => $input['raw']);
}
return false;
}
@ -338,10 +414,10 @@ class ComplexTypeTool
if ($this->__resolveFilename($compositeParts[0])) {
$hash = $this->__resolveHash($compositeParts[1]);
if ($hash) {
return array('types' => $hash['composite'], 'to_ids' => true, 'default_type' => $hash['composite'][0], 'value' => $input['raw']);
return array('types' => $hash['composite'], 'default_type' => $hash['composite'][0], 'value' => $input['raw']);
}
if ($this->__resolveSsdeep($compositeParts[1])) {
return array('types' => array('filename|ssdeep'), 'to_ids' => true, 'default_type' => 'filename|ssdeep', 'value' => $input['raw']);
return array('types' => array('filename|ssdeep'), 'default_type' => 'filename|ssdeep', 'value' => $input['raw']);
}
}
}
@ -354,11 +430,11 @@ class ComplexTypeTool
if ($this->__checkForBTC($input)) {
$types[] = 'btc';
}
return array('types' => $types, 'to_ids' => true, 'default_type' => $types[0], 'value' => $input['raw']);
return array('types' => $types, 'default_type' => $types[0], 'value' => $input['raw']);
}
// ssdeep has a different pattern
if ($this->__resolveSsdeep($input['raw'])) {
return array('types' => array('ssdeep'), 'to_ids' => true, 'default_type' => 'ssdeep', 'value' => $input['raw']);
return array('types' => array('ssdeep'), 'default_type' => 'ssdeep', 'value' => $input['raw']);
}
return false;
}
@ -401,15 +477,15 @@ class ComplexTypeTool
if (preg_match("#^cve-[0-9]{4}-[0-9]{4,9}$#i", $input['raw'])) {
return [
'types' => ['vulnerability'],
'to_ids' => false,
'default_type' => 'vulnerability',
'value' => strtoupper($input['raw']), // 'CVE' must be uppercase
];
}
// Phone numbers - for automatic recognition, needs to start with + or include dashes
if ($input['raw'][0] === '+' || strpos($input['raw'], '-')) {
if (!preg_match('#^[0-9]{4}-[0-9]{2}-[0-9]{2}$#i', $input['raw']) && preg_match("#^(\+)?([0-9]{1,3}(\(0\))?)?[0-9\/\-]{5,}[0-9]$#i", $input['raw'])) {
return array('types' => array('phone-number', 'prtn', 'whois-registrant-phone'), 'to_ids' => false, 'default_type' => 'phone-number', 'value' => $input['raw']);
return array('types' => array('phone-number', 'prtn', 'whois-registrant-phone'), 'default_type' => 'phone-number', 'value' => $input['raw']);
}
}
return false;
@ -419,16 +495,15 @@ class ComplexTypeTool
{
if (filter_var($input['refanged_no_port'], FILTER_VALIDATE_IP)) {
if (isset($input['port'])) {
return array('types' => array('ip-dst|port', 'ip-src|port', 'ip-src|port/ip-dst|port'), 'to_ids' => true, 'default_type' => 'ip-dst|port', 'comment' => $input['comment'], 'value' => $input['refanged_no_port'] . '|' . $input['port']);
return array('types' => array('ip-dst|port', 'ip-src|port', 'ip-src|port/ip-dst|port'), 'default_type' => 'ip-dst|port', 'comment' => $input['comment'], 'value' => $input['refanged_no_port'] . '|' . $input['port']);
} else {
return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'to_ids' => true, 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
}
}
// IPv6 address that is considered as IP address with port
if (filter_var($input['refanged'], FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) {
return [
'types' => ['ip-dst', 'ip-src', 'ip-src/ip-dst'],
'to_ids' => true,
'default_type' => 'ip-dst',
'comment' => '',
'value' => $input['refanged'],
@ -443,7 +518,6 @@ class ComplexTypeTool
$value = substr($input['refanged_no_port'], 1, -1); // remove brackets
return [
'types' => ['ip-dst|port', 'ip-src|port', 'ip-src|port/ip-dst|port'],
'to_ids' => true,
'default_type' => 'ip-dst|port',
'comment' => $input['comment'],
'value' => "$value|{$input['port']}",
@ -453,7 +527,7 @@ class ComplexTypeTool
if (strpos($input['refanged_no_port'], '/')) {
$temp = explode('/', $input['refanged_no_port']);
if (count($temp) === 2 && filter_var($temp[0], FILTER_VALIDATE_IP) && is_numeric($temp[1])) {
return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'to_ids' => true, 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
}
}
return false;
@ -473,25 +547,24 @@ class ComplexTypeTool
}
if ($domainDetection) {
if (count($temp) > 2) {
return array('types' => array('hostname', 'domain', 'url', 'filename'), 'to_ids' => true, 'default_type' => 'hostname', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
return array('types' => array('hostname', 'domain', 'url', 'filename'), 'default_type' => 'hostname', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
} else {
return array('types' => array('domain', 'filename'), 'to_ids' => true, 'default_type' => 'domain', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
return array('types' => array('domain', 'filename'), 'default_type' => 'domain', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
}
} else {
// check if it is a URL
// Adding http:// infront of the input in case it was left off. github.com/MISP/MISP should still be counted as a valid link
if (count($temp) > 1 && (filter_var($input['refanged_no_port'], FILTER_VALIDATE_URL) || filter_var('http://' . $input['refanged_no_port'], FILTER_VALIDATE_URL))) {
// Even though some domains are valid, we want to exclude them as they are known security vendors / etc
// TODO, replace that with the appropriate warninglist.
if (preg_match('/^(https:\/\/(www.)?virustotal.com\/|https:\/\/www\.hybrid-analysis\.com\/)/i', $input['refanged_no_port'])) {
return array('types' => array('link'), 'to_ids' => false, 'default_type' => 'link', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
if ($this->isLink($input['refanged_no_port'])) {
return array('types' => array('link'), 'default_type' => 'link', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
}
if (strpos($input['refanged_no_port'], '/')) {
return array('types' => array('url'), 'to_ids' => true, 'default_type' => 'url', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
return array('types' => array('url'), 'default_type' => 'url', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
}
}
if ($this->__resolveFilename($input['raw'])) {
return array('types' => array('filename'), 'to_ids' => true, 'default_type' => 'filename', 'value' => $input['raw']);
return array('types' => array('filename'), 'default_type' => 'filename', 'value' => $input['raw']);
}
}
}
@ -499,10 +572,10 @@ class ComplexTypeTool
$temp = explode('\\', $input['raw']);
if (strpos(end($temp), '.') || preg_match('/^.:/i', $temp[0])) {
if ($this->__resolveFilename(end($temp))) {
return array('types' => array('filename'), 'to_ids' => true, 'default_type' => 'filename', 'value' => $input['raw']);
return array('types' => array('filename'), 'default_type' => 'filename', 'value' => $input['raw']);
}
} else if (!empty($temp[0])) {
return array('types' => array('regkey'), 'to_ids' => false, 'default_type' => 'regkey', 'value' => $input['raw']);
return array('types' => array('regkey'), 'default_type' => 'regkey', 'value' => $input['raw']);
}
}
return false;
@ -525,7 +598,7 @@ class ComplexTypeTool
*/
private function __resolveSsdeep($value)
{
return preg_match('#^[0-9]+:[0-9a-zA-Z\/\+]+:[0-9a-zA-Z\/\+]+$#', $value) && !preg_match('#^[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$#', $value);
return preg_match('#^[0-9]+:[0-9a-zA-Z/+]+:[0-9a-zA-Z/+]+$#', $value) && !preg_match('#^[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$#', $value);
}
/**
@ -553,6 +626,29 @@ class ComplexTypeTool
return isset($this->__tlds[strtolower($tld)]);
}
/**
* Check if URL should be considered as link attribute type
* @param string $value
* @return bool
*/
private function isLink($value)
{
if (!preg_match('/^https:\/\/([^\/]*)/i', $value, $matches)) {
return false;
}
$domainToCheck = '';
$domainParts = array_reverse(explode('.', strtolower($matches[1])));
foreach ($domainParts as $domainPart) {
$domainToCheck = $domainPart . $domainToCheck;
if (in_array($domainToCheck, $this->securityVendorDomains, true)) {
return true;
}
$domainToCheck = '.' . $domainToCheck;
}
return false;
}
private function __generateTLDList()
{
$tlds = array('biz', 'cat', 'com', 'edu', 'gov', 'int', 'mil', 'net', 'org', 'pro', 'tel', 'aero', 'arpa', 'asia', 'coop', 'info', 'jobs', 'mobi', 'name', 'museum', 'travel', 'onion');

11
app/Lib/Tools/HttpSocketExtended.php
View File

@ -110,16 +110,7 @@ class HttpSocketResponseExtended extends HttpSocketResponse
public function json()
{
try {
if (defined('JSON_THROW_ON_ERROR')) {
// JSON_THROW_ON_ERROR is supported since PHP 7.3
$decoded = json_decode($this->body, true, 512, JSON_THROW_ON_ERROR);
} else {
$decoded = json_decode($this->body, true);
if ($decoded === null) {
throw new UnexpectedValueException('Could not parse JSON: ' . json_last_error_msg(), json_last_error());
}
}
return $decoded;
return JsonTool::decode($this->body);
} catch (Exception $e) {
throw new HttpSocketJsonException('Could not parse response as JSON.', $this, $e);
}

27
app/Lib/Tools/JsonTool.php
View File

@ -26,6 +26,14 @@ class JsonTool
*/
public static function decode($value)
{
if (function_exists('simdjson_decode')) {
try {
return simdjson_decode($value, true);
} catch (SimdJsonException $e) {
throw new JsonException($e->getMessage(), $e->getCode(), $e);
}
}
if (defined('JSON_THROW_ON_ERROR')) {
// JSON_THROW_ON_ERROR is supported since PHP 7.3
return json_decode($value, true, 512, JSON_THROW_ON_ERROR);
@ -37,4 +45,23 @@ class JsonTool
}
return $decoded;
}
/**
* Check if string is valid JSON
* @param string $value
* @return bool
*/
public static function isValid($value)
{
if (function_exists('simdjson_is_valid')) {
return simdjson_is_valid($value);
}
try {
self::decode($value);
return true;
} catch (Exception $e) {
return false;
}
}
}

2
app/Lib/Tools/SecurityAudit.php
View File

@ -497,7 +497,7 @@ class SecurityAudit
if ($linuxVersion) {
list($name, $version) = $linuxVersion;
if ($name === 'Ubuntu') {
if (in_array($version, ['14.04', '16.04', '19.10', '20.10', '21.04'], true)) {
if (in_array($version, ['14.04', '16.04', '19.10', '20.10', '21.04', '21.10'], true)) {
$output['System'][] = [
'warning',
__('You are using Ubuntu %s. This version doesn\'t receive security support anymore.', $version),

52
app/Model/AppModel.php
View File

@ -2263,8 +2263,7 @@ class AppModel extends Model
public function valueIsJson($value)
{
$value = array_values($value)[0];
$json_decoded = json_decode($value);
if ($json_decoded === null) {
if (!JsonTool::isValid($value)) {
return __('Invalid JSON.');
}
return true;
@ -3368,27 +3367,35 @@ class AppModel extends Model
return (new RandomTool())->random_str(false, 12);
}
/**
* @param string|int $delta
* @return int Timestamp
*/
public function resolveTimeDelta($delta)
{
if (is_numeric($delta)) {
return $delta;
return (int)$delta;
}
$multiplierArray = array('d' => 86400, 'h' => 3600, 'm' => 60, 's' => 1);
$multiplierArray = ['d' => 86400, 'h' => 3600, 'm' => 60, 's' => 1];
$lastChar = strtolower(substr($delta, -1));
if (!is_numeric($lastChar) && isset($multiplierArray[$lastChar])) {
$multiplier = $multiplierArray[$lastChar];
$delta = substr($delta, 0, -1);
} else if (strtotime($delta) !== false) {
return strtotime($delta);
} else {
// invalid filter, make sure we don't return anything
return time() + 1;
$timeDelta = substr($delta, 0, -1);
if (!is_numeric($timeDelta)) {
$this->log('Invalid time filter format ' . $delta, LOG_NOTICE);
return time() + 1;
}
return time() - ($timeDelta * $multiplier);
}
if (!is_numeric($delta)) {
// Same here. (returning false dumps the whole database)
return time() + 1;
$time = strtotime($delta);
if ($time !== false) {
return $time;
}
return time() - ($delta * $multiplier);
$this->log('Invalid time filter format ' . $delta, LOG_NOTICE);
return time() + 1;
}
private function __fixServerPullPushRules()
@ -3728,7 +3735,7 @@ class AppModel extends Model
* @param array $logging If the execution failure should be logged
* @return boolean If the execution for the blocking path was a success
*/
public function executeTrigger($trigger_id, array $data=[], array &$blockingErrors=[], array $logging=[]): bool
protected function executeTrigger($trigger_id, array $data=[], array &$blockingErrors=[], array $logging=[]): bool
{
if ($this->isTriggerCallable($trigger_id)) {
$success = $this->Workflow->executeWorkflowForTriggerRouter($trigger_id, $data, $blockingErrors, $logging);
@ -3742,8 +3749,17 @@ class AppModel extends Model
return true;
}
public function isTriggerCallable($trigger_id): bool
protected function isTriggerCallable($trigger_id): bool
{
static $workflowEnabled;
if ($workflowEnabled === null) {
$workflowEnabled = (bool)Configure::read('Plugin.Workflow_enable');
}
if (!$workflowEnabled) {
return false;
}
if ($this->Workflow === null) {
$this->Workflow = ClassRegistry::init('Workflow');
}
@ -3790,7 +3806,7 @@ class AppModel extends Model
$this->Correlation->validEngines['Legacy'],
'Job created.'
);
$this->Correlation->Attribute->getBackgroundJobsTool()->enqueue(
$this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_ADMIN,
[
@ -3810,7 +3826,7 @@ class AppModel extends Model
'Job created.'
);
$this->Attribute->getBackgroundJobsTool()->enqueue(
$this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_ADMIN,
[

201
app/Model/Attribute.php
View File

@ -515,14 +515,16 @@ class Attribute extends AppModel
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $attributeForPublish, $action);
}
$workflowErrors = [];
$logging = [
if ($isTriggerCallable) {
$workflowErrors = [];
$logging = [
'model' => 'Attribute',
'action' => $action,
'id' => $attributeForPublish['Attribute']['id'],
];
$triggerData = $attributeForPublish;
$this->executeTrigger('attribute-after-save', $triggerData, $workflowErrors, $logging);
$triggerData = $attributeForPublish;
$this->executeTrigger('attribute-after-save', $triggerData, $workflowErrors, $logging);
}
}
}
if ($created && isset($attribute['event_id']) && empty($attribute['skip_auto_increment'])) {
@ -545,16 +547,15 @@ class Attribute extends AppModel
}
// update correlation..
$this->Correlation->beforeSaveCorrelation($attribute['Attribute']);
if (!empty($attribute['Attribute']['id'])) {
if ($this->pubToZmq('attribute')) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->attribute_save($attribute, 'delete');
}
$kafkaTopic = $this->kafkaTopic('attribute');
if ($kafkaTopic) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $attribute, 'delete');
}
if ($this->pubToZmq('attribute')) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->attribute_save($attribute, 'delete');
}
$kafkaTopic = $this->kafkaTopic('attribute');
if ($kafkaTopic) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $attribute, 'delete');
}
}
@ -1194,116 +1195,23 @@ class Attribute extends AppModel
}
/**
* @param int|false $jobId
* @param int|false $eventId
* @param int|false $attributeId
* @return int Number of processed attributes
* @param $jobId
* @param $eventId
* @param $attributeId
* @return void
* @throws Exception
* @deprecated Use Correlation::generateCorrelation directly
*/
public function generateCorrelation($jobId = false, $eventId = false, $attributeId = false)
{
$this->purgeCorrelations($eventId);
$this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep');
$this->FuzzyCorrelateSsdeep->purge($eventId, $attributeId);
$this->query('TRUNCATE TABLE over_correlating_values');
// get all attributes..
if (!$eventId) {
$eventIds = $this->Event->find('column', [
'fields' => ['Event.id'],
'conditions' => ['Event.disable_correlation' => 0],
]);
$full = true;
} else {
$eventIds = [$eventId];
$full = false;
}
$attributeCount = 0;
if (Configure::read('MISP.background_jobs') && $jobId) {
$this->Job = ClassRegistry::init('Job');
} else {
$jobId = false;
}
if (!empty($eventIds)) {
$eventCount = count($eventIds);
foreach ($eventIds as $j => $currentEventId) {
$attributeCount = $this->__iteratedCorrelation(
$jobId,
$full,
$attributeCount,
$attributeId,
$eventCount,
$currentEventId,
$j
);
}
} else {
// Not sure why that line was added. If there are no events, there are no correlations to save
// $attributeCount = $this->__iteratedCorrelation($jobId, $full, $attributeCount);
}
if ($jobId) {
$this->Job->saveStatus($jobId, true);
}
return $attributeCount;
}
private function __iteratedCorrelation(
$jobId = false,
$full = false,
$attributeCount = 0,
$attributeId = null,
$eventCount = null,
$eventId = null,
$j = 0
)
{
if ($jobId) {
$message = $attributeId ? __('Correlating Attribute %s', $attributeId) : __('Correlating Event %s (%s MB used)', $eventId, intval(memory_get_usage() / 1024 / 1024));
$this->Job->saveProgress($jobId, $message, !empty($eventCount) ? ($j / $eventCount) * 100 : 0);
}
$attributeConditions = [
'Attribute.deleted' => 0,
'Attribute.disable_correlation' => 0,
'NOT' => [
'Attribute.type' => Attribute::NON_CORRELATING_TYPES,
],
];
if ($eventId) {
$attributeConditions['Attribute.event_id'] = $eventId;
}
if ($attributeId) {
$attributeConditions['Attribute.id'] = $attributeId;
}
$query = [
'recursive' => -1,
'conditions' => $attributeConditions,
// fetch just necessary fields to save memory
'fields' => $this->Correlation->getFieldRules(),
'order' => 'Attribute.id',
'limit' => 5000,
'callbacks' => false, // memory leak fix
];
do {
$attributes = $this->find('all', $query);
foreach ($attributes as $attribute) {
$attribute['Attribute']['event_id'] = $eventId;
$this->Correlation->afterSaveCorrelation($attribute['Attribute'], $full);
}
$fetchedAttributes = count($attributes);
unset($attributes);
$attributeCount += $fetchedAttributes;
if ($fetchedAttributes === 5000) { // maximum number of attributes fetched, continue in next loop
$query['conditions']['Attribute.id >'] = $attribute['Attribute']['id'];
} else {
break;
}
} while (true);
// Generating correlations can take long time, so clear CIDR cache after each event to refresh cache
$this->Correlation->clearCidrCache();
return $attributeCount;
$this->Correlation->generateCorrelation($jobId, $eventId, $attributeId);
}
/**
* @param $eventId
* @return void
* @deprecated Use Correlation::purgeCorrelations directly
*/
public function purgeCorrelations($eventId = false)
{
$this->Correlation->purgeCorrelations($eventId);
@ -2389,8 +2297,8 @@ class Attribute extends AppModel
throw new InvalidArgumentException('Invalid date specification, must be string or array with two elements');
}
$timestamp[0] = intval($this->resolveTimeDelta($timestamp[0]));
$timestamp[1] = intval($this->resolveTimeDelta($timestamp[1]));
$timestamp[0] = $this->resolveTimeDelta($timestamp[0]);
$timestamp[1] = $this->resolveTimeDelta($timestamp[1]);
if ($timestamp[0] > $timestamp[1]) {
$temp = $timestamp[0];
$timestamp[0] = $timestamp[1];
@ -2399,7 +2307,7 @@ class Attribute extends AppModel
$conditions['AND'][] = array($scope . ' >=' => $timestamp[0]);
$conditions['AND'][] = array($scope . ' <=' => $timestamp[1]);
} else {
$timestamp = intval($this->resolveTimeDelta($timestamp));
$timestamp = $this->resolveTimeDelta($timestamp);
$conditions['AND'][] = array($scope . ' >=' => $timestamp);
}
if ($returnRaw) {
@ -2753,28 +2661,32 @@ class Attribute extends AppModel
return true;
}
/**
* @param int $id Attribute ID
* @param array $user
* @param bool $hard
* @return bool
* @throws Exception
*/
public function deleteAttribute($id, array $user, $hard = false)
{
$result = $this->fetchAttributes($user, array(
'conditions' => array('Attribute.id' => $id),
'flatten' => 1,
'deleted' => [0,1],
$attribute = $this->find('first', [
'conditions' => ['Attribute.id' => $id],
'contain' => ['Event'],
'recursive' => -1,
'contain' => array('Event')
));
if (empty($result)) {
throw new ForbiddenException(__('Invalid attribute'));
]);
if (empty($attribute)) {
throw new NotFoundException(__('Invalid attribute'));
}
$result = $result[0];
// check for permissions
if (!$user['Role']['perm_site_admin']) {
if ($result['Event']['locked']) {
if ($user['org_id'] != $result['Event']['org_id'] || !$user['Role']['perm_sync']) {
if ($attribute['Event']['locked']) {
if ($user['org_id'] != $attribute['Event']['org_id'] || !$user['Role']['perm_sync']) {
throw new ForbiddenException(__('You do not have permission to do that.'));
}
} else {
if ($user['org_id'] != $result['Event']['orgc_id']) {
if ($user['org_id'] != $attribute['Event']['orgc_id']) {
throw new ForbiddenException(__('You do not have permission to do that.'));
}
}
@ -2783,15 +2695,15 @@ class Attribute extends AppModel
$save = $this->delete($id);
} else {
if (Configure::read('Security.sanitise_attribute_on_delete')) {
$result['Attribute']['category'] = 'Other';
$result['Attribute']['type'] = 'comment';
$result['Attribute']['value'] = 'deleted';
$result['Attribute']['comment'] = '';
$result['Attribute']['to_ids'] = 0;
$attribute['Attribute']['category'] = 'Other';
$attribute['Attribute']['type'] = 'comment';
$attribute['Attribute']['value'] = 'deleted';
$attribute['Attribute']['comment'] = '';
$attribute['Attribute']['to_ids'] = 0;
}
$result['Attribute']['deleted'] = 1;
$result['Attribute']['timestamp'] = time();
$save = $this->save($result);
$attribute['Attribute']['deleted'] = 1;
$attribute['Attribute']['timestamp'] = time();
$save = $this->save($attribute);
$object_refs = $this->Object->ObjectReference->find('all', array(
'conditions' => array(
'ObjectReference.referenced_type' => 0,
@ -2810,11 +2722,10 @@ class Attribute extends AppModel
$this->Event->ShadowAttribute->deleteAll(array('ShadowAttribute.old_id' => $id), false);
// remove the published flag from the event
$this->Event->unpublishEvent($result['Event']['id']);
$this->Event->unpublishEvent($attribute);
return true;
} else {
return false;
}
return false;
}
public function attachValidationWarnings($adata)

144
app/Model/Correlation.php
View File

@ -12,6 +12,7 @@ App::uses('AppModel', 'Model');
* @method getFieldRules
* @method getContainRules($filter = null)
* @method updateContainedCorrelations(array $data, string $type, array $options = [])
* @method purgeCorrelations(int $evenId = null)
*/
class Correlation extends AppModel
{
@ -107,6 +108,138 @@ class Correlation extends AppModel
}
}
/**
* Generate correlation for given attributes or events.
*
* @param int|false $jobId
* @param int|false $eventId
* @param int|false $attributeId
* @return int Number of processed attributes
* @throws Exception
*/
public function generateCorrelation($jobId = false, $eventId = false, $attributeId = false)
{
$this->purgeCorrelations($eventId);
$this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep');
$this->FuzzyCorrelateSsdeep->purge($eventId, $attributeId);
$this->OverCorrelatingValue->truncateTable();
if (!$eventId) {
$eventIds = $this->Event->find('column', [
'fields' => ['Event.id'],
'conditions' => ['Event.disable_correlation' => 0],
]);
$full = true;
} else {
$eventIds = [$eventId];
$full = false;
}
$attributeCount = 0;
if (Configure::read('MISP.background_jobs') && $jobId) {
$this->Job = ClassRegistry::init('Job');
} else {
$jobId = false;
}
if (!empty($eventIds)) {
$eventCount = count($eventIds);
foreach ($eventIds as $j => $currentEventId) {
$attributeCount += $this->__iteratedCorrelation(
$jobId,
$full,
$attributeId,
$eventCount,
$currentEventId,
$j
);
}
}
if ($jobId) {
$this->Job->saveStatus($jobId, true);
}
return $attributeCount;
}
/**
* @param int|false $jobId
* @param bool $full
* @param int $attributeId
* @param int $eventCount
* @param int $eventId
* @param int $j
* @return int
* @throws Exception
*/
private function __iteratedCorrelation(
$jobId = false,
$full = false,
$attributeId = null,
$eventCount = null,
$eventId = null,
$j = 0
)
{
if ($jobId) {
$message = $attributeId ? __('Correlating Attribute %s', $attributeId) : __('Correlating Event %s (%s MB used)', $eventId, intval(memory_get_usage() / 1024 / 1024));
$this->Job->saveProgress($jobId, $message, !empty($eventCount) ? ($j / $eventCount) * 100 : 0);
}
$attributeConditions = [
'Attribute.deleted' => 0,
'Attribute.disable_correlation' => 0,
'NOT' => [
'Attribute.type' => Attribute::NON_CORRELATING_TYPES,
],
];
if ($eventId) {
$attributeConditions['Attribute.event_id'] = $eventId;
$event = $this->Event->find('first', [
'conditions' => ['Event.id' => $eventId],
'recursive' => -1,
'fields' => $this->getContainRules('Event')['fields'],
]);
if (empty($event)) {
return 0; // event not found, skip
}
} else {
$event = false;
}
if ($attributeId) {
$attributeConditions['Attribute.id'] = $attributeId;
}
$query = [
'recursive' => -1,
'conditions' => $attributeConditions,
// fetch just necessary fields to save memory
'fields' => $this->getFieldRules(),
'order' => 'Attribute.id',
'limit' => 5000,
'callbacks' => false, // memory leak fix
];
$attributeCount = 0;
do {
$attributes = $this->Attribute->find('all', $query);
foreach ($attributes as $attribute) {
$this->afterSaveCorrelation($attribute['Attribute'], $full, $event);
}
$fetchedAttributes = count($attributes);
unset($attributes);
$attributeCount += $fetchedAttributes;
if ($fetchedAttributes === 5000) { // maximum number of attributes fetched, continue in next loop
$query['conditions']['Attribute.id >'] = $attribute['Attribute']['id'];
} else {
break;
}
} while (true);
// Generating correlations can take long time, so clear caches after each event to refresh them
$this->cidrListCache = null;
$this->OverCorrelatingValue->cleanCache();
$this->containCache = [];
return $attributeCount;
}
/**
* @param array $attribute Simple attribute array
* @return array|null
@ -326,6 +459,9 @@ class Correlation extends AppModel
if ($cV === null) {
continue;
}
if ($full && $this->OverCorrelatingValue->isBlocked($cV)) {
continue; // skip already blocked values when doing full correlation
}
$conditions = [
'OR' => [
'Attribute.value1' => $cV,
@ -777,14 +913,6 @@ class Correlation extends AppModel
return $cidrList;
}
/**
* @return void
*/
public function clearCidrCache()
{
$this->cidrListCache = null;
}
/**
* @return array
*/

87
app/Model/Event.php
View File

@ -2823,15 +2823,21 @@ class Event extends AppModel
return $conditions;
}
/**
* @param array $params
* @param array $conditions
* @param array $options
* @return array
*/
public function set_filter_timestamp(&$params, $conditions, $options)
{
if ($options['filter'] == 'from') {
if ($options['filter'] === 'from') {
if (is_numeric($params['from'])) {
$conditions['AND']['Event.date >='] = date('Y-m-d', $params['from']);
} else {
$conditions['AND']['Event.date >='] = $params['from'];
}
} elseif ($options['filter'] == 'to') {
} elseif ($options['filter'] === 'to') {
if (is_numeric($params['to'])) {
$conditions['AND']['Event.date <='] = date('Y-m-d', $params['to']);
} else {
@ -3706,47 +3712,9 @@ class Event extends AppModel
if ($fromXml) {
$created_id = $this->id;
}
$userForWorkflow = $this->User->getAuthUser(Configure::read('CurrentUserId'), true);
$userForWorkflow['Role']['perm_site_admin'] = 1;
if ($fromPull) {
if ($this->isTriggerCallable('event-after-save-new-from-pull')) {
$fullSavedEvent = $this->fetchEvent($userForWorkflow, [
'eventid' => $this->id,
'includeAttachments' => 1
])[0];
$workflowErrors = [];
$logging = [
'model' => 'Event',
'action' => 'add',
'id' => $this->id,
];
$triggerData = $fullSavedEvent;
$triggerData['_server'] = $server;
$success = $this->executeTrigger('event-after-save-new-from-pull', $triggerData, $workflowErrors, $logging);
if (empty($success)) {
$errorMessage = implode(', ', $workflowErrors);
return $errorMessage;
}
}
} else {
if ($this->isTriggerCallable('event-after-save-new')) {
$fullSavedEvent = $this->fetchEvent($userForWorkflow, [
'eventid' => $this->id,
'includeAttachments' => 1
])[0];
$workflowErrors = [];
$logging = [
'model' => 'Event',
'action' => 'add',
'id' => $this->id,
];
$triggerData = $fullSavedEvent;
$success = $this->executeTrigger('event-after-save-new', $triggerData, $workflowErrors, $logging);
if (empty($success)) {
$errorMessage = implode(', ', $workflowErrors);
return $errorMessage;
}
}
$workflowResult = $this->afterAddWorkflow($this->id, $fromPull);
if (is_array($workflowResult)) {
return implode(', ', $workflowResult);
}
if (!empty($data['Event']['published']) && 1 == $data['Event']['published']) {
// do the necessary actions to publish the event (email, upload,...)
@ -3798,6 +3766,38 @@ class Event extends AppModel
}
}
/**
* @param int $eventId
* @param bool $fromPull
* @return true|array
*/
private function afterAddWorkflow($eventId, $fromPull)
{
$triggerId = $fromPull ? 'event-after-save-new-from-pull' : 'event-after-save-new';
if (!$this->isTriggerCallable($triggerId)) {
return true;
}
$userForWorkflow = $this->User->getAuthUser(Configure::read('CurrentUserId'), true);
$userForWorkflow['Role']['perm_site_admin'] = 1;
$fullSavedEvent = $this->fetchEvent($userForWorkflow, [
'eventid' => $eventId,
'includeAttachments' => 1
])[0];
$workflowErrors = [];
$logging = [
'model' => 'Event',
'action' => 'add',
'id' => $eventId,
];
$success = $this->executeTrigger($triggerId, $fullSavedEvent, $workflowErrors, $logging);
if (!$success) {
return $workflowErrors;
}
return true;
}
public function _edit(array &$data, array $user, $id = null, $jobId = null, $passAlong = null, $force = false)
{
$data = $this->cleanupEventArrayFromXML($data);
@ -5446,6 +5446,7 @@ class Event extends AppModel
}
$this->Warninglist = ClassRegistry::init('Warninglist');
$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
$complexTypeTool->setSecurityVendorDomains($this->Warninglist->fetchSecurityVendorDomains());
$freetextResults = array_merge($freetextResults, $complexTypeTool->checkFreeText($value));
if (!empty($freetextResults)) {
foreach ($freetextResults as &$ft) {

1
app/Model/EventReport.php
View File

@ -738,6 +738,7 @@ class EventReport extends AppModel
$complexTypeTool = new ComplexTypeTool();
$this->Warninglist = ClassRegistry::init('Warninglist');
$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
$complexTypeTool->setSecurityVendorDomains($this->Warninglist->fetchSecurityVendorDomains());
$complexTypeToolResult = $complexTypeTool->checkFreeText($report['EventReport']['content']);
$replacementResult = $this->transformFreeTextIntoReplacement($user, $report, $complexTypeToolResult);

1
app/Model/Feed.php
View File

@ -377,6 +377,7 @@ class Feed extends AppModel
$complexTypeTool = new ComplexTypeTool();
$this->Warninglist = ClassRegistry::init('Warninglist');
$complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
$complexTypeTool->setSecurityVendorDomains($this->Warninglist->fetchSecurityVendorDomains());
$settings = array();
if (!empty($feed['Feed']['settings']) && !is_array($feed['Feed']['settings'])) {
$feed['Feed']['settings'] = json_decode($feed['Feed']['settings'], true);

6
app/Model/Noticelist.php
View File

@ -25,8 +25,6 @@ class Noticelist extends AppModel
)
);
private $__entries = array();
public function beforeValidate($options = array())
{
parent::beforeValidate();
@ -44,9 +42,7 @@ class Noticelist extends AppModel
$directories = glob(APP . 'files' . DS . 'noticelists' . DS . 'lists' . DS . '*', GLOB_ONLYDIR);
$updated = array();
foreach ($directories as $dir) {
$file = new File($dir . DS . 'list.json');
$list = json_decode($file->read(), true);
$file->close();
$list = FileAccessTool::readJsonFromFile($dir . DS . 'list.json');
if (!isset($list['version'])) {
$list['version'] = 1;
}

6
app/Model/ObjectTemplateElement.php
View File

@ -11,13 +11,13 @@ class ObjectTemplateElement extends AppModel
{
foreach ($results as &$result) {
if (isset($result['ObjectTemplateElement']['categories'])) {
$result['ObjectTemplateElement']['categories'] = json_decode($result['ObjectTemplateElement']['categories'], true);
$result['ObjectTemplateElement']['categories'] = JsonTool::decode($result['ObjectTemplateElement']['categories']);
}
if (isset($result['ObjectTemplateElement']['values_list'])) {
$result['ObjectTemplateElement']['values_list'] = json_decode($result['ObjectTemplateElement']['values_list'], true);
$result['ObjectTemplateElement']['values_list'] = JsonTool::decode($result['ObjectTemplateElement']['values_list']);
}
if (isset($result['ObjectTemplateElement']['sane_default'])) {
$result['ObjectTemplateElement']['sane_default'] = json_decode($result['ObjectTemplateElement']['sane_default'], true);
$result['ObjectTemplateElement']['sane_default'] = JsonTool::decode($result['ObjectTemplateElement']['sane_default']);
}
}
return $results;

61
app/Model/OverCorrelatingValue.php
View File

@ -5,17 +5,8 @@ class OverCorrelatingValue extends AppModel
{
public $recursive = -1;
public function beforeValidate($options = array())
{
$this->data['OverCorrelatingValue']['value'] = self::truncate($this->data['OverCorrelatingValue']['value']);
return true;
}
public function beforeSave($options = array())
{
$this->data['OverCorrelatingValue']['value'] = self::truncate($this->data['OverCorrelatingValue']['value']);
return true;
}
/** @var array */
private $blockedValues = [];
public static function truncate(string $value): string
{
@ -31,27 +22,41 @@ class OverCorrelatingValue extends AppModel
/**
* @param string $value
* @param int $count
* @return bool
*/
public function isBlocked($value)
{
$value = self::truncate($value);
if (isset($this->blockedValues[$value])) {
return $this->blockedValues[$value];
}
$isBlocked = $this->hasAny(['value' => $value]);
return $this->blockedValues[$value] = $isBlocked;
}
/**
* @param string $value
* @return void
* @throws Exception
*/
public function block($value, $count = 0)
public function block($value)
{
$this->unblock($value);
$this->create();
$this->save(
[
'value' => $value,
'occurrence' => $count
]
);
if (!$this->isBlocked($value)) {
$this->create();
$this->save([
'value' => self::truncate($value),
'occurrence' => 0
]);
$this->blockedValues[$value] = true;
}
}
/**
* @param string $value
* @return void
*/
public function unBlock($value)
public function unblock($value)
{
$this->deleteAll(
[
@ -59,6 +64,12 @@ class OverCorrelatingValue extends AppModel
],
false
);
$this->blockedValues[$value] = false;
}
public function cleanCache()
{
$this->blockedValues = [];
}
/**
@ -151,4 +162,10 @@ class OverCorrelatingValue extends AppModel
}
$this->saveMany($overCorrelations);
}
public function truncateTable()
{
$this->query('TRUNCATE TABLE over_correlating_values');
$this->cleanCache();
}
}

18
app/Model/Server.php
View File

@ -2686,16 +2686,6 @@ class Server extends AppModel
];
}
public function isJson($string)
{
try {
$this->jsonDecode($string);
return true;
} catch (Exception $e) {
return false;
}
}
public function captureServer($server, $user)
{
if (isset($server[0])) {
@ -2961,8 +2951,7 @@ class Server extends AppModel
public function getExpectedDBSchema()
{
try {
$content = FileAccessTool::readFromFile(ROOT . DS . 'db_schema.json');
return JsonTool::decode($content);
return FileAccessTool::readJsonFromFile(ROOT . DS . 'db_schema.json');
} catch (Exception $e) {
return false;
}
@ -3337,7 +3326,7 @@ class Server extends AppModel
$scriptFile = APP . 'files' . DS . 'scripts' . DS . 'yaratest.py';
try {
$scriptResult = ProcessTool::execute([ProcessTool::pythonBin(), $scriptFile]);
$scriptResult = json_decode($scriptResult, true);
$scriptResult = JsonTool::decode($scriptResult);
} catch (Exception $exception) {
$this->logException('Failed to run yara diagnostics.', $exception);
return array(
@ -3366,7 +3355,7 @@ class Server extends AppModel
}
try {
$scriptResult = $this->jsonDecode($scriptResult);
$scriptResult = JsonTool::decode($scriptResult);
} catch (Exception $e) {
$this->logException('Invalid JSON returned from stixtest', $e);
return array(
@ -3521,7 +3510,6 @@ class Server extends AppModel
{
$sessionCount = null;
$sessionHandler = null;
$errorCode = 9;
switch (Configure::read('Session.defaults')) {
case 'php':

7
app/Model/Sighting.php
View File

@ -891,6 +891,13 @@ class Sighting extends AppModel
return $sightings;
}
/**
* @param array $user
* @param string $returnFormat
* @param array $filters
* @return TmpFileTool
* @throws Exception
*/
public function restSearch($user, $returnFormat, $filters)
{
$allowedContext = array('event', 'attribute');

29
app/Model/User.php
View File

@ -1663,19 +1663,9 @@ class User extends AppModel
'trending_period_amount' => 2,
];
$periodicSettings = $this->UserSetting->find('first', [
'recursive' => -1,
'conditions' => [
'user_id' => $userId,
'setting' => self::PERIODIC_USER_SETTING_KEY,
],
'fields' => ['value'],
]);
if (empty($periodicSettings)) {
$periodicSettings = $defaultPeriodicSettings;
} else {
$periodicSettings = $periodicSettings['UserSetting']['value'];
}
$periodicSettings = $this->UserSetting->getValueForUser($userId, self::PERIODIC_USER_SETTING_KEY);
$periodicSettings = $periodicSettings ?: $defaultPeriodicSettings;
$periodicSettingsIndexed = [];
foreach ($filterNames as $filterName) {
$periodicSettingsIndexed[$filterName] = $periodicSettings[$filterName] ?? $defaultPeriodicSettings[$filterName];
@ -1786,11 +1776,12 @@ class User extends AppModel
* @param int $userId
* @param string $period Can be 'daily', 'weekly' or 'monthly'
* @param bool $rendered When false, instance of SendEmailTemplate will returned
* @return string|SendEmailTemplate
* @return string|SendEmailTemplate|null
* @throws NotFoundException
* @throws InvalidArgumentException
* @throws JsonException
*/
public function generatePeriodicSummary(int $userId, string $period, $rendered=true)
public function generatePeriodicSummary(int $userId, string $period, $rendered = true)
{
$allowedPeriods = array_map(function($period) {
return substr($period, strlen('notification_'));
@ -1814,6 +1805,10 @@ class User extends AppModel
$filters['includeScoresOnEvent'] = true;
$events = $this->Event->fetchEvent($user, $filters);
if (empty($events)) {
return null;
}
$elementCounter = 0;
$renderView = false;
$filtersForRestSearch['publish_timestamp'] = $filtersForRestSearch['last'];
@ -1838,7 +1833,7 @@ class User extends AppModel
$securityRecommendationsData = [
'course_of_action' => $this->Event->extractRelatedCourseOfActions($events),
];
$security_recommendations = $this->__renderSecurityRecommenrations($securityRecommendationsData);
$security_recommendations = $this->__renderSecurityRecommendations($securityRecommendationsData);
$emailTemplate = $this->prepareEmailTemplate($period);
$emailTemplate->set('baseurl', $this->Event->__getAnnounceBaseurl());
@ -1869,7 +1864,7 @@ class User extends AppModel
return $this->__renderGeneric('Elements' . DS . 'Events', 'trendingSummary', $trendData);
}
private function __renderSecurityRecommenrations(array $data): string
private function __renderSecurityRecommendations(array $data): string
{
return $this->__renderGeneric('Elements' . DS . 'Events', 'securityRecommendations', $data);
}

29
app/Model/Warninglist.php
View File

@ -727,22 +727,33 @@ class Warninglist extends AppModel
'conditions' => array('Warninglist.name' => self::TLDS),
'fields' => array('Warninglist.id')
));
$tlds = array();
if (!empty($tldLists)) {
$tlds = $this->WarninglistEntry->find('column', array(
'conditions' => array('WarninglistEntry.warninglist_id' => $tldLists),
'fields' => array('WarninglistEntry.value')
));
foreach ($tlds as $key => $value) {
$tlds[$key] = strtolower($value);
}
$tlds = [];
foreach ($tldLists as $warninglistId) {
$tlds = array_merge($tlds, $this->getWarninglistEntries($warninglistId));
}
$tlds = array_map('strtolower', $tlds);
if (!in_array('onion', $tlds, true)) {
$tlds[] = 'onion';
}
return $tlds;
}
/**
* @return array
*/
public function fetchSecurityVendorDomains()
{
$securityVendorList = $this->find('column', array(
'conditions' => array('Warninglist.name' => 'List of known domains used by automated malware analysis services & security vendors'),
'fields' => array('Warninglist.id')
));
$domains = [];
foreach ($securityVendorList as $warninglistId) {
$domains = array_merge($domains, $this->getWarninglistEntries($warninglistId));
}
return $domains;
}
/**
* @param array $attribute
* @param array|null $warninglists If null, all enabled warninglists will be used

17
app/Model/Workflow.php
View File

@ -157,9 +157,23 @@ class Workflow extends AppModel
$this->toggleModules($this->modules_enabled_by_default, true, false);
}
/**
* @param string $trigger_id
* @return bool
* @throws WorkflowDuplicatedModuleIDException
*/
protected function checkTriggerEnabled($trigger_id)
{
$settingName = sprintf('Plugin.Workflow_triggers_%s', $trigger_id);
static $enabled;
if ($enabled === null) {
$enabled = (bool)Configure::read('Plugin.Workflow_enable');
}
if (!$enabled) {
return false;
}
$settingName = "Plugin.Workflow_triggers_$trigger_id";
$module_disabled = empty(Configure::read($settingName));
if ($module_disabled) {
return false;
@ -415,6 +429,7 @@ class Workflow extends AppModel
*
* @param string $trigger_id
* @param array $data
* @return bool
* @throws TriggerNotFoundException
*/
public function executeWorkflowForTriggerRouter($trigger_id, array $data, array &$blockingErrors=[], array $logging=[]): bool

8
app/Model/WorkflowBlueprint.php
View File

@ -1,7 +1,6 @@
<?php
App::uses('AppModel', 'Model');
App::uses('Folder', 'Utility');
App::uses('File', 'Utility');
class WorkflowBlueprint extends AppModel
{
@ -18,10 +17,9 @@ class WorkflowBlueprint extends AppModel
],
];
public $belongsTo = [
];
public $validate = [
'name' => 'stringNotEmpty',
'value' => [
'stringNotEmpty' => [
'rule' => ['stringNotEmpty']
@ -92,11 +90,11 @@ class WorkflowBlueprint extends AppModel
return $blueprint;
}
/**
* __readBlueprintsFromRepo Reads blueprints from the misp-workflow-blueprints repository
*
* @return array
* @throws Exception
*/
private function __readBlueprintsFromRepo(): array
{
@ -114,6 +112,7 @@ class WorkflowBlueprint extends AppModel
*
* @param boolean $force
* @return void
* @throws Exception
*/
public function update($force=false)
{
@ -133,7 +132,6 @@ class WorkflowBlueprint extends AppModel
if ($force || $blueprint_from_repo['timestamp'] > $existing_blueprint['timestamp']) {
$blueprint_from_repo['id'] = $existing_blueprint['id'];
$this->save($blueprint_from_repo);
continue;
}
} else {
$this->create();

3
app/Test/ComplexTypeToolTest.php
View File

@ -361,11 +361,9 @@ EOT;
$this->assertEquals('https://www.virustotal.com/example', $results[0]['value']);
$this->assertEquals('link', $results[0]['default_type']);
$this->assertFalse($results[0]['to_ids']);
$this->assertEquals('https://virustotal.com/example', $results[1]['value']);
$this->assertEquals('link', $results[1]['default_type']);
$this->assertFalse($results[1]['to_ids']);
}
public function testCheckFreeTextUrlHybridAnalysis(): void
@ -375,7 +373,6 @@ EOT;
$this->assertCount(1, $results);
$this->assertEquals('https://www.hybrid-analysis.com/example', $results[0]['value']);
$this->assertEquals('link', $results[0]['default_type']);
$this->assertFalse($results[0]['to_ids']);
}
// Issue https://github.com/MISP/MISP/issues/4908

4
app/View/Dashboards/widget_loader.ctp
View File

@ -6,11 +6,11 @@
<?= $scopedHtml['bundle']; ?>
</div>
<?php if ($config['autoRefreshDelay']): ?>
<script type="text/javascript">
<script>
$(function() {
setTimeout(function() {
updateDashboardWidget("#widget_<?= h($widget_id) ?>")},
<?= $config['autoRefreshDelay'] ? $config['autoRefreshDelay'] : 1 ?> * 1000
<?= $config['autoRefreshDelay'] ?: 1 ?> * 1000
);
});
</script>

19
app/View/Elements/genericElements/IndexTable/headers.ctp
View File

@ -1,5 +1,6 @@
<?php
$headersHtml = '';
$selectAllCheckbox = false;
echo '<thead>';
foreach ($fields as $k => $header) {
if (!isset($header['requirement']) || $header['requirement']) {
$header_data = '';
@ -11,6 +12,7 @@
}
} else {
if (!empty($header['element']) && $header['element'] === 'selector') {
$selectAllCheckbox = true;
$header_data = sprintf(
'<input id="select_all" class="%s" type="checkbox" %s>',
empty($header['select_all_class']) ? 'select_all' : $header['select_all_class'],
@ -29,7 +31,7 @@
$header_data = "<div><span>$header_data</span></div>";
}
$headersHtml .= sprintf(
echo sprintf(
'<th%s%s>%s</th>',
!empty($classes) ? ' class="' . implode(' ', $classes) .'"' : '',
!empty($header['header_title']) ? ' title="' . h($header['header_title']) . '"' : '',
@ -38,18 +40,16 @@
}
}
if ($actions) {
$headersHtml .= sprintf(
echo sprintf(
'<th class="actions">%s</th>',
__('Actions')
);
}
$thead = '<thead>';
$thead .= $headersHtml;
$thead .= '</thead>';
echo $thead;
echo '</thead>';
?>
<script type="text/javascript">
$(document).ready(function() {
<?php if ($selectAllCheckbox): ?>
<script>
$(function() {
$('.select_attribute').add('#select_all').on('change', function() {
if ($('.select_attribute:checked').length > 0) {
$('.mass-select').show();
@ -59,3 +59,4 @@
});
});
</script>
<?php endif; ?>

6
app/View/Elements/genericElements/SideMenu/side_menu.ctp
View File

@ -1664,13 +1664,13 @@ $divider = $this->element('/genericElements/SideMenu/side_menu_divider');
if ($isSiteAdmin) {
echo $this->element('/genericElements/SideMenu/side_menu_link', array(
'element_id' => 'edit',
'url' => '/workflows/edit/' . h($id),
'url' => '/workflowBlueprints/edit/' . h($id),
'text' => __('Edit Workflow Blueprint')
));
}
echo $this->element('/genericElements/SideMenu/side_menu_link', array(
'url' => '/admin/audit_logs/index/model:WorkflowBlueprints/model_id:' . h($id),
'text' => __('View worflow blueprint history'),
'url' => '/admin/audit_logs/index/model:WorkflowBlueprint/model_id:' . h($id),
'text' => __('View workflow blueprint history'),
'requirement' => Configure::read('MISP.log_new_audit') && $canAccess('auditLogs', 'admin_index'),
));
}

2
app/View/Helper/ScopedCSSHelper.php
View File

@ -28,7 +28,6 @@ App::uses('AppHelper', 'View/Helper');
return sprintf("<style>%s%s%s</style>", PHP_EOL, $cssScopedLines, PHP_EOL);
}
/**
* Replace a declared CSS scoped style and prepend a random CSS data filter to any CSS selector discovered.
* Usage: Add the following style tag `<style widget-scoped>` to use the scoped feature. Nearly every selector path will have their rule modified to adhere to the scope
@ -61,7 +60,6 @@ App::uses('AppHelper', 'View/Helper');
*/
public function createScopedCSS($html)
{
$css = "";
$seed = "";
$originalHtml = $html;
$bundle = $originalHtml;

13
app/View/Users/view_periodic_summary.ctp
View File

@ -1,8 +1,8 @@
<div class="index">
<div class="btn-group">
<a class="btn <?= $period == 'daily' ? 'btn-primary' : 'btn-inverse' ?>" href="<?= $baseurl . '/users/viewPeriodicSummary/daily' ?>"><?= __('Daily') ?></a>
<a class="btn <?= $period == 'weekly' ? 'btn-primary' : 'btn-inverse' ?>" href="<?= $baseurl . '/users/viewPeriodicSummary/weekly' ?>"><?= __('Weekly') ?></a>
<a class="btn <?= $period == 'monthly' ? 'btn-primary' : 'btn-inverse' ?>" href="<?= $baseurl . '/users/viewPeriodicSummary/monthly' ?>"><?= __('Monthly') ?></a>
<a class="btn <?= $period === 'daily' ? 'btn-primary' : 'btn-inverse' ?>" href="<?= $baseurl . '/users/viewPeriodicSummary/daily' ?>"><?= __('Daily') ?></a>
<a class="btn <?= $period === 'weekly' ? 'btn-primary' : 'btn-inverse' ?>" href="<?= $baseurl . '/users/viewPeriodicSummary/weekly' ?>"><?= __('Weekly') ?></a>
<a class="btn <?= $period === 'monthly' ? 'btn-primary' : 'btn-inverse' ?>" href="<?= $baseurl . '/users/viewPeriodicSummary/monthly' ?>"><?= __('Monthly') ?></a>
</div>
<h2><?= __('MISP %s summary', h($period)); ?></h2>
@ -17,10 +17,7 @@
</pre>
</div>
<div class="report-container" style="margin-top: 2em;">
<?= $summary; ?>
<?= $summary ?: __('No new events for this period'); ?>
</div>
</div>
<?php
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event-collection', 'menuItem' => 'viewPeriodicSummary'));
?>
<?= $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event-collection', 'menuItem' => 'viewPeriodicSummary'));

2
app/View/WorkflowBlueprints/import.ctp
View File

@ -5,7 +5,7 @@ echo $this->element('genericElements/Form/genericForm', [
'enctype' => 'multipart/form-data',
],
'data' => [
'model' => 'Workflow',
'model' => 'WorkflowBlueprint',
'title' => __('Import Workflow Blueprint'),
'description' => __('Paste a JSON of a Workflow blueprint to import it or provide a JSON file below.'),
'fields' => [

16
app/View/WorkflowBlueprints/index.ctp
View File

@ -32,10 +32,10 @@
'name' => __('Timestamp'),
'sort' => 'WorkflowBlueprint.timestamp',
'data_path' => 'WorkflowBlueprint.timestamp',
'element' => 'datetime',
],
];
echo $this->element('genericElements/IndexTable/scaffold', [
'scaffold_data' => [
'data' => [
@ -73,11 +73,13 @@
'url_params_data_paths' => ['WorkflowBlueprint.id'],
'icon' => 'eye',
'dbclickAction' => true,
'title' => __('View'),
],
[
'url' => $baseurl . '/workflowBlueprints/edit',
'url_params_data_paths' => ['WorkflowBlueprint.id'],
'icon' => 'edit',
'title' => __('Edit'),
],
[
'url' => $baseurl . '/workflowBlueprints/export',
@ -86,13 +88,11 @@
'icon' => 'download',
],
[
'onclick' => sprintf(
'openGenericModal(\'%s/workflowBlueprints/delete/[onclick_params_data_path]\');',
$baseurl
),
'onclick_params_data_path' => 'WorkflowBlueprint.id',
'icon' => 'trash'
'class' => 'modal-open',
'url' => $baseurl . '/workflowBlueprints/delete/',
'url_params_data_paths' => 'WorkflowBlueprint.id',
'icon' => 'trash',
'title' => __('Delete'),
]
]
]

1
app/composer.json
View File

@ -31,6 +31,7 @@
"ext-bcmath": "For faster validating IBAN numbers",
"ext-rdkafka": "Required for publishing events to Kafka broker",
"ext-apcu": "To cache data in memory instead of file system",
"ext-simdjson": "To decode JSON structures faster",
"elasticsearch/elasticsearch": "For logging to elasticsearch",
"aws/aws-sdk-php": "To upload samples to S3",
"jakub-onderka/openid-connect-php": "For OIDC authentication",

6
app/webroot/doc/openapi.yaml
View File

@ -5378,8 +5378,10 @@ components:
$ref: "#/components/schemas/DateRestSearchFilter"
LastRestSearchFilter:
description: "Published within the last x amount of time, where x can be defined in days, hours, minutes (for example 5d or 12h or 30m)"
type: integer
description: "Events published within the last x amount of time, where x can be defined in days, hours, minutes (for example 5d or 12h or 30m), ISO 8601 datetime format or timestamp"
oneOf:
- type: integer
- type: string
nullable: true
TagsRestSearchFilter:

31
tests/testlive_comprehensive_local.py
View File

@ -565,6 +565,7 @@ class TestComprehensive(unittest.TestCase):
with MISPSetting(self.admin_misp_connector, {"MISP.correlation_engine": "NoAcl"}):
self.test_correlations()
self.test_correlations_object()
self.test_recorrelate()
def test_advanced_correlations(self):
with MISPSetting(self.admin_misp_connector, {"MISP.enable_advanced_correlations": True}):
@ -594,6 +595,36 @@ class TestComprehensive(unittest.TestCase):
check_response(result)
self.assertIn("message", result)
def test_recorrelate(self):
first = create_simple_event()
dom_ip_obj = DomainIPObject({'ip': ['10.0.0.1']})
first.add_object(dom_ip_obj)
first = check_response(self.admin_misp_connector.add_event(first))
second = create_simple_event()
dom_ip_obj = DomainIPObject({'ip': ['10.0.0.1']})
second.add_object(dom_ip_obj)
second = check_response(self.admin_misp_connector.add_event(second))
check_response(self.admin_misp_connector.set_server_setting('MISP.background_jobs', 0, force=True))
result = self.admin_misp_connector._check_json_response(self.admin_misp_connector._prepare_request('POST', 'attributes/generateCorrelation'))
check_response(result)
self.assertIn("message", result)
check_response(self.admin_misp_connector.set_server_setting('MISP.background_jobs', 1, force=True))
first = check_response(self.admin_misp_connector.get_event(first))
second = check_response(self.admin_misp_connector.get_event(second))
try:
self.assertEqual(1, len(first.RelatedEvent), first.RelatedEvent)
self.assertEqual(1, len(second.RelatedEvent), second.RelatedEvent)
except:
raise
finally:
# Delete events
for event in (first, second):
check_response(self.admin_misp_connector.delete_event(event))
def test_restsearch_event_by_tags(self):
first = create_simple_event()
first.add_tag('test_search_tag')