MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'develop' into add_simple_background_jobs

pull/7939/head
Luciano Righetti 2021-10-26 10:55:13 +02:00
parent 5b08bc3578 02895c30ba
commit 35cf34fab2
45 changed files with 935 additions and 685 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.github/workflows/main.yml vendored
View File

@ -6,9 +6,9 @@ name: misp
# events but only for the 2.4 and develop branches
on:
push:
branches: [ 2.4, develop ]
branches: [ 2.4, develop, misp-stix ]
pull_request:
branches: [ 2.4, develop ]
branches: [ 2.4, develop, misp-stix ]
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
@ -200,10 +200,13 @@ jobs:
pushd ./app/files/scripts/cti-python-stix2
pip install .
popd
pushd ./app/files/scripts/python-stix
pip install .
popd
pushd PyMISP
pip install .[fileobjects,email]
popd
pip install stix zmq redis plyara
pip install zmq redis plyara
deactivate
- name: Test if apache is working

3
.gitignore vendored
View File

@ -45,9 +45,6 @@ tools/mkdocs
!/app/files/misp-objects/*
!/app/files/misp-decaying-models
!/app/files/misp-decaying-models/*
/app/files/scripts/python-stix/
/app/files/scripts/python-cybox/
/app/files/scripts/mixbox/
/app/files/scripts/*.pyc
/app/files/scripts/*.py~
/app/files/scripts/__pycache__

13
INSTALL/INSTALL.sh
View File

@ -1403,6 +1403,9 @@ installCore () {
sudo mkdir /var/www/.cache/
sudo chown ${WWW_USER}:${WWW_USER} /var/www/.cache
# install python-stix dependencies
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ordered-set python-dateutil six weakrefmethod
debug "Install PyMISP"
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/PyMISP
@ -1725,6 +1728,9 @@ coreCAKE () {
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.block_old_event_alert" false
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.block_old_event_alert_age" ""
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.block_old_event_alert_by_date" ""
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.event_alert_republish_ban" false
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.event_alert_republish_ban_threshold" 5
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.event_alert_republish_ban_refresh_on_retry" false
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.incoming_tags_disabled_by_default" false
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.maintenance_message" "Great things are happening! MISP is undergoing maintenance, but will return shortly. You can contact the administration at \$email."
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.footermidleft" "This is an initial install"
@ -2329,6 +2335,7 @@ installCoreRHEL7 () {
cd $PATH_TO_MISP
# Fetch submodules
$SUDO_WWW git submodule sync
$SUDO_WWW git submodule update --init --recursive
# Make git ignore filesystem permission differences for submodules
$SUDO_WWW git submodule foreach --recursive git config core.filemode false
@ -2347,6 +2354,9 @@ installCoreRHEL7 () {
UMASK=$(umask)
umask 0022
# install python-stix dependencies
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install ordered-set python-dateutil six weakrefmethod
# install zmq
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U zmq
@ -2458,6 +2468,9 @@ installCoreRHEL8 () {
UMASK=$(umask)
umask 0022
# install python-stix dependencies
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install ordered-set python-dateutil six weakrefmethod
# install zmq, redis
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U zmq redis

6
INSTALL/INSTALL.sh.sfv
View File

@ -1,5 +1,5 @@
; Generated by RHash v1.4.2 on 2021-10-08 at 10:34.14
; Generated by RHash v1.3.9 on 2021-10-18 at 10:56.53
; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/
;
; 159470 10:34.13 2021-10-08 INSTALL.sh
INSTALL.sh 9E3B4D450C3F35EEAA14547662979DF376549652 18C310BEC8A91335D8430C0C5B8AC7E60D9B3FFD0B8E3415CF4B32A10A1A6782 316CC204FB4FADA321923109797782A7CA41C258D5841A8A8396BBB3A06B21FEB0A1F37A11A4B6D4C7FEA6B060B1EAD8 967DBCCE0D1E26B2CF285C52625F4859395032B8F502A42FE32D9B3237172FDC4CAE2DC4B0AA9C23B4D278102EA58DE83BDB2760748B049816788395345B42A7
; 160201 10:56.53 2021-10-18 INSTALL.sh
INSTALL.sh 8F59974F7AE69DFBF7B1C492E35F0B421AAC10C1 6F9E9C2C24880D2E69E04AB6AE490F72D8B5CBE5BB98596F4FA50C1CFEAA632F CBCFBA692B57E027A9861C4D4FB1D4808511A23148516946802B0364D428638E60087AD6EA7E2F016B2F65CD216DE288 7221893A49C924974F7D28C094C6CB27FC8ACA6E07FECD7B8DE4D55D283C9D6A5FF63409F55EEC110BF6612E8578BD1373E39B83A7986A6369ACF32A6A92F538

2
INSTALL/INSTALL.sh.sha1
View File

@ -1 +1 @@
9e3b4d450c3f35eeaa14547662979df376549652 INSTALL.sh
8f59974f7ae69dfbf7b1c492e35f0b421aac10c1 INSTALL.sh

2
INSTALL/INSTALL.sh.sha256
View File

@ -1 +1 @@
18c310bec8a91335d8430c0c5b8ac7e60d9b3ffd0b8e3415cf4b32a10a1a6782 INSTALL.sh
6f9e9c2c24880d2e69e04ab6ae490f72d8b5cbe5bb98596f4fa50c1cfeaa632f INSTALL.sh

2
INSTALL/INSTALL.sh.sha384
View File

@ -1 +1 @@
316cc204fb4fada321923109797782a7ca41c258d5841a8a8396bbb3a06b21feb0a1f37a11a4b6d4c7fea6b060b1ead8 INSTALL.sh
cbcfba692b57e027a9861c4d4fb1d4808511a23148516946802b0364d428638e60087ad6ea7e2f016b2f65cd216de288 INSTALL.sh

2
INSTALL/INSTALL.sh.sha512
View File

@ -1 +1 @@
967dbcce0d1e26b2cf285c52625f4859395032b8f502a42fe32d9b3237172fdc4cae2dc4b0aa9c23b4d278102ea58de83bdb2760748b049816788395345b42a7 INSTALL.sh
7221893a49c924974f7d28c094c6cb27fc8aca6e07fecd7b8de4d55d283c9d6a5ff63409f55eec110bf6612e8578bd1373e39b83a7986a6369acf32a6a92f538 INSTALL.sh

1
INSTALL/MYSQL.sql
View File

@ -1056,6 +1056,7 @@ CREATE TABLE IF NOT EXISTS `sharing_groups` (
INDEX `org_id` (`org_id`),
INDEX `sync_user_id` (`sync_user_id`),
UNIQUE INDEX `uuid` (`uuid`),
UNIQUE INDEX `name` (`name`),
INDEX `organisation_uuid` (`organisation_uuid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;

2
app/Console/Command/EventShell.php
View File

@ -447,7 +447,7 @@ class EventShell extends AppShell
public function publish_galaxy_clusters()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || empty($this->args[3])) {
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || !array_key_exists(3, $this->args)) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Publish Galaxy clusters'] . PHP_EOL);
}

39
app/Console/Command/UserShell.php
View File

@ -82,28 +82,31 @@ class UserShell extends AppShell
public function list()
{
// do not fetch sensitive or big values
$schema = $this->User->schema();
unset($schema['authkey']);
unset($schema['password']);
unset($schema['gpgkey']);
unset($schema['certif_public']);
$fields = array_keys($schema);
$fields[] = 'Role.*';
$fields[] = 'Organisation.*';
$users = $this->User->find('all', [
'recursive' => -1,
'fields' => $fields,
'contain' => ['Organisation', 'Role'],
]);
if ($this->params['json']) {
// do not fetch sensitive or big values
$schema = $this->User->schema();
unset($schema['authkey']);
unset($schema['password']);
unset($schema['gpgkey']);
unset($schema['certif_public']);
$fields = array_keys($schema);
$fields[] = 'Role.*';
$fields[] = 'Organisation.*';
$users = $this->User->find('all', [
'recursive' => -1,
'fields' => $fields,
'contain' => ['Organisation', 'Role', 'UserSetting'],
]);
$this->out($this->json($users));
} else {
$users = $this->User->find('column', [
'fields' => ['email'],
]);
foreach ($users as $user) {
$this->out($user['User']['email']);
$this->out($user);
}
}
}

8
app/Controller/AppController.php
View File

@ -366,12 +366,8 @@ class AppController extends Controller
// Notifications and homepage is not necessary for AJAX or REST requests
if ($user && !$this->_isRest() && !$isAjax) {
if ($this->request->params['controller'] === 'users' && $this->request->params['action'] === 'dashboard') {
$notifications = $this->User->populateNotifications($user);
} else {
$notifications = $this->User->populateNotifications($user, 'fast');
}
$this->set('notifications', $notifications);
$hasNotifications = $this->User->hasNotifications($user);
$this->set('hasNotifications', $hasNotifications);
$homepage = $this->User->UserSetting->getValueForUser($user['id'], 'homepage');
if (!empty($homepage)) {

11
app/Controller/AttributesController.php
View File

@ -2673,16 +2673,11 @@ class AttributesController extends AppController
}
}
} else {
$conditions = array('LOWER(Tag.name)' => strtolower(trim($tag_id)));
if (!$this->_isSiteAdmin()) {
$conditions['Tag.org_id'] = array('0', $this->Auth->user('org_id'));
$conditions['Tag.user_id'] = array('0', $this->Auth->user('id'));
}
$tag = $this->Attribute->AttributeTag->Tag->find('first', array('recursive' => -1, 'conditions' => $conditions));
if (empty($tag)) {
$tagId = $this->Attribute->AttributeTag->Tag->lookupTagIdForUser($this->Auth->user(), trim($tag_id));
if (empty($tagId)) {
return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Invalid Tag.')), 'status'=>200, 'type' => 'json'));
}
$tag_id = $tag['Tag']['id'];
$tag_id = $tagId;
}
}
}

103
app/Controller/EventsController.php
View File

@ -28,20 +28,22 @@ class EventsController extends AppController
)
);
private $acceptedFilteringNamedParams = array(
'sort', 'direction', 'focus', 'extended', 'overrideLimit', 'filterColumnsOverwrite', 'attributeFilter', 'extended', 'page',
// private
const ACCEPTED_FILTERING_NAMED_PARAMS = array(
'sort', 'direction', 'focus', 'extended', 'overrideLimit', 'filterColumnsOverwrite', 'attributeFilter', 'page',
'searchFor', 'proposal', 'correlation', 'warning', 'deleted', 'includeRelatedTags', 'includeDecayScore', 'distribution',
'taggedAttributes', 'galaxyAttachedAttributes', 'objectType', 'attributeType', 'focus', 'extended', 'overrideLimit',
'filterColumnsOverwrite', 'feed', 'server', 'toIDS', 'sighting', 'includeSightingdb', 'warninglistId'
'taggedAttributes', 'galaxyAttachedAttributes', 'objectType', 'attributeType', 'feed', 'server', 'toIDS',
'sighting', 'includeSightingdb', 'warninglistId'
);
public $defaultFilteringRules = array(
// private
const DEFAULT_FILTERING_RULE = array(
'searchFor' => '',
'attributeFilter' => 'all',
'proposal' => 0,
'correlation' => 0,
'warning' => 0,
'deleted' => 2,
'deleted' => 0,
'includeRelatedTags' => 0,
'includeDecayScore' => 0,
'toIDS' => 0,
@ -700,7 +702,9 @@ class EventsController extends AppController
$this->paginate['contain']['ThreatLevel'] = [
'fields' => array('ThreatLevel.name')
];
$this->paginate['contain'][] = 'EventTag';
$this->paginate['contain']['EventTag'] = [
'fields' => ['EventTag.event_id', 'EventTag.tag_id', 'EventTag.local'],
];
if ($this->_isSiteAdmin()) {
$this->paginate['contain'][] = 'User.email';
}
@ -1178,7 +1182,7 @@ class EventsController extends AppController
{
$filterData = array(
'request' => $this->request,
'paramArray' => $this->acceptedFilteringNamedParams,
'paramArray' => self::ACCEPTED_FILTERING_NAMED_PARAMS,
'named_params' => $this->request->params['named']
);
$exception = false;
@ -1211,9 +1215,9 @@ class EventsController extends AppController
if ($filters['deleted'] == 1) { // both
$conditions['deleted'] = [0, 1];
} elseif ($filters['deleted'] == 0) { // not-deleted only
$conditions['deleted'] = 1;
} else { // only deleted
$conditions['deleted'] = 0;
} else { // only deleted
$conditions['deleted'] = 1;
}
}
if (isset($filters['toIDS']) && $filters['toIDS'] != 0) {
@ -1325,33 +1329,28 @@ class EventsController extends AppController
}
$this->params->params['paging'] = array($this->modelClass => $params);
$this->set('event', $event);
$deleted = 0;
if (isset($filters['deleted'])) {
$deleted = $filters['deleted'] != 2 ? 1 : 0;
}
$this->set('includeSightingdb', (!empty($filters['includeSightingdb']) && Configure::read('Plugin.Sightings_sighting_db_enable')));
$this->set('deleted', $deleted);
$this->set('deleted', isset($filters['deleted']) && $filters['deleted'] != 0);
$this->set('attributeFilter', isset($filters['attributeFilter']) ? $filters['attributeFilter'] : 'all');
$this->set('filters', $filters);
$advancedFiltering = $this->__checkIfAdvancedFiltering($filters);
$this->set('advancedFilteringActive', $advancedFiltering['active'] ? 1 : 0);
$this->set('advancedFilteringActiveRules', $advancedFiltering['activeRules']);
$this->response->disableCache();
$uriArray = explode('/', $this->params->here);
// Remove `focus` attribute from URI
$uriArray = explode('/', $this->request->here);
foreach ($uriArray as $k => $v) {
if (strpos($v, ':')) {
$temp = explode(':', $v);
if ($temp[0] == 'focus') {
unset($uriArray[$k]);
}
if (strpos($v, 'focus:') === 0) {
unset($uriArray[$k]);
}
$this->params->here = implode('/', $uriArray);
$this->request->here = implode('/', $uriArray);
}
if (!empty($filters['includeSightingdb']) && Configure::read('Plugin.Sightings_sighting_db_enable')) {
$this->set('sightingdbs', $this->Sightingdb->getSightingdbList($this->Auth->user()));
}
$this->set('currentUri', $this->params->here);
$this->set('currentUri', $this->request->here);
$this->layout = false;
$this->__eventViewCommon($this->Auth->user());
$this->render('/Elements/eventattribute');
@ -1368,7 +1367,7 @@ class EventsController extends AppController
$this->loadModel('Taxonomy');
$filterData = array(
'request' => $this->request,
'paramArray' => $this->acceptedFilteringNamedParams,
'paramArray' => self::ACCEPTED_FILTERING_NAMED_PARAMS,
'named_params' => $this->request->params['named']
);
$exception = false;
@ -1588,7 +1587,7 @@ class EventsController extends AppController
private function __eventViewCommon(array $user)
{
$this->set('defaultFilteringRules', $this->defaultFilteringRules);
$this->set('defaultFilteringRules', self::DEFAULT_FILTERING_RULE);
$this->set('typeGroups', array_keys($this->Event->Attribute->typeGroupings));
$orgTable = $this->Event->Orgc->find('list', array(
@ -1663,20 +1662,18 @@ class EventsController extends AppController
if (isset($this->request->data['deleted'])) {
$deleted = $this->request->data['deleted'];
}
if (isset($deleted)) {
// workaround for old instances trying to pull events with both deleted / non deleted data
if (($this->userRole['perm_sync'] && $this->_isRest() && !$this->userRole['perm_site_admin']) && $deleted == 1) {
$conditions['deleted'] = array(0, 1);
} else {
if (is_array($deleted)) {
$conditions['deleted'] = $deleted;
} else if ($deleted == 1) { // both
$conditions['deleted'] = [0, 1];
} elseif ($deleted == 0) { // not-deleted only
$conditions['deleted'] = 0;
} else { // only deleted
$conditions['deleted'] = 1;
}
// workaround for old instances trying to pull events with both deleted / non deleted data
if (($this->userRole['perm_sync'] && $this->_isRest() && !$this->userRole['perm_site_admin']) && $deleted == 1) {
$conditions['deleted'] = array(0, 1);
} else {
if (is_array($deleted)) {
$conditions['deleted'] = $deleted;
} else if ($deleted == 1) { // both
$conditions['deleted'] = [0, 1];
} elseif ($deleted == 0) { // not-deleted only
$conditions['deleted'] = 0;
} else { // only deleted
$conditions['deleted'] = 1;
}
}
if (isset($namedParams['toIDS']) && $namedParams['toIDS'] != 0) {
@ -1786,7 +1783,7 @@ class EventsController extends AppController
return $this->__restResponse($event);
}
$this->set('deleted', isset($deleted) ? ($deleted > 0 ? 1 : 0) : 0);
$this->set('deleted', $deleted > 0);
$this->set('includeRelatedTags', (!empty($namedParams['includeRelatedTags'])) ? 1 : 0);
$this->set('includeDecayScore', (!empty($namedParams['includeDecayScore'])) ? 1 : 0);
@ -1996,7 +1993,7 @@ class EventsController extends AppController
unset($filters['direction']);
$activeRules = array();
foreach ($filters as $k => $v) {
if (isset($this->defaultFilteringRules[$k]) && $this->defaultFilteringRules[$k] != $v) {
if (isset(self::DEFAULT_FILTERING_RULE[$k]) && self::DEFAULT_FILTERING_RULE[$k] != $v) {
$activeRules[$k] = $v;
}
}
@ -2273,17 +2270,20 @@ class EventsController extends AppController
}
}
public function upload_stix($stix_version = '1')
public function upload_stix($stix_version = '1', $publish = false)
{
if ($this->request->is('post')) {
if ($this->_isRest()) {
if (isset($this->params['named']['publish'])) {
$publish = $this->params['named']['publish'];
}
$filePath = FileAccessTool::writeToTempFile($this->request->input());
$result = $this->Event->upload_stix(
$this->Auth->user(),
$filePath,
$stix_version,
'uploaded_stix_file.' . ($stix_version == '1' ? 'xml' : 'json'),
false
$publish
);
if (is_numeric($result)) {
$event = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $result));
@ -3467,16 +3467,11 @@ class EventsController extends AppController
}
}
} else {
$conditions = array('LOWER(Tag.name)' => strtolower(trim($tag_id)));
if (!$this->_isSiteAdmin()) {
$conditions['Tag.org_id'] = array('0', $this->Auth->user('org_id'));
$conditions['Tag.user_id'] = array('0', $this->Auth->user('id'));
}
$tag = $this->Event->EventTag->Tag->find('first', array('recursive' => -1, 'conditions' => $conditions));
if (empty($tag)) {
$tagId = $this->Event->EventTag->Tag->lookupTagIdForUser($this->Auth->user(), trim($tag_id));
if (empty($tagId)) {
return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Invalid Tag.')), 'status'=>200, 'type' => 'json'));
}
$tag_id = $tag['Tag']['id'];
$tag_id = $tagId;
}
}
}
@ -5661,13 +5656,13 @@ class EventsController extends AppController
$objectRef['object_id'] = $ObjectResult;
$objectRef['relationship_type'] = "preceded-by";
$this->loadModel('MispObject');
$result = $this->MispObject->ObjectReference->captureReference($objectRef, $eventId, $this->Auth->user(), false);
$result = $this->MispObject->ObjectReference->captureReference($objectRef, $eventId);
$objectRef['referenced_id'] = $temp['Object']['id'];
$objectRef['referenced_uuid'] = $temp['Object']['uuid'];
$objectRef['object_id'] = $PreviousObjRef['Object']['id'];
$objectRef['relationship_type'] = "followed-by";
$this->loadModel('MispObject');
$result = $this->MispObject->ObjectReference->captureReference($objectRef, $eventId, $this->Auth->user(), false);
$result = $this->MispObject->ObjectReference->captureReference($objectRef, $eventId);
$PreviousObjRef = $temp;
} else {
$PreviousObjRef = $temp;

2
app/Controller/ShadowAttributesController.php
View File

@ -708,7 +708,7 @@ class ShadowAttributesController extends AppController
}
throw new InternalErrorException(__('Could not save the proposal. Errors: %s', $message));
} else {
$this->Flash->error(__('The ShadowAttribute could not be saved. Please, try again.'));
$this->Flash->error(__('The proposed Attribute could not be saved. Please, try again.'));
}
}
} else {

2
app/Lib/Dashboard/MispStatusWidget.php
View File

@ -44,7 +44,7 @@ class MispStatusWidget
'View'
)
);
$notifications = $this->Event->populateNotifications($user);
$notifications = $this->Event->User->populateNotifications($user);
if (!empty($notifications['proposalCount'])) {
$data[] = array(
'title' => __('Pending proposals'),

72
app/Model/AppModel.php
View File

@ -86,7 +86,8 @@ class AppModel extends Model
51 => false, 52 => false, 53 => false, 54 => false, 55 => false, 56 => false,
57 => false, 58 => false, 59 => false, 60 => false, 61 => false, 62 => false,
63 => true, 64 => false, 65 => false, 66 => false, 67 => false, 68 => false,
69 => false, 70 => false, 71 => true, 72 => true,
69 => false, 70 => false, 71 => true, 72 => true, 73 => false, 74 => false,
75 => false,
);
public $advanced_updates_description = array(
@ -1581,6 +1582,20 @@ class AppModel extends Model
case 72:
$sqlArray[] = "ALTER TABLE `auth_keys` ADD `read_only` tinyint(1) NOT NULL DEFAULT 0 AFTER `expiration`;";
break;
case 73:
$this->__dropIndex('user_settings', 'timestamp'); // index is not used
$sqlArray[] = "ALTER TABLE `user_settings` ADD UNIQUE INDEX `unique_setting` (`user_id`, `setting`)";
break;
case 74:
$sqlArray[] = "ALTER TABLE `users` MODIFY COLUMN `change_pw` tinyint(1) NOT NULL DEFAULT 0;";
break;
case 75:
$this->__addIndex('object_references', 'event_id');
$this->__dropIndex('object_references', 'timestamp');
$this->__dropIndex('object_references', 'source_uuid');
$this->__dropIndex('object_references', 'relationship_type');
$this->__dropIndex('object_references', 'referenced_uuid');
break;
case 'fixNonEmptySharingGroupID':
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
$sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
@ -2423,60 +2438,6 @@ class AppModel extends Model
return true;
}
public function populateNotifications($user, $mode = 'full')
{
$notifications = array();
list($notifications['proposalCount'], $notifications['proposalEventCount']) = $this->_getProposalCount($user, $mode);
$notifications['total'] = $notifications['proposalCount'];
if (Configure::read('MISP.delegation')) {
$notifications['delegationCount'] = $this->_getDelegationCount($user);
$notifications['total'] += $notifications['delegationCount'];
}
return $notifications;
}
// if not using $mode === 'full', simply check if an entry exists. We really don't care about the real count for the top menu.
private function _getProposalCount($user, $mode = 'full')
{
$this->ShadowAttribute = ClassRegistry::init('ShadowAttribute');
$results[0] = $this->ShadowAttribute->find(
'count',
array(
'recursive' => -1,
'conditions' => array(
'ShadowAttribute.event_org_id' => $user['org_id'],
'ShadowAttribute.deleted' => 0,
)
)
);
if ($mode === 'full') {
$results[1] = $this->ShadowAttribute->find(
'count',
array(
'recursive' => -1,
'conditions' => array(
'ShadowAttribute.event_org_id' => $user['org_id'],
'ShadowAttribute.deleted' => 0,
),
'fields' => 'distinct event_id'
)
);
} else {
$results[1] = $results[0];
}
return $results;
}
private function _getDelegationCount($user)
{
$this->EventDelegation = ClassRegistry::init('EventDelegation');
$delegations = $this->EventDelegation->find('count', array(
'recursive' => -1,
'conditions' => array('EventDelegation.org_id' => $user['org_id'])
));
return $delegations;
}
public function checkFilename($filename)
{
return preg_match('@^([a-z0-9_.]+[a-z0-9_.\- ]*[a-z0-9_.\-]|[a-z0-9_.])+$@i', $filename);
@ -3211,6 +3172,7 @@ class AppModel extends Model
'conditions' => $conditions,
'recursive' => -1,
'callbacks' => false,
'order' => [], // disable order
));
}

158
app/Model/Attribute.php
View File

@ -794,7 +794,6 @@ class Attribute extends AppModel
public function runValidation($value, $type)
{
$returnValue = false;
// check data validation
switch ($type) {
case 'md5':
@ -822,24 +821,19 @@ class Attribute extends AppModel
case 'git-commit-id':
if ($this->isHashValid($type, $value)) {
return true;
} else {
$length = self::HEX_HAS_LENGTHS[$type];
return __('Checksum has an invalid length or format (expected: %s hexadecimal characters). Please double check the value or select type "other".', $length);
}
$length = self::HEX_HAS_LENGTHS[$type];
return __('Checksum has an invalid length or format (expected: %s hexadecimal characters). Please double check the value or select type "other".', $length);
case 'tlsh':
if (preg_match("#^[0-9a-f]{35,}$#", $value)) {
$returnValue = true;
} else {
$returnValue = __('Checksum has an invalid length or format (expected: at least 35 hexadecimal characters). Please double check the value or select type "other".');
if (preg_match("#^t?[0-9a-f]{35,}$#i", $value)) {
return true;
}
break;
return __('Checksum has an invalid length or format (expected: at least 35 hexadecimal characters, optionally starting with t1 instead of hexadecimal characters). Please double check the value or select type "other".');
case 'pehash':
if ($this->isHashValid('pehash', $value)) {
$returnValue = true;
} else {
$returnValue = __('The input doesn\'t match the expected sha1 format (expected: 40 hexadecimal characters). Keep in mind that MISP currently only supports SHA1 for PEhashes, if you would like to get the support extended to other hash types, make sure to create a github ticket about it at https://github.com/MISP/MISP!');
return true;
}
break;
return __('The input doesn\'t match the expected sha1 format (expected: 40 hexadecimal characters). Keep in mind that MISP currently only supports SHA1 for PEhashes, if you would like to get the support extended to other hash types, make sure to create a github ticket about it at https://github.com/MISP/MISP!');
case 'ssdeep':
if (substr_count($value, ':') === 2) {
$parts = explode(':', $value);
@ -852,35 +846,26 @@ class Attribute extends AppModel
if (substr_count($value, ':') === 2) {
$parts = explode(':', $value);
if ($this->isPositiveInteger($parts[0])) {
$returnValue = true;
return true;
}
}
if (!$returnValue) {
$returnValue = __('Invalid impfuzzy format. The format has to be imports:hash:hash');
}
break;
return __('Invalid impfuzzy format. The format has to be imports:hash:hash');
case 'cdhash':
if (preg_match("#^[0-9a-f]{40,}$#", $value)) {
$returnValue = true;
} else {
$returnValue = __('The input doesn\'t match the expected format (expected: 40 or more hexadecimal characters)');
return true;
}
break;
return __('The input doesn\'t match the expected format (expected: 40 or more hexadecimal characters)');
case 'http-method':
if (preg_match("#(OPTIONS|GET|HEAD|POST|PUT|DELETE|TRACE|CONNECT|PROPFIND|PROPPATCH|MKCOL|COPY|MOVE|LOCK|UNLOCK|VERSION-CONTROL|REPORT|CHECKOUT|CHECKIN|UNCHECKOUT|MKWORKSPACE|UPDATE|LABEL|MERGE|BASELINE-CONTROL|MKACTIVITY|ORDERPATCH|ACL|PATCH|SEARCH)#", $value)) {
$returnValue = true;
} else {
$returnValue = __('Unknown HTTP method.');
return true;
}
break;
return __('Unknown HTTP method.');
case 'filename|pehash':
// no newline
if (preg_match("#^.+\|[0-9a-f]{40}$#", $value)) {
$returnValue = true;
} else {
$returnValue = __('The input doesn\'t match the expected filename|sha1 format (expected: filename|40 hexadecimal characters). Keep in mind that MISP currently only supports SHA1 for PEhashes, if you would like to get the support extended to other hash types, make sure to create a github ticket about it at https://github.com/MISP/MISP!');
return true;
}
break;
return __('The input doesn\'t match the expected filename|sha1 format (expected: filename|40 hexadecimal characters). Keep in mind that MISP currently only supports SHA1 for PEhashes, if you would like to get the support extended to other hash types, make sure to create a github ticket about it at https://github.com/MISP/MISP!');
case 'filename|md5':
case 'filename|sha1':
case 'filename|imphash':
@ -898,42 +883,33 @@ class Attribute extends AppModel
$parts = explode('|', $type);
$length = self::HEX_HAS_LENGTHS[$parts[1]];
if (preg_match("#^.+\|[0-9a-f]{" . $length . "}$#", $value)) {
$returnValue = true;
} else {
$returnValue = __('Checksum has an invalid length or format (expected: filename|%s hexadecimal characters). Please double check the value or select type "other".', $length);
return true;
}
break;
return __('Checksum has an invalid length or format (expected: filename|%s hexadecimal characters). Please double check the value or select type "other".', $length);
case 'filename|ssdeep':
if (substr_count($value, '|') != 1 || !preg_match("#^.+\|.+$#", $value)) {
$returnValue = __('Invalid composite type. The format has to be %s.', $type);
return __('Invalid composite type. The format has to be %s.', $type);
} else {
$composite = explode('|', $value);
$value = $composite[1];
if (substr_count($value, ':') == 2) {
$parts = explode(':', $value);
if ($this->isPositiveInteger($parts[0])) {
$returnValue = true;
return true;
}
}
if (!$returnValue) {
$returnValue = __('Invalid SSDeep hash (expected: blocksize:hash:hash).');
}
}
break;
return __('Invalid SSDeep hash (expected: blocksize:hash:hash).');
case 'filename|tlsh':
if (preg_match("#^.+\|[0-9a-f]{35,}$#", $value)) {
$returnValue = true;
} else {
$returnValue = __('Checksum has an invalid length or format (expected: filename|at least 35 hexadecimal characters). Please double check the value or select type "other".');
return true;
}
break;
return __('Checksum has an invalid length or format (expected: filename|at least 35 hexadecimal characters). Please double check the value or select type "other".');
case 'filename|vhash':
if (preg_match('#^.+\|.+$#', $value)) {
$returnValue = true;
} else {
$returnValue = __('Checksum has an invalid length or format (expected: filename|string characters). Please double check the value or select type "other".');
return true;
}
break;
return __('Checksum has an invalid length or format (expected: filename|string characters). Please double check the value or select type "other".');
case 'ip-src':
case 'ip-dst':
if (strpos($value, '/') !== false) {
@ -957,14 +933,11 @@ class Attribute extends AppModel
return __('IP address has an invalid format.');
}
return true;
case 'port':
if (!$this->isPortValid($value)) {
$returnValue = __('Port numbers have to be integers between 1 and 65535.');
} else {
$returnValue = true;
return __('Port numbers have to be integers between 1 and 65535.');
}
break;
return true;
case 'ip-dst|port':
case 'ip-src|port':
$parts = explode('|', $value);
@ -977,22 +950,20 @@ class Attribute extends AppModel
return true;
case 'mac-address':
if (preg_match('/^([a-fA-F0-9]{2}[:]?){6}$/', $value)) {
$returnValue = true;
return true;
}
break;
case 'mac-eui-64':
if (preg_match('/^([a-fA-F0-9]{2}[:]?){8}$/', $value)) {
$returnValue = true;
return true;
}
break;
case 'hostname':
case 'domain':
if ($this->isDomainValid($value)) {
$returnValue = true;
} else {
$returnValue = __('%s has an invalid format. Please double check the value or select type "other".', ucfirst($type));
return true;
}
break;
return __('%s has an invalid format. Please double check the value or select type "other".', ucfirst($type));
case 'hostname|port':
$parts = explode('|', $value);
if (!$this->isDomainValid($parts[0])) {
@ -1006,14 +977,12 @@ class Attribute extends AppModel
if (preg_match("#^[A-Z0-9.\-_]+\.[A-Z0-9\-]{2,}\|.*$#i", $value)) {
$parts = explode('|', $value);
if (filter_var($parts[1], FILTER_VALIDATE_IP)) {
$returnValue = true;
return true;
} else {
$returnValue = __('IP address has an invalid format.');
return __('IP address has an invalid format.');
}
} else {
$returnValue = __('Domain name has an invalid format.');
}
break;
return __('Domain name has an invalid format.');
case 'email':
case 'email-src':
case 'eppn':
@ -1024,38 +993,30 @@ class Attribute extends AppModel
case 'jabber-id':
// we don't use the native function to prevent issues with partial email addresses
if (preg_match("#^.*\@.*\..*$#i", $value)) {
$returnValue = true;
} else {
$returnValue = __('Email address has an invalid format. Please double check the value or select type "other".');
return true;
}
break;
return __('Email address has an invalid format. Please double check the value or select type "other".');
case 'vulnerability':
if (preg_match("#^(CVE-)[0-9]{4}(-)[0-9]{4,}$#", $value)) {
$returnValue = true;
} else {
$returnValue = __('Invalid format. Expected: CVE-xxxx-xxxx...');
return true;
}
break;
return __('Invalid format. Expected: CVE-xxxx-xxxx...');
case 'weakness':
if (preg_match("#^(CWE-)[0-9]{1,}$#", $value)) {
$returnValue = true;
} else {
$returnValue = __('Invalid format. Expected: CWE-x...');
return true;
}
break;
return __('Invalid format. Expected: CWE-x...');
case 'named pipe':
if (!preg_match("#\n#", $value)) {
$returnValue = true;
return true;
}
break;
case 'windows-service-name':
case 'windows-service-displayname':
if (strlen($value) > 256 || preg_match('#[\\\/]#', $value)) {
$returnValue = __('Invalid format. Only values shorter than 256 characters that don\'t include any forward or backward slashes are allowed.');
} else {
$returnValue = true;
return __('Invalid format. Only values shorter than 256 characters that don\'t include any forward or backward slashes are allowed.');
}
break;
return true;
case 'mutex':
case 'process-state':
case 'snort':
@ -1090,12 +1051,11 @@ class Attribute extends AppModel
case 'middle-name':
case 'last-name':
case 'full-name':
$returnValue = true;
break;
return true;
case 'link':
// Moved to a native function whilst still enforcing the scheme as a requirement
if (filter_var($value, FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED) && !preg_match("#\n#", $value)) {
$returnValue = true;
return true;
}
break;
case 'hex':
@ -1164,38 +1124,33 @@ class Attribute extends AppModel
}
return true;
case 'datetime':
try {
new DateTime($value);
$returnValue = true;
} catch (Exception $e) {
$returnValue = __('Datetime has to be in the ISO 8601 format.');
if (strtotime($value) !== false) {
return true;
}
break;
return __('Datetime has to be in the ISO 8601 format.');
case 'size-in-bytes':
case 'counter':
if ($this->isPositiveInteger($value)) {
return true;
}
return __('The value has to be a whole number greater or equal 0.');
case 'targeted-threat-index':
/* case 'targeted-threat-index':
if (!is_numeric($value) || $value < 0 || $value > 10) {
$returnValue = __('The value has to be a number between 0 and 10.');
} else {
$returnValue = true;
return __('The value has to be a number between 0 and 10.');
}
break;
return true;*/
case 'iban':
case 'bic':
case 'btc':
case 'dash':
case 'xmr':
if (preg_match('/^[a-zA-Z0-9]+$/', $value)) {
$returnValue = true;
return true;
}
break;
case 'vhash':
if (preg_match('/^.+$/', $value)) {
$returnValue = true;
return true;
}
break;
case 'bin':
@ -1206,18 +1161,17 @@ class Attribute extends AppModel
case 'phone-number':
case 'whois-registrant-phone':
if (is_numeric($value)) {
$returnValue = true;
return true;
}
break;
case 'cortex':
json_decode($value);
$returnValue = (json_last_error() == JSON_ERROR_NONE);
break;
return json_last_error() === JSON_ERROR_NONE;
case 'float':
return is_numeric($value);
case 'boolean':
if ($value == 1 || $value == 0) {
$returnValue = true;
return true;
}
break;
case 'AS':
@ -1226,7 +1180,7 @@ class Attribute extends AppModel
}
return __('AS number have to be integers between 1 and 4294967295');
}
return $returnValue;
return false;
}
// do some last second modifications before the validation

351
app/Model/Event.php
View File

@ -539,23 +539,23 @@ class Event extends AppModel
{
$tagsToFetch = array();
foreach ($events as $event) {
if (!empty($event['EventTag'])) {
foreach ($event['EventTag'] as $et) {
$tagsToFetch[$et['tag_id']] = $et['tag_id'];
}
foreach ($event['EventTag'] as $et) {
$tagsToFetch[$et['tag_id']] = $et['tag_id'];
}
}
if (empty($tagsToFetch)) {
return $events;
}
$tags = $this->EventTag->Tag->find('all', array(
'conditions' => array('Tag.id' => $tagsToFetch),
'recursive' => -1,
'fields' => ['id', 'name', 'colour', 'is_galaxy'], // fetch just necessary columns
'order' => false
));
$tags = array_column(array_column($tags, 'Tag'), null, 'id');
foreach ($events as &$event) {
if (!empty($event['EventTag'])) {
foreach ($event['EventTag'] as &$et) {
$et['Tag'] = $tags[$et['tag_id']];
}
foreach ($event['EventTag'] as &$et) {
$et['Tag'] = $tags[$et['tag_id']];
}
}
return $events;
@ -1960,7 +1960,7 @@ class Event extends AppModel
$conditions['AND'][] = array('Event.published' => 1);
$conditionsAttributes['AND'][] = array('Attribute.to_ids' => 1);
}
$softDeletables = array('Attribute', 'Object', 'ObjectReference', 'EventReport');
$softDeletables = array('Attribute', 'Object', 'EventReport');
if (isset($options['deleted'])) {
if (!is_array($options['deleted'])) {
$options['deleted'] = array($options['deleted']);
@ -2040,7 +2040,6 @@ class Event extends AppModel
$fieldsAtt = array('Attribute.id', 'Attribute.type', 'Attribute.category', 'Attribute.value', 'Attribute.to_ids', 'Attribute.uuid', 'Attribute.event_id', 'Attribute.distribution', 'Attribute.timestamp', 'Attribute.comment', 'Attribute.sharing_group_id', 'Attribute.deleted', 'Attribute.disable_correlation', 'Attribute.object_id', 'Attribute.object_relation', 'Attribute.first_seen', 'Attribute.last_seen');
$fieldsShadowAtt = array('ShadowAttribute.id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.value', 'ShadowAttribute.to_ids', 'ShadowAttribute.uuid', 'ShadowAttribute.event_uuid', 'ShadowAttribute.event_id', 'ShadowAttribute.old_id', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.proposal_to_delete', 'ShadowAttribute.timestamp', 'ShadowAttribute.first_seen', 'ShadowAttribute.last_seen');
$fieldsOrg = array('id', 'name', 'uuid', 'local');
$fieldsEventReport = array('*');
$params = array(
'conditions' => $conditions,
'recursive' => 0,
@ -2052,17 +2051,11 @@ class Event extends AppModel
'Attribute' => array(
'fields' => $fieldsAtt,
'conditions' => $conditionsAttributes,
'AttributeTag' => array(
'order' => false
),
'order' => false
),
'Object' => array(
'conditions' => $conditionsObjects,
'order' => false,
'ObjectReference' => array(
'order' => false
)
),
'ShadowAttribute' => array(
'fields' => $fieldsShadowAtt,
@ -2074,7 +2067,6 @@ class Event extends AppModel
'order' => false
),
'EventReport' => array(
'fields' => $fieldsEventReport,
'conditions' => $conditionsEventReport,
'order' => false
)
@ -2084,9 +2076,6 @@ class Event extends AppModel
$params['contain']['EventTag']['conditions'] = array(
'EventTag.local' => 0
);
$params['contain']['Attribute']['AttributeTag']['conditions'] = array(
'AttributeTag.local' => 0
);
}
if ($flatten) {
unset($params['contain']['Object']);
@ -2124,6 +2113,9 @@ class Event extends AppModel
if (!empty($options['includeDecayScore']) && !isset($this->DecayingModel)) {
$this->DecayingModel = ClassRegistry::init('DecayingModel');
}
if ($options['includeServerCorrelations'] && !$isSiteAdmin && $user['org_id'] != Configure::read('MISP.host_org_id')) {
$options['includeServerCorrelations'] = false; // not permission to see server correlations
}
if (($options['includeFeedCorrelations'] || $options['includeServerCorrelations']) && !isset($this->Feed)) {
$this->Feed = ClassRegistry::init('Feed');
}
@ -2139,6 +2131,20 @@ class Event extends AppModel
$justExportableTags = false;
}
$overrideLimit = !empty($options['overrideLimit']);
if (!empty($options['allow_proposal_blocking']) && !Configure::read('MISP.proposals_block_attributes')) {
$options['allow_proposal_blocking'] = false; // proposal blocking is not enabled
}
if (!$options['metadata']) {
$this->__attachAttributeTags($results, $options['excludeLocalTags']);
}
if (!$options['metadata'] && !$flatten) {
$this->__attachReferences($results);
}
foreach ($results as &$event) {
/*
// REMOVING THIS FOR NOW - users should see data they own, even if they're not in the sharing group.
@ -2164,7 +2170,6 @@ class Event extends AppModel
$this->Warninglist->attachWarninglistToAttributes($event['ShadowAttribute']);
$event['warnings'] = $eventWarnings;
}
$this->__attachReferences($event);
$this->__attachTags($event, $justExportableTags);
$this->__attachGalaxies($event, $user, $options['excludeGalaxy'], $options['fetchFullClusters']);
$event = $this->Orgc->attachOrgs($event, $fieldsOrg);
@ -2202,29 +2207,36 @@ class Event extends AppModel
}
$event['RelatedShadowAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id'], true);
}
if (!empty($event['ShadowAttribute']) && $options['includeAttachments']) {
foreach ($event['ShadowAttribute'] as $k => $sa) {
if ($this->ShadowAttribute->typeIsAttachment($sa['type'])) {
$encodedFile = $this->ShadowAttribute->base64EncodeAttachment($sa);
$event['ShadowAttribute'][$k]['data'] = $encodedFile;
}
$shadowAttributeByOldId = [];
if (!empty($event['ShadowAttribute'])) {
if ($isSiteAdmin && $options['includeFeedCorrelations']) {
$event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user, $event['Event'], $overrideLimit);
}
if ($options['includeServerCorrelations']) {
$event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user, $event['Event'], $overrideLimit, 'Server');
}
if ($options['includeAttachments']) {
foreach ($event['ShadowAttribute'] as &$sa) {
if ($this->ShadowAttribute->typeIsAttachment($sa['type'])) {
$encodedFile = $this->ShadowAttribute->base64EncodeAttachment($sa);
$sa['data'] = $encodedFile;
}
}
unset($sa);
}
foreach ($event['ShadowAttribute'] as $sa) {
$shadowAttributeByOldId[$sa['old_id']][] = $sa;
}
// Assign just shadow attributes that are linked to event (that means they have old_id set to `0`)
$event['ShadowAttribute'] = $shadowAttributeByOldId[0] ?? [];
}
if (!empty($event['Attribute'])) {
if ($options['includeFeedCorrelations']) {
if (!empty($options['overrideLimit'])) {
$overrideLimit = true;
} else {
$overrideLimit = false;
}
$event['Attribute'] = $this->Feed->attachFeedCorrelations($event['Attribute'], $user, $event['Event'], $overrideLimit);
}
if (!empty($options['includeServerCorrelations']) && ($user['Role']['perm_site_admin'] || $user['org_id'] == Configure::read('MISP.host_org_id'))) {
if (!empty($options['overrideLimit'])) {
$overrideLimit = true;
} else {
$overrideLimit = false;
}
if ($options['includeServerCorrelations']) {
$event['Attribute'] = $this->Feed->attachFeedCorrelations($event['Attribute'], $user, $event['Event'], $overrideLimit, 'Server');
}
$event = $this->__filterBlockedAttributesByTags($event, $options, $user);
@ -2232,54 +2244,42 @@ class Event extends AppModel
$event['Attribute'] = $this->__attachSharingGroups($event['Attribute'], $sharingGroupData);
}
$proposalBlockAttributes = Configure::read('MISP.proposals_block_attributes');
// move all object attributes to a temporary container
$tempObjectAttributeContainer = array();
foreach ($event['Attribute'] as $key => $attribute) {
foreach ($event['Attribute'] as $key => &$attribute) {
if ($options['enforceWarninglist'] && !empty($attribute['warnings'])) {
unset($event['Attribute'][$key]);
continue;
}
if ($attribute['category'] === 'Financial fraud') {
$event['Attribute'][$key] = $this->Attribute->attachValidationWarnings($event['Attribute'][$key]);
$attribute = $this->Attribute->attachValidationWarnings($attribute);
}
if ($options['includeAttachments'] && $this->Attribute->typeIsAttachment($attribute['type'])) {
$encodedFile = $this->Attribute->base64EncodeAttachment($attribute);
$event['Attribute'][$key]['data'] = $encodedFile;
$attribute['data'] = $encodedFile;
}
if (!empty($options['includeDecayScore'])) {
if (isset($event['EventTag'])) { // include EventTags for score computation
$event['Attribute'][$key]['EventTag'] = $event['EventTag'];
$attribute['EventTag'] = $event['EventTag'];
}
$event['Attribute'][$key] = $this->DecayingModel->attachScoresToAttribute($user, $event['Attribute'][$key]);
$attribute = $this->DecayingModel->attachScoresToAttribute($user, $attribute);
if (isset($event['EventTag'])) { // remove included EventTags
unset($event['Attribute'][$key]['EventTag']);
unset($attribute['EventTag']);
}
}
$event['Attribute'][$key]['ShadowAttribute'] = array();
// If a shadowattribute can be linked to an attribute, link it to it then remove it from the event
// If a shadowattribute can be linked to an attribute, link it to it
// This is to differentiate between proposals that were made to an attribute for modification and between proposals for new attributes
if (isset($event['ShadowAttribute'])) {
foreach ($event['ShadowAttribute'] as $k => $sa) {
if (!empty($sa['old_id'])) {
if ($event['ShadowAttribute'][$k]['old_id'] == $attribute['id']) {
$event['Attribute'][$key]['ShadowAttribute'][] = $sa;
unset($event['ShadowAttribute'][$k]);
}
}
}
}
if ($proposalBlockAttributes && !empty($options['allow_proposal_blocking'])) {
foreach ($event['Attribute'][$key]['ShadowAttribute'] as $sa) {
$attribute['ShadowAttribute'] = $shadowAttributeByOldId[$attribute['id']] ?? [];
if (!empty($options['allow_proposal_blocking'])) {
foreach ($attribute['ShadowAttribute'] as $sa) {
if ($sa['proposal_to_delete'] || $sa['to_ids'] == 0) {
unset($event['Attribute'][$key]);
continue;
continue 2;
}
}
}
if (!$flatten && $attribute['object_id'] != 0) {
$tempObjectAttributeContainer[$attribute['object_id']][] = $event['Attribute'][$key];
$tempObjectAttributeContainer[$attribute['object_id']][] = $attribute;
unset($event['Attribute'][$key]);
}
}
@ -2289,9 +2289,9 @@ class Event extends AppModel
if (!$sharingGroupReferenceOnly) {
$event['Object'] = $this->__attachSharingGroups($event['Object'], $sharingGroupData);
}
foreach ($event['Object'] as $objectKey => $objectValue) {
foreach ($event['Object'] as &$objectValue) {
if (isset($tempObjectAttributeContainer[$objectValue['id']])) {
$event['Object'][$objectKey]['Attribute'] = $tempObjectAttributeContainer[$objectValue['id']];
$objectValue['Attribute'] = $tempObjectAttributeContainer[$objectValue['id']];
}
}
unset($tempObjectAttributeContainer);
@ -2299,33 +2299,6 @@ class Event extends AppModel
if (!$sharingGroupReferenceOnly && !empty($event['EventReport'])) {
$event['EventReport'] = $this->__attachSharingGroups($event['EventReport'], $sharingGroupData);
}
if (!empty($event['ShadowAttribute'])) {
if ($isSiteAdmin && $options['includeFeedCorrelations']) {
if (!empty($options['overrideLimit'])) {
$overrideLimit = true;
} else {
$overrideLimit = false;
}
$event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user, $event['Event'], $overrideLimit);
}
if (!empty($options['includeServerCorrelations']) && $user['org_id'] == Configure::read('MISP.host_org_id')) {
if (!empty($options['overrideLimit'])) {
$overrideLimit = true;
} else {
$overrideLimit = false;
}
$event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user, $event['Event'], $overrideLimit, 'Server');
}
// remove proposals to attributes that we cannot see
// if the shadow attribute wasn't moved within an attribute before, this is the case
foreach ($event['ShadowAttribute'] as $k => $sa) {
if (!empty($sa['old_id'])) {
unset($event['ShadowAttribute'][$k]);
}
}
$event['ShadowAttribute'] = array_values($event['ShadowAttribute']);
}
if (empty($options['metadata']) && empty($options['noSightings'])) {
$event['Sighting'] = $this->Sighting->attachToEvent($event, $user);
}
@ -3880,9 +3853,12 @@ class Event extends AppModel
foreach ($referencesToCapture as $referenceToCapture) {
$result = $this->Object->ObjectReference->captureReference(
$referenceToCapture,
$this->id,
$user
$this->id
);
if ($result !== true) {
$title = "Could not save object reference when capturing event with ID {$this->id}";
$this->loadLog()->validationError($user, 'add', 'ObjectReference', $title, $result, $referenceToCapture);
}
}
}
@ -4099,7 +4075,11 @@ class Event extends AppModel
foreach ($object['ObjectReference'] as $objectRef) {
$nothingToChange = false;
$objectRef['source_uuid'] = $object['uuid'];
$result = $this->Object->ObjectReference->captureReference($objectRef, $this->id, $user);
$result = $this->Object->ObjectReference->captureReference($objectRef, $this->id);
if ($result !== true) {
$title = "Could not save object reference when capturing event with ID {$this->id}";
$this->loadLog()->validationError($user, 'edit', 'ObjectReference', $title, $result, $objectRef);
}
if ($result && !$nothingToChange) {
$changed = true;
}
@ -6841,6 +6821,123 @@ class Event extends AppModel
return $this->processModuleResultsData($user, $resolved_data, $id, $default_comment);
}
/**
* Attach references to objects faster than CakePHP.
* @param array $events
*/
private function __attachReferences(array &$events)
{
$eventIds = [];
foreach ($events as $event) {
if (!empty($event['Object'])) {
$eventIds[] = $event['Event']['id']; // event contains objects
}
}
if (!empty($eventIds)) {
// Do not fetch fields that we already know to reduce memory usage
$schema = $this->Object->ObjectReference->schema();
unset($schema['event_id']);
unset($schema['source_uuid']);
$references = $this->Object->ObjectReference->find('all', [
'conditions' => ['ObjectReference.event_id' => $eventIds],
'fields' => array_keys($schema),
'recursive' => -1,
]);
}
if (empty($references)) {
// Assign empty object reference object
foreach ($events as &$event) {
foreach ($event['Object'] as &$object) {
$object['ObjectReference'] = [];
}
}
return;
}
$referencesForObject = [];
foreach ($references as $reference) {
$referencesForObject[$reference['ObjectReference']['object_id']][] = $reference['ObjectReference'];
}
$fieldsToCopy = array(
'common' => array('distribution', 'sharing_group_id', 'uuid'),
'Attribute' => array('value', 'type', 'category', 'to_ids'),
'Object' => array('name', 'meta-category')
);
foreach ($events as &$event) {
$eventIdCache = [];
foreach ($event['Object'] as &$object) {
$objectReferences = $referencesForObject[$object['id']] ?? [];
foreach ($objectReferences as &$reference) {
$reference['event_id'] = $event['Event']['id'];
$reference['source_uuid'] = $object['uuid'];
// find referenced object in current event
$type = $reference['referenced_type'] == 0 ? 'Attribute' : 'Object';
// construct array with ID in key, so we can search attributes and objects by ID faster
if (!isset($eventIdCache[$type])) {
$eventIdCache[$type] = array_column($event[$type], null, 'id');
}
$found = $eventIdCache[$type][$reference['referenced_id']] ?? null;
if ($found) {
// copy requested fields
$copied = [];
foreach (array_merge($fieldsToCopy['common'], $fieldsToCopy[$type]) as $field) {
$copied[$field] = $found[$field];
}
$reference[$type] = $copied;
} else { // object / attribute might be from an extended event
$otherEventText = __('%s from another event', $type);
$reference[$type] = [
'name' => '',
'meta-category' => $otherEventText,
'category' => $otherEventText,
'type' => '',
'value' => '',
'uuid' => $reference['referenced_uuid']
];
}
}
$object['ObjectReference'] = $objectReferences;
}
}
}
/**
* Faster way how to attach tags to events that integrated in CakePHP.
* @param array $events
* @param bool $excludeLocalTags
*/
private function __attachAttributeTags(array &$events, $excludeLocalTags = false)
{
$eventIds = array_column(array_column($events, 'Event'), 'id');
$conditions = ['AttributeTag.event_id' => $eventIds];
if ($excludeLocalTags) {
$conditions['AttributeTag.local'] = false;
}
$ats = $this->Attribute->AttributeTag->find('all', [
'conditions' => $conditions,
'fields' => ['AttributeTag.attribute_id', 'AttributeTag.tag_id', 'AttributeTag.local'], // we don't need id or event_id
'recursive' => -1,
]);
if (empty($ats)) {
foreach ($events as &$event) {
foreach ($event['Attribute'] as &$attribute) {
$attribute['AttributeTag'] = [];
}
}
return;
}
$atForAttributes = [];
foreach ($ats as $at) {
$atForAttributes[$at['AttributeTag']['attribute_id']][] = $at['AttributeTag'];
}
foreach ($events as &$event) {
foreach ($event['Attribute'] as &$attribute) {
$attribute['AttributeTag'] = $atForAttributes[$attribute['id']] ?? [];
}
}
}
/**
* Get tag from cache by given ID.
*
@ -6877,21 +6974,19 @@ class Event extends AppModel
if (!empty($event['Attribute'])) {
foreach ($event['Attribute'] as $attribute) {
if (!empty($attribute['AttributeTag'])) {
foreach ($attribute['AttributeTag'] as $attributeTag) {
$tagIds[$attributeTag['tag_id']] = true;
}
foreach ($attribute['AttributeTag'] as $attributeTag) {
$tagIds[$attributeTag['tag_id']] = true;
}
}
}
$notCachedTags = array_diff(array_keys($tagIds), isset($this->assetCache['tags']) ? array_keys($this->assetCache['tags']) : []);
$notCachedTags = array_diff_key($tagIds, isset($this->assetCache['tags']) ? $this->assetCache['tags'] : []);
if (empty($notCachedTags)) {
return;
}
$conditions = ['id' => $notCachedTags];
$conditions = ['Tag.id' => array_keys($notCachedTags)];
if ($justExportable) {
$conditions['exportable'] = 1;
$conditions['Tag.exportable'] = 1;
}
$tags = $this->EventTag->Tag->find('all', [
'recursive' => -1,
@ -6942,58 +7037,6 @@ class Event extends AppModel
}
}
/**
* Attach referenced object to ObjectReference. Since reference can be just to attribute or object in the same event,
* we just find proper element in event.
*
* @param array $event
*/
private function __attachReferences(array &$event)
{
if (!isset($event['Object'])) {
return;
}
$fieldsToCopy = array(
'common' => array('distribution', 'sharing_group_id', 'uuid'),
'Attribute' => array('value', 'type', 'category', 'to_ids'),
'Object' => array('name', 'meta-category')
);
foreach ($event['Object'] as $k => $object) {
foreach ($object['ObjectReference'] as $k2 => $reference) {
// find referenced object in current event
$type = $reference['referenced_type'] == 0 ? 'Attribute' : 'Object';
$found = null;
foreach ($event[$type] as $o) {
if ($o['id'] == $reference['referenced_id']) {
$found = $o;
break;
}
}
if ($found) {
// copy requested fields
$reference = [];
foreach (array_merge($fieldsToCopy['common'], $fieldsToCopy[$type]) as $field) {
$reference[$field] = $found[$field];
}
$event['Object'][$k]['ObjectReference'][$k2][$type] = $reference;
} else { // object / attribute might be from an extended event
$otherEventText = __('%s from another event', $reference['referenced_type'] == 0 ? 'Attribute' : 'Object');
$event['Object'][$k]['ObjectReference'][$k2][$type] = [
'name' => '',
'meta-category' => $otherEventText,
'category' => $otherEventText,
'type' => '',
'value' => '',
'uuid' => $reference['referenced_uuid']
];
}
}
}
}
/**
* @param array $user
* @param string $returnFormat
@ -7350,7 +7393,7 @@ class Event extends AppModel
public function addFiltersFromUserSettings($user, $filters)
{
$this->UserSetting = ClassRegistry::init('UserSetting');
$defaultParameters = $this->UserSetting->getDefaulRestSearchParameters($user);
$defaultParameters = $this->UserSetting->getDefaultRestSearchParameters($user);
$filters = array_replace_recursive($defaultParameters, $filters);
return $filters;
}

8
app/Model/GalaxyCluster.php
View File

@ -457,10 +457,6 @@ class GalaxyCluster extends AppModel
} else {
return false;
}
$this->Event = ClassRegistry::init('Event');
$job_type = 'publish_cluster';
$function = 'publish_galaxy_clusters';
$message = 'Publishing.';
$job = ClassRegistry::init('Job');
$job->create();
$data = array(
@ -471,14 +467,14 @@ class GalaxyCluster extends AppModel
'retries' => 0,
'org_id' => $user['org_id'],
'org' => $user['Organisation']['name'],
'message' => $message
'message' => 'Publishing.'
);
$job->save($data);
$jobId = $job->id;
$process_id = CakeResque::enqueue(
'prio',
'EventShell',
array($function, $clusterId, $jobId, $user['id'], $passAlong),
array('publish_galaxy_clusters', $clusterId, $jobId, $user['id'], $passAlong),
true
);
$job->saveField('process_id', $process_id);

19
app/Model/Log.php
View File

@ -240,6 +240,9 @@ class Log extends AppModel
if ($action === 'request' && !empty(Configure::read('MISP.log_paranoid_skip_db'))) {
return null;
}
if (!empty(Configure::read('MISP.log_skip_db_logs_completely'))) {
return null;
}
throw new Exception("Cannot save log because of validation errors: " . json_encode($this->validationErrors));
}
@ -247,6 +250,22 @@ class Log extends AppModel
return $result;
}
/**
* @param array|string $user
* @param string $action
* @param string $model
* @param string $title
* @param array $validationErrors
* @param array $fullObject
* @throws Exception
*/
public function validationError($user, $action, $model, $title, array $validationErrors, array $fullObject)
{
$this->log($title, LOG_WARNING);
$change = 'Validation errors: ' . json_encode($validationErrors) . ' Full ' . $model . ': ' . json_encode($fullObject);
$this->createLogEntry($user, $action, $model, 0, $title, $change);
}
// to combat a certain bug that causes the upgrade scripts to loop without being able to set the correct version
// this function remedies a fixed upgrade bug instance by eliminating the massive number of erroneous upgrade log entries
public function pruneUpdateLogs($jobId = false, $user)

49
app/Model/ObjectReference.php
View File

@ -38,17 +38,38 @@ class ObjectReference extends AppModel
)
);
public $validate = [
'uuid' => 'uuid',
'object_id' => [
'rule' => 'numeric',
'required' => true,
'on' => 'create',
],
'event_id' => [
'rule' => 'numeric',
'required' => true,
'on' => 'create',
],
'source_uuid' => 'uuid',
'referenced_uuid' => 'uuid',
'referenced_id' => 'numeric',
'referenced_type' => [
'rule' => ['inList', ['0', '1']],
],
'deleted' => 'boolean',
];
public function beforeValidate($options = array())
{
parent::beforeValidate();
if (empty($this->data['ObjectReference']['uuid'])) {
$this->data['ObjectReference']['uuid'] = CakeText::uuid();
$reference = &$this->data['ObjectReference'];
if (empty($reference['uuid'])) {
$reference['uuid'] = CakeText::uuid();
}
if (empty($this->data['ObjectReference']['timestamp'])) {
$this->data['ObjectReference']['timestamp'] = time();
if (empty($reference['timestamp'])) {
$reference['timestamp'] = time();
}
if (!isset($this->data['ObjectReference']['comment'])) {
$this->data['ObjectReference']['comment'] = '';
if (!isset($reference['comment'])) {
$reference['comment'] = '';
}
return true;
}
@ -170,7 +191,13 @@ class ObjectReference extends AppModel
return true;
}
public function captureReference($reference, $eventId, $user)
/**
* @param array $reference
* @param int $eventId
* @return array|bool
* @throws Exception
*/
public function captureReference(array $reference, $eventId)
{
if (isset($reference['uuid'])) {
$existingReference = $this->find('first', array(
@ -257,6 +284,9 @@ class ObjectReference extends AppModel
$reference['object_uuid'] = $sourceObject['Object']['uuid'];
$reference['event_id'] = $eventId;
$result = $this->save(array('ObjectReference' => $reference));
if (!$result) {
return $this->validationErrors;
}
return true;
}
@ -308,7 +338,8 @@ class ObjectReference extends AppModel
return array($referenced_id, $referenced_uuid, $referenced_type);
}
function isValidExtendedEventForReference($sourceEvent, $targetEventID, $user) {
private function isValidExtendedEventForReference(array $sourceEvent, $targetEventID, array $user)
{
if ($sourceEvent['Event']['orgc_id'] != $user['org_id']) {
return false;
}

137
app/Model/Server.php
View File

@ -3020,71 +3020,70 @@ class Server extends AppModel
);
}
/**
* @throws Exception
*/
private function compareDBIndexes(array $actualIndex, array $expectedIndex, array $dbExpectedSchema)
{
$allowedlistTables = array();
$indexDiff = array();
foreach ($expectedIndex as $tableName => $indexes) {
if (!array_key_exists($tableName, $actualIndex)) {
continue; // If table does not exists, it is covered by the schema diagnostic
} elseif(in_array($tableName, $allowedlistTables)) {
continue; // Ignore allowedlisted tables
} else {
$tableIndexDiff = array_diff(array_keys($indexes), array_keys($actualIndex[$tableName])); // check for missing indexes
foreach ($tableIndexDiff as $columnDiff) {
$shouldBeUnique = $indexes[$columnDiff];
if ($shouldBeUnique && !$this->checkIfColumnContainsJustUniqueValues($tableName, $columnDiff)) {
$indexDiff[$tableName][$columnDiff] = array(
'message' => __('Column `%s` should be unique indexed, but contains duplicate values', $columnDiff),
'sql' => '',
}
$tableIndexDiff = array_diff(array_keys($indexes), array_keys($actualIndex[$tableName])); // check for missing indexes
foreach ($tableIndexDiff as $columnDiff) {
$shouldBeUnique = $indexes[$columnDiff];
if ($shouldBeUnique && !$this->checkIfColumnContainsJustUniqueValues($tableName, $columnDiff)) {
$indexDiff[$tableName][$columnDiff] = array(
'message' => __('Column `%s` should be unique indexed, but contains duplicate values', $columnDiff),
'sql' => '',
);
continue;
}
$message = __('Column `%s` should be indexed', $columnDiff);
$indexDiff[$tableName][$columnDiff] = array(
'message' => $message,
'sql' => $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $columnDiff, $shouldBeUnique),
);
}
$tableIndexDiff = array_diff(array_keys($actualIndex[$tableName]), array_keys($indexes)); // check for additional indexes
foreach ($tableIndexDiff as $columnDiff) {
$message = __('Column `%s` is indexed but should not', $columnDiff);
$indexDiff[$tableName][$columnDiff] = array(
'message' => $message,
'sql' => $this->generateSqlDropIndexQuery($tableName, $columnDiff),
);
}
foreach ($indexes as $column => $unique) {
if (isset($actualIndex[$tableName][$column]) && $actualIndex[$tableName][$column] != $unique) {
if ($actualIndex[$tableName][$column]) {
$sql = $this->generateSqlDropIndexQuery($tableName, $column);
$sql .= '<br>' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, false);
$message = __('Column `%s` has unique index, but should be non unique', $column);
$indexDiff[$tableName][$column] = array(
'message' => $message,
'sql' => $sql,
);
continue;
}
$message = __('Column `%s` should be indexed', $columnDiff);
$indexDiff[$tableName][$columnDiff] = array(
'message' => $message,
'sql' => $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $columnDiff, $shouldBeUnique),
);
}
$tableIndexDiff = array_diff(array_keys($actualIndex[$tableName]), array_keys($indexes)); // check for additional indexes
foreach ($tableIndexDiff as $columnDiff) {
$message = __('Column `%s` is indexed but should not', $columnDiff);
$indexDiff[$tableName][$columnDiff] = array(
'message' => $message,
'sql' => $this->generateSqlDropIndexQuery($tableName, $columnDiff),
);
}
foreach ($indexes as $column => $unique) {
if (isset($actualIndex[$tableName][$column]) && $actualIndex[$tableName][$column] != $unique) {
if ($actualIndex[$tableName][$column]) {
$sql = $this->generateSqlDropIndexQuery($tableName, $column);
$sql .= '<br>' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, false);
$message = __('Column `%s` has unique index, but should be non unique', $column);
} else {
if (!$this->checkIfColumnContainsJustUniqueValues($tableName, $column)) {
$message = __('Column `%s` should be unique index, but contains duplicate values', $column);
$indexDiff[$tableName][$column] = array(
'message' => $message,
'sql' => $sql,
);
} else {
if (!$this->checkIfColumnContainsJustUniqueValues($tableName, $column)) {
$message = __('Column `%s` should be unique index, but contains duplicate values', $column);
$indexDiff[$tableName][$column] = array(
'message' => $message,
'sql' => '',
);
continue;
}
$sql = $this->generateSqlDropIndexQuery($tableName, $column);
$sql .= '<br>' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, true);
$message = __('Column `%s` should be unique index', $column);
$indexDiff[$tableName][$column] = array(
'message' => $message,
'sql' => $sql,
'sql' => '',
);
continue;
}
$sql = $this->generateSqlDropIndexQuery($tableName, $column);
$sql .= '<br>' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, true);
$message = __('Column `%s` should be unique index', $column);
$indexDiff[$tableName][$column] = array(
'message' => $message,
'sql' => $sql,
);
}
}
}
@ -3121,21 +3120,21 @@ class Server extends AppModel
App::uses('Folder', 'Utility');
// check writeable directories
$writeableDirs = array(
'/tmp' => 0,
APP . 'tmp' => 0,
APP . 'files' => 0,
APP . 'files' . DS . 'scripts' . DS . 'tmp' => 0,
APP . 'tmp' . DS . 'csv_all' => 0,
APP . 'tmp' . DS . 'csv_sig' => 0,
APP . 'tmp' . DS . 'md5' => 0,
APP . 'tmp' . DS . 'sha1' => 0,
APP . 'tmp' . DS . 'snort' => 0,
APP . 'tmp' . DS . 'suricata' => 0,
APP . 'tmp' . DS . 'text' => 0,
APP . 'tmp' . DS . 'xml' => 0,
APP . 'tmp' . DS . 'files' => 0,
APP . 'tmp' . DS . 'logs' => 0,
APP . 'tmp' . DS . 'bro' => 0,
'/tmp' => 0,
APP . 'tmp' => 0,
APP . 'files' => 0,
APP . 'files' . DS . 'scripts' . DS . 'tmp' => 0,
APP . 'tmp' . DS . 'csv_all' => 0,
APP . 'tmp' . DS . 'csv_sig' => 0,
APP . 'tmp' . DS . 'md5' => 0,
APP . 'tmp' . DS . 'sha1' => 0,
APP . 'tmp' . DS . 'snort' => 0,
APP . 'tmp' . DS . 'suricata' => 0,
APP . 'tmp' . DS . 'text' => 0,
APP . 'tmp' . DS . 'xml' => 0,
APP . 'tmp' . DS . 'files' => 0,
APP . 'tmp' . DS . 'logs' => 0,
APP . 'tmp' . DS . 'bro' => 0,
);
foreach ($writeableDirs as $path => &$error) {
$dir = new Folder($path);

63
app/Model/ShadowAttribute.php
View File

@ -129,13 +129,13 @@ class ShadowAttribute extends AppModel
'first_seen' => array(
'rule' => array('datetimeOrNull'),
'required' => false,
'message' => array('Invalid ISO 8601 format')
'message' => array('Invalid ISO 8601 format'),
),
'last_seen' => array(
'datetimeOrNull' => array(
'rule' => array('datetimeOrNull'),
'required' => false,
'message' => array('Invalid ISO 8601 format')
'message' => array('Invalid ISO 8601 format'),
),
'validateLastSeenValue' => array(
'rule' => array('validateLastSeenValue'),
@ -173,7 +173,7 @@ class ShadowAttribute extends AppModel
$compositeTypes = $this->getCompositeTypes();
// explode composite types in value1 and value2
$pieces = explode('|', $this->data['ShadowAttribute']['value']);
if (in_array($this->data['ShadowAttribute']['type'], $compositeTypes)) {
if (in_array($this->data['ShadowAttribute']['type'], $compositeTypes, true)) {
if (2 != count($pieces)) {
throw new InternalErrorException('Composite type, but value not explodable');
}
@ -300,51 +300,54 @@ class ShadowAttribute extends AppModel
public function beforeValidate($options = array())
{
parent::beforeValidate();
// remove leading and trailing blanks
//$this->trimStringFields(); // TODO
if (!isset($this->data['ShadowAttribute']['comment'])) {
$this->data['ShadowAttribute']['comment'] = '';
}
if (!isset($this->data['ShadowAttribute']['type'])) {
$proposal = &$this->data['ShadowAttribute'];
if (!isset($proposal['type'])) {
$this->invalidate('type', 'No value provided.');
return false;
}
if (!isset($proposal['comment'])) {
$proposal['comment'] = '';
}
// make some changes to the inserted value
if (isset($this->data['ShadowAttribute']['value'])) {
$value = trim($this->data['ShadowAttribute']['value']);
$value = ComplexTypeTool::refangValue($value, $this->data['ShadowAttribute']['type']);
$value = $this->Attribute->modifyBeforeValidation($this->data['ShadowAttribute']['type'], $value);
$this->data['ShadowAttribute']['value'] = $value;
if (isset($proposal['value'])) {
$value = trim($proposal['value']);
$value = ComplexTypeTool::refangValue($value, $proposal['type']);
$value = $this->Attribute->modifyBeforeValidation($proposal['type'], $value);
$proposal['value'] = $value;
}
if (!isset($this->data['ShadowAttribute']['org'])) {
$this->data['ShadowAttribute']['org'] = '';
if (!isset($proposal['org'])) {
$proposal['org'] = '';
}
if (empty($this->data['ShadowAttribute']['timestamp'])) {
$date = new DateTime();
$this->data['ShadowAttribute']['timestamp'] = $date->getTimestamp();
if (empty($proposal['timestamp'])) {
$proposal['timestamp'] = time();
}
if (!isset($this->data['ShadowAttribute']['proposal_to_delete'])) {
$this->data['ShadowAttribute']['proposal_to_delete'] = 0;
if (!isset($proposal['proposal_to_delete'])) {
$proposal['proposal_to_delete'] = 0;
}
// generate UUID if it doesn't exist
if (empty($this->data['ShadowAttribute']['uuid'])) {
$this->data['ShadowAttribute']['uuid'] = CakeText::uuid();
if (empty($proposal['uuid'])) {
$proposal['uuid'] = CakeText::uuid();
} else {
$this->data['ShadowAttribute']['uuid'] = strtolower($this->data['ShadowAttribute']['uuid']);
$proposal['uuid'] = strtolower($proposal['uuid']);
}
if (!empty($this->data['ShadowAttribute']['type']) && empty($this->data['ShadowAttribute']['category'])) {
$this->data['ShadowAttribute']['category'] = $this->Attribute->typeDefinitions[$this->data['ShadowAttribute']['type']]['default_category'];
if (empty($proposal['category'])) {
$proposal['category'] = $this->Attribute->typeDefinitions[$proposal['type']]['default_category'];
}
if (isset($proposal['first_seen'])) {
$proposal['first_seen'] = $proposal['first_seen'] === '' ? null : $proposal['first_seen'];
}
if (isset($proposal['last_seen'])) {
$proposal['last_seen'] = $proposal['last_seen'] === '' ? null : $proposal['last_seen'];
}
// always return true, otherwise the object cannot be saved
return true;
}

9
app/Model/SharingGroup.php
View File

@ -5,6 +5,9 @@ App::uses('AppModel', 'Model');
* @property SharingGroupOrg $SharingGroupOrg
* @property SharingGroupServer $SharingGroupServer
* @property Organisation $Organisation
* @property Event $Event
* @property Attribute $Attribute
* @property Thread $Thread
*/
class SharingGroup extends AppModel
{
@ -101,6 +104,12 @@ class SharingGroup extends AppModel
if ($this->Attribute->hasAny(['sharing_group_id' => $this->id])) {
return false;
}
if ($this->Attribute->Object->hasAny(['sharing_group_id' => $this->id])) {
return false;
}
if ($this->Event->EventReport->hasAny(['sharing_group_id' => $this->id])) {
return false;
}
return true;
}

47
app/Model/SharingGroupOrg.php
View File

@ -1,29 +1,41 @@
<?php
App::uses('AppModel', 'Model');
class SharingGroupOrg extends AppModel
{
public $actsAs = array('AuditLog', 'Containable');
public $belongsTo = array(
'SharingGroup' => array(
'className' => 'SharingGroup',
'foreignKey' => 'sharing_group_id'
),
'Organisation' => array(
'className' => 'Organisation',
'foreignKey' => 'org_id',
//'conditions' => array('SharingGroupElement.organisation_uuid' => 'Organisation.uuid')
)
'SharingGroup' => array(
'className' => 'SharingGroup',
'foreignKey' => 'sharing_group_id'
),
'Organisation' => array(
'className' => 'Organisation',
'foreignKey' => 'org_id',
//'conditions' => array('SharingGroupElement.organisation_uuid' => 'Organisation.uuid')
)
);
public function beforeValidate($options = array())
{
parent::beforeValidate();
$data = $this->data[$this->alias];
$conditions = [
'sharing_group_id' => $data['sharing_group_id'],
'org_id' => $data['org_id'],
];
if (isset($data['id'])) {
$conditions['id !='] = $data['id'];
}
if ($this->hasAny($conditions)) {
$this->log("Trying to save duplicate organisation `{$data['org_id']}` for sharing group `{$data['sharing_group_id']}. This should never happened.");
$this->invalidate('org_id', 'The same organisation is already assigned to this sharing group.');
return false;
}
}
public function updateOrgsForSG($id, $new_orgs, $old_orgs, $user)
{
$log = ClassRegistry::init('Log');
// Loop through all of the organisations we want to add.
foreach ($new_orgs as $org) {
$SgO = array(
@ -54,16 +66,16 @@ class SharingGroupOrg extends AppModel
}
if ($this->save($SgO)) {
if ($isChange) {
$log->createLogEntry($user, 'edit', 'SharingGroupOrg', $this->id, 'Sharing group (' . $id . '): Modified right to alter sharing group for organisation (' . $org['id'] . ').', ($org['extend'] ? 'Organisation (' . $org['id'] . ') can now extend the sharing group.' : 'Organisation (' . $org['id'] . ') can no longer extend the sharing group.'));
$this->loadLog()->createLogEntry($user, 'edit', 'SharingGroupOrg', $this->id, 'Sharing group (' . $id . '): Modified right to alter sharing group for organisation (' . $org['id'] . ').', ($org['extend'] ? 'Organisation (' . $org['id'] . ') can now extend the sharing group.' : 'Organisation (' . $org['id'] . ') can no longer extend the sharing group.'));
} else {
$log->createLogEntry($user, 'add', 'SharingGroupOrg', $this->id, 'Sharing group (' . $id . '): Added organisation (' . $org['id'] . ').', 'Organisation (' . $org['id'] . ') added to Sharing group.' . ($org['extend'] ? ' Organisation (' . $org['id'] . ') can extend the sharing group.' : ''));
$this->loadLog()->createLogEntry($user, 'add', 'SharingGroupOrg', $this->id, 'Sharing group (' . $id . '): Added organisation (' . $org['id'] . ').', 'Organisation (' . $org['id'] . ') added to Sharing group.' . ($org['extend'] ? ' Organisation (' . $org['id'] . ') can extend the sharing group.' : ''));
}
}
}
// We are left with some "old orgs" that are not in the new list. This means that they can be safely deleted.
foreach ($old_orgs as $old_org) {
if ($this->delete($old_org['id'])) {
$log->createLogEntry($user, 'delete', 'SharingGroupOrg', $old_org['id'], 'Sharing group (' . $id . '): Removed organisation (' . $old_org['id'] . ').', 'Organisation (' . $org['id'] . ') removed from Sharing group.');
$this->loadLog()->createLogEntry($user, 'delete', 'SharingGroupOrg', $old_org['id'], 'Sharing group (' . $id . '): Removed organisation (' . $old_org['id'] . ').', 'Organisation (' . $org['id'] . ') removed from Sharing group.');
}
}
}
@ -83,7 +95,12 @@ class SharingGroupOrg extends AppModel
return $sgs;
}
// pass a sharing group ID and an organisation ID, returns true if it has a matching attached organisation object
/**
* Pass a sharing group ID and an organisation ID, returns true if it has a matching attached organisation object
* @param int $id
* @param int $org_id
* @return bool
*/
public function checkIfAuthorised($id, $org_id)
{
return $this->hasAny([

143
app/Model/Tag.php
View File

@ -22,28 +22,28 @@ class Tag extends AppModel
);
public $validate = array(
'name' => array(
'required' => array(
'rule' => array('notBlank', 'name'),
'message' => 'This field is required.'
),
'valueNotEmpty' => array(
'rule' => array('valueNotEmpty', 'name'),
),
'unique' => array(
'rule' => 'isUnique',
'message' => 'A similar name already exists.',
),
'name' => array(
'required' => array(
'rule' => array('notBlank', 'name'),
'message' => 'This field is required.'
),
'colour' => array(
'valueNotEmpty' => array(
'rule' => array('valueNotEmpty', 'colour'),
),
'userdefined' => array(
'rule' => 'validateColour',
'message' => 'Colour has to be in the RGB format (#FFFFFF)',
),
'valueNotEmpty' => array(
'rule' => array('valueNotEmpty', 'name'),
),
'unique' => array(
'rule' => 'isUnique',
'message' => 'A similar name already exists.',
),
),
'colour' => array(
'valueNotEmpty' => array(
'rule' => array('valueNotEmpty', 'colour'),
),
'userdefined' => array(
'rule' => 'validateColour',
'message' => 'Colour has to be in the RGB format (#FFFFFF)',
),
),
);
public $hasMany = array(
@ -83,7 +83,6 @@ class Tag extends AppModel
public function beforeValidate($options = array())
{
parent::beforeValidate();
if (!isset($this->data['Tag']['org_id'])) {
$this->data['Tag']['org_id'] = 0;
}
@ -149,8 +148,7 @@ class Tag extends AppModel
public function afterFind($results, $primary = false)
{
$results = $this->checkForOverride($results);
return $results;
return $this->checkForOverride($results);
}
public function validateColour($fields)
@ -161,12 +159,41 @@ class Tag extends AppModel
return true;
}
/**
* @param array $user
* @param string $tagName
* @return mixed|null
*/
public function lookupTagIdForUser(array $user, $tagName)
{
$conditions = ['LOWER(Tag.name)' => mb_strtolower($tagName)];
if (!$user['Role']['perm_site_admin']) {
$conditions['Tag.org_id'] = [0, $user['org_id']];
$conditions['Tag.user_id'] = [0, $user['id']];
}
$tagId = $this->find('first', array(
'conditions' => $conditions,
'recursive' => -1,
'fields' => array('Tag.id'),
'callbacks' => false,
));
if (empty($tagId)) {
return null;
}
return $tagId['Tag']['id'];
}
/**
* @param string $tagName
* @return int|mixed
*/
public function lookupTagIdFromName($tagName)
{
$tagId = $this->find('first', array(
'conditions' => array('LOWER(Tag.name)' => strtolower($tagName)),
'conditions' => array('LOWER(Tag.name)' => mb_strtolower($tagName)),
'recursive' => -1,
'fields' => array('Tag.id')
'fields' => array('Tag.id'),
'callbacks' => false,
));
if (empty($tagId)) {
return -1;
@ -286,7 +313,7 @@ class Tag extends AppModel
return array_values($tag_ids);
}
public function findEventIdsByTagNames($array)
private function findEventIdsByTagNames($array)
{
$ids = array();
foreach ($array as $a) {
@ -310,26 +337,6 @@ class Tag extends AppModel
return $ids;
}
public function findAttributeIdsByAttributeTagNames($array)
{
$ids = array();
foreach ($array as $a) {
$conditions['OR'][] = array('LOWER(name) LIKE' => strtolower($a));
}
$params = array(
'recursive' => 1,
'contain' => 'AttributeTag',
'conditions' => $conditions
);
$result = $this->find('all', $params);
foreach ($result as $tag) {
foreach ($tag['AttributeTag'] as $attributeTag) {
$ids[] = $attributeTag['attribute_id'];
}
}
return $ids;
}
/**
* @param array $tag
* @param array $user
@ -337,17 +344,18 @@ class Tag extends AppModel
* @return false|int
* @throws Exception
*/
public function captureTag($tag, $user, $force=false)
public function captureTag(array $tag, array $user, $force=false)
{
$existingTag = $this->find('first', array(
'recursive' => -1,
'conditions' => array('LOWER(name)' => mb_strtolower($tag['name'])),
'fields' => ['id', 'org_id', 'user_id'],
'callbacks' => false,
));
if (empty($existingTag)) {
if ($force || $user['Role']['perm_tag_editor']) {
$this->create();
if (!isset($tag['colour']) || empty($tag['colour'])) {
if (empty($tag['colour'])) {
$tag['colour'] = $this->random_color();
}
$tag = array(
@ -363,22 +371,21 @@ class Tag extends AppModel
} else {
return false;
}
} else {
if (
!$user['Role']['perm_site_admin'] &&
}
if (
!$user['Role']['perm_site_admin'] &&
(
(
(
$existingTag['Tag']['org_id'] != 0 &&
$existingTag['Tag']['org_id'] != $user['org_id']
) ||
(
$existingTag['Tag']['user_id'] != 0 &&
$existingTag['Tag']['user_id'] != $user['id']
)
$existingTag['Tag']['org_id'] != 0 &&
$existingTag['Tag']['org_id'] != $user['org_id']
) ||
(
$existingTag['Tag']['user_id'] != 0 &&
$existingTag['Tag']['user_id'] != $user['id']
)
) {
return false;
}
)
) {
return false;
}
return $existingTag['Tag']['id'];
}
@ -434,15 +441,21 @@ class Tag extends AppModel
}
/**
* Recover user_id from the session and override numerical_values from userSetting
*/
public function checkForOverride($tags)
* Recover user_id from the session and override numerical_values from userSetting.
*
* @param array $tags
* @return array
*/
private function checkForOverride($tags)
{
$userId = Configure::read('CurrentUserId');
if ($this->tagOverrides === false && $userId > 0) {
$this->UserSetting = ClassRegistry::init('UserSetting');
$this->tagOverrides = $this->UserSetting->getTagNumericalValueOverride($userId);
}
if (empty($this->tagOverrides)) {
return $tags;
}
foreach ($tags as $k => $tag) {
if (isset($tag['Tag']['name'])) {
$tagName = $tag['Tag']['name'];

70
app/Model/User.php
View File

@ -231,9 +231,9 @@ class User extends AppModel
public function __construct($id = false, $table = null, $ds = null)
{
parent::__construct($id, $table, $ds);
$this->AdminSetting = ClassRegistry::init('AdminSetting');
$db_version = $this->AdminSetting->getSetting('db_version');
if ($db_version >= 62) {
// bind AuthKey just when authkey table already exists. This is important for updating from old versions
if (in_array('auth_keys', $this->getDataSource()->listSources(), true)) {
$this->bindModel([
'hasMany' => ['AuthKey']
], false);
@ -1504,4 +1504,68 @@ class User extends AppModel
$banStatus['message'] = __('User email notification ban setting is not enabled');
return $banStatus;
}
/**
* @param array $user
* @return bool
*/
public function hasNotifications(array $user)
{
$hasProposal = $this->Event->ShadowAttribute->hasAny([
'ShadowAttribute.event_org_id' => $user['org_id'],
'ShadowAttribute.deleted' => 0,
]);
if ($hasProposal) {
return true;
}
if (Configure::read('MISP.delegation') && $this->_getDelegationCount($user)) {
return true;
}
return false;
}
/**
* @param array $user
* @return array
*/
public function populateNotifications(array $user)
{
$notifications = array();
list($notifications['proposalCount'], $notifications['proposalEventCount']) = $this->_getProposalCount($user);
$notifications['total'] = $notifications['proposalCount'];
if (Configure::read('MISP.delegation')) {
$notifications['delegationCount'] = $this->_getDelegationCount($user);
$notifications['total'] += $notifications['delegationCount'];
}
return $notifications;
}
// if not using $mode === 'full', simply check if an entry exists. We really don't care about the real count for the top menu.
private function _getProposalCount($user, $mode = 'full')
{
$results[0] = $this->Event->ShadowAttribute->find('count', [
'conditions' => array(
'ShadowAttribute.event_org_id' => $user['org_id'],
'ShadowAttribute.deleted' => 0,
)
]);
$results[1] = $this->Event->ShadowAttribute->find('count', [
'conditions' => array(
'ShadowAttribute.event_org_id' => $user['org_id'],
'ShadowAttribute.deleted' => 0,
),
'fields' => 'distinct event_id'
]);
return $results;
}
private function _getDelegationCount($user)
{
$this->EventDelegation = ClassRegistry::init('EventDelegation');
return $this->EventDelegation->find('count', array(
'recursive' => -1,
'conditions' => array('EventDelegation.org_id' => $user['org_id'])
));
}
}

83
app/Model/UserSetting.php
View File

@ -1,5 +1,9 @@
<?php
App::uses('AppModel', 'Model');
/**
* @property User $User
*/
class UserSetting extends AppModel
{
public $useTable = 'user_settings';
@ -92,12 +96,14 @@ class UserSetting extends AppModel
'event_index_hide_columns' => [
'placeholder' => ['clusters'],
],
'oidc' => [ // Data saved by OIDC plugin
'restricted' => 'perm_site_admin',
],
);
// massage the data before we send it off for validation before saving anything
public function beforeValidate($options = array())
{
parent::beforeValidate();
// add a timestamp if it is not set
if (empty($this->data['UserSetting']['timestamp'])) {
$this->data['UserSetting']['timestamp'] = time();
@ -120,7 +126,9 @@ class UserSetting extends AppModel
public function afterFind($results, $primary = false)
{
foreach ($results as $k => $v) {
$results[$k]['UserSetting']['value'] = json_decode($v['UserSetting']['value'], true);
if (isset($v['UserSetting']['value'])) {
$results[$k]['UserSetting']['value'] = json_decode($v['UserSetting']['value'], true);
}
}
return $results;
}
@ -130,25 +138,27 @@ class UserSetting extends AppModel
return isset($this->validSettings[$setting]);
}
public function checkSettingAccess($user, $setting)
/**
* @param array $user
* @param string $setting
* @return bool|string
*/
public function checkSettingAccess(array $user, $setting)
{
if (!empty($this->validSettings[$setting]['restricted'])) {
$role_check = $this->validSettings[$setting]['restricted'];
if (!is_array($role_check)) {
$role_check = array($role_check);
$roleCheck = $this->validSettings[$setting]['restricted'];
if (!is_array($roleCheck)) {
$roleCheck = array($roleCheck);
}
$userHasValidRole = false;
foreach ($role_check as $role) {
foreach ($roleCheck as $role) {
if (!empty($user['Role'][$role])) {
return true;
}
}
if (!$userHasValidRole) {
foreach ($role_check as &$role) {
$role = substr($role, 5);
}
return implode(', ', $role_check);
foreach ($roleCheck as &$role) {
$role = substr($role, 5);
}
return implode(', ', $roleCheck);
}
return true;
}
@ -199,7 +209,7 @@ class UserSetting extends AppModel
return false;
}
public function getDefaulRestSearchParameters($user)
public function getDefaultRestSearchParameters($user)
{
return $this->getValueForUser($user['id'], 'default_restsearch_parameters') ?: [];
}
@ -232,8 +242,8 @@ class UserSetting extends AppModel
/**
* Check whether the event is something the user is interested (to be alerted on)
* @param $user
* @param $event
* @param array $user
* @param array $event
* @return bool
*/
public function checkPublishFilter(array $user, array $event)
@ -352,7 +362,13 @@ class UserSetting extends AppModel
return false;
}
public function setSetting($user, &$data)
/**
* @param array $user
* @param array $data
* @return bool
* @throws Exception
*/
public function setSetting(array $user, array $data)
{
$userSetting = array();
if (!empty($data['UserSetting']['user_id']) && is_numeric($data['UserSetting']['user_id'])) {
@ -387,21 +403,42 @@ class UserSetting extends AppModel
} else {
$userSetting['value'] = '';
}
return $this->setSettingInternal($userSetting['user_id'], $userSetting['setting'], $userSetting['value']);
}
/**
* Set user setting without checking permission.
* @param int $userId
* @param string $setting
* @param mixed $value
* @return array|bool|mixed|null
* @throws Exception
*/
public function setSettingInternal($userId, $setting, $value)
{
$userSetting = [
'user_id' => $userId,
'setting' => $setting,
'value' => $value,
];
$existingSetting = $this->find('first', array(
'recursive' => -1,
'conditions' => array(
'UserSetting.user_id' => $userSetting['user_id'],
'UserSetting.setting' => $userSetting['setting']
)
'UserSetting.user_id' => $userId,
'UserSetting.setting' => $setting,
),
'fields' => ['UserSetting.id'],
'callbacks' => false,
));
if (empty($existingSetting)) {
$this->create();
} else {
$userSetting['id'] = $existingSetting['UserSetting']['id'];
}
// save the setting
$result = $this->save(array('UserSetting' => $userSetting));
return true;
return $this->save($userSetting);
}
/**

22
app/Plugin/OidcAuth/Controller/Component/Auth/OidcAuthenticate.php
View File

@ -86,7 +86,7 @@ class OidcAuthenticate extends BaseAuthenticate
$this->log($mispUsername, "Unblocking user.");
$user['disabled'] = false;
}
$this->storeMetadata($user['id'], $verifiedClaims);
$this->log($mispUsername, 'Logged in.');
return $user;
}
@ -106,6 +106,8 @@ class OidcAuthenticate extends BaseAuthenticate
throw new RuntimeException("Could not save user `$mispUsername` to database.");
}
$this->storeMetadata($this->userModel()->id, $verifiedClaims);
$this->log($mispUsername, "Saved in database with ID {$this->userModel()->id}");
$this->log($mispUsername, 'Logged in.');
return $this->_findUser($mispUsername);
@ -227,6 +229,24 @@ class OidcAuthenticate extends BaseAuthenticate
return $value;
}
/**
* @param int $userId
* @param stdClass $verifiedClaims
* @return array|bool|mixed|null
* @throws Exception
*/
private function storeMetadata($userId, $verifiedClaims)
{
$value = [];
foreach (['sub', 'preferred_username', 'given_name', 'family_name'] as $field) {
if (property_exists($verifiedClaims, $field)) {
$value[$field] = $verifiedClaims->{$field};
}
}
return $this->userModel()->UserSetting->setSettingInternal($userId, 'oidc', $value);
}
/**
* @param string $username
* @param string $message

38
app/View/Elements/Events/View/eventFilteringQueryBuilder.ctp
View File

@ -5,20 +5,19 @@ foreach ($warninglists as $id => $name) {
}
$warninglistsValues = json_encode($warninglistsValues);
?>
<div id="eventFilteringQBWrapper" style="padding: 5px; display: none; border: 1px solid #dddddd; border-bottom: 0px;">
<div id="eventFilteringQBWrapper" style="padding: 5px; display: none; border: 1px solid #dddddd; border-bottom: 0;">
<div id="eventFilteringQB" style="overflow-y: auto; padding-right: 5px; resize: vertical; max-height: 750px; height: 400px;"></div>
<div style="display: flex; justify-content: flex-end; margin-top: 5px;">
<input id="eventFilteringQBLinkInput" class="form-control" style="width: 66%;">
<button id="eventFilteringQBLinkCopy" type="button" class="btn btn-inverse" style="margin-right: 5px; margin-left: 5px;" onclick="clickMessage(this);"> <i class="fa fa-clipboard"></i> <?php echo h('Copy to clipboard'); ?> </button>
<button id="eventFilteringQBSubmit" type="button" class="btn btn-success" style="margin-right: 5px;"> <i class="fa fa-filter"></i> <?php echo h('Filter'); ?> </button>
<button id="eventFilteringQBClear" type="button" class="btn btn-xs btn-danger" style="" title="<?php echo h('Clear filtering rules'); ?>"> <i class="fa fa-times"></i> <?php echo h('Clear'); ?> </button>
<input id="eventFilteringQBLinkInput" class="form-control" style="width: 66%;">
<button id="eventFilteringQBLinkCopy" type="button" class="btn btn-inverse" style="margin-right: 5px; margin-left: 5px;"> <i class="fa fa-clipboard"></i> Copy to clipboard</button>
<button id="eventFilteringQBSubmit" type="button" class="btn btn-success" style="margin-right: 5px;"> <i class="fa fa-filter"></i> Filter</button>
<button id="eventFilteringQBClear" type="button" class="btn btn-xs btn-danger" style="" title="Clear filtering rules"> <i class="fa fa-times"></i> Clear</button>
</div>
</div>
<?php
?>
<script>
var defaultFilteringRules = <?php echo json_encode($defaultFilteringRules); ?>;
var defaultFilteringRules = <?= json_encode($defaultFilteringRules); ?>;
var querybuilderTool;
function triggerEventFilteringTool(hide) {
var qbOptions = {
@ -119,9 +118,9 @@ function triggerEventFilteringTool(hide) {
"id": "deleted",
"label": "Deleted",
"values": {
0: "Deleted only",
0: "Exclude deleted",
1: "Both",
2: "Exclude deleted"
2: "Deleted only",
}
},
{
@ -303,7 +302,7 @@ function triggerEventFilteringTool(hide) {
<?php if (isset($filters['attributeFilter'])): ?>
value: "<?php echo in_array($filters['attributeFilter'], array('all', 'network', 'financial', 'file')) ? h($filters['attributeFilter']) : 'all'; ?>"
<?php else: ?>
value: "<?php echo 'all'; ?>"
value: 'all'
<?php endif; ?>
},
<?php endif; ?>
@ -339,7 +338,7 @@ function triggerEventFilteringTool(hide) {
{
field: 'deleted',
id: 'deleted',
value: <?php echo isset($filters['deleted']) ? h($filters['deleted']) : 2; ?>
value: <?php echo isset($filters['deleted']) ? h($filters['deleted']) : 0; ?>
},
<?php endif; ?>
<?php if (empty($advancedFilteringActiveRules) || isset($advancedFilteringActiveRules['includeRelatedTags'])): ?>
@ -425,7 +424,6 @@ function triggerEventFilteringTool(hide) {
},
};
var filters = <?php echo json_encode($filters); ?>;
var $wrapper = $('#eventFilteringQBWrapper');
var $ev = $('#eventFilteringQB');
querybuilderTool = $ev.queryBuilder(qbOptions);
@ -446,6 +444,7 @@ function triggerEventFilteringTool(hide) {
$('#eventFilteringQBLinkCopy').off('click').on('click', function() {
copyToClipboard($('#eventFilteringQBLinkInput'));
clickMessage(this);
});
$('#eventFilteringQBClear').off('click').on('click', function() {
@ -525,27 +524,22 @@ function cleanRules(rules) {
function performQuery(rules) {
var res = cleanRules(rules);
var url = "<?php echo $baseurl; ?>/events/viewEventAttributes/<?php echo h($event['Event']['id']); ?>";
$.ajax({
var url = "/events/viewEventAttributes/<?php echo h($event['Event']['id']); ?>";
xhr({
type: "post",
url: url,
data: res,
beforeSend: function () {
$(".loading").show();
},
success:function (data) {
success: function (data) {
$("#attributes_div").html(data);
$(".loading").hide();
},
error:function() {
error: function() {
showMessage('fail', 'Something went wrong - could not fetch attributes.');
}
});
}
function clickMessage(clicked) {
$clicked = $(clicked);
var $clicked = $(clicked);
$clicked.tooltip({
title: 'Copied!',
trigger: 'manual',

6
app/View/Elements/global_menu.ctp
View File

@ -513,10 +513,10 @@
h($me['email']),
$this->UserName->prepend($me['email']),
h($this->UserName->convertEmailToName($me['email'])),
isset($notifications) ? sprintf(
isset($hasNotifications) ? sprintf(
'<i class="fa fa-envelope %s" role="img" aria-label="%s"></i>',
(($notifications['total'] == 0) ? 'white' : 'red'),
__('Notifications') . ': ' . $notifications['total']
$hasNotifications ? 'red' : 'white',
__('Notifications')
) : ''
)
),

2
app/files/misp-galaxy

@ -1 +1 @@
Subproject commit 20f14c0090f1963ee485b98fa99c62f7830f734f
Subproject commit aeb57194485cb9fa8483b6bc21308d44e921ff75

2
app/files/scripts/misp-stix

@ -1 +1 @@
Subproject commit be3f694cd46ac26619ed8e8eaa73f45b2f62d7df
Subproject commit 04b8c09a56b230789726bc1019efe2c304964f22

13
db_schema.json
View File

@ -7414,7 +7414,7 @@
"character_maximum_length": null,
"numeric_precision": "3",
"collation_name": null,
"column_type": "tinyint(4)",
"column_type": "tinyint(1)",
"column_default": "0",
"extra": ""
},
@ -7978,12 +7978,9 @@
},
"object_references": {
"id": true,
"source_uuid": false,
"referenced_uuid": false,
"timestamp": false,
"object_id": false,
"referenced_id": false,
"relationship_type": false
"event_id": false
},
"object_relationships": {
"id": true,
@ -8062,6 +8059,7 @@
"sharing_groups": {
"id": true,
"uuid": true,
"name": true,
"org_id": false,
"sync_user_id": false,
"organisation_uuid": false
@ -8168,8 +8166,7 @@
"user_settings": {
"id": true,
"setting": false,
"user_id": false,
"timestamp": false
"user_id": false
},
"warninglists": {
"id": true
@ -8182,5 +8179,5 @@
"id": true
}
},
"db_version": "72"
"db_version": "75"
}

4
docs/INSTALL.rhel7.md
View File

@ -219,6 +219,7 @@ installCoreRHEL7 () {
cd $PATH_TO_MISP
# Fetch submodules
$SUDO_WWW git submodule sync
$SUDO_WWW git submodule update --init --recursive
# Make git ignore filesystem permission differences for submodules
$SUDO_WWW git submodule foreach --recursive git config core.filemode false
@ -237,6 +238,9 @@ installCoreRHEL7 () {
UMASK=$(umask)
umask 0022
# install python-stix dependencies
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install ordered-set python-dateutil six weakrefmethod
# install zmq
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U zmq

3
docs/INSTALL.rhel8.md
View File

@ -272,6 +272,9 @@ installCoreRHEL8 () {
UMASK=$(umask)
umask 0022
# install python-stix dependencies
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install ordered-set python-dateutil six weakrefmethod
# install zmq, redis
$SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U zmq redis

3
docs/INSTALL.ubuntu1804.md
View File

@ -145,6 +145,9 @@ installCore () {
sudo mkdir /var/www/.cache/
sudo chown ${WWW_USER}:${WWW_USER} /var/www/.cache
# install python-stix dependencies
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ordered-set python-dateutil six weakrefmethod
debug "Install PyMISP"
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/PyMISP

3
docs/INSTALL.ubuntu2004.md
View File

@ -137,6 +137,9 @@ installCore () {
sudo mkdir /var/www/.cache/
sudo chown ${WWW_USER}:${WWW_USER} /var/www/.cache
# install python-stix dependencies
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ordered-set python-dateutil six weakrefmethod
debug "Install PyMISP"
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/PyMISP

3
docs/generic/MISP_CAKE_init.md
View File

@ -212,6 +212,9 @@ coreCAKE () {
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.block_old_event_alert" false
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.block_old_event_alert_age" ""
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.block_old_event_alert_by_date" ""
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.event_alert_republish_ban" false
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.event_alert_republish_ban_threshold" 5
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.event_alert_republish_ban_refresh_on_retry" false
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.incoming_tags_disabled_by_default" false
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.maintenance_message" "Great things are happening! MISP is undergoing maintenance, but will return shortly. You can contact the administration at \$email."
${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.footermidleft" "This is an initial install"

13
docs/xINSTALL.OpenBSD.md
View File

@ -1,5 +1,5 @@
# INSTALLATION INSTRUCTIONS
## for OpenBSD 6.9-amd64
## for OpenBSD 7.0-amd64
!!! warning
This is not fully working yet. Mostly it is a template for our ongoing documentation efforts :spider:
@ -13,11 +13,6 @@
!!! notice
This guide attempts to offer native httpd or apache2/nginx.
!!! warning
As of 20181018 the native httpd server is NOT useable with MISP on OpenBSD 6.3.
Thus ONLY Apache 2.x available.
NO *rewrite* available, just yet. It will be in [the next release](https://marc.info/?l=openbsd-tech&m=152761257806283&w=2)
!!! notice
As of OpenBSD 6.4 the native httpd has rewrite rules and php 5.6 is gone too.
@ -86,7 +81,7 @@ doas pkg_add -v mariadb-server
#### Install misc dependencies
```bash
doas pkg_add -v curl git python--%3.8 redis libmagic autoconf--%2.71 automake--%1.16 libtool unzip--iconv rust
doas pkg_add -v curl git sqlite3 python--%3.9 redis libmagic autoconf--%2.71 automake--%1.16 libtool unzip--iconv rust
```
```bash
@ -229,8 +224,8 @@ doas rcctl enable httpd
#### Install Python virtualenv
```bash
doas pkg_add -v py3-virtualenv py3-pip
doas ln -sf /usr/local/bin/pip3.8 /usr/local/bin/pip
doas ln -s /usr/local/bin/python3.8 /usr/local/bin/python
doas ln -sf /usr/local/bin/pip3.9 /usr/local/bin/pip
doas ln -s /usr/local/bin/python3.9 /usr/local/bin/python
doas mkdir /usr/local/virtualenvs
doas /usr/local/bin/virtualenv /usr/local/virtualenvs/MISP
```

2
mkdocs.yml
View File

@ -76,7 +76,7 @@ nav:
- 'Warning': 'xINSTALL.md'
- 'Debian 10': 'xINSTALL.debian10.md'
- 'Tsurugi Linux': 'xINSTALL.tsurugi.md'
- 'OpenBSD 6.8': 'xINSTALL.OpenBSD.md'
- 'OpenBSD 7.0': 'xINSTALL.OpenBSD.md'
- Config Guides:
- 'Elastic Search Logging': 'CONFIG.elasticsearch-logging.md'
- 'Amazon S3 attachments': 'CONFIG.s3-attachments.md'

61
tests/testlive_comprehensive_local.py
View File

@ -62,6 +62,7 @@ class TestComprehensive(unittest.TestCase):
def setUp(self):
self.user_misp_connector.global_pythonify = True
self.admin_misp_connector.global_pythonify = True
def test_search_index(self):
# Search all events
@ -330,7 +331,7 @@ class TestComprehensive(unittest.TestCase):
# Create test event
event = create_simple_event()
event = self.user_misp_connector.add_event(event, pythonify=True)
event = self.user_misp_connector.add_event(event)
check_response(event)
# Delete event
@ -338,6 +339,64 @@ class TestComprehensive(unittest.TestCase):
check_response(self.admin_misp_connector.set_server_setting('MISP.enableEventBlocklisting', 0))
def test_deleted_attributes(self):
# Create test event
event = create_simple_event()
event.add_attribute('text', "deleted", deleted=True)
event.add_attribute('text', "not-deleted")
event = self.user_misp_connector.add_event(event)
check_response(event)
# Not deleted
fetched_event = self.user_misp_connector.get_event(event)
check_response(fetched_event)
self.assertEqual(len(fetched_event.attributes), 2, fetched_event)
# Not deleted
fetched_event = self.user_misp_connector.get_event(event, deleted=0)
check_response(fetched_event)
self.assertEqual(len(fetched_event.attributes), 2, fetched_event)
# Include deleted
fetched_event = self.user_misp_connector.get_event(event, deleted=1)
check_response(fetched_event)
self.assertEqual(len(fetched_event.attributes), 3, fetched_event)
# Deleted only
fetched_event = self.user_misp_connector.get_event(event, deleted=2)
check_response(fetched_event)
self.assertEqual(len(fetched_event.attributes), 1, fetched_event)
# Both
fetched_event = self.user_misp_connector.get_event(event, deleted=[0, 1])
check_response(fetched_event)
self.assertEqual(len(fetched_event.attributes), 3, fetched_event)
check_response(self.user_misp_connector.delete_event(event))
def test_view_event_exclude_local_tags(self):
event = create_simple_event()
event.add_tag({"name": "local", "local": 1})
event.add_tag({"name": "global", "local": 0})
event.attributes[0].add_tag({"name": "local", "local": 1})
event.attributes[0].add_tag({"name": "global", "local": 0})
event = self.admin_misp_connector.add_event(event)
check_response(event)
event_with_local_tags = self.admin_misp_connector.get_event(event)
check_response(event_with_local_tags)
self.assertEqual(len(event_with_local_tags.tags), 2)
self.assertEqual(len(event_with_local_tags.attributes[0].tags), 2)
event_without_local_tags = self.admin_misp_connector._check_json_response(self.admin_misp_connector._prepare_request('GET', f'events/view/{event.id}/excludeLocalTags:1'))
check_response(event_without_local_tags)
self.assertEqual(event_without_local_tags["Event"]["Tag"][0]["local"], 0, event_without_local_tags)
self.assertEqual(event_without_local_tags["Event"]["Attribute"][0]["Tag"][0]["local"], 0, event_without_local_tags)
check_response(self.admin_misp_connector.delete_event(event))
if __name__ == '__main__':
unittest.main()

8
tests/testlive_security.py
View File

@ -268,14 +268,14 @@ class TestSecurity(unittest.TestCase):
def test_user_must_change_password(self):
updated_user = self.admin_misp_connector.update_user({'change_pw': 1}, self.test_usr)
check_response(updated_user)
self.assertEqual(updated_user.change_pw, "1")
self.assertTrue(updated_user.change_pw)
# Try to login, should still work because key is still valid
PyMISP(url, self.test_usr.authkey)
updated_user = self.admin_misp_connector.update_user({'change_pw': 0}, self.test_usr)
check_response(updated_user)
self.assertEqual(updated_user.change_pw, "0")
self.assertFalse(updated_user.change_pw)
# Try to login, should also still works
PyMISP(url, self.test_usr.authkey)
@ -284,7 +284,7 @@ class TestSecurity(unittest.TestCase):
# Admin set that user must change password
updated_user = self.admin_misp_connector.update_user({'change_pw': 1}, self.test_usr)
check_response(updated_user)
self.assertEqual(updated_user.change_pw, "1")
self.assertTrue(updated_user.change_pw)
# User try to change back trough API
logged_in = PyMISP(url, self.test_usr.authkey)
@ -292,7 +292,7 @@ class TestSecurity(unittest.TestCase):
updated_user = self.admin_misp_connector.get_user(self.test_usr)
# Should not be possible
self.assertEqual(updated_user.change_pw, "1")
self.assertTrue(updated_user.change_pw)
def test_disabled_user(self):
# Disable user

21
tests/testlive_sync.py
View File

@ -1,5 +1,5 @@
import os
from pymisp import PyMISP, MISPEvent
from pymisp import PyMISP, MISPEvent, MISPGalaxyCluster
def check_response(response):
@ -53,6 +53,24 @@ check_response(event)
# Publish that event
check_response(pymisp.publish(event))
# Publish event inline
url = f'events/publish/{event.id}/disable_background_processing:1'
push_event = pymisp._check_json_response(pymisp._prepare_request('POST', url))
check_response(push_event)
# Create testing galaxy cluster
galaxy = pymisp.galaxies()[0]
galaxy_cluster = MISPGalaxyCluster()
galaxy_cluster.value = "Test Cluster"
galaxy_cluster.authors = ["MISP"]
galaxy_cluster.distribution = 1
galaxy_cluster.description = "Example test cluster"
galaxy_cluster = pymisp.add_galaxy_cluster(galaxy.id, galaxy_cluster)
check_response(galaxy_cluster)
# Publish that galaxy cluster
check_response(pymisp.publish_galaxy_cluster(galaxy_cluster))
# Preview event
url = f'servers/previewEvent/{remote_server["id"]}/{event.uuid}'
event_preview = pymisp._check_json_response(pymisp._prepare_request('GET', url))
@ -92,3 +110,4 @@ check_response(rules_response)
check_response(pymisp.delete_server(remote_server))
check_response(pymisp.delete_event(event))
check_response(pymisp.delete_event_blocklist(event))
check_response(pymisp.delete_galaxy_cluster(galaxy_cluster))