MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'develop' into local_tags

pull/7892/head
Loïc Fortemps 2021-10-27 17:06:02 +02:00
parent a1ba1c053e fb8112befc
commit dc71f6e451
No known key found for this signature in database
GPG Key ID: 98878A7B36234F0D
63 changed files with 2337 additions and 1648 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/main.yml vendored
View File

@ -255,6 +255,7 @@ jobs:
poetry add lxml
poetry run python ../tests/testlive_security.py -v
poetry run python ../tests/testlive_sync.py
poetry run python ../tests/testlive_comprehensive_local.py -v
poetry run python tests/test_mispevent.py
popd
cp PyMISP/tests/keys.py PyMISP/examples/events/

1
INSTALL/MYSQL.sql
View File

@ -1056,6 +1056,7 @@ CREATE TABLE IF NOT EXISTS `sharing_groups` (
INDEX `org_id` (`org_id`),
INDEX `sync_user_id` (`sync_user_id`),
UNIQUE INDEX `uuid` (`uuid`),
UNIQUE INDEX `name` (`name`),
INDEX `organisation_uuid` (`organisation_uuid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;

20
app/Console/Command/EventShell.php
View File

@ -1,6 +1,7 @@
<?php
App::uses('Folder', 'Utility');
App::uses('File', 'Utility');
App::uses('FileAccessTool', 'Tools');
require_once 'AppShell.php';
/**
@ -404,7 +405,6 @@ class EventShell extends AppShell
public function publish_sightings()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[2]) || empty($this->args[3])) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Publish sightings'] . PHP_EOL);
}
@ -414,14 +414,8 @@ class EventShell extends AppShell
$sightingsUuidsToPush = [];
if (isset($this->args[4])) { // push just specific sightings
$path = APP . 'tmp/cache/ingest' . DS . $this->args[4];
$tempFile = new File($path);
$inputData = $tempFile->read();
if ($inputData === false) {
$this->error("File `$path` not found.");
}
$sightingsUuidsToPush = $this->Event->jsonDecode($inputData);
$tempFile->delete();
$path = $this->args[4][0] === '/' ? $this->args[4] : (APP . 'tmp/cache/ingest' . DS . $this->args[4]);
$sightingsUuidsToPush = $this->Event->jsonDecode(FileAccessTool::readAndDelete($path));
}
$this->Event->Behaviors->unload('SysLogLogable.SysLogLogable');
@ -444,7 +438,7 @@ class EventShell extends AppShell
public function publish_galaxy_clusters()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || empty($this->args[3])) {
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || !array_key_exists(3, $this->args)) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Publish Galaxy clusters'] . PHP_EOL);
}
@ -528,9 +522,8 @@ class EventShell extends AppShell
$inputFile = $this->args[0];
$inputFile = $inputFile[0] === '/' ? $inputFile : APP . 'tmp/cache/ingest' . DS . $inputFile;
$inputData = FileAccessTool::readFromFile($inputFile);
$inputData = FileAccessTool::readAndDelete($inputFile);
$inputData = $this->Event->jsonDecode($inputData);
FileAccessTool::deleteFile($inputFile);
Configure::write('CurrentUserId', $inputData['user']['id']);
$this->Event->processFreeTextData(
$inputData['user'],
@ -552,9 +545,8 @@ class EventShell extends AppShell
$inputFile = $this->args[0];
$inputFile = $inputFile[0] === '/' ? $inputFile : APP . 'tmp/cache/ingest' . DS . $inputFile;
$inputData = FileAccessTool::readFromFile($inputFile);
$inputData = FileAccessTool::readAndDelete($inputFile);
$inputData = $this->Event->jsonDecode($inputData);
FileAccessTool::deleteFile($inputFile);
Configure::write('CurrentUserId', $inputData['user']['id']);
$this->Event->processModuleResultsData(
$inputData['user'],

4
app/Console/Command/ServerShell.php
View File

@ -112,7 +112,7 @@ class ServerShell extends AppShell
$force = true;
}
try {
$result = $this->Server->pull($user, $serverId, $technique, $server, $jobId, $force);
$result = $this->Server->pull($user, $technique, $server, $jobId, $force);
if (is_array($result)) {
$message = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4]);
$this->Job->saveStatus($jobId, true, $message);
@ -399,7 +399,7 @@ class ServerShell extends AppShell
);
$this->Job->save($data);
$jobId = $this->Job->id;
$result = $this->Server->pull($user, $server['Server']['id'], 'full', $server, $jobId);
$result = $this->Server->pull($user, 'full', $server, $jobId);
$this->Job->save(array(
'id' => $jobId,
'message' => 'Job done.',

39
app/Console/Command/UserShell.php
View File

@ -82,28 +82,31 @@ class UserShell extends AppShell
public function list()
{
// do not fetch sensitive or big values
$schema = $this->User->schema();
unset($schema['authkey']);
unset($schema['password']);
unset($schema['gpgkey']);
unset($schema['certif_public']);
$fields = array_keys($schema);
$fields[] = 'Role.*';
$fields[] = 'Organisation.*';
$users = $this->User->find('all', [
'recursive' => -1,
'fields' => $fields,
'contain' => ['Organisation', 'Role'],
]);
if ($this->params['json']) {
// do not fetch sensitive or big values
$schema = $this->User->schema();
unset($schema['authkey']);
unset($schema['password']);
unset($schema['gpgkey']);
unset($schema['certif_public']);
$fields = array_keys($schema);
$fields[] = 'Role.*';
$fields[] = 'Organisation.*';
$users = $this->User->find('all', [
'recursive' => -1,
'fields' => $fields,
'contain' => ['Organisation', 'Role', 'UserSetting'],
]);
$this->out($this->json($users));
} else {
$users = $this->User->find('column', [
'fields' => ['email'],
]);
foreach ($users as $user) {
$this->out($user['User']['email']);
$this->out($user);
}
}
}

29
app/Controller/AppController.php
View File

@ -132,7 +132,8 @@ class AppController extends Controller
$this->_setupDatabaseConnection();
$this->set('debugMode', Configure::read('debug') >= 1 ? 'debugOn' : 'debugOff');
$this->set('ajax', $this->request->is('ajax'));
$isAjax = $this->request->is('ajax');
$this->set('ajax', $isAjax);
$this->set('queryVersion', $this->__queryVersion);
$this->User = ClassRegistry::init('User');
@ -245,7 +246,7 @@ class AppController extends Controller
$this->__logAccess($user);
// Try to run updates
if ($user['Role']['perm_site_admin'] || (!$this->_isRest() && $this->_isLive())) {
if ($user['Role']['perm_site_admin'] || (!$this->_isRest() && !$isAjax && $this->_isLive())) {
$this->User->runUpdates();
}
@ -265,7 +266,7 @@ class AppController extends Controller
$user = $this->Auth->user(); // user info in session could change (see __verifyUser) method, so reload user variable
if (isset($user['logged_by_authkey']) && $user['logged_by_authkey'] && !($this->_isRest() || $this->_isAutomation())) {
throw new ForbiddenException("When user is authenticated by authkey, just REST request can be processed");
throw new ForbiddenException("When user is authenticated by authkey, just REST request can be processed.");
}
// Put token expiration time to response header that can be processed by automation tool
@ -321,12 +322,15 @@ class AppController extends Controller
$preAuthActions[] = 'email_otp';
}
if (!$this->_isControllerAction(['users' => $preAuthActions, 'servers' => ['cspReport']])) {
if (!$this->request->is('ajax')) {
if ($isAjax) {
$response = $this->RestResponse->throwException(401, "Unauthorized");
$response->send();
$this->_stop();
} else {
$this->Session->write('pre_login_requested_url', $this->request->here);
$this->_redirectToLogin();
}
$this->_redirectToLogin();
}
$this->set('me', false);
}
@ -361,13 +365,9 @@ class AppController extends Controller
}
// Notifications and homepage is not necessary for AJAX or REST requests
if ($user && !$this->_isRest() && !$this->request->is('ajax')) {
if ($this->request->params['controller'] === 'users' && $this->request->params['action'] === 'dashboard') {
$notifications = $this->User->populateNotifications($user);
} else {
$notifications = $this->User->populateNotifications($user, 'fast');
}
$this->set('notifications', $notifications);
if ($user && !$this->_isRest() && !$isAjax) {
$hasNotifications = $this->User->hasNotifications($user);
$this->set('hasNotifications', $hasNotifications);
$homepage = $this->User->UserSetting->getValueForUser($user['id'], 'homepage');
if (!empty($homepage)) {
@ -1224,6 +1224,9 @@ class AppController extends Controller
if ($user === false) {
return $exception;
}
session_write_close(); // Rest search can be longer, so close session to allow concurrent requests
if (isset($filters['returnFormat'])) {
$returnFormat = $filters['returnFormat'];
if ($returnFormat === 'download') {

11
app/Controller/AttributesController.php
View File

@ -2673,16 +2673,11 @@ class AttributesController extends AppController
}
}
} else {
$conditions = array('LOWER(Tag.name)' => strtolower(trim($tag_id)));
if (!$this->_isSiteAdmin()) {
$conditions['Tag.org_id'] = array('0', $this->Auth->user('org_id'));
$conditions['Tag.user_id'] = array('0', $this->Auth->user('id'));
}
$tag = $this->Attribute->AttributeTag->Tag->find('first', array('recursive' => -1, 'conditions' => $conditions));
if (empty($tag)) {
$tagId = $this->Attribute->AttributeTag->Tag->lookupTagIdForUser($this->Auth->user(), trim($tag_id));
if (empty($tagId)) {
return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Invalid Tag.')), 'status'=>200, 'type' => 'json'));
}
$tag_id = $tag['Tag']['id'];
$tag_id = $tagId;
}
}
}

9
app/Controller/Component/RestResponseComponent.php
View File

@ -512,6 +512,7 @@ class RestResponseComponent extends Component
* @param bool $download
* @param array $headers
* @return CakeResponse
* @throws Exception
*/
private function __sendResponse($response, $code, $format = false, $raw = false, $download = false, $headers = array())
{
@ -535,7 +536,7 @@ class RestResponseComponent extends Component
$type = 'xml';
} elseif ($format === 'openioc') {
$type = 'xml';
} elseif ($format === 'csv') {
} elseif ($format === 'csv' || $format === 'text/csv') {
$type = 'csv';
} else {
if (empty($format)) {
@ -582,6 +583,12 @@ class RestResponseComponent extends Component
}
App::uses('TmpFileTool', 'Tools');
if ($response instanceof Generator) {
$tmpFile = new TmpFileTool();
$tmpFile->writeWithSeparator($response, null);
$response = $tmpFile;
}
if ($response instanceof TmpFileTool) {
App::uses('CakeResponseTmp', 'Tools');
$cakeResponse = new CakeResponseTmp(['status' => $code, 'type' => $type]);

606
app/Controller/EventsController.php
View File

@ -28,20 +28,22 @@ class EventsController extends AppController
)
);
private $acceptedFilteringNamedParams = array(
'sort', 'direction', 'focus', 'extended', 'overrideLimit', 'filterColumnsOverwrite', 'attributeFilter', 'extended', 'page',
// private
const ACCEPTED_FILTERING_NAMED_PARAMS = array(
'sort', 'direction', 'focus', 'extended', 'overrideLimit', 'filterColumnsOverwrite', 'attributeFilter', 'page',
'searchFor', 'proposal', 'correlation', 'warning', 'deleted', 'includeRelatedTags', 'includeDecayScore', 'distribution',
'taggedAttributes', 'galaxyAttachedAttributes', 'objectType', 'attributeType', 'focus', 'extended', 'overrideLimit',
'filterColumnsOverwrite', 'feed', 'server', 'toIDS', 'sighting', 'includeSightingdb', 'warninglistId'
'taggedAttributes', 'galaxyAttachedAttributes', 'objectType', 'attributeType', 'feed', 'server', 'toIDS',
'sighting', 'includeSightingdb', 'warninglistId'
);
public $defaultFilteringRules = array(
// private
const DEFAULT_FILTERING_RULE = array(
'searchFor' => '',
'attributeFilter' => 'all',
'proposal' => 0,
'correlation' => 0,
'warning' => 0,
'deleted' => 2,
'deleted' => 0,
'includeRelatedTags' => 0,
'includeDecayScore' => 0,
'toIDS' => 0,
@ -115,7 +117,7 @@ class EventsController extends AppController
$excludeIDs = [];
if (!empty($value)) {
if (!is_array($value)) {
$pieces = explode('|', strtolower($value));
$pieces = explode('|', mb_strtolower($value));
} else {
$pieces = $value;
}
@ -131,8 +133,8 @@ class EventsController extends AppController
if (!empty($include)) {
$includeConditions = [];
foreach ($include as $i) {
$includeConditions['OR'][] = array('lower(Attribute.value1) LIKE' => $i);
$includeConditions['OR'][] = array('lower(Attribute.value2) LIKE' => $i);
$includeConditions['OR'][] = array('Attribute.value1 LIKE' => $i);
$includeConditions['OR'][] = array('Attribute.value2 LIKE' => $i);
}
$includeIDs = $this->Event->Attribute->fetchAttributes($this->Auth->user(), array(
@ -146,8 +148,8 @@ class EventsController extends AppController
if (!empty($exclude)) {
$excludeConditions = [];
foreach ($exclude as $e) {
$excludeConditions['OR'][] = array('lower(Attribute.value1) LIKE' => $e);
$excludeConditions['OR'][] = array('lower(Attribute.value2) LIKE' => $e);
$excludeConditions['OR'][] = array('Attribute.value1 LIKE' => $e);
$excludeConditions['OR'][] = array('Attribute.value2 LIKE' => $e);
}
$excludeIDs = $this->Event->Attribute->fetchAttributes($this->Auth->user(), array(
@ -257,9 +259,10 @@ class EventsController extends AppController
/**
* @param array $passedArgs
* @param string $urlparams
* @param bool $nothing True when nothing should be fetched from database
* @return array
*/
private function __setIndexFilterConditions(array $passedArgs, &$urlparams)
private function __setIndexFilterConditions(array $passedArgs, &$urlparams, &$nothing = false)
{
$passedArgsArray = array();
foreach ($passedArgs as $k => $v) {
@ -275,8 +278,8 @@ class EventsController extends AppController
$searchTerm = strtolower(substr($k, 6));
switch ($searchTerm) {
case 'all':
if (!empty($passedArgs['searchall'])) {
$this->paginate['conditions']['AND'][] = array('Event.id' => $this->__quickFilter($passedArgs['searchall']));
if (!empty($v)) {
$this->paginate['conditions']['AND'][] = array('Event.id' => $this->__quickFilter($v));
}
break;
case 'attribute':
@ -292,6 +295,9 @@ class EventsController extends AppController
if ($v === 2 || $v === '2') { // both
continue 2;
}
if (is_array($v) && in_array(0, $v) && in_array(1, $v)) {
continue 2; // both
}
$this->paginate['conditions']['AND'][] = array('Event.published' => $v);
break;
case 'hasproposal':
@ -312,7 +318,7 @@ class EventsController extends AppController
$eventidConditions = array();
foreach ($pieces as $piece) {
$piece = trim($piece);
if ($piece[0] == '!') {
if ($piece[0] === '!') {
if (strlen($piece) === 37) {
$eventidConditions['NOT']['uuid'][] = substr($piece, 1);
} else {
@ -385,16 +391,15 @@ class EventsController extends AppController
$this->Event->Org->virtualFields = [
'upper_name' => 'UPPER(name)',
'lower_uuid' => 'LOWER(name)',
'lower_uuid' => 'LOWER(uuid)',
];
$orgs = array_column($this->Event->Org->find('all', [
'fields' => ['Org.id', 'Org.upper_name', 'Org.lower_uuid'],
'recursive' => -1,
]), 'Org');
unset($this->Event->Org->virtualFields['upper_name']);
unset($this->Event->Org->virtualFields['lower_uuid']);
$orgByName = array_column($orgs, null, 'upper_name');
$orgByUuid = array_column($orgs, null, 'lower_uuid');
$this->Event->Org->virtualFields = [];
$orgByName = array_column($orgs, 'id', 'upper_name');
$orgByUuid = array_column($orgs, 'id', 'lower_uuid');
// if the first character is '!', search for NOT LIKE the rest of the string (excluding the '!' itself of course)
$pieces = is_array($v) ? $v : explode('|', $v);
$test = array();
@ -404,28 +409,28 @@ class EventsController extends AppController
if (is_numeric($piece)) {
$orgId = $piece;
} else if (Validation::uuid($piece)) {
$orgId = isset($orgByUuid[$piece]) ? $orgByUuid[$piece]['id'] : null;
$orgId = isset($orgByUuid[$piece]) ? $orgByUuid[$piece] : null;
} else {
$orgName = mb_strtoupper($piece);
$orgId = isset($orgByName[$orgName]) ? $orgByName[$orgName]['id'] : null;
$orgId = isset($orgByName[$orgName]) ? $orgByName[$orgName] : null;
}
if ($orgId) {
$this->paginate['conditions']['AND'][] = array('Event.orgc_id !=' => $orgId);
}
} else {
if (is_numeric($piece)) {
$test['OR'][] = array('Event.orgc_id' => array('Event.orgc_id' => $piece));
$test['OR'][] = array('Event.orgc_id' => $piece);
} else {
if (Validation::uuid($piece)) {
$orgId = isset($orgByUuid[$piece]) ? $orgByUuid[$piece]['id'] : null;
$orgId = isset($orgByUuid[$piece]) ? $orgByUuid[$piece] : null;
} else {
$orgName = mb_strtoupper($piece);
$orgId = isset($orgByName[$orgName]) ? $orgByName[$orgName]['id'] : null;
$orgId = isset($orgByName[$orgName]) ? $orgByName[$orgName] : null;
}
if ($orgId) {
$test['OR'][] = array('Event.orgc_id' => $orgId);
} else {
$test['OR'][] = array('Event.orgc_id' => -1);
$nothing = true;
}
}
}
@ -436,7 +441,7 @@ class EventsController extends AppController
$pieces = explode('|', $v);
$test = array();
foreach ($pieces as $piece) {
if ($piece[0] == '!') {
if ($piece[0] === '!') {
$this->paginate['conditions']['AND'][] = array('Event.sharing_group_id !=' => substr($piece, 1));
} else {
$test['OR'][] = array('Event.sharing_group_id' => $piece);
@ -454,10 +459,10 @@ class EventsController extends AppController
$pieces = explode('|', $v);
$test = array();
foreach ($pieces as $piece) {
if ($piece[0] == '!') {
$this->paginate['conditions']['AND'][] = array('lower(Event.info) NOT LIKE' => '%' . strtolower(substr($piece, 1)) . '%');
if ($piece[0] === '!') {
$this->paginate['conditions']['AND'][] = array('lower(Event.info) NOT LIKE' => '%' . mb_strtolower(substr($piece, 1)) . '%');
} else {
$test['OR'][] = array('lower(Event.info) LIKE' => '%' . strtolower($piece) . '%');
$test['OR'][] = array('lower(Event.info) LIKE' => '%' . mb_strtolower($piece) . '%');
}
}
$this->paginate['conditions']['AND'][] = $test;
@ -470,14 +475,13 @@ class EventsController extends AppController
$pieces = is_array($v) ? $v : explode('|', $v);
$filterString = "";
$expectOR = false;
$setOR = false;
$tagRules = [];
foreach ($pieces as $piece) {
if ($piece[0] == '!') {
if ($piece[0] === '!') {
if (is_numeric(substr($piece, 1))) {
$conditions = array('OR' => array('Tag.id' => substr($piece, 1)));
$conditions = array('Tag.id' => substr($piece, 1));
} else {
$conditions = array('OR' => array('Tag.name' => substr($piece, 1)));
$conditions = array('Tag.name' => substr($piece, 1));
}
$tagName = $this->Event->EventTag->Tag->find('first', array(
'conditions' => $conditions,
@ -492,26 +496,18 @@ class EventsController extends AppController
$filterString .= '!' . $piece;
continue;
}
$block = $this->Event->EventTag->find('column', array(
'conditions' => array('EventTag.tag_id' => $tagName['Tag']['id']),
'fields' => ['EventTag.event_id'],
));
if (!empty($block)) {
$sqlSubQuery = 'Event.id NOT IN (' . implode(",", $block) . ')';
$tagRules['AND'][] = $sqlSubQuery;
}
$tagRules['block'][] = $tagName['Tag']['id'];
if ($filterString != "") {
$filterString .= "|";
}
$filterString .= '!' . (isset($tagName['Tag']['name']) ? $tagName['Tag']['name'] : $piece);
$filterString .= '!' . $tagName['Tag']['name'];
} else {
$expectOR = true;
if (is_numeric($piece)) {
$conditions = array('OR' => array('Tag.id' => $piece));
$conditions = array('Tag.id' => $piece);
} else {
$conditions = array('OR' => array('Tag.name' => $piece));
$conditions = array('Tag.name' => $piece);
}
$tagName = $this->Event->EventTag->Tag->find('first', array(
'conditions' => $conditions,
'fields' => array('id', 'name'),
@ -524,61 +520,87 @@ class EventsController extends AppController
$filterString .= $piece;
continue;
}
$allow = $this->Event->EventTag->find('column', array(
'conditions' => array('EventTag.tag_id' => $tagName['Tag']['id']),
'fields' => ['EventTag.event_id'],
));
if (!empty($allow)) {
$setOR = true;
$sqlSubQuery = 'Event.id IN (' . implode(",", $allow) . ')';
$tagRules['OR'][] = $sqlSubQuery;
}
$tagRules['include'][] = $tagName['Tag']['id'];
if ($filterString != "") {
$filterString .= "|";
}
$filterString .= isset($tagName['Tag']['name']) ? $tagName['Tag']['name'] : $piece;
$filterString .= $tagName['Tag']['name'];
}
}
$this->paginate['conditions']['AND'][] = $tagRules;
// If we have a list of OR-d arguments, we expect to end up with a list of allowed event IDs
// If we don't however, it means that none of the tags was found. To prevent displaying the entire event index in this case:
if ($expectOR && !$setOR) {
$this->paginate['conditions']['AND'][] = array('Event.id' => -1);
if (!empty($tagRules['block'])) {
$block = $this->Event->EventTag->find('column', array(
'conditions' => array('EventTag.tag_id' => $tagRules['block']),
'fields' => ['EventTag.event_id'],
));
if (!empty($block)) {
$this->paginate['conditions']['AND'][] = 'Event.id NOT IN (' . implode(",", $block) . ')';
}
}
if (!empty($tagRules['include'])) {
$include = $this->Event->EventTag->find('column', array(
'conditions' => array('EventTag.tag_id' => $tagRules['include']),
'fields' => ['EventTag.event_id'],
));
if (!empty($include)) {
$this->paginate['conditions']['AND'][] = 'Event.id IN (' . implode(",", $include) . ')';
} else {
$nothing = true;
}
} else if ($expectOR) {
// If we have a list of OR-d arguments, we expect to end up with a list of allowed event IDs
// If we don't however, it means that none of the tags was found. To prevent displaying the entire event index in this case:
$nothing = true;
}
$v = $filterString;
break;
case 'email':
if ($v == "" || (strtolower($this->Auth->user('email')) !== strtolower(trim($v)) && !$this->_isSiteAdmin())) {
if ($v == "") {
continue 2;
}
if (!$this->_isSiteAdmin()) {
// Special case to filter own events
if (strtolower($this->Auth->user('email')) === strtolower(trim($v))) {
$this->paginate['conditions']['AND'][] = ['Event.user_id' => $this->Auth->user('id')];
break;
} else {
$nothing = true;
continue 2;
}
}
// if the first character is '!', search for NOT LIKE the rest of the string (excluding the '!' itself of course)
$pieces = explode('|', $v);
$test = array();
$usersToMatch = array();
$positiveQuery = false;
foreach ($pieces as $piece) {
if ($piece[0] === '!') {
$users = $this->Event->User->find('column', array(
'recursive' => -1,
'fields' => array('User.id'),
'conditions' => array('lower(User.email) LIKE' => '%' . strtolower(substr($piece, 1)) . '%')
'conditions' => array('User.email LIKE' => '%' . strtolower(substr($piece, 1)) . '%')
));
if (!empty($users)) {
$this->paginate['conditions']['AND'][] = array('Event.user_id !=' => $users);
}
} else {
$positiveQuery = true;
$users = $this->Event->User->find('column', array(
'recursive' => -1,
'fields' => array('User.id'),
'conditions' => array('lower(User.email) LIKE' => '%' . strtolower($piece) . '%')
'conditions' => array('User.email LIKE' => '%' . strtolower($piece) . '%')
));
if (!empty($users)) {
$test['OR'][] = array('Event.user_id' => $users);
}
$usersToMatch = array_merge($usersToMatch, $users);
}
}
if (!empty($test)) {
$this->paginate['conditions']['AND'][] = $test;
if ($positiveQuery) {
if (empty($usersToMatch)) {
$nothing = true;
} else {
$this->paginate['conditions']['AND'][] = ['Event.user_id' => array_unique($usersToMatch)];
}
}
break;
case 'distribution':
@ -597,7 +619,7 @@ class EventsController extends AppController
} else {
$terms = $this->Event->distributionLevels;
}
$pieces = is_array($v) ? $v : explode('|', $v);
$pieces = is_array($v) ? $v : explode('|', $v);
$test = array();
foreach ($pieces as $piece) {
if ($filterString != "") {
@ -616,7 +638,7 @@ class EventsController extends AppController
break;
case 'minimal':
$tableName = $this->Event->EventReport->table;
$eventReportQuery = sprintf('EXISTS (SELECT id, deleted FROM %s WHERE %s.event_id = Event.id and %s.deleted = 0)', $tableName, $tableName, $tableName);
$eventReportQuery = sprintf('EXISTS (SELECT id FROM %s WHERE %s.event_id = Event.id AND %s.deleted = 0)', $tableName, $tableName, $tableName);
$this->paginate['conditions']['AND'][] = [
'OR' => [
['Event.attribute_count >' => 0],
@ -639,15 +661,15 @@ class EventsController extends AppController
$overrideAbleParams = array('all', 'attribute', 'published', 'eventid', 'datefrom', 'dateuntil', 'org', 'eventinfo', 'tag', 'tags', 'distribution', 'sharinggroup', 'analysis', 'threatlevel', 'email', 'hasproposal', 'timestamp', 'publishtimestamp', 'publish_timestamp', 'minimal');
$paginationParams = array('limit', 'page', 'sort', 'direction', 'order');
$passedArgs = $this->passedArgs;
if (isset($this->request->data)) {
if (!empty($this->request->data)) {
if (isset($this->request->data['request'])) {
$this->request->data = $this->request->data['request'];
}
foreach ($this->request->data as $k => $v) {
if (substr($k, 0, 6) === 'search' && in_array(strtolower(substr($k, 6)), $overrideAbleParams)) {
if (substr($k, 0, 6) === 'search' && in_array(strtolower(substr($k, 6)), $overrideAbleParams, true)) {
unset($this->request->data[$k]);
$this->request->data[strtolower(substr($k, 6))] = $v;
} else if (in_array(strtolower($k), $overrideAbleParams)) {
} else if (in_array(strtolower($k), $overrideAbleParams, true)) {
unset($this->request->data[$k]);
$this->request->data[strtolower($k)] = $v;
}
@ -665,150 +687,38 @@ class EventsController extends AppController
}
// check each of the passed arguments whether they're a filter (could also be a sort for example) and if yes, add it to the pagination conditions
$passedArgsArray = $this->__setIndexFilterConditions($passedArgs, $urlparams);
$nothing = false;
$passedArgsArray = $this->__setIndexFilterConditions($passedArgs, $urlparams, $nothing);
$this->loadModel('GalaxyCluster');
// for REST, don't use the pagination. With this, we'll escape the limit of events shown on the index.
if ($this->_isRest()) {
$rules = array();
$fieldNames = array_keys($this->Event->getColumnTypes());
$directions = array('ASC', 'DESC');
if (isset($passedArgs['sort']) && in_array($passedArgs['sort'], $fieldNames)) {
if (isset($passedArgs['direction']) && in_array(strtoupper($passedArgs['direction']), $directions)) {
$rules['order'] = array('Event.' . $passedArgs['sort'] => $passedArgs['direction']);
} else {
$rules['order'] = array('Event.' . $passedArgs['sort'] => 'ASC');
}
if ($nothing) {
return $this->RestResponse->viewData([], $this->response->type(), false, false, false, ['X-Result-Count' => 0]);
}
$rules['contain'] = $this->paginate['contain'];
if (isset($this->paginate['conditions'])) {
$rules['conditions'] = $this->paginate['conditions'];
}
$minimal = !empty($passedArgs['searchminimal']) || !empty($passedArgs['minimal']);
if ($minimal) {
$rules['recursive'] = -1;
$rules['fields'] = array('id', 'timestamp', 'sighting_timestamp', 'published', 'uuid');
$rules['contain'] = array('Orgc.uuid');
} else {
$rules['contain'][] = 'EventTag';
}
$paginationRules = array('page', 'limit', 'sort', 'direction', 'order');
foreach ($paginationRules as $paginationRule) {
if (isset($passedArgs[$paginationRule])) {
$rules[$paginationRule] = $passedArgs[$paginationRule];
}
}
if (empty($rules['limit'])) {
$events = array();
$i = 1;
$rules['limit'] = 20000;
while (true) {
$rules['page'] = $i;
$temp = $this->Event->find('all', $rules);
$resultCount = count($temp);
if ($resultCount !== 0) {
$events = array_merge($events, $temp);
}
if ($resultCount < $rules['limit']) {
break;
}
$i += 1;
}
$absolute_total = count($events);
} else {
$counting_rules = $rules;
unset($counting_rules['limit']);
unset($counting_rules['page']);
$absolute_total = $this->Event->find('count', $counting_rules);
$events = $absolute_total === 0 ? [] : $this->Event->find('all', $rules);
}
if (!$minimal) {
$tagIds = [];
foreach (array_column($events, 'EventTag') as $eventTags) {
foreach (array_column($eventTags, 'tag_id') as $tagId) {
$tagIds[$tagId] = true;
}
}
if (!empty($tagIds)) {
$tags = $this->Event->EventTag->Tag->find('all', [
'conditions' => [
'Tag.id' => array_keys($tagIds),
'Tag.exportable' => 1,
],
'recursive' => -1,
'fields' => ['Tag.id', 'Tag.name', 'Tag.colour', 'Tag.is_galaxy'],
]);
unset($tagIds);
$tags = array_column(array_column($tags, 'Tag'), null, 'id');
foreach ($events as $k => $event) {
if (empty($event['EventTag'])) {
continue;
}
foreach ($event['EventTag'] as $k2 => $et) {
if (!isset($tags[$et['tag_id']])) {
unset($events[$k]['EventTag'][$k2]); // tag not exists or is not exportable
} else {
$events[$k]['EventTag'][$k2]['Tag'] = $tags[$et['tag_id']];
}
}
$events[$k]['EventTag'] = array_values($events[$k]['EventTag']);
}
$events = $this->GalaxyCluster->attachClustersToEventIndex($this->Auth->user(), $events, false);
}
foreach ($events as $key => $event) {
if (empty($event['SharingGroup']['name'])) {
unset($event['SharingGroup']);
}
$temp = $event['Event'];
$temp['Org'] = $event['Org'];
$temp['Orgc'] = $event['Orgc'];
unset($temp['user_id']);
$rearrangeObjects = array('GalaxyCluster', 'EventTag', 'SharingGroup');
foreach ($rearrangeObjects as $ro) {
if (isset($event[$ro])) {
$temp[$ro] = $event[$ro];
}
}
$events[$key] = $temp;
}
if ($this->response->type() === 'application/xml') {
$events = array('Event' => $events);
}
} else {
foreach ($events as $key => $event) {
$event['Event']['orgc_uuid'] = $event['Orgc']['uuid'];
$events[$key] = $event['Event'];
}
}
return $this->RestResponse->viewData($events, $this->response->type(), false, false, false, ['X-Result-Count' => $absolute_total]);
return $this->__indexRestResponse($passedArgs);
}
$this->paginate['contain']['ThreatLevel'] = [
'fields' => array('ThreatLevel.name')
];
$this->paginate['contain'][] = 'EventTag';
$this->paginate['contain']['EventTag'] = [
'fields' => ['EventTag.event_id', 'EventTag.tag_id', 'EventTag.local'],
];
if ($this->_isSiteAdmin()) {
$this->paginate['contain'][] = 'User.email';
}
if ($nothing) {
$this->paginate['conditions']['AND'][] = ['Event.id' => -1]; // do not fetch any event
}
$events = $this->paginate();
if (count($events) === 1 && isset($this->passedArgs['searchall'])) {
$this->redirect(array('controller' => 'events', 'action' => 'view', $events[0]['Event']['id']));
}
if ($this->params['ext'] === 'csv') {
$events = $this->__attachInfoToEvents(['tags'], $events);
App::uses('CsvExport', 'Export');
$export = new CsvExport();
return $this->RestResponse->viewData($export->eventIndex($events), 'csv');
}
list($possibleColumns, $enabledColumns) = $this->__indexColumns();
$events = $this->__attachInfoToEvents($enabledColumns, $events);
@ -832,6 +742,183 @@ class EventsController extends AppController
}
}
/**
* @param array $passedArgs
* @return CakeResponse
*/
private function __indexRestResponse(array $passedArgs)
{
$fieldNames = $this->Event->schema();
$minimal = !empty($passedArgs['searchminimal']) || !empty($passedArgs['minimal']);
if ($minimal) {
$rules = [
'recursive' => -1,
'fields' => array('id', 'timestamp', 'sighting_timestamp', 'published', 'uuid'),
'contain' => array('Orgc.uuid'),
];
} else {
// Remove user ID from fetched fields
unset($fieldNames['user_id']);
$rules = [
'contain' => ['EventTag'],
'fields' => array_keys($fieldNames),
];
}
if (isset($passedArgs['sort']) && isset($fieldNames[$passedArgs['sort']])) {
if (isset($passedArgs['direction']) && in_array(strtoupper($passedArgs['direction']), ['ASC', 'DESC'])) {
$rules['order'] = array('Event.' . $passedArgs['sort'] => $passedArgs['direction']);
} else {
$rules['order'] = array('Event.' . $passedArgs['sort'] => 'ASC');
}
}
if (isset($this->paginate['conditions'])) {
$rules['conditions'] = $this->paginate['conditions'];
}
$paginationRules = array('page', 'limit', 'sort', 'direction', 'order');
foreach ($paginationRules as $paginationRule) {
if (isset($passedArgs[$paginationRule])) {
$rules[$paginationRule] = $passedArgs[$paginationRule];
}
}
if (empty($rules['limit'])) {
$events = array();
$i = 1;
$rules['limit'] = 20000;
while (true) {
$rules['page'] = $i;
$temp = $this->Event->find('all', $rules);
$resultCount = count($temp);
if ($resultCount !== 0) {
// this is faster and memory efficient than array_merge
foreach ($temp as $tempEvent) {
$events[] = $tempEvent;
}
}
if ($resultCount < $rules['limit']) {
break;
}
$i++;
}
unset($temp);
$absolute_total = count($events);
} else {
$counting_rules = $rules;
unset($counting_rules['limit']);
unset($counting_rules['page']);
$absolute_total = $this->Event->find('count', $counting_rules);
$events = $absolute_total === 0 ? [] : $this->Event->find('all', $rules);
}
$isCsvResponse = $this->response->type() === 'text/csv';
if (!$minimal) {
// Collect all tag IDs that are events
$tagIds = [];
foreach (array_column($events, 'EventTag') as $eventTags) {
foreach (array_column($eventTags, 'tag_id') as $tagId) {
$tagIds[$tagId] = true;
}
}
if (!empty($tagIds)) {
$tags = $this->Event->EventTag->Tag->find('all', [
'conditions' => [
'Tag.id' => array_keys($tagIds),
'Tag.exportable' => 1,
],
'recursive' => -1,
'fields' => ['Tag.id', 'Tag.name', 'Tag.colour', 'Tag.is_galaxy'],
]);
unset($tagIds);
$tags = array_column(array_column($tags, 'Tag'), null, 'id');
foreach ($events as $k => $event) {
if (empty($event['EventTag'])) {
continue;
}
foreach ($event['EventTag'] as $k2 => $et) {
if (!isset($tags[$et['tag_id']])) {
unset($events[$k]['EventTag'][$k2]); // tag not exists or is not exportable
} else {
$events[$k]['EventTag'][$k2]['Tag'] = $tags[$et['tag_id']];
}
}
$events[$k]['EventTag'] = array_values($events[$k]['EventTag']);
}
if (!$isCsvResponse) {
$events = $this->GalaxyCluster->attachClustersToEventIndex($this->Auth->user(), $events, false);
}
}
// Fetch all org and sharing groups that are in events
$orgIds = [];
$sharingGroupIds = [];
foreach ($events as $event) {
$orgIds[$event['Event']['org_id']] = true;
$orgIds[$event['Event']['orgc_id']] = true;
$sharingGroupIds[$event['Event']['sharing_group_id']] = true;
}
if (!empty($orgIds)) {
$orgs = $this->Event->Org->find('all', [
'conditions' => ['Org.id' => array_keys($orgIds)],
'recursive' => -1,
'fields' => $this->paginate['contain']['Org']['fields'],
]);
unset($orgIds);
$orgs = array_column(array_column($orgs, 'Org'), null, 'id');
} else {
$orgs = [];
}
unset($sharingGroupIds[0]);
if (!empty($sharingGroupIds)) {
$sharingGroups = $this->Event->SharingGroup->find('all', [
'conditions' => ['SharingGroup.id' => array_keys($sharingGroupIds)],
'recursive' => -1,
'fields' => $this->paginate['contain']['SharingGroup']['fields'],
]);
unset($sharingGroupIds);
$sharingGroups = array_column(array_column($sharingGroups, 'SharingGroup'), null, 'id');
}
foreach ($events as $key => $event) {
$temp = $event['Event'];
$temp['Org'] = $orgs[$temp['org_id']];
$temp['Orgc'] = $orgs[$temp['orgc_id']];
if ($temp['sharing_group_id'] != 0) {
$temp['SharingGroup'] = $sharingGroups[$temp['sharing_group_id']];
}
$rearrangeObjects = array('GalaxyCluster', 'EventTag');
foreach ($rearrangeObjects as $ro) {
if (isset($event[$ro])) {
$temp[$ro] = $event[$ro];
}
}
$events[$key] = $temp;
}
unset($sharingGroups);
unset($orgs);
if ($this->response->type() === 'application/xml') {
$events = array('Event' => $events);
}
} else {
foreach ($events as $key => $event) {
$event['Event']['orgc_uuid'] = $event['Orgc']['uuid'];
$events[$key] = $event['Event'];
}
}
if ($isCsvResponse) {
App::uses('CsvExport', 'Export');
$export = new CsvExport();
$events = $export->eventIndex($events);
}
return $this->RestResponse->viewData($events, $this->response->type(), false, false, false, ['X-Result-Count' => $absolute_total]);
}
private function __indexColumns()
{
$possibleColumns = [];
@ -1095,7 +1182,7 @@ class EventsController extends AppController
{
$filterData = array(
'request' => $this->request,
'paramArray' => $this->acceptedFilteringNamedParams,
'paramArray' => self::ACCEPTED_FILTERING_NAMED_PARAMS,
'named_params' => $this->request->params['named']
);
$exception = false;
@ -1128,9 +1215,9 @@ class EventsController extends AppController
if ($filters['deleted'] == 1) { // both
$conditions['deleted'] = [0, 1];
} elseif ($filters['deleted'] == 0) { // not-deleted only
$conditions['deleted'] = 1;
} else { // only deleted
$conditions['deleted'] = 0;
} else { // only deleted
$conditions['deleted'] = 1;
}
}
if (isset($filters['toIDS']) && $filters['toIDS'] != 0) {
@ -1170,7 +1257,7 @@ class EventsController extends AppController
}
$event = $results[0];
$attributeTagsName = $this->Event->Attribute->AttributeTag->extractAttributeTagsNameFromEvent($event, 'both');
$attributeTagsName = $this->Event->Attribute->AttributeTag->extractAttributeTagsNameFromEvent($event);
$this->set('attributeTags', array_values($attributeTagsName['tags']));
$this->set('attributeClusters', array_values($attributeTagsName['clusters']));
@ -1242,33 +1329,28 @@ class EventsController extends AppController
}
$this->params->params['paging'] = array($this->modelClass => $params);
$this->set('event', $event);
$deleted = 0;
if (isset($filters['deleted'])) {
$deleted = $filters['deleted'] != 2 ? 1 : 0;
}
$this->set('includeSightingdb', (!empty($filters['includeSightingdb']) && Configure::read('Plugin.Sightings_sighting_db_enable')));
$this->set('deleted', $deleted);
$this->set('deleted', isset($filters['deleted']) && $filters['deleted'] != 0);
$this->set('attributeFilter', isset($filters['attributeFilter']) ? $filters['attributeFilter'] : 'all');
$this->set('filters', $filters);
$advancedFiltering = $this->__checkIfAdvancedFiltering($filters);
$this->set('advancedFilteringActive', $advancedFiltering['active'] ? 1 : 0);
$this->set('advancedFilteringActiveRules', $advancedFiltering['activeRules']);
$this->response->disableCache();
$uriArray = explode('/', $this->params->here);
// Remove `focus` attribute from URI
$uriArray = explode('/', $this->request->here);
foreach ($uriArray as $k => $v) {
if (strpos($v, ':')) {
$temp = explode(':', $v);
if ($temp[0] == 'focus') {
unset($uriArray[$k]);
}
if (strpos($v, 'focus:') === 0) {
unset($uriArray[$k]);
}
$this->params->here = implode('/', $uriArray);
$this->request->here = implode('/', $uriArray);
}
if (!empty($filters['includeSightingdb']) && Configure::read('Plugin.Sightings_sighting_db_enable')) {
$this->set('sightingdbs', $this->Sightingdb->getSightingdbList($this->Auth->user()));
}
$this->set('currentUri', $this->params->here);
$this->set('currentUri', $this->request->here);
$this->layout = false;
$this->__eventViewCommon($this->Auth->user());
$this->render('/Elements/eventattribute');
@ -1285,7 +1367,7 @@ class EventsController extends AppController
$this->loadModel('Taxonomy');
$filterData = array(
'request' => $this->request,
'paramArray' => $this->acceptedFilteringNamedParams,
'paramArray' => self::ACCEPTED_FILTERING_NAMED_PARAMS,
'named_params' => $this->request->params['named']
);
$exception = false;
@ -1376,7 +1458,7 @@ class EventsController extends AppController
}
$event['Attribute'][$k]['tagConflicts'] = $tagConflicts;
}
$attributeTagsName = $this->Event->Attribute->AttributeTag->extractAttributeTagsNameFromEvent($event, 'both');
$attributeTagsName = $this->Event->Attribute->AttributeTag->extractAttributeTagsNameFromEvent($event);
$this->set('attributeTags', array_values($attributeTagsName['tags']));
$this->set('attributeClusters', array_values($attributeTagsName['clusters']));
@ -1505,7 +1587,7 @@ class EventsController extends AppController
private function __eventViewCommon(array $user)
{
$this->set('defaultFilteringRules', $this->defaultFilteringRules);
$this->set('defaultFilteringRules', self::DEFAULT_FILTERING_RULE);
$this->set('typeGroups', array_keys($this->Event->Attribute->typeGroupings));
$orgTable = $this->Event->Orgc->find('list', array(
@ -1580,20 +1662,18 @@ class EventsController extends AppController
if (isset($this->request->data['deleted'])) {
$deleted = $this->request->data['deleted'];
}
if (isset($deleted)) {
// workaround for old instances trying to pull events with both deleted / non deleted data
if (($this->userRole['perm_sync'] && $this->_isRest() && !$this->userRole['perm_site_admin']) && $deleted == 1) {
$conditions['deleted'] = array(0, 1);
} else {
if (is_array($deleted)) {
$conditions['deleted'] = $deleted;
} else if ($deleted == 1) { // both
$conditions['deleted'] = [0, 1];
} elseif ($deleted == 0) { // not-deleted only
$conditions['deleted'] = 0;
} else { // only deleted
$conditions['deleted'] = 1;
}
// workaround for old instances trying to pull events with both deleted / non deleted data
if (($this->userRole['perm_sync'] && $this->_isRest() && !$this->userRole['perm_site_admin']) && $deleted == 1) {
$conditions['deleted'] = array(0, 1);
} else {
if (is_array($deleted)) {
$conditions['deleted'] = $deleted;
} else if ($deleted == 1) { // both
$conditions['deleted'] = [0, 1];
} elseif ($deleted == 0) { // not-deleted only
$conditions['deleted'] = 0;
} else { // only deleted
$conditions['deleted'] = 1;
}
}
if (isset($namedParams['toIDS']) && $namedParams['toIDS'] != 0) {
@ -1703,7 +1783,7 @@ class EventsController extends AppController
return $this->__restResponse($event);
}
$this->set('deleted', isset($deleted) ? ($deleted > 0 ? 1 : 0) : 0);
$this->set('deleted', $deleted > 0);
$this->set('includeRelatedTags', (!empty($namedParams['includeRelatedTags'])) ? 1 : 0);
$this->set('includeDecayScore', (!empty($namedParams['includeDecayScore'])) ? 1 : 0);
@ -1913,8 +1993,8 @@ class EventsController extends AppController
unset($filters['direction']);
$activeRules = array();
foreach ($filters as $k => $v) {
if (isset($this->defaultFilteringRules[$k]) && $this->defaultFilteringRules[$k] != $v) {
$activeRules[$k] = 1;
if (isset(self::DEFAULT_FILTERING_RULE[$k]) && self::DEFAULT_FILTERING_RULE[$k] != $v) {
$activeRules[$k] = $v;
}
}
return array('active' => $activeRules > 0 ? $res : false, 'activeRules' => $activeRules);
@ -2190,24 +2270,20 @@ class EventsController extends AppController
}
}
public function upload_stix($stix_version = '1')
public function upload_stix($stix_version = '1', $publish = false)
{
if ($this->request->is('post')) {
$scriptDir = APP . 'files' . DS . 'scripts';
if ($this->_isRest()) {
$randomFileName = $this->Event->generateRandomFileName();
$tempFile = new File($scriptDir . DS . 'tmp' . DS . $randomFileName, true, 0644);
if (!$tempFile->write($this->request->input())) {
throw new Exception("Could not write content of STIX file.");
if (isset($this->params['named']['publish'])) {
$publish = $this->params['named']['publish'];
}
$tempFile->close();
$filePath = FileAccessTool::writeToTempFile($this->request->input());
$result = $this->Event->upload_stix(
$this->Auth->user(),
$scriptDir,
$randomFileName,
$filePath,
$stix_version,
'uploaded_stix_file.' . ($stix_version == '1' ? 'xml' : 'json'),
false
$publish
);
if (is_numeric($result)) {
$event = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $result));
@ -2222,14 +2298,13 @@ class EventsController extends AppController
} else {
$original_file = !empty($this->data['Event']['original_file']) ? $this->data['Event']['stix']['name'] : '';
if (isset($this->data['Event']['stix']) && $this->data['Event']['stix']['size'] > 0 && is_uploaded_file($this->data['Event']['stix']['tmp_name'])) {
$randomFileName = $this->Event->generateRandomFileName();
if (!move_uploaded_file($this->data['Event']['stix']['tmp_name'], $scriptDir . DS . 'tmp' . DS . $randomFileName)) {
$filePath = FileAccessTool::createTempFile();
if (!move_uploaded_file($this->data['Event']['stix']['tmp_name'], $filePath)) {
throw new Exception("Could not move uploaded STIX file.");
}
$result = $this->Event->upload_stix(
$this->Auth->user(),
$scriptDir,
$randomFileName,
$filePath,
$stix_version,
$original_file,
$this->data['Event']['publish']
@ -2534,8 +2609,7 @@ class EventsController extends AppController
foreach ($idList as $eid) {
$event = $this->Event->find('first', array(
'conditions' => Validation::uuid($eid) ? ['Event.uuid' => $eid] : ['Event.id' => $eid],
'fields' => array('Event.orgc_id', 'Event.id', 'Event.user_id'),
'recursive' => -1
'recursive' => -1,
));
if (empty($event)) {
$fails[] = $eid; // event not found
@ -3127,10 +3201,7 @@ class EventsController extends AppController
$iocData = FileAccessTool::readFromFile($this->data['Event']['submittedioc']['tmp_name'], $this->data['Event']['submittedioc']['size']);
// write
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$attachments_dir = $this->Event->getDefaultAttachments_dir();
}
$attachments_dir = Configure::read('MISP.attachments_dir') ?: (APP . 'files');
$rootDir = $attachments_dir . DS . $id . DS;
App::uses('Folder', 'Utility');
$dir = new Folder($rootDir . 'ioc', true);
@ -3396,16 +3467,11 @@ class EventsController extends AppController
}
}
} else {
$conditions = array('LOWER(Tag.name)' => strtolower(trim($tag_id)));
if (!$this->_isSiteAdmin()) {
$conditions['Tag.org_id'] = array('0', $this->Auth->user('org_id'));
$conditions['Tag.user_id'] = array('0', $this->Auth->user('id'));
}
$tag = $this->Event->EventTag->Tag->find('first', array('recursive' => -1, 'conditions' => $conditions));
if (empty($tag)) {
$tagId = $this->Event->EventTag->Tag->lookupTagIdForUser($this->Auth->user(), trim($tag_id));
if (empty($tagId)) {
return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Invalid Tag.')), 'status'=>200, 'type' => 'json'));
}
$tag_id = $tag['Tag']['id'];
$tag_id = $tagId;
}
}
}
@ -5590,13 +5656,13 @@ class EventsController extends AppController
$objectRef['object_id'] = $ObjectResult;
$objectRef['relationship_type'] = "preceded-by";
$this->loadModel('MispObject');
$result = $this->MispObject->ObjectReference->captureReference($objectRef, $eventId, $this->Auth->user(), false);
$result = $this->MispObject->ObjectReference->captureReference($objectRef, $eventId);
$objectRef['referenced_id'] = $temp['Object']['id'];
$objectRef['referenced_uuid'] = $temp['Object']['uuid'];
$objectRef['object_id'] = $PreviousObjRef['Object']['id'];
$objectRef['relationship_type'] = "followed-by";
$this->loadModel('MispObject');
$result = $this->MispObject->ObjectReference->captureReference($objectRef, $eventId, $this->Auth->user(), false);
$result = $this->MispObject->ObjectReference->captureReference($objectRef, $eventId);
$PreviousObjRef = $temp;
} else {
$PreviousObjRef = $temp;

49
app/Controller/ServersController.php
View File

@ -704,32 +704,36 @@ class ServersController extends AppController
* incremental - only new events
* <int> - specific id of the event to pull
*/
public function pull($id = null, $technique='full')
public function pull($id = null, $technique = 'full')
{
if (!empty($id)) {
$this->Server->id = $id;
} else if (!empty($this->request->data['id'])) {
$this->Server->id = $this->request->data['id'];
} else {
if (empty($id)) {
if (!empty($this->request->data['id'])) {
$id = $this->request->data['id'];
} else {
throw new NotFoundException(__('Invalid server'));
}
}
$s = $this->Server->find('first', [
'conditions' => ['id' => $id],
'recursive' => -1,
]);
if (empty($s)) {
throw new NotFoundException(__('Invalid server'));
}
if (!$this->Server->exists()) {
throw new NotFoundException(__('Invalid server'));
}
$s = $this->Server->read(null, $id);
$error = false;
if (!$this->_isSiteAdmin() && !($s['Server']['org_id'] == $this->Auth->user('org_id') && $this->_isAdmin())) {
throw new MethodNotAllowedException(__('You are not authorised to do that.'));
}
if (false == $this->Server->data['Server']['pull'] && ($technique == 'full' || $technique == 'incremental')) {
if (false == $s['Server']['pull'] && ($technique === 'full' || $technique === 'incremental')) {
$error = __('Pull setting not enabled for this server.');
}
if (false == $this->Server->data['Server']['pull_galaxy_clusters'] && ($technique == 'pull_relevant_clusters')) {
if (false == $s['Server']['pull_galaxy_clusters'] && ($technique === 'pull_relevant_clusters')) {
$error = __('Pull setting not enabled for this server.');
}
if (empty($error)) {
if (!Configure::read('MISP.background_jobs')) {
$result = $this->Server->pull($this->Auth->user(), $id, $technique, $s);
$result = $this->Server->pull($this->Auth->user(), $technique, $s);
if (is_array($result)) {
$success = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4]);
} else {
@ -741,22 +745,11 @@ class ServersController extends AppController
$this->set('pulledSightings', $result[3]);
} else {
$this->loadModel('Job');
$this->Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'pull',
'job_input' => 'Server: ' . $id,
'status' => 0,
'retries' => 0,
'org' => $this->Auth->user('Organisation')['name'],
'message' => __('Pulling.'),
);
$this->Job->save($data);
$jobId = $this->Job->id;
$jobId = $this->Job->createJob($this->Auth->user(), Job::WORKER_DEFAULT, 'pull', 'Server: ' . $id, __('Pulling.'));
$process_id = CakeResque::enqueue(
'default',
'ServerShell',
array('pull', $this->Auth->user('id'), $id, $technique, $jobId)
Job::WORKER_DEFAULT,
'ServerShell',
array('pull', $this->Auth->user('id'), $id, $technique, $jobId)
);
$this->Job->saveField('process_id', $process_id);
$success = __('Pull queued for background execution. Job ID: %s', $jobId);

2
app/Controller/ShadowAttributesController.php
View File

@ -708,7 +708,7 @@ class ShadowAttributesController extends AppController
}
throw new InternalErrorException(__('Could not save the proposal. Errors: %s', $message));
} else {
$this->Flash->error(__('The ShadowAttribute could not be saved. Please, try again.'));
$this->Flash->error(__('The proposed Attribute could not be saved. Please, try again.'));
}
}
} else {

2
app/Lib/Dashboard/MispStatusWidget.php
View File

@ -44,7 +44,7 @@ class MispStatusWidget
'View'
)
);
$notifications = $this->Event->populateNotifications($user);
$notifications = $this->Event->User->populateNotifications($user);
if (!empty($notifications['proposalCount'])) {
$data[] = array(
'title' => __('Pending proposals'),

64
app/Lib/Export/CsvExport.php
View File

@ -64,7 +64,7 @@ class CsvExport
$attribute['decay_score_score'] = 0;
$attribute['decay_score_decayed'] = false;
}
return $this->__addLine($attribute, $options);
return $this->__addLine($attribute);
}
private function __sightingsHandler($sighting, $options)
@ -89,7 +89,7 @@ class CsvExport
$sighting['Sighting'][$new_key] = $attribute_val;
}
}
$lines .= $this->__addLine($sighting['Sighting'], $options);
$lines .= $this->__addLine($sighting['Sighting']);
return $lines;
}
@ -97,20 +97,20 @@ class CsvExport
{
$lines = '';
if (!empty($event['Attribute'])) {
foreach ($event['Attribute'] as $k => $attribute) {
foreach ($event['Attribute'] as $attribute) {
$attribute = $this->__addMetadataToAttribute($event, $attribute);
$lines .= $this->__addLine($attribute, $options);
$lines .= $this->__addLine($attribute);
}
}
if (!empty($event['Object'])) {
foreach ($event['Object'] as $k => $object) {
foreach ($event['Object'] as $object) {
if (!empty($object['Attribute'])) {
foreach ($object['Attribute'] as $attribute) {
$attribute = $this->__addMetadataToAttribute($event, $attribute);
$attribute['object_uuid'] = $object['uuid'];
$attribute['object_name'] = $object['name'];
$attribute['object_meta-category'] = $object['meta-category'];
$lines .= $this->__addLine($attribute, $options);
$lines .= $this->__addLine($attribute);
}
}
}
@ -118,22 +118,27 @@ class CsvExport
return $lines;
}
private function __addLine($attribute, $options = array()) {
$line = '';
/**
* @param array $attribute
* @return string
*/
private function __addLine($attribute)
{
$parts = [];
foreach ($this->requested_fields as $req_att) {
if (empty($line)) {
$line = $this->__escapeCSVField($attribute[$req_att]);
} else {
$line .= ',' . $this->__escapeCSVField($attribute[$req_att]);
}
if (isset($attribute[$req_att])) {
$parts[] = $this->__escapeCSVField($attribute[$req_att]);
} else {
$parts[] = '""'; // keep it consistent with old CSV format
}
}
return $line . PHP_EOL;
return implode(',', $parts) . PHP_EOL;
}
private function __escapeCSVField(&$field)
private function __escapeCSVField($field)
{
if (is_bool($field)) {
return ($field ? '1' : '0');
return $field ? '1' : '0';
}
if (is_numeric($field)) {
return $field;
@ -257,15 +262,18 @@ class CsvExport
return '';
}
public function eventIndex($events)
/**
* @param array $events
* @return Generator[string]
*/
public function eventIndex(array $events)
{
$fields = array(
'id', 'date', 'info', 'tags', 'uuid', 'published', 'analysis', 'attribute_count', 'orgc_id', 'orgc_name', 'orgc_uuid', 'timestamp', 'distribution', 'sharing_group_id', 'threat_level_id',
'publish_timestamp', 'extends_uuid'
);
$result = implode(',', $fields) . PHP_EOL;
foreach ($events as $key => $event) {
$event['tags'] = '';
yield implode(',', $fields) . PHP_EOL;
foreach ($events as $event) {
if (!empty($event['EventTag'])) {
$tags = array();
foreach ($event['EventTag'] as $et) {
@ -275,16 +283,18 @@ class CsvExport
} else {
$tags = '';
}
$event['Event']['tags'] = $tags;
$event['Event']['orgc_name'] = $event['Orgc']['name'];
$event['Event']['orgc_uuid'] = $event['Orgc']['uuid'];
$event['tags'] = $tags;
$event['orgc_name'] = $event['Orgc']['name'];
$event['orgc_uuid'] = $event['Orgc']['uuid'];
$current = array();
foreach ($fields as $field) {
$current[] = $this->__escapeCSVField($event['Event'][$field]);
if (isset($event[$field])) {
$current[] = $this->__escapeCSVField($event[$field]);
} else {
$current[] = '';
}
}
$result .= implode(', ', $current) . PHP_EOL;
yield implode(',', $current) . PHP_EOL;
}
return $result;
}
}

1
app/Lib/Export/OpendataExport.php
View File

@ -3,7 +3,6 @@
class OpendataExport
{
public $non_restrictive_export = true;
public $use_default_filters = true;
public $mock_query_only = true;
private $__default_filters = null;

21
app/Lib/Export/Stix1Export.php
View File

@ -1,5 +1,4 @@
<?php
App::uses('StixExport', 'Export');
class Stix1Export extends StixExport
@ -7,22 +6,20 @@ class Stix1Export extends StixExport
protected $__attributes_limit = 15000;
protected $__default_version = '1.1.1';
protected $__sane_versions = array('1.1.1', '1.2');
private $__script_name = 'misp2stix.py ';
private $__baseurl = null;
private $__org = null;
protected function __initiate_framing_params()
{
$this->__baseurl = escapeshellarg(Configure::read('MISP.baseurl'));
$this->__org = escapeshellarg(Configure::read('MISP.org'));
$my_server = ClassRegistry::init('Server');
return $my_server->getPythonVersion() . ' ' . $this->__framing_script . ' stix1 -v ' . $this->__version . ' -n ' . $this->__baseurl . ' -o ' . $this->__org . ' -f ' . $this->__return_format . ' ' . $this->__end_of_cmd;
$baseurl = escapeshellarg(Configure::read('MISP.baseurl'));
$org = escapeshellarg(Configure::read('MISP.org'));
return $this->pythonBin() . ' ' . $this->__framing_script . ' stix1 -v ' . $this->__version . ' -n ' . $baseurl . ' -o ' . $org . ' -f ' . $this->__return_format . ' ' . $this->__end_of_cmd;
}
protected function __parse_misp_events($filenames)
protected function __parse_misp_events(array $filenames)
{
$scriptFile = $this->__scripts_dir . $this->__script_name;
$my_server = ClassRegistry::init('Server');
return shell_exec($my_server->getPythonVersion() . ' ' . $scriptFile . '-v ' . $this->__version . ' -f ' . $this->__return_format . ' -o ' . $this->__org . ' -i ' . $this->__tmp_dir . $filenames . $this->__end_of_cmd);
$org = escapeshellarg(Configure::read('MISP.org'));
$filenames = implode(' ', $filenames);
$scriptFile = $this->__scripts_dir . 'misp2stix.py';
$command = $this->pythonBin() . ' ' . $scriptFile . ' -v ' . $this->__version . ' -f ' . $this->__return_format . ' -o ' . $org . ' -i ' . $filenames . $this->__end_of_cmd;
return shell_exec($command);
}
}

14
app/Lib/Export/Stix2Export.php
View File

@ -1,5 +1,4 @@
<?php
App::uses('StixExport', 'Export');
class Stix2Export extends StixExport
@ -7,20 +6,17 @@ class Stix2Export extends StixExport
protected $__attributes_limit = 15000;
protected $__default_version = '2.0';
protected $__sane_versions = array('2.0', '2.1');
private $__script_name = 'stix2/misp2stix2.py ';
protected function __initiate_framing_params()
{
$my_server = ClassRegistry::init('Server');
return $my_server->getPythonVersion() . ' ' . $this->__framing_script . ' stix2 -v ' . $this->__version . ' --uuid ' . escapeshellarg(CakeText::uuid()) . $this->__end_of_cmd;
return $this->pythonBin() . ' ' . $this->__framing_script . ' stix2 -v ' . $this->__version . ' --uuid ' . escapeshellarg(CakeText::uuid()) . $this->__end_of_cmd;
}
protected function __parse_misp_events($filenames)
protected function __parse_misp_events(array $filenames)
{
$scriptFile = $this->__scripts_dir . $this->__script_name;
$filenames = implode(' ' . $this->__tmp_dir, $this->__filenames);
$my_server = ClassRegistry::init('Server');
$result = shell_exec($my_server->getPythonVersion() . ' ' . $scriptFile . '-v ' . $this->__version . ' -i ' . $this->__tmp_dir . $filenames . $this->__end_of_cmd);
$scriptFile = $this->__scripts_dir . 'stix2/misp2stix2.py';
$filenames = implode(' ', $filenames);
$result = shell_exec($this->pythonBin() . ' ' . $scriptFile . ' -v ' . $this->__version . ' -i ' . $filenames . $this->__end_of_cmd);
$result = preg_split("/\r\n|\n|\r/", trim($result));
return end($result);
}

157
app/Lib/Export/StixExport.php
View File

@ -1,6 +1,9 @@
<?php
App::uses('JSONConverterTool', 'Tools');
App::uses('TmpFileTool', 'Tools');
App::uses('JsonTool', 'Tools');
class StixExport
abstract class StixExport
{
public $additional_params = array(
'includeEventTags' => 1,
@ -8,60 +11,55 @@ class StixExport
);
protected $__return_format = 'json';
protected $__scripts_dir = APP . 'files/scripts/';
protected $__tmp_dir = APP . 'files/scripts/tmp/';
protected $__framing_script = APP . 'files/scripts/misp_framing.py';
protected $__end_of_cmd = ' 2>' . APP . 'tmp/logs/exec-errors.log';
protected $__return_type = null;
/** @var array Full paths to files to convert */
protected $__filenames = array();
protected $__default_filters = null;
protected $__version = null;
private $__current_filename = null;
private $__empty_file = null;
private $__framing = null;
private $__stix_file = null;
/** @var File */
private $__tmp_file = null;
private $__n_attributes = 0;
public $non_restrictive_export = true;
public $use_default_filters = true;
private $Server;
public function setDefaultFilters($filters)
{
$sane_version = (!empty($filters['stix-version']) && in_array($filters['stix-version'], $this->__sane_versions));
$sane_version = !empty($filters['stix-version']) && in_array($filters['stix-version'], $this->__sane_versions, true);
$this->__version = $sane_version ? $filters['stix-version'] : $this->__default_version;
}
public function handler($data, $options = array())
{
$attributes_count = count($data['Attribute']);
foreach ($data['Object'] as $_object) {
if (isset($_object['Attribute'])) {
$attributes_count += count($_object['Attribute']);
$attributesCount = count($data['Attribute']);
foreach ($data['Object'] as $object) {
if (isset($object['Attribute'])) {
$attributesCount += count($object['Attribute']);
}
}
App::uses('JSONConverterTool', 'Tools');
$converter = new JSONConverterTool();
$event = $converter->convert($data);
if ($this->__n_attributes + $attributes_count < $this->__attributes_limit) {
$this->__tmp_file->append($this->__n_attributes == 0 ? $event : ',' . $event);
$this->__n_attributes += $attributes_count;
$event = JsonTool::encode($converter->convert($data, false, true)); // we don't need pretty printed JSON
if ($this->__n_attributes + $attributesCount < $this->__attributes_limit) {
$this->__tmp_file->append($this->__n_attributes === 0 ? $event : ',' . $event);
$this->__n_attributes += $attributesCount;
$this->__empty_file = false;
} elseif ($attributesCount > $this->__attributes_limit) {
$filePath = FileAccessTool::writeToTempFile($event);
$this->__filenames[] = $filePath;
} else {
if ($attributes_count > $this->__attributes_limit) {
$randomFileName = $this->__generateRandomFileName();
$tmpFile = new File($this->__tmp_dir . $randomFileName, true, 0644);
$tmpFile->write($event);
$tmpFile->close();
array_push($this->__filenames, $randomFileName);
} else {
$this->__tmp_file->append(']}');
$this->__tmp_file->close();
array_push($this->__filenames, $this->__current_filename);
$this->__initialize_misp_file();
$this->__tmp_file->append($event);
$this->__n_attributes = $attributes_count;
}
$this->__tmp_file->append(']}');
$this->__tmp_file->close();
$this->__filenames[] = $this->__current_filename;
$this->__initialize_misp_file();
$this->__tmp_file->append($event);
$this->__n_attributes = $attributesCount;
}
return '';
}
@ -69,21 +67,19 @@ class StixExport
public function header($options = array())
{
$this->__return_type = $options['returnFormat'];
if ($this->__return_type == 'stix-json') {
if ($this->__return_type === 'stix-json') {
$this->__return_type = 'stix';
} else if ($this->__return_type == 'stix') {
} else if ($this->__return_type === 'stix') {
$this->__return_format = 'xml';
}
$framing_cmd = $this->__initiate_framing_params();
$randomFileName = $this->__generateRandomFileName();
$this->__framing = json_decode(shell_exec($framing_cmd), true);
$this->__stix_file = new File($this->__tmp_dir . $randomFileName . '.' . $this->__return_type);
unset($randomFileName);
$this->__stix_file->write($this->__framing['header']);
$this->__initialize_misp_file();
return '';
}
/**
* @return TmpFileTool
* @throws Exception
*/
public function footer()
{
if ($this->__empty_file) {
@ -92,31 +88,27 @@ class StixExport
} else {
$this->__tmp_file->append(']}');
$this->__tmp_file->close();
array_push($this->__filenames, $this->__current_filename);
$this->__filenames[] = $this->__current_filename;
}
$filenames = implode(' ' . $this->__tmp_dir, $this->__filenames);
$result = $this->__parse_misp_events($filenames);
$result = $this->__parse_misp_events($this->__filenames);
$this->__delete_temporary_files();
$decoded = json_decode($result, true);
if (!isset($decoded['success']) || !$decoded['success']) {
$this->__delete_temporary_files();
$error = $decoded && !empty($decoded['error']) ? $decoded['error'] : $result;
return 'Error while processing your query: ' . $error;
throw new Exception('Error while processing your query during STIX export: ' . $error);
}
foreach ($this->__filenames as $f => $filename) {
$file = new File($this->__tmp_dir . $filename . '.out');
$stix_event = ($this->__return_type == 'stix') ? $file->read() : substr($file->read(), 1, -1);
$file->close();
$file->delete();
@unlink($this->__tmp_dir . $filename);
$this->__stix_file->append($stix_event . $this->__framing['separator']);
unset($stix_event);
$framing = $this->getFraming();
$stixFile = new TmpFileTool();
$stixFile->write($framing['header']);
foreach ($this->__filenames as $filename) {
$stixEvent = FileAccessTool::readAndDelete($filename . '.out');
$stixEvent = $this->__return_type === 'stix' ? $stixEvent : substr($stixEvent, 1, -1);
$stixFile->writeWithSeparator($stixEvent, $framing['separator']);
}
$stix_event = $this->__stix_file->read();
$this->__stix_file->close();
$this->__stix_file->delete();
$sep_len = strlen($this->__framing['separator']);
$stix_event = (empty($this->__filenames) ? $stix_event : substr($stix_event, 0, -$sep_len)) . $this->__framing['footer'];
return $stix_event;
$stixFile->write($framing['footer']);
return $stixFile;
}
public function separator()
@ -126,23 +118,52 @@ class StixExport
private function __initialize_misp_file()
{
$this->__current_filename = $this->__generateRandomFileName();
$this->__tmp_file = new File($this->__tmp_dir . $this->__current_filename, true, 0644);
$this->__current_filename = FileAccessTool::createTempFile();
$this->__tmp_file = new File($this->__current_filename);
$this->__tmp_file->write('{"response": [');
$this->__empty_file = true;
}
private function __generateRandomFileName()
{
return (new RandomTool())->random_str(false, 12);
}
private function __delete_temporary_files()
{
foreach ($this->__filenames as $f => $filename) {
@unlink($this->__tmp_dir . $filename);
foreach ($this->__filenames as $filename) {
FileAccessTool::deleteFileIfExists($filename);
}
$this->__stix_file->close();
$this->__stix_file->delete();
}
/**
* @return array
* @throws Exception
*/
private function getFraming()
{
$framingCmd = $this->__initiate_framing_params();
$framing = json_decode(shell_exec($framingCmd), true);
if ($framing === null || isset($framing['error'])) {
throw new Exception("Could not get results from framing cmd when exporting STIX file.");
}
return $framing;
}
/**
* @return string
*/
protected function pythonBin()
{
if (!isset($this->Server)) {
$this->Server = ClassRegistry::init('Server');
}
return $this->Server->getPythonVersion();
}
/**
* @param array $filenames Paths to files to process
* @return string|false|null
*/
abstract protected function __parse_misp_events(array $filenames);
/**
* @return string
*/
abstract protected function __initiate_framing_params();
}

27
app/Lib/Tools/FileAccessTool.php
View File

@ -2,6 +2,21 @@
class FileAccessTool
{
/**
* @param string $path
* @param int $permissions
* @throws Exception
*/
public static function createFile($path, $permissions = 0600)
{
if (!file_exists($path)) {
if (!touch($path)) {
throw new Exception("Could not create file `$path`.");
}
}
@chmod($path, $permissions); // hide error if current user is not file owner
}
/**
* Creates temporary file, but you have to delete it after use.
* @param string|null $dir
@ -47,6 +62,18 @@ class FileAccessTool
return $content;
}
/**
* @param string $file
* @return string
* @throws Exception
*/
public static function readAndDelete($file)
{
$content = self::readFromFile($file);
self::deleteFile($file);
return $content;
}
/**
* @param string $file
* @param mixed $content

9
app/Lib/Tools/JSONConverterTool.php
View File

@ -48,7 +48,7 @@ class JSONConverterTool
}
}
if (isset($event['Event']['SharingGroup']) && empty($event['Event']['SharingGroup'])) {
if (empty($event['Event']['SharingGroup'])) {
unset($event['Event']['SharingGroup']);
}
@ -86,11 +86,6 @@ class JSONConverterTool
}
unset($tempSightings);
unset($event['Event']['RelatedAttribute']);
if (isset($event['Event']['RelatedEvent'])) {
foreach ($event['Event']['RelatedEvent'] as $key => $value) {
unset($event['Event']['RelatedEvent'][$key]['Event']['user_id']);
}
}
$result = array('Event' => $event['Event']);
if (isset($event['errors'])) {
$result = array_merge($result, array('errors' => $event['errors']));
@ -143,7 +138,7 @@ class JSONConverterTool
{
// remove value1 and value2 from the output and remove invalid utf8 characters for the xml parser
foreach ($attributes as $key => $attribute) {
if (isset($attribute['SharingGroup']) && empty($attribute['SharingGroup'])) {
if (empty($attribute['SharingGroup'])) {
unset($attributes[$key]['SharingGroup']);
}
unset($attributes[$key]['value1']);

20
app/Lib/Tools/JsonTool.php Normal file
View File

@ -0,0 +1,20 @@
<?php
class JsonTool
{
/**
* @param mixed $value
* @param bool $prettyPrint
* @returns string
*/
public static function encode($value, $prettyPrint = false)
{
$flags = JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE;
if (defined('JSON_THROW_ON_ERROR')) {
$flags |= JSON_THROW_ON_ERROR; // Throw exception on error if supported
}
if ($prettyPrint) {
$flags |= JSON_PRETTY_PRINT;
}
return json_encode($value, $flags);
}
}

61
app/Lib/Tools/PubSubTool.php
View File

@ -1,4 +1,7 @@
<?php
App::uses('FileAccessTool', 'Tools');
App::uses('JsonTool', 'Tools');
class PubSubTool
{
const SCRIPTS_TMP = APP . 'files' . DS . 'scripts' . DS . 'tmp' . DS;
@ -9,18 +12,12 @@ class PubSubTool
*/
private $redis;
/**
* @var array
*/
private $settings;
public function initTool()
{
if (!$this->redis) {
$settings = $this->getSetSettings();
$this->setupPubServer($settings);
$this->redis = $this->createRedisConnection($settings);
$this->settings = $settings;
}
}
@ -57,8 +54,8 @@ class PubSubTool
{
$settings = $this->getSetSettings();
$redis = $this->createRedisConnection($settings);
$redis->rPush($settings['redis_namespace'] . ':command', 'status');
$response = $redis->blPop($settings['redis_namespace'] . ':status', 5);
$redis->rPush( 'command', 'status');
$response = $redis->blPop('status', 5);
if ($response === null) {
throw new Exception("No response from status command returned after 5 seconds.");
}
@ -80,7 +77,7 @@ class PubSubTool
App::uses('JSONConverterTool', 'Tools');
$jsonTool = new JSONConverterTool();
$json = $jsonTool->convert($event);
return $this->pushToRedis(':data:misp_json', $json);
return $this->pushToRedis('data:misp_json', $json);
}
public function event_save(array $event, $action)
@ -88,7 +85,7 @@ class PubSubTool
if (!empty($action)) {
$event['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_event', $event);
return $this->pushToRedis('data:misp_json_event', $event);
}
public function object_save(array $object, $action)
@ -96,7 +93,7 @@ class PubSubTool
if (!empty($action)) {
$object['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_object', $object);
return $this->pushToRedis('data:misp_json_object', $object);
}
public function object_reference_save(array $object_reference, $action)
@ -104,12 +101,12 @@ class PubSubTool
if (!empty($action)) {
$object_reference['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_object_reference', $object_reference);
return $this->pushToRedis('data:misp_json_object_reference', $object_reference);
}
public function publishConversation(array $message)
{
return $this->pushToRedis(':data:misp_json_conversation', $message);
return $this->pushToRedis('data:misp_json_conversation', $message);
}
public function attribute_save(array $attribute, $action = false)
@ -117,7 +114,7 @@ class PubSubTool
if (!empty($action)) {
$attribute['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_attribute', $attribute);
return $this->pushToRedis('data:misp_json_attribute', $attribute);
}
public function tag_save(array $tag, $action = false)
@ -125,7 +122,7 @@ class PubSubTool
if (!empty($action)) {
$tag['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_tag', $tag);
return $this->pushToRedis('data:misp_json_tag', $tag);
}
public function sighting_save(array $sighting, $action = false)
@ -133,7 +130,7 @@ class PubSubTool
if (!empty($action)) {
$sighting['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_sighting', $sighting);
return $this->pushToRedis('data:misp_json_sighting', $sighting);
}
public function warninglist_save(array $warninglist, $action = false)
@ -141,7 +138,7 @@ class PubSubTool
if (!empty($action)) {
$warninglist['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_warninglist', $warninglist);
return $this->pushToRedis('data:misp_json_warninglist', $warninglist);
}
/**
@ -155,7 +152,7 @@ class PubSubTool
if (!empty($action)) {
$data['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_' . $type, $data);
return $this->pushToRedis('data:misp_json_' . $type, $data);
}
public function publish($data, $type, $action = false)
@ -163,7 +160,7 @@ class PubSubTool
if (!empty($action)) {
$data['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_' . $type, $data);
return $this->pushToRedis('data:misp_json_' . $type, $data);
}
public function killService()
@ -171,7 +168,7 @@ class PubSubTool
if ($this->checkIfRunning()) {
$settings = $this->getSetSettings();
$redis = $this->createRedisConnection($settings);
$redis->rPush($settings['redis_namespace'] . ':command', 'kill');
$redis->rPush('command', 'kill');
sleep(1);
if ($this->checkIfRunning()) {
// Still running.
@ -194,7 +191,7 @@ class PubSubTool
if ($this->checkIfRunning()) {
$redis = $this->createRedisConnection($settings);
$redis->rPush($settings['redis_namespace'] . ':command', 'reload');
$redis->rPush( 'command', 'reload');
} else {
return 'Setting saved, but something is wrong with the ZeroMQ server. Please check the diagnostics page for more information.';
}
@ -226,7 +223,7 @@ class PubSubTool
if ($this->checkIfRunning(self::OLD_PID_LOCATION)) {
// Old version is running, kill it and start again new one.
$redis = $this->createRedisConnection($settings);
$redis->rPush($settings['redis_namespace'] . ':command', 'kill');
$redis->rPush('command', 'kill');
sleep(1);
}
@ -244,10 +241,10 @@ class PubSubTool
private function pushToRedis($ns, $data)
{
if (is_array($data)) {
$data = json_encode($data, JSON_UNESCAPED_UNICODE);
$data = JsonTool::encode($data);
}
$this->redis->rPush($this->settings['redis_namespace'] . $ns, $data);
$this->redis->rPush($ns, $data);
return true;
}
@ -264,6 +261,7 @@ class PubSubTool
$redis->auth($redisPassword);
}
$redis->select($settings['redis_database']);
$redis->setOption(Redis::OPT_PREFIX, $settings['redis_namespace'] . ':');
return $redis;
}
@ -274,18 +272,12 @@ class PubSubTool
private function saveSettingToFile(array $settings)
{
$settingFilePath = self::SCRIPTS_TMP . 'mispzmq_settings.json';
$settingsFile = new File($settingFilePath, true, 0644);
if (!$settingsFile->exists()) {
throw new Exception("Could not create zmq config file '$settingFilePath'.");
}
// Because setting file contains secrets, it should be readable just by owner. But because in Travis test,
// config file is created under one user and then changed under other user, file must be readable and writable
// also by group.
@chmod($settingsFile->pwd(), 0660); // hide error if current user is not file owner
if (!$settingsFile->write(json_encode($settings))) {
throw new Exception("Could not write zmq config file '$settingFilePath'.");
}
$settingsFile->close();
FileAccessTool::createFile($settingFilePath, 0660);
FileAccessTool::writeToFile($settingFilePath, JsonTool::encode($settings));
}
private function getSetSettings()
@ -302,8 +294,9 @@ class PubSubTool
'password' => null,
);
$pluginConfig = Configure::read('Plugin');
foreach ($settings as $key => $setting) {
$temp = Configure::read('Plugin.ZeroMQ_' . $key);
$temp = isset($pluginConfig['ZeroMQ_' . $key]) ? $pluginConfig['ZeroMQ_' . $key] : null;
if ($temp) {
$settings[$key] = $temp;
}

23
app/Lib/Tools/ServerSyncTool.php
View File

@ -50,7 +50,9 @@ class ServerSyncTool
public function eventExists(array $event)
{
$url = $this->server['Server']['url'] . '/events/view/' . $event['Event']['uuid'];
$start = microtime(true);
$exists = $this->socket->head($url, [], $this->request);
$this->log($start, 'HEAD', $url, $exists);
if ($exists->code == '404') {
return false;
}
@ -235,7 +237,9 @@ class ServerSyncTool
private function get($url)
{
$url = $this->server['Server']['url'] . $url;
$start = microtime(true);
$response = $this->socket->get($url, [], $this->request);
$this->log($start, 'GET', $url, $response);
if (!$response->isOk()) {
throw new HttpSocketHttpException($response, $url);
}
@ -261,7 +265,7 @@ class ServerSyncTool
$logMessage,
$data
);
file_put_contents(APP . 'files/scripts/tmp/debug_server_' . $this->server['Server']['id'] . '.log', $pushLogEntry, FILE_APPEND);
file_put_contents(APP . 'files/scripts/tmp/debug_server_' . $this->server['Server']['id'] . '.log', $pushLogEntry, FILE_APPEND | LOCK_EX);
}
$request = $this->request;
@ -275,7 +279,9 @@ class ServerSyncTool
}
}
$url = $this->server['Server']['url'] . $url;
$start = microtime(true);
$response = $this->socket->post($url, $data, $request);
$this->log($start, 'POST', $url, $response);
if (!$response->isOk()) {
throw new HttpSocketHttpException($response, $url);
}
@ -300,4 +306,19 @@ class ServerSyncTool
}
return $url;
}
/**
* @param float $start
* @param string $method HTTP method
* @param string $url
* @param HttpSocketResponse $response
*/
private function log($start, $method, $url, HttpSocketResponse $response)
{
$duration = round(microtime(true) - $start, 3);
$responseSize = strlen($response->body);
$ce = $response->getHeader('Content-Encoding');
$logEntry = '[' . date("Y-m-d H:i:s") . "] \"$method $url\" {$response->code} $responseSize $duration $ce\n";
file_put_contents(APP . 'tmp/logs/server-sync.log', $logEntry, FILE_APPEND | LOCK_EX);
}
}

27
app/Model/AdminSetting.php
View File

@ -18,16 +18,27 @@ class AdminSetting extends AppModel
public function changeSetting($setting, $value = false)
{
$setting_object = $this->find('first', array(
'conditions' => array('setting' => $setting)
$existing = $this->find('first', array(
'conditions' => array('setting' => $setting),
'fields' => ['id'],
));
$this->deleteAll(array('setting' => $setting));
$this->create();
$setting_object['AdminSetting'] = array('setting' => $setting, 'value' => $value);
if ($this->save($setting_object)) {
return true;
if ($existing) {
if ($this->save([
'id' => $existing['AdminSetting']['id'],
'value' => $value,
])) {
return true;
} else {
return $this->validationErrors;
}
} else {
return $this->validationErrors;
$this->create();
$existing['AdminSetting'] = array('setting' => $setting, 'value' => $value);
if ($this->save($existing)) {
return true;
} else {
return $this->validationErrors;
}
}
}

161
app/Model/AppModel.php
View File

@ -83,7 +83,8 @@ class AppModel extends Model
51 => false, 52 => false, 53 => false, 54 => false, 55 => false, 56 => false,
57 => false, 58 => false, 59 => false, 60 => false, 61 => false, 62 => false,
63 => true, 64 => false, 65 => false, 66 => false, 67 => false, 68 => false,
69 => false, 70 => false, 71 => true, 72 => true, 73 => true,
69 => false, 70 => false, 71 => true, 72 => true, 73 => false, 74 => false,
75 => false, 76 => false
);
public $advanced_updates_description = array(
@ -1579,6 +1580,20 @@ class AppModel extends Model
$sqlArray[] = "ALTER TABLE `auth_keys` ADD `read_only` tinyint(1) NOT NULL DEFAULT 0 AFTER `expiration`;";
break;
case 73:
$this->__dropIndex('user_settings', 'timestamp'); // index is not used
$sqlArray[] = "ALTER TABLE `user_settings` ADD UNIQUE INDEX `unique_setting` (`user_id`, `setting`)";
break;
case 74:
$sqlArray[] = "ALTER TABLE `users` MODIFY COLUMN `change_pw` tinyint(1) NOT NULL DEFAULT 0;";
break;
case 75:
$this->__addIndex('object_references', 'event_id');
$this->__dropIndex('object_references', 'timestamp');
$this->__dropIndex('object_references', 'source_uuid');
$this->__dropIndex('object_references', 'relationship_type');
$this->__dropIndex('object_references', 'referenced_uuid');
break;
case 76:
$sqlArray[] = "ALTER TABLE `tags` ADD `local_only` tinyint(1) NOT NULL DEFAULT 0 AFTER `is_custom_galaxy`;";
$sqlArray[] = "ALTER TABLE `galaxies` ADD `local_only` tinyint(1) NOT NULL DEFAULT 0 AFTER `enabled`;";
break;
@ -1918,11 +1933,7 @@ class AppModel extends Model
public function getPythonVersion()
{
if (!empty(Configure::read('MISP.python_bin'))) {
return Configure::read('MISP.python_bin');
} else {
return 'python3';
}
return Configure::read('MISP.python_bin') ?: 'python3';
}
public function validateAuthkey($value)
@ -1939,10 +1950,9 @@ class AppModel extends Model
// alternative to the build in notempty/notblank validation functions, compatible with cakephp <= 2.6 and cakephp and cakephp >= 2.7
public function valueNotEmpty($value)
{
$field = array_keys($value);
$field = $field[0];
$value[$field] = trim($value[$field]);
if (!empty($value[$field])) {
$field = array_keys($value)[0];
$value = trim($value[$field]);
if (!empty($value)) {
return true;
}
return ucfirst($field) . ' cannot be empty.';
@ -1950,32 +1960,17 @@ class AppModel extends Model
public function valueIsJson($value)
{
$field = array_keys($value);
$field = $field[0];
$json_decoded = json_decode($value[$field]);
$value = array_values($value)[0];
$json_decoded = json_decode($value);
if ($json_decoded === null) {
return __('Invalid JSON.');
}
return true;
}
public function valueIsJsonOrNull($value)
{
$field = array_keys($value);
$field = $field[0];
if (!is_null($value[$field])) {
$json_decoded = json_decode($value[$field]);
if ($json_decoded === null) {
return __('Invalid JSON.');
}
}
return true;
}
public function valueIsID($value)
{
$field = array_keys($value);
$field = $field[0];
$field = array_keys($value)[0];
if (!is_numeric($value[$field]) || $value[$field] < 0) {
return 'Invalid ' . ucfirst($field) . ' ID';
}
@ -1984,10 +1979,9 @@ class AppModel extends Model
public function stringNotEmpty($value)
{
$field = array_keys($value);
$field = $field[0];
$value[$field] = trim($value[$field]);
if (!isset($value[$field]) || ($value[$field] == false && $value[$field] !== "0")) {
$field = array_keys($value)[0];
$value = trim($value[$field]);
if (!isset($value) || ($value == false && $value !== "0")) {
return ucfirst($field) . ' cannot be empty.';
}
return true;
@ -2020,8 +2014,7 @@ class AppModel extends Model
{
$this->AdminSetting = ClassRegistry::init('AdminSetting');
$this->Job = ClassRegistry::init('Job');
$this->Log = ClassRegistry::init('Log');
$this->Server = ClassRegistry::init('Server');
$db = ConnectionManager::getDataSource('default');
$tables = $db->listSources();
$requiresLogout = false;
@ -2031,7 +2024,10 @@ class AppModel extends Model
$requiresLogout = true;
} else {
$this->__runCleanDB();
$db_version = $this->AdminSetting->find('all', array('conditions' => array('setting' => 'db_version')));
$db_version = $this->AdminSetting->find('all', [
'conditions' => array('setting' => 'db_version'),
'fields' => ['id', 'value'],
]);
if (count($db_version) > 1) {
// we rgan into a bug where we have more than one db_version entry. This bug happened in some rare circumstances around 2.4.50-2.4.57
foreach ($db_version as $k => $v) {
@ -2050,6 +2046,8 @@ class AppModel extends Model
$job = null;
}
if (!empty($updates)) {
$this->Log = ClassRegistry::init('Log');
$this->Server = ClassRegistry::init('Server');
// Exit if updates are locked.
// This is not as reliable as a real lock implementation
// However, as all updates are re-playable, there is no harm if they
@ -2371,16 +2369,10 @@ class AppModel extends Model
private function __runCleanDB()
{
$cleanDB = $this->AdminSetting->find('first', array('conditions' => array('setting' => 'clean_db')));
if (empty($cleanDB) || $cleanDB['AdminSetting']['value'] == 1) {
$cleanDB = $this->AdminSetting->getSetting('clean_db');
if ($cleanDB === false || $cleanDB == 1) {
$this->cleanCacheFiles();
if (empty($cleanDB)) {
$this->AdminSetting->create();
$cleanDB = array('AdminSetting' => array('setting' => 'clean_db', 'value' => 0));
} else {
$cleanDB['AdminSetting']['value'] = 0;
}
$this->AdminSetting->save($cleanDB);
$this->AdminSetting->changeSetting('clean_db', 0);
}
}
@ -2447,60 +2439,6 @@ class AppModel extends Model
return true;
}
public function populateNotifications($user, $mode = 'full')
{
$notifications = array();
list($notifications['proposalCount'], $notifications['proposalEventCount']) = $this->_getProposalCount($user, $mode);
$notifications['total'] = $notifications['proposalCount'];
if (Configure::read('MISP.delegation')) {
$notifications['delegationCount'] = $this->_getDelegationCount($user);
$notifications['total'] += $notifications['delegationCount'];
}
return $notifications;
}
// if not using $mode === 'full', simply check if an entry exists. We really don't care about the real count for the top menu.
private function _getProposalCount($user, $mode = 'full')
{
$this->ShadowAttribute = ClassRegistry::init('ShadowAttribute');
$results[0] = $this->ShadowAttribute->find(
'count',
array(
'recursive' => -1,
'conditions' => array(
'ShadowAttribute.event_org_id' => $user['org_id'],
'ShadowAttribute.deleted' => 0,
)
)
);
if ($mode === 'full') {
$results[1] = $this->ShadowAttribute->find(
'count',
array(
'recursive' => -1,
'conditions' => array(
'ShadowAttribute.event_org_id' => $user['org_id'],
'ShadowAttribute.deleted' => 0,
),
'fields' => 'distinct event_id'
)
);
} else {
$results[1] = $results[0];
}
return $results;
}
private function _getDelegationCount($user)
{
$this->EventDelegation = ClassRegistry::init('EventDelegation');
$delegations = $this->EventDelegation->find('count', array(
'recursive' => -1,
'conditions' => array('EventDelegation.org_id' => $user['org_id'])
));
return $delegations;
}
public function checkFilename($filename)
{
return preg_match('@^([a-z0-9_.]+[a-z0-9_.\- ]*[a-z0-9_.\-]|[a-z0-9_.])+$@i', $filename);
@ -2933,11 +2871,6 @@ class AppModel extends Model
return $val / (1024 * 1024);
}
public function getDefaultAttachments_dir()
{
return APP . 'files';
}
private function __bumpReferences()
{
$this->Event = ClassRegistry::init('Event');
@ -3021,10 +2954,10 @@ class AppModel extends Model
}
$multiplierArray = array('d' => 86400, 'h' => 3600, 'm' => 60, 's' => 1);
$lastChar = strtolower(substr($delta, -1));
if (!is_numeric($lastChar) && array_key_exists($lastChar, $multiplierArray)) {
if (!is_numeric($lastChar) && isset($multiplierArray[$lastChar])) {
$multiplier = $multiplierArray[$lastChar];
$delta = substr($delta, 0, -1);
} else if(strtotime($delta) !== false) {
} else if (strtotime($delta) !== false) {
return strtotime($delta);
} else {
// invalid filter, make sure we don't return anything
@ -3171,23 +3104,6 @@ class AppModel extends Model
return $this->log($message, $type);
}
/**
* Generates random file name in tmp dir.
* @return string
*/
protected function tempFileName()
{
return $this->tempDir() . DS . $this->generateRandomFileName();
}
/**
* @return string
*/
protected function tempDir()
{
return Configure::read('MISP.tmpdir') ?: sys_get_temp_dir();
}
/**
* Decodes JSON string and throws exception if string is not valid JSON or if is not array.
*
@ -3243,6 +3159,7 @@ class AppModel extends Model
'conditions' => $conditions,
'recursive' => -1,
'callbacks' => false,
'order' => [], // disable order
));
}

276
app/Model/Attribute.php
View File

@ -362,6 +362,9 @@ class Attribute extends AppModel
public function beforeSave($options = array())
{
if (empty($this->data['Attribute']['uuid'])) {
$this->data['Attribute']['uuid'] = CakeText::uuid();
}
if (!empty($this->data['Attribute']['id'])) {
$this->old = $this->find('first', array(
'recursive' => -1,
@ -393,12 +396,28 @@ class Attribute extends AppModel
return true;
}
/**
* @param int $event_id
* @param bool $increment True for increment, false for decrement,
* @return bool
*/
private function __alterAttributeCount($event_id, $increment = true)
{
return $this->Event->updateAll(
array('Event.attribute_count' => $increment ? 'Event.attribute_count+1' : 'GREATEST(Event.attribute_count, 1) - 1'),
array('Event.id' => $event_id)
);
// Temporary unbind models that we don't need to prevent deadlocks
$this->Event->unbindModel([
'belongsTo' => array_keys($this->Event->belongsTo),
]);
try {
return $this->Event->updateAll(
array('Event.attribute_count' => $increment ? 'Event.attribute_count+1' : 'GREATEST(Event.attribute_count, 1) - 1'),
array('Event.id' => $event_id)
);
} catch (Exception $e) {
$this->logException('Exception when updating event attribute count', $e);
return false;
} finally {
$this->Event->resetAssociations();
}
}
public function afterSave($created, $options = array())
@ -547,81 +566,78 @@ class Attribute extends AppModel
public function beforeValidate($options = array())
{
if (empty($this->data['Attribute']['type'])) {
$attribute = &$this->data['Attribute'];
if (empty($attribute['type'])) {
$this->validationErrors['type'] = ['No type set.'];
return false;
}
$type = $this->data['Attribute']['type'];
if (is_array($this->data['Attribute']['value'])) {
$type = $attribute['type'];
if (is_array($attribute['value'])) {
$this->validationErrors['value'] = ['Value is an array.'];
return false;
}
if (!empty($this->data['Attribute']['object_id']) && empty($this->data['Attribute']['object_relation'])) {
if (!empty($attribute['object_id']) && empty($attribute['object_relation'])) {
$this->validationErrors['object_relation'] = ['Object attribute sent, but no object_relation set.'];
return false;
}
// If `value1` or `value2` provided and `value` is empty, merge them into `value` because of validation
if (empty($this->data['Attribute']['value'])) {
if (!empty($this->data['Attribute']['value1']) && !empty($this->data['Attribute']['value2'])) {
$this->data['Attribute']['value'] = "{$this->data['Attribute']['value1']}|{$this->data['Attribute']['value2']}";
} else if (!empty($this->data['Attribute']['value1'])) {
$this->data['Attribute']['value'] = $this->data['Attribute']['value1'];
if (empty($attribute['value'])) {
if (!empty($attribute['value1']) && !empty($attribute['value2'])) {
$attribute['value'] = "{$attribute['value1']}|{$attribute['value2']}";
} else if (!empty($attribute['value1'])) {
$attribute['value'] = $attribute['value1'];
}
}
// remove leading and trailing blanks and refang value and
$this->data['Attribute']['value'] = ComplexTypeTool::refangValue(trim($this->data['Attribute']['value']), $type);
$attribute['value'] = ComplexTypeTool::refangValue(trim($attribute['value']), $type);
// make some changes to the inserted value
$this->data['Attribute']['value'] = $this->modifyBeforeValidation($type, $this->data['Attribute']['value']);
$attribute['value'] = $this->modifyBeforeValidation($type, $attribute['value']);
// Run user defined regexp to attribute value
$result = $this->runRegexp($type, $this->data['Attribute']['value']);
$result = $this->runRegexp($type, $attribute['value']);
if ($result === false) {
$this->invalidate('value', 'This value is blocked by a regular expression in the import filters.');
} else {
$this->data['Attribute']['value'] = $result;
$attribute['value'] = $result;
}
if (empty($this->data['Attribute']['comment'])) {
$this->data['Attribute']['comment'] = "";
if (empty($attribute['comment'])) {
$attribute['comment'] = "";
}
// generate UUID if it doesn't exist
if (empty($this->data['Attribute']['uuid'])) {
$this->data['Attribute']['uuid'] = CakeText::uuid();
} else {
$this->data['Attribute']['uuid'] = strtolower($this->data['Attribute']['uuid']);
if (!empty($attribute['uuid'])) {
$attribute['uuid'] = strtolower($attribute['uuid']);
}
// generate timestamp if it doesn't exist
if (empty($this->data['Attribute']['timestamp'])) {
$this->data['Attribute']['timestamp'] = time();
if (empty($attribute['timestamp'])) {
$attribute['timestamp'] = time();
}
// parse first_seen different formats
if (isset($this->data['Attribute']['first_seen'])) {
$this->data['Attribute']['first_seen'] = $this->data['Attribute']['first_seen'] === '' ? null : $this->data['Attribute']['first_seen'];
if (isset($attribute['first_seen'])) {
$attribute['first_seen'] = $attribute['first_seen'] === '' ? null : $attribute['first_seen'];
}
// parse last_seen different formats
if (isset($this->data['Attribute']['last_seen'])) {
$this->data['Attribute']['last_seen'] = $this->data['Attribute']['last_seen'] === '' ? null : $this->data['Attribute']['last_seen'];
if (isset($attribute['last_seen'])) {
$attribute['last_seen'] = $attribute['last_seen'] === '' ? null : $attribute['last_seen'];
}
// Set defaults for when some of the mandatory fields don't have defaults
// These fields all have sane defaults either based on another field, or due to server settings
if (!isset($this->data['Attribute']['distribution'])) {
$this->data['Attribute']['distribution'] = $this->defaultDistribution();
if (!isset($attribute['distribution'])) {
$attribute['distribution'] = $this->defaultDistribution();
}
if ($attribute['distribution'] != 4) {
$attribute['sharing_group_id'] = 0;
}
// If category is not provided, assign default category by type
if (empty($this->data['Attribute']['category'])) {
$this->data['Attribute']['category'] = $this->typeDefinitions[$type]['default_category'];
if (empty($attribute['category'])) {
$attribute['category'] = $this->typeDefinitions[$type]['default_category'];
}
if (!isset($this->data['Attribute']['to_ids'])) {
$this->data['Attribute']['to_ids'] = $this->typeDefinitions[$type]['to_ids'];
}
if ($this->data['Attribute']['distribution'] != 4) {
$this->data['Attribute']['sharing_group_id'] = 0;
if (!isset($attribute['to_ids'])) {
$attribute['to_ids'] = $this->typeDefinitions[$type]['to_ids'];
}
// return true, otherwise the object cannot be saved
return true;
@ -726,21 +742,17 @@ class Attribute extends AppModel
// check whether the variable is null or datetime
public function datetimeOrNull($fields)
{
$k = array_keys($fields)[0];
$seen = $fields[$k];
try {
new DateTime($seen);
$returnValue = true;
} catch (Exception $e) {
$returnValue = false;
$seen = array_values($fields)[0];
if ($seen === null) {
return true;
}
return $returnValue || is_null($seen);
return strtotime($seen) !== false;
}
public function validateLastSeenValue($fields)
{
$ls = $fields['last_seen'];
if (!isset($this->data['Attribute']['first_seen']) || is_null($ls)) {
if (!isset($this->data['Attribute']['first_seen']) || $ls === null) {
return true;
}
$converted = $this->ISODatetimeToUTC(['Attribute' => [
@ -782,7 +794,6 @@ class Attribute extends AppModel
public function runValidation($value, $type)
{
$returnValue = false;
// check data validation
switch ($type) {
case 'md5':
@ -810,24 +821,19 @@ class Attribute extends AppModel
case 'git-commit-id':
if ($this->isHashValid($type, $value)) {
return true;
} else {
$length = self::HEX_HAS_LENGTHS[$type];
return __('Checksum has an invalid length or format (expected: %s hexadecimal characters). Please double check the value or select type "other".', $length);
}
$length = self::HEX_HAS_LENGTHS[$type];
return __('Checksum has an invalid length or format (expected: %s hexadecimal characters). Please double check the value or select type "other".', $length);
case 'tlsh':
if (preg_match("#^t?[0-9a-f]{35,}$#i", $value)) {
$returnValue = true;
} else {
$returnValue = __('Checksum has an invalid length or format (expected: at least 35 hexadecimal characters, optionally starting with t1 instead of hexadecimal characters). Please double check the value or select type "other".');
return true;
}
break;
return __('Checksum has an invalid length or format (expected: at least 35 hexadecimal characters, optionally starting with t1 instead of hexadecimal characters). Please double check the value or select type "other".');
case 'pehash':
if ($this->isHashValid('pehash', $value)) {
$returnValue = true;
} else {
$returnValue = __('The input doesn\'t match the expected sha1 format (expected: 40 hexadecimal characters). Keep in mind that MISP currently only supports SHA1 for PEhashes, if you would like to get the support extended to other hash types, make sure to create a github ticket about it at https://github.com/MISP/MISP!');
return true;
}
break;
return __('The input doesn\'t match the expected sha1 format (expected: 40 hexadecimal characters). Keep in mind that MISP currently only supports SHA1 for PEhashes, if you would like to get the support extended to other hash types, make sure to create a github ticket about it at https://github.com/MISP/MISP!');
case 'ssdeep':
if (substr_count($value, ':') === 2) {
$parts = explode(':', $value);
@ -840,35 +846,26 @@ class Attribute extends AppModel
if (substr_count($value, ':') === 2) {
$parts = explode(':', $value);
if ($this->isPositiveInteger($parts[0])) {
$returnValue = true;
return true;
}
}
if (!$returnValue) {
$returnValue = __('Invalid impfuzzy format. The format has to be imports:hash:hash');
}
break;
return __('Invalid impfuzzy format. The format has to be imports:hash:hash');
case 'cdhash':
if (preg_match("#^[0-9a-f]{40,}$#", $value)) {
$returnValue = true;
} else {
$returnValue = __('The input doesn\'t match the expected format (expected: 40 or more hexadecimal characters)');
return true;
}
break;
return __('The input doesn\'t match the expected format (expected: 40 or more hexadecimal characters)');
case 'http-method':
if (preg_match("#(OPTIONS|GET|HEAD|POST|PUT|DELETE|TRACE|CONNECT|PROPFIND|PROPPATCH|MKCOL|COPY|MOVE|LOCK|UNLOCK|VERSION-CONTROL|REPORT|CHECKOUT|CHECKIN|UNCHECKOUT|MKWORKSPACE|UPDATE|LABEL|MERGE|BASELINE-CONTROL|MKACTIVITY|ORDERPATCH|ACL|PATCH|SEARCH)#", $value)) {
$returnValue = true;
} else {
$returnValue = __('Unknown HTTP method.');
return true;
}
break;
return __('Unknown HTTP method.');
case 'filename|pehash':
// no newline
if (preg_match("#^.+\|[0-9a-f]{40}$#", $value)) {
$returnValue = true;
} else {
$returnValue = __('The input doesn\'t match the expected filename|sha1 format (expected: filename|40 hexadecimal characters). Keep in mind that MISP currently only supports SHA1 for PEhashes, if you would like to get the support extended to other hash types, make sure to create a github ticket about it at https://github.com/MISP/MISP!');
return true;
}
break;
return __('The input doesn\'t match the expected filename|sha1 format (expected: filename|40 hexadecimal characters). Keep in mind that MISP currently only supports SHA1 for PEhashes, if you would like to get the support extended to other hash types, make sure to create a github ticket about it at https://github.com/MISP/MISP!');
case 'filename|md5':
case 'filename|sha1':
case 'filename|imphash':
@ -886,42 +883,33 @@ class Attribute extends AppModel
$parts = explode('|', $type);
$length = self::HEX_HAS_LENGTHS[$parts[1]];
if (preg_match("#^.+\|[0-9a-f]{" . $length . "}$#", $value)) {
$returnValue = true;
} else {
$returnValue = __('Checksum has an invalid length or format (expected: filename|%s hexadecimal characters). Please double check the value or select type "other".', $length);
return true;
}
break;
return __('Checksum has an invalid length or format (expected: filename|%s hexadecimal characters). Please double check the value or select type "other".', $length);
case 'filename|ssdeep':
if (substr_count($value, '|') != 1 || !preg_match("#^.+\|.+$#", $value)) {
$returnValue = __('Invalid composite type. The format has to be %s.', $type);
return __('Invalid composite type. The format has to be %s.', $type);
} else {
$composite = explode('|', $value);
$value = $composite[1];
if (substr_count($value, ':') == 2) {
$parts = explode(':', $value);
if ($this->isPositiveInteger($parts[0])) {
$returnValue = true;
return true;
}
}
if (!$returnValue) {
$returnValue = __('Invalid SSDeep hash (expected: blocksize:hash:hash).');
}
}
break;
return __('Invalid SSDeep hash (expected: blocksize:hash:hash).');
case 'filename|tlsh':
if (preg_match("#^.+\|[0-9a-f]{35,}$#", $value)) {
$returnValue = true;
} else {
$returnValue = __('Checksum has an invalid length or format (expected: filename|at least 35 hexadecimal characters). Please double check the value or select type "other".');
return true;
}
break;
return __('Checksum has an invalid length or format (expected: filename|at least 35 hexadecimal characters). Please double check the value or select type "other".');
case 'filename|vhash':
if (preg_match('#^.+\|.+$#', $value)) {
$returnValue = true;
} else {
$returnValue = __('Checksum has an invalid length or format (expected: filename|string characters). Please double check the value or select type "other".');
return true;
}
break;
return __('Checksum has an invalid length or format (expected: filename|string characters). Please double check the value or select type "other".');
case 'ip-src':
case 'ip-dst':
if (strpos($value, '/') !== false) {
@ -945,14 +933,11 @@ class Attribute extends AppModel
return __('IP address has an invalid format.');
}
return true;
case 'port':
if (!$this->isPortValid($value)) {
$returnValue = __('Port numbers have to be integers between 1 and 65535.');
} else {
$returnValue = true;
return __('Port numbers have to be integers between 1 and 65535.');
}
break;
return true;
case 'ip-dst|port':
case 'ip-src|port':
$parts = explode('|', $value);
@ -965,22 +950,20 @@ class Attribute extends AppModel
return true;
case 'mac-address':
if (preg_match('/^([a-fA-F0-9]{2}[:]?){6}$/', $value)) {
$returnValue = true;
return true;
}
break;
case 'mac-eui-64':
if (preg_match('/^([a-fA-F0-9]{2}[:]?){8}$/', $value)) {
$returnValue = true;
return true;
}
break;
case 'hostname':
case 'domain':
if ($this->isDomainValid($value)) {
$returnValue = true;
} else {
$returnValue = __('%s has an invalid format. Please double check the value or select type "other".', ucfirst($type));
return true;
}
break;
return __('%s has an invalid format. Please double check the value or select type "other".', ucfirst($type));
case 'hostname|port':
$parts = explode('|', $value);
if (!$this->isDomainValid($parts[0])) {
@ -994,14 +977,12 @@ class Attribute extends AppModel
if (preg_match("#^[A-Z0-9.\-_]+\.[A-Z0-9\-]{2,}\|.*$#i", $value)) {
$parts = explode('|', $value);
if (filter_var($parts[1], FILTER_VALIDATE_IP)) {
$returnValue = true;
return true;
} else {
$returnValue = __('IP address has an invalid format.');
return __('IP address has an invalid format.');
}
} else {
$returnValue = __('Domain name has an invalid format.');
}
break;
return __('Domain name has an invalid format.');
case 'email':
case 'email-src':
case 'eppn':
@ -1012,38 +993,30 @@ class Attribute extends AppModel
case 'jabber-id':
// we don't use the native function to prevent issues with partial email addresses
if (preg_match("#^.*\@.*\..*$#i", $value)) {
$returnValue = true;
} else {
$returnValue = __('Email address has an invalid format. Please double check the value or select type "other".');
return true;
}
break;
return __('Email address has an invalid format. Please double check the value or select type "other".');
case 'vulnerability':
if (preg_match("#^(CVE-)[0-9]{4}(-)[0-9]{4,}$#", $value)) {
$returnValue = true;
} else {
$returnValue = __('Invalid format. Expected: CVE-xxxx-xxxx...');
return true;
}
break;
return __('Invalid format. Expected: CVE-xxxx-xxxx...');
case 'weakness':
if (preg_match("#^(CWE-)[0-9]{1,}$#", $value)) {
$returnValue = true;
} else {
$returnValue = __('Invalid format. Expected: CWE-x...');
return true;
}
break;
return __('Invalid format. Expected: CWE-x...');
case 'named pipe':
if (!preg_match("#\n#", $value)) {
$returnValue = true;
return true;
}
break;
case 'windows-service-name':
case 'windows-service-displayname':
if (strlen($value) > 256 || preg_match('#[\\\/]#', $value)) {
$returnValue = __('Invalid format. Only values shorter than 256 characters that don\'t include any forward or backward slashes are allowed.');
} else {
$returnValue = true;
return __('Invalid format. Only values shorter than 256 characters that don\'t include any forward or backward slashes are allowed.');
}
break;
return true;
case 'mutex':
case 'process-state':
case 'snort':
@ -1079,12 +1052,11 @@ class Attribute extends AppModel
case 'middle-name':
case 'last-name':
case 'full-name':
$returnValue = true;
break;
return true;
case 'link':
// Moved to a native function whilst still enforcing the scheme as a requirement
if (filter_var($value, FILTER_VALIDATE_URL, FILTER_FLAG_SCHEME_REQUIRED) && !preg_match("#\n#", $value)) {
$returnValue = true;
return true;
}
break;
case 'hex':
@ -1153,38 +1125,33 @@ class Attribute extends AppModel
}
return true;
case 'datetime':
try {
new DateTime($value);
$returnValue = true;
} catch (Exception $e) {
$returnValue = __('Datetime has to be in the ISO 8601 format.');
if (strtotime($value) !== false) {
return true;
}
break;
return __('Datetime has to be in the ISO 8601 format.');
case 'size-in-bytes':
case 'counter':
if ($this->isPositiveInteger($value)) {
return true;
}
return __('The value has to be a whole number greater or equal 0.');
case 'targeted-threat-index':
/* case 'targeted-threat-index':
if (!is_numeric($value) || $value < 0 || $value > 10) {
$returnValue = __('The value has to be a number between 0 and 10.');
} else {
$returnValue = true;
return __('The value has to be a number between 0 and 10.');
}
break;
return true;*/
case 'iban':
case 'bic':
case 'btc':
case 'dash':
case 'xmr':
if (preg_match('/^[a-zA-Z0-9]+$/', $value)) {
$returnValue = true;
return true;
}
break;
case 'vhash':
if (preg_match('/^.+$/', $value)) {
$returnValue = true;
return true;
}
break;
case 'bin':
@ -1195,18 +1162,17 @@ class Attribute extends AppModel
case 'phone-number':
case 'whois-registrant-phone':
if (is_numeric($value)) {
$returnValue = true;
return true;
}
break;
case 'cortex':
json_decode($value);
$returnValue = (json_last_error() == JSON_ERROR_NONE);
break;
return json_last_error() === JSON_ERROR_NONE;
case 'float':
return is_numeric($value);
case 'boolean':
if ($value == 1 || $value == 0) {
$returnValue = true;
return true;
}
break;
case 'AS':
@ -1215,7 +1181,7 @@ class Attribute extends AppModel
}
return __('AS number have to be integers between 1 and 4294967295');
}
return $returnValue;
return false;
}
// do some last second modifications before the validation
@ -1395,7 +1361,7 @@ class Attribute extends AppModel
break;
case 'datetime':
try {
$value = (new DateTime($value))->setTimezone(new DateTimeZone('GMT'))->format('Y-m-d\TH:i:s.uO'); // ISO8601 formating with microseconds
$value = (new DateTime($value, new DateTimeZone('GMT')))->format('Y-m-d\TH:i:s.uO'); // ISO8601 formating with microseconds
} catch (Exception $e) {
// silently skip. Rejection will be done in runValidation()
}
@ -1683,8 +1649,7 @@ class Attribute extends AppModel
{
// convert into utc and micro sec
if (!empty($data[$alias]['first_seen'])) {
$d = new DateTime($data[$alias]['first_seen']);
$d->setTimezone(new DateTimeZone('GMT'));
$d = new DateTime($data[$alias]['first_seen'], new DateTimeZone('GMT'));
$fs_sec = $d->format('U');
$fs_micro = $d->format('u');
$fs_micro = str_pad($fs_micro, 6, "0", STR_PAD_LEFT);
@ -1692,8 +1657,7 @@ class Attribute extends AppModel
$data[$alias]['first_seen'] = $fs;
}
if (!empty($data[$alias]['last_seen'])) {
$d = new DateTime($data[$alias]['last_seen']);
$d->setTimezone(new DateTimeZone('GMT'));
$d = new DateTime($data[$alias]['last_seen'], new DateTimeZone('GMT'));
$ls_sec = $d->format('U');
$ls_micro = $d->format('u');
$ls_micro = str_pad($ls_micro, 6, "0", STR_PAD_LEFT);
@ -3916,7 +3880,7 @@ class Attribute extends AppModel
}
App::uses($this->validFormats[$returnFormat][1], 'Export');
$exportTool = new $this->validFormats[$returnFormat][1]();
if (!empty($exportTool->use_default_filters)) {
if (method_exists($exportTool, 'setDefaultFilters')) {
$exportTool->setDefaultFilters($filters);
}
if (empty($exportTool->non_restrictive_export)) {

48
app/Model/AttributeTag.php
View File

@ -342,42 +342,40 @@ class AttributeTag extends AppModel
return $allClusters;
}
public function extractAttributeTagsNameFromEvent(&$event, $to_extract='both')
/**
* @param array $event
* @return array|array[]
*/
public function extractAttributeTagsNameFromEvent(array $event)
{
$attribute_tags_name = array('tags' => array(), 'clusters' => array());
foreach ($event['Attribute'] as $i => $attribute) {
if ($to_extract == 'tags' || $to_extract == 'both') {
foreach ($attribute['AttributeTag'] as $tag) {
$attribute_tags_name['tags'][] = $tag['Tag']['name'];
}
$extractedTags = [];
$extractedClusters = [];
foreach ($event['Attribute'] as $attribute) {
foreach ($attribute['AttributeTag'] as $tag) {
$extractedTags[$tag['Tag']['id']] = $tag['Tag']['name'];
}
if ($to_extract == 'clusters' || $to_extract == 'both') {
foreach ($attribute['Galaxy'] as $galaxy) {
foreach ($galaxy['GalaxyCluster'] as $cluster) {
$attribute_tags_name['clusters'][] = $cluster['tag_name'];
}
foreach ($attribute['Galaxy'] as $galaxy) {
foreach ($galaxy['GalaxyCluster'] as $cluster) {
$extractedClusters[$cluster['tag_id']] = $cluster['tag_name'];
}
}
}
foreach ($event['Object'] as $i => $object) {
foreach ($event['Object'] as $object) {
if (!empty($object['Attribute'])) {
foreach ($object['Attribute'] as $j => $object_attribute) {
if ($to_extract == 'tags' || $to_extract == 'both') {
foreach ($object_attribute['AttributeTag'] as $tag) {
$attribute_tags_name['tags'][] = $tag['Tag']['name'];
}
foreach ($object['Attribute'] as $object_attribute) {
foreach ($object_attribute['AttributeTag'] as $tag) {
$extractedTags[$tag['Tag']['id']] = $tag['Tag']['name'];
}
if ($to_extract == 'clusters' || $to_extract == 'both') {
foreach ($object_attribute['Galaxy'] as $galaxy) {
foreach ($galaxy['GalaxyCluster'] as $cluster) {
$attribute_tags_name['clusters'][] = $cluster['tag_name'];
}
foreach ($object_attribute['Galaxy'] as $galaxy) {
foreach ($galaxy['GalaxyCluster'] as $cluster) {
$extractedClusters[$cluster['tag_id']] = $cluster['tag_name'];
}
}
}
}
}
$attribute_tags_name['tags'] = array_diff_key($attribute_tags_name['tags'], $attribute_tags_name['clusters']); // de-dup if needed.
return $attribute_tags_name;
$extractedTags = array_diff_key($extractedTags, $extractedClusters); // de-dup if needed.
return ['tags' => $extractedTags, 'clusters' => $extractedClusters];
}
}

15
app/Model/Behavior/AuditLogBehavior.php
View File

@ -81,11 +81,26 @@ class AuditLogBehavior extends ModelBehavior
if (!$this->enabled) {
return true;
}
// Do not fetch old version when just few fields will be fetched
$fieldToFetch = [];
if (!empty($options['fieldList'])) {
foreach ($options['fieldList'] as $field) {
if (!isset($this->skipFields[$field])) {
$fieldToFetch[] = $field;
}
}
if (empty($fieldToFetch)) {
$this->old = null;
return true;
}
}
if ($model->id) {
$this->old = $model->find('first', [
'conditions' => [$model->alias . '.' . $model->primaryKey => $model->id],
'recursive' => -1,
'callbacks' => false,
'fields' => $fieldToFetch,
]);
} else {
$this->old = null;

747
app/Model/Event.php
View File

File diff suppressed because it is too large Load Diff

12
app/Model/EventGraph.php
View File

@ -30,7 +30,7 @@ class EventGraph extends AppModel
public $validate = array(
'network_json' => array(
'rule' => array('isValidJson'),
'rule' => 'valueIsJson',
'message' => 'The provided eventGraph is not a valid json format',
'required' => true,
),
@ -44,16 +44,6 @@ class EventGraph extends AppModel
return true;
}
public function isValidJson($fields)
{
$text = $fields['network_json'];
$check = json_decode($text);
if ($check === null) {
return false;
}
return true;
}
public function getPictureData($eventGraph)
{
$b64 = str_replace('data:image/png;base64,', '', $eventGraph['EventGraph']['preview_img']);

26
app/Model/Feed.php
View File

@ -1932,16 +1932,14 @@ class Feed extends AppModel
$contentType = $response->getHeader('Content-Type');
if ($contentType === 'application/zip') {
$zipFile = new File($this->tempFileName());
$zipFile->write($response->body);
$zipFile->close();
$zipFilePath = FileAccessTool::writeToTempFile($response->body);
try {
$response->body = $this->unzipFirstFile($zipFile);
$response->body = $this->unzipFirstFile($zipFilePath);
} catch (Exception $e) {
throw new Exception("Fetching the '$uri' failed: {$e->getMessage()}");
} finally {
$zipFile->delete();
FileAccessTool::deleteFile($zipFilePath);
}
}
@ -2047,18 +2045,18 @@ class Feed extends AppModel
}
/**
* @param File $zipFile
* @param string $zipFile
* @return string Uncompressed data
* @throws Exception
*/
private function unzipFirstFile(File $zipFile)
private function unzipFirstFile($zipFile)
{
if (!class_exists('ZipArchive')) {
throw new Exception('ZIP archive decompressing is not supported. ZIP extension is missing in PHP.');
}
$zip = new ZipArchive();
$result = $zip->open($zipFile->pwd());
$result = $zip->open($zipFile);
if ($result !== true) {
$errorCodes = [
ZipArchive::ER_EXISTS => 'file already exists',
@ -2086,18 +2084,12 @@ class Feed extends AppModel
$zip->close();
$destinationFile = $this->tempFileName();
$result = copy("zip://{$zipFile->pwd()}#$filename", $destinationFile);
$destinationFile = FileAccessTool::createTempFile();
$result = copy("zip://$zipFile#$filename", $destinationFile);
if ($result === false) {
throw new Exception("Remote server returns ZIP file, that contains '$filename' file, but this file cannot be extracted.");
}
$unzipped = new File($destinationFile);
$data = $unzipped->read();
if ($data === false) {
throw new Exception("Couldn't read extracted file content.");
}
$unzipped->delete();
return $data;
return FileAccessTool::readAndDelete($destinationFile);
}
}

14
app/Model/GalaxyCluster.php
View File

@ -457,10 +457,6 @@ class GalaxyCluster extends AppModel
} else {
return false;
}
$this->Event = ClassRegistry::init('Event');
$job_type = 'publish_cluster';
$function = 'publish_galaxy_clusters';
$message = 'Publishing.';
$job = ClassRegistry::init('Job');
$job->create();
$data = array(
@ -471,14 +467,14 @@ class GalaxyCluster extends AppModel
'retries' => 0,
'org_id' => $user['org_id'],
'org' => $user['Organisation']['name'],
'message' => $message
'message' => 'Publishing.'
);
$job->save($data);
$jobId = $job->id;
$process_id = CakeResque::enqueue(
'prio',
'EventShell',
array($function, $clusterId, $jobId, $user['id'], $passAlong),
array('publish_galaxy_clusters', $clusterId, $jobId, $user['id'], $passAlong),
true
);
$job->saveField('process_id', $process_id);
@ -1142,12 +1138,12 @@ class GalaxyCluster extends AppModel
$sharingGroupData = $this->Event->__cacheSharingGroupData($user, false);
foreach ($clusters as $i => $cluster) {
if (!empty($cluster['GalaxyCluster']['sharing_group_id']) && isset($sharingGroupData[$cluster['GalaxyCluster']['sharing_group_id']])) {
$clusters[$i]['SharingGroup'] = $sharingGroupData[$cluster['GalaxyCluster']['sharing_group_id']]['SharingGroup'];
$clusters[$i]['SharingGroup'] = $sharingGroupData[$cluster['GalaxyCluster']['sharing_group_id']];
}
if (isset($cluster['GalaxyClusterRelation'])) {
foreach ($cluster['GalaxyClusterRelation'] as $j => $relation) {
if (!empty($relation['sharing_group_id']) && isset($sharingGroupData[$relation['sharing_group_id']])) {
$clusters[$i]['GalaxyClusterRelation'][$j]['SharingGroup'] = $sharingGroupData[$relation['sharing_group_id']]['SharingGroup'];
$clusters[$i]['GalaxyClusterRelation'][$j]['SharingGroup'] = $sharingGroupData[$relation['sharing_group_id']];
}
foreach ($relation['GalaxyClusterRelationTag'] as $relationTag) {
if (isset($tags[$relationTag['tag_id']])) {
@ -1662,7 +1658,7 @@ class GalaxyCluster extends AppModel
{
$this->Server = ClassRegistry::init('Server');
$this->Log = ClassRegistry::init('Log');
$push = $this->Server->checkVersionCompatibility($server, false, $HttpSocket);
$push = $this->Server->checkVersionCompatibility($server, false);
if (empty($push['canPush']) && empty($push['canPushGalaxyCluster'])) {
return __('The remote user does not have the permission to manipulate galaxies - the upload of the galaxy clusters has been blocked.');
}

19
app/Model/Log.php
View File

@ -240,6 +240,9 @@ class Log extends AppModel
if ($action === 'request' && !empty(Configure::read('MISP.log_paranoid_skip_db'))) {
return null;
}
if (!empty(Configure::read('MISP.log_skip_db_logs_completely'))) {
return null;
}
throw new Exception("Cannot save log because of validation errors: " . json_encode($this->validationErrors));
}
@ -247,6 +250,22 @@ class Log extends AppModel
return $result;
}
/**
* @param array|string $user
* @param string $action
* @param string $model
* @param string $title
* @param array $validationErrors
* @param array $fullObject
* @throws Exception
*/
public function validationError($user, $action, $model, $title, array $validationErrors, array $fullObject)
{
$this->log($title, LOG_WARNING);
$change = 'Validation errors: ' . json_encode($validationErrors) . ' Full ' . $model . ': ' . json_encode($fullObject);
$this->createLogEntry($user, $action, $model, 0, $title, $change);
}
// to combat a certain bug that causes the upgrade scripts to loop without being able to set the correct version
// this function remedies a fixed upgrade bug instance by eliminating the massive number of erroneous upgrade log entries
public function pruneUpdateLogs($jobId = false, $user)

69
app/Model/MispObject.php
View File

@ -69,7 +69,6 @@ class MispObject extends AppModel
'unique' => array(
'rule' => 'isUnique',
'message' => 'The UUID provided is not unique',
'required' => true,
'on' => 'create'
),
),
@ -216,23 +215,19 @@ class MispObject extends AppModel
}
// check whether the variable is null or datetime
public function datetimeOrNull($fields)
{
$k = array_keys($fields)[0];
$seen = $fields[$k];
try {
new DateTime($seen);
$returnValue = true;
} catch (Exception $e) {
$returnValue = false;
}
return $returnValue || is_null($seen);
}
public function datetimeOrNull($fields)
{
$seen = array_values($fields)[0];
if ($seen === null) {
return true;
}
return strtotime($seen) !== false;
}
public function validateLastSeenValue($fields)
{
$ls = $fields['last_seen'];
if (!isset($this->data['Object']['first_seen']) || is_null($ls)) {
if (!isset($this->data['Object']['first_seen']) || $ls === null) {
return true;
}
$converted = $this->Attribute->ISODatetimeToUTC(['Object' => [
@ -253,44 +248,44 @@ class MispObject extends AppModel
return $results;
}
public function beforeSave($options = array()) {
public function beforeSave($options = array())
{
// generate UUID if it doesn't exist
if (empty($this->data['Object']['uuid'])) {
$this->data['Object']['uuid'] = CakeText::uuid();
}
$this->data = $this->Attribute->ISODatetimeToUTC($this->data, $this->alias);
}
public function beforeValidate($options = array())
{
parent::beforeValidate();
if (empty($this->data[$this->alias]['comment'])) {
$this->data[$this->alias]['comment'] = "";
}
// generate UUID if it doesn't exist
if (empty($this->data[$this->alias]['uuid'])) {
$this->data[$this->alias]['uuid'] = CakeText::uuid();
$object = &$this->data['Object'];
if (empty($object['comment'])) {
$object['comment'] = "";
}
// generate timestamp if it doesn't exist
if (empty($this->data[$this->alias]['timestamp'])) {
$date = new DateTime();
$this->data[$this->alias]['timestamp'] = $date->getTimestamp();
if (empty($object['timestamp'])) {
$object['timestamp'] = time();
}
// parse first_seen different formats
if (isset($this->data[$this->alias]['first_seen'])) {
$this->data[$this->alias]['first_seen'] = $this->data[$this->alias]['first_seen'] === '' ? null : $this->data[$this->alias]['first_seen'];
if (isset($object['first_seen'])) {
$object['first_seen'] = $object['first_seen'] === '' ? null : $object['first_seen'];
}
// parse last_seen different formats
if (isset($this->data[$this->alias]['last_seen'])) {
$this->data[$this->alias]['last_seen'] = $this->data[$this->alias]['last_seen'] === '' ? null : $this->data[$this->alias]['last_seen'];
if (isset($object['last_seen'])) {
$object['last_seen'] = $object['last_seen'] === '' ? null : $object['last_seen'];
}
if (empty($this->data[$this->alias]['template_version'])) {
$this->data[$this->alias]['template_version'] = 1;
if (empty($object['template_version'])) {
$object['template_version'] = 1;
}
if (isset($this->data[$this->alias]['deleted']) && empty($this->data[$this->alias]['deleted'])) {
$this->data[$this->alias]['deleted'] = 0;
if (isset($object['deleted']) && empty($object['deleted'])) {
$object['deleted'] = 0;
}
if (!isset($this->data[$this->alias]['distribution']) || $this->data['Object']['distribution'] != 4) {
$this->data['Object']['sharing_group_id'] = 0;
if (!isset($object['distribution']) || $object['distribution'] != 4) {
$object['sharing_group_id'] = 0;
}
if (!isset($this->data[$this->alias]['distribution'])) {
$this->data['Object']['distribution'] = 5;
if (!isset($object['distribution'])) {
$object['distribution'] = 5;
}
return true;
}

45
app/Model/Module.php
View File

@ -1,5 +1,6 @@
<?php
App::uses('AppModel', 'Model');
App::uses('JsonTool', 'Tools');
class Module extends AppModel
{
@ -234,28 +235,27 @@ class Module extends AppModel
* @param array|null $postData
* @param string $moduleFamily
* @return array
* @throws JsonException
* @throws HttpSocketJsonException
* @throws Exception
*/
public function sendRequest($uri, $timeout, $postData = null, $moduleFamily = 'Enrichment')
{
$url = $this->__getModuleServer($moduleFamily);
if (!$url) {
$serverUrl = $this->__getModuleServer($moduleFamily);
if (!$serverUrl) {
throw new Exception("Module type $moduleFamily is not enabled.");
}
App::uses('HttpSocket', 'Network/Http');
App::uses('HttpSocketExtended', 'Tools');
$httpSocketSetting = ['timeout' => $timeout];
$sslSettings = array('ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_verify_peer', 'ssl_cafile');
foreach ($sslSettings as $sslSetting) {
if (Configure::check('Plugin.' . $moduleFamily . '_' . $sslSetting) && Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting) !== '') {
$settings[$sslSetting] = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
$value = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
if ($value && $value !== '') {
$httpSocketSetting[$sslSetting] = $value;
}
}
$httpSocket = new HttpSocket(['timeout' => $timeout]);
$request = array(
'header' => array(
'Content-Type' => 'application/json',
)
);
if ($moduleFamily == 'Cortex') {
$httpSocket = new HttpSocketExtended($httpSocketSetting);
$request = [];
if ($moduleFamily === 'Cortex') {
if (!empty(Configure::read('Plugin.' . $moduleFamily . '_authkey'))) {
$request['header']['Authorization'] = 'Bearer ' . Configure::read('Plugin.' . $moduleFamily . '_authkey');
}
@ -264,26 +264,23 @@ class Module extends AppModel
if (!is_array($postData)) {
throw new InvalidArgumentException("Post data must be array, " . gettype($postData) . " given.");
}
$post = json_encode($postData);
$response = $httpSocket->post($url . $uri, $post, $request);
$post = JsonTool::encode($postData);
$request['header']['Content-Type'] = 'application/json';
$response = $httpSocket->post($serverUrl . $uri, $post, $request);
} else {
if ($moduleFamily == 'Cortex') {
unset($request['header']['Content-Type']);
}
$response = $httpSocket->get($url . $uri, false, $request);
$response = $httpSocket->get($serverUrl . $uri, false, $request);
}
if (!$response->isOk()) {
if ($httpSocket->lastError()) {
throw new Exception("Failed to get response from $moduleFamily module: " . $httpSocket->lastError['str']);
}
throw new Exception("Failed to get response from $moduleFamily module: HTTP $response->reasonPhrase", (int)$response->code);
$e = new HttpSocketHttpException($response, $serverUrl . $uri);
throw new Exception("Failed to get response from `$moduleFamily` module", 0, $e);
}
return $this->jsonDecode($response->body);
return $response->json();
}
/**
* @param string $moduleFamily
* @return array
* @throws JsonException
*/
public function getModuleSettings($moduleFamily = 'Enrichment')
{

49
app/Model/ObjectReference.php
View File

@ -38,17 +38,38 @@ class ObjectReference extends AppModel
)
);
public $validate = [
'uuid' => 'uuid',
'object_id' => [
'rule' => 'numeric',
'required' => true,
'on' => 'create',
],
'event_id' => [
'rule' => 'numeric',
'required' => true,
'on' => 'create',
],
'source_uuid' => 'uuid',
'referenced_uuid' => 'uuid',
'referenced_id' => 'numeric',
'referenced_type' => [
'rule' => ['inList', ['0', '1']],
],
'deleted' => 'boolean',
];
public function beforeValidate($options = array())
{
parent::beforeValidate();
if (empty($this->data['ObjectReference']['uuid'])) {
$this->data['ObjectReference']['uuid'] = CakeText::uuid();
$reference = &$this->data['ObjectReference'];
if (empty($reference['uuid'])) {
$reference['uuid'] = CakeText::uuid();
}
if (empty($this->data['ObjectReference']['timestamp'])) {
$this->data['ObjectReference']['timestamp'] = time();
if (empty($reference['timestamp'])) {
$reference['timestamp'] = time();
}
if (!isset($this->data['ObjectReference']['comment'])) {
$this->data['ObjectReference']['comment'] = '';
if (!isset($reference['comment'])) {
$reference['comment'] = '';
}
return true;
}
@ -170,7 +191,13 @@ class ObjectReference extends AppModel
return true;
}
public function captureReference($reference, $eventId, $user)
/**
* @param array $reference
* @param int $eventId
* @return array|bool
* @throws Exception
*/
public function captureReference(array $reference, $eventId)
{
if (isset($reference['uuid'])) {
$existingReference = $this->find('first', array(
@ -257,6 +284,9 @@ class ObjectReference extends AppModel
$reference['object_uuid'] = $sourceObject['Object']['uuid'];
$reference['event_id'] = $eventId;
$result = $this->save(array('ObjectReference' => $reference));
if (!$result) {
return $this->validationErrors;
}
return true;
}
@ -308,7 +338,8 @@ class ObjectReference extends AppModel
return array($referenced_id, $referenced_uuid, $referenced_type);
}
function isValidExtendedEventForReference($sourceEvent, $targetEventID, $user) {
private function isValidExtendedEventForReference(array $sourceEvent, $targetEventID, array $user)
{
if ($sourceEvent['Event']['orgc_id'] != $user['org_id']) {
return false;
}

274
app/Model/Server.php
View File

@ -444,12 +444,22 @@ class Server extends AppModel
return true;
}
public function pull($user, $id = null, $technique=false, $server, $jobId = false, $force = false)
/**
* @param array $user
* @param string $technique
* @param array $server
* @param int|false $jobId
* @param bool $force
* @return array|string
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
* @throws JsonException
*/
public function pull(array $user, $technique, array $server, $jobId = false, $force = false)
{
if ($jobId) {
Configure::write('CurrentUserId', $user['id']);
$job = ClassRegistry::init('Job');
$job->read(null, $jobId);
$email = "Scheduled job";
} else {
$email = $user['email'];
@ -474,19 +484,17 @@ class Server extends AppModel
if (!empty($server['Server']['pull_galaxy_clusters'])) {
$this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
if ($jobId) {
$job->saveField('message', $technique == 'pull_relevant_clusters' ? __('Pulling relevant galaxy clusters.') : __('Pulling galaxy clusters.'));
$job->saveProgress($jobId, $technique === 'pull_relevant_clusters' ? __('Pulling relevant galaxy clusters.') : __('Pulling galaxy clusters.'));
}
$pulledClusters = $this->GalaxyCluster->pullGalaxyClusters($user, $server, $technique);
if ($technique == 'pull_relevant_clusters') {
if ($technique === 'pull_relevant_clusters') {
if ($jobId) {
$job->saveField('progress', 100);
$job->saveField('message', 'Pulling complete.');
$job->saveStatus($jobId, true, 'Pulling complete.');
}
return array(array(), array(), 0, 0, $pulledClusters);
}
if ($jobId) {
$job->saveField('progress', 10);
$job->saveField('message', 'Pulling events.');
$job->saveProgress($jobId, 'Pulling events.', 10);
}
}
@ -516,16 +524,12 @@ class Server extends AppModel
}
foreach ($eventIds as $k => $eventId) {
$this->__pullEvent($eventId, $successes, $fails, $eventModel, $serverSync, $user, $jobId, $force);
if ($jobId) {
if ($k % 10 == 0) {
$job->saveProgress($jobId, null, 10 + 40 * (($k + 1) / count($eventIds)));
}
if ($jobId && $k % 10 === 0) {
$job->saveProgress($jobId, null, 10 + 40 * (($k + 1) / count($eventIds)));
}
}
}
if (!empty($fails)) {
foreach ($fails as $eventid => $message) {
$this->loadLog()->createLogEntry($user, 'pull', 'Server', $id, "Failed to pull event #$eventid.", 'Reason: ' . $message);
$this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['Server']['id'], "Failed to pull event #$eventid.", 'Reason: ' . $message);
}
}
if ($jobId) {
@ -541,27 +545,18 @@ class Server extends AppModel
$pulledSightings = $eventModel->Sighting->pullSightings($user, $serverSync);
}
if ($jobId) {
$job->saveProgress($jobId, 'Pull completed.', 100);
$job->saveStatus($jobId, true, 'Pull completed.');
}
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'Server',
'model_id' => $id,
'email' => $user['email'],
'action' => 'pull',
'user_id' => $user['id'],
'title' => 'Pull from ' . $server['Server']['url'] . ' initiated by ' . $email,
'change' => sprintf(
'%s events, %s proposals, %s sightings and %s galaxyClusters pulled or updated. %s events failed or didn\'t need an update.',
count($successes),
$pulledProposals,
$pulledSightings,
$pulledClusters,
count($fails)
)
));
$change = sprintf(
'%s events, %s proposals, %s sightings and %s galaxyClusters pulled or updated. %s events failed or didn\'t need an update.',
count($successes),
$pulledProposals,
$pulledSightings,
$pulledClusters,
count($fails)
);
$this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['Server']['id'], 'Pull from ' . $server['Server']['url'] . ' initiated by ' . $email, $change);
return array($successes, $fails, $pulledProposals, $pulledSightings, $pulledClusters);
}
@ -731,7 +726,7 @@ class Server extends AppModel
* @throws HttpSocketJsonException
* @throws InvalidArgumentException
*/
public function getEventIdsFromServer(ServerSyncTool $serverSync, $all = false, $ignoreFilterRules = false, $scope = 'events', $force = false)
private function getEventIdsFromServer(ServerSyncTool $serverSync, $all = false, $ignoreFilterRules = false, $scope = 'events', $force = false)
{
if (!in_array($scope, ['events', 'sightings'], true)) {
throw new InvalidArgumentException("Scope must be 'events' or 'sightings', '$scope' given.");
@ -2481,61 +2476,47 @@ class Server extends AppModel
return ['status' => 1, 'content-encoding' => $contentEncoding];
}
public function checkVersionCompatibility(array $server, $user = array(), $HttpSocket = false)
/**
* @param array $server
* @param array $user
* @param ServerSyncTool|null $serverSync
* @return array|string
* @throws JsonException
*/
public function checkVersionCompatibility(array $server, $user = [], ServerSyncTool $serverSync = null)
{
// for event publishing when we don't have a user.
if (empty($user)) {
$user = array('Organisation' => array('name' => 'SYSTEM'), 'email' => 'SYSTEM', 'id' => 0);
$user = 'SYSTEM';
}
$localVersion = $this->checkMISPVersion();
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server);
$uri = $server['Server']['url'] . '/servers/getVersion';
$serverSync = $serverSync ? $serverSync : new ServerSyncTool($server, $this->setupSyncRequest($server));
try {
$response = $HttpSocket->get($uri, '', $request);
$remoteVersion = $serverSync->info();
} catch (Exception $e) {
$error = $e->getMessage();
}
if (!isset($response) || $response->code != '200') {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
if (isset($response->code)) {
$title = 'Error: Connection to the server has failed.' . (isset($response->code) ? ' Returned response code: ' . $response->code : '');
$this->logException("Connection to the server {$server['Server']['id']} has failed", $e);
if ($e instanceof HttpSocketHttpException) {
$title = 'Error: Connection to the server has failed. Returned response code: ' . $e->getCode();
} else {
$title = 'Error: Connection to the server has failed. The returned exception\'s error message was: ' . $error;
$title = 'Error: Connection to the server has failed. The returned exception\'s error message was: ' . $e->getMessage();
}
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'Server',
'model_id' => $server['Server']['id'],
'email' => $user['email'],
'action' => 'error',
'user_id' => $user['id'],
'title' => $title
));
$this->loadLog()->createLogEntry($user, 'error', 'Server', $server['Server']['id'], $title);
return $title;
}
$remoteVersion = $this->jsonDecode($response->body);
$canPush = isset($remoteVersion['perm_sync']) ? $remoteVersion['perm_sync'] : false;
$canSight = isset($remoteVersion['perm_sighting']) ? $remoteVersion['perm_sighting'] : false;
$supportEditOfGalaxyCluster = isset($remoteVersion['perm_galaxy_editor']);
$canEditGalaxyCluster = isset($remoteVersion['perm_galaxy_editor']) ? $remoteVersion['perm_galaxy_editor'] : false;
$remoteVersion = explode('.', $remoteVersion['version']);
if (!isset($remoteVersion[0])) {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$message = __('Error: Server didn\'t send the expected response. This may be because the remote server version is outdated.');
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'Server',
'model_id' => $server['Server']['id'],
'email' => $user['email'],
'action' => 'error',
'user_id' => $user['id'],
'title' => $message,
));
$this->loadLog()->createLogEntry($user, 'error', 'Server', $server['Server']['id'], $message);
return $message;
}
$localVersion = $this->checkMISPVersion();
$response = false;
$success = false;
$issueLevel = "warning";
@ -2569,17 +2550,7 @@ class Server extends AppModel
}
if ($response !== false) {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'Server',
'model_id' => $server['Server']['id'],
'email' => $user['email'],
'action' => $issueLevel,
'user_id' => $user['id'],
'title' => ucfirst($issueLevel) . ': ' . $response,
));
$this->loadLog()->createLogEntry($user, $issueLevel, 'Server', $server['Server']['id'], ucfirst($issueLevel) . ': ' . $response);
}
return [
'success' => $success,
@ -3049,71 +3020,70 @@ class Server extends AppModel
);
}
/**
* @throws Exception
*/
private function compareDBIndexes(array $actualIndex, array $expectedIndex, array $dbExpectedSchema)
{
$allowedlistTables = array();
$indexDiff = array();
foreach ($expectedIndex as $tableName => $indexes) {
if (!array_key_exists($tableName, $actualIndex)) {
continue; // If table does not exists, it is covered by the schema diagnostic
} elseif(in_array($tableName, $allowedlistTables)) {
continue; // Ignore allowedlisted tables
} else {
$tableIndexDiff = array_diff(array_keys($indexes), array_keys($actualIndex[$tableName])); // check for missing indexes
foreach ($tableIndexDiff as $columnDiff) {
$shouldBeUnique = $indexes[$columnDiff];
if ($shouldBeUnique && !$this->checkIfColumnContainsJustUniqueValues($tableName, $columnDiff)) {
$indexDiff[$tableName][$columnDiff] = array(
'message' => __('Column `%s` should be unique indexed, but contains duplicate values', $columnDiff),
'sql' => '',
}
$tableIndexDiff = array_diff(array_keys($indexes), array_keys($actualIndex[$tableName])); // check for missing indexes
foreach ($tableIndexDiff as $columnDiff) {
$shouldBeUnique = $indexes[$columnDiff];
if ($shouldBeUnique && !$this->checkIfColumnContainsJustUniqueValues($tableName, $columnDiff)) {
$indexDiff[$tableName][$columnDiff] = array(
'message' => __('Column `%s` should be unique indexed, but contains duplicate values', $columnDiff),
'sql' => '',
);
continue;
}
$message = __('Column `%s` should be indexed', $columnDiff);
$indexDiff[$tableName][$columnDiff] = array(
'message' => $message,
'sql' => $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $columnDiff, $shouldBeUnique),
);
}
$tableIndexDiff = array_diff(array_keys($actualIndex[$tableName]), array_keys($indexes)); // check for additional indexes
foreach ($tableIndexDiff as $columnDiff) {
$message = __('Column `%s` is indexed but should not', $columnDiff);
$indexDiff[$tableName][$columnDiff] = array(
'message' => $message,
'sql' => $this->generateSqlDropIndexQuery($tableName, $columnDiff),
);
}
foreach ($indexes as $column => $unique) {
if (isset($actualIndex[$tableName][$column]) && $actualIndex[$tableName][$column] != $unique) {
if ($actualIndex[$tableName][$column]) {
$sql = $this->generateSqlDropIndexQuery($tableName, $column);
$sql .= '<br>' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, false);
$message = __('Column `%s` has unique index, but should be non unique', $column);
$indexDiff[$tableName][$column] = array(
'message' => $message,
'sql' => $sql,
);
continue;
}
$message = __('Column `%s` should be indexed', $columnDiff);
$indexDiff[$tableName][$columnDiff] = array(
'message' => $message,
'sql' => $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $columnDiff, $shouldBeUnique),
);
}
$tableIndexDiff = array_diff(array_keys($actualIndex[$tableName]), array_keys($indexes)); // check for additional indexes
foreach ($tableIndexDiff as $columnDiff) {
$message = __('Column `%s` is indexed but should not', $columnDiff);
$indexDiff[$tableName][$columnDiff] = array(
'message' => $message,
'sql' => $this->generateSqlDropIndexQuery($tableName, $columnDiff),
);
}
foreach ($indexes as $column => $unique) {
if (isset($actualIndex[$tableName][$column]) && $actualIndex[$tableName][$column] != $unique) {
if ($actualIndex[$tableName][$column]) {
$sql = $this->generateSqlDropIndexQuery($tableName, $column);
$sql .= '<br>' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, false);
$message = __('Column `%s` has unique index, but should be non unique', $column);
} else {
if (!$this->checkIfColumnContainsJustUniqueValues($tableName, $column)) {
$message = __('Column `%s` should be unique index, but contains duplicate values', $column);
$indexDiff[$tableName][$column] = array(
'message' => $message,
'sql' => $sql,
);
} else {
if (!$this->checkIfColumnContainsJustUniqueValues($tableName, $column)) {
$message = __('Column `%s` should be unique index, but contains duplicate values', $column);
$indexDiff[$tableName][$column] = array(
'message' => $message,
'sql' => '',
);
continue;
}
$sql = $this->generateSqlDropIndexQuery($tableName, $column);
$sql .= '<br>' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, true);
$message = __('Column `%s` should be unique index', $column);
$indexDiff[$tableName][$column] = array(
'message' => $message,
'sql' => $sql,
'sql' => '',
);
continue;
}
$sql = $this->generateSqlDropIndexQuery($tableName, $column);
$sql .= '<br>' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, true);
$message = __('Column `%s` should be unique index', $column);
$indexDiff[$tableName][$column] = array(
'message' => $message,
'sql' => $sql,
);
}
}
}
@ -3150,21 +3120,21 @@ class Server extends AppModel
App::uses('Folder', 'Utility');
// check writeable directories
$writeableDirs = array(
'/tmp' => 0,
APP . 'tmp' => 0,
APP . 'files' => 0,
APP . 'files' . DS . 'scripts' . DS . 'tmp' => 0,
APP . 'tmp' . DS . 'csv_all' => 0,
APP . 'tmp' . DS . 'csv_sig' => 0,
APP . 'tmp' . DS . 'md5' => 0,
APP . 'tmp' . DS . 'sha1' => 0,
APP . 'tmp' . DS . 'snort' => 0,
APP . 'tmp' . DS . 'suricata' => 0,
APP . 'tmp' . DS . 'text' => 0,
APP . 'tmp' . DS . 'xml' => 0,
APP . 'tmp' . DS . 'files' => 0,
APP . 'tmp' . DS . 'logs' => 0,
APP . 'tmp' . DS . 'bro' => 0,
'/tmp' => 0,
APP . 'tmp' => 0,
APP . 'files' => 0,
APP . 'files' . DS . 'scripts' . DS . 'tmp' => 0,
APP . 'tmp' . DS . 'csv_all' => 0,
APP . 'tmp' . DS . 'csv_sig' => 0,
APP . 'tmp' . DS . 'md5' => 0,
APP . 'tmp' . DS . 'sha1' => 0,
APP . 'tmp' . DS . 'snort' => 0,
APP . 'tmp' . DS . 'suricata' => 0,
APP . 'tmp' . DS . 'text' => 0,
APP . 'tmp' . DS . 'xml' => 0,
APP . 'tmp' . DS . 'files' => 0,
APP . 'tmp' . DS . 'logs' => 0,
APP . 'tmp' . DS . 'bro' => 0,
);
foreach ($writeableDirs as $path => &$error) {
$dir = new Folder($path);

63
app/Model/ShadowAttribute.php
View File

@ -129,13 +129,13 @@ class ShadowAttribute extends AppModel
'first_seen' => array(
'rule' => array('datetimeOrNull'),
'required' => false,
'message' => array('Invalid ISO 8601 format')
'message' => array('Invalid ISO 8601 format'),
),
'last_seen' => array(
'datetimeOrNull' => array(
'rule' => array('datetimeOrNull'),
'required' => false,
'message' => array('Invalid ISO 8601 format')
'message' => array('Invalid ISO 8601 format'),
),
'validateLastSeenValue' => array(
'rule' => array('validateLastSeenValue'),
@ -173,7 +173,7 @@ class ShadowAttribute extends AppModel
$compositeTypes = $this->getCompositeTypes();
// explode composite types in value1 and value2
$pieces = explode('|', $this->data['ShadowAttribute']['value']);
if (in_array($this->data['ShadowAttribute']['type'], $compositeTypes)) {
if (in_array($this->data['ShadowAttribute']['type'], $compositeTypes, true)) {
if (2 != count($pieces)) {
throw new InternalErrorException('Composite type, but value not explodable');
}
@ -300,51 +300,54 @@ class ShadowAttribute extends AppModel
public function beforeValidate($options = array())
{
parent::beforeValidate();
// remove leading and trailing blanks
//$this->trimStringFields(); // TODO
if (!isset($this->data['ShadowAttribute']['comment'])) {
$this->data['ShadowAttribute']['comment'] = '';
}
if (!isset($this->data['ShadowAttribute']['type'])) {
$proposal = &$this->data['ShadowAttribute'];
if (!isset($proposal['type'])) {
$this->invalidate('type', 'No value provided.');
return false;
}
if (!isset($proposal['comment'])) {
$proposal['comment'] = '';
}
// make some changes to the inserted value
if (isset($this->data['ShadowAttribute']['value'])) {
$value = trim($this->data['ShadowAttribute']['value']);
$value = ComplexTypeTool::refangValue($value, $this->data['ShadowAttribute']['type']);
$value = $this->Attribute->modifyBeforeValidation($this->data['ShadowAttribute']['type'], $value);
$this->data['ShadowAttribute']['value'] = $value;
if (isset($proposal['value'])) {
$value = trim($proposal['value']);
$value = ComplexTypeTool::refangValue($value, $proposal['type']);
$value = $this->Attribute->modifyBeforeValidation($proposal['type'], $value);
$proposal['value'] = $value;
}
if (!isset($this->data['ShadowAttribute']['org'])) {
$this->data['ShadowAttribute']['org'] = '';
if (!isset($proposal['org'])) {
$proposal['org'] = '';
}
if (empty($this->data['ShadowAttribute']['timestamp'])) {
$date = new DateTime();
$this->data['ShadowAttribute']['timestamp'] = $date->getTimestamp();
if (empty($proposal['timestamp'])) {
$proposal['timestamp'] = time();
}
if (!isset($this->data['ShadowAttribute']['proposal_to_delete'])) {
$this->data['ShadowAttribute']['proposal_to_delete'] = 0;
if (!isset($proposal['proposal_to_delete'])) {
$proposal['proposal_to_delete'] = 0;
}
// generate UUID if it doesn't exist
if (empty($this->data['ShadowAttribute']['uuid'])) {
$this->data['ShadowAttribute']['uuid'] = CakeText::uuid();
if (empty($proposal['uuid'])) {
$proposal['uuid'] = CakeText::uuid();
} else {
$this->data['ShadowAttribute']['uuid'] = strtolower($this->data['ShadowAttribute']['uuid']);
$proposal['uuid'] = strtolower($proposal['uuid']);
}
if (!empty($this->data['ShadowAttribute']['type']) && empty($this->data['ShadowAttribute']['category'])) {
$this->data['ShadowAttribute']['category'] = $this->Attribute->typeDefinitions[$this->data['ShadowAttribute']['type']]['default_category'];
if (empty($proposal['category'])) {
$proposal['category'] = $this->Attribute->typeDefinitions[$proposal['type']]['default_category'];
}
if (isset($proposal['first_seen'])) {
$proposal['first_seen'] = $proposal['first_seen'] === '' ? null : $proposal['first_seen'];
}
if (isset($proposal['last_seen'])) {
$proposal['last_seen'] = $proposal['last_seen'] === '' ? null : $proposal['last_seen'];
}
// always return true, otherwise the object cannot be saved
return true;
}

9
app/Model/SharingGroup.php
View File

@ -5,6 +5,9 @@ App::uses('AppModel', 'Model');
* @property SharingGroupOrg $SharingGroupOrg
* @property SharingGroupServer $SharingGroupServer
* @property Organisation $Organisation
* @property Event $Event
* @property Attribute $Attribute
* @property Thread $Thread
*/
class SharingGroup extends AppModel
{
@ -101,6 +104,12 @@ class SharingGroup extends AppModel
if ($this->Attribute->hasAny(['sharing_group_id' => $this->id])) {
return false;
}
if ($this->Attribute->Object->hasAny(['sharing_group_id' => $this->id])) {
return false;
}
if ($this->Event->EventReport->hasAny(['sharing_group_id' => $this->id])) {
return false;
}
return true;
}

47
app/Model/SharingGroupOrg.php
View File

@ -1,29 +1,41 @@
<?php
App::uses('AppModel', 'Model');
class SharingGroupOrg extends AppModel
{
public $actsAs = array('AuditLog', 'Containable');
public $belongsTo = array(
'SharingGroup' => array(
'className' => 'SharingGroup',
'foreignKey' => 'sharing_group_id'
),
'Organisation' => array(
'className' => 'Organisation',
'foreignKey' => 'org_id',
//'conditions' => array('SharingGroupElement.organisation_uuid' => 'Organisation.uuid')
)
'SharingGroup' => array(
'className' => 'SharingGroup',
'foreignKey' => 'sharing_group_id'
),
'Organisation' => array(
'className' => 'Organisation',
'foreignKey' => 'org_id',
//'conditions' => array('SharingGroupElement.organisation_uuid' => 'Organisation.uuid')
)
);
public function beforeValidate($options = array())
{
parent::beforeValidate();
$data = $this->data[$this->alias];
$conditions = [
'sharing_group_id' => $data['sharing_group_id'],
'org_id' => $data['org_id'],
];
if (isset($data['id'])) {
$conditions['id !='] = $data['id'];
}
if ($this->hasAny($conditions)) {
$this->log("Trying to save duplicate organisation `{$data['org_id']}` for sharing group `{$data['sharing_group_id']}. This should never happened.");
$this->invalidate('org_id', 'The same organisation is already assigned to this sharing group.');
return false;
}
}
public function updateOrgsForSG($id, $new_orgs, $old_orgs, $user)
{
$log = ClassRegistry::init('Log');
// Loop through all of the organisations we want to add.
foreach ($new_orgs as $org) {
$SgO = array(
@ -54,16 +66,16 @@ class SharingGroupOrg extends AppModel
}
if ($this->save($SgO)) {
if ($isChange) {
$log->createLogEntry($user, 'edit', 'SharingGroupOrg', $this->id, 'Sharing group (' . $id . '): Modified right to alter sharing group for organisation (' . $org['id'] . ').', ($org['extend'] ? 'Organisation (' . $org['id'] . ') can now extend the sharing group.' : 'Organisation (' . $org['id'] . ') can no longer extend the sharing group.'));
$this->loadLog()->createLogEntry($user, 'edit', 'SharingGroupOrg', $this->id, 'Sharing group (' . $id . '): Modified right to alter sharing group for organisation (' . $org['id'] . ').', ($org['extend'] ? 'Organisation (' . $org['id'] . ') can now extend the sharing group.' : 'Organisation (' . $org['id'] . ') can no longer extend the sharing group.'));
} else {
$log->createLogEntry($user, 'add', 'SharingGroupOrg', $this->id, 'Sharing group (' . $id . '): Added organisation (' . $org['id'] . ').', 'Organisation (' . $org['id'] . ') added to Sharing group.' . ($org['extend'] ? ' Organisation (' . $org['id'] . ') can extend the sharing group.' : ''));
$this->loadLog()->createLogEntry($user, 'add', 'SharingGroupOrg', $this->id, 'Sharing group (' . $id . '): Added organisation (' . $org['id'] . ').', 'Organisation (' . $org['id'] . ') added to Sharing group.' . ($org['extend'] ? ' Organisation (' . $org['id'] . ') can extend the sharing group.' : ''));
}
}
}
// We are left with some "old orgs" that are not in the new list. This means that they can be safely deleted.
foreach ($old_orgs as $old_org) {
if ($this->delete($old_org['id'])) {
$log->createLogEntry($user, 'delete', 'SharingGroupOrg', $old_org['id'], 'Sharing group (' . $id . '): Removed organisation (' . $old_org['id'] . ').', 'Organisation (' . $org['id'] . ') removed from Sharing group.');
$this->loadLog()->createLogEntry($user, 'delete', 'SharingGroupOrg', $old_org['id'], 'Sharing group (' . $id . '): Removed organisation (' . $old_org['id'] . ').', 'Organisation (' . $org['id'] . ') removed from Sharing group.');
}
}
}
@ -83,7 +95,12 @@ class SharingGroupOrg extends AppModel
return $sgs;
}
// pass a sharing group ID and an organisation ID, returns true if it has a matching attached organisation object
/**
* Pass a sharing group ID and an organisation ID, returns true if it has a matching attached organisation object
* @param int $id
* @param int $org_id
* @return bool
*/
public function checkIfAuthorised($id, $org_id)
{
return $this->hasAny([

143
app/Model/Tag.php
View File

@ -22,28 +22,28 @@ class Tag extends AppModel
);
public $validate = array(
'name' => array(
'required' => array(
'rule' => array('notBlank', 'name'),
'message' => 'This field is required.'
),
'valueNotEmpty' => array(
'rule' => array('valueNotEmpty', 'name'),
),
'unique' => array(
'rule' => 'isUnique',
'message' => 'A similar name already exists.',
),
'name' => array(
'required' => array(
'rule' => array('notBlank', 'name'),
'message' => 'This field is required.'
),
'colour' => array(
'valueNotEmpty' => array(
'rule' => array('valueNotEmpty', 'colour'),
),
'userdefined' => array(
'rule' => 'validateColour',
'message' => 'Colour has to be in the RGB format (#FFFFFF)',
),
'valueNotEmpty' => array(
'rule' => array('valueNotEmpty', 'name'),
),
'unique' => array(
'rule' => 'isUnique',
'message' => 'A similar name already exists.',
),
),
'colour' => array(
'valueNotEmpty' => array(
'rule' => array('valueNotEmpty', 'colour'),
),
'userdefined' => array(
'rule' => 'validateColour',
'message' => 'Colour has to be in the RGB format (#FFFFFF)',
),
),
);
public $hasMany = array(
@ -83,7 +83,6 @@ class Tag extends AppModel
public function beforeValidate($options = array())
{
parent::beforeValidate();
if (!isset($this->data['Tag']['org_id'])) {
$this->data['Tag']['org_id'] = 0;
}
@ -152,8 +151,7 @@ class Tag extends AppModel
public function afterFind($results, $primary = false)
{
$results = $this->checkForOverride($results);
return $results;
return $this->checkForOverride($results);
}
public function validateColour($fields)
@ -164,12 +162,41 @@ class Tag extends AppModel
return true;
}
/**
* @param array $user
* @param string $tagName
* @return mixed|null
*/
public function lookupTagIdForUser(array $user, $tagName)
{
$conditions = ['LOWER(Tag.name)' => mb_strtolower($tagName)];
if (!$user['Role']['perm_site_admin']) {
$conditions['Tag.org_id'] = [0, $user['org_id']];
$conditions['Tag.user_id'] = [0, $user['id']];
}
$tagId = $this->find('first', array(
'conditions' => $conditions,
'recursive' => -1,
'fields' => array('Tag.id'),
'callbacks' => false,
));
if (empty($tagId)) {
return null;
}
return $tagId['Tag']['id'];
}
/**
* @param string $tagName
* @return int|mixed
*/
public function lookupTagIdFromName($tagName)
{
$tagId = $this->find('first', array(
'conditions' => array('LOWER(Tag.name)' => strtolower($tagName)),
'conditions' => array('LOWER(Tag.name)' => mb_strtolower($tagName)),
'recursive' => -1,
'fields' => array('Tag.id')
'fields' => array('Tag.id'),
'callbacks' => false,
));
if (empty($tagId)) {
return -1;
@ -289,7 +316,7 @@ class Tag extends AppModel
return array_values($tag_ids);
}
public function findEventIdsByTagNames($array)
private function findEventIdsByTagNames($array)
{
$ids = array();
foreach ($array as $a) {
@ -313,26 +340,6 @@ class Tag extends AppModel
return $ids;
}
public function findAttributeIdsByAttributeTagNames($array)
{
$ids = array();
foreach ($array as $a) {
$conditions['OR'][] = array('LOWER(name) LIKE' => strtolower($a));
}
$params = array(
'recursive' => 1,
'contain' => 'AttributeTag',
'conditions' => $conditions
);
$result = $this->find('all', $params);
foreach ($result as $tag) {
foreach ($tag['AttributeTag'] as $attributeTag) {
$ids[] = $attributeTag['attribute_id'];
}
}
return $ids;
}
/**
* @param array $tag
* @param array $user
@ -340,17 +347,18 @@ class Tag extends AppModel
* @return false|int
* @throws Exception
*/
public function captureTag($tag, $user, $force=false)
public function captureTag(array $tag, array $user, $force=false)
{
$existingTag = $this->find('first', array(
'recursive' => -1,
'conditions' => array('LOWER(name)' => mb_strtolower($tag['name'])),
'fields' => ['id', 'org_id', 'user_id'],
'callbacks' => false,
));
if (empty($existingTag)) {
if ($force || $user['Role']['perm_tag_editor']) {
$this->create();
if (!isset($tag['colour']) || empty($tag['colour'])) {
if (empty($tag['colour'])) {
$tag['colour'] = $this->random_color();
}
$tag = array(
@ -367,22 +375,21 @@ class Tag extends AppModel
} else {
return false;
}
} else {
if (
!$user['Role']['perm_site_admin'] &&
}
if (
!$user['Role']['perm_site_admin'] &&
(
(
(
$existingTag['Tag']['org_id'] != 0 &&
$existingTag['Tag']['org_id'] != $user['org_id']
) ||
(
$existingTag['Tag']['user_id'] != 0 &&
$existingTag['Tag']['user_id'] != $user['id']
)
$existingTag['Tag']['org_id'] != 0 &&
$existingTag['Tag']['org_id'] != $user['org_id']
) ||
(
$existingTag['Tag']['user_id'] != 0 &&
$existingTag['Tag']['user_id'] != $user['id']
)
) {
return false;
}
)
) {
return false;
}
return $existingTag['Tag']['id'];
}
@ -441,15 +448,21 @@ class Tag extends AppModel
}
/**
* Recover user_id from the session and override numerical_values from userSetting
*/
public function checkForOverride($tags)
* Recover user_id from the session and override numerical_values from userSetting.
*
* @param array $tags
* @return array
*/
private function checkForOverride($tags)
{
$userId = Configure::read('CurrentUserId');
if ($this->tagOverrides === false && $userId > 0) {
$this->UserSetting = ClassRegistry::init('UserSetting');
$this->tagOverrides = $this->UserSetting->getTagNumericalValueOverride($userId);
}
if (empty($this->tagOverrides)) {
return $tags;
}
foreach ($tags as $k => $tag) {
if (isset($tag['Tag']['name'])) {
$tagName = $tag['Tag']['name'];

70
app/Model/User.php
View File

@ -231,9 +231,9 @@ class User extends AppModel
public function __construct($id = false, $table = null, $ds = null)
{
parent::__construct($id, $table, $ds);
$this->AdminSetting = ClassRegistry::init('AdminSetting');
$db_version = $this->AdminSetting->getSetting('db_version');
if ($db_version >= 62) {
// bind AuthKey just when authkey table already exists. This is important for updating from old versions
if (in_array('auth_keys', $this->getDataSource()->listSources(), true)) {
$this->bindModel([
'hasMany' => ['AuthKey']
], false);
@ -1504,4 +1504,68 @@ class User extends AppModel
$banStatus['message'] = __('User email notification ban setting is not enabled');
return $banStatus;
}
/**
* @param array $user
* @return bool
*/
public function hasNotifications(array $user)
{
$hasProposal = $this->Event->ShadowAttribute->hasAny([
'ShadowAttribute.event_org_id' => $user['org_id'],
'ShadowAttribute.deleted' => 0,
]);
if ($hasProposal) {
return true;
}
if (Configure::read('MISP.delegation') && $this->_getDelegationCount($user)) {
return true;
}
return false;
}
/**
* @param array $user
* @return array
*/
public function populateNotifications(array $user)
{
$notifications = array();
list($notifications['proposalCount'], $notifications['proposalEventCount']) = $this->_getProposalCount($user);
$notifications['total'] = $notifications['proposalCount'];
if (Configure::read('MISP.delegation')) {
$notifications['delegationCount'] = $this->_getDelegationCount($user);
$notifications['total'] += $notifications['delegationCount'];
}
return $notifications;
}
// if not using $mode === 'full', simply check if an entry exists. We really don't care about the real count for the top menu.
private function _getProposalCount($user, $mode = 'full')
{
$results[0] = $this->Event->ShadowAttribute->find('count', [
'conditions' => array(
'ShadowAttribute.event_org_id' => $user['org_id'],
'ShadowAttribute.deleted' => 0,
)
]);
$results[1] = $this->Event->ShadowAttribute->find('count', [
'conditions' => array(
'ShadowAttribute.event_org_id' => $user['org_id'],
'ShadowAttribute.deleted' => 0,
),
'fields' => 'distinct event_id'
]);
return $results;
}
private function _getDelegationCount($user)
{
$this->EventDelegation = ClassRegistry::init('EventDelegation');
return $this->EventDelegation->find('count', array(
'recursive' => -1,
'conditions' => array('EventDelegation.org_id' => $user['org_id'])
));
}
}

89
app/Model/UserSetting.php
View File

@ -1,5 +1,9 @@
<?php
App::uses('AppModel', 'Model');
/**
* @property User $User
*/
class UserSetting extends AppModel
{
public $useTable = 'user_settings';
@ -16,11 +20,7 @@ class UserSetting extends AppModel
);
public $validate = array(
'json' => array(
'isValidJson' => array(
'rule' => array('isValidJson'),
)
)
'value' => 'valueIsJson',
);
public $belongsTo = array(
@ -96,12 +96,14 @@ class UserSetting extends AppModel
'event_index_hide_columns' => [
'placeholder' => ['clusters'],
],
'oidc' => [ // Data saved by OIDC plugin
'restricted' => 'perm_site_admin',
],
);
// massage the data before we send it off for validation before saving anything
public function beforeValidate($options = array())
{
parent::beforeValidate();
// add a timestamp if it is not set
if (empty($this->data['UserSetting']['timestamp'])) {
$this->data['UserSetting']['timestamp'] = time();
@ -124,7 +126,9 @@ class UserSetting extends AppModel
public function afterFind($results, $primary = false)
{
foreach ($results as $k => $v) {
$results[$k]['UserSetting']['value'] = json_decode($v['UserSetting']['value'], true);
if (isset($v['UserSetting']['value'])) {
$results[$k]['UserSetting']['value'] = json_decode($v['UserSetting']['value'], true);
}
}
return $results;
}
@ -134,25 +138,27 @@ class UserSetting extends AppModel
return isset($this->validSettings[$setting]);
}
public function checkSettingAccess($user, $setting)
/**
* @param array $user
* @param string $setting
* @return bool|string
*/
public function checkSettingAccess(array $user, $setting)
{
if (!empty($this->validSettings[$setting]['restricted'])) {
$role_check = $this->validSettings[$setting]['restricted'];
if (!is_array($role_check)) {
$role_check = array($role_check);
$roleCheck = $this->validSettings[$setting]['restricted'];
if (!is_array($roleCheck)) {
$roleCheck = array($roleCheck);
}
$userHasValidRole = false;
foreach ($role_check as $role) {
foreach ($roleCheck as $role) {
if (!empty($user['Role'][$role])) {
return true;
}
}
if (!$userHasValidRole) {
foreach ($role_check as &$role) {
$role = substr($role, 5);
}
return implode(', ', $role_check);
foreach ($roleCheck as &$role) {
$role = substr($role, 5);
}
return implode(', ', $roleCheck);
}
return true;
}
@ -203,7 +209,7 @@ class UserSetting extends AppModel
return false;
}
public function getDefaulRestSearchParameters($user)
public function getDefaultRestSearchParameters($user)
{
return $this->getValueForUser($user['id'], 'default_restsearch_parameters') ?: [];
}
@ -236,8 +242,8 @@ class UserSetting extends AppModel
/**
* Check whether the event is something the user is interested (to be alerted on)
* @param $user
* @param $event
* @param array $user
* @param array $event
* @return bool
*/
public function checkPublishFilter(array $user, array $event)
@ -356,7 +362,13 @@ class UserSetting extends AppModel
return false;
}
public function setSetting($user, &$data)
/**
* @param array $user
* @param array $data
* @return bool
* @throws Exception
*/
public function setSetting(array $user, array $data)
{
$userSetting = array();
if (!empty($data['UserSetting']['user_id']) && is_numeric($data['UserSetting']['user_id'])) {
@ -391,21 +403,42 @@ class UserSetting extends AppModel
} else {
$userSetting['value'] = '';
}
return $this->setSettingInternal($userSetting['user_id'], $userSetting['setting'], $userSetting['value']);
}
/**
* Set user setting without checking permission.
* @param int $userId
* @param string $setting
* @param mixed $value
* @return array|bool|mixed|null
* @throws Exception
*/
public function setSettingInternal($userId, $setting, $value)
{
$userSetting = [
'user_id' => $userId,
'setting' => $setting,
'value' => $value,
];
$existingSetting = $this->find('first', array(
'recursive' => -1,
'conditions' => array(
'UserSetting.user_id' => $userSetting['user_id'],
'UserSetting.setting' => $userSetting['setting']
)
'UserSetting.user_id' => $userId,
'UserSetting.setting' => $setting,
),
'fields' => ['UserSetting.id'],
'callbacks' => false,
));
if (empty($existingSetting)) {
$this->create();
} else {
$userSetting['id'] = $existingSetting['UserSetting']['id'];
}
// save the setting
$result = $this->save(array('UserSetting' => $userSetting));
return true;
return $this->save($userSetting);
}
/**

22
app/Plugin/OidcAuth/Controller/Component/Auth/OidcAuthenticate.php
View File

@ -86,7 +86,7 @@ class OidcAuthenticate extends BaseAuthenticate
$this->log($mispUsername, "Unblocking user.");
$user['disabled'] = false;
}
$this->storeMetadata($user['id'], $verifiedClaims);
$this->log($mispUsername, 'Logged in.');
return $user;
}
@ -106,6 +106,8 @@ class OidcAuthenticate extends BaseAuthenticate
throw new RuntimeException("Could not save user `$mispUsername` to database.");
}
$this->storeMetadata($this->userModel()->id, $verifiedClaims);
$this->log($mispUsername, "Saved in database with ID {$this->userModel()->id}");
$this->log($mispUsername, 'Logged in.');
return $this->_findUser($mispUsername);
@ -227,6 +229,24 @@ class OidcAuthenticate extends BaseAuthenticate
return $value;
}
/**
* @param int $userId
* @param stdClass $verifiedClaims
* @return array|bool|mixed|null
* @throws Exception
*/
private function storeMetadata($userId, $verifiedClaims)
{
$value = [];
foreach (['sub', 'preferred_username', 'given_name', 'family_name'] as $field) {
if (property_exists($verifiedClaims, $field)) {
$value[$field] = $verifiedClaims->{$field};
}
}
return $this->userModel()->UserSetting->setSettingInternal($userId, 'oidc', $value);
}
/**
* @param string $username
* @param string $message

1
app/View/Attributes/index.ctp
View File

@ -7,6 +7,7 @@ echo '<div class="index">';
echo $this->element('/genericElements/IndexTable/index_table', [
'data' => [
'title' => __('Attributes'),
'primary_id_path' => 'Attribute.id',
'data' => $attributes,
'fields' => [
[

8
app/View/Elements/Events/View/attribute_correlations.ctp
View File

@ -1,6 +1,7 @@
<?php
$i = 0;
$linkColour = ($scope == 'Attribute') ? 'red' : 'white';
$withPivot = isset($withPivot) ? $withPivot : false;
// remove duplicates
$relatedEvents = array();
foreach ($event['Related' . $scope][$object['id']] as $k => $relatedAttribute) {
@ -10,7 +11,6 @@ foreach ($event['Related' . $scope][$object['id']] as $k => $relatedAttribute) {
$relatedEvents[$relatedAttribute['id']] = true;
}
}
$event['Related' . $scope][$object['id']] = array_values($event['Related' . $scope][$object['id']]);
$count = count($event['Related' . $scope][$object['id']]);
foreach ($event['Related' . $scope][$object['id']] as $relatedAttribute) {
if ($i == 4 && $count > 5) {
@ -33,8 +33,10 @@ foreach ($event['Related' . $scope][$object['id']] as $relatedAttribute) {
}
$link = $this->Html->link(
$relatedAttribute['id'],
array('controller' => 'events', 'action' => 'view', $relatedAttribute['id'], true, $event['id']),
array('class' => ($relatedAttribute['org_id'] == $me['org_id']) ? $linkColour : 'blue')
$withPivot ?
['controller' => 'events', 'action' => 'view', $relatedAttribute['id'], true, $event['Event']['id']] :
['controller' => 'events', 'action' => 'view', $relatedAttribute['id']],
['class' => ($relatedAttribute['org_id'] == $me['org_id']) ? $linkColour : 'blue']
);
echo sprintf(
'<li class="no-side-padding %s" %s data-toggle="popover" data-content="%s" data-trigger="hover">%s&nbsp;</li>',

38
app/View/Elements/Events/View/eventFilteringQueryBuilder.ctp
View File

@ -5,20 +5,19 @@ foreach ($warninglists as $id => $name) {
}
$warninglistsValues = json_encode($warninglistsValues);
?>
<div id="eventFilteringQBWrapper" style="padding: 5px; display: none; border: 1px solid #dddddd; border-bottom: 0px;">
<div id="eventFilteringQBWrapper" style="padding: 5px; display: none; border: 1px solid #dddddd; border-bottom: 0;">
<div id="eventFilteringQB" style="overflow-y: auto; padding-right: 5px; resize: vertical; max-height: 750px; height: 400px;"></div>
<div style="display: flex; justify-content: flex-end; margin-top: 5px;">
<input id="eventFilteringQBLinkInput" class="form-control" style="width: 66%;">
<button id="eventFilteringQBLinkCopy" type="button" class="btn btn-inverse" style="margin-right: 5px; margin-left: 5px;" onclick="clickMessage(this);"> <i class="fa fa-clipboard"></i> <?php echo h('Copy to clipboard'); ?> </button>
<button id="eventFilteringQBSubmit" type="button" class="btn btn-success" style="margin-right: 5px;"> <i class="fa fa-filter"></i> <?php echo h('Filter'); ?> </button>
<button id="eventFilteringQBClear" type="button" class="btn btn-xs btn-danger" style="" title="<?php echo h('Clear filtering rules'); ?>"> <i class="fa fa-times"></i> <?php echo h('Clear'); ?> </button>
<input id="eventFilteringQBLinkInput" class="form-control" style="width: 66%;">
<button id="eventFilteringQBLinkCopy" type="button" class="btn btn-inverse" style="margin-right: 5px; margin-left: 5px;"> <i class="fa fa-clipboard"></i> Copy to clipboard</button>
<button id="eventFilteringQBSubmit" type="button" class="btn btn-success" style="margin-right: 5px;"> <i class="fa fa-filter"></i> Filter</button>
<button id="eventFilteringQBClear" type="button" class="btn btn-xs btn-danger" style="" title="Clear filtering rules"> <i class="fa fa-times"></i> Clear</button>
</div>
</div>
<?php
?>
<script>
var defaultFilteringRules = <?php echo json_encode($defaultFilteringRules); ?>;
var defaultFilteringRules = <?= json_encode($defaultFilteringRules); ?>;
var querybuilderTool;
function triggerEventFilteringTool(hide) {
var qbOptions = {
@ -119,9 +118,9 @@ function triggerEventFilteringTool(hide) {
"id": "deleted",
"label": "Deleted",
"values": {
0: "Deleted only",
0: "Exclude deleted",
1: "Both",
2: "Exclude deleted"
2: "Deleted only",
}
},
{
@ -303,7 +302,7 @@ function triggerEventFilteringTool(hide) {
<?php if (isset($filters['attributeFilter'])): ?>
value: "<?php echo in_array($filters['attributeFilter'], array('all', 'network', 'financial', 'file')) ? h($filters['attributeFilter']) : 'all'; ?>"
<?php else: ?>
value: "<?php echo 'all'; ?>"
value: 'all'
<?php endif; ?>
},
<?php endif; ?>
@ -339,7 +338,7 @@ function triggerEventFilteringTool(hide) {
{
field: 'deleted',
id: 'deleted',
value: <?php echo isset($filters['deleted']) ? h($filters['deleted']) : 2; ?>
value: <?php echo isset($filters['deleted']) ? h($filters['deleted']) : 0; ?>
},
<?php endif; ?>
<?php if (empty($advancedFilteringActiveRules) || isset($advancedFilteringActiveRules['includeRelatedTags'])): ?>
@ -425,7 +424,6 @@ function triggerEventFilteringTool(hide) {
},
};
var filters = <?php echo json_encode($filters); ?>;
var $wrapper = $('#eventFilteringQBWrapper');
var $ev = $('#eventFilteringQB');
querybuilderTool = $ev.queryBuilder(qbOptions);
@ -446,6 +444,7 @@ function triggerEventFilteringTool(hide) {
$('#eventFilteringQBLinkCopy').off('click').on('click', function() {
copyToClipboard($('#eventFilteringQBLinkInput'));
clickMessage(this);
});
$('#eventFilteringQBClear').off('click').on('click', function() {
@ -525,27 +524,22 @@ function cleanRules(rules) {
function performQuery(rules) {
var res = cleanRules(rules);
var url = "<?php echo $baseurl; ?>/events/viewEventAttributes/<?php echo h($event['Event']['id']); ?>";
$.ajax({
var url = "/events/viewEventAttributes/<?php echo h($event['Event']['id']); ?>";
xhr({
type: "post",
url: url,
data: res,
beforeSend: function () {
$(".loading").show();
},
success:function (data) {
success: function (data) {
$("#attributes_div").html(data);
$(".loading").hide();
},
error:function() {
error: function() {
showMessage('fail', 'Something went wrong - could not fetch attributes.');
}
});
}
function clickMessage(clicked) {
$clicked = $(clicked);
var $clicked = $(clicked);
$clicked.tooltip({
title: 'Copied!',
trigger: 'manual',

1
app/View/Elements/Events/View/row_attribute.ctp
View File

@ -210,6 +210,7 @@ $quickEdit = function($fieldName) use ($editScope, $object, $event) {
'scope' => 'Attribute',
'object' => $object,
'event' => $event,
'withPivot' => true,
));
echo '</ul>';
}

1
app/View/Elements/Events/View/row_proposal.ctp
View File

@ -110,6 +110,7 @@
'scope' => 'ShadowAttribute',
'object' => $object,
'event' => $event,
'withPivot' => true,
));
}
?>

4
app/View/Elements/ajaxTags.ctp
View File

@ -64,7 +64,7 @@
);
if (!empty($tag['Tag']['id'])) {
$span_tag = sprintf(
'<a href="%s" style="%s" class="%s" title="%s" data-tag-id="%s">%s</a>',
'<a href="%s" style="%s" class="%s"%s data-tag-id="%s">%s</a>',
sprintf(
'%s%s%s',
$baseurl,
@ -73,7 +73,7 @@
),
$aStyle,
$aClass,
$aText,
isset($aTextModified) ? ' title="' . $aText . '"' : '',
h($tag['Tag']['id']),
isset($aTextModified) ? $aTextModified : $aText
);

1
app/View/Elements/eventattribute.ctp
View File

@ -33,6 +33,7 @@
$focus = $params['focus'];
}
unset($params['focus']);
$params += $advancedFilteringActiveRules;
$url = array_merge(array('controller' => 'events', 'action' => 'viewEventAttributes', $event['Event']['id']), $params);
$this->Paginator->options(array(
'url' => $url,

4
app/View/Elements/genericElements/IndexTable/Fields/attributeTags.ctp
View File

@ -6,8 +6,7 @@ $mayModify = ($isSiteAdmin || ($isAclModify && $event['user_id'] == $me['id'] &&
$objectId = intval($attribute['id']);
?>
<div id="Attribute_<?= $objectId ?>" class="attributeTagContainer">
<div class="attributeTagContainer">
<?php echo $this->element(
'ajaxTags',
array(
@ -21,7 +20,6 @@ $objectId = intval($attribute['id']);
)
); ?>
</div>
<?php
if (!empty($includeRelatedTags)) {
$element = '';

36
app/View/Elements/genericElements/IndexTable/index_table.ctp
View File

@ -70,23 +70,25 @@
if (!empty($data['primary_id_path'])) {
$primary = Hash::extract($data_row, $data['primary_id_path'])[0];
}
$rows .= sprintf(
'<tr data-row-id="%s" %s %s>%s</tr>',
h($k),
empty($dblclickActionArray) ? '' : 'class="dblclickElement"',
empty($primary) ? '' : 'data-primary-id="' . $primary . '"',
$this->element(
'/genericElements/IndexTable/' . $row_element,
array(
'k' => $k,
'row' => $data_row,
'fields' => $data['fields'],
'options' => $options,
'actions' => $actions,
'primary' => $primary
)
)
);
$row = '<tr data-row-id="' . h($k) . '"';
if (!empty($dblclickActionArray)) {
$row .= ' class="dblclickElement"';
}
if (!empty($primary)) {
$row .= ' data-primary-id="' . $primary . '"';
}
$row .= '>';
$row .= $this->element('/genericElements/IndexTable/' . $row_element, [
'k' => $k,
'row' => $data_row,
'fields' => $data['fields'],
'options' => $options,
'actions' => $actions,
'primary' => $primary,
]);
$rows .= $row;
}
$tbody = '<tbody>' . $rows . '</tbody>';
echo sprintf(

6
app/View/Elements/global_menu.ctp
View File

@ -513,10 +513,10 @@
h($me['email']),
$this->UserName->prepend($me['email']),
h($this->UserName->convertEmailToName($me['email'])),
isset($notifications) ? sprintf(
isset($hasNotifications) ? sprintf(
'<i class="fa fa-envelope %s" role="img" aria-label="%s"></i>',
(($notifications['total'] == 0) ? 'white' : 'red'),
__('Notifications') . ': ' . $notifications['total']
$hasNotifications ? 'red' : 'white',
__('Notifications')
) : ''
)
),

9
app/View/Events/view.ctp
View File

@ -574,10 +574,13 @@ $(function () {
queryEventLock('<?= h($event['Event']['id']); ?>', <?= (int)$event['Event']['timestamp'] ?>);
popoverStartup();
$("th, td, dt, div, span, li").tooltip({
'placement': 'top',
'container' : 'body',
$(document.body).tooltip({
selector: 'span[title], td[title], time[title]',
placement: 'top',
container: 'body',
delay: { show: 500, hide: 100 }
}).on('shown', function() {
$('.tooltip').not(":last").remove();
});
$.get("<?php echo $baseurl; ?>/threads/view/<?php echo h($event['Event']['id']); ?>/true", function(data) {

15
app/View/Helper/TimeHelper.php
View File

@ -3,6 +3,10 @@ App::uses('AppHelper', 'View/Helper');
class TimeHelper extends AppHelper
{
/**
* @param string|int $time
* @return string
*/
public function time($time)
{
if (empty($time)) {
@ -10,9 +14,16 @@ class TimeHelper extends AppHelper
}
if (is_numeric($time)) {
$time = date('Y-m-d H:i:s', $time);
} else if (is_string($time)) { // time string with timezone
$timezonePosition = strpos($time, '+00:00'); // first and last seen format
if ($timezonePosition === false) {
$timezonePosition = strpos($time, '+0000'); // datetime attribute format
}
if ($timezonePosition !== false) {
return '<time title="' . __('In UTC') . '">' . h(substr($time, 0, $timezonePosition)) . '</time>';
}
}
$time = h($time);
return '<time>' . $time . '</time>';
return '<time>' . h($time) . '</time>';
}
}

2
app/files/scripts/stix2/stix2misp.py
View File

@ -2061,7 +2061,7 @@ def from_misp(stix_objects):
def main(args):
filename = Path(os.path.dirname(args[0]), args[1])
filename = args[1] if args[1][0] == '/' else Path(os.path.dirname(args[0]), args[1])
with open(filename, 'rt', encoding='utf-8') as f:
event = stix2.parse(f.read(), allow_custom=True, interoperability=True)
stix_parser = StixFromMISPParser() if from_misp(event.objects) else ExternalStixParser()

2
app/files/scripts/stix2misp.py
View File

@ -1540,7 +1540,7 @@ def is_from_misp(event):
def main(args):
filename = '{}/tmp/{}'.format(os.path.dirname(args[0]), args[1])
filename = args[1] if args[1][0] == '/' else '{}/tmp/{}'.format(os.path.dirname(args[0]), args[1])
event = generate_event(filename)
from_misp = is_from_misp(event)
stix_parser = StixFromMISPParser() if from_misp else ExternalStixParser()

13
app/webroot/js/misp.js
View File

@ -1132,10 +1132,11 @@ function removeEventTag(event, tag) {
function loadAttributeTags(id) {
$.ajax({
dataType:"html",
dataType: "html",
cache: false,
success:function (data) {
$("#Attribute_"+id+".attributeTagContainer").html(data);
success: function (data) {
// different approach for event view and attribute view
$("#Attribute_" + id + "_tr .attributeTagContainer, [data-primary-id=" + id + "] .attributeTagContainer").html(data);
},
error: xhrFailCallback,
url: baseurl + "/tags/showAttributeTag/" + id
@ -4224,7 +4225,11 @@ function checkAndSetPublishedInfo(skip_reload) {
if (typeof skip_reload === "undefined") {
skip_reload = false;
}
var id = $('#hiddenSideMenuData').data('event-id');
var $el = $('#hiddenSideMenuData');
if ($el.length === 0) {
return;
}
var id = $el.data('event-id');
if (id !== 'undefined' && !skip_reload) {
$.get(baseurl + "/events/checkPublishedStatus/" + id, function(data) {
if (data == 1) {

13
db_schema.json
View File

@ -7436,7 +7436,7 @@
"character_maximum_length": null,
"numeric_precision": "3",
"collation_name": null,
"column_type": "tinyint(4)",
"column_type": "tinyint(1)",
"column_default": "0",
"extra": ""
},
@ -8000,12 +8000,9 @@
},
"object_references": {
"id": true,
"source_uuid": false,
"referenced_uuid": false,
"timestamp": false,
"object_id": false,
"referenced_id": false,
"relationship_type": false
"event_id": false
},
"object_relationships": {
"id": true,
@ -8084,6 +8081,7 @@
"sharing_groups": {
"id": true,
"uuid": true,
"name": true,
"org_id": false,
"sync_user_id": false,
"organisation_uuid": false
@ -8190,8 +8188,7 @@
"user_settings": {
"id": true,
"setting": false,
"user_id": false,
"timestamp": false
"user_id": false
},
"warninglists": {
"id": true
@ -8204,5 +8201,5 @@
"id": true
}
},
"db_version": "72"
"db_version": "75"
}

5
tests/build-test.sh
View File

@ -22,5 +22,6 @@ python3 ./../app/files/scripts/stix2/stix2misp.py ./../tmp/test-stix2.json 1 1 .
rm -f ./../app/files/scripts/tmp/{test-stix2.json,test-stix2.json.stix2}
# Test converting MISP to STIX2
python3 ./../app/files/scripts/stix2/misp2stix2.py -i event.json | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if data["success"] == 1 else 1)'
rm -f event.json.out
cp event.json /tmp/
python3 ./../app/files/scripts/stix2/misp2stix2.py -i /tmp/event.json | python3 -c 'import sys, json; data = json.load(sys.stdin); print(data); sys.exit(0 if data["success"] == 1 else 1)'
rm -f /tmp/{event.json,event.json.out}

402
tests/testlive_comprehensive_local.py Normal file
View File

@ -0,0 +1,402 @@
#!/usr/bin/env python3
import os
import unittest
import uuid
import urllib3 # type: ignore
import logging
logging.disable(logging.CRITICAL)
logger = logging.getLogger('pymisp')
from pymisp import PyMISP, MISPOrganisation, MISPUser, MISPRole, MISPSharingGroup, MISPEvent, MISPLog, MISPSighting, Distribution, ThreatLevel, Analysis
# Load access information for env variables
url = "http://" + os.environ["HOST"]
key = os.environ["AUTH"]
urllib3.disable_warnings()
def create_simple_event():
mispevent = MISPEvent()
mispevent.info = 'This is a super simple test'
mispevent.distribution = Distribution.your_organisation_only
mispevent.threat_level_id = ThreatLevel.low
mispevent.analysis = Analysis.completed
mispevent.add_attribute('text', str(uuid.uuid4()))
return mispevent
def check_response(response):
if isinstance(response, dict) and "errors" in response:
raise Exception(response["errors"])
class TestComprehensive(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.maxDiff = None
# Connect as admin
cls.admin_misp_connector = PyMISP(url, key, ssl=False, debug=False)
cls.admin_misp_connector.set_server_setting('debug', 1, force=True)
# Creates an org
organisation = MISPOrganisation()
organisation.name = 'Test Org'
cls.test_org = cls.admin_misp_connector.add_organisation(organisation, pythonify=True)
# Set the refault role (id 3 on the VM)
cls.admin_misp_connector.set_default_role(3)
# Creates a user
user = MISPUser()
user.email = 'testusr@user.local'
user.org_id = cls.test_org.id
cls.test_usr = cls.admin_misp_connector.add_user(user, pythonify=True)
cls.user_misp_connector = PyMISP(url, cls.test_usr.authkey, ssl=False, debug=True)
@classmethod
def tearDownClass(cls):
# Delete user
cls.admin_misp_connector.delete_user(cls.test_usr)
# Delete org
cls.admin_misp_connector.delete_organisation(cls.test_org)
def setUp(self):
self.user_misp_connector.global_pythonify = True
self.admin_misp_connector.global_pythonify = True
def test_search_index(self):
# Search all events
index = self.user_misp_connector.search_index()
self.assertGreater(len(index), 0)
# Search published
index_published = self.user_misp_connector.search_index(published=True)
self.assertEqual(len(index_published), 0, "No event should be published.")
# Create test event
event = create_simple_event()
event = self.user_misp_connector.add_event(event)
check_response(event)
# Search by org name
index_org = self.user_misp_connector.search_index(org="Test Org")
self.assertGreater(len(index_org), 0)
# Search by org name with different case
index_org_lower = self.user_misp_connector.search_index(org="test org")
self.assertGreater(len(index_org_lower), 0)
# Search by org uuid
index_org_uuid = self.user_misp_connector.search_index(org=self.test_org.uuid)
self.assertGreater(len(index_org_uuid), 0)
# Search by org ID
index_org_id = self.user_misp_connector.search_index(org=self.test_org.id)
self.assertGreater(len(index_org_id), 0)
self.assertEqual(len(index_org), len(index_org_lower))
self.assertEqual(len(index_org), len(index_org_uuid))
self.assertEqual(len(index_org), len(index_org_id))
self.user_misp_connector.delete_event(event)
def test_search_index_by_info(self):
event = create_simple_event()
event.info = uuid.uuid4()
# No event should exists
index = self.user_misp_connector.search_index(eventinfo=event.info)
self.assertEqual(len(index), 0, "No event should exists")
event = self.user_misp_connector.add_event(event)
check_response(event)
# One event should exists
index = self.user_misp_connector.search_index(eventinfo=event.info)
self.assertEqual(len(index), 1)
self.assertEqual(index[0].uuid, event.uuid)
index = self.user_misp_connector.search_index(eventinfo="!" + str(event.info))
for index_event in index:
self.assertNotEqual(event.uuid, index_event.uuid, index)
self.user_misp_connector.delete_event(event)
def test_search_index_by_all(self):
event = create_simple_event()
index = self.user_misp_connector.search_index(all=event.attributes[0].value)
self.assertEqual(len(index), 0, "No event should exists")
event = self.user_misp_connector.add_event(event)
check_response(event)
index = self.user_misp_connector.search_index(all=event.attributes[0].value)
self.assertEqual(len(index), 1, "One event should exists")
self.assertEqual(index[0].uuid, event.uuid)
index = self.user_misp_connector.search_index(all=event.attributes[0].value.upper())
self.assertEqual(len(index), 1, "One event should exists")
self.assertEqual(index[0].uuid, event.uuid)
self.user_misp_connector.delete_event(event)
def test_search_index_by_attribute(self):
event = create_simple_event()
index = self.user_misp_connector.search_index(attribute=event.attributes[0].value)
self.assertEqual(len(index), 0, "No event should exists")
event = self.user_misp_connector.add_event(event)
check_response(event)
index = self.user_misp_connector.search_index(attribute=event.attributes[0].value)
self.assertEqual(len(index), 1, "One event should exists")
self.assertEqual(index[0].uuid, event.uuid)
index = self.user_misp_connector.search_index(attribute=event.attributes[0].value.upper())
self.assertEqual(len(index), 1, "One event should exists")
self.assertEqual(index[0].uuid, event.uuid)
self.user_misp_connector.delete_event(event)
def test_search_index_by_tag(self):
tags = self.user_misp_connector.search_tags("tlp:red", True)
index = self.user_misp_connector.search_index(tags="tlp:red")
self.assertEqual(len(index), 0, "No event should exists")
index = self.user_misp_connector.search_index(tags=tags[0].id)
self.assertEqual(len(index), 0, "No event should exists")
event = create_simple_event()
event.add_tag("tlp:red")
event = self.user_misp_connector.add_event(event)
check_response(event)
index = self.user_misp_connector.search_index(tags="tlp:red")
self.assertEqual(len(index), 1, "One event should exists")
index = self.user_misp_connector.search_index(tags="tlp:red|not_exists")
self.assertEqual(len(index), 1, "One event should exists")
index = self.user_misp_connector.search_index(tags=["tlp:red", "not_exists"])
self.assertEqual(len(index), 1, "One event should exists")
index = self.user_misp_connector.search_index(tags=tags[0].id)
self.assertEqual(len(index), 1, "One event should exists")
index = self.user_misp_connector.search_index(tags="!tlp:red")
for index_event in index:
self.assertNotEqual(event.uuid, index_event.uuid, index)
index = self.user_misp_connector.search_index(tags="!" + str(tags[0].id))
for index_event in index:
self.assertNotEqual(event.uuid, index_event.uuid, index)
self.user_misp_connector.delete_event(event)
def test_search_index_by_email(self):
index = self.user_misp_connector.search_index(email=self.test_usr.email)
self.assertEqual(len(index), 0, index)
event = create_simple_event()
event = self.user_misp_connector.add_event(event)
check_response(event)
index = self.user_misp_connector.search_index(email=self.test_usr.email)
self.assertEqual(len(index), 1, "One event should exists")
self.user_misp_connector.delete_event(event)
def test_search_index_by_email_admin(self):
index = self.admin_misp_connector.search_index(email="no_existing_exmail@example.com")
self.assertEqual(len(index), 0, index)
index = self.admin_misp_connector.search_index(email=self.test_usr.email)
self.assertEqual(len(index), 0, index)
event = create_simple_event()
event = self.user_misp_connector.add_event(event)
check_response(event)
index = self.admin_misp_connector.search_index(email=self.test_usr.email)
self.assertEqual(len(index), 1, index)
# Search by partial match
index = self.admin_misp_connector.search_index(email="testusr@user")
self.assertEqual(len(index), 1, index)
self.user_misp_connector.delete_event(event)
def test_search_index_minimal(self):
# pythonify is not supported for minimal results
self.user_misp_connector.global_pythonify = False
minimal = self.user_misp_connector.search_index(minimal=True)
self.assertGreater(len(minimal), 0)
minimal_event = minimal[0]
self.assertIn("id", minimal_event)
self.assertIn("timestamp", minimal_event)
self.assertIn("sighting_timestamp", minimal_event)
self.assertIn("published", minimal_event)
self.assertIn("uuid", minimal_event)
self.assertIn("orgc_uuid", minimal_event)
for event in minimal:
self.assertFalse(event["published"], "No event should be published.")
def test_search_index_minimal_published(self):
# pythonify is not supported for minimal results
self.user_misp_connector.global_pythonify = False
index = self.user_misp_connector.search_index(minimal=True, published=True)
self.assertEqual(len(index), 0, "No event should be published.")
index = self.user_misp_connector.search_index(minimal=True)
not_published = self.user_misp_connector.search_index(minimal=True, published=0)
both_2 = self.user_misp_connector.search_index(minimal=True, published=2)
both_array = self.user_misp_connector.search_index(minimal=True, published=[0, 1])
self.assertEqual(len(index), len(not_published))
self.assertEqual(len(index), len(both_2))
self.assertEqual(len(index), len(both_array))
def test_search_index_minimal_by_org(self):
# pythonify is not supported for minimal results
self.user_misp_connector.global_pythonify = False
# Create test event
event = create_simple_event()
event = self.user_misp_connector.add_event(event, pythonify=True)
check_response(event)
# Search by org name
minimal_org = self.user_misp_connector.search_index(minimal=True, org="Test Org")
self.assertGreater(len(minimal_org), 0)
for event in minimal_org:
self.assertEqual(event["orgc_uuid"], self.test_org.uuid)
# Search by org name with different case
minimal_org_lower = self.user_misp_connector.search_index(minimal=True, org="test org")
self.assertGreater(len(minimal_org), 0)
for event in minimal_org:
self.assertEqual(event["orgc_uuid"], self.test_org.uuid)
# Search by non exists org name
minimal_org_non_existing = self.user_misp_connector.search_index(minimal=True, org="Test Org that doesn't exists")
self.assertEqual(len(minimal_org_non_existing), 0)
# Search by org uuid
minimal_org_uuid = self.user_misp_connector.search_index(minimal=True, org=self.test_org.uuid)
self.assertGreater(len(minimal_org), 0)
for event in minimal_org:
self.assertEqual(event["orgc_uuid"], self.test_org.uuid)
# Search by non existing uuid
minimal_org_uuid_non_existing = self.user_misp_connector.search_index(minimal=True, org=uuid.uuid4())
self.assertEqual(len(minimal_org_uuid_non_existing), 0)
# Search by org ID
minimal_org_id = self.user_misp_connector.search_index(minimal=True, org=self.test_org.id)
self.assertGreater(len(minimal_org), 0)
for event in minimal_org:
self.assertEqual(event["orgc_uuid"], self.test_org.uuid)
self.assertEqual(len(minimal_org), len(minimal_org_lower))
self.assertEqual(len(minimal_org), len(minimal_org_uuid))
self.assertEqual(len(minimal_org), len(minimal_org_id))
# Search not by org
minimal_org_not = self.user_misp_connector.search_index(minimal=True, org="!Test Org")
for event in minimal_org_not:
self.assertNotEqual(event["orgc_uuid"], self.test_org.uuid)
minimal_org_lower_not = self.user_misp_connector.search_index(minimal=True, org="!test org")
for event in minimal_org_lower_not:
self.assertNotEqual(event["orgc_uuid"], self.test_org.uuid)
minimal_org_uuid_not = self.user_misp_connector.search_index(minimal=True, org="!" + self.test_org.uuid)
for event in minimal_org_uuid_not:
self.assertNotEqual(event["orgc_uuid"], self.test_org.uuid)
minimal_org_id_not = self.user_misp_connector.search_index(minimal=True, org="!" + self.test_org.id)
for event in minimal_org_id_not:
self.assertNotEqual(event["orgc_uuid"], self.test_org.uuid)
self.assertEqual(len(minimal_org_not), len(minimal_org_lower_not))
self.assertEqual(len(minimal_org_not), len(minimal_org_uuid_not))
self.assertEqual(len(minimal_org_not), len(minimal_org_id_not))
self.user_misp_connector.delete_event(event)
def test_delete_event_blocklist(self):
check_response(self.admin_misp_connector.set_server_setting('MISP.enableEventBlocklisting', 1))
# Create test event
event = create_simple_event()
event = self.user_misp_connector.add_event(event)
check_response(event)
# Delete event
check_response(self.user_misp_connector.delete_event(event))
check_response(self.admin_misp_connector.set_server_setting('MISP.enableEventBlocklisting', 0))
def test_deleted_attributes(self):
# Create test event
event = create_simple_event()
event.add_attribute('text', "deleted", deleted=True)
event.add_attribute('text', "not-deleted")
event = self.user_misp_connector.add_event(event)
check_response(event)
# Not deleted
fetched_event = self.user_misp_connector.get_event(event)
check_response(fetched_event)
self.assertEqual(len(fetched_event.attributes), 2, fetched_event)
# Not deleted
fetched_event = self.user_misp_connector.get_event(event, deleted=0)
check_response(fetched_event)
self.assertEqual(len(fetched_event.attributes), 2, fetched_event)
# Include deleted
fetched_event = self.user_misp_connector.get_event(event, deleted=1)
check_response(fetched_event)
self.assertEqual(len(fetched_event.attributes), 3, fetched_event)
# Deleted only
fetched_event = self.user_misp_connector.get_event(event, deleted=2)
check_response(fetched_event)
self.assertEqual(len(fetched_event.attributes), 1, fetched_event)
# Both
fetched_event = self.user_misp_connector.get_event(event, deleted=[0, 1])
check_response(fetched_event)
self.assertEqual(len(fetched_event.attributes), 3, fetched_event)
check_response(self.user_misp_connector.delete_event(event))
def test_view_event_exclude_local_tags(self):
event = create_simple_event()
event.add_tag({"name": "local", "local": 1})
event.add_tag({"name": "global", "local": 0})
event.attributes[0].add_tag({"name": "local", "local": 1})
event.attributes[0].add_tag({"name": "global", "local": 0})
event = self.admin_misp_connector.add_event(event)
check_response(event)
event_with_local_tags = self.admin_misp_connector.get_event(event)
check_response(event_with_local_tags)
self.assertEqual(len(event_with_local_tags.tags), 2)
self.assertEqual(len(event_with_local_tags.attributes[0].tags), 2)
event_without_local_tags = self.admin_misp_connector._check_json_response(self.admin_misp_connector._prepare_request('GET', f'events/view/{event.id}/excludeLocalTags:1'))
check_response(event_without_local_tags)
self.assertEqual(event_without_local_tags["Event"]["Tag"][0]["local"], 0, event_without_local_tags)
self.assertEqual(event_without_local_tags["Event"]["Attribute"][0]["Tag"][0]["local"], 0, event_without_local_tags)
check_response(self.admin_misp_connector.delete_event(event))
if __name__ == '__main__':
unittest.main()

8
tests/testlive_security.py
View File

@ -268,14 +268,14 @@ class TestSecurity(unittest.TestCase):
def test_user_must_change_password(self):
updated_user = self.admin_misp_connector.update_user({'change_pw': 1}, self.test_usr)
check_response(updated_user)
self.assertEqual(updated_user.change_pw, "1")
self.assertTrue(updated_user.change_pw)
# Try to login, should still work because key is still valid
PyMISP(url, self.test_usr.authkey)
updated_user = self.admin_misp_connector.update_user({'change_pw': 0}, self.test_usr)
check_response(updated_user)
self.assertEqual(updated_user.change_pw, "0")
self.assertFalse(updated_user.change_pw)
# Try to login, should also still works
PyMISP(url, self.test_usr.authkey)
@ -284,7 +284,7 @@ class TestSecurity(unittest.TestCase):
# Admin set that user must change password
updated_user = self.admin_misp_connector.update_user({'change_pw': 1}, self.test_usr)
check_response(updated_user)
self.assertEqual(updated_user.change_pw, "1")
self.assertTrue(updated_user.change_pw)
# User try to change back trough API
logged_in = PyMISP(url, self.test_usr.authkey)
@ -292,7 +292,7 @@ class TestSecurity(unittest.TestCase):
updated_user = self.admin_misp_connector.get_user(self.test_usr)
# Should not be possible
self.assertEqual(updated_user.change_pw, "1")
self.assertTrue(updated_user.change_pw)
def test_disabled_user(self):
# Disable user

21
tests/testlive_sync.py
View File

@ -1,5 +1,5 @@
import os
from pymisp import PyMISP, MISPEvent
from pymisp import PyMISP, MISPEvent, MISPGalaxyCluster
def check_response(response):
@ -53,6 +53,24 @@ check_response(event)
# Publish that event
check_response(pymisp.publish(event))
# Publish event inline
url = f'events/publish/{event.id}/disable_background_processing:1'
push_event = pymisp._check_json_response(pymisp._prepare_request('POST', url))
check_response(push_event)
# Create testing galaxy cluster
galaxy = pymisp.galaxies()[0]
galaxy_cluster = MISPGalaxyCluster()
galaxy_cluster.value = "Test Cluster"
galaxy_cluster.authors = ["MISP"]
galaxy_cluster.distribution = 1
galaxy_cluster.description = "Example test cluster"
galaxy_cluster = pymisp.add_galaxy_cluster(galaxy.id, galaxy_cluster)
check_response(galaxy_cluster)
# Publish that galaxy cluster
check_response(pymisp.publish_galaxy_cluster(galaxy_cluster))
# Preview event
url = f'servers/previewEvent/{remote_server["id"]}/{event.uuid}'
event_preview = pymisp._check_json_response(pymisp._prepare_request('GET', url))
@ -92,3 +110,4 @@ check_response(rules_response)
check_response(pymisp.delete_server(remote_server))
check_response(pymisp.delete_event(event))
check_response(pymisp.delete_event_blocklist(event))
check_response(pymisp.delete_galaxy_cluster(galaxy_cluster))