MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'develop' of https://github.com/MISP/MISP into develop

pull/8053/head
chrisr3d 2021-12-20 12:05:42 +01:00
parent 7ebf3818d3 b2343fc95d
commit f796071d64
No known key found for this signature in database
GPG Key ID: 6BBED1B63A6D639F
83 changed files with 1784 additions and 957 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/Config/bootstrap.default.php
View File

@ -132,7 +132,6 @@ if (Configure::read('MISP.baseurl')) {
CakePlugin::load('SysLog');
CakePlugin::load('Assets'); // having Logable
CakePlugin::load('SysLogLogable');
CakePlugin::load('UrlCache');
/**
* Uncomment the following line to enable client SSL certificate authentication.

1
app/Config/config.default.php
View File

@ -104,6 +104,7 @@ $config = array(
'user_email_notification_ban_amount_threshold' => 10,
'user_email_notification_ban_refresh_on_retry' => true,
'warning_for_all' => true,
'enable_synchronisation_filtering_on_type' => false,
),
'GnuPG' => array(
'onlyencrypted' => false,

37
app/Console/Command/AdminShell.php
View File

@ -1,5 +1,6 @@
<?php
App::uses('AppShell', 'Console/Command');
App::uses('ProcessTool', 'Tools');
/**
* @property Server $Server
@ -125,13 +126,6 @@ class AdminShell extends AppShell
echo $this->Server->update($status) . PHP_EOL;
}
public function restartWorkers()
{
$this->ConfigLoad->execute();
$this->Server->restartWorkers();
echo PHP_EOL . 'Workers restarted.' . PHP_EOL;
}
public function updateAfterPull()
{
$this->ConfigLoad->execute();
@ -155,8 +149,23 @@ class AdminShell extends AppShell
}
}
public function restartWorkers()
{
if (Configure::read('SimpleBackgroundJobs.enabled')) {
$this->error('This method does nothing when SimpleBackgroundJobs are enabled.');
}
$this->ConfigLoad->execute();
$this->Server->restartWorkers();
echo PHP_EOL . 'Workers restarted.' . PHP_EOL;
}
public function restartWorker()
{
if (Configure::read('SimpleBackgroundJobs.enabled')) {
$this->error('This method does nothing when SimpleBackgroundJobs are enabled.');
}
$this->ConfigLoad->execute();
if (empty($this->args[0]) || !is_numeric($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['worker_management_tasks']['data']['Restart a worker'] . PHP_EOL);
@ -179,6 +188,10 @@ class AdminShell extends AppShell
public function killWorker()
{
if (Configure::read('SimpleBackgroundJobs.enabled')) {
$this->error('This method does nothing when SimpleBackgroundJobs are enabled.');
}
$this->ConfigLoad->execute();
if (empty($this->args[0]) || !is_numeric($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['worker_management_tasks']['data']['Kill a worker'] . PHP_EOL);
@ -196,6 +209,10 @@ class AdminShell extends AppShell
public function startWorker()
{
if (Configure::read('SimpleBackgroundJobs.enabled')) {
$this->error('This method does nothing when SimpleBackgroundJobs are enabled.');
}
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['worker_management_tasks']['data']['Start a worker'] . PHP_EOL);
@ -462,11 +479,7 @@ class AdminShell extends AppShell
public function runUpdates()
{
if (function_exists('posix_getpwuid') && function_exists('posix_geteuid')) {
$whoami = posix_getpwuid(posix_geteuid())['name'];
} else {
$whoami = exec('whoami');
}
$whoami = ProcessTool::whoami();
if (in_array($whoami, ['httpd', 'www-data', 'apache', 'wwwrun', 'travis', 'www'], true) || $whoami === Configure::read('MISP.osuser')) {
$this->out('Executing all updates to bring the database up to date with the current version.');
$processId = empty($this->args[0]) ? false : $this->args[0];

30
app/Console/Command/ServerShell.php
View File

@ -79,16 +79,19 @@ class ServerShell extends AppShell
));
foreach ($servers as $serverId => $serverName) {
$backgroundJobId = $this->Server->getBackgroundJobsTool()->enqueue(
$jobId = $this->Job->createJob($user, Job::WORKER_DEFAULT, 'pull', "Server: $serverId", 'Pulling.');
$backgroundJobId = $this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_SERVER,
[
'pull',
$user['id'],
$serverId,
$technique
]
$technique,
$jobId,
],
true,
$jobId
);
$this->out("Enqueued pulling from $serverName server as job $backgroundJobId");
@ -189,8 +192,7 @@ class ServerShell extends AppShell
));
foreach ($servers as $serverId => $serverName) {
$jobId = $this->Server->getBackgroundJobsTool()->enqueue(
$jobId = $this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_SERVER,
[
@ -309,8 +311,7 @@ class ServerShell extends AppShell
));
foreach ($servers as $serverId => $serverName) {
$jobId = $this->Server->getBackgroundJobsTool()->enqueue(
$jobId = $this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::DEFAULT_QUEUE,
BackgroundJobsTool::CMD_SERVER,
[
@ -320,7 +321,7 @@ class ServerShell extends AppShell
]
);
$this->out("Enqueued cacheServer from {$serverName} server as job $jobId");
$this->out("Enqueued cacheServer from $serverName server as job $jobId");
}
}
@ -605,4 +606,15 @@ class ServerShell extends AppShell
}
return $server;
}
/**
* @return BackgroundJobsTool
*/
private function getBackgroundJobsTool()
{
if (!$this->BackgroundJobsTool) {
$this->BackgroundJobsTool = new BackgroundJobsTool(Configure::read('SimpleBackgroundJobs'));
}
return $this->BackgroundJobsTool;
}
}

49
app/Console/Command/StartWorkerShell.php
View File

@ -3,6 +3,7 @@
declare(strict_types=1);
App::uses('BackgroundJobsTool', 'Tools');
App::uses('ProcessTool', 'Tools');
class StartWorkerShell extends AppShell
{
@ -50,7 +51,7 @@ class StartWorkerShell extends AppShell
[
'pid' => getmypid(),
'queue' => $this->args[0],
'user' => $this->whoami()
'user' => ProcessTool::whoami(),
]
);
@ -62,21 +63,38 @@ class StartWorkerShell extends AppShell
$this->checkMaxExecutionTime();
$job = $this->BackgroundJobsTool->dequeue($this->worker->queue());
if ($job) {
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}...");
try {
$this->BackgroundJobsTool->run($job);
} catch (Exception $exception) {
CakeLog::error("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - job ID: {$job->id()} failed with exception: {$exception->getMessage()}");
$job->setStatus(BackgroundJob::STATUS_FAILED);
$this->BackgroundJobsTool->update($job);
}
$this->runJob($job);
}
}
}
/**
* @param BackgroundJob $job
*/
private function runJob(BackgroundJob $job)
{
CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}...");
try {
$job->setStatus(BackgroundJob::STATUS_RUNNING);
CakeLog::info("[JOB ID: {$job->id()}] - started.");
$this->BackgroundJobsTool->update($job);
$job->run();
if ($job->status() === BackgroundJob::STATUS_COMPLETED) {
CakeLog::info("[JOB ID: {$job->id()}] - completed.");
} else {
CakeLog::error("[JOB ID: {$job->id()}] - failed with error code {$job->returnCode()}. STDERR: {$job->error()}. STDOUT: {$job->output()}.");
}
} catch (Exception $exception) {
CakeLog::error("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - job ID: {$job->id()} failed with exception: {$exception->getMessage()}");
$job->setStatus(BackgroundJob::STATUS_FAILED);
}
$this->BackgroundJobsTool->update($job);
}
/**
* Checks if worker maximum execution time is reached, and exits if so.
*
@ -92,13 +110,4 @@ class StartWorkerShell extends AppShell
exit;
}
}
private function whoami(): string
{
if (function_exists('posix_getpwuid') && function_exists('posix_geteuid')) {
return posix_getpwuid(posix_geteuid())['name'];
} else {
return trim(shell_exec('whoami'));
}
}
}

88
app/Console/Command/UserShell.php
View File

@ -19,6 +19,17 @@ class UserShell extends AppShell
],
]
]);
$parser->addSubcommand('authkey', [
'help' => __('Get information about given authkey.'),
'parser' => [
'arguments' => [
'authkey' => ['help' => __('Authentication key. If not provide, it will be read from STDIN.')],
],
]
]);
$parser->addSubcommand('authkey_valid', [
'help' => __('Check if given authkey by STDIN is valid.'),
]);
$parser->addSubcommand('block', [
'help' => __('Immediately block user.'),
'parser' => [
@ -111,6 +122,83 @@ class UserShell extends AppShell
}
}
public function authkey()
{
if (isset($this->args[0])) {
$authkey = $this->args[0];
} else {
$authkey = fgets(STDIN); // read line from STDIN
}
$authkey = trim($authkey);
if (strlen($authkey) !== 40) {
$this->error('Authkey has not valid format.');
}
if (Configure::read('Security.advanced_authkeys')) {
$user = $this->User->AuthKey->getAuthUserByAuthKey($authkey, true);
if (empty($user)) {
$this->error("Given authkey doesn't belong to any user.");
}
$isExpired = $user['authkey_expiration'] && $user['authkey_expiration'] < time();
$this->out($this->json([
'user_id' => $user['id'],
'email' => $user['email'],
'org_id' => $user['org_id'],
'authkey_id' => $user['authkey_id'],
'authkey_expiration' => $user['authkey_expiration'],
'authkey_expired' => $isExpired,
'allowed_ips' => $user['allowed_ips'],
'authkey_read_only' => $user['authkey_read_only'],
]));
$this->_stop($isExpired ? 2 : 0);
} else {
$user = $this->User->getAuthUserByAuthkey($authkey);
if (empty($user)) {
$this->error("Given authkey doesn't belong to any user.");
}
$this->out($this->json([
'user_id' => $user['id'],
'email' => $user['email'],
'org_id' => $user['org_id'],
]));
}
}
/**
* Reads line from stdin and checks if authkey is valid. Returns '1' to stdout if key is valid and '0' if not.
*/
public function authkey_valid()
{
$cache = [];
do {
$authkey = fgets(STDIN); // read line from STDIN
$authkey = trim($authkey);
if (strlen($authkey) !== 40) {
fwrite(STDOUT, "0\n"); // authkey is not in valid format
continue;
}
$time = time();
// Generate hash from authkey to not store raw authkey in memory
$keyHash = hash('sha256', $authkey, true);
if (isset($cache[$keyHash]) && $cache[$keyHash][1] > $time) {
fwrite(STDOUT, $cache[$keyHash][0] ? "1\n" : "0\n");
continue;
}
if (Configure::read('Security.advanced_authkeys')) {
$user = $this->User->AuthKey->getAuthUserByAuthKey($authkey);
} else {
$user = $this->User->getAuthUserByAuthkey($authkey);
}
$user = (bool)$user;
// Cache results for 5 seconds
$cache[$keyHash] = [$user, $time + 5];
fwrite(STDOUT, $user ? "1\n" : "0\n");
} while (true);
}
public function block()
{
list($userId) = $this->args;

7
app/Console/cake.php
View File

@ -23,13 +23,14 @@ if (!defined('DS')) {
$dispatcher = 'Cake' . DS . 'Console' . DS . 'ShellDispatcher.php';
if (function_exists('ini_set')) {
$root = dirname(dirname(dirname(__FILE__)));
$appDir = basename(dirname(dirname(__FILE__)));
$install = $root . DS . $appDir . DS . 'Lib' . DS . 'cakephp' . DS . 'lib';
$root = dirname(__DIR__, 2);
$appDir = basename(dirname(__DIR__));
$composerInstall = $root . DS . $appDir . DS . 'Vendor' . DS . 'cakephp' . DS . 'cakephp' . DS . 'lib';
if (file_exists($composerInstall . DS . $dispatcher)) {
$install = $composerInstall; // prefer compose install
} else {
$install = $root . DS . $appDir . DS . 'Lib' . DS . 'cakephp' . DS . 'lib';
}
ini_set('include_path', $install . PATH_SEPARATOR . ini_get('include_path'));

23
app/Controller/AppController.php
View File

@ -33,7 +33,7 @@ class AppController extends Controller
public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
private $__queryVersion = '131';
private $__queryVersion = '135';
public $pyMispVersion = '2.4.151';
public $phpmin = '7.2';
public $phprec = '7.4';
@ -180,7 +180,7 @@ class AppController extends Controller
Configure::write('CurrentController', $this->request->params['controller']);
Configure::write('CurrentAction', $this->request->params['action']);
$versionArray = $this->User->checkMISPVersion();
$this->mispVersion = implode('.', array_values($versionArray));
$this->mispVersion = implode('.', $versionArray);
$this->Security->blackHoleCallback = 'blackHole';
// send users away that are using ancient versions of IE
@ -368,9 +368,13 @@ class AppController extends Controller
}
}
}
}
public function beforeRender()
{
// Notifications and homepage is not necessary for AJAX or REST requests
if ($user && !$this->_isRest() && !$isAjax) {
$user = $this->Auth->user();
if ($user && !$this->_isRest() && !$this->request->is('ajax')) {
$hasNotifications = $this->User->hasNotifications($user);
$this->set('hasNotifications', $hasNotifications);
@ -1409,4 +1413,17 @@ class AppController extends Controller
return true;
}
}
/**
* Override default View class
* @return View
*/
protected function _getViewObject()
{
if ($this->viewClass === 'View') {
App::uses('AppView', 'View');
return new AppView($this);
}
return parent::_getViewObject();
}
}

74
app/Controller/AttributesController.php
View File

@ -315,28 +315,27 @@ class AttributesController extends AppController
if (empty($attributes)) {
throw new UnauthorizedException(__('Attribute does not exists or you do not have the permission to download this attribute.'));
}
$this->__downloadAttachment($attributes[0]['Attribute']);
return $this->__downloadAttachment($attributes[0]['Attribute']);
}
private function __downloadAttachment($attribute)
private function __downloadAttachment(array $attribute)
{
$file = $this->Attribute->getAttachmentFile($attribute);
if ('attachment' == $attribute['type']) {
if ('attachment' === $attribute['type']) {
$filename = $attribute['value'];
$fileExt = pathinfo($filename, PATHINFO_EXTENSION);
$filename = substr($filename, 0, strlen($filename) - strlen($fileExt) - 1);
} elseif ('malware-sample' == $attribute['type']) {
} elseif ('malware-sample' === $attribute['type']) {
$filenameHash = explode('|', $attribute['value']);
$filename = substr($filenameHash[0], strrpos($filenameHash[0], '\\'));
$fileExt = "zip";
} else {
throw new NotFoundException(__('Attribute not an attachment or malware-sample'));
}
$this->autoRender = false;
$this->response->type($fileExt);
$download_attachments_on_load = Configure::check('MISP.download_attachments_on_load') ? Configure::read('MISP.download_attachments_on_load') : true;
$this->response->file($file->path, array('download' => $download_attachments_on_load, 'name' => $filename . '.' . $fileExt));
return $this->RestResponse->sendFile($file, $fileExt, $download_attachments_on_load, $filename . '.' . $fileExt);
}
public function add_attachment($eventId = null)
@ -998,37 +997,50 @@ class AttributesController extends AppController
{
$conditions = $this->__idToConditions($id);
$conditions['Attribute.type'] = 'attachment';
$options = array(
'conditions' => $conditions,
'includeAllTags' => false,
'includeAttributeUuid' => true,
'flatten' => true,
'deleted' => [0, 1]
);
if ($this->_isRest()) {
$options['withAttachments'] = true;
$options = array(
'conditions' => $conditions,
'includeAllTags' => false,
'includeAttributeUuid' => true,
'flatten' => true,
'deleted' => [0, 1],
'withAttachments' => true,
);
$attribute = $this->Attribute->fetchAttributes($this->Auth->user(), $options);
if (empty($attribute)) {
throw new MethodNotAllowedException('Invalid attribute');
}
$attribute = $attribute[0];
if (!$this->Attribute->isImage($attribute['Attribute'])) {
throw new NotFoundException("Attribute is not an image.");
}
return $this->RestResponse->viewData($attribute['Attribute']['data'], $this->response->type());
}
$attribute = $this->Attribute->fetchAttributes($this->Auth->user(), $options);
$attribute = $this->Attribute->fetchAttributeSimple($this->Auth->user(), [
'conditions' => $conditions,
'fields' => ['Attribute.id', 'Attribute.event_id', 'Attribute.type', 'Attribute.value'],
]);
if (empty($attribute)) {
throw new MethodNotAllowedException('Invalid attribute');
}
$attribute = $attribute[0];
if (!$this->Attribute->isImage($attribute['Attribute'])) {
throw new NotFoundException("Attribute is not an image.");
}
if ($this->_isRest()) {
return $this->RestResponse->viewData($attribute['Attribute']['data'], $this->response->type());
} else {
$width = isset($this->request->params['named']['width']) ? $this->request->params['named']['width'] : 200;
$height = isset($this->request->params['named']['height']) ? $this->request->params['named']['height'] : 200;
$imageData = $this->Attribute->getPictureData($attribute, $thumbnail, $width, $height);
$extension = pathinfo($attribute['Attribute']['value'], PATHINFO_EXTENSION);
return new CakeResponse(array('body' => $imageData, 'type' => strtolower($extension)));
$width = isset($this->request->params['named']['width']) ? $this->request->params['named']['width'] : 200;
$height = isset($this->request->params['named']['height']) ? $this->request->params['named']['height'] : 200;
$extension = pathinfo($attribute['Attribute']['value'], PATHINFO_EXTENSION);
$imageData = $this->Attribute->getPictureData($attribute, $thumbnail, $width, $height);
if ($imageData instanceof File) {
return $this->RestResponse->sendFile($imageData, strtolower($extension));
}
$this->response->body($imageData);
$this->response->type(strtolower($extension));
return $this->response;
}
public function delete($id, $hard = false)
@ -1397,7 +1409,7 @@ class AttributesController extends AppController
|| ($clusters_ids_remove === null || count($clusters_ids_remove) > 0)
|| ($clusters_ids_add === null || count($clusters_ids_add) > 0);
$changeInAttribute = ($this->request->data['Attribute']['to_ids'] != 2) || ($this->request->data['Attribute']['distribution'] != 6) || ($this->request->data['Attribute']['comment'] != null);
$changeInAttribute = ($this->request->data['Attribute']['to_ids'] != 2) || ($this->request->data['Attribute']['distribution'] != 6) || ($this->request->data['Attribute']['comment'] != null) || ($this->request->data['Attribute']['disable_correlation'] != 2);
if (!$changeInAttribute && !$changeInTagOrCluster) {
return new CakeResponse(array('body'=> json_encode(array('saved' => true)), 'status' => 200, 'type' => 'json'));
@ -1435,6 +1447,12 @@ class AttributesController extends AppController
}
}
if ($this->request->data['Attribute']['disable_correlation'] != 2) {
foreach ($attributes as $key => $attribute) {
$attributes[$key]['Attribute']['disable_correlation'] = $this->request->data['Attribute']['disable_correlation'] === '0' ? false : true;
}
}
$date = new DateTime();
$timestamp = $date->getTimestamp();
foreach ($attributes as $key => $attribute) {
@ -1753,7 +1771,7 @@ class AttributesController extends AppController
if (empty($attributes)) {
throw new UnauthorizedException(__('Attribute does not exists or you do not have the permission to download this attribute.'));
}
$this->__downloadAttachment($attributes[0]['Attribute']);
return $this->__downloadAttachment($attributes[0]['Attribute']);
}
// returns an XML with attributes that belong to an event. The type of attributes to be returned can be restricted by type using the 3rd parameter.

14
app/Controller/Component/ACLComponent.php
View File

@ -12,7 +12,7 @@ class ACLComponent extends Component
// $action == array('OR' => array()) - any role in the array has access
// $action == array('AND' => array()) - roles with all permissions in the array have access
// If we add any new functionality to MISP and we don't add it to this list, it will only be visible to site admins.
private $__aclList = array(
const ACL_LIST = array(
'*' => array(
'blackhole' => array(),
'debugACL' => array(),
@ -906,10 +906,10 @@ class ACLComponent extends Component
if ($user && $user['Role']['perm_site_admin']) {
return true;
}
if (!isset($this->__aclList[$controller])) {
if (!isset(self::ACL_LIST[$controller])) {
throw new NotFoundException('Invalid controller.');
}
$controllerAclList = array_change_key_case($this->__aclList[$controller]);
$controllerAclList = array_change_key_case(self::ACL_LIST[$controller]);
if (!empty($controllerAclList[$action])) {
$rules = $controllerAclList[$action];
if (in_array('*', $rules, true)) {
@ -970,7 +970,7 @@ class ACLComponent extends Component
$fileContents = preg_replace('/\/\*[^\*]+?\*\//', '', $fileContents);
preg_match_all($functionFinder, $fileContents, $functionArray);
foreach ($functionArray[1] as $function) {
if (substr($function, 0, 1) !== '_' && $function !== 'beforeFilter' && $function !== 'afterFilter') {
if ($function[0] !== '_' && $function !== 'beforeFilter' && $function !== 'afterFilter' && $function !== 'beforeRender') {
$results[$controllerName][] = $function;
}
}
@ -991,8 +991,8 @@ class ACLComponent extends Component
$missing = array();
foreach ($results as $controller => $functions) {
foreach ($functions as $function) {
if (!isset($this->__aclList[$controller])
|| !in_array($function, array_keys($this->__aclList[$controller]))) {
if (!isset(self::ACL_LIST[$controller])
|| !in_array($function, array_keys(self::ACL_LIST[$controller]))) {
$missing[$controller][] = $function;
}
}
@ -1026,7 +1026,7 @@ class ACLComponent extends Component
{
$result = array();
$fakeUser = ['Role' => $role, 'org_id' => Configure::read('MISP.host_org_id')];
foreach ($this->__aclList as $controller => $actions) {
foreach (self::ACL_LIST as $controller => $actions) {
$controllerNames = Inflector::variable($controller) === Inflector::underscore($controller) ?
array(Inflector::variable($controller)) :
array(Inflector::variable($controller), Inflector::underscore($controller));

35
app/Controller/Component/RestResponseComponent.php
View File

@ -9,7 +9,7 @@ class RestResponseComponent extends Component
public $headers = array();
private $__convertActionToMessage = array(
const CONVERT_ACTION_TO_MESSAGE = array(
'SharingGroup' => array(
'addOrg' => 'add Organisation to',
'removeOrg' => 'remove Organisation from',
@ -473,8 +473,8 @@ class RestResponseComponent extends Component
$response = array();
$action = $this->__dissectAdminRouting($action);
$stringifiedAction = $action['action'];
if (isset($this->__convertActionToMessage[$controller][$action['action']])) {
$stringifiedAction = $this->__convertActionToMessage[$controller][$action['action']];
if (isset(self::CONVERT_ACTION_TO_MESSAGE[$controller][$action['action']])) {
$stringifiedAction = self::CONVERT_ACTION_TO_MESSAGE[$controller][$action['action']];
}
$response['saved'] = false;
$response['name'] = 'Could not ' . $stringifiedAction . ' ' . Inflector::singularize($controller);
@ -590,8 +590,8 @@ class RestResponseComponent extends Component
}
if ($response instanceof TmpFileTool) {
App::uses('CakeResponseTmp', 'Tools');
$cakeResponse = new CakeResponseTmp(['status' => $code, 'type' => $type]);
App::uses('CakeResponseFile', 'Tools');
$cakeResponse = new CakeResponseFile(['status' => $code, 'type' => $type]);
$cakeResponse->file($response);
} else {
$cakeResponse = new CakeResponse(array('body' => $response, 'status' => $code, 'type' => $type));
@ -640,7 +640,7 @@ class RestResponseComponent extends Component
private function __dissectAdminRouting($action)
{
$admin = false;
if (strlen($action) > 6 && substr($action, 0, 6) == 'admin_') {
if (strlen($action) > 6 && substr($action, 0, 6) === 'admin_') {
$action = substr($action, 6);
$admin = true;
}
@ -655,13 +655,24 @@ class RestResponseComponent extends Component
return $this->__sendResponse($data, 200, $format, $raw, $download, $headers);
}
public function sendFile($path, $format = false, $download = false, $name = 'download')
/**
* @param string|File|TmpFileTool $path
* @param string|null $type
* @param bool $download
* @param string $name
* @return CakeResponseFile
* @throws Exception
*/
public function sendFile($path, $type = null, $download = false, $name = 'download')
{
$cakeResponse = new CakeResponse(array(
'status' => 200,
'type' => $format
));
$cakeResponse->file($path, array('name' => $name, 'download' => true));
App::uses('CakeResponseFile', 'Tools');
$cakeResponse = new CakeResponseFile([
'type' => $type
]);
$cakeResponse->file($path, ['name' => $name, 'download' => $download]);
if (Configure::read('Security.disable_browser_cache')) {
$cakeResponse->disableCache();
}
return $cakeResponse;
}

48
app/Controller/EventsController.php
View File

@ -56,6 +56,12 @@ class EventsController extends AppController
'warninglistId' => '',
);
// private
const DEFAULT_HIDDEN_INDEX_COLUMNS = [
'timestmap',
'publish_timestamp'
];
public $paginationFunctions = array('index', 'proposalEventIndex');
public function beforeFilter()
@ -358,10 +364,21 @@ class EventsController extends AppController
if ($v == "") {
continue 2;
}
if (preg_match('/^[0-9]+[mhdw]$/i', $v)) {
$v = $this->Event->resolveTimeDelta($v);
if (is_array($v) && isset($v[0]) && isset($v[1])) {
if (!is_int($v[0])) {
$v[0] = $this->Event->resolveTimeDelta($v[0]);
}
if (!is_int($v[1])) {
$v[1] = $this->Event->resolveTimeDelta($v[1]);
}
$this->paginate['conditions']['AND'][] = array('Event.timestamp >=' => $v[0]);
$this->paginate['conditions']['AND'][] = array('Event.timestamp <=' => $v[1]);
} else {
if (!is_int($v)) {
$v = $this->Event->resolveTimeDelta($v);
}
$this->paginate['conditions']['AND'][] = array('Event.timestamp >=' => $v);
}
$this->paginate['conditions']['AND'][] = array('Event.timestamp >=' => $v);
break;
case 'publish_timestamp':
case 'publishtimestamp':
@ -369,16 +386,16 @@ class EventsController extends AppController
continue 2;
}
if (is_array($v) && isset($v[0]) && isset($v[1])) {
if (preg_match('/^[0-9]+[mhdw]$/i', $v[0])) {
if (!is_int($v[0])) {
$v[0] = $this->Event->resolveTimeDelta($v[0]);
}
if (preg_match('/^[0-9]+[mhdw]$/i', $v[1])) {
if (!is_int($v[1])) {
$v[1] = $this->Event->resolveTimeDelta($v[1]);
}
$this->paginate['conditions']['AND'][] = array('Event.publish_timestamp >=' => $v[0]);
$this->paginate['conditions']['AND'][] = array('Event.publish_timestamp <=' => $v[1]);
} else {
if (preg_match('/^[0-9]+[mhdw]$/i', $v)) {
if (!is_int($v)) {
$v = $this->Event->resolveTimeDelta($v);
}
$this->paginate['conditions']['AND'][] = array('Event.publish_timestamp >=' => $v);
@ -933,6 +950,8 @@ class EventsController extends AppController
}
$possibleColumns[] = 'attribute_count';
$possibleColumns[] = 'timestamp';
$possibleColumns[] = 'publish_timestamp';
if (Configure::read('MISP.showCorrelationsOnIndex')) {
$possibleColumns[] = 'correlations';
@ -958,12 +977,12 @@ class EventsController extends AppController
$possibleColumns[] = 'creator_user';
}
$userEnabledColumns = $this->User->UserSetting->getValueForUser($this->Auth->user()['id'], 'event_index_hide_columns');
if ($userEnabledColumns === null) {
$userEnabledColumns = [];
$userDisabledColumns = $this->User->UserSetting->getValueForUser($this->Auth->user()['id'], 'event_index_hide_columns');
if ($userDisabledColumns === null) {
$userDisabledColumns = self::DEFAULT_HIDDEN_INDEX_COLUMNS;
}
$enabledColumns = array_diff($possibleColumns, $userEnabledColumns);
$enabledColumns = array_diff($possibleColumns, $userDisabledColumns);
return [$possibleColumns, $enabledColumns];
}
@ -1044,6 +1063,8 @@ class EventsController extends AppController
'analysis' => array('OR' => array(), 'NOT' => array()),
'attribute' => array('OR' => array(), 'NOT' => array()),
'hasproposal' => 2,
'timestamp' => array('from' => "", 'until' => ""),
'publishtimestamp' => array('from' => "", 'until' => ""),
);
if ($this->_isSiteAdmin()) {
@ -1112,6 +1133,8 @@ class EventsController extends AppController
'analysis' => __('Analysis'),
'attribute' => __('Attribute'),
'hasproposal' => __('Has proposal'),
'timestamp' => __('Last change at'),
'publishtimestamp' => __('Published at'),
];
if ($this->_isSiteAdmin()) {
@ -5890,13 +5913,12 @@ class EventsController extends AppController
if ($this->request->is('json')) {
App::uses('JSONConverterTool', 'Tools');
$converter = new JSONConverterTool();
if ($this->RestResponse->isAutomaticTool()) {
foreach ($converter->streamConvert($event) as $part) {
foreach (JSONConverterTool::streamConvert($event) as $part) {
$tmpFile->write($part);
}
} else {
$tmpFile->write($converter->convert($event));
$tmpFile->write(JSONConverterTool::convert($event));
}
$format = 'json';
} elseif ($this->request->is('xml')) {

6
app/Controller/FeedsController.php
View File

@ -124,7 +124,9 @@ class FeedsController extends AppController
'menuItem' => 'index'
]);
$this->loadModel('Event');
$this->set('distributionLevels', $this->Event->distributionLevels);
$distributionLevels = $this->Event->distributionLevels;
$distributionLevels[5] = __('Inherit from feed');
$this->set('distributionLevels', $distributionLevels);
$this->set('scope', $scope);
}
@ -286,6 +288,7 @@ class FeedsController extends AppController
$this->loadModel('Event');
$sharingGroups = $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1);
$distributionLevels = $this->Event->distributionLevels;
$distributionLevels[5] = __('Inherit from feed');
if (empty($sharingGroups)) {
unset($distributionLevels[4]);
}
@ -424,6 +427,7 @@ class FeedsController extends AppController
$this->loadModel('Event');
$sharingGroups = $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1);
$distributionLevels = $this->Event->distributionLevels;
$distributionLevels[5] = __('Inherit from feed');
if (empty($sharingGroups)) {
unset($distributionLevels[4]);
}

8
app/Controller/ObjectsController.php
View File

@ -1,6 +1,7 @@
<?php
App::uses('AppController', 'Controller');
App::uses('JsonTool', 'Tools');
/**
* @property MispObject $MispObject
@ -61,7 +62,7 @@ class ObjectsController extends AppController
}
$multiple_template_elements = Hash::extract($template['ObjectTemplateElement'], sprintf('{n}[multiple=true]'));
$multiple_attribute_allowed = array();
foreach ($multiple_template_elements as $k => $template_element) {
foreach ($multiple_template_elements as $template_element) {
$relation_type = $template_element['object_relation'] . ':' . $template_element['type'];
$multiple_attribute_allowed[$relation_type] = true;
}
@ -89,6 +90,9 @@ class ObjectsController extends AppController
if (isset($this->request->data['Attribute'])) {
foreach ($this->request->data['Attribute'] as &$attribute) {
if (empty($attribute['uuid'])) {
$attribute['uuid'] = CakeText::uuid();
}
$validation = $this->MispObject->Attribute->validateAttribute($attribute, false);
if ($validation !== true) {
$attribute['validation'] = $validation;
@ -414,7 +418,7 @@ class ObjectsController extends AppController
$this->request->data['Object'] = $this->request->data;
}
if (isset($this->request->data['Object']['data'])) {
$this->request->data = json_decode($this->request->data['Object']['data'], true);
$this->request->data = JsonTool::decode($this->request->data['Object']['data']);
}
if (isset($this->request->data['Object'])) {
$this->request->data = array_merge($this->request->data, $this->request->data['Object']);

48
app/Controller/ServersController.php
View File

@ -311,7 +311,7 @@ class ServersController extends AppController
if (!$fail) {
if ($this->_isRest()) {
$defaultPushRules = json_encode(["tags" => ["OR" => [], "NOT" => []], "orgs" => ["OR" => [], "NOT" => []]]);
$defaultPullRules = json_encode(["tags" => ["OR" => [], "NOT" => []], "orgs" => ["OR" => [], "NOT" => []], "url_params" => ""]);
$defaultPullRules = json_encode(["tags" => ["OR" => [], "NOT" => []], "orgs" => ["OR" => [], "NOT" => []], "type_attributes" => ["NOT" => []], "type_objects" => ["NOT" => []], "url_params" => ""]);
$defaults = array(
'push' => 0,
'pull' => 0,
@ -442,11 +442,25 @@ class ServersController extends AppController
$allOrgs[] = array('id' => $o['Organisation']['id'], 'name' => $o['Organisation']['name']);
}
$allTypes = [];
$this->loadModel('Attribute');
$this->loadModel('ObjectTemplate');
$objects = $this->ObjectTemplate->find('list', [
'fields' => ['name'],
'group' => ['name', 'id'],
]);
$allTypes = [
'attribute' => array_unique(Hash::extract(Hash::extract($this->Attribute->categoryDefinitions, '{s}.types'), '{n}.{n}')),
'object' => $objects
];
$this->set('host_org_id', Configure::read('MISP.host_org_id'));
$this->set('organisationOptions', $organisationOptions);
$this->set('localOrganisations', $localOrganisations);
$this->set('externalOrganisations', $externalOrganisations);
$this->set('allOrganisations', $allOrgs);
$this->set('allAttributeTypes', $allTypes['attribute']);
$this->set('allObjectTypes', $allTypes['object']);
$this->set('allTags', $this->__getTags());
$this->set('host_org_id', Configure::read('MISP.host_org_id'));
@ -497,6 +511,16 @@ class ServersController extends AppController
$this->Flash->error($error_msg);
}
}
$pushRules = $this->Server->jsonDecode($this->request->data['Server']['push_rules']);
$this->loadModel('Tag');
foreach ($pushRules['tags'] as $operator => $list) {
foreach ($list as $i => $tagName) {
if (!is_numeric($tagName)) { // tag added from freetext
$tag_id = $this->Tag->captureTag(['name' => $tagName], $this->Auth->user());
$list[$i] = $tag_id;
}
}
}
if (!$fail) {
// say what fields are to be updated
$fieldList = array('id', 'url', 'push', 'pull', 'push_sightings', 'push_galaxy_clusters', 'pull_galaxy_clusters', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
@ -613,6 +637,21 @@ class ServersController extends AppController
$allOrgs[] = array('id' => $o['Organisation']['id'], 'name' => $o['Organisation']['name']);
}
$allTypes = [];
$this->loadModel('Attribute');
$this->loadModel('ObjectTemplate');
$objects = $this->ObjectTemplate->find('all', [
'recursive' => -1,
'fields' => ['uuid', 'name'],
'group' => ['uuid', 'name'],
]);
$allTypes = [
'attribute' => array_unique(Hash::extract(Hash::extract($this->Attribute->categoryDefinitions, '{s}.types'), '{n}.{n}')),
'object' => Hash::map($objects, '{n}.ObjectTemplate', function ($item) {
return ['id' => $item['uuid'], 'name' => sprintf('%s (%s)', $item['name'], $item['uuid'])];
})
];
$oldRemoteSetting = 0;
if (!$this->Server->data['RemoteOrg']['local']) {
$oldRemoteSetting = 1;
@ -627,6 +666,8 @@ class ServersController extends AppController
$this->set('allOrganisations', $allOrgs);
$this->set('allTags', $this->__getTags());
$this->set('allAttributeTypes', $allTypes['attribute']);
$this->set('allObjectTypes', $allTypes['object']);
$this->set('server', $s);
$this->set('id', $id);
$this->set('host_org_id', Configure::read('MISP.host_org_id'));
@ -1974,7 +2015,7 @@ class ServersController extends AppController
}
$header = sprintf(
"Authorization: %s \nAccept: application/json\nContent-type: application/json",
empty(Configure::read('Security.advanced_authkeys')) ? $this->Auth->user('authkey') : __('YOUR_API_KEY')
__('YOUR_API_KEY')
);
$this->set('header', $header);
$this->set('allValidApis', $allValidApis);
@ -2054,7 +2095,6 @@ class ServersController extends AppController
$temp_headers = empty($request['header']) ? [] : explode("\n", $request['header']);
$request['header'] = array(
'Authorization' => $this->Auth->user('authkey'),
'Accept' => 'application/json',
'Content-Type' => 'application/json',
'User-Agent' => 'MISP REST Client',
@ -2301,7 +2341,7 @@ misp.direct_call(relative_path, body)
'Server' => array(
'url' => $baseurl,
'uuid' => Configure::read('MISP.uuid'),
'authkey' => $this->Auth->user('authkey'),
'authkey' => __('YOUR_API_KEY'),
'Organisation' => array(
'name' => $host_org['Organisation']['name'],
'uuid' => $host_org['Organisation']['uuid'],

2
app/Controller/TagsController.php
View File

@ -147,7 +147,7 @@ class TagsController extends AppController
$this->request->data['Tag'] = $this->request->data['Tag']['request'];
}
if (!isset($this->request->data['Tag']['colour'])) {
$this->request->data['Tag']['colour'] = $this->Tag->random_color();
$this->request->data['Tag']['colour'] = $this->Tag->tagColor($this->request->data['Tag']['name']);
}
if (isset($this->request->data['Tag']['id'])) {
unset($this->request->data['Tag']['id']);

15
app/Lib/Export/JsonExport.php
View File

@ -2,7 +2,6 @@
class JsonExport
{
private $__converter = false;
public $non_restrictive_export = true;
/**
@ -32,19 +31,13 @@ class JsonExport
*/
private function __eventHandler($event, $options = array())
{
if ($this->__converter === false) {
App::uses('JSONConverterTool', 'Tools');
$this->__converter = new JSONConverterTool();
}
return $this->__converter->streamConvert($event);
App::uses('JSONConverterTool', 'Tools');
return JSONConverterTool::streamConvert($event);
}
private function __objectHandler($object, $options = array()) {
if ($this->__converter === false) {
App::uses('JSONConverterTool', 'Tools');
$this->__converter = new JSONConverterTool();
}
return json_encode($this->__converter->convertObject($object, false, true));
App::uses('JSONConverterTool', 'Tools');
return json_encode(JSONConverterTool::convertObject($object, false, true));
}
private function __attributeHandler($attribute, $options = array())

18
app/Lib/Export/OpendataExport.php
View File

@ -1,4 +1,5 @@
<?php
App::uses('ProcessTool', 'Tools');
class OpendataExport
{
@ -67,12 +68,11 @@ class OpendataExport
public function footer()
{
$my_server = ClassRegistry::init('Server');
$cmd = $my_server->getPythonVersion() . ' ' . $this->__scripts_dir . $this->__script_name;
$cmd = [ProcessTool::pythonBin(), $this->__scripts_dir . $this->__script_name];
if (!empty($this->__auth)) {
$this->__request_object['auth'] = $this->__auth;
}
if ($this->__search){
if ($this->__search) {
return $this->__search_query($cmd);
}
return $this->__delete ? $this->__delete_query($cmd) : $this->__add_query($cmd);
@ -83,7 +83,7 @@ class OpendataExport
return '';
}
private function __add_query($cmd)
private function __add_query(array $cmd)
{
unset($this->__default_filters['returnFormat']);
$body = json_encode($this->__default_filters);
@ -95,7 +95,9 @@ class OpendataExport
$this->__request_object['setup'] = $setupFilename;
$this->__request_object['misp_url'] = $this->__url;
$commandFile = $this->__generateCommandFile();
$results = shell_exec($cmd . ' --query_data ' . $commandFile);
$cmd[] = '--query_data';
$cmd[] = $commandFile;
$results = ProcessTool::execute($cmd);
unlink($commandFile);
unlink($bodyFilename);
unlink($setupFilename);
@ -128,13 +130,15 @@ class OpendataExport
return $this->__simple_query($cmd);
}
private function __simple_query($cmd)
private function __simple_query(array $cmd)
{
if (!empty($this->__setup['resources'])) {
$this->__request_object['search'] = $this->__setup['resources'];
}
$commandFile = $this->__generateCommandFile();
$results = shell_exec($cmd . ' --query_data ' . $commandFile);
$cmd[] = '--query_data';
$cmd[] = $commandFile;
$results = ProcessTool::execute($cmd);
unlink($commandFile);
return $results;
}

153
app/Lib/Export/OpeniocExport.php
View File

@ -2,92 +2,58 @@
class OpeniocExport
{
const MAPPING = array(
'composite' => array(
'regkey|value' => array(array('Network', 'RegistryItem/KeyPath', 'string'), array('Network', 'RegistryItem/Value', 'string')),
'filename|md5' => array(array('FileItem', 'FileItem/FileName', 'string'), array('FileItem', 'FileItem/Md5sum', 'md5')),
'filename|sha1' => array(array('FileItem', 'FileItem/FileName', 'string'), array('FileItem', 'FileItem/Sha1sum', 'sha1')),
'filename|sha256' => array(array('FileItem', 'FileItem/FileName', 'string'), array('FileItem', 'FileItem/Sha256sum', 'sha256')),
'malware-sample' => array(array('FileItem', 'FileItem/FileName', 'string'), array('FileItem', 'FileItem/Md5sum', 'md5')),
'domain|ip' => array(array('Network', 'Network/DNS', 'string'), array('PortItem', 'PortItem/remoteIP', 'IP')),
),
'simple' => array(
'md5' => array('FileItem', 'FileItem/Md5sum', 'md5'),
'sha1' => array('FileItem', 'FileItem/Sha1sum', 'sha1'),
'sha256' => array('FileItem', 'FileItem/Sha256sum', 'sha256'),
'filename' => array('FileItem', 'FileItem/FileName', 'string'),
'ip-src' => array('PortItem', 'PortItem/remoteIP', 'IP'),
'ip-dst' => array('RouteEntryItem', 'RouteEntryItem/Destination', 'IP'),
'hostname' => array('RouteEntryItem', 'RouteEntryItem/Destination', 'string'),
'email' => array('Email', 'Email/From', 'string'),
'email-src' => array('Email', 'Email/From', 'string'),
'email-dst' => array('Email', 'Email/To', 'string'),
'email-subject' => array('Email', 'Email/Subject', 'string'),
'email-attachment' => array('Email', 'Email/Attachment/Name', 'string'),
'domain' => array('Network', 'Network/DNS', 'string'),
'url' => array('UrlHistoryItem', 'UrlHistoryItem/URL', 'string'),
'user-agent' => array('Network', 'Network/UserAgent', 'string'),
'regkey' => array('Network', 'RegistryItem/KeyPath', 'string'),
'snort' => array('Snort', 'Snort/Snort', 'string'),
'attachment' => array('FileItem', 'FileItem/FileName', 'string'),
'link' => array('URL', 'UrlHistoryItem/URL', 'md5')
)
);
public $additional_params = array(
'flatten' => 1
);
public function buildAll($user, $data, $scope = 'event')
{
$final = '';
if (!isset($data['Attribute'])) {
$data = array('Attribute' => $data);
}
if ($scope == 'event') {
$final = $this->generateSingleTop($data);
} else {
$final = $this->generateTop($user, $data);
}
foreach ($data['Attribute'] as $attribute) {
$final .= $this->generateAttribute($attribute);
}
$final .= $this->generateBottom();
return $final;
}
public function convert($attributes)
{
$final = '';
foreach ($attributes as $attribute) {
$final .= $this->generateAttribute($attribute);
}
return $final;
}
public function getResult()
{
return $this->__final;
}
public $mapping = array(
'composite' => array(
'regkey|value' => array(array('Network', 'RegistryItem/KeyPath', 'string'), array('Network', 'RegistryItem/Value', 'string')),
'filename|md5' => array(array('FileItem', 'FileItem/FileName', 'string'), array('FileItem', 'FileItem/Md5sum', 'md5')),
'filename|sha1' => array(array('FileItem', 'FileItem/FileName', 'string'), array('FileItem', 'FileItem/Sha1sum', 'sha1')),
'filename|sha256' => array(array('FileItem', 'FileItem/FileName', 'string'), array('FileItem', 'FileItem/Sha256sum', 'sha256')),
'malware-sample' => array(array('FileItem', 'FileItem/FileName', 'string'), array('FileItem', 'FileItem/Md5sum', 'md5')),
'domain|ip' => array(array('Network', 'Network/DNS', 'string'), array('PortItem', 'PortItem/remoteIP', 'IP')),
),
'simple' => array(
'md5' => array('FileItem', 'FileItem/Md5sum', 'md5'),
'sha1' => array('FileItem', 'FileItem/Sha1sum', 'sha1'),
'sha256' => array('FileItem', 'FileItem/Sha256sum', 'sha256'),
'filename' => array('FileItem', 'FileItem/FileName', 'string'),
'ip-src' => array('PortItem', 'PortItem/remoteIP', 'IP'),
'ip-dst' => array('RouteEntryItem', 'RouteEntryItem/Destination', 'IP'),
'hostname' => array('RouteEntryItem', 'RouteEntryItem/Destination', 'string'),
'email' => array('Email', 'Email/From', 'string'),
'email-src' => array('Email', 'Email/From', 'string'),
'email-dst' => array('Email', 'Email/To', 'string'),
'email-subject' => array('Email', 'Email/Subject', 'string'),
'email-attachment' => array('Email', 'Email/Attachment/Name', 'string'),
'domain' => array('Network', 'Network/DNS', 'string'),
'url' => array('UrlHistoryItem', 'UrlHistoryItem/URL', 'string'),
'user-agent' => array('Network', 'Network/UserAgent', 'string'),
'regkey' => array('Network', 'RegistryItem/KeyPath', 'string'),
'snort' => array('Snort', 'Snort/Snort', 'string'),
'attachment' => array('FileItem', 'FileItem/FileName', 'string'),
'link' => array('URL', 'UrlHistoryItem/URL', 'md5')
)
);
public function frameComposite($attribute)
private function frameComposite($attribute)
{
$mapping = self::MAPPING['composite'][$attribute['type']];
$temp = '';
$values = explode('|', $attribute['value']);
$temp .= ' <Indicator operator="AND" id="' . h($attribute['uuid']) . '">' . PHP_EOL;
$temp .= $this->frameIndicator($this->mapping['composite'][$attribute['type']][0], $attribute['uuid'], $values[0], true);
$temp .= $this->frameIndicator($this->mapping['composite'][$attribute['type']][1], $attribute['uuid'], $values[1], true);
$temp .= $this->frameIndicator($mapping[0], $attribute['uuid'], $values[0], true);
$temp .= $this->frameIndicator($mapping[1], $attribute['uuid'], $values[1], true);
$temp .= ' </Indicator>' . PHP_EOL;
return $temp;
}
public function frameIndicator($mapping, $uuid, $value, $extraIndent = false)
private function frameIndicator($mapping, $uuid, $value, $extraIndent = false)
{
$temp = '';
$padding = 6;
if ($extraIndent) {
$padding = 8;
}
$padding = $extraIndent ? 8 : 6;
$temp .= str_repeat(' ', $padding) . '<IndicatorItem id="' . h($uuid) . '" condition="is">' . PHP_EOL;
$temp .= str_repeat(' ', ($padding + 2)) . '<Context document="' . $mapping[0] . '" search="' . $mapping[1] . '" type="mir" />' . PHP_EOL;
$temp .= str_repeat(' ', ($padding + 2)) . '<Content type="' . $mapping[2] . '">' . h($value) . '</Content>' . PHP_EOL;
@ -95,26 +61,16 @@ class OpeniocExport
return $temp;
}
// This method will turn each eligible attribute into an indicator
public function generateAttribute($attribute)
{
}
// Simple check for valid categories and types for IOC generation
public function checkValidTypeForIOC($attribute)
private function checkValidTypeForIOC($attribute)
{
// categories that should be included
$category = array('Payload delivery', 'Artifacts dropped', 'Payload installation', 'Persistence mechanism', 'Network activity');
if (!in_array($attribute['category'], $category)) {
return false;
}
return true;
return in_array($attribute['category'], $category, true);
}
private function __attributeHandler($attribute, $options = array()) {
$temp = '';
private function attributeHandler($attribute, $options = array())
{
if (isset($attribute['Attribute'])) {
$attribute = $attribute['Attribute'];
}
@ -122,44 +78,37 @@ class OpeniocExport
if (!$this->checkValidTypeForIOC($attribute) || $attribute['to_ids'] == 0) {
return false;
}
if ($attribute['type'] == 'malware-sample') {
if ($attribute['type'] === 'malware-sample') {
$attribute['type'] = 'filename|md5';
}
if (strpos($attribute['type'], '|')) {
if (isset($this->mapping['composite'][$attribute['type']])) {
$temp .= $this->frameComposite($attribute);
if (isset(self::MAPPING['composite'][$attribute['type']])) {
return $this->frameComposite($attribute);
}
} else {
if (isset($this->mapping['simple'][$attribute['type']])) {
$temp .= $this->frameIndicator($this->mapping['simple'][$attribute['type']], $attribute['uuid'], $attribute['value'], false);
if (isset(self::MAPPING['simple'][$attribute['type']])) {
return $this->frameIndicator(self::MAPPING['simple'][$attribute['type']], $attribute['uuid'], $attribute['value'], false);
}
}
return $temp;
return false;
}
public function handler($data, $options = array())
{
if ($options['scope'] === 'Attribute') {
return $this->__attributeHandler($data, $options);
return $this->attributeHandler($data, $options);
} else if ($options['scope'] === 'Event') {
$result = '';
if (!empty($data['Attribute'])) {
$first = true;
foreach ($data['Attribute'] as $attribute) {
$temp = $this->__attributeHandler($attribute, $options);
$temp = $this->attributeHandler($attribute, $options);
if (!empty($temp)) {
if (!$first) {
$result .= $this->separator($options);
}
$result .= $temp;
$first = false;
}
}
}
return $result;
}
}
public function header($options = array())
@ -179,7 +128,6 @@ class OpeniocExport
$temp .= ' <definition>' . PHP_EOL;
$temp .= ' <Indicator operator="OR" id="' . CakeText::uuid() . '">' . PHP_EOL;
return $temp;
}
public function footer()
@ -195,5 +143,4 @@ class OpeniocExport
{
return '';
}
}

24
app/Lib/Export/StixExport.php
View File

@ -22,7 +22,6 @@ abstract class StixExport
protected $stixFile = null;
private $__cluster_uuids = array();
private $__current_filename = null;
private $__empty_file = null;
private $__event_galaxies = array();
/** @var File */
@ -90,7 +89,7 @@ abstract class StixExport
}
$this->__tmp_file->append($this->__scope === 'Attribute' ? ']}}' : ']}');
$this->__tmp_file->close();
$this->__filenames[] = $this->__current_filename;
$this->__filenames[] = $this->__tmp_file->path;
}
$result = $this->__parse_misp_data();
$decoded = json_decode($result, true);
@ -265,8 +264,7 @@ abstract class StixExport
$attributes_count += count($_object['Attribute']);
}
}
$converter = new JSONConverterTool();
$event = JsonTool::encode($converter->convert($event, false, true)); // we don't need pretty printed JSON
$event = JsonTool::encode(JSONConverterTool::convert($event, false, true)); // we don't need pretty printed JSON
if ($this->__n_attributes + $attributes_count <= $this->__attributes_limit) {
$this->__tmp_file->append($this->__n_attributes == 0 ? $event : ', ' . $event);
$this->__n_attributes += $attributes_count;
@ -304,8 +302,8 @@ abstract class StixExport
private function __initialize_misp_file()
{
$this->__current_filename = FileAccessTool::createTempFile();
$this->__tmp_file = new File($this->__current_filename);
$tmpFile = FileAccessTool::createTempFile();
$this->__tmp_file = new File($tmpFile);
$this->__tmp_file->write('{"response": ' . ($this->__scope === 'Attribute' ? '{"Attribute": [' : '['));
$this->__empty_file = true;
}
@ -332,11 +330,15 @@ abstract class StixExport
private function getFraming()
{
$framingCmd = $this->__initiate_framing_params();
$framing = json_decode(ProcessTool::execute($framingCmd, null, true), true);
if ($framing === null || isset($framing['error'])) {
throw new Exception("Could not get results from framing cmd when exporting STIX file.");
try {
$framing = JsonTool::decode(ProcessTool::execute($framingCmd, null, true));
if (isset($framing['error'])) {
throw new Exception("Framing command error: " . $framing['error']);
}
return $framing;
} catch (Exception $e) {
throw new Exception("Could not get results from framing cmd when exporting STIX file.", 0, $e);
}
return $framing;
}
private function __merge_galaxy_tag(&$galaxies, $tag_name)
@ -354,7 +356,7 @@ abstract class StixExport
{
$this->__tmp_file->append($this->__scope === 'Attribute' ? ']}}' : ']}');
$this->__tmp_file->close();
$this->__filenames[] = $this->__current_filename;
$this->__filenames[] = $this->__tmp_file->path;
$this->__initialize_misp_file();
$this->__tmp_file->append($content);
}

129
app/Lib/Export/YaraExport.php
View File

@ -1,59 +1,57 @@
<?php
App::uses('JsonExport', 'Export');
App::uses('AppModel', 'Model');
App::uses('ProcessTool', 'Tools');
App::uses('TmpFileTool', 'Tools');
class YaraExport
{
private $__script_path = APP . 'files/scripts/yara/yaraexport.py';
private $__tmp_dir = APP . 'tmp/yara/';
private $__end_of_cmd = ' 2>>' . APP . 'tmp/logs/yara_export.log';
private $__n_attributes = 0;
private $__MAX_n_attributes = 15000;
private $__yara_file_gen = null;
private $__yara_file_asis = null;
/** @var null|File */
private $__curr_input_file = null;
private $__scope = false;
private $__curr_input_is_empty = true;
private $__JsonExporter = false;
/** @var JsonExport */
private $__JsonExporter;
private $__raw_mode = true;
public $non_restrictive_export = true;
public function __construct()
{
$this->__JsonExporter = new JsonExport();
}
private static function __count_atributes($data)
{
$attributes_count = count($data['Attribute']);
// foreach ($data['Object'] as $_object) {
// $attributes_count += count($_object['Attribute']);
// }
$attributes_count = count($data['Attribute']);
foreach ($data['Object'] as $_object) {
$attributes_count += count($_object['Attribute']);
}
return $attributes_count;
}
public function header($options = array())
{
if($this->__JsonExporter === false){
$this->__JsonExporter = new JsonExport();
}
$this->__initialize_yara_file();
$this->__initialize_misp_file($options);
if($options['returnFormat'] === 'yara-json'){
$this->__yara_file_gen = FileAccessTool::createTempFile();
$this->__yara_file_asis = FileAccessTool::createTempFile();
if ($options['returnFormat'] === 'yara-json') {
$this->__raw_mode = false;
}
return '';
}
private function __initialize_yara_file()
{
$yaraFileName = $this->generateRandomFileName();
$this->__yara_file_gen = new File($this->__tmp_dir . $yaraFileName . '_generated', true, 0644);
$this->__yara_file_asis = new File($this->__tmp_dir . $yaraFileName . '_asis', true, 0644);
$this->__yara_file_gen->close();
$this->__yara_file_asis->close();
}
/**
* @throws Exception
*/
private function __initialize_misp_file($options)
{
$mispFileName = $this->generateRandomFileName();
$this->__curr_input_file = new File($this->__tmp_dir . $mispFileName, true, 0644);
$this->__curr_input_file = new File(FileAccessTool::createTempFile());
$header = $this->__JsonExporter->header($options);
$this->__curr_input_file->append($header);
$this->__curr_input_is_empty = true;
@ -64,11 +62,11 @@ class YaraExport
// convert attribute(s) to json and write them to input queue file
if ($options['scope'] === 'Attribute') {
$attr_count = 1;
} else if($options['scope'] === 'Event') {
$attr_count = YaraExport::__count_atributes($data);
} else if ($options['scope'] === 'Event') {
$attr_count = $this->__count_atributes($data);
}
if(!empty($data)){
if(!$this->__curr_input_is_empty){
if (!empty($data)) {
if (!$this->__curr_input_is_empty) {
$this->separator(); // calling separator since returning '' will prevent it
}
$jsonData = $this->__JsonExporter->handler($data, $options);
@ -83,42 +81,48 @@ class YaraExport
}
$this->__n_attributes += $attr_count;
// if the file exceeds the max_attributes, process it, delete it and reset the counter
if ($this->__n_attributes >= $this->__MAX_n_attributes){
if ($this->__n_attributes >= $this->__MAX_n_attributes) {
$this->__process_file($options);
$this->__initialize_misp_file($options);
}
return '';
}
/**
* @param array $options
* @return TmpFileTool
* @throws Exception
*/
public function footer($options = array())
{
if(!($this->__curr_input_is_empty)){
if (!$this->__curr_input_is_empty) {
$this->__process_file($options);
}
$file = new File($this->__yara_file_gen->path);
$data_gen = $file->read(true, 'r');
$file->close();
$file->delete();
$file = new File($this->__yara_file_asis->path);
$data_asis = $file->read(true, 'r');
$file->close();
$file->delete();
if($this->__raw_mode){
$output =
'// ===================================== GENERATED ===================================='. PHP_EOL .
$data_gen . PHP_EOL .
'// ===================================== AS-IS ===================================='. PHP_EOL .
$data_asis;
}else{
$output = '{"generated":['. $data_gen .'],'.
'"as-is":[' . $data_asis . ']}';
$output = new TmpFileTool();
if ($this->__raw_mode) {
$output->write('// ===================================== GENERATED ===================================='. PHP_EOL);
$output->writeFromFile($this->__yara_file_gen);
$output->write(PHP_EOL . '// ===================================== AS-IS ===================================='. PHP_EOL);
$output->writeFromFile($this->__yara_file_asis);
} else {
$output->write('{"generated":[');
$output->writeFromFile($this->__yara_file_gen);
$output->write('],"as-is":[');
$output->writeFromFile($this->__yara_file_asis);
$output->write(']}');
}
FileAccessTool::deleteFile($this->__yara_file_gen);
FileAccessTool::deleteFile($this->__yara_file_asis);
return $output;
}
public function separator()
{
if(!$this->__curr_input_is_empty){
if (!$this->__curr_input_is_empty) {
$this->__curr_input_file->append(',');
}
return '';
@ -128,21 +132,22 @@ class YaraExport
{
$footer = $this->__JsonExporter->footer($options);
$this->__curr_input_file->append($footer);
$pythonScript = $this->__script_path;
$in = $this->__curr_input_file->path;
$out1 = $this->__yara_file_gen->path;
$out2 = $this->__yara_file_asis->path;
$logging = $this->__end_of_cmd;
$raw_flag = $this->__raw_mode ? '--raw' : '';
$my_server = ClassRegistry::init('Server');
$result = shell_exec($my_server->getPythonVersion() . " $pythonScript --input $in --out-generated $out1 --out-asis $out2 $raw_flag $logging");
$this->__curr_input_file->close();
$command = [
ProcessTool::pythonBin(),
$this->__script_path,
'--input', $this->__curr_input_file->path,
'--out-generated', $this->__yara_file_gen,
'--out-asis', $this->__yara_file_asis,
];
if ($this->__raw_mode) {
$command[] = '--raw';
}
ProcessTool::execute($command, null, true);
$this->__curr_input_file->delete();
$this->__n_attributes = 0;
}
public function generateRandomFileName()
{
return (new RandomTool())->random_str(false, 12);
}
}

6
app/Lib/Tools/AttachmentTool.php
View File

@ -156,7 +156,7 @@ class AttachmentTool
} else {
$filepath = $this->attachmentDir() . DS . $path;
$file = new File($filepath);
if (!$file->exists()) {
if (!is_file($file->path)) {
throw new NotFoundException("File '$filepath' does not exists.");
}
}
@ -281,6 +281,7 @@ class AttachmentTool
$s3 = $this->loadS3Client();
$s3->deleteDirectory($eventId);
} else {
App::uses('Folder', 'Utility');
$dirPath = $this->attachmentDir();
foreach (array($dirPath, $dirPath . DS . 'shadow') as $dirPath) {
@ -425,8 +426,7 @@ class AttachmentTool
// Output image to string
ob_start();
imagepng($imageThumbnail, null, 9);
$imageData = ob_get_contents();
ob_end_clean();
$imageData = ob_get_clean();
imagedestroy($imageThumbnail);
return $imageData;

19
app/Lib/Tools/BackgroundJobs/BackgroundJob.php
View File

@ -66,10 +66,8 @@ class BackgroundJob implements JsonSerializable
/**
* Run the job command
*
* @return self
*/
public function run(): self
public function run(): void
{
$descriptorSpec = [
1 => ["pipe", "w"], // stdout
@ -90,28 +88,17 @@ class BackgroundJob implements JsonSerializable
['BACKGROUND_JOB_ID' => $this->id]
);
$stdout = stream_get_contents($pipes[1]);
$this->setOutput($stdout);
fclose($pipes[1]);
$stderr = stream_get_contents($pipes[2]);
$this->setError($stderr);
fclose($pipes[2]);
$this->output = stream_get_contents($pipes[1]);
$this->error = stream_get_contents($pipes[2]);
$this->returnCode = proc_close($process);
if ($this->returnCode === 0 && empty($stderr)) {
$this->setStatus(BackgroundJob::STATUS_COMPLETED);
$this->setProgress(100);
CakeLog::info("[JOB ID: {$this->id()}] - completed.");
} else {
$this->setStatus(BackgroundJob::STATUS_FAILED);
CakeLog::error("[JOB ID: {$this->id()}] - failed with error code {$this->returnCode}. STDERR: {$stderr}. STDOUT: {$stdout}.");
}
return $this;
}
public function jsonSerialize(): array

54
app/Lib/Tools/BackgroundJobsTool.php
View File

@ -278,9 +278,14 @@ class BackgroundJobsTool
*/
public function getWorkers(): array
{
$workers = [];
$procs = $this->getSupervisor()->getAllProcesses();
try {
$procs = $this->getSupervisor()->getAllProcesses();
} catch (\Exception $exception) {
CakeLog::error("An error occured when getting the workers statuses via Supervisor API: {$exception->getMessage()}");
return [];
}
$workers = [];
foreach ($procs as $proc) {
if ($proc->offsetGet('group') === self::MISP_WORKERS_PROCESS_GROUP) {
if ($proc->offsetGet('pid') > 0) {
@ -335,27 +340,6 @@ class BackgroundJobsTool
);
}
/**
* Run job
*
* @param BackgroundJob $job
*
* @return integer Process return code.
*/
public function run(BackgroundJob $job): int
{
$job->setStatus(BackgroundJob::STATUS_RUNNING);
CakeLog::info("[JOB ID: {$job->id()}] - started.");
$this->update($job);
$job = $job->run();
$this->update($job);
return $job->returnCode();
}
/**
* Start worker by name
*
@ -492,7 +476,7 @@ class BackgroundJobsTool
}
try {
$supervisorStatus = $this->getSupervisor()->getState()['statecode'] === \Supervisor\Supervisor::RUNNING;
$supervisorStatus = $this->getSupervisorStatus();
} catch (Exception $exception) {
CakeLog::error("SimpleBackgroundJobs Supervisor error: {$exception->getMessage()}");
$supervisorStatus = false;
@ -509,6 +493,16 @@ class BackgroundJobsTool
}
}
/**
* Return true if Supervisor process is running.
*
* @return boolean
*/
public function getSupervisorStatus(): bool
{
return $this->getSupervisor()->getState()['statecode'] === \Supervisor\Supervisor::RUNNING;
}
/**
* Validate queue
*
@ -603,6 +597,7 @@ class BackgroundJobsTool
/**
* @return \Supervisor\Supervisor
* @throws Exception
*/
private function createSupervisorConnection(): \Supervisor\Supervisor
{
@ -616,10 +611,19 @@ class BackgroundJobsTool
];
}
$host = null;
if (substr($this->settings['supervisor_host'], 0, 5) === 'unix:') {
if (!defined('CURLOPT_UNIX_SOCKET_PATH')) {
throw new Exception("For unix socket connection, cURL is required.");
}
$httpOptions['curl'][CURLOPT_UNIX_SOCKET_PATH] = substr($this->settings['supervisor_host'], 5);
$host = 'localhost';
}
$client = new \fXmlRpc\Client(
sprintf(
'http://%s:%s/RPC2',
$this->settings['supervisor_host'],
$host ?: $this->settings['supervisor_host'],
$this->settings['supervisor_port']
),
new \fXmlRpc\Transport\HttpAdapterTransport(

107
app/Lib/Tools/CakeResponseFile.php Normal file
View File

@ -0,0 +1,107 @@
<?php
class CakeResponseFile extends CakeResponse
{
/**
* Modified version that supports also TmpFileTool and File
*
* @param string|TmpFileTool|File $path
* @param array $options
* @throws Exception
*/
public function file($path, $options = array())
{
if ($path instanceof TmpFileTool) {
$this->header('Content-Length', $path->size());
$this->_clearBuffer();
$this->_file = $path;
} else if ($path instanceof File) {
$options += array(
'name' => null,
'download' => null
);
if ($options['download']) {
$name = $options['name'] === null ? $path->name : $options['name'];
$this->download($name);
$this->header('Content-Transfer-Encoding', 'binary');
}
$this->header('Accept-Ranges', 'bytes');
$httpRange = env('HTTP_RANGE');
if (isset($httpRange)) {
$this->_fileRange($path, $httpRange);
} else {
$this->header('Content-Length', filesize($path->path));
}
$this->_clearBuffer();
$this->_file = $path;
} else {
parent::file($path, $options);
}
}
/**
* This method supports TmpFileTool and also provides optimised variant for sending file from `File` object
* @param File|TmpFileTool $file
* @param array $range
* @return bool
* @throws Exception
*/
protected function _sendFile($file, $range)
{
set_time_limit(0);
session_write_close();
if ($file instanceof TmpFileTool) {
foreach ($file->intoChunks() as $chunk) {
if (!$this->_isActive()) {
$file->close();
return false;
}
echo $chunk;
}
} else {
$handler = fopen($file->path, 'rb');
if ($handler === false) {
throw new Exception("File $file->path doesn't exists anymore or is not readable.");
}
$end = $start = false;
if ($range && is_array($range)) {
list($start, $end) = $range;
}
if ($start !== false) {
fseek($handler, $start);
}
$bufferSize = 8192;
while (!feof($handler)) {
if (!$this->_isActive()) {
$file->close();
return false;
}
$offset = ftell($handler);
if ($end && $offset >= $end) {
break;
}
if ($end && $offset + $bufferSize >= $end) {
$bufferSize = $end - $offset + 1;
}
echo fread($handler, $bufferSize);
}
fclose($handler);
}
$this->_flushBuffer();
return true;
}
/**
* Faster version that do not do redundant check
* @return bool
*/
protected function _isActive()
{
return connection_status() === CONNECTION_NORMAL;
}
}

40
app/Lib/Tools/CakeResponseTmp.php
View File

@ -1,40 +0,0 @@
<?php
class CakeResponseTmp extends CakeResponse
{
public function file($path, $options = array())
{
if ($path instanceof TmpFileTool) {
$this->header('Content-Length', $path->size());
$this->_clearBuffer();
$this->_file = $path;
} else {
parent::file($path, $options);
}
}
/**
* @param File|TmpFileTool $file
* @param array $range
* @return bool
* @throws Exception
*/
protected function _sendFile($file, $range)
{
if ($file instanceof TmpFileTool) {
set_time_limit(0);
session_write_close();
foreach ($file->intoChunks() as $chunk) {
if (!$this->_isActive()) {
$file->close();
return false;
}
echo $chunk;
$this->_flushBuffer();
}
return true;
} else {
return parent::_sendFile($file, $range);
}
}
}

36
app/Lib/Tools/ComplexTypeTool.php
View File

@ -3,7 +3,7 @@ require_once __DIR__ . '/TmpFileTool.php';
class ComplexTypeTool
{
const REFANG_REGEX__TABLE = array(
const REFANG_REGEX_TABLE = array(
array(
'from' => '/^(hxxp|hxtp|htxp|meow|h\[tt\]p)/i',
'to' => 'http',
@ -31,11 +31,23 @@ class ComplexTypeTool
)
);
const HEX_HASH_TYPES = array(
32 => array('single' => array('md5', 'imphash', 'x509-fingerprint-md5'), 'composite' => array('filename|md5', 'filename|imphash')),
40 => array('single' => array('sha1', 'pehash', 'x509-fingerprint-sha1', 'cdhash'), 'composite' => array('filename|sha1', 'filename|pehash')),
56 => array('single' => array('sha224', 'sha512/224'), 'composite' => array('filename|sha224', 'filename|sha512/224')),
64 => array('single' => array('sha256', 'authentihash', 'sha512/256', 'x509-fingerprint-sha256'), 'composite' => array('filename|sha256', 'filename|authentihash', 'filename|sha512/256')),
96 => array('single' => array('sha384'), 'composite' => array('filename|sha384')),
128 => array('single' => array('sha512'), 'composite' => array('filename|sha512'))
);
// algorithms to run through in order, without Hashes that are checked separately
const CHECKS = array('Email', 'IP', 'DomainOrFilename', 'SimpleRegex', 'AS', 'BTC');
private $__tlds = null;
public static function refangValue($value, $type)
{
foreach (self::REFANG_REGEX__TABLE as $regex) {
foreach (self::REFANG_REGEX_TABLE as $regex) {
if (in_array($type, $regex['types'], true)) {
$value = preg_replace($regex['from'], $regex['to'], $value);
}
@ -223,18 +235,6 @@ class ComplexTypeTool
return array_values($resultArray);
}
private $__hexHashTypes = array(
32 => array('single' => array('md5', 'imphash', 'x509-fingerprint-md5'), 'composite' => array('filename|md5', 'filename|imphash')),
40 => array('single' => array('sha1', 'pehash', 'x509-fingerprint-sha1', 'cdhash'), 'composite' => array('filename|sha1', 'filename|pehash')),
56 => array('single' => array('sha224', 'sha512/224'), 'composite' => array('filename|sha224', 'filename|sha512/224')),
64 => array('single' => array('sha256', 'authentihash', 'sha512/256', 'x509-fingerprint-sha256'), 'composite' => array('filename|sha256', 'filename|authentihash', 'filename|sha512/256')),
96 => array('single' => array('sha384'), 'composite' => array('filename|sha384')),
128 => array('single' => array('sha512'), 'composite' => array('filename|sha512'))
);
// algorithms to run through in order, without Hashes that are checked separately
private $__checks = array('Email', 'IP', 'DomainOrFilename', 'SimpleRegex', 'AS', 'BTC');
/**
* @param string $raw_input Trimmed value
* @return array|false
@ -263,7 +263,7 @@ class ComplexTypeTool
$input = $this->__refangInput($input);
$input = $this->__extractPort($input);
foreach ($this->__checks as $check) {
foreach (self::CHECKS as $check) {
$result = $this->{'__checkFor' . $check}($input);
if ($result) {
return $result;
@ -357,7 +357,7 @@ class ComplexTypeTool
private function __refangInput($input)
{
$input['refanged'] = $input['raw'];
foreach (self::REFANG_REGEX__TABLE as $regex) {
foreach (self::REFANG_REGEX_TABLE as $regex) {
$input['refanged'] = preg_replace($regex['from'], $regex['to'], $input['refanged']);
}
$input['refanged'] = rtrim($input['refanged'], ".");
@ -511,8 +511,8 @@ class ComplexTypeTool
private function __resolveHash($value)
{
$strlen = strlen($value);
if (isset($this->__hexHashTypes[$strlen]) && ctype_xdigit($value)) {
return $this->__hexHashTypes[$strlen];
if (isset(self::HEX_HASH_TYPES[$strlen]) && ctype_xdigit($value)) {
return self::HEX_HASH_TYPES[$strlen];
}
return false;
}

49
app/Lib/Tools/EventTimelineTool.php
View File

@ -9,6 +9,10 @@
# Will be use latter on
private $__related_events = array();
private $__related_attributes = array();
# Feature not implemented yet.
# This is to let user configure a polling frequency. The idea is that it's not because we don't have data during a specific period that it means it should be considered as positive (or fp).
# For example, if we poll every Monday. If it was positive on week 1 and negative on week 2, the timeline will show that it was positive until monday on week 2, which is most probably incorrect as it might be actually negative on Wednesday.
private $extrapolateWithPollingFrequency = false;
public function construct($eventModel, $user, $filterRules, $extended_view=0)
{
@ -88,6 +92,8 @@
'timestamp' => $attr['timestamp'],
'first_seen' => $attr['first_seen'],
'last_seen' => $attr['last_seen'],
'attribute_type' => $attr['type'],
'is_image' => $this->__eventModel->Attribute->isImage($attr),
);
$this->__json['items'][] = $toPush;
}
@ -127,6 +133,8 @@
'event_id' => $obj_attr['event_id'],
'group' => 'object_attribute',
'timestamp' => $obj_attr['timestamp'],
'attribute_type' => $obj_attr['type'],
'is_image' => $this->__eventModel->Attribute->isImage($obj_attr),
);
$toPush_obj['Attribute'][] = $toPush_attr;
}
@ -168,18 +176,18 @@
$event['Sighting'][$k]['date_sighting'] *= 1000; // adapt to use micro
$regroupedSightings[$sighting['attribute_id']][] = &$event['Sighting'][$k];
}
// make sure sightings are ordered
uksort($regroupedSightings, function ($a, $b) {
return $a['date_sighting'] > $b['date_sighting'];
});
// generate extrapolation
$now = time()*1000;
foreach ($regroupedSightings as $attributeId => $sightings) {
usort($sightings, function ($a, $b) { // make sure sightings are ordered
return $a['date_sighting'] > $b['date_sighting'];
});
$i = 0;
while ($i < count($sightings)) {
$sighting = $sightings[$i];
$attribute = $lookupAttribute[$attributeId];
$fpSightingIndex = $this->getNextFalsePositiveSightingIndex($sightings, $i+1);
$group = $sighting['type'] == '1' ? 'sighting_negative' : 'sighting_positive';
if ($fpSightingIndex === false) { // No next FP, extrapolate to now
$this->__json['items'][] = array(
'attribute_id' => $attributeId,
@ -187,7 +195,7 @@
'uuid' => $sighting['uuid'],
'content' => $attribute['value'],
'event_id' => $attribute['event_id'],
'group' => 'sighting_positive',
'group' => $group,
'timestamp' => $attribute['timestamp'],
'first_seen' => $sighting['date_sighting'],
'last_seen' => $now,
@ -198,11 +206,18 @@
$pSightingIndex = $fpSightingIndex - 1;
$halfTime = 0;
if ($pSightingIndex == $i) {
// we have only one positive sighting, thus the UP time should be take from a pooling frequence
// we have only one positive sighting, thus the UP time should be take from a pooling frequence (feature not existing yet)
// for now, consider it UP only for half the time until the next FP
$halfTime = ($sightings[$i+1]['date_sighting'] - $sighting['date_sighting'])/2;
if ($this->extrapolateWithPollingFrequency) {
$halfTime = ($sightings[$i+1]['date_sighting'] - $sighting['date_sighting'])/2;
}
}
$pSighting = $sightings[$pSightingIndex];
if ($this->extrapolateWithPollingFrequency) {
$lastSeenPositive = $pSighting['date_sighting'] + $halfTime;
} else {
$lastSeenPositive = $sightings[$i+1]['date_sighting'];
}
$this->__json['items'][] = array(
'attribute_id' => $attributeId,
'id' => sprintf('%s-%s', $attributeId, $sighting['id']),
@ -212,21 +227,26 @@
'group' => 'sighting_positive',
'timestamp' => $attribute['timestamp'],
'first_seen' => $sighting['date_sighting'],
'last_seen' => $pSighting['date_sighting'] + $halfTime,
'last_seen' => $lastSeenPositive,
);
// No next FP, extrapolate to now
$fpSighting = $sightings[$fpSightingIndex];
$secondNextPSightingIndex = $this->getNextPositiveSightingIndex($sightings, $fpSightingIndex+1);
if ($secondNextPSightingIndex === false) { // No next P, extrapolate to now
if ($this->extrapolateWithPollingFrequency) {
$firstSeenNegative = $fpSighting['date_sighting'] - $halfTime;
} else {
$firstSeenNegative = $fpSighting['date_sighting'];
}
$this->__json['items'][] = array(
'attribute_id' => $attributeId,
'id' => sprintf('%s-%s', $attributeId, $sighting['id']),
'uuid' => $sighting['uuid'],
'uuid' => $fpSighting['uuid'],
'content' => $attribute['value'],
'event_id' => $attribute['event_id'],
'group' => 'sighting_negative',
'timestamp' => $attribute['timestamp'],
'first_seen' => $pSighting['date_sighting'] - $halfTime,
'first_seen' => $firstSeenNegative,
'last_seen' => $now,
);
break;
@ -235,17 +255,22 @@
$pSightingIndex = $pSightingIndex+1;
$pSighting = $sightings[$pSightingIndex];
}
if ($this->extrapolateWithPollingFrequency) {
$firstSeenNegative = $fpSighting['date_sighting'] - $halfTime;
} else {
$firstSeenNegative = $fpSighting['date_sighting'];
}
// set down until next postive
$secondNextPSighting = $sightings[$secondNextPSightingIndex];
$this->__json['items'][] = array(
'attribute_id' => $attributeId,
'id' => sprintf('%s-%s', $attributeId, $sighting['id']),
'uuid' => $pSighting['uuid'],
'uuid' => $fpSighting['uuid'],
'content' => $attribute['value'],
'event_id' => $attribute['event_id'],
'group' => 'sighting_negative',
'timestamp' => $attribute['timestamp'],
'first_seen' => $pSighting['date_sighting'] - $halfTime,
'first_seen' => $firstSeenNegative,
'last_seen' => $secondNextPSighting['date_sighting'],
);
$i = $secondNextPSightingIndex;

32
app/Lib/Tools/JSONConverterTool.php
View File

@ -1,7 +1,7 @@
<?php
class JSONConverterTool
{
public function convertAttribute($attribute, $raw = false)
public static function convertAttribute($attribute, $raw = false)
{
$toRearrange = array('AttributeTag');
foreach ($toRearrange as $object) {
@ -12,14 +12,14 @@ class JSONConverterTool
}
// Submit as list to the attribute cleaner but obtain the only attribute
$attribute['Attribute'] = $this->__cleanAttributes(array($attribute['Attribute']))[0];
$attribute['Attribute'] = self::__cleanAttributes(array($attribute['Attribute']))[0];
if ($raw) {
return $attribute;
}
return json_encode($attribute, JSON_PRETTY_PRINT);
}
public function convertObject($object, $isSiteAdmin = false, $raw = false)
public static function convertObject($object, $isSiteAdmin = false, $raw = false)
{
$toRearrange = array('SharingGroup', 'Attribute', 'ShadowAttribute', 'Event');
foreach ($toRearrange as $element) {
@ -38,7 +38,7 @@ class JSONConverterTool
return json_encode($result, JSON_PRETTY_PRINT);
}
public function convert($event, $isSiteAdmin=false, $raw = false)
public static function convert($event, $isSiteAdmin=false, $raw = false)
{
$toRearrange = array('Org', 'Orgc', 'SharingGroup', 'Attribute', 'ShadowAttribute', 'RelatedAttribute', 'RelatedEvent', 'Galaxy', 'Object', 'EventReport');
foreach ($toRearrange as $object) {
@ -79,10 +79,10 @@ class JSONConverterTool
}
unset($event['Event']['user_id']);
if (isset($event['Event']['Attribute'])) {
$event['Event']['Attribute'] = $this->__cleanAttributes($event['Event']['Attribute'], $tempSightings);
$event['Event']['Attribute'] = self::__cleanAttributes($event['Event']['Attribute'], $tempSightings);
}
if (isset($event['Event']['Object'])) {
$event['Event']['Object'] = $this->__cleanObjects($event['Event']['Object'], $tempSightings);
$event['Event']['Object'] = self::__cleanObjects($event['Event']['Object'], $tempSightings);
}
unset($tempSightings);
unset($event['Event']['RelatedAttribute']);
@ -101,15 +101,15 @@ class JSONConverterTool
* @param array $event
* @return Generator<string>
*/
public function streamConvert(array $event)
public static function streamConvert(array $event)
{
$event = $this->convert($event, false, true);
$event = self::convert($event, false, true);
// Fast and inaccurate way how to check if event is too big for to convert in one call. This can be changed in future.
$isBigEvent = (isset($event['Event']['Attribute']) ? count($event['Event']['Attribute']) : 0) +
(isset($event['Event']['Object']) ? count($event['Event']['Object']) : 0) > 100;
if (!$isBigEvent) {
yield json_encode($event, JSON_UNESCAPED_UNICODE);
yield json_encode($event, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
return;
}
@ -120,11 +120,11 @@ class JSONConverterTool
yield ($firstKey === $key ? '' : ',') . json_encode($key) . ":[";
$firstInnerKey = key($value);
foreach ($value as $i => $attribute) {
yield ($firstInnerKey === $i ? '' : ',') . json_encode($attribute, JSON_UNESCAPED_UNICODE);
yield ($firstInnerKey === $i ? '' : ',') . json_encode($attribute, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
}
yield "]";
} else {
yield ($firstKey === $key ? '' : ',') . json_encode($key) . ":" . json_encode($value, JSON_UNESCAPED_UNICODE);
yield ($firstKey === $key ? '' : ',') . json_encode($key) . ":" . json_encode($value, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
}
}
if (isset($event['errors'])) {
@ -134,7 +134,7 @@ class JSONConverterTool
}
}
private function __cleanAttributes($attributes, $tempSightings = array())
private static function __cleanAttributes($attributes, $tempSightings = array())
{
// remove value1 and value2 from the output and remove invalid utf8 characters for the xml parser
foreach ($attributes as $key => $attribute) {
@ -157,11 +157,11 @@ class JSONConverterTool
return $attributes;
}
private function __cleanObjects($objects, $tempSightings = array())
private static function __cleanObjects($objects, $tempSightings = array())
{
foreach ($objects as $k => $object) {
if (!empty($object['Attribute'])) {
$objects[$k]['Attribute'] = $this->__cleanAttributes($object['Attribute'], $tempSightings);
$objects[$k]['Attribute'] = self::__cleanAttributes($object['Attribute'], $tempSightings);
} else {
unset($objects[$k]);
}
@ -170,12 +170,12 @@ class JSONConverterTool
return $objects;
}
public function arrayPrinter($array, $root = true)
public static function arrayPrinter($array, $root = true)
{
if (is_array($array)) {
$resultArray = array();
foreach ($array as $k => $element) {
$temp = $this->arrayPrinter($element, false);
$temp = self::arrayPrinter($element, false);
if (!is_array($temp)) {
$resultArray[] = '[' . $k .']' . $temp;
} else {

16
app/Lib/Tools/ProcessTool.php
View File

@ -73,13 +73,11 @@ class ProcessTool
$commandForException = self::commandFormat($command);
throw new Exception("Could not get STDOUT of command '$commandForException'.");
}
fclose($pipes[1]);
if ($stderrToFile) {
$stderr = null;
} else {
$stderr = stream_get_contents($pipes[2]);
fclose($pipes[2]);
}
$returnCode = proc_close($process);
@ -95,6 +93,20 @@ class ProcessTool
return $stdout;
}
/**
* Get current process user name
* @return string
* @throws ProcessException
*/
public static function whoami()
{
if (function_exists('posix_getpwuid') && function_exists('posix_geteuid')) {
return posix_getpwuid(posix_geteuid())['name'];
} else {
return rtrim(self::execute(['whoami']));
}
}
/**
* @return string
*/

3
app/Lib/Tools/PubSubTool.php
View File

@ -78,8 +78,7 @@ class PubSubTool
public function publishEvent($event)
{
App::uses('JSONConverterTool', 'Tools');
$jsonTool = new JSONConverterTool();
$json = $jsonTool->convert($event, false, true);
$json = JSONConverterTool::convert($event, false, true);
return $this->pushToRedis('data:misp_json', $json);
}

16
app/Lib/Tools/QueryTool.php
View File

@ -2,24 +2,24 @@
class QueryTool
{
private $__pdoMap = array(
'integer' => PDO::PARAM_INT,
'float' => PDO::PARAM_STR,
'boolean' => PDO::PARAM_BOOL,
'string' => PDO::PARAM_STR,
'text' => PDO::PARAM_STR
const PDO_MAP = array(
'integer' => PDO::PARAM_INT,
'float' => PDO::PARAM_STR,
'boolean' => PDO::PARAM_BOOL,
'string' => PDO::PARAM_STR,
'text' => PDO::PARAM_STR,
);
public function quickDelete($table, $field, $value, $model)
{
$db = $model->getDataSource();
$connection = $db->getConnection();
if ($db->config['datasource'] == 'Database/Mysql' || $db->config['datasource'] == 'Database/MysqlObserver') {
if ($db->config['datasource'] === 'Database/Mysql' || $db->config['datasource'] === 'Database/MysqlObserver') {
$query = $connection->prepare('DELETE FROM ' . $table . ' WHERE ' . $field . ' = :value');
} elseif ($db->config['datasource'] == 'Database/Postgres' ) {
$query = $connection->prepare('DELETE FROM "' . $table . '" WHERE "' . $field . '" = :value');
}
$query->bindValue(':value', $value, $this->__pdoMap[$db->introspectType($value)]);
$query->bindValue(':value', $value, self::PDO_MAP[$db->introspectType($value)]);
$query->execute();
}
}

80
app/Lib/Tools/SecurityAudit.php
View File

@ -43,6 +43,10 @@ class SecurityAudit
$output['Database'][] = ['warning', __('Database password is too short, should be at least %s chars long.', self::STRONG_PASSWORD_LENGTH)];
}
if (!Configure::read('Security.encryption_key')) {
$output['Database'][] = ['warning', __('Sensitive information like keys to remote server are stored in database unencrypted. Set `Security.encryption_key` to encrypt these values.')];
}
$passwordPolicyLength = Configure::read('Security.password_policy_length') ?: $server->serverSettings['Security']['password_policy_length']['value'];
if ($passwordPolicyLength < 8) {
$output['Password'][] = ['error', __('Minimum password length is set to %s, it is highly advised to increase it.', $passwordPolicyLength)];
@ -209,19 +213,28 @@ class SecurityAudit
}
} catch (RuntimeException $e) {}
if (version_compare(PHP_VERSION, '7.3.0', '<')) {
if (version_compare(PHP_VERSION, '7.4.0', '<')) {
$output['PHP'][] = [
'warning',
__('PHP version %s is not supported anymore. It can be still supported by your distribution. ', PHP_VERSION),
'https://www.php.net/supported-versions.php'
];
} else if (version_compare(PHP_VERSION, '7.4.0', '<')) {
$output['PHP'][] = [
'hint',
__('PHP version 7.3 will not be supported after 6 Dec 2021. Even beyond that date, it can be still supported by your distribution.'),
__('PHP version %s is not supported anymore. It can be still supported by your distribution.', PHP_VERSION),
'https://www.php.net/supported-versions.php'
];
}
if (ini_get('expose_php')) {
$output['PHP'][] = [
'hint',
__('PHP `expose_php` setting is enabled. That means that PHP version will be send in `X-Powered-By` header. This can help attackers.'),
];
}
if (extension_loaded('xdebug')) {
$output['PHP'][] = [
'error',
__('The xdebug extension can reveal code and data to an attacker.'),
];
}
if (ini_get('session.use_strict_mode') != 1) {
$output['PHP'][] = [
'warning',
@ -256,7 +269,7 @@ class SecurityAudit
];
}
$this->system($server, $output);
$this->system($output);
return $output;
}
@ -369,7 +382,7 @@ class SecurityAudit
}
}
private function system(Server $server, array &$output)
private function system(array &$output)
{
$kernelBuildTime = $this->getKernelBuild();
if ($kernelBuildTime) {
@ -422,27 +435,30 @@ class SecurityAudit
} catch (Exception $e) {
}
$ubuntuVersion = $this->getUbuntuVersion();
if ($ubuntuVersion) {
if (in_array($ubuntuVersion, ['14.04', '19.10'])) {
$output['System'][] = [
'warning',
__('You are using Ubuntu %s. This version doesn\'t receive security support anymore.', $ubuntuVersion),
'https://endoflife.date/ubuntu',
];
} else if (in_array($ubuntuVersion, ['16.04'])) {
$output['System'][] = [
'hint',
__('You are using Ubuntu %s. This version will be not supported after 02 Apr 2021.', $ubuntuVersion),
'https://endoflife.date/ubuntu',
];
$linuxVersion = $this->getLinuxVersion();
if ($linuxVersion) {
list($name, $version) = $linuxVersion;
if ($name === 'Ubuntu') {
if (in_array($version, ['14.04', '19.10'], true)) {
$output['System'][] = [
'warning',
__('You are using Ubuntu %s. This version doesn\'t receive security support anymore.', $version),
'https://endoflife.date/ubuntu',
];
} else if (in_array($version, ['16.04'], true)) {
$output['System'][] = [
'hint',
__('You are using Ubuntu %s. This version will be not supported after 02 Apr 2021.', $version),
'https://endoflife.date/ubuntu',
];
}
}
}
}
private function getKernelBuild()
{
if (!php_uname('s') !== 'Linux') {
if (PHP_OS !== 'Linux') {
return false;
}
@ -454,9 +470,9 @@ class SecurityAudit
return new DateTime($buildTime);
}
private function getUbuntuVersion()
private function getLinuxVersion()
{
if (!php_uname('s') !== 'Linux') {
if (PHP_OS !== 'Linux') {
return false;
}
if (!is_readable('/etc/os-release')) {
@ -470,16 +486,10 @@ class SecurityAudit
if ($parsed === false) {
return false;
}
if (!isset($parsed['NAME'])) {
if (!isset($parsed['NAME']) || !isset($parsed['VERSION_ID'])) {
return false;
}
if ($parsed['NAME'] !== 'Ubuntu') {
return false;
}
if (!isset($parsed['VERSION_ID'])) {
return false;
}
return $parsed['VERSION_ID'];
return [$parsed['NAME'], $parsed['VERSION_ID']];
}
/**

28
app/Lib/Tools/ServerSyncTool.php
View File

@ -219,6 +219,34 @@ class ServerSyncTool
return $this->server['Server']['id'];
}
/**
* @param string $key
* @return array
*/
public function decodeRule($key)
{
$rules = $this->server['Server'][$key];
return json_decode($rules, true);
}
/**
* @return array
*/
public function pullRules()
{
return $this->decodeRule('pull_rules');
}
/**
* @return array
*/
public function pushRules()
{
return $this->decodeRule('push_rules');
}
/**
* @param string $flag
* @return bool

15
app/Lib/Tools/TmpFileTool.php
View File

@ -67,6 +67,21 @@ class TmpFileTool
}
}
/**
* @param string $path
* @throws Exception
*/
public function writeFromFile($path)
{
$file = fopen($path, 'r');
if (!$file) {
throw new Exception("Could not open file $file.");
}
if (stream_copy_to_stream($file, $this->tmpfile) === false) {
throw new Exception("Could not copy content of file $file into TmpFile.");
}
}
/**
* Returns generator of parsed CSV line from file.
*

5
app/Model/AppModel.php
View File

@ -1948,11 +1948,6 @@ class AppModel extends Model
return true;
}
public function getPythonVersion()
{
return Configure::read('MISP.python_bin') ?: 'python3';
}
public function validateAuthkey($value)
{
if (empty($value['authkey'])) {

61
app/Model/Attribute.php
View File

@ -9,6 +9,7 @@ App::uses('AttachmentTool', 'Tools');
App::uses('TmpFileTool', 'Tools');
App::uses('ComplexTypeTool', 'Tools');
App::uses('AttributeValidationTool', 'Tools');
App::uses('JsonTool', 'Tools');
/**
* @property Event $Event
@ -125,7 +126,7 @@ class Attribute extends AppModel
'hostname|port',
);
public $captureFields = array(
const CAPTURE_FIELDS = array(
'event_id',
'category',
'type',
@ -483,7 +484,7 @@ class Attribute extends AppModel
$result = $this->saveAttachment($attribute);
}
}
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_attribute_notifications_enable');
$pubToZmq = $this->pubToZmq('attribute');
$kafkaTopic = $this->kafkaTopic('attribute');
if ($pubToZmq || $kafkaTopic) {
$attributeForPublish = $this->fetchAttribute($this->id);
@ -535,7 +536,7 @@ class Attribute extends AppModel
// update correlation..
$this->Correlation->beforeSaveCorrelation($this->data['Attribute']);
if (!empty($this->data['Attribute']['id'])) {
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_attribute_notifications_enable')) {
if ($this->pubToZmq('attribute')) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->attribute_save($this->data, 'delete');
}
@ -952,16 +953,16 @@ class Attribute extends AppModel
* @param bool $thumbnail
* @param int $maxWidth - When $thumbnail is true
* @param int $maxHeight - When $thumbnail is true
* @return string
* @return string|File
* @throws Exception
*/
public function getPictureData(array $attribute, $thumbnail=false, $maxWidth=200, $maxHeight=200)
public function getPictureData(array $attribute, $thumbnail = false, $maxWidth = 200, $maxHeight = 200)
{
if ($thumbnail && extension_loaded('gd')) {
if ($maxWidth == 200 && $maxHeight == 200) {
// Return thumbnail directly if already exists
try {
return $this->getAttachment($attribute['Attribute'], $path_suffix = '_thumbnail');
return $this->loadAttachmentTool()->getFile($attribute['Attribute']['event_id'], $attribute['Attribute']['id'], $path_suffix = '_thumbnail');
} catch (NotFoundException $e) {
// pass
}
@ -976,11 +977,10 @@ class Attribute extends AppModel
$attribute['Attribute']['data'] = $imageData;
$this->saveAttachment($attribute['Attribute'], $path_suffix='_thumbnail');
}
} else {
$imageData = $this->getAttachment($attribute['Attribute']);
return $imageData;
}
return $imageData;
return $this->loadAttachmentTool()->getFile($attribute['Attribute']['event_id'], $attribute['Attribute']['id']);
}
/**
@ -1724,7 +1724,6 @@ class Attribute extends AppModel
}
}
public function checkTemplateAttributes($template, $data, $event_id)
{
$result = array();
@ -2878,10 +2877,7 @@ class Attribute extends AppModel
if (!$this->Warninglist->filterWarninglistAttribute($attribute)) {
$this->validationErrors['warninglist'] = 'Attribute could not be saved as it trips over a warninglist and enforceWarninglist is enforced.';
$validationErrors = $this->validationErrors['warninglist'];
$this->loadLog()->createLogEntry($user, 'add', 'Attribute', 0,
'Attribute dropped due to validation for Event ' . $eventId . ' failed',
'Validation errors: ' . json_encode($this->validationErrors) . ' Full Attribute: ' . json_encode($attribute)
);
$this->logDropped($user, $attribute);
return $attribute;
}
}
@ -2893,7 +2889,7 @@ class Attribute extends AppModel
$attribute['distribution'] = $this->defaultDistribution();
}
$params = array(
'fieldList' => $this->captureFields
'fieldList' => self::CAPTURE_FIELDS,
);
if (!empty($parentEvent)) {
$params['parentEvent'] = $parentEvent;
@ -2905,12 +2901,8 @@ class Attribute extends AppModel
unset($attribute['sharing_group_id']);
}
}
if (!$this->save($attribute, $params)) {
$attribute_short = (isset($attribute['category']) ? $attribute['category'] : 'N/A') . '/' . (isset($attribute['type']) ? $attribute['type'] : 'N/A') . ' ' . (isset($attribute['value']) ? $attribute['value'] : 'N/A');
$this->loadLog()->createLogEntry($user, 'add', 'Attribute', 0,
'Attribute dropped due to validation for Event ' . $eventId . ' failed: ' . $attribute_short,
'Validation errors: ' . json_encode($this->validationErrors) . ' Full Attribute: ' . json_encode($attribute)
);
if (!$this->save(['Attribute' => $attribute], $params)) {
$this->logDropped($user, $attribute);
} else {
if (!empty($attribute['AttributeTag'])) {
$toSave = [];
@ -3027,12 +3019,8 @@ class Attribute extends AppModel
$fieldList[] = 'object_id';
$fieldList[] = 'object_relation';
}
if (!$this->save($attribute, ['fieldList' => $fieldList, 'parentEvent' => $event])) {
$attribute_short = (isset($attribute['category']) ? $attribute['category'] : 'N/A') . '/' . (isset($attribute['type']) ? $attribute['type'] : 'N/A') . ' ' . (isset($attribute['value']) ? $attribute['value'] : 'N/A');
$this->loadLog()->createLogEntry($user, 'edit', 'Attribute', 0,
'Attribute dropped due to validation for Event ' . $eventId . ' failed: ' . $attribute_short,
'Validation errors: ' . json_encode($this->validationErrors) . ' Full Attribute: ' . json_encode($attribute)
);
if (!$this->save(['Attribute' => $attribute], ['fieldList' => $fieldList, 'parentEvent' => $event])) {
$this->logDropped($user, $attribute, 'edit');
return $this->validationErrors;
}
if (!empty($attribute['Sighting'])) {
@ -3462,6 +3450,25 @@ class Attribute extends AppModel
return $distribution;
}
/**
* Log when attribute was dropped due to validation errors.
*
* @param array $user
* @param array $attribute
* @param string $action
* @throws JsonException
*/
public function logDropped(array $user, array $attribute, $action = 'add')
{
$attribute_short = (isset($attribute['category']) ? $attribute['category'] : 'N/A') . '/' . (isset($attribute['type']) ? $attribute['type'] : 'N/A') . ' ' . (isset($attribute['value']) ? $attribute['value'] : 'N/A');
$eventId = $attribute['event_id'];
$modelId = $action === 'add' ? 0 : $this->id;
$this->loadLog()->createLogEntry($user, 'add', 'Attribute', $modelId,
"Attribute dropped due to validation for Event $eventId failed: $attribute_short",
'Validation errors: ' . JsonTool::encode($this->validationErrors) . ' Full Attribute: ' . JsonTool::encode($attribute)
);
}
public function __isset($name)
{
if ($name === 'typeDefinitions' || $name === 'categoryDefinitions') {

25
app/Model/AuthKey.php
View File

@ -129,23 +129,30 @@ class AuthKey extends AppModel
/**
* @param string $authkey
* @param bool $includeExpired
* @return array|false
*/
public function getAuthUserByAuthKey($authkey)
public function getAuthUserByAuthKey($authkey, $includeExpired = false)
{
$start = substr($authkey, 0, 4);
$end = substr($authkey, -4);
$conditions = [
'authkey_start' => $start,
'authkey_end' => $end,
];
if (!$includeExpired) {
$conditions['OR'] = [
'expiration >' => time(),
'expiration' => 0
];
}
$possibleAuthkeys = $this->find('all', [
'recursive' => -1,
'fields' => ['id', 'authkey', 'user_id', 'expiration', 'allowed_ips', 'read_only'],
'conditions' => [
'OR' => [
'expiration >' => time(),
'expiration' => 0
],
'authkey_start' => $start,
'authkey_end' => $end,
]
'conditions' => $conditions,
]);
$passwordHasher = $this->getHasher();
foreach ($possibleAuthkeys as $possibleAuthkey) {

53
app/Model/Behavior/AuditLogBehavior.php
View File

@ -4,7 +4,10 @@ App::uses('AuditLog', 'Model');
class AuditLogBehavior extends ModelBehavior
{
/** @var array|null */
private $old;
private $beforeSave;
/** @var array|null */
private $beforeDelete;
/** @var AuditLog|null */
private $AuditLog;
@ -13,7 +16,7 @@ class AuditLogBehavior extends ModelBehavior
private $enabled;
// Hash is faster that in_array
private $skipFields = [
const SKIP_FIELDS = [
'id' => true,
'lastpushedid' => true,
'timestamp' => true,
@ -85,7 +88,7 @@ class AuditLogBehavior extends ModelBehavior
$fieldToFetch = [];
if (!empty($options['fieldList'])) {
foreach ($options['fieldList'] as $field) {
if (!isset($this->skipFields[$field])) {
if (!isset(self::SKIP_FIELDS[$field])) {
$fieldToFetch[] = $field;
}
}
@ -96,20 +99,25 @@ class AuditLogBehavior extends ModelBehavior
$fieldToFetch[] = 'event_id';
}
// Fetch fields that are necessary to fill object title
if (isset($this->modelInfo[$model->name]) && is_string($this->modelInfo[$model->name]) && !in_array($this->modelInfo[$model->name], $fieldToFetch, true)) {
$fieldToFetch[] = $this->modelInfo[$model->name];
}
if (empty($fieldToFetch)) {
$this->old = null;
$this->beforeSave = null;
return true;
}
}
if ($model->id) {
$this->old = $model->find('first', [
$this->beforeSave = $model->find('first', [
'conditions' => [$model->alias . '.' . $model->primaryKey => $model->id],
'recursive' => -1,
'callbacks' => false,
'fields' => $fieldToFetch,
]);
} else {
$this->old = null;
$this->beforeSave = null;
}
return true;
}
@ -130,13 +138,13 @@ class AuditLogBehavior extends ModelBehavior
if (isset($data['deleted'])) {
if ($data['deleted']) {
$action = AuditLog::ACTION_SOFT_DELETE;
} else if (isset($this->old[$model->alias]['deleted']) && $this->old[$model->alias]['deleted']) {
} else if (isset($this->beforeSave[$model->alias]['deleted']) && $this->beforeSave[$model->alias]['deleted']) {
$action = AuditLog::ACTION_UNDELETE;
}
}
}
$changedFields = $this->changedFields($model, $options['fieldList']);
$changedFields = $this->changedFields($model, $this->beforeSave, $options['fieldList']);
if (empty($changedFields)) {
return;
}
@ -145,8 +153,8 @@ class AuditLogBehavior extends ModelBehavior
$eventId = $id;
} else if (isset($data['event_id'])) {
$eventId = $data['event_id'];
} else if (isset($this->old[$model->alias]['event_id'])) {
$eventId = $this->old[$model->alias]['event_id'];
} else if (isset($this->beforeSave[$model->alias]['event_id'])) {
$eventId = $this->beforeSave[$model->alias]['event_id'];
} else {
$eventId = null;
}
@ -155,11 +163,11 @@ class AuditLogBehavior extends ModelBehavior
if (isset($this->modelInfo[$model->name])) {
$modelTitleField = $this->modelInfo[$model->name];
if (is_callable($modelTitleField)) {
$modelTitle = $modelTitleField($data, isset($this->old[$model->alias]) ? $this->old[$model->alias] : []);
$modelTitle = $modelTitleField($data, isset($this->beforeSave[$model->alias]) ? $this->beforeSave[$model->alias] : []);
} else if (isset($data[$modelTitleField])) {
$modelTitle = $data[$modelTitleField];
} else if ($this->old[$model->alias][$modelTitleField]) {
$modelTitle = $this->old[$model->alias][$modelTitleField];
} else if ($this->beforeSave[$model->alias][$modelTitleField]) {
$modelTitle = $this->beforeSave[$model->alias][$modelTitleField];
}
}
@ -198,6 +206,8 @@ class AuditLogBehavior extends ModelBehavior
'event_id' => $eventId,
'change' => $changedFields,
]]);
$this->beforeSave = null; // cleanup
}
public function beforeDelete(Model $model, $cascade = true)
@ -206,7 +216,7 @@ class AuditLogBehavior extends ModelBehavior
return true;
}
$this->old = $model->find('first', [
$this->beforeDelete = $model->find('first', [
'conditions' => array($model->alias . '.' . $model->primaryKey => $model->id),
'recursive' => -1,
'callbacks' => false,
@ -219,8 +229,8 @@ class AuditLogBehavior extends ModelBehavior
if (!$this->enabled) {
return;
}
$model->data = $this->old;
$this->old = null;
$model->data = $this->beforeDelete;
$this->beforeDelete = null;
if ($model->name === 'Event') {
$eventId = $model->id;
} else {
@ -266,7 +276,7 @@ class AuditLogBehavior extends ModelBehavior
'model_id' => $id,
'model_title' => $modelTitle,
'event_id' => $eventId,
'change' => $this->changedFields($model),
'change' => $this->changedFields($model, null),
]]);
}
@ -311,16 +321,17 @@ class AuditLogBehavior extends ModelBehavior
/**
* @param Model $model
* @param array|null $oldData Array with alias
* @param array|null $fieldsToSave
* @return array
*/
private function changedFields(Model $model, $fieldsToSave = null)
private function changedFields(Model $model, $oldData, $fieldsToSave = null)
{
$dbFields = $model->schema();
$changedFields = [];
$hasPrimaryField = isset($model->data[$model->alias][$model->primaryKey]);
foreach ($model->data[$model->alias] as $key => $value) {
if (isset($this->skipFields[$key])) {
if (isset(self::SKIP_FIELDS[$key])) {
continue;
}
if (!isset($dbFields[$key])) {
@ -330,8 +341,8 @@ class AuditLogBehavior extends ModelBehavior
continue;
}
if ($hasPrimaryField && isset($this->old[$model->alias][$key])) {
$old = $this->old[$model->alias][$key];
if ($hasPrimaryField && isset($oldData[$model->alias][$key])) {
$old = $oldData[$model->alias][$key];
} else {
$old = null;
}

34
app/Model/Event.php
View File

@ -395,7 +395,7 @@ class Event extends AppModel
}
if (!empty($this->data['Event']['id'])) {
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) {
if ($this->pubToZmq('event')) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->event_save(array('Event' => $this->data['Event']), 'delete');
}
@ -522,7 +522,7 @@ class Event extends AppModel
$this->Attribute->Correlation->updateAll($updateCorrelation, ['Correlation.event_id' => (int)$event['id']]);
}
}
if (empty($event['unpublishAction']) && empty($event['skip_zmq']) && Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) {
if (empty($event['unpublishAction']) && empty($event['skip_zmq']) && $this->pubToZmq('event')) {
$pubSubTool = $this->getPubSubTool();
$eventForZmq = $this->quickFetchEvent($event['id']);
if (!empty($event)) {
@ -1117,11 +1117,15 @@ class Event extends AppModel
return $data;
}
private function __prepareAttributesForSync($data, $server)
private function __prepareAttributesForSync($data,$server, $pushRules)
{
// prepare attribute for sync
if (!empty($data['Attribute'])) {
foreach ($data['Attribute'] as $key => $attribute) {
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && in_array($attribute['type'], $pushRules['type_attributes']['NOT'])) {
unset($data['Attribute'][$key]);
continue;
}
$data['Attribute'][$key] = $this->__updateAttributeForSync($attribute, $server);
if (empty($data['Attribute'][$key])) {
unset($data['Attribute'][$key]);
@ -1134,16 +1138,20 @@ class Event extends AppModel
return $data;
}
private function __prepareObjectsForSync($data, $server)
private function __prepareObjectsForSync($data,$server, $pushRules)
{
// prepare Object for sync
if (!empty($data['Object'])) {
foreach ($data['Object'] as $key => $object) {
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && in_array($object['template_uuid'], $pushRules['type_objects']['NOT'])) {
unset($data['Object'][$key]);
continue;
}
$data['Object'][$key] = $this->__updateObjectForSync($object, $server);
if (empty($data['Object'][$key])) {
unset($data['Object'][$key]);
} else {
$data['Object'][$key] = $this->__prepareAttributesForSync($data['Object'][$key], $server);
$data['Object'][$key] = $this->__prepareAttributesForSync($data['Object'][$key], $server, $pushRules);
}
}
$data['Object'] = array_values($data['Object']);
@ -1184,14 +1192,17 @@ class Event extends AppModel
}
}
}
$event['Event'] = $this->__prepareAttributesForSync($event['Event'], $server);
$event['Event'] = $this->__prepareObjectsForSync($event['Event'], $server);
$event['Event'] = $this->__prepareEventReportForSync($event['Event'], $server);
$pushRules = $this->jsonDecode($server['Server']['push_rules']);
$event['Event'] = $this->__prepareAttributesForSync($event['Event'], $server, $pushRules);
$event['Event'] = $this->__prepareObjectsForSync($event['Event'], $server, $pushRules);
$event['Event'] = $this->__prepareEventReportForSync($event['Event'], $server, $pushRules);
// Downgrade the event from connected communities to community only
if (!$server['Server']['internal'] && $event['Event']['distribution'] == 2) {
$event['Event']['distribution'] = 1;
}
return $event;
}
@ -3963,7 +3974,7 @@ class Event extends AppModel
$server['Server']['internal'] = false;
}
// If the event exists...
if (count($existingEvent)) {
if (!empty($existingEvent)) {
$data['Event']['id'] = $existingEvent['Event']['id'];
$id = $existingEvent['Event']['id'];
// Conditions affecting all:
@ -4008,7 +4019,7 @@ class Event extends AppModel
$changed = false;
// If a field is not set in the request, just reuse the old value
// Also, compare the event to the existing event and see whether this is a meaningful change
$recoverFields = array('analysis', 'threat_level_id', 'info', 'distribution', 'date');
$recoverFields = array('analysis', 'threat_level_id', 'info', 'distribution', 'date', 'org_id');
foreach ($recoverFields as $rF) {
if (!isset($data['Event'][$rF])) {
$data['Event'][$rF] = $existingEvent['Event'][$rF];
@ -5742,9 +5753,8 @@ class Event extends AppModel
$standard_format = !empty($module['meta']['require_standard_format']);
if ($standard_format) {
App::uses('JSONConverterTool', 'Tools');
$converter = new JSONConverterTool();
foreach ($events as $k => $event) {
$events[$k] = $converter->convert($event, false, true);
$events[$k] = JSONConverterTool::convert($event, false, true);
}
}
$modulePayload['data'] = $events;

61
app/Model/Feed.php
View File

@ -915,11 +915,49 @@ class Feed extends AppModel
$event['Event']['Orgc'] = $event['Event']['orgc'];
unset($event['Event']['orgc']);
}
$event['Event']['distribution'] = $feed['Feed']['distribution'];
$event['Event']['sharing_group_id'] = $feed['Feed']['sharing_group_id'];
if (!empty($event['Event']['Attribute'])) {
foreach ($event['Event']['Attribute'] as $key => $attribute) {
$event['Event']['Attribute'][$key]['distribution'] = 5;
if (5 == $feed['Feed']['distribution'] && isset($event['Event']['distribution'])) {
// We inherit the distribution from the feed and should not rewrite the distributions.
// MISP magically parses the Sharing Group info and creates the SG if it didn't exist.
} else {
// rewrite the distributions to the one configured by the Feed settings
// overwrite Event distribution
if (5 == $feed['Feed']['distribution']) {
// We said to inherit the distribution from the feed, but the feed does not contain distribution
// rewrite the event to My org only distribution, and set all attributes to inherit the event distribution
$event['Event']['distribution'] = 0;
$event['Event']['sharing_group_id'] = 0;
} else {
$event['Event']['distribution'] = $feed['Feed']['distribution'];
$event['Event']['sharing_group_id'] = $feed['Feed']['sharing_group_id'];
if ($feed['Feed']['sharing_group_id']) {
$sg = $this->SharingGroup->find('first', array(
'recursive' => -1,
'conditions' => array('SharingGroup.id' => $feed['Feed']['sharing_group_id'])
));
if (!empty($sg)) {
$event['Event']['SharingGroup'] = $sg['SharingGroup'];
} else {
// We have an SG ID for the feed, but the SG is gone. Make the event private as a fall-back.
$event['Event']['distribution'] = 0;
$event['Event']['sharing_group_id'] = 0;
}
}
}
// overwrite Attributes and Objects distribution to Inherit
if (!empty($event['Event']['Attribute'])) {
foreach ($event['Event']['Attribute'] as $key => $attribute) {
$event['Event']['Attribute'][$key]['distribution'] = 5;
}
}
if (!empty($event['Event']['Object'])) {
foreach ($event['Event']['Object'] as $key => $object) {
$event['Event']['Object'][$key]['distribution'] = 5;
if (!empty($event['Event']['Object'][$key]['Attribute'])) {
foreach ($event['Event']['Object'][$key]['Attribute'] as $a_key => $attribute) {
$event['Event']['Object'][$key]['Attribute'][$a_key]['distribution'] = 5;
}
}
}
}
}
if ($feed['Feed']['tag_id']) {
@ -940,19 +978,6 @@ class Feed extends AppModel
}
}
}
if ($feed['Feed']['sharing_group_id']) {
$sg = $this->SharingGroup->find('first', array(
'recursive' => -1,
'conditions' => array('SharingGroup.id' => $feed['Feed']['sharing_group_id'])
));
if (!empty($sg)) {
$event['Event']['SharingGroup'] = $sg['SharingGroup'];
} else {
// We have an SG ID for the feed, but the SG is gone. Make the event private as a fall-back.
$event['Event']['distribution'] = 0;
$event['Event']['sharing_group_id'] = 0;
}
}
if (!$this->__checkIfEventBlockedByFilter($event, $filterRules)) {
return 'blocked';
}

4
app/Model/GalaxyCluster.php
View File

@ -127,12 +127,12 @@ class GalaxyCluster extends AppModel
}
if (isset($result[$this->alias]['org_id']) && $results[$k][$this->alias]['org_id'] == 0) {
if (isset($results[$k]['Org'])) {
$results[$k]['Org'] = $this->Org->genericMISPOrganisation;
$results[$k]['Org'] = Organisation::GENERIC_MISP_ORGANISATION;
}
}
if (isset($result[$this->alias]['orgc_id']) && $results[$k][$this->alias]['orgc_id'] == 0) {
if (isset($results[$k]['Orgc'])) {
$results[$k]['Orgc'] = $this->Org->genericMISPOrganisation;
$results[$k]['Orgc'] = Organisation::GENERIC_MISP_ORGANISATION;
}
}

8
app/Model/Log.php
View File

@ -3,14 +3,14 @@ App::uses('AppModel', 'Model');
class Log extends AppModel
{
public $warningActions = array(
const WARNING_ACTIONS = array(
'warning',
'change_pw',
'login_fail',
'version_warning',
'auth_fail'
);
public $errorActions = array(
const ERROR_ACTIONS = array(
'error'
);
public $validate = array(
@ -382,10 +382,10 @@ class Log extends AppModel
if ($this->syslog) {
$action = 'info';
if (isset($data['Log']['action'])) {
if (in_array($data['Log']['action'], $this->errorActions, true)) {
if (in_array($data['Log']['action'], self::ERROR_ACTIONS, true)) {
$action = 'err';
}
if (in_array($data['Log']['action'], $this->warningActions, true)) {
if (in_array($data['Log']['action'], self::WARNING_ACTIONS, true)) {
$action = 'warning';
}
}

43
app/Model/MispObject.php
View File

@ -300,9 +300,7 @@ class MispObject extends AppModel
public function afterSave($created, $options = array())
{
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable') &&
Configure::read('Plugin.ZeroMQ_object_notifications_enable') &&
empty($this->data['Object']['skip_zmq']);
$pubToZmq = $this->pubToZmq('object') && empty($this->data['Object']['skip_zmq']);
$kafkaTopic = $this->kafkaTopic('object');
$pubToKafka = $kafkaTopic && empty($this->data['Object']['skip_kafka']);
if ($pubToZmq || $pubToKafka) {
@ -329,10 +327,9 @@ class MispObject extends AppModel
public function beforeDelete($cascade = true)
{
if (!empty($this->data['Object']['id'])) {
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_object_notifications_enable');
$kafkaTopic = Configure::read('Plugin.Kafka_object_notifications_topic');
$pubToKafka = Configure::read('Plugin.Kafka_enable') && Configure::read('Plugin.Kafka_object_notifications_enable') && !empty($kafkaTopic);
if ($pubToZmq || $pubToKafka) {
$pubToZmq = $this->pubToZmq('object');
$kafkaTopic = $this->kafkaTopic('object');
if ($pubToZmq || $kafkaTopic) {
$object = $this->find('first', array(
'recursive' => -1,
'conditions' => array('Object.id' => $this->data['Object']['id'])
@ -341,7 +338,7 @@ class MispObject extends AppModel
$pubSubTool = $this->getPubSubTool();
$pubSubTool->object_save($object, 'delete');
}
if ($pubToKafka) {
if ($kafkaTopic) {
$kafkaPubTool = $this->getKafkaPubTool();
$kafkaPubTool->publishJson($kafkaTopic, $object, 'delete');
}
@ -737,7 +734,7 @@ class MispObject extends AppModel
* @throws InternalErrorException
* @throws Exception
*/
public function attributeCleanup($attributes)
public function attributeCleanup(array $attributes)
{
if (empty($attributes['Attribute'])) {
return $attributes;
@ -820,7 +817,7 @@ class MispObject extends AppModel
return $object;
}
public function deltaMerge($object, $objectToSave, $onlyAddNewAttribute=false, $user)
public function deltaMerge(array $object, array $objectToSave, $onlyAddNewAttribute=false, array $user)
{
if (!isset($objectToSave['Object'])) {
$dataToBackup = array('ObjectReferences', 'Attribute', 'ShadowAttribute');
@ -851,8 +848,8 @@ class MispObject extends AppModel
$object['Object']['sharing_group_id'] = $objectToSave['Object']['sharing_group_id'];
}
}
$date = new DateTime();
$object['Object']['timestamp'] = $date->getTimestamp();
$time = time();
$object['Object']['timestamp'] = $time;
$forcedSeenOnElements = array();
if (isset($objectToSave['Object']['first_seen'])) {
$forcedSeenOnElements['first_seen'] = $objectToSave['Object']['first_seen'];
@ -886,10 +883,12 @@ class MispObject extends AppModel
$newAttribute['id'] = $originalAttribute['id'];
$newAttribute['event_id'] = $object['Object']['event_id'];
$newAttribute['object_id'] = $object['Object']['id'];
$newAttribute['timestamp'] = $date->getTimestamp();
$newAttribute['timestamp'] = $time;
$result = $this->Event->Attribute->save(array('Attribute' => $newAttribute), array('fieldList' => Attribute::EDITABLE_FIELDS));
if ($result) {
$this->Event->Attribute->AttributeTag->handleAttributeTags($user, $newAttribute, $newAttribute['event_id'], $capture=true);
} else {
$this->Event->Attribute->logDropped($user, $newAttribute, 'edit');
}
}
unset($object['Attribute'][$origKey]);
@ -897,7 +896,6 @@ class MispObject extends AppModel
}
}
}
$this->Event->Attribute->create();
$newAttribute['event_id'] = $object['Object']['event_id'];
$newAttribute['object_id'] = $object['Object']['id'];
// Set seen of object at attribute level
@ -914,20 +912,19 @@ class MispObject extends AppModel
}
}
if (!isset($newAttribute['distribution'])) {
$newAttribute['distribution'] = Configure::read('MISP.default_attribute_distribution');
if ($newAttribute['distribution'] == 'event') {
$newAttribute['distribution'] = 5;
}
$newAttribute['distribution'] = $this->Event->Attribute->defaultDistribution();
}
$this->Event->Attribute->create();
$saveResult = $this->Event->Attribute->save($newAttribute);
if ($saveResult) {
$newAttribute['id'] = $this->Event->Attribute->id;
$this->Event->Attribute->AttributeTag->handleAttributeTags($user, $newAttribute, $newAttribute['event_id'], $capture=true);
} else {
$this->Event->Attribute->logDropped($user, $newAttribute, 'add');
}
$attributeArrays['add'][] = $newAttribute;
unset($objectToSave['Attribute'][$newKey]);
}
foreach ($object['Attribute'] as $origKey => $originalAttribute) {
foreach ($object['Attribute'] as $originalAttribute) {
$originalAttribute['deleted'] = 1;
$this->Event->Attribute->save($originalAttribute, array('fieldList' => Attribute::EDITABLE_FIELDS));
}
@ -951,12 +948,6 @@ class MispObject extends AppModel
$newAttribute['last_seen'] = $object['Object']['last_seen'];
$different = true;
}
if (!isset($newAttribute['distribution'])) {
$newAttribute['distribution'] = Configure::read('MISP.default_attribute_distribution');
if ($newAttribute['distribution'] == 'event') {
$newAttribute['distribution'] = 5;
}
}
$saveAttributeResult = $this->Attribute->saveAttributes(array($newAttribute), $user);
return $saveAttributeResult ? $this->id : $this->validationErrors;
}

6
app/Model/Organisation.php
View File

@ -87,7 +87,7 @@ class Organisation extends AppModel
'User' => array('table' => 'users', 'fields' => array('org_id'))
);
public $genericMISPOrganisation = array(
const GENERIC_MISP_ORGANISATION = [
'id' => '0',
'name' => 'MISP',
'date_created' => '',
@ -102,7 +102,7 @@ class Organisation extends AppModel
'local' => true,
'restricted_to_domain' => [],
'landingpage' => null
);
];
public function beforeValidate($options = array())
{
@ -150,7 +150,7 @@ class Organisation extends AppModel
public function afterSave($created, $options = array())
{
if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_organisation_notifications_enable')) {
if ($this->pubToZmq('organisation')) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->modified($this->data, 'organisation');
}

8
app/Model/Role.php
View File

@ -52,7 +52,7 @@ class Role extends AppModel
'permission' => "CASE WHEN (Role.perm_add AND Role.perm_modify AND Role.perm_publish) THEN '3' WHEN (Role.perm_add AND Role.perm_modify_org) THEN '2' WHEN (Role.perm_add) THEN '1' ELSE '0' END",
);
public $permissionConstants = array(
const PERMISSION_CONSTANTS = array(
'read_only' => 0,
'manage_own' => 1,
'manage_org' => 2,
@ -69,8 +69,8 @@ class Role extends AppModel
} elseif (!is_numeric($this->data['Role']['permission'])) {
// If a constant was passed via the API, convert it to the numeric value
// For invalid entries, choose permission level 0
if (isset($this->permissionConstants[$this->data['Role']['permission']])) {
$this->data['Role']['permission'] = $this->permissionConstants[$this->data['Role']['permission']];
if (isset(self::PERMISSION_CONSTANTS[$this->data['Role']['permission']])) {
$this->data['Role']['permission'] = self::PERMISSION_CONSTANTS[$this->data['Role']['permission']];
} else {
$this->data['Role']['permission'] = 0;
}
@ -149,7 +149,7 @@ class Role extends AppModel
unset($results[$key]['Role']['perm_full']);
if (isset($results[$key]['Role']['permission'])) {
$results[$key]['Role']['permission_description'] =
array_flip($this->permissionConstants)[$results[$key]['Role']['permission']];
array_flip(self::PERMISSION_CONSTANTS)[$results[$key]['Role']['permission']];
}
}
}

267
app/Model/Server.php
View File

@ -240,7 +240,7 @@ class Server extends AppModel
* @param array $server
* @param array $user
*/
private function __updatePulledEventBeforeInsert(array &$event, array $server, array $user)
private function __updatePulledEventBeforeInsert(array &$event, array $server, array $user, array $pullRules, bool &$pullRulesEmptiedEvent=false)
{
// we have an Event array
// The event came from a pull, so it should be locked.
@ -268,8 +268,14 @@ class Server extends AppModel
}
}
}
if (isset($event['Event']['Attribute'])) {
$originalCount = count($event['Event']['Attribute']);
foreach ($event['Event']['Attribute'] as $key => $attribute) {
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && in_array($attribute['type'], $pullRules['type_attributes']['NOT'])) {
unset($event['Event']['Attribute'][$key]);
continue;
}
switch ($attribute['distribution']) {
case '1':
$event['Event']['Attribute'][$key]['distribution'] = '0';
@ -287,9 +293,17 @@ class Server extends AppModel
}
}
}
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalCount > 0 && count($event['Event']['Attribute']) == 0) {
$pullRulesEmptiedEvent = true;
}
}
if (isset($event['Event']['Object'])) {
$originalObjectCount = count($event['Event']['Object']);
foreach ($event['Event']['Object'] as $i => $object) {
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && in_array($object['template_uuid'], $pullRules['type_objects']['NOT'])) {
unset($event['Event']['Object'][$i]);
continue;
}
switch ($object['distribution']) {
case '1':
$event['Event']['Object'][$i]['distribution'] = '0';
@ -299,7 +313,12 @@ class Server extends AppModel
break;
}
if (isset($object['Attribute'])) {
$originalAttributeCount = count($object['Attribute']);
foreach ($object['Attribute'] as $j => $a) {
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && in_array($a['type'], $pullRules['type_attributes']['NOT'])) {
unset($event['Event']['Object'][$i]['Attribute'][$j]);
continue;
}
switch ($a['distribution']) {
case '1':
$event['Event']['Object'][$i]['Attribute'][$j]['distribution'] = '0';
@ -317,8 +336,15 @@ class Server extends AppModel
}
}
}
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalAttributeCount > 0 && empty($event['Event']['Object'][$i]['Attribute'])) {
unset($event['Event']['Object'][$i]); // Object is empty, get rid of it
$pullRulesEmptiedEvent = true;
}
}
}
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && $originalObjectCount > 0 && count($event['Event']['Object']) == 0) {
$pullRulesEmptiedEvent = true;
}
}
if (isset($event['Event']['EventReport'])) {
foreach ($event['Event']['EventReport'] as $key => $r) {
@ -441,9 +467,13 @@ class Server extends AppModel
if ($this->__checkIfEventIsBlockedBeforePull($event)) {
return false;
}
$this->__updatePulledEventBeforeInsert($event, $serverSync->server(), $user);
$pullRulesEmptiedEvent = false;
$this->__updatePulledEventBeforeInsert($event, $serverSync->server(), $user, $serverSync->pullRules(), $pullRulesEmptiedEvent);
if (!$this->__checkIfEventSaveAble($event)) {
$fails[$eventId] = __('Empty event detected.');
if (!$pullRulesEmptiedEvent) { // The event is empty because of the filtering rule. This is not considered a failure
$fails[$eventId] = __('Empty event detected.');
}
} else {
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force);
}
@ -1377,11 +1407,6 @@ class Server extends AppModel
public function serverSettingReadSingle($settingObject, $settingName, $leafKey)
{
// invalidate config.php from php opcode cache
if (function_exists('opcache_reset')) {
opcache_reset();
}
$setting = Configure::read($settingName);
$result = $this->__evaluateLeaf($settingObject, $leafKey, $setting);
$result['setting'] = $settingName;
@ -1580,19 +1605,20 @@ class Server extends AppModel
if (substr($value, 0, 7) === "phar://") {
return 'Phar protocol not allowed.';
}
$finfo = finfo_open(FILEINFO_MIME_TYPE);
if ($value === '') {
return true;
}
if (is_executable($value)) {
if (finfo_file($finfo, $value) == "application/x-executable" || finfo_file($finfo, $value) == "application/x-pie-executable" || finfo_file($finfo, $value) == "application/x-sharedlib") {
finfo_close($finfo);
$finfo = finfo_open(FILEINFO_MIME_TYPE);
$type = finfo_file($finfo, $value);
finfo_close($finfo);
if ($type === "application/x-executable" || $type === "application/x-pie-executable" || $type === "application/x-sharedlib") {
return true;
} else {
return 'Binary file not executable. It is of type: ' . finfo_file($finfo, $value);
return 'Binary file not executable. It is of type: ' . $type;
}
} else {
return false;
return 'Binary file not executable.';
}
}
@ -2100,7 +2126,7 @@ class Server extends AppModel
return true;
}
private function __serverSettingNormaliseValue($data, $value, $setting)
private function __serverSettingNormaliseValue($data, $value)
{
if (!empty($data['type'])) {
if ($data['type'] === 'boolean') {
@ -2112,41 +2138,39 @@ class Server extends AppModel
return $value;
}
public function getSettingData($setting_name)
/**
* @param string $setting_name
* @return array|false False if setting doesn't exists
*/
public function getSettingData($setting_name, $withOptions = true)
{
// invalidate config.php from php opcode cache
if (function_exists('opcache_reset')) {
opcache_reset();
}
// This is just hack to reset opcache, so for next request cache will be reloaded.
$this->opcacheResetConfig();
if (strpos($setting_name, 'Plugin.Enrichment') !== false || strpos($setting_name, 'Plugin.Import') !== false || strpos($setting_name, 'Plugin.Export') !== false || strpos($setting_name, 'Plugin.Cortex') !== false) {
$serverSettings = $this->getCurrentServerSettings();
} else {
$serverSettings = $this->serverSettings;
}
$setting = false;
foreach ($serverSettings as $k => $s) {
if (isset($s['branch'])) {
foreach ($s as $ek => $es) {
if ($ek != 'branch') {
if ($setting_name == $k . '.' . $ek) {
$setting = $es;
continue 2;
}
}
}
$setting = $serverSettings;
$parts = explode('.', $setting_name);
foreach ($parts as $part) {
if (isset($setting[$part])) {
$setting = $setting[$part];
} else {
if ($setting_name == $k) {
$setting = $s;
continue;
}
$setting = false;
break;
}
}
if (!empty($setting)) {
if (isset($setting['level'])) {
$setting['name'] = $setting_name;
if ($withOptions && isset($setting['optionsSource'])) {
$setting['options'] = $setting['optionsSource']();
}
}
if (!empty($setting['optionsSource'])) {
$setting['options'] = $setting['optionsSource']();
}
return $setting;
}
@ -2264,19 +2288,10 @@ class Server extends AppModel
throw new Exception("Could not create config backup `$backupFilePath`.");
}
}
$settingObject = $this->getCurrentServerSettings();
foreach ($settingObject as $branchName => $branch) {
if (!isset($branch['level'])) {
foreach ($branch as $settingName => $settingObject) {
if ($setting === $branchName . '.' . $settingName) {
$value = $this->__serverSettingNormaliseValue($settingObject, $value, $setting);
}
}
} else {
if ($setting === $branchName) {
$value = $this->__serverSettingNormaliseValue($branch, $value, $setting);
}
}
$settingObject = $this->getSettingData($setting, false);
if ($settingObject) {
$value = $this->__serverSettingNormaliseValue($settingObject, $value);
}
/** @var array $config */
@ -2315,9 +2330,7 @@ class Server extends AppModel
FileAccessTool::deleteFile($tmpFile);
throw new Exception("Could not rename `$tmpFile` to config file `$configFilePath`.");
}
if (function_exists('opcache_reset')) {
opcache_reset();
}
$this->opcacheResetConfig();
chmod($configFilePath, octdec($previous_file_perm));
$config_saved = FileAccessTool::readFromFile($configFilePath);
// if the saved config file is empty, restore the backup.
@ -2330,56 +2343,58 @@ class Server extends AppModel
}
} else {
FileAccessTool::writeToFile($configFilePath, $settingsString);
if (function_exists('opcache_reset')) {
opcache_reset();
}
$this->opcacheResetConfig();
}
return true;
}
public function getFileRules()
{
$validItems = array(
'orgs' => array(
'name' => __('Organisation logos'),
'description' => __('The logo used by an organisation on the event index, event view, discussions, proposals, etc. Make sure that the filename is in the org.png format, where org is the case-sensitive organisation name.'),
'expected' => array(),
'valid_format' => __('48x48 pixel .png files'),
'path' => APP . 'webroot' . DS . 'img' . DS . 'orgs',
'regex' => '.*\.(png|PNG)$',
'regex_error' => __('Filename must be in the following format: *.png'),
'files' => array(),
),
'img' => array(
'name' => __('Additional image files'),
'description' => __('Image files uploaded into this directory can be used for various purposes, such as for the login page logos'),
'expected' => array(
'MISP.footer_logo' => Configure::read('MISP.footer_logo'),
'MISP.home_logo' => Configure::read('MISP.home_logo'),
'MISP.welcome_logo' => Configure::read('MISP.welcome_logo'),
'MISP.welcome_logo2' => Configure::read('MISP.welcome_logo2'),
),
'valid_format' => __('text/html if served inline, anything that conveys the terms of use if served as download'),
'path' => APP . 'webroot' . DS . 'img' . DS . 'custom',
'regex' => '.*\.(png|PNG)$',
'regex_error' => __('Filename must be in the following format: *.png'),
'files' => array(),
return array(
'orgs' => array(
'name' => __('Organisation logos'),
'description' => __('The logo used by an organisation on the event index, event view, discussions, proposals, etc. Make sure that the filename is in the org.png format, where org is the case-sensitive organisation name.'),
'expected' => array(),
'valid_format' => __('48x48 pixel .png files'),
'path' => APP . 'webroot' . DS . 'img' . DS . 'orgs',
'regex' => '.*\.(png|PNG)$',
'regex_error' => __('Filename must be in the following format: *.png'),
'files' => array(),
),
'img' => array(
'name' => __('Additional image files'),
'description' => __('Image files uploaded into this directory can be used for various purposes, such as for the login page logos'),
'expected' => array(
'MISP.footer_logo' => Configure::read('MISP.footer_logo'),
'MISP.home_logo' => Configure::read('MISP.home_logo'),
'MISP.welcome_logo' => Configure::read('MISP.welcome_logo'),
'MISP.welcome_logo2' => Configure::read('MISP.welcome_logo2'),
),
'valid_format' => __('PNG or SVG file'),
'path' => APP . 'webroot' . DS . 'img' . DS . 'custom',
'regex' => '.*\.(png|svg)$',
'regex_error' => __('Filename must be in the following format: *.png or *.svg'),
'files' => array(),
),
);
return $validItems;
}
public function grabFiles()
{
$validItems = $this->getFileRules();
App::uses('Folder', 'Utility');
App::uses('File', 'Utility');
foreach ($validItems as $k => $item) {
$dir = new Folder($item['path']);
$files = $dir->find($item['regex'], true);
foreach ($files as $file) {
$f = new File($item['path'] . DS . $file);
$validItems[$k]['files'][] = array('filename' => $file, 'filesize' => $f->size(), 'read' => $f->readable(), 'write' => $f->writable(), 'execute' => $f->executable());
$f = new SplFileInfo($item['path'] . DS . $file);
$validItems[$k]['files'][] = [
'filename' => $file,
'filesize' => $f->getSize(),
'read' => $f->isReadable(),
'write' => $f->isWritable(),
'execute' => $f->isExecutable(),
];
}
}
return $validItems;
@ -3155,6 +3170,17 @@ class Server extends AppModel
APP . 'tmp' . DS . 'logs' => 0,
APP . 'tmp' . DS . 'bro' => 0,
);
$attachmentDir = Configure::read('MISP.attachments_dir');
if ($attachmentDir && !isset($writeableDirs[$attachmentDir])) {
$writeableDirs[$attachmentDir] = 0;
}
$tmpDir = Configure::read('MISP.tmpdir');
if ($tmpDir && !isset($writeableDirs[$tmpDir])) {
$writeableDirs[$tmpDir] = 0;
}
foreach ($writeableDirs as $path => &$error) {
if (!file_exists($path)) {
// Try to create directory if not exists
@ -3208,9 +3234,18 @@ class Server extends AppModel
public function yaraDiagnostics(&$diagnostic_errors)
{
$scriptFile = APP . 'files' . DS . 'scripts' . DS . 'yaratest.py';
$scriptResult = ProcessTool::execute([ProcessTool::pythonBin(), $scriptFile]);
$scriptResult = json_decode($scriptResult, true);
return array('operational' => $scriptResult['success'], 'plyara' => $scriptResult['plyara']);
try {
$scriptResult = ProcessTool::execute([ProcessTool::pythonBin(), $scriptFile]);
$scriptResult = json_decode($scriptResult, true);
} catch (Exception $exception) {
$this->logException('Failed to run yara diagnostics.', $exception);
return array(
'operational' => 0,
'plyara' => 0,
'test_run' => false
);
}
return array('operational' => $scriptResult['success'], 'plyara' => $scriptResult['plyara'], 'test_run' => true);
}
public function stixDiagnostics(&$diagnostic_errors)
@ -3218,7 +3253,17 @@ class Server extends AppModel
$expected = array('stix' => '>1.2.0.11', 'cybox' => '>2.1.0.21', 'mixbox' => '>1.0.5', 'maec' => '>4.1.0.17', 'stix2' => '>3.0.0', 'pymisp' => '>2.4.120');
// check if the STIX and Cybox libraries are working using the test script stixtest.py
$scriptFile = APP . 'files' . DS . 'scripts' . DS . 'stixtest.py';
$scriptResult = ProcessTool::execute([ProcessTool::pythonBin(), $scriptFile]);
try {
$scriptResult = ProcessTool::execute([ProcessTool::pythonBin(), $scriptFile]);
} catch (Exception $exception) {
$this->logException('Failed to run STIX diagnostics.', $exception);
return [
'operational' => 0,
'invalid_version' => false,
'test_run' => false
];
}
try {
$scriptResult = $this->jsonDecode($scriptResult);
} catch (Exception $e) {
@ -3241,6 +3286,7 @@ class Server extends AppModel
$result = [
'operational' => $scriptResult['operational'],
'invalid_version' => false,
'test_run' => true
];
foreach ($expected as $package => $expectedVersion) {
$result[$package]['version'] = $scriptResult[$package];
@ -3400,12 +3446,7 @@ class Server extends AppModel
$workers = $this->getWorkers();
if (function_exists('posix_getpwuid')) {
$currentUser = posix_getpwuid(posix_geteuid());
$currentUser = $currentUser['name'];
} else {
$currentUser = trim(ProcessTool::execute(['whoami']));
}
$currentUser = ProcessTool::whoami();
$procAccessible = file_exists('/proc');
foreach ($workers as $pid => $worker) {
$entry = ($worker['type'] == 'regular') ? $worker['queue'] : $worker['type'];
@ -3452,6 +3493,16 @@ class Server extends AppModel
if (Configure::check('MISP.manage_workers')) {
$worker_array['controls'] = Configure::read('MISP.manage_workers');
}
if (Configure::read('SimpleBackgroundJobs.enabled')) {
try {
$worker_array['supervisord_status'] = $this->getBackgroundJobsTool()->getSupervisorStatus();
} catch (Exception $exception) {
$this->logException('Error getting supervisor status.', $exception);
$worker_array['supervisord_status'] = false;
}
}
return $worker_array;
}
@ -3758,7 +3809,11 @@ class Server extends AppModel
];
}
if (is_readable(APP . '/files/scripts/selftest.php')) {
$execResult = exec('php ' . APP . '/files/scripts/selftest.php ' . escapeshellarg(json_encode(array_keys($extensions))));
try {
$execResult = ProcessTool::execute(['php', APP . '/files/scripts/selftest.php', json_encode(array_keys($extensions))]);
} catch (Exception $e) {
// pass
}
if (!empty($execResult)) {
$execResult = $this->jsonDecode($execResult);
$results['cli']['phpversion'] = $execResult['phpversion'];
@ -4602,6 +4657,16 @@ class Server extends AppModel
return $this->saveMany($toSave, ['validate' => false, 'fields' => ['authkey']]);
}
/**
* Invalidate config.php from php opcode cache
*/
private function opcacheResetConfig()
{
if (function_exists('opcache_invalidate')) {
opcache_invalidate(APP . 'Config' . DS . 'config.php', true);
}
}
/**
* Generate just when required
* @return array[]
@ -5623,6 +5688,14 @@ class Server extends AppModel
'type' => 'string',
'null' => true,
],
'enable_synchronisation_filtering_on_type' => [
'level' => self::SETTING_OPTIONAL,
'description' => __('Allows server synchronisation connections to be filtered on Attribute type or Object name. Warning: This feature has can potentially cause your synchronisation partners to receive incomplete versions of the events you are propagating on behalf of others. This means that even if they would be receiving the unfiltered version through another instance, your filtered version might be the one they receive on a first-come-first-serve basis.'),
'value' => false,
'test' => 'testBoolFalse',
'type' => 'boolean',
'null' => true,
],
),
'GnuPG' => array(
'branch' => 1,

3
app/Model/Sighting.php
View File

@ -2,6 +2,7 @@
App::uses('AppModel', 'Model');
App::uses('TmpFileTool', 'Tools');
App::uses('ServerSyncTool', 'Tools');
App::uses('ProcessTool', 'Tools');
/**
* @property Attribute $Attribute
@ -744,7 +745,7 @@ class Sighting extends AppModel
$tempFile->close();
$scriptFile = APP . "files" . DS . "scripts" . DS . "stixsighting2misp.py";
// Execute the python script and point it to the temporary filename
$result = shell_exec($this->getPythonVersion() . ' ' . $scriptFile . ' ' . $randomFileName);
$result = ProcessTool::execute([ProcessTool::pythonBin(), $scriptFile, $randomFileName]);
// The result of the script will be a returned JSON object with 2 variables: success (boolean) and message
// If success = 1 then the temporary output file was successfully written, otherwise an error message is passed along
$result = json_decode($result, true);

25
app/Model/Tag.php
View File

@ -79,8 +79,8 @@ class Tag extends AppModel
)
);
public $reGalaxy = '/misp-galaxy:[^:="]+="[^:="]+/i';
public $reCustomGalaxy = '/misp-galaxy:[^:="]+="[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}"/i';
const RE_GALAXY = '/misp-galaxy:[^:="]+="[^:="]+/i';
const RE_CUSTOM_GALAXY = '/misp-galaxy:[^:="]+="[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}"/i';
private $tagOverrides = false;
public function beforeValidate($options = array())
@ -104,8 +104,8 @@ class Tag extends AppModel
if (isset($tag['name']) && strlen($tag['name']) >= 255) {
$tag['name'] = substr($tag['name'], 0, 255);
}
$tag['is_galaxy'] = preg_match($this->reGalaxy, $tag['name']);
$tag['is_custom_galaxy'] = preg_match($this->reCustomGalaxy, $tag['name']);
$tag['is_galaxy'] = preg_match(self::RE_GALAXY, $tag['name']);
$tag['is_custom_galaxy'] = preg_match(self::RE_CUSTOM_GALAXY, $tag['name']);
return true;
}
@ -312,7 +312,7 @@ class Tag extends AppModel
if ($force || $user['Role']['perm_tag_editor']) {
$this->create();
if (empty($tag['colour'])) {
$tag['colour'] = $this->random_color();
$tag['colour'] = $this->tagColor($tag['name']);
}
$tag = array(
'name' => $tag['name'],
@ -347,13 +347,14 @@ class Tag extends AppModel
return $existingTag['Tag']['id'];
}
public function random_color()
/**
* Generate tag color according to name. So color will be same on all instances.
* @param string $tagName
* @return string
*/
public function tagColor($tagName)
{
$colour = '#';
for ($i = 0; $i < 3; $i++) {
$colour .= str_pad(dechex(mt_rand(0, 255)), 2, '0', STR_PAD_LEFT);
}
return $colour;
return '#' . substr(md5($tagName), 0, 6);
}
/**
@ -367,7 +368,7 @@ class Tag extends AppModel
{
$this->create();
if ($colour === false) {
$colour = $this->random_color();
$colour = $this->tagColor($name);
}
$data = array(
'name' => $name,

4
app/Model/User.php
View File

@ -19,8 +19,6 @@ class User extends AppModel
{
public $displayField = 'email';
public $orgField = array('Organisation', 'name');
public $validate = array(
'role_id' => array(
'numeric' => array(
@ -278,7 +276,7 @@ class User extends AppModel
public function afterSave($created, $options = array())
{
$pubToZmq = Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_user_notifications_enable');
$pubToZmq = $this->pubToZmq('user');
$kafkaTopic = $this->kafkaTopic('user');
if ($pubToZmq || $kafkaTopic) {
if (!empty($this->data)) {

6
app/Model/Warninglist.php
View File

@ -53,7 +53,7 @@ class Warninglist extends AppModel
)
);
private $__tlds = array(
const TLDS = array(
'TLDs as known by IANA'
);
@ -725,7 +725,7 @@ class Warninglist extends AppModel
public function fetchTLDLists()
{
$tldLists = $this->find('column', array(
'conditions' => array('Warninglist.name' => $this->__tlds),
'conditions' => array('Warninglist.name' => self::TLDS),
'fields' => array('Warninglist.id')
));
$tlds = array();
@ -769,7 +769,7 @@ class Warninglist extends AppModel
public function missingTldLists()
{
$missingTldLists = array();
foreach ($this->__tlds as $tldList) {
foreach (self::TLDS as $tldList) {
$temp = $this->find('first', array(
'recursive' => -1,
'conditions' => array('Warninglist.name' => $tldList),

6
app/Model/WarninglistEntry.php
View File

@ -23,10 +23,4 @@ class WarninglistEntry extends AppModel
'counterCache' => true
)
);
public function beforeValidate($options = array())
{
parent::beforeValidate();
return true;
}
}

6
app/Model/WarninglistType.php
View File

@ -19,10 +19,4 @@ class WarninglistType extends AppModel
public $belongsTo = array(
'Warninglist'
);
public function beforeValidate($options = array())
{
parent::beforeValidate();
return true;
}
}

1
app/Plugin/UrlCache/keep.txt Normal file
View File

@ -0,0 +1 @@
This plugin is not used anymore, but it is here to keep backward compatibility.

3
app/Test/JSONConverterToolTest.php
View File

@ -37,9 +37,8 @@ class JSONConverterToolTest extends TestCase
private function check(array $event): void
{
$complexTypeTool = new JSONConverterTool();
$json = '';
foreach ($complexTypeTool->streamConvert($event) as $part) {
foreach (JSONConverterTool::streamConvert($event) as $part) {
$json .= $part;
}
if (defined('JSON_THROW_ON_ERROR')) {

23
app/View/AppView.php Normal file
View File

@ -0,0 +1,23 @@
<?php
class AppView extends View
{
/**
* Negative results of `file_exists` methods are not cached, so we provide our cache
* @var array
*/
private $elementFileCache = [];
/**
* @param string $name
* @return false|string
*/
protected function _getElementFileName($name)
{
if (isset($this->elementFileCache[$name])) {
return $this->elementFileCache[$name];
}
$result = parent::_getElementFileName($name);
$this->elementFileCache[$name] = $result;
return $result;
}
}

20
app/View/Attributes/ajax/attributeEditMassForm.ctp
View File

@ -27,17 +27,29 @@
?>
</div>
<?php
echo '<div class="input clear"></div>';
echo $this->Form->input('to_ids', array(
'options' => array(__('No'), __('Yes'), __('Do not alter current settings')),
'data-content' => isset($attrDescriptions['signature']['formdesc']) ? $attrDescriptions['signature']['formdesc'] : $attrDescriptions['signature']['desc'],
'label' => __('For Intrusion Detection System'),
'selected' => 2,
'options' => array(__('No'), __('Yes'), __('Do not alter current settings')),
'data-content' => isset($attrDescriptions['signature']['formdesc']) ? $attrDescriptions['signature']['formdesc'] : $attrDescriptions['signature']['desc'],
'label' => __('For Intrusion Detection System'),
'selected' => 2
));
echo '<div class="input clear"></div>';
echo $this->Form->input('is_proposal', array(
'type' => 'checkbox',
'label' => __('Create proposals'),
'checked' => false
));
echo '<div class="input clear"></div>';
echo $this->Form->input('disable_correlation', array(
'label' => __('Correlations'),
'options' => [
'2' => __('Do not alter current settings'),
'0' => __('Enable correlations'),
'1' => __('Disable correlations')
],
'selected' => '2'
));
?>
<div class="input clear"></div>

7
app/View/Elements/Events/View/value_field.ctp
View File

@ -27,13 +27,12 @@ switch ($object['type']) {
case 'malware-sample':
if ($object['type'] === 'attachment' && isset($object['image'])) {
if ($object['image'] === true) {
$img = '<it class="fa fa-spin fa-spinner" style="font-size: large; left: 50%; top: 50%;"></it>';
$img .= '<img class="screenshot screenshot-collapsed useCursorPointer img-rounded hidden" src="' . $baseurl . sprintf('/%s/viewPicture/', $object['objectType'] == 'proposal' ? 'shadowAttributes' : 'attributes') . h($object['id']) . '/1' . '" title="' . h($object['value']) . '" onload="$(this).show(200); $(this).parent().find(\'.fa-spinner\').remove();"/>';
echo $img;
$src = $baseurl . '/' . ($object['objectType'] === 'proposal' ? 'shadowAttributes' : 'attributes') . '/viewPicture/' . (int)$object['id'] . '/1';
echo '<img class="screenshot screenshot-collapsed useCursorPointer img-rounded" src="' . $src . '" title="' . h($object['value']) . '" loading="lazy">';
} else {
$extension = pathinfo($object['value'], PATHINFO_EXTENSION);
$uri = 'data:image/' . strtolower(h($extension)) . ';base64,' . h($object['image']);
echo '<img class="screenshot screenshot-collapsed useCursorPointer" src="' . $uri . '" title="' . h($object['value']) . '" />';
echo '<img class="screenshot screenshot-collapsed useCursorPointer" src="' . $uri . '" title="' . h($object['value']) . '">';
}
} else {
$filenameHash = explode('|', h($object['value']));

20
app/View/Elements/Events/eventIndexTable.ctp
View File

@ -13,19 +13,19 @@
<?php
if (Configure::read('MISP.showorgalternate') && Configure::read('MISP.showorg')):
?>
<th class="filter"><?php echo $this->Paginator->sort('Org', 'Source org'); ?></th>
<th class="filter"><?php echo $this->Paginator->sort('Org', 'Member org'); ?></th>
<th class="filter"><?php echo $this->Paginator->sort('Orgc.name', __('Source org')); ?></th>
<th class="filter"><?php echo $this->Paginator->sort('Orgc.name', __('Member org')); ?></th>
<?php
elseif (Configure::read('MISP.showorg') || $isAdmin):
?>
<th class="filter"><?php echo $this->Paginator->sort('Org', __('Creator org')); ?></th>
<th class="filter"><?php echo $this->Paginator->sort('Orgc.name', __('Creator org')); ?></th>
<?php
endif;
$date = time();
$day = 86400;
?>
<?php if (in_array('owner_org', $columns, true)): ?><th class="filter"><?= $this->Paginator->sort('owner org', __('Owner org')) ?></th><?php endif; ?>
<?php if (in_array('owner_org', $columns, true)): ?><th class="filter"><?= $this->Paginator->sort('Org.name', __('Owner org')) ?></th><?php endif; ?>
<th><?= $this->Paginator->sort('id', __('ID'), ['direction' => 'desc']) ?></th>
<?php if (in_array('clusters', $columns, true)): ?><th><?= __('Clusters') ?></th><?php endif; ?>
<?php if (in_array('tags', $columns, true)): ?><th><?= __('Tags') ?></th><?php endif; ?>
@ -37,6 +37,8 @@
<?php if (in_array('discussion', $columns, true)): ?><th title="<?= __('Post Count') ?>"><?= __('#Posts') ?></th><?php endif; ?>
<?php if (in_array('creator_user', $columns, true)): ?><th><?= $this->Paginator->sort('user_id', __('Creator user')) ?></th><?php endif; ?>
<th class="filter"><?= $this->Paginator->sort('date', null, array('direction' => 'desc'));?></th>
<?php if (in_array('timestamp', $columns, true)): ?><th title="<?= __('Last modified at') ?>"><?= $this->Paginator->sort('timestamp', __('Last modified at')) ?></th><?php endif; ?>
<?php if (in_array('publish_timestamp', $columns, true)): ?><th title="<?= __('Last modified at') ?>"><?= $this->Paginator->sort('publish_timestamp', __('Published at')) ?></th><?php endif; ?>
<th class="filter"><?= $this->Paginator->sort('info');?></th>
<th title="<?= $eventDescriptions['distribution']['desc'];?>">
<?= $this->Paginator->sort('distribution');?>
@ -167,6 +169,16 @@
<td class="short dblclickElement">
<?= $event['Event']['date'] ?>
</td>
<?php if (in_array('timestamp', $columns, true)): ?>
<td class="short dblclickElement">
<?= $this->Time->time($event['Event']['timestamp']) ?>
</td>
<?php endif; ?>
<?php if (in_array('publish_timestamp', $columns, true)): ?>
<td class="short dblclickElement">
<?= $this->Time->time($event['Event']['publish_timestamp']) ?>
</td>
<?php endif; ?>
<td class="dblclickElement">
<?= nl2br(h($event['Event']['info']), false) ?>
</td>

9
app/View/Elements/Feeds/View/row_attribute.ctp
View File

@ -80,6 +80,15 @@
<td class="shortish">
&nbsp;
</td>
<td>
<?php
if ($object['distribution'] == 4) {
echo h($object['SharingGroup']['name']);
} else {
echo $distributionLevels[$object['distribution']];
}
?>
</td>
<td class="short">
<div id="Attribute_<?php echo $object['uuid']; ?>_to_ids_solid" class="inline-field-solid">
<?php echo $object['to_ids'] ? __('Yes') : __('No'); ?>

1
app/View/Elements/Feeds/View/row_object.ctp
View File

@ -37,6 +37,7 @@
</td>
<td>&nbsp;</td>
<td>&nbsp;</td>
<td>&nbsp;</td>
</tr>
<?php
if (!empty($object['Attribute'])) {

1
app/View/Elements/Feeds/eventattribute.ctp
View File

@ -47,6 +47,7 @@
<th><?php echo $this->Paginator->sort('comment');?></th>
<th><?php echo __('Related Events');?></th>
<th><?php echo __('Feed hits');?></th>
<th><?php echo __('Distribution');?></th>
<th title="<?php echo $attrDescriptions['signature']['desc'];?>"><?php echo $this->Paginator->sort('to_ids', 'IDS');?></th>
</tr>
<?php

3
app/View/Elements/genericElements/Form/genericForm.ctp
View File

@ -39,8 +39,10 @@ $fieldsArrayForPersistence = array();
$formOptions = isset($formOptions) ? $formOptions : array();
$formOptions = array_merge(['class' => 'genericForm'], $formOptions);
$formCreate = $this->Form->create($modelForForm, $formOptions);
$fieldsArray = [];
if (!empty($data['fields'])) {
foreach ($data['fields'] as $fieldData) {
$fieldsArray[] = $modelForForm . Inflector::camelize($fieldData['field']);
if (isset($fieldData['requirements']) && !$fieldData['requirements']) {
continue;
}
@ -124,6 +126,7 @@ if (!empty($ajax)) {
?>
<script type="text/javascript">
var fieldsArray = <?= json_encode($fieldsArray) ?>;
$(function() {
popoverStartup();
});

12
app/View/Elements/healthElements/diagnostics.ctp
View File

@ -335,7 +335,9 @@
<?php else: ?>
<b><?= __('Current libraries status') ?>:</b>
<?php if ($stix['operational'] === 0): ?>
<?php if ($stix['test_run'] === false): ?>
<b class="red bold"><?= __('Failed to run STIX diagnostics tool.') ?></b>
<?php elseif ($stix['operational'] === 0): ?>
<b class="red bold"><?= __('Some of the libraries related to STIX are not installed. Make sure that all libraries listed below are correctly installed.') ?></b>
<?php elseif ($stix['invalid_version']): ?>
<span class="orange"><?= __('Some versions should be updated.') ?></span>
@ -369,10 +371,14 @@
<div class="diagnostics-box">
<?php
$colour = 'green';
$message = __('OK');
if ($yaraStatus['operational'] == 0) {
if ($yaraStatus['test_run'] === false) {
$colour = 'red';
$message = __('Failed to run yara diagnostics tool.');
}elseif ($yaraStatus['operational'] == 0) {
$colour = 'red';
$message = __('Invalid plyara version / plyara not installed. Please run pip3 install plyara');
}else{
$message = __('OK');
}
echo __('plyara library installed') . '…<span style="color:' . $colour . ';">' . $message . '</span>';
?>

9
app/View/Elements/healthElements/workers.ctp
View File

@ -1,10 +1,17 @@
<div style="border:1px solid #dddddd; margin-top:1px; width:100%; padding:10px">
<?php
<?php
if (!$worker_array['proc_accessible']):
?>
<div style="background-color:red !important;color:white;"><b><?php echo __('Warning');?></b>: <?php echo __('MISP cannot access your /proc directory to check the status of the worker processes, which means that dead workers will not be detected by the diagnostic tool. If you would like to regain this functionality, make sure that the open_basedir directive is not set, or that /proc is included in it.');?></div>
<?php
endif;
if(Configure::read('SimpleBackgroundJobs.enabled') && !$worker_array['supervisord_status']):
?>
<div style="background-color:red !important;color:white;"><b><?php echo __('Warning');?></b>: <?php echo __('MISP cannot connect to the Supervisord API, check the following settings are correct: [`supervisor_host`, `supervisor_port`, `supervisor_user`, `supervisor_password`] and restart the service. For details check the MISP error logs.');?></div>
<?php
endif;
if (!$worker_array['controls']):
?>
<div><b><?php echo __('Note:');?></b>: <?php echo __('You have set the "manage_workers" variable to "false", therefore worker controls have been disabled.');?></div>

291
app/View/Elements/serverRuleElements/pull.ctp
View File

@ -1,5 +1,5 @@
<div style="display: flex; flex-direction: column;" class="server-rule-container-pull">
<?php if ($context == 'servers'): ?>
<?php if ($context == 'servers') : ?>
<div class="alert alert-primary notice-pull-rule-fetched">
<button type="button" class="close" data-dismiss="alert">&times;</button>
<i class="<?= $this->FontAwesome->getClass('spinner') ?> fa-spin"></i>
@ -13,68 +13,86 @@
<button type="button" class="close" data-dismiss="alert">&times;</button>
<?= __('Issues while trying to fetch Organisations and Tags from the remote server.') ?>
<div><strong><?= __('Reason:') ?></strong></div>
<div><pre class="reason" style="margin-bottom: 0;"></pre></div>
<div>
<pre class="reason" style="margin-bottom: 0;"></pre>
</div>
</div>
<?php endif; ?>
<?php
$tagAllowRules = [];
$tagBlockRules = [];
$orgAllowRules = [];
$orgBlockRules = [];
$ruleUrlParams = [];
if (!empty($ruleObject)) {
$tagAllowRules = $ruleObject['tags']['OR'];
$tagBlockRules = $ruleObject['tags']['NOT'];
$orgAllowRules = $ruleObject['orgs']['OR'];
$orgBlockRules = $ruleObject['orgs']['NOT'];
$ruleUrlParams = $ruleObject['url_params'];
}
$tagAllowRules = [];
$tagBlockRules = [];
$orgAllowRules = [];
$orgBlockRules = [];
$ruleUrlParams = [];
$attributeTypeBlockRules = [];
$objectTypeBlockRules = [];
if (!empty($ruleObject)) {
$tagAllowRules = $ruleObject['tags']['OR'];
$tagBlockRules = $ruleObject['tags']['NOT'];
$orgAllowRules = $ruleObject['orgs']['OR'];
$orgBlockRules = $ruleObject['orgs']['NOT'];
$ruleUrlParams = $ruleObject['url_params'];
$attributeTypeBlockRules = !empty($ruleObject['type_attributes']['NOT']) ? $ruleObject['type_attributes']['NOT'] : [];
$objectTypeBlockRules = !empty($ruleObject['type_objects']['NOT']) ? $ruleObject['type_objects']['NOT'] : [];
}
?>
<?php
echo $this->element('serverRuleElements/rules_widget', [
'scope' => 'tag',
'scopeI18n' => __('tag'),
'technique' => 'pull',
'allowEmptyOptions' => true,
'options' => $allTags,
'optionNoValue' => true,
'initAllowOptions' => $tagAllowRules,
'initBlockOptions' => $tagBlockRules
]);
echo $this->element('serverRuleElements/rules_widget', [
'scope' => 'tag',
'scopeI18n' => __('tag'),
'technique' => 'pull',
'allowEmptyOptions' => true,
'options' => $allTags,
'optionNoValue' => true,
'initAllowOptions' => $tagAllowRules,
'initBlockOptions' => $tagBlockRules
]);
?>
<div style="display: flex;">
<h4 class="bold green" style=""><?= __('AND');?></h4>
<h4 class="bold red" style="margin-left: auto;"><?= __('AND NOT');?></h4>
<h4 class="bold green" style=""><?= __('AND'); ?></h4>
<h4 class="bold red" style="margin-left: auto;"><?= __('AND NOT'); ?></h4>
</div>
<?php
echo $this->element('serverRuleElements/rules_widget', [
'scope' => 'org',
'scopeI18n' => __('org'),
'technique' => 'pull',
'allowEmptyOptions' => true,
'options' => $allOrganisations,
'optionNoValue' => true,
'initAllowOptions' => $orgAllowRules,
'initBlockOptions' => $orgBlockRules
]);
echo $this->element('serverRuleElements/rules_widget', [
'scope' => 'org',
'scopeI18n' => __('org'),
'technique' => 'pull',
'allowEmptyOptions' => true,
'options' => $allOrganisations,
'optionNoValue' => true,
'initAllowOptions' => $orgAllowRules,
'initBlockOptions' => $orgBlockRules
]);
?>
<div style="display: flex;">
<h4 class="bold green" style=""><?= __('AND');?></h4>
<h4 class="bold green" style=""><?= __('AND'); ?></h4>
</div>
<div style="display: flex; flex-direction: column;">
<div class="bold green">
<?= __('Additional sync parameters (based on the event index filters)');?>
<?= __('Additional sync parameters (based on the event index filters)'); ?>
</div>
<div style="display: flex;">
<textarea style="width:100%;" placeholder='{"timestamp": "30d"}' type="text" value="" id="urlParams" data-original-title="" title="" rows="3"
><?= !empty($ruleUrlParams) ? json_encode(h($ruleUrlParams), JSON_PRETTY_PRINT) : '' ?></textarea>
<textarea style="width:100%;" placeholder='{"timestamp": "30d"}' type="text" value="" id="urlParams" data-original-title="" title="" rows="3"><?= !empty($ruleUrlParams) ? json_encode(h($ruleUrlParams), JSON_PRETTY_PRINT) : '' ?></textarea>
</div>
</div>
</div>
<?php
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type'))) {
echo $this->element('serverRuleElements/rules_filtering_type', [
'technique' => 'pull',
'allowEmptyOptions' => true,
'allAttributeTypes' => $allAttributeTypes,
'attributeTypeBlockRules' => $attributeTypeBlockRules,
'allObjectTypes' => $allObjectTypes,
'objectTypeBlockRules' => $objectTypeBlockRules,
]);
}
?>
</div>
<?php
echo $this->element('genericElements/assetLoader', array(
@ -95,104 +113,111 @@ echo $this->element('genericElements/assetLoader', array(
?>
<script>
var pullRemoteRules404Error = '<?= __('Connection error or the remote version is not supporting remote filter lookups (v2.4.142+). Make sure that the remote instance is accessible and that it is up to date.') ?>'
var cm;
$(function() {
var serverID = "<?= isset($id) ? $id : '' ?>"
<?php if ($context == 'servers'): ?>
addPullFilteringRulesToPicker()
<?php endif; ?>
setupCodeMirror()
var pullRemoteRules404Error = '<?= __('Connection error or the remote version is not supporting remote filter lookups (v2.4.142+). Make sure that the remote instance is accessible and that it is up to date.') ?>'
var cm;
$(function() {
var serverID = "<?= isset($id) ? $id : '' ?>"
<?php if ($context == 'servers') : ?>
addPullFilteringRulesToPicker()
<?php endif; ?>
setupCodeMirror()
<?php if (empty($attributeTypeBlockRules) && empty($objectTypeBlockRules)) : ?>
$('div.server-rule-container-pull .type-filtering-subcontainer').hide()
<?php else : ?>
$('div.server-rule-container-pull #type-filtering-cb').prop('checked', true)
$('div.server-rule-container-pull #type-filtering-notice-cb').prop('checked', true)
$('div.server-rule-container-pull .type-filtering-container').show()
<?php endif; ?>
function addPullFilteringRulesToPicker() {
var $rootContainer = $('div.server-rule-container-pull')
var $pickerTags = $rootContainer.find('select.rules-select-picker-tag')
var $pickerOrgs = $rootContainer.find('select.rules-select-picker-org')
if (serverID !== "") {
$pickerOrgs.parent().children().prop('disabled', true)
$pickerTags.parent().children().prop('disabled', true)
getPullFilteringRules(
function(data) {
addOptions($pickerTags, data.tags)
addOptions($pickerOrgs, data.organisations)
$('div.notice-pull-rule-fetched.alert-success').show()
},
function(errorMessage) {
var regex = /Reponse was not OK\. \(HTTP code: (?<code>\d+)\)/m
var matches = errorMessage.match(regex)
if (matches !== null) {
if (matches.groups !== undefined && matches.groups.code !== undefined) {
errorMessage += '\n ↳ ' + pullRemoteRules404Error
function addPullFilteringRulesToPicker() {
var $rootContainer = $('div.server-rule-container-pull')
var $pickerTags = $rootContainer.find('select.rules-select-picker-tag')
var $pickerOrgs = $rootContainer.find('select.rules-select-picker-org')
if (serverID !== "") {
$pickerOrgs.parent().children().prop('disabled', true)
$pickerTags.parent().children().prop('disabled', true)
getPullFilteringRules(
function(data) {
addOptions($pickerTags, data.tags)
addOptions($pickerOrgs, data.organisations)
$('div.notice-pull-rule-fetched.alert-success').show()
},
function(errorMessage) {
var regex = /Reponse was not OK\. \(HTTP code: (?<code>\d+)\)/m
var matches = errorMessage.match(regex)
if (matches !== null) {
if (matches.groups !== undefined && matches.groups.code !== undefined) {
errorMessage += '\n ↳ ' + pullRemoteRules404Error
}
}
}
$('div.notice-pull-rule-fetched.alert-warning').show().find('.reason').text(errorMessage)
$pickerTags.parent().remove()
$pickerOrgs.parent().remove()
$rootContainer.find('.freetext-button-toggle-tag').collapse('show').remove()
$rootContainer.find('.freetext-button-toggle-org').collapse('show').remove()
$rootContainer.find('.collapse-freetext-tag').removeClass('collapse')
$rootContainer.find('.collapse-freetext-org').removeClass('collapse')
},
function() {
$('div.notice-pull-rule-fetched.alert-primary').hide()
$pickerTags.parent().children().prop('disabled', false).trigger('chosen:updated')
$pickerOrgs.parent().children().prop('disabled', false).trigger('chosen:updated')
},
)
} else {
$('div.notice-pull-rule-fetched.alert-warning').show().find('.reason').text('<?= __('The server must first be saved in order to fetch remote synchronisation rules.') ?>')
$pickerTags.parent().remove()
$pickerOrgs.parent().remove()
$('div.notice-pull-rule-fetched.alert-primary').hide()
}
}
function getPullFilteringRules(callback, failCallback, alwaysCallback) {
$.getJSON('/servers/queryAvailableSyncFilteringRules/' + serverID, function(availableRules) {
if (availableRules.error.length == 0) {
callback(availableRules.data)
$('div.notice-pull-rule-fetched.alert-warning').show().find('.reason').text(errorMessage)
$pickerTags.parent().remove()
$pickerOrgs.parent().remove()
$rootContainer.find('.freetext-button-toggle-tag').collapse('show').remove()
$rootContainer.find('.freetext-button-toggle-org').collapse('show').remove()
$rootContainer.find('.collapse-freetext-tag').removeClass('collapse')
$rootContainer.find('.collapse-freetext-org').removeClass('collapse')
},
function() {
$('div.notice-pull-rule-fetched.alert-primary').hide()
$pickerTags.parent().children().prop('disabled', false).trigger('chosen:updated')
$pickerOrgs.parent().children().prop('disabled', false).trigger('chosen:updated')
},
)
} else {
failCallback(availableRules.error)
$('div.notice-pull-rule-fetched.alert-warning').show().find('.reason').text('<?= __('The server must first be saved in order to fetch remote synchronisation rules.') ?>')
$pickerTags.parent().remove()
$pickerOrgs.parent().remove()
$('div.notice-pull-rule-fetched.alert-primary').hide()
}
})
.always(function() {
alwaysCallback()
})
}
function addOptions($select, data) {
data.forEach(function(entry) {
$select.append($('<option/>', {
value: entry,
text : entry
}));
});
}
function setupCodeMirror() {
var cmOptions = {
mode: "application/json",
theme:'default',
gutters: ["CodeMirror-lint-markers"],
lint: true,
lineNumbers: true,
indentUnit: 4,
showCursorWhenSelecting: true,
lineWrapping: true,
autoCloseBrackets: true,
}
cm = CodeMirror.fromTextArea(document.getElementById('urlParams'), cmOptions);
cm.on("keyup", function (cm, event) {
$('#urlParams').val(cm.getValue())
});
}
})
function getPullFilteringRules(callback, failCallback, alwaysCallback) {
$.getJSON('/servers/queryAvailableSyncFilteringRules/' + serverID, function(availableRules) {
if (availableRules.error.length == 0) {
callback(availableRules.data)
} else {
failCallback(availableRules.error)
}
})
.always(function() {
alwaysCallback()
})
}
function addOptions($select, data) {
data.forEach(function(entry) {
$select.append($('<option/>', {
value: entry,
text: entry
}));
});
}
function setupCodeMirror() {
var cmOptions = {
mode: "application/json",
theme: 'default',
gutters: ["CodeMirror-lint-markers"],
lint: true,
lineNumbers: true,
indentUnit: 4,
showCursorWhenSelecting: true,
lineWrapping: true,
autoCloseBrackets: true,
}
cm = CodeMirror.fromTextArea(document.getElementById('urlParams'), cmOptions);
cm.on("keyup", function(cm, event) {
$('#urlParams').val(cm.getValue())
});
}
})
</script>
<style>
div.server-rule-container-pull .CodeMirror {
height: 100px;
width: 100%;
border: 1px solid #ddd;
}
div.server-rule-container-pull .CodeMirror {
height: 100px;
width: 100%;
border: 1px solid #ddd;
}
</style>

32
app/View/Elements/serverRuleElements/push.ctp
View File

@ -4,11 +4,15 @@
$tagBlockRules = [];
$orgAllowRules = [];
$orgBlockRules = [];
$attributeTypeBlockRules = [];
$objectTypeBlockRules = [];
if (!empty($ruleObject)) {
$tagAllowRules = mapIDsToObject($allTags, $ruleObject['tags']['OR']);
$tagBlockRules = mapIDsToObject($allTags, $ruleObject['tags']['NOT']);
$orgAllowRules = mapIDsToObject($allOrganisations, $ruleObject['orgs']['OR']);
$orgBlockRules = mapIDsToObject($allOrganisations, $ruleObject['orgs']['NOT']);
$attributeTypeBlockRules = !empty($ruleObject['type_attributes']['NOT']) ? $ruleObject['type_attributes']['NOT'] : [];
$objectTypeBlockRules = !empty($ruleObject['type_objects']['NOT']) ? $ruleObject['type_objects']['NOT'] : [];
}
function mapIDsToObject($data, $ids) {
$result = [];
@ -58,5 +62,33 @@
'disableFreeText' => true
]);
?>
<?php
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type'))) {
echo $this->element('serverRuleElements/rules_filtering_type', [
'technique' => 'push',
'allowEmptyOptions' => true,
'allAttributeTypes' => $allAttributeTypes,
'attributeTypeBlockRules' => $attributeTypeBlockRules,
'allObjectTypes' => $allObjectTypes,
'objectTypeBlockRules' => $objectTypeBlockRules,
]);
}
?>
<div style="height: 50px;"></div>
</div>
<script>
var pullRemoteRules404Error = '<?= __('Connection error or the remote version is not supporting remote filter lookups (v2.4.142+). Make sure that the remote instance is accessible and that it is up to date.') ?>'
var cm;
$(function() {
var serverID = "<?= isset($id) ? $id : '' ?>"
<?php if (empty($attributeTypeBlockRules) && empty($objectTypeBlockRules)) : ?>
$('div.server-rule-container-push .type-filtering-subcontainer').hide()
<?php else : ?>
$('div.server-rule-container-push #type-filtering-cb').prop('checked', true)
$('div.server-rule-container-push #type-filtering-notice-cb').prop('checked', true)
$('div.server-rule-container-push .type-filtering-container').show()
<?php endif; ?>
})
</script>

57
app/View/Elements/serverRuleElements/rules_filtering_type.ctp Normal file
View File

@ -0,0 +1,57 @@
<div id="eventreport_div">
<span class="report-title-section">
<label class="checkbox">
<input id="type-filtering-cb" type="checkbox" onclick="$('div.server-rule-container-<?= $technique ?> .type-filtering-container').toggle()"><?= __('Type filtering') ?>
</label>
</span>
<div class="type-filtering-container hidden">
<div class="alert alert-error">
<button type="button" class="close" data-dismiss="alert">&times;</button>
<strong><?= __('Warning!') ?></strong>
<?= __('Use this feature only if you know exactly what you are doing as it might introduce unwanted behaviour:') ?>
<ul>
<li><?= __('This instance will potentially receive incomplete events (missing the filtered-out types)') ?></li>
<li><?= __('If later on you were to decide to have the previously filtered types included, the only way for this instance to receive them is to completely delete the affected events, as a full sync is needed') ?></li>
<li><?= __('Any instances synchronising with this instances will also receive incomplete events') ?></li>
</ul>
<strong><?= __('Any instance being synchronised with this one will also be affected by these shortcomings!') ?></strong>
<label class="checkbox">
<input id="type-filtering-notice-cb" type="checkbox" onclick="$('div.server-rule-container-<?= $technique ?> .type-filtering-subcontainer').toggle()"><?= __('I understand the caveats mentioned above resulting from the use of these filters') ?>
</label>
</div>
<div class="type-filtering-subcontainer" style="display: flex; flex-direction: column;">
<div style="display: flex;">
<h4 class="bold green" style=""></h4>
<h4 class="bold red" style="margin-left: auto;"><?= __('AND NOT'); ?></h4>
</div>
<?php
echo $this->element('serverRuleElements/rules_widget', [
'scope' => 'type_attributes',
'scopeI18n' => __('Attribute Types'),
'technique' => $technique,
'allowEmptyOptions' => true,
'options' => $allAttributeTypes,
'optionNoValue' => true,
'initAllowOptions' => [],
'initBlockOptions' => $attributeTypeBlockRules,
'disableAllow' => true,
'disableFreeText' => true,
]);
?>
<?php
echo $this->element('serverRuleElements/rules_widget', [
'scope' => 'type_objects',
'scopeI18n' => __('Object Types'),
'technique' => $technique,
'allowEmptyOptions' => true,
'options' => $allObjectTypes,
'optionNoValue' => false,
'initAllowOptions' => [],
'initBlockOptions' => $objectTypeBlockRules,
'disableAllow' => true,
'disableFreeText' => true,
]);
?>
</div>
</div>
</div>

30
app/View/Elements/serverRuleElements/rules_widget.ctp
View File

@ -4,6 +4,7 @@ $pickerDisplayed = false;
?>
<div>
<div style="display: flex;" class="rules-widget-container container-seed-<?= $seed ?> scope-<?= Inflector::pluralize(h($scope)) ?>" data-funname="initRuleWidgetPicker<?= $seed ?>">
<?php if (empty($disableAllow)): ?>
<div style="flex-grow: 1;">
<div class="bold green" style="display: flex; align-items: center;">
<?= __('Allowed %s (OR)', Inflector::pluralize(h($scopeI18n)));?>
@ -28,11 +29,13 @@ $pickerDisplayed = false;
<?php endforeach; ?>
</select>
</div>
<?php endif; ?>
<div style="display: flex; margin: 0 0.5em; flex-shrink: 1; padding-top: 20px;">
<div style="display: flex; flex-direction: column;">
<?php if(!empty($options) || $allowEmptyOptions): ?>
<?php $pickerDisplayed = true; ?>
<div class="input-prepend input-append">
<?php if (empty($disableAllow)): ?>
<button
class="btn"
type="button"
@ -43,19 +46,31 @@ $pickerDisplayed = false;
>
<i class="<?= $this->FontAwesome->getClass('caret-left') ?>"></i>
</button>
<?php endif; ?>
<select
class="rules-select-picker rules-select-picker-<?= h($scope) ?>"
multiple
placeholder="<?= sprintf('%s name', h($scopeI18n)) ?>"
>
<?php foreach($options as $option): ?>
<?php if(is_array($option)): ?>
<?php foreach($options as $optGroup => $option): ?>
<?php if(!is_numeric($optGroup) && is_array($option)): ?>
<optgroup label="<?= h($optGroup) ?>">
<?php foreach($option as $subOption): ?>
<?php if(is_array($subOption)): ?>
<option value="<?= !empty($optionNoValue) ? h($subOption['name']) : h($subOption['id']) ?>"><?= h($subOption['name']) ?></option>
<?php else: ?>
<option value="<?= h($subOption) ?>"><?= h($subOption) ?></option>
<?php endif; ?>
<?php endforeach; ?>
</optgroup>
<?php elseif(is_array($option)): ?>
<option value="<?= !empty($optionNoValue) ? h($option['name']) : h($option['id']) ?>"><?= h($option['name']) ?></option>
<?php else: ?>
<option value="<?= h($option) ?>"><?= h($option) ?></option>
<?php endif; ?>
<?php endforeach; ?>
</select>
<?php if (empty($disableBlock)): ?>
<button
class="btn"
type="button"
@ -66,6 +81,7 @@ $pickerDisplayed = false;
>
<i class="<?= $this->FontAwesome->getClass('caret-right') ?>"></i>
</button>
<?php endif; ?>
</div>
<?php endif; ?>
<?php if(!isset($disableFreeText) || !$disableFreeText): ?>
@ -110,6 +126,7 @@ $pickerDisplayed = false;
<?php endif; ?>
</div>
</div>
<?php if (empty($disableBlock)): ?>
<div style="flex-grow: 1;">
<div class="bold red" style="display: flex; align-items: center;">
<?php echo __('Blocked %s (AND NOT)', Inflector::pluralize(h($scopeI18n)));?>
@ -134,14 +151,17 @@ $pickerDisplayed = false;
<?php endforeach; ?>
</select>
</div>
<?php endif; ?>
</div>
</div>
<script>
function initRuleWidgetPicker<?= $seed ?>() {
var $baseContainer = $('.container-seed-<?= $seed ?>');
$baseContainer.find('select.rules-select-picker').chosen({
placeholder_text_multiple: "<?= __('Select some %s', Inflector::humanize(Inflector::pluralize(h($scopeI18n)))); ?>"
var $select = $baseContainer.find('select.rules-select-picker')
$select.chosen({
placeholder_text_multiple: "<?= __('Select some %s', Inflector::humanize(Inflector::pluralize(h($scopeI18n)))); ?>",
width: $select.is(":visible") ? undefined : 220,
})
$baseContainer.find('select.rules-select-data').keydown(function(evt) {
var $select = $(this)
@ -194,7 +214,7 @@ function handleFreetextButtonClick(targetClass, clicked) {
function handlePickerButtonClick(targetClass, clicked) {
var $select = $(clicked).parent().find('select');
var values = $select.val()
$select.children().each(function() {
$select.find('option').each(function() {
if (values.includes($(this).val())) {
var $target = $select.closest('.rules-widget-container').find('select.' + targetClass)
moveItemToSelect($target, $(this))

3
app/View/Events/add_misp_export_result.ctp
View File

@ -9,7 +9,6 @@
</tr>
<?php
App::uses('JSONConverterTool', 'Tools');
$converter = new JSONConverterTool();
foreach ($results as &$result):
if ($result['result'] === false) {
$status = __('Failed');
@ -25,7 +24,7 @@
} else if (is_numeric($result['result'])) {
$text = __('Event with this UUID already exists.');
}
if (!empty($result['validationIssues'])) $result['validationIssues'] = $converter->arrayPrinter($result['validationIssues']);
if (!empty($result['validationIssues'])) $result['validationIssues'] = JSONConverterTool::arrayPrinter($result['validationIssues']);
else $result['validationIssues'] = false;
?>
<tr>

41
app/View/Events/filter_event_index.ctp
View File

@ -10,7 +10,7 @@
'class' => 'input',
//'label' => 'Add Filtering Rule',
'onchange' => "indexRuleChange();",
'style' => 'margin-right:3px;width:120px;',
'style' => 'margin-right:3px;width:130px;',
'div' => false
));
echo $this->Form->input('searchbool', array(
@ -87,6 +87,39 @@
'style' => 'display:none;width:236px;',
'div' => false
));
echo $this->Form->input('searchtimestampfrom', array(
'class' => 'input',
'label' => false,
'style' => 'display:none;width:236px;margin-right:3px;',
'div' => false,
'placeholder' => __("YYYY-MM-DD HH:mm:ss")
));
echo $this->Form->input('searchtimestampuntil', array(
'class' => 'input',
'label' => false,
'style' => 'display:none;width:236px;margin-right:3px;',
'div' => false,
'placeholder' => __("YYYY-MM-DD HH:mm:ss")
));
echo $this->Form->input('searchpublishtimestampfrom', array(
'class' => 'input',
'label' => false,
'style' => 'display:none;width:236px;margin-right:3px;',
'div' => false,
'placeholder' => __("YYYY:MM:DD HH:MM:SS")
));
echo $this->Form->input('searchpublishtimestampuntil', array(
'class' => 'input',
'label' => false,
'style' => 'display:none;width:236px;margin-right:3px;',
'div' => false,
'placeholder' => __("YYYY:MM:DD HH:MM:SS")
));
echo $this->Form->input('searcheventinfo', array(
'label' => false,
'class' => 'input-large',
@ -134,7 +167,7 @@
<th style="width:10px;border:1px solid #cccccc;border-left:0px;text-align: left;"></th>
</tr>
<?php
$fields = array('published', 'org', 'tag', 'date', 'eventinfo', 'eventid', 'threatlevel', 'analysis', 'distribution', 'sharinggroup', 'attribute', 'hasproposal');
$fields = array('published', 'org', 'tag', 'date', 'eventinfo', 'eventid', 'threatlevel', 'analysis', 'distribution', 'sharinggroup', 'attribute', 'hasproposal', 'timestamp', 'publishtimestamp');
if ($isSiteAdmin) $fields[] = 'email';
foreach ($fields as $k => $field):
?>
@ -209,11 +242,11 @@ var filtering = <?php echo $filtering; ?>;
var operators = ["<?php echo __('OR');?>", "<?php echo __('NOT');?>"];
var allFields = ["published", "tag", "date", "eventinfo", "eventid", "threatlevel", "distribution", "sharinggroup", "analysis", "attribute", "hasproposal"];
var allFields = ["published", "tag", "date", "eventinfo", "eventid", "threatlevel", "distribution", "sharinggroup", "analysis", "attribute", "hasproposal", "timestamp", "publishtimestamp"];
var simpleFilters = ["tag", "eventinfo", "eventid", "threatlevel", "distribution", "sharinggroup", "analysis", "attribute"];
var differentFilters = ["published", "date", "hasproposal"];
var differentFilters = ["published", "date", "hasproposal", "timestamp", "publishtimestamp"];
var typedFields = ["tag", "threatlevel", "distribution", "analysis"];

6
app/View/Events/index.ctp
View File

@ -16,7 +16,7 @@
$filterParamsString[] = sprintf(
'%s: %s',
h(ucfirst($k)),
h($v)
h(is_array($v) ? http_build_query($v) : h($v) )
);
}
@ -30,7 +30,9 @@
'sightings' => __('Sightings'),
'proposals' => __('Proposals'),
'discussion' => __('Posts'),
'report_count' => __('Report count')
'report_count' => __('Report count'),
'timestamp' => __('Last change at'),
'publish_timestamp' => __('Published at')
];
$columnsMenu = [];

3
app/View/Events/json/view.ctp
View File

@ -1,6 +1,5 @@
<?php
App::uses('JSONConverterTool', 'Tools');
$converter = new JSONConverterTool();
foreach ($converter->streamConvert($event) as $part) {
foreach (JSONConverterTool::streamConvert($event) as $part) {
echo $part;
}

9
app/View/Feeds/preview_event.ctp
View File

@ -26,6 +26,15 @@ $tableData[] = [
'key_title' => $eventDescriptions['analysis']['desc'],
'value' => $analysisLevels[$event['Event']['analysis']],
];
if ($event['Event']['distribution'] == 4) {
$distributionText = $event['SharingGroup']['name'];
} else {
$distributionText = $distributionLevels[$event['Event']['distribution']];
}
$tableData[] = [
'key' => __('Distribution'),
'value' => $distributionText
];
$tableData[] = [
'key' => __('Info'),
'value' => $event['Event']['info']

15
app/View/Helper/AppHelper.php
View File

@ -20,9 +20,7 @@
* @license MIT License (http://www.opensource.org/licenses/mit-license.php)
*/
App::uses('UrlCacheAppHelper', 'UrlCache.View/Helper');
App::uses('Helper', 'View');
/**
* Application helper
@ -32,16 +30,13 @@ App::uses('UrlCacheAppHelper', 'UrlCache.View/Helper');
*
* @package app.View.Helper
*/
class AppHelper extends UrlCacheAppHelper {
public function afterLayout($layoutFile) {
}
public function url($url = null, $full = false) {
class AppHelper extends Helper
{
public function url($url = null, $full = false)
{
if (is_array($url) && !isset($url['admin'])) {
$url['admin'] = false;
}
return parent::url($url, $full);
}
}

18
app/View/Helper/OrgImgHelper.php
View File

@ -6,6 +6,9 @@ class OrgImgHelper extends AppHelper
{
const IMG_PATH = APP . WEBROOT_DIR . DS . 'img' . DS . 'orgs' . DS;
/** @var array */
private $imageCache = [];
public function getNameWithImg(array $organisation, $link = null)
{
if (!isset($organisation['Organisation'])) {
@ -97,15 +100,26 @@ class OrgImgHelper extends AppHelper
*/
private function findOrgImage(array $options)
{
if (isset($options['id']) && array_key_exists($options['id'], $this->imageCache)) {
return $this->imageCache[$options['id']];
}
$image = null;
foreach (['id', 'name', 'uuid'] as $field) {
if (isset($options[$field])) {
foreach (['png', 'svg'] as $extensions) {
if (file_exists(self::IMG_PATH . $options[$field] . '.' . $extensions)) {
return $options[$field] . '.' . $extensions;
$image = $options[$field] . '.' . $extensions;
break 2;
}
}
}
}
return null;
if (isset($options['id'])) {
$this->imageCache[$options['id']] = $image;
}
return $image;
}
}

17
app/View/Servers/edit.ctp
View File

@ -157,12 +157,20 @@
<span id="push_tags_NOT" style="display:none;"><?php echo __('Events with the following tags blocked: ');?><span id="push_tags_NOT_text" style="color:red;"></span><br /></span>
<span id="push_orgs_OR" style="display:none;"><?php echo __('Events with the following organisations allowed: ');?><span id="push_orgs_OR_text" style="color:green;"></span><br /></span>
<span id="push_orgs_NOT" style="display:none;"><?php echo __('Events with the following organisations blocked: ');?><span id="push_orgs_NOT_text" style="color:red;"></span><br /></span>
<?php if(!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type'))): ?>
<span id="push_type_attributes_NOT" style="display:none;"><?php echo __('Attributes of the following types blocked: ');?><span id="push_type_attributes_NOT_text" style="color:red;"></span><br /></span>
<span id="push_type_objects_NOT" style="display:none;"><?php echo __('Objects of the following uuids blocked: ');?><span id="push_type_objects_NOT_text" style="color:red;"></span><br /></span>
<?php endif; ?>
<span id="push_modify" class="btn btn-inverse" style="line-height:10px; padding: 4px 4px;"><?php echo __('Modify');?></span><br /><br />
<b><?php echo __('Pull rules:');?></b><br />
<span id="pull_tags_OR" style="display:none;"><?php echo __('Events with the following tags allowed: ');?><span id="pull_tags_OR_text" style="color:green;"></span><br /></span>
<span id="pull_tags_NOT" style="display:none;"><?php echo __('Events with the following tags blocked: ');?><span id="pull_tags_NOT_text" style="color:red;"></span><br /></span>
<span id="pull_orgs_OR" style="display:none;"><?php echo __('Events with the following organisations allowed: ');?><span id="pull_orgs_OR_text" style="color:green;"></span><br /></span>
<span id="pull_orgs_NOT" style="display:none;"><?php echo __('Events with the following organisations blocked: ');?><span id="pull_orgs_NOT_text" style="color:red;"></span><br /></span>
<?php if(!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type'))): ?>
<span id="pull_type_attributes_NOT" style="display:none;"><?php echo __('Attributes of the following types blocked: ');?><span id="pull_type_attributes_NOT_text" style="color:red;"></span><br /></span>
<span id="pull_type_objects_NOT" style="display:none;"><?php echo __('Objects of the following uuids blocked: ');?><span id="pull_type_objects_NOT_text" style="color:red;"></span><br /></span>
<?php endif; ?>
<span id="pull_url_params" style="display:none;"><?php echo __('Additional parameters: ');?><span id="pull_url_params_text" style="color:green;"></span><br /></span>
<span id="pull_modify" class="btn btn-inverse" style="line-height:10px; padding: 4px 4px;"><?php echo __('Modify');?></span><br /><br />
<?php
@ -245,18 +253,23 @@ var formInfoValues = {
var rules = {
"push": {
"tags": {"OR":[], "NOT":[]},
"orgs": {"OR":[], "NOT":[]}
"orgs": {"OR":[], "NOT":[]},
"type_attributes": {"NOT":[]},
"type_objects": {"NOT":[]},
},
"pull": {
"tags": {"OR":[], "NOT":[]},
"orgs": {"OR":[], "NOT":[]},
"type_attributes": {"NOT":[]},
"type_objects": {"NOT":[]},
"url_params": ""
}
};
var validOptions = ['pull', 'push'];
var validFields = ['tags', 'orgs'];
var validFields = ['tags', 'orgs', 'type_attributes', 'type_objects'];
var tags = <?php echo json_encode($allTags); ?>;
var orgs = <?php echo json_encode($allOrganisations); ?>;
var type_objects = <?php echo json_encode($allObjectTypes); ?>;
var delete_cert = false;
var delete_client_cert = false;
var host_org_id = "<?php echo h($host_org_id); ?>";

3
app/View/Servers/index.ctp
View File

@ -42,6 +42,9 @@ foreach ($servers as $server):
$rules['pull'] = json_decode($server['Server']['pull_rules'], true);
$syncOptions = array('pull', 'push');
$fieldOptions = array('tags', 'orgs');
if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type'))) {
$fieldOptions = array_merge($fieldOptions, ['type_attributes', 'type_objects']);
}
$typeOptions = array('OR' => array('colour' => 'green', 'text' => 'allowed'), 'NOT' => array('colour' => 'red', 'text' => 'blocked'));
$ruleDescription = array('pull' => '', 'push' => '');
foreach ($syncOptions as $syncOption) {

10
app/webroot/index.php
View File

@ -32,17 +32,15 @@ if (!defined('DS')) {
/**
* The full path to the directory which holds "app", WITHOUT a trailing DS.
*
*/
if (!defined('ROOT')) {
define('ROOT', dirname(dirname(dirname(__FILE__))));
define('ROOT', dirname(__DIR__, 2));
}
/**
* The actual directory name for the "app".
*
*/
if (!defined('APP_DIR')) {
define('APP_DIR', basename(dirname(dirname(__FILE__))));
define('APP_DIR', basename(dirname(__DIR__)));
}
/**
@ -73,10 +71,10 @@ if (!defined('CAKE_CORE_INCLUDE_PATH') && file_exists($vendorPath . DS . $dispat
* Change at your own risk.
*/
if (!defined('WEBROOT_DIR')) {
define('WEBROOT_DIR', basename(dirname(__FILE__)));
define('WEBROOT_DIR', basename(__DIR__));
}
if (!defined('WWW_ROOT')) {
define('WWW_ROOT', dirname(__FILE__) . DS);
define('WWW_ROOT', __DIR__ . DS);
}
if (!defined('CAKE_CORE_INCLUDE_PATH')) {

34
app/webroot/js/event-timeline.js
View File

@ -1,4 +1,5 @@
var max_displayed_char_timeline = 64;
var imgSize = '80';
var eventTimeline;
var items_timeline;
var items_backup;
@ -179,7 +180,19 @@ function build_attr_template(attr) {
if (!attr.seen_enabled) {
span.addClass('timestamp-attr');
}
span.text(attr.content);
if (attr.attribute_type == 'attachment' && attr.is_image) {
var $img = $('<img/>')
.addClass('screenshot img-rounded')
.attr('alt', attr.content)
.attr('title', attr.content)
.attr('loading', 'lazy')
.attr('src', '/attributes/viewPicture/' + attr.orig_id + '/1')
.attr('width', imgSize)
.attr('height', imgSize);
span.append($img)
} else {
span.text(attr.content);
}
span.data('seen_enabled', attr.seen_enabled);
var html = span[0].outerHTML;
return html;
@ -191,8 +204,26 @@ function build_object_template(obj) {
if (!obj.seen_enabled) {
table.addClass('timestamp-obj');
}
var $imgContainer = $('<div/>');
for (var attr of obj.Attribute) {
if (attr.attribute_type == 'attachment' && attr.is_image) {
var $img = $('<img/>')
.addClass('screenshot img-rounded')
.attr('alt', attr.content)
.attr('title', attr.content)
.attr('loading', 'lazy')
.attr('src', '/attributes/viewPicture/' + attr.id + '/1')
.attr('width', imgSize)
.attr('height', imgSize)
.attr('style', $imgContainer.length > 0 ? 'margin-left: 5px' : '');
$imgContainer.append($img)
}
}
var bolt_html = obj.overwrite_enabled ? " <i class=\"fa fa-bolt\" style=\"color: yellow; font-size: large;\" title=\"The Object is overwritten by its attributes\">" : "";
table.append($('<tr class="timeline-objectName"><th>'+obj.content+bolt_html+'</th><th></th></tr>'));
if ($imgContainer.length > 0) {
table.append($('<tr/>').append('<td colspan="2"/>').append($imgContainer));
}
for (var attr of obj.Attribute) {
var overwritten = obj.overwrite_enabled && (attr.contentType == "first-seen" || attr.contentType == "last-seen") ? " <i class=\"fa fa-bolt\" style=\"color: yellow;\" title=\"Overwrite object "+attr.contentType+"\"></i>" : "";
table.append(
@ -479,6 +510,7 @@ function reload_timeline() {
eventTimeline.setGroups(null);
}
items_timeline.add(data.items);
eventTimeline.fit()
},
error: function( jqXhr, textStatus, errorThrown ){
console.log( errorThrown );

98
app/webroot/js/misp.js
View File

@ -2094,6 +2094,27 @@ function indexEvaluateFiltering() {
}
}
$('#value_date').html(text);
if (filtering.timestamp.from != null) {
var text = "";
if (filtering.timestamp.from != "") text = "From: " + $('<span>').text(filtering.timestamp.from).html();
if (filtering.timestamp.until != "") {
if (text != "") text += " ";
text += "Until: " + $('<span>').text(filtering.timestamp.until).html();
}
}
$('#value_timestamp').html(text);
if (filtering.publishtimestamp.from != null) {
var text = "";
if (filtering.publishtimestamp.from != "") text = "From: " + $('<span>').text(filtering.publishtimestamp.from).html();
if (filtering.publishtimestamp.until != "") {
if (text != "") text += " ";
text += "Until: " + $('<span>').text(filtering.publishtimestamp.until).html();
}
}
$('#value_publishtimestamp').html(text);
for (var i = 0; i < simpleFilters.length; i++) {
indexEvaluateSimpleFiltering(simpleFilters[i]);
}
@ -2318,6 +2339,22 @@ function indexCreateFilters() {
if (text != "") text += "/";
text += "searchDateuntil:" + filtering.date.until;
}
if (filtering.timestamp.from) {
if (text != "") text += "/";
text += "searchTimestamp:" + filtering.timestamp.from;
}
if (filtering.timestamp.until) {
if (text != "") text += "/";
text += "searchTimestamp:" + filtering.timestamp.until;
}
if (filtering.publishtimestamp.from) {
if (text != "") text += "/";
text += "searchPublishTimestamp:" + filtering.publishtimestamp.from;
}
if (filtering.publishtimestamp.until) {
if (text != "") text += "/";
text += "searchPublishTimestamp:" + filtering.publishtimestamp.until;
}
return baseurl + '/events/index/' + text;
} else {
return baseurl + '/admin/users/index/' + text;
@ -2393,11 +2430,11 @@ function indexEvaluateSimpleFiltering(field) {
function indexAddRule(param) {
var found = false;
if (filterContext == 'event') {
if (param.data.param1 == "date") {
if (param.data.param1 == "date" || param.data.param1 == "timestamp" || param.data.param1 == "publishtimestamp") {
var val1 = encodeURIComponent($('#EventSearch' + param.data.param1 + 'from').val());
var val2 = encodeURIComponent($('#EventSearch' + param.data.param1 + 'until').val());
if (val1 != "") filtering.date.from = val1;
if (val2 != "") filtering.date.until = val2;
if (val1 != "") filtering[param.data.param1].from = val1;
if (val2 != "") filtering[param.data.param1].until = val2;
} else if (param.data.param1 == "published") {
var value = encodeURIComponent($('#EventSearchpublished').val());
if (value != "") filtering.published = value;
@ -2436,7 +2473,7 @@ function indexRuleChange() {
$('[id^=' + context + 'Search]').hide();
var rule = $('#' + context + 'Rule').val();
var fieldName = '#' + context + 'Search' + rule;
if (fieldName === '#' + context + 'Searchdate') {
if (fieldName === '#' + context + 'Searchdate' || fieldName === '#' + context + 'Searchtimestamp' || fieldName === '#' + context + 'Searchpublishtimestamp') {
$(fieldName + 'from').show();
$(fieldName + 'until').show();
} else {
@ -2459,6 +2496,12 @@ function indexFilterClearRow(field) {
if (field == "date") {
filtering.date.from = "";
filtering.date.until = "";
} else if (field == "timestamp") {
filtering.timestamp.from = "";
filtering.timestamp.until = "";
} else if (field == "publishtimestamp") {
filtering.publishtimestamp.from = "";
filtering.publishtimestamp.until = "";
} else if (field == "published") {
filtering.published = 2;
} else if (field == "hasproposal") {
@ -3506,7 +3549,7 @@ function convertServerFilterRules(rules) {
rules[type] = JSON.parse($(container).val());
} else {
if (type === 'pull') {
rules[type] = {"tags": {"OR": [], "NOT": []}, "orgs": {"OR": [], "NOT": []}, "url_params": ""}
rules[type] = {"tags": {"OR": [], "NOT": []}, "orgs": {"OR": [], "NOT": []}, "type_attributes": {"NOT": []}, "type_objects": {"NOT": []}, "url_params": ""}
} else {
rules[type] = {"tags": {"OR": [], "NOT": []}, "orgs": {"OR": [], "NOT": []}}
}
@ -3520,26 +3563,35 @@ function serverRuleUpdate() {
var statusOptions = ["OR", "NOT"];
validOptions.forEach(function(type) {
validFields.forEach(function(field) {
if (type === 'push') {
var indexedList = {};
window[field].forEach(function(item) {
indexedList[item.id] = item.name;
});
var indexedList = {};
if (type === 'push' || field == 'type_objects') {
if (window[field] !== undefined) {
window[field].forEach(function(item) {
indexedList[item.id] = item.name;
});
}
}
statusOptions.forEach(function(status) {
if (rules[type][field][status].length > 0) {
$('#' + type + '_' + field + '_' + status).show();
var t = '';
rules[type][field][status].forEach(function(item) {
if (t.length > 0) t += ', ';
if (type === 'pull') t += item;
else {
t += indexedList[item] !== undefined ? indexedList[item] : item;
}
});
$('#' + type + '_' + field + '_' + status + '_text').text(t);
} else {
$('#' + type + '_' + field + '_' + status).hide();
if (rules[type][field] !== undefined && rules[type][field][status] !== undefined) {
if (rules[type][field][status].length > 0) {
$('#' + type + '_' + field + '_' + status).show();
var t = '';
rules[type][field][status].forEach(function(item) {
if (t.length > 0) t += ', ';
if (type === 'pull') {
if (indexedList[item] !== undefined) {
t += indexedList[item];
} else {
t += item;
}
} else {
t += indexedList[item] !== undefined ? indexedList[item] : item;
}
});
$('#' + type + '_' + field + '_' + status + '_text').text(t);
} else {
$('#' + type + '_' + field + '_' + status).hide();
}
}
});
});

28
tests/testlive_comprehensive_local.py
View File

@ -2,6 +2,7 @@
import os
import unittest
import uuid
from xml.etree import ElementTree as ET
from io import BytesIO
import urllib3 # type: ignore
@ -565,6 +566,33 @@ class TestComprehensive(unittest.TestCase):
self.assertEqual(1, len(fetched_event.tags), fetched_event.tags)
self.assertTrue(fetched_event.tags[0].local, fetched_event.tags[0])
def test_export(self):
event = create_simple_event()
event.add_attribute("ip-src", "1.2.4.5", to_ids=True)
event = check_response(self.admin_misp_connector.add_event(event))
result = self._search({'returnFormat': "openioc", 'eventid': event.id, "published": [0, 1]})
ET.fromstring(result) # check if result is valid XML
self.assertTrue("1.2.4.5" in result, result)
result = self._search({'returnFormat': "yara", 'eventid': event.id, "published": [0, 1]})
self.assertTrue("1.2.4.5" in result, result)
self.assertTrue("GENERATED" in result, result)
self.assertTrue("AS-IS" in result, result)
result = self._search({'returnFormat': "yara-json", 'eventid': event.id, "published": [0, 1]})
self.assertIn("generated", result)
self.assertEqual(len(result["generated"]), 1, result)
self.assertIn("as-is", result)
check_response(self.admin_misp_connector.delete_event(event))
def _search(self, query: dict):
response = self.admin_misp_connector._prepare_request('POST', 'events/restSearch', data=query)
response = self.admin_misp_connector._check_response(response)
check_response(response)
return response
if __name__ == '__main__':
unittest.main()

1
travis/bootstrap.php
View File

@ -106,7 +106,6 @@ if (!Configure::read('MISP.baseurl')) {
CakePlugin::load('SysLog');
CakePlugin::load('Assets'); // having Logable
CakePlugin::load('SysLogLogable');
CakePlugin::load('UrlCache');
/**
* Uncomment the following line to enable client SSL certificate authentication.