MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'develop' of github.com:MISP/MISP into develop

pull/9570/head
Christian Studer 2024-01-04 13:42:24 +01:00
parent 6da4d82106 0f268782cc
commit 7dc2a0e562
No known key found for this signature in database
GPG Key ID: 6BBED1B63A6D639F
52 changed files with 1605 additions and 335 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -24,7 +24,6 @@ tools/mkdocs
/.idea
.DS_Store
/.htaccess
/app/Vendor
/README
/app/tmp/GPG*
/app/tmp/sessions/sess_*
@ -40,6 +39,7 @@ tools/mkdocs
/app/tmp/cache/feeds/*.etag
app/Lib/EventWarning/Custom/*
!app/Lib/EventWarning/Custom/empty
!/app/files/certs/empty
!/app/files/feed-metadata
!/app/files/empty
!/app/files/scripts/
@ -69,7 +69,6 @@ app/Lib/EventWarning/Custom/*
/app/files/scripts/stix2/*
!/app/files/scripts/stix2/misp2stix2*.py
!/app/files/scripts/stix2/stix2misp*.py
!/app/files/empty
/app/files/terms/*
!/app/files/terms/empty
!/app/files/browscap

1
INSTALL/INSTALL.sh
View File

@ -3628,6 +3628,7 @@ armv7l-ubuntu-bionic
armv7l-ubuntu-focal
aarch64-ubuntu-focal
aarch64-ubuntu-hirsute
aarch64-ubuntu-jammy
"
# Check if we actually support this configuration

2
INSTALL/INSTALL.sh.sha1
View File

@ -1 +1 @@
8624b1d834a4c958b16dce32ceae88c3d1dc15d7 INSTALL.sh
9be7870e15e262913fc6a007d93d09d672675a7c INSTALL.sh

2
INSTALL/INSTALL.sh.sha256
View File

@ -1 +1 @@
d7b9e370c85b53bbf7b0f81f5c263b6ab2e534f59b40a6499277f39407ff194a INSTALL.sh
ee53f64a2996c551ffcccf4a928dc4a67cb5d373026b430e14df6b72a984c42d INSTALL.sh

2
INSTALL/INSTALL.sh.sha384
View File

@ -1 +1 @@
ef5b68e9d0d634c2cadd4fd9b1e5c2a93dbd938f488417f67a2b9c87e8867cb0200293a150cdaeda369e7fdc476eec2b INSTALL.sh
9f2ebadefb74bff235d86913bc825da045ff67ae23c9833683ff0780bbcbf63fefaa056ee169a97fd1c1765a483e7af4 INSTALL.sh

2
INSTALL/INSTALL.sh.sha512
View File

@ -1 +1 @@
1445710924bc029647cc5aa0ebffd0a3b6ddaf39d26b5a0e951fffeef621677c59470f250ecfb6059c9e200951f98d4f21646f152d6f8931438531f9516a8748 INSTALL.sh
c600dd0b2a98b570bfc66128b3637b55c2414147ea5d92c7cd5670b23f0862df3828b6e3e172777e08e2f810a88a7619f24bc7ff4a522e90f469136f791cfe14 INSTALL.sh

1
INSTALL/INSTALL.tpl.sh
View File

@ -867,6 +867,7 @@ armv7l-ubuntu-bionic
armv7l-ubuntu-focal
aarch64-ubuntu-focal
aarch64-ubuntu-hirsute
aarch64-ubuntu-jammy
"
# Check if we actually support this configuration

2
PyMISP

@ -1 +1 @@
Subproject commit 190f82923687cebaf06138724e7c03239e529cb4
Subproject commit 81f5d596a7dd5cb1ca7213ac4fbdf07b402420b7

2
VERSION.json
View File

@ -1 +1 @@
{"major":2, "minor":4, "hotfix":181}
{"major":2, "minor":4, "hotfix":182}

4
app/Console/Command/AdminShell.php
View File

@ -540,6 +540,10 @@ class AdminShell extends AppShell
$whoami = ProcessTool::whoami();
if (in_array($whoami, ['httpd', 'www-data', 'apache', 'wwwrun', 'travis', 'www'], true) || $whoami === Configure::read('MISP.osuser')) {
$this->out('Executing all updates to bring the database up to date with the current version.');
$lock = $this->AdminSetting->find('first', array('conditions' => array('setting' => 'update_locked')));
if (!empty($lock)) {
$this->AdminSetting->delete($lock['AdminSetting']['id']);
}
$processId = empty($this->args[0]) ? false : $this->args[0];
$this->Server->runUpdates(true, false, $processId);
$this->Server->cleanCacheFiles();

11
app/Console/Command/UserShell.php
View File

@ -213,16 +213,18 @@ class UserShell extends AppShell
public function authkey_valid()
{
$cache = [];
$randomKey = random_bytes(16);
do {
$authkey = fgets(STDIN); // read line from STDIN
$authkey = trim($authkey);
if (strlen($authkey) !== 40) {
fwrite(STDOUT, "0\n"); // authkey is not in valid format
$this->log("Authkey in incorrect format provided.", LOG_WARNING);
continue;
}
$time = time();
// Generate hash from authkey to not store raw authkey in memory
$keyHash = hash('sha256', $authkey, true);
$keyHash = sha1($authkey . $randomKey, true);
if (isset($cache[$keyHash]) && $cache[$keyHash][1] > $time) {
fwrite(STDOUT, $cache[$keyHash][0] ? "1\n" : "0\n");
continue;
@ -250,6 +252,13 @@ class UserShell extends AppShell
}
$user = (bool)$user;
if (!$user) {
$start = substr($authkey, 0, 4);
$end = substr($authkey, -4);
$authKeyToStore = $start . str_repeat('*', 32) . $end;
$this->log("Not valid authkey $authKeyToStore provided.", LOG_WARNING);
}
// Cache results for 5 seconds
$cache[$keyHash] = [$user, $time + 5];
fwrite(STDOUT, $user ? "1\n" : "0\n");

37
app/Controller/AppController.php
View File

@ -34,7 +34,7 @@ class AppController extends Controller
public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
private $__queryVersion = '157';
public $pyMispVersion = '2.4.182';
public $pyMispVersion = '2.4.183';
public $phpmin = '7.2';
public $phprec = '7.4';
public $phptoonew = '8.0';
@ -362,6 +362,10 @@ class AppController extends Controller
}
}
}
if (Configure::read('MISP.enable_automatic_garbage_collection') && mt_rand(1,100) % 100 == 0) {
$this->loadModel('AdminSetting');
$this->AdminSetting->garbageCollect();
}
}
public function beforeRender()
@ -440,8 +444,7 @@ class AppController extends Controller
// User found in the db, add the user info to the session
if (Configure::read('MISP.log_auth')) {
$this->loadModel('Log');
$this->UserLoginProfile = ClassRegistry::init('UserLoginProfile');
$change = $this->UserLoginProfile->_getUserProfile();
$change = $this->User->UserLoginProfile->_getUserProfile();
$change['http_method'] = $_SERVER['REQUEST_METHOD'];
$change['target'] = $this->request->here;
$this->Log->createLogEntry(
@ -564,8 +567,7 @@ class AppController extends Controller
if ($user['disabled'] || (isset($user['logged_by_authkey']) && $user['logged_by_authkey']) && !$this->User->checkIfUserIsValid($user)) {
if ($this->_shouldLog('disabled:' . $user['id'])) {
$this->Log = ClassRegistry::init('Log');
$this->UserLoginProfile = ClassRegistry::init('UserLoginProfile');
$change = $this->UserLoginProfile->_getUserProfile();
$change = $this->User->UserLoginProfile->_getUserProfile();
$this->Log->createLogEntry($user, 'auth_fail', 'User', $user['id'], 'Login attempt by disabled user.', json_encode($change));
}
@ -585,9 +587,9 @@ class AppController extends Controller
if ($user['authkey_expiration'] < $time) {
if ($this->_shouldLog('expired:' . $user['authkey_id'])) {
$this->Log = ClassRegistry::init('Log');
$this->UserLoginProfile = ClassRegistry::init('UserLoginProfile');
$change = $this->UserLoginProfile->_getUserProfile();
$this->Log->createLogEntry($user, 'auth_fail', 'User', $user['id'], "Login attempt by expired auth key {$user['authkey_id']}.", json_encode($change)); }
$change = $this->User->UserLoginProfile->_getUserProfile();
$this->Log->createLogEntry($user, 'auth_fail', 'User', $user['id'], "Login attempt by expired auth key {$user['authkey_id']}.", json_encode($change));
}
$this->Auth->logout();
throw new ForbiddenException('Auth key is expired');
}
@ -604,9 +606,9 @@ class AppController extends Controller
if (!$cidrTool->contains($remoteIp)) {
if ($this->_shouldLog('not_allowed_ip:' . $user['authkey_id'] . ':' . $remoteIp)) {
$this->Log = ClassRegistry::init('Log');
$this->UserLoginProfile = ClassRegistry::init('UserLoginProfile');
$change = $this->UserLoginProfile->_getUserProfile();
$this->Log->createLogEntry($user, 'auth_fail', 'User', $user['id'], "Login attempt from not allowed IP address {$remoteIp} for auth key {$user['authkey_id']}.", json_encode($change)); }
$change = $this->User->UserLoginProfile->_getUserProfile();
$this->Log->createLogEntry($user, 'auth_fail', 'User', $user['id'], "Login attempt from not allowed IP address {$remoteIp} for auth key {$user['authkey_id']}.", json_encode($change));
}
$this->Auth->logout();
throw new ForbiddenException('It is not possible to use this Auth key from your IP address');
}
@ -878,7 +880,7 @@ class AppController extends Controller
ConnectionManager::create('default', $db->config);
}
$dataSource = $dataSourceConfig['datasource'];
if (!in_array($dataSource, ['Database/Mysql', 'Database/Postgres', 'Database/MysqlObserver', 'Database/MysqlExtended'], true)) {
if (!in_array($dataSource, ['Database/Mysql', 'Database/Postgres', 'Database/MysqlObserver', 'Database/MysqlExtended', 'Database/MysqlObserverExtended'], true)) {
throw new Exception('Datasource not supported: ' . $dataSource);
}
}
@ -1149,8 +1151,7 @@ class AppController extends Controller
$this->Session->write(AuthComponent::$sessionKey, $user['User']);
if (Configure::read('MISP.log_auth')) {
$this->Log = ClassRegistry::init('Log');
$this->UserLoginProfile = ClassRegistry::init('UserLoginProfile');
$change = $this->UserLoginProfile->_getUserProfile();
$change = $this->User->UserLoginProfile->_getUserProfile();
$change['http_method'] = $_SERVER['REQUEST_METHOD'];
$change['target'] = $this->request->here;
$this->Log->createLogEntry(
@ -1166,8 +1167,7 @@ class AppController extends Controller
// User not authenticated correctly
// reset the session information
$this->Log = ClassRegistry::init('Log');
$this->UserLoginProfile = ClassRegistry::init('UserLoginProfile');
$change = $this->UserLoginProfile->_getUserProfile();
$change = $this->User->UserLoginProfile->_getUserProfile();
$this->Log->createLogEntry(
'SYSTEM',
'auth_fail',
@ -1434,14 +1434,13 @@ class AppController extends Controller
/**
* @return string|null
* @deprecated Use User::_remoteIp() instead
*/
protected function _remoteIp()
{
$ipHeader = Configure::read('MISP.log_client_ip_header') ?: 'REMOTE_ADDR';
return isset($_SERVER[$ipHeader]) ? trim($_SERVER[$ipHeader]) : $_SERVER['REMOTE_ADDR'];
return $this->User->_remoteIp();
}
/**
* @param string $key
* @return bool Returns true if the same log defined by $key was not stored in last hour

5
app/Controller/AttributesController.php
View File

@ -2671,12 +2671,15 @@ class AttributesController extends AppController
$fails = 0;
$this->Taxonomy = ClassRegistry::init('Taxonomy');
foreach ($idList as $id) {
$conditions = $this->__idToConditions($id);
$conditions['Attribute.deleted'] = 0;
$attribute = $this->Attribute->fetchAttributeSimple($this->Auth->user(), [
'conditions' => array('Attribute.id' => $id, 'Attribute.deleted' => 0),
'conditions' => $conditions,
]);
if (empty($attribute)) {
throw new NotFoundException(__('Invalid attribute'));
}
$id = $attribute['Attribute']['id'];
if (!$this->__canModifyTag($attribute, $local)) {
$fails++;
continue;

2
app/Controller/Component/ACLComponent.php
View File

@ -596,7 +596,9 @@ class ACLComponent extends Component
'delete' => array('perm_sharing_group'),
'detach' => array('perm_sharing_group'),
'edit' => array('perm_sharing_group'),
'encodeSyncRule' => ['perm_site_admin'],
'execute' => array('perm_sharing_group'),
'generateUuidList' => ['perm_sharing_group'],
'index' => array('perm_sharing_group'),
'view' => array('perm_sharing_group'),
'viewOrgs' => array('perm_sharing_group'),

40
app/Controller/EventsController.php
View File

@ -2469,7 +2469,7 @@ class EventsController extends AppController
$original_file,
$this->data['Event']['publish'],
$this->data['Event']['distribution'],
$this->data['Event']['sharing_group_id'],
$this->data['Event']['sharing_group_id'] ?? null,
$this->data['Event']['galaxies_handling'],
$debug
);
@ -2501,15 +2501,31 @@ class EventsController extends AppController
foreach ($distributionLevels as $key => $value) {
$fieldDesc['distribution'][$key] = $this->Event->distributionDescriptions[$key]['formdesc'];
}
$debugOptions = $this->Event->debugOptions;
$debugOptions = [
0 => __('Standard debugging'),
1 => __('Advanced debugging'),
];
$debugDescriptions = [
0 => __('The critical errors are logged in the usual log file.'),
1 => __('All the errors and warnings are logged in the usual log file.'),
];
$galaxiesOptions = [
0 => __('As MISP standard format'),
1 => __('As tag names'),
];
$galaxiesOptionsDescriptions = [
0 => __('Galaxies and Clusters are passed as MISP standard format. New generic Galaxies and Clusters are created when there is no match with existing ones.'),
1 => __('Galaxies are passed as tags and there is only a simple search with existing galaxy tag names.'),
];
$this->set('debugOptions', $debugOptions);
foreach ($debugOptions as $key => $value) {
$fieldDesc['debug'][$key] = $this->Event->debugDescriptions[$key];
$fieldDesc['debug'][$key] = $debugDescriptions[$key];
}
$galaxiesOptions = $this->Event->galaxiesOptions;
$this->set('galaxiesOptions', $galaxiesOptions);
foreach ($galaxiesOptions as $key => $value) {
$fieldDesc['galaxies_handling'][$key] = $this->Event->galaxiesOptionsDescriptions[$key];
$fieldDesc['galaxies_handling'][$key] = $galaxiesOptionsDescriptions[$key];
}
$this->set('sharingGroups', $sgs);
$this->set('fieldDesc', $fieldDesc);
@ -3792,11 +3808,21 @@ class EventsController extends AppController
if ($id === false) {
$id = $this->request->data['event'];
}
$this->Event->recursive = -1;
$event = $this->Event->read(array(), $id);
$conditions = ['Event.id' => $id];
if (Validation::uuid($id)) {
$conditions = ['Event.uuid' => $id];
}
$event = $this->Event->find(
'first',
[
'recursive' => -1,
'conditions' => $conditions
]
);
if (empty($event)) {
return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Invalid event.')), 'status'=>200, 'type' => 'json'));
}
$id = $event['Event']['id'];
$local = !empty($this->params['named']['local']);
if (!$this->request->is('post')) {
$this->set('local', $local);

87
app/Controller/SharingGroupBlueprintsController.php
View File

@ -193,4 +193,91 @@ class SharingGroupBlueprintsController extends AppController
$this->render('/genericTemplates/confirm');
}
}
public function generateUuidList($id)
{
$orgs = $this->__getUuidList($id);
return $this->RestResponse->viewData($orgs, 'json');
}
private function __getUuidList($id)
{
$conditions = [];
if (empty($id)) {
throw new MethodNotAllowedException(__('No ID specified.'));
}
$conditions['SharingGroupBlueprint.id'] = $id;
if (!$this->Auth->user('Role')['perm_admin']) {
$conditions['SharingGroupBlueprint.org_id'] = $this->Auth->user('org_id');
}
$sharingGroupBlueprint = $this->SharingGroupBlueprint->find('first', ['conditions' => $conditions, 'recursive' => -1]);
if (empty($sharingGroupBlueprint)) {
throw new NotFoundException(__('Invalid Sharing Group Blueprint'));
}
// we create a fake user to restrict the visible sharing groups to the creator of the SharingGroupBlueprint, in case an admin wants to update it
$fake_user = [
'Role' => [
'perm_site_admin' => false
],
'org_id' => $sharingGroupBlueprint['SharingGroupBlueprint']['org_id'],
'id' => 1
];
$temp = $this->SharingGroupBlueprint->evaluateSharingGroupBlueprint($sharingGroupBlueprint, $fake_user);
$orgs = $this->SharingGroupBlueprint->SharingGroup->Organisation->find('list', [
'recursive' => -1,
'fields' => ['uuid'],
'conditions' => ['id' => $temp['orgs']]
]);
return array_values($orgs);
}
public function encodeSyncRule($id)
{
$org_uuids = $this->__getUuidList($id);
$this->loadModel('Server');
if ($this->request->is('post')) {
if (!isset($this->request->data['SharingGroupBlueprint'])) {
$this->request->data = ['SharingGroupBlueprint' => $this->request->data];
}
$server = $this->Server->find('first', [
'conditions' => ['Server.id' => $this->request->data['SharingGroupBlueprint']['server_id']],
'recursive' => -1
]);
if (empty($server)) {
throw new NotFoundException(__('Invalid server.'));
}
$server['Server']['pull_rules'] = json_decode($server['Server']['pull_rules'], true);
$server['Server']['push_rules'] = json_decode($server['Server']['push_rules'], true);
$rules = [];
$type_to_update = empty($this->request->data['SharingGroupBlueprint']['type']) ? 'pull' : $this->request->data['SharingGroupBlueprint']['type'];
$rule_to_update = empty($this->request->data['SharingGroupBlueprint']['rule']) ? 'OR' : $this->request->data['SharingGroupBlueprint']['rule'];
$rules[$type_to_update][$rule_to_update] = $org_uuids;
$server['Server'][$type_to_update . '_rules']['orgs'][$rule_to_update] = $rules[$type_to_update][$rule_to_update];
$server['Server']['pull_rules'] = json_encode($server['Server']['pull_rules']);
$server['Server']['push_rules'] = json_encode($server['Server']['push_rules']);
if (!$this->Server->save($server)) {
throw new InvalidArgumentException(__('Could not update the server - something went wrong.'));
} else {
if ($this->_isRest()) {
$server = $this->Server->find('first', [
'recursive' => -1,
'conditions' => ['Server.id' => $this->request->data['SharingGroupBlueprint']['server_id']]
]);
return $this->RestResponse->viewData($server, 'json');
} else {
$this->Flash->success(__('Server %s\'s %s rules\' %s branch updated with the blueprint\'s rules.', $server['Server']['id'], $type_to_update, $rule_to_update));
$this->redirect('/servers/index');
}
}
}
$servers = $this->Server->find('all', ['recursive' => -1]);
if (empty($servers)) {
throw new NotFoundException(__('No valid servers found.'));
}
$server_data = [];
foreach ($servers as $s) {
$server_data[$s['Server']['id']] = $s['Server']['name'];
}
$this->set('servers', $server_data);
}
}

2
app/Controller/UsersController.php
View File

@ -3179,7 +3179,7 @@ class UsersController extends AppController
// We didn't filter the data at SQL query too much, nor by age, as we want to show "enough" data, even if old
$rows = 0;
// group authentications by type of loginprofile, to make the list shorter
foreach($logs as $logEntry) {
foreach ($logs as $logEntry) {
$loginProfile = $this->UserLoginProfile->_fromLog($logEntry['Log']);
if (!$loginProfile) continue; // skip if empty log
$loginProfile['ip'] = $logEntry['Log']['ip'] ?? null; // transitional workaround

10
app/Lib/Export/Stix1Export.php
View File

@ -11,7 +11,7 @@ class Stix1Export extends StixExport
{
return [
ProcessTool::pythonBin(),
$this->__framing_script,
self::FRAMING_SCRIPT,
'stix1',
'-s', $this->__scope,
'-v', $this->__version,
@ -25,7 +25,7 @@ class Stix1Export extends StixExport
{
$command = [
ProcessTool::pythonBin(),
$this->__scripts_dir . 'misp2stix.py',
self::SCRIPTS_DIR . 'misp2stix.py',
'-s', $this->__scope,
'-v', $this->__version,
'-f', $this->__return_format,
@ -33,6 +33,10 @@ class Stix1Export extends StixExport
'-i',
];
$command = array_merge($command, $this->__filenames);
return ProcessTool::execute($command, null, true);
try {
return ProcessTool::execute($command, null, true);
} catch (ProcessException $e) {
return $e->stdout();
}
}
}

14
app/Lib/Export/Stix2Export.php
View File

@ -11,16 +11,20 @@ class Stix2Export extends StixExport
{
return [
ProcessTool::pythonBin(),
$this->__framing_script,
self::FRAMING_SCRIPT,
'stix2',
'-v', $this->__version,
'--uuid', CakeText::uuid(),
];
}
/**
* @return string
* @throws Exception
*/
protected function __parse_misp_data()
{
$scriptFile = $this->__scripts_dir . 'stix2/misp2stix2.py';
$scriptFile = self::SCRIPTS_DIR . 'stix2/misp2stix2.py';
$command = [
ProcessTool::pythonBin(),
$scriptFile,
@ -28,7 +32,11 @@ class Stix2Export extends StixExport
'-i',
];
$command = array_merge($command, $this->__filenames);
$result = ProcessTool::execute($command, null, true);
try {
$result = ProcessTool::execute($command, null, true);
} catch (ProcessException $e) {
$result = $e->stdout();
}
$result = preg_split("/\r\n|\n|\r/", trim($result));
return end($result);
}

5
app/Lib/Export/StixExport.php
View File

@ -6,13 +6,14 @@ App::uses('ProcessTool', 'Tools');
abstract class StixExport
{
const SCRIPTS_DIR = APP . 'files/scripts/',
FRAMING_SCRIPT = APP . 'files/scripts/misp_framing.py';
public $additional_params = array(
'includeEventTags' => 1,
'includeGalaxy' => 1
);
protected $__return_format = 'json';
protected $__scripts_dir = APP . 'files/scripts/';
protected $__framing_script = APP . 'files/scripts/misp_framing.py';
protected $__return_type = null;
/** @var array Full paths to files to convert */

42
app/Lib/Tools/ProcessTool.php
View File

@ -1,7 +1,7 @@
<?php
class ProcessException extends Exception
{
/** @var string|null */
/** @var string */
private $stderr;
/** @var string */
@ -10,14 +10,13 @@ class ProcessException extends Exception
/**
* @param string|array $command
* @param int $returnCode
* @param string|null $stderr
* @param string $stderr
* @param string $stdout
*/
public function __construct($command, $returnCode, $stderr, $stdout)
{
$commandForException = is_array($command) ? implode(' ', $command) : $command;
$stderrToMessage = $stderr === null ? 'Logged to tmp/logs/exec-errors.log' : "'$stderr'";
$message = "Command '$commandForException' finished with error code $returnCode.\nSTDERR: $stderrToMessage\nSTDOUT: '$stdout'";
$message = "Command '$commandForException' finished with error code $returnCode.\nSTDERR: '$stderr'\nSTDOUT: '$stdout'";
$this->stderr = $stderr;
$this->stdout = $stdout;
parent::__construct($message, $returnCode);
@ -41,21 +40,20 @@ class ProcessTool
/**
* @param array $command If command is array, it is not necessary to escape arguments
* @param string|null $cwd
* @param bool $stderrToFile IF true, log stderrr output to LOG_FILE
* @param bool $logToFile If true, log stderr output to LOG_FILE
* @return string Stdout
* @throws ProcessException
* @throws Exception
*/
public static function execute(array $command, $cwd = null, $stderrToFile = false)
public static function execute(array $command, $cwd = null, $logToFile = false)
{
$descriptorSpec = [
1 => ['pipe', 'w'], // stdout
2 => ['pipe', 'w'], // stderr
];
if ($stderrToFile) {
if ($logToFile) {
self::logMessage('Running command ' . implode(' ', $command));
$descriptorSpec[2] = ['file', self::LOG_FILE, 'a'];
}
// PHP older than 7.4 do not support proc_open with array, so we need to convert values to string manually
@ -75,20 +73,24 @@ class ProcessTool
throw new Exception("Could not get STDOUT of command '$commandForException'.");
}
if ($stderrToFile) {
$stderr = null;
} else {
$stderr = stream_get_contents($pipes[2]);
$stderr = stream_get_contents($pipes[2]);
if ($stderr === false) {
$commandForException = self::commandFormat($command);
throw new Exception("Could not get STDERR of command '$commandForException'.");
}
$returnCode = proc_close($process);
if ($stderrToFile) {
self::logMessage("Process finished with return code $returnCode");
if ($logToFile) {
self::logMessage("Process finished with return code $returnCode", $stderr);
}
if ($returnCode !== 0) {
throw new ProcessException($command, $returnCode, $stderr, $stdout);
$exception = new ProcessException($command, $returnCode, $stderr, $stdout);
if ($logToFile && Configure::read('Security.ecs_log')) {
EcsLog::handleException($exception);
}
throw $exception;
}
return $stdout;
@ -116,9 +118,17 @@ class ProcessTool
return Configure::read('MISP.python_bin') ?: 'python3';
}
private static function logMessage($message)
/**
* @param string $message
* @param string|null $stderr
* @return void
*/
private static function logMessage($message, $stderr = null)
{
$logMessage = '[' . date("Y-m-d H:i:s") . ' ' . getmypid() . "] $message\n";
if ($stderr) {
$logMessage = rtrim($stderr) . "\n" . $logMessage;
}
file_put_contents(self::LOG_FILE, $logMessage, FILE_APPEND | LOCK_EX);
}

14
app/Lib/Tools/SendEmail.php
View File

@ -555,14 +555,18 @@ class SendEmail
}
try {
return [
'contents' => $email->send(),
'encrypted' => $encrypted,
'subject' => $subject,
];
$content = $email->send();
} catch (Exception $e) {
throw new SendEmailException('The message could not be sent.', 0, $e);
}
return [
'to' => $user['User']['email'],
'message_id' => $email->messageId(),
'contents' => $content,
'encrypted' => $encrypted,
'subject' => $subject,
];
}
/**

65
app/Model/AdminSetting.php
View File

@ -69,4 +69,69 @@ class AdminSetting extends AppModel
return empty($this->findUpgrades($db_version['AdminSetting']['value']));
}
}
public function garbageCollect()
{
$last_collection = $this->find('first', [
'conditions' => ['setting' => 'last_gc_timestamp'],
'recursive' => -1
]);
if (empty($last_collection)) {
$last_collection = 0;
} else {
$last_collection = $last_collection['AdminSetting']['value'];
}
if ((time()) > ($last_collection + 3600)) {
$this->__cleanTmpFiles();
}
}
private function __cleanTmpFiles() {
$time = time();
$this->__deleteScriptTmpFiles($time);
$this->__deleteTaxiiTmpFiles($time);
}
private function __deleteScriptTmpFiles($time) {
$scripts_tmp_path = APP . 'files/scripts/tmp';
$dir = new Folder($scripts_tmp_path);
$contents = $dir->read(false, false);
foreach ($contents[1] as $file) {
if (preg_match('/^[a-zA-Z0-9]{12}$/', $file)) {
$tmp_file = new File($scripts_tmp_path . '/' . $file);
if ($time > $tmp_file->lastChange() + 3600) {
$tmp_file->delete();
}
unlink($scripts_tmp_path . '/' . $file);
}
}
}
private function __deleteTaxiiTmpFiles($time) {
$taxii_path = APP . 'files/scripts/tmp/Taxii';
$taxii_dir = new Folder($taxii_path);
$taxii_contents = $taxii_dir->read(false, false);
if (!empty($taxii_contents[0])) {
foreach ($taxii_contents[0] as $taxii_temp_dir) {
if (preg_match('/^[a-zA-Z0-9]{12}$/', $taxii_temp_dir)) {
$tmp_dir = new Folder($taxii_path . $taxii_temp_dir);
$taxii_temp_dir_contents = $tmp_dir->read(false, false);
if (!empty(count($taxii_temp_dir_contents[1]))) {
$files_count = count($taxii_temp_dir_contents[1]);
$files_removed = 0;
foreach ($taxii_temp_dir_contents[1] as $tmp_file) {
$tmp_file = new File($taxii_path . $taxii_temp_dir . '/' . $tmp_file);
if ($time > $tmp_file->lastChange() + 3600) {
$tmp_file->delete();
$files_removed += 1;
}
}
if ($files_count === $files_removed) {
$tmp_dir->delete();
}
}
}
}
}
}
}

21
app/Model/AppModel.php
View File

@ -4022,14 +4022,20 @@ class AppModel extends Model
*/
public function _remoteIp()
{
$ipHeader = Configure::read('MISP.log_client_ip_header') ?: null;
if ($ipHeader && isset($_SERVER[$ipHeader])) {
return trim($_SERVER[$ipHeader]);
$clientIpHeader = Configure::read('MISP.log_client_ip_header');
if ($clientIpHeader && isset($_SERVER[$clientIpHeader])) {
$headerValue = $_SERVER[$clientIpHeader];
// X-Forwarded-For can contain multiple IPs, see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For
if (($commaPos = strpos($headerValue, ',')) !== false) {
$headerValue = substr($headerValue, 0, $commaPos);
}
return trim($headerValue);
}
return $_SERVER['REMOTE_ADDR'] ?? null;
}
public function find($type = 'first', $query = array()) {
public function find($type = 'first', $query = array())
{
if (!empty($query['order']) && $this->validOrderClause($query['order']) === false) {
throw new InvalidArgumentException('Invalid order clause');
}
@ -4037,9 +4043,10 @@ class AppModel extends Model
return parent::find($type, $query);
}
private function validOrderClause($order){
private function validOrderClause($order)
{
$pattern = '/^[\w\_\-\.\(\) ]+$/';
if(is_string($order) && preg_match($pattern, $order)){
if (is_string($order) && preg_match($pattern, $order)) {
return true;
}
@ -4048,7 +4055,7 @@ class AppModel extends Model
if (is_string($key) && is_string($value) && preg_match($pattern, $key) && in_array(strtolower($value), ['asc', 'desc'])) {
return true;
}
if(is_numeric($key) && is_string($value) && preg_match($pattern, $value)){
if (is_numeric($key) && is_string($value) && preg_match($pattern, $value)) {
return true;
}
}

6
app/Model/AuditLog.php
View File

@ -25,11 +25,7 @@ class AuditLog extends AppModel
ACTION_REMOVE_GALAXY = 'remove_galaxy',
ACTION_REMOVE_GALAXY_LOCAL = 'remove_local_galaxy',
ACTION_PUBLISH = 'publish',
ACTION_PUBLISH_SIGHTINGS = 'publish_sightings',
ACTION_LOGIN = 'login',
ACTION_PASSWDCHANGE = 'password_change',
ACTION_LOGOUT = 'logout',
ACTION_LOGIN_FAILED = 'login_failed';
ACTION_PUBLISH_SIGHTINGS = 'publish_sightings';
const REQUEST_TYPE_DEFAULT = 0,
REQUEST_TYPE_API = 1,

41
app/Model/AuthKey.php
View File

@ -162,6 +162,7 @@ class AuthKey extends AppModel
* @param string $authkey
* @param bool $includeExpired
* @return array|false
* @throws Exception
*/
public function getAuthUserByAuthKey($authkey, $includeExpired = false)
{
@ -187,24 +188,8 @@ class AuthKey extends AppModel
]);
$passwordHasher = $this->getHasher();
foreach ($possibleAuthkeys as $possibleAuthkey) {
if ($passwordHasher->check($authkey, $possibleAuthkey['AuthKey']['authkey'])) { // valid authkey
// store IP in db if not there yet
if (!Configure::read("MISP.disable_seen_ips_authkeys")) {
$remote_ip = $this->_remoteIp();
$update_db_ip = true;
if (in_array($remote_ip, $possibleAuthkey['AuthKey']['unique_ips'])) {
$update_db_ip = false; // IP already seen, skip saving in DB
} else { // first time this IP is seen for this API key
$possibleAuthkey['AuthKey']['unique_ips'][] = $remote_ip;
}
if ($update_db_ip) {
// prevent double entries due to race condition
$possibleAuthkey['AuthKey']['unique_ips'] = array_unique($possibleAuthkey['AuthKey']['unique_ips']);
// save in db
$this->save($possibleAuthkey, ['fieldList' => ['unique_ips']]);
}
}
// fetch user
if ($passwordHasher->check($authkey, $possibleAuthkey['AuthKey']['authkey'])) {
$this->updateUniqueIp($possibleAuthkey);
$user = $this->User->getAuthUser($possibleAuthkey['AuthKey']['user_id']);
if ($user) {
$user = $this->setUserData($user, $possibleAuthkey);
@ -215,6 +200,26 @@ class AuthKey extends AppModel
return false;
}
/**
* @param array $authkey
* @return void
* @throws Exception
*/
private function updateUniqueIp(array $authkey)
{
if (Configure::read("MISP.disable_seen_ips_authkeys")) {
return;
}
$remoteIp = $this->_remoteIp();
if ($remoteIp === null || in_array($remoteIp, $authkey['AuthKey']['unique_ips'], true)) {
return;
}
$authkey['AuthKey']['unique_ips'][] = $remoteIp;
$this->save($authkey, ['fieldList' => ['unique_ips']]);
}
/**
* @param array $user
* @param array $authkey

241
app/Model/Datasource/Database/MysqlObserverExtended.php Normal file
View File

@ -0,0 +1,241 @@
<?php
App::uses('Mysql', 'Model/Datasource/Database');
/**
* Overrides the default MySQL database implementation to support the following features:
* - Set query hints to optimize queries
*/
class MysqlObserverExtended extends Mysql
{
/**
* Output MD5 as binary, that is faster and uses less memory
* @param string $value
* @return string
*/
public function cacheMethodHasher($value)
{
return md5($value, true);
}
/**
* Builds and generates an SQL statement from an array. Handles final clean-up before conversion.
*
* @param array $query An array defining an SQL query.
* @param Model $Model The model object which initiated the query.
* @return string An executable SQL statement.
* @see DboSource::renderStatement()
*/
public function buildStatement($query, Model $Model)
{
$query = array_merge($this->_queryDefaults, $query);
if (!empty($query['joins'])) {
$count = count($query['joins']);
for ($i = 0; $i < $count; $i++) {
if (is_array($query['joins'][$i])) {
$query['joins'][$i] = $this->buildJoinStatement($query['joins'][$i]);
}
}
}
return $this->renderStatement('select', array(
'conditions' => $this->conditions($query['conditions'], true, true, $Model),
'fields' => implode(', ', $query['fields']),
'table' => $query['table'],
'alias' => $this->alias . $this->name($query['alias']),
'order' => $this->order($query['order'], 'ASC', $Model),
'limit' => $this->limit($query['limit'], $query['offset']),
'joins' => implode(' ', $query['joins']),
'group' => $this->group($query['group'], $Model),
'having' => $this->having($query['having'], true, $Model),
'lock' => $this->getLockingHint($query['lock']),
'indexHint' => $this->__buildIndexHint($query['forceIndexHint'] ?? null),
));
}
/**
* Builds an SQL statement.
*
* This is merely a convenient wrapper to DboSource::buildStatement().
*
* @param Model $Model The model to build an association query for.
* @param array $queryData An array of queryData information containing keys similar to Model::find().
* @return string String containing an SQL statement.
* @see DboSource::buildStatement()
* @see DboSource::buildAssociationQuery()
*/
public function buildAssociationQuery(Model $Model, $queryData)
{
$queryData = $this->_scrubQueryData($queryData);
return $this->buildStatement(
array(
'fields' => $this->prepareFields($Model, $queryData),
'table' => $this->fullTableName($Model),
'alias' => $Model->alias,
'limit' => $queryData['limit'],
'offset' => $queryData['offset'],
'joins' => $queryData['joins'],
'conditions' => $queryData['conditions'],
'order' => $queryData['order'],
'group' => $queryData['group'],
'having' => $queryData['having'],
'lock' => $queryData['lock'],
'forceIndexHint' => $queryData['forceIndexHint'] ?? null,
),
$Model
);
}
/**
* Renders a final SQL statement by putting together the component parts in the correct order
*
* Edit: Added support for query hints
*
* @param string $type type of query being run. e.g select, create, update, delete, schema, alter.
* @param array $data Array of data to insert into the query.
* @return string|null Rendered SQL expression to be run, otherwise null.\
* @see DboSource::renderStatement()
*/
public function renderStatement($type, $data)
{
if ($type === 'select') {
extract($data);
$having = !empty($having) ? " $having" : '';
$lock = !empty($lock) ? " $lock" : '';
return rtrim("SELECT {$fields} FROM {$table} {$alias} {$indexHint} {$joins} {$conditions} {$group}{$having} {$order} {$limit}{$lock}");
}
return parent::renderStatement($type, $data);
}
/**
* Builds the index hint for the query
*
* @param string|null $forceIndexHint FORCE INDEX hint
* @return string
*/
private function __buildIndexHint($forceIndexHint = null): ?string
{
return isset($forceIndexHint) ? ('FORCE INDEX ' . $forceIndexHint) : null;
}
/**
* - Do not call microtime when not necessary
* - Count query count even when logging is disabled
*
* @param string $sql
* @param array $options
* @param array $params
* @return mixed
*/
public function execute($sql, $options = [], $params = [])
{
$log = $options['log'] ?? $this->fullDebug;
$comment = sprintf(
'%s%s%s',
empty(Configure::read('CurrentUserId')) ? '' : sprintf(
'[User: %s] ',
intval(Configure::read('CurrentUserId'))
),
empty(Configure::read('CurrentController')) ? '' : preg_replace('/[^a-zA-Z0-9_]/', '', Configure::read('CurrentController')) . ' :: ',
empty(Configure::read('CurrentAction')) ? '' : preg_replace('/[^a-zA-Z0-9_]/', '', Configure::read('CurrentAction'))
);
$sql = '/* ' . $comment . ' */ ' . $sql;
if ($log) {
$t = microtime(true);
$this->_result = $this->_execute($sql, $params);
$this->took = round((microtime(true) - $t) * 1000);
$this->numRows = $this->affected = $this->lastAffected();
$this->logQuery($sql, $params);
} else {
$this->_result = $this->_execute($sql, $params);
$this->_queriesCnt++;
}
return $this->_result;
}
/**
* Reduce memory usage for insertMulti
*
* @param string $table
* @param array $fields
* @param array $values
* @return bool
*/
public function insertMulti($table, $fields, $values)
{
$table = $this->fullTableName($table);
$holder = implode(',', array_fill(0, count($fields), '?'));
$fields = implode(',', array_map([$this, 'name'], $fields));
$pdoMap = [
'integer' => PDO::PARAM_INT,
'float' => PDO::PARAM_STR,
'boolean' => PDO::PARAM_BOOL,
'string' => PDO::PARAM_STR,
'text' => PDO::PARAM_STR
];
$columnMap = [];
foreach ($values[key($values)] as $key => $val) {
if (is_int($val)) {
$columnMap[$key] = PDO::PARAM_INT;
} elseif (is_bool($val)) {
$columnMap[$key] = PDO::PARAM_BOOL;
} else {
$type = $this->introspectType($val);
$columnMap[$key] = $pdoMap[$type];
}
}
$sql = "INSERT INTO $table ($fields) VALUES ";
$sql .= implode(',', array_fill(0, count($values), "($holder)"));
$statement = $this->_connection->prepare($sql);
$valuesList = array();
$i = 1;
foreach ($values as $value) {
foreach ($value as $col => $val) {
if ($this->fullDebug) {
$valuesList[] = $val;
}
$statement->bindValue($i++, $val, $columnMap[$col]);
}
}
$result = $statement->execute();
$statement->closeCursor();
if ($this->fullDebug) {
$this->logQuery($sql, $valuesList);
}
return $result;
}
/**
* {@inheritDoc}
*/
public function value($data, $column = null, $null = true)
{
// Fast check if data is int, then return value
if (is_int($data)) {
return $data;
}
// No need to quote bool values
if (is_bool($data)) {
return $data ? '1' : '0';
}
// No need to call expensive array_map
if (is_array($data) && !empty($data)) {
$output = [];
foreach ($data as $d) {
if (is_int($d)) {
$output[] = $d;
} else {
$output[] = parent::value($d, $column);
}
}
return $output;
}
return parent::value($data, $column, $null);
}
}

153
app/Model/Event.php
View File

@ -63,11 +63,6 @@ class Event extends AppModel
2 => array('desc' => '*Complete* means that the event\'s creation is complete', 'formdesc' => 'The event creator considers the analysis complete')
);
public $debugDescriptions = array(
0 => 'The critical errors are logged in the usual log file.',
1 => 'All the errors and warnings are logged in the usual log file.'
);
public $distributionDescriptions = [
self::DISTRIBUTION_ORGANISATION => [
'desc' => 'This field determines the current distribution of the event',
@ -91,16 +86,6 @@ class Event extends AppModel
],
];
public $galaxiesOptionsDescriptions = array(
0 => 'Galaxies and Clusters are passed as MISP standard format. New generic Galaxies and Clusters are created when there is no match with existing ones.',
1 => 'Galaxies are passed as tags and there is only a simple search with existing galaxy tag names.'
);
public $debugOptions = array(
0 => 'Standard debugging',
1 => 'Advanced debugging'
);
public $distributionLevels = [
self::DISTRIBUTION_ORGANISATION => 'Your organisation only',
self::DISTRIBUTION_COMMUNITY => 'This community only',
@ -109,11 +94,6 @@ class Event extends AppModel
self::DISTRIBUTION_SHARING_GROUP => 'Sharing group',
];
public $galaxiesOptions = array(
0 => 'As MISP standard format',
1 => 'As tag names'
);
public $analysisLevels = array(
0 => 'Initial', 1 => 'Ongoing', 2 => 'Completed'
);
@ -5922,61 +5902,24 @@ class Event extends AppModel
/**
* @param array $user
* @param string $file Path
* @param string $stix_version
* @param string $original_file
* @param string $stixVersion
* @param string $originalFile
* @param bool $publish
* @param int $distribution
* @param int|null $sharingGroupId
* @param bool $galaxiesAsTags
* @param bool $debug
* @return int|string|array
* @throws JsonException
* @throws InvalidArgumentException
* @throws Exception
*/
public function upload_stix(array $user, $file, $stix_version, $original_file, $publish, $distribution, $sharingGroupId, $galaxiesAsTags, $debug = false)
public function upload_stix(array $user, $file, $stixVersion, $originalFile, $publish, $distribution, $sharingGroupId, $galaxiesAsTags, $debug = false)
{
$scriptDir = APP . 'files' . DS . 'scripts';
if ($stix_version == '2' || $stix_version == '2.0' || $stix_version == '2.1') {
$scriptFile = $scriptDir . DS . 'stix2' . DS . 'stix2misp.py';
$output_path = $file . '.out';
$shell_command = [
ProcessTool::pythonBin(),
$scriptFile,
'-i', $file,
'--distribution', $distribution
];
if ($distribution == 4) {
array_push($shell_command, '--sharing_group_id', $sharingGroupId);
}
if ($galaxiesAsTags) {
$shell_command[] = '--galaxies_as_tags';
}
if ($debug) {
$shell_command[] = '--debug';
}
$stix_version = "STIX 2.1";
} elseif ($stix_version == '1' || $stix_version == '1.1' || $stix_version == '1.2') {
$scriptFile = $scriptDir . DS . 'stix2misp.py';
$output_path = $file . '.json';
$shell_command = [
ProcessTool::pythonBin(),
$scriptFile,
$file,
Configure::read('MISP.default_event_distribution'),
Configure::read('MISP.default_attribute_distribution'),
$this->__getTagNamesFromSynonyms($scriptDir)
];
$stix_version = "STIX 1.1";
} else {
throw new InvalidArgumentException('Invalid STIX version');
}
$decoded = $this->convertStixToMisp($stixVersion, $file, $distribution, $sharingGroupId, $galaxiesAsTags, $debug);
$result = ProcessTool::execute($shell_command, null, true);
$result = preg_split("/\r\n|\n|\r/", trim($result));
$result = trim(end($result));
$tempFile = file_get_contents($file);
unlink($file);
$decoded = JsonTool::decode($result);
if (!empty($decoded['success'])) {
$data = FileAccessTool::readAndDelete($output_path);
$data = $this->jsonDecode($data);
$data = JsonTool::decodeArray($decoded['converted']);
if (empty($data['Event'])) {
$data = array('Event' => $data);
}
@ -6000,15 +5943,13 @@ class Event extends AppModel
}
}
}
if (!empty($decoded['stix_version'])) {
$stix_version = 'STIX ' . $decoded['stix_version'];
}
$stixVersion = $decoded['stix_version'];
$created_id = false;
$validationIssues = false;
$result = $this->_add($data, true, $user, '', null, false, null, $created_id, $validationIssues);
if ($result === true) {
if ($original_file) {
$this->add_original_file($tempFile, $original_file, $created_id, $stix_version);
if ($originalFile) {
$this->add_original_file($decoded['original'], $originalFile, $created_id, $stixVersion);
}
if ($publish && $user['Role']['perm_publish']) {
$this->publish($created_id);
@ -6031,6 +5972,76 @@ class Event extends AppModel
return $response;
}
/**
* @param string $stixVersion
* @param string $file
* @param int $distribution
* @param int|null $sharingGroupId
* @param bool $galaxiesAsTags
* @param bool $debug
* @return array
* @throws Exception
*/
private function convertStixToMisp($stixVersion, $file, $distribution, $sharingGroupId, $galaxiesAsTags, $debug)
{
$scriptDir = APP . 'files' . DS . 'scripts';
if ($stixVersion === '2' || $stixVersion === '2.0' || $stixVersion === '2.1') {
$scriptFile = $scriptDir . DS . 'stix2' . DS . 'stix2misp.py';
$outputPath = $file . '.out';
$shellCommand = [
ProcessTool::pythonBin(),
$scriptFile,
'-i', $file,
'--distribution', $distribution,
];
if ($distribution == 4) {
array_push($shellCommand, '--sharing_group_id', $sharingGroupId);
}
if ($galaxiesAsTags) {
$shellCommand[] = '--galaxies_as_tags';
}
if ($debug) {
$shellCommand[] = '--debug';
}
$stixVersion = "STIX 2.1";
} else if ($stixVersion === '1' || $stixVersion === '1.1' || $stixVersion === '1.2') {
$scriptFile = $scriptDir . DS . 'stix2misp.py';
$outputPath = $file . '.json';
$shellCommand = [
ProcessTool::pythonBin(),
$scriptFile,
$file,
Configure::read('MISP.default_event_distribution'),
Configure::read('MISP.default_attribute_distribution'),
$this->__getTagNamesFromSynonyms($scriptDir)
];
$stixVersion = "STIX 1.1";
} else {
throw new InvalidArgumentException('Invalid STIX version');
}
try {
$stdout = ProcessTool::execute($shellCommand, null, true);
} catch (ProcessException $e) {
$stdout = $e->stdout();
}
$stdout = preg_split("/\r\n|\n|\r/", trim($stdout));
$stdout = trim(end($stdout));
$decoded = JsonTool::decode($stdout);
if (empty($decoded['stix_version'])) {
$decoded['stix_version'] = $stixVersion;
}
$decoded['original'] = FileAccessTool::readAndDelete($file);
if (!empty($decoded['success'])) {
$decoded['converted'] = FileAccessTool::readAndDelete($outputPath);
}
return $decoded;
}
private function __handleGalaxiesAndClusters($user, &$data)
{
if (!empty($data['Galaxy'])) {

70
app/Model/Log.php
View File

@ -159,11 +159,6 @@ class Log extends AppModel
if (!in_array($this->data['Log']['model'], ['Log', 'Workflow'])) {
$trigger_id = 'log-after-save';
$workflowErrors = [];
$logging = [
'model' => 'Log',
'action' => 'execute_workflow',
'id' => $this->data['Log']['user_id']
];
$this->executeTrigger($trigger_id, $this->data, $workflowErrors);
}
return true;
@ -206,7 +201,7 @@ class Log extends AppModel
$validDates = $this->query($sql);
}
$data = array();
foreach ($validDates as $k => $date) {
foreach ($validDates as $date) {
$data[$date[0]['Date']] = intval($date[0]['count']);
}
return $data;
@ -286,7 +281,7 @@ class Log extends AppModel
public function validationError($user, $action, $model, $title, array $validationErrors, array $fullObject)
{
$this->log($title, LOG_WARNING);
$change = 'Validation errors: ' . json_encode($validationErrors) . ' Full ' . $model . ': ' . json_encode($fullObject);
$change = 'Validation errors: ' . JsonTool::encode($validationErrors) . ' Full ' . $model . ': ' . JsonTool::encode($fullObject);
$this->createLogEntry($user, $action, $model, 0, $title, $change);
}
@ -365,7 +360,7 @@ class Log extends AppModel
}
}
public function logData($data)
private function logData(array $data)
{
if ($this->pubToZmq('audit')) {
$this->getPubSubTool()->publish($data, 'audit', 'log');
@ -386,6 +381,14 @@ class Log extends AppModel
}
// write to syslogd as well if enabled
$this->sendToSyslog($data);
$this->sendToEcs($data);
return true;
}
private function sendToSyslog(array $data)
{
if ($this->syslog === null) {
if (Configure::read('Security.syslog')) {
$options = [];
@ -407,8 +410,7 @@ class Log extends AppModel
if (isset($data['Log']['action'])) {
if (in_array($data['Log']['action'], self::ERROR_ACTIONS, true)) {
$action = LOG_ERR;
}
if (in_array($data['Log']['action'], self::WARNING_ACTIONS, true)) {
} else if (in_array($data['Log']['action'], self::WARNING_ACTIONS, true)) {
$action = LOG_WARNING;
}
}
@ -420,11 +422,49 @@ class Log extends AppModel
if (!empty($data['Log']['description'])) {
$entry .= " -- {$data['Log']['description']}";
} else if (!empty($data['Log']['change'])) {
$entry .= " -- " . json_encode($data['Log']['change']);
$entry .= " -- " . JsonTool::encode($data['Log']['change']);
}
$this->syslog->write($action, $entry);
}
return true;
}
/**
* @param array $data
* @return void
* @throws JsonException
*/
private function sendToEcs(array $data)
{
if (!Configure::read('Security.ecs_log')) {
return;
}
$log = $data['Log'];
$action = $log['action'];
if ($action === 'email') {
return; // do not log email actions as it is logged with more details by `writeEmailLog` function
}
if (in_array($action, self::ERROR_ACTIONS, true)) {
$type = 'error';
} else if (in_array($action, self::WARNING_ACTIONS, true)) {
$type = 'warning';
} else {
$type = 'info';
}
$message = $action;
if (!empty($log['title'])) {
$message .= " -- {$log['title']}";
}
if (!empty($log['description'])) {
$message .= " -- {$log['description']}";
} else if (!empty($log['change'])) {
$message .= " -- " . (is_string($log['change']) ? $log['change'] : JsonTool::encode($log['change']));
}
EcsLog::writeApplicationLog($type, $action, $message);
}
public function filterSiteAdminSensitiveLogs($list)
@ -1176,11 +1216,17 @@ class Log extends AppModel
return $this->elasticSearchClient;
}
/**
* @param $data
* @param $options
* @return array|bool|mixed
*/
public function saveOrFailSilently($data, $options = null)
{
try {
return $this->save($data, $options);
} catch (Exception $e) {
$this->logException('Could not save log to database', $e);
return false;
}
}

8
app/Model/Server.php
View File

@ -5062,6 +5062,14 @@ class Server extends AppModel
'type' => 'boolean',
'null' => true
],
'enable_automatic_garbage_collection' => [
'level' => 1,
'description' => __('Enable to execute an automatic garbage collection of temporary data such as export files. When enabled, on agerage every 100th query will check whether to garbage collect. Garbage collection can run at maximum once an hour.'),
'value' => false,
'test' => 'testBool',
'type' => 'boolean',
'null' => true,
],
'server_settings_skip_backup_rotate' => array(
'level' => 1,
'description' => __('Enable this setting to directly save the config.php file without first creating a temporary file and moving it to avoid concurency issues. Generally not recommended, but useful when for example other tools modify/maintain the config.php file.'),

36
app/Model/User.php
View File

@ -12,6 +12,7 @@ App::uses('BlowfishConstantPasswordHasher', 'Controller/Component/Auth');
* @property Organisation $Organisation
* @property Role $Role
* @property UserSetting $UserSetting
* @property UserLoginProfile $UserLoginProfile
* @property Event $Event
* @property AuthKey $AuthKey
* @property Server $Server
@ -889,7 +890,11 @@ class User extends AppModel
$logTitle = $result['encrypted'] ? 'Encrypted email' : 'Email';
// Intentional two spaces to pass test :)
$logTitle .= $replyToLog . ' to ' . $user['User']['email'] . ' sent, titled "' . $result['subject'] . '".';
$logTitle .= $replyToLog . ' to ' . $result['to'] . ' sent, titled "' . $result['subject'] . '".';
if (Configure::read('Security.ecs_log')) {
EcsLog::writeEmailLog($logTitle, $result, $replyToUser ? $replyToUser['User']['email'] : null);
}
$log->create();
$log->saveOrFailSilently(array(
@ -1264,37 +1269,44 @@ class User extends AppModel
return $newkey;
}
public function extralog($user, $action = null, $description = null, $fieldsResult = null, $modifiedUser = null)
/**
* @param string|array $user
* @param string $action
* @param string $description
* @param string $fieldsResult
* @param array|null $modifiedUser
* @return void
* @throws JsonException
*/
public function extralog($user, $action, $description = null, $fieldsResult = null, $modifiedUser = null)
{
if (!is_array($user) && $user === 'SYSTEM') {
if ($user === 'SYSTEM') {
$user = [
'id' => 0,
'email' => 'SYSTEM',
'Organisation' => [
'name' => 'SYSTEM'
]
],
];
}
// new data
$model = 'User';
$modelId = $user['id'];
if (!empty($modifiedUser)) {
$modelId = $modifiedUser['User']['id'];
}
if ($action == 'login') {
if ($action === 'login') {
$description = "User (" . $user['id'] . "): " . $user['email'];
$fieldsResult = json_encode($this->UserLoginProfile->_getUserProfile());
} elseif ($action == 'logout') {
$fieldsResult = JsonTool::encode($this->UserLoginProfile->_getUserProfile());
} else if ($action === 'logout') {
$description = "User (" . $user['id'] . "): " . $user['email'];
} elseif ($action == 'edit') {
} else if ($action === 'edit') {
$description = "User (" . $modifiedUser['User']['id'] . "): " . $modifiedUser['User']['email'];
} elseif ($action == 'change_pw') {
} else if ($action === 'change_pw') {
$description = "User (" . $modifiedUser['User']['id'] . "): " . $modifiedUser['User']['email'];
$fieldsResult = "Password changed.";
}
// query
$result = $this->loadLog()->createLogEntry($user, $action, $model, $modelId, $description, $fieldsResult);
$result = $this->loadLog()->createLogEntry($user, $action, 'User', $modelId, $description, $fieldsResult);
// write to syslogd as well
if ($result) {
App::import('Lib', 'SysLog.SysLog');

179
app/Model/UserLoginProfile.php
View File

@ -1,7 +1,9 @@
<?php
App::uses('AppModel', 'Model');
/**
* @property User $User
*/
class UserLoginProfile extends AppModel
{
public $actsAs = array(
@ -20,34 +22,37 @@ class UserLoginProfile extends AppModel
'rule' => '/^(trusted|malicious)$/',
'message' => 'Must be one of: trusted, malicious'
],
];
public $order = array("UserLoginProfile.id" => "DESC");
public $belongsTo = [
'User' => [
'className' => 'User',
'foreignKey' => 'user_id',
'conditions' => '',
'fields' => '',
'order' => ''
]];
'User' => [
'className' => 'User',
'foreignKey' => 'user_id',
'conditions' => '',
'fields' => '',
'order' => ''
]
];
protected $browscapCacheDir = APP . DS . 'tmp' . DS . 'browscap';
protected $browscapIniFile = APP . DS . 'files' . DS . 'browscap'. DS . 'browscap.ini'; // Browscap file managed by MISP - https://browscap.org/stream?q=Lite_PHP_BrowsCapINI
protected $geoIpDbFile = APP . DS . 'files' . DS . 'geo-open' . DS . 'GeoOpen-Country.mmdb'; // GeoIP file managed by MISP - https://data.public.lu/en/datasets/geo-open-ip-address-geolocation-per-country-in-mmdb-format/
const BROWSER_CACHE_DIR = APP . DS . 'tmp' . DS . 'browscap';
const BROWSER_INI_FILE = APP . DS . 'files' . DS . 'browscap'. DS . 'browscap.ini'; // Browscap file managed by MISP - https://browscap.org/stream?q=Lite_PHP_BrowsCapINI
const GEOIP_DB_FILE = APP . DS . 'files' . DS . 'geo-open' . DS . 'GeoOpen-Country.mmdb'; // GeoIP file managed by MISP - https://data.public.lu/en/datasets/geo-open-ip-address-geolocation-per-country-in-mmdb-format/
private $userProfile;
private $knownUserProfiles = [];
public function _buildBrowscapCache() {
$this->log("Browscap - building new cache from browscap.ini file.", "info");
$fileCache = new \Doctrine\Common\Cache\FilesystemCache($this->browscapCacheDir);
private function _buildBrowscapCache()
{
$this->log("Browscap - building new cache from browscap.ini file.", LOG_INFO);
$fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR);
$cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache);
$logger = new \Monolog\Logger('name');
$bc = new \BrowscapPHP\BrowscapUpdater($cache, $logger);
$bc->convertFile($this->browscapIniFile);
$bc->convertFile(UserLoginProfile::BROWSER_INI_FILE);
}
public function beforeSave($options = [])
@ -56,7 +61,8 @@ class UserLoginProfile extends AppModel
return true;
}
public function hash($data) {
public function hash($data)
{
unset($data['hash']);
unset($data['created_at']);
return md5(serialize($data));
@ -66,12 +72,13 @@ class UserLoginProfile extends AppModel
* slow function - don't call it too often
* @return array
*/
public function _getUserProfile() {
public function _getUserProfile()
{
if (!$this->userProfile) {
// below uses https://github.com/browscap/browscap-php
if (class_exists('\BrowscapPHP\Browscap')) {
try {
$fileCache = new \Doctrine\Common\Cache\FilesystemCache($this->browscapCacheDir);
$fileCache = new \Doctrine\Common\Cache\FilesystemCache(UserLoginProfile::BROWSER_CACHE_DIR);
$cache = new \Roave\DoctrineSimpleCache\SimpleCacheAdapter($fileCache);
$logger = new \Monolog\Logger('name');
$bc = new \BrowscapPHP\Browscap($cache, $logger);
@ -82,7 +89,7 @@ class UserLoginProfile extends AppModel
}
} else {
// a primitive OS & browser extraction capability
$ua = env('HTTP_USER_AGENT');
$ua = $_SERVER['HTTP_USER_AGENT'] ?? null;
$browser = new stdClass();
$browser->browser_name_pattern = $ua;
if (mb_strpos($ua, 'Linux') !== false) $browser->platform = "Linux";
@ -95,16 +102,21 @@ class UserLoginProfile extends AppModel
}
$ip = $this->_remoteIp();
if (class_exists('GeoIp2\Database\Reader')) {
$geoDbReader = new GeoIp2\Database\Reader($this->geoIpDbFile);
$record = $geoDbReader->country($ip);
$country = $record->country->isoCode;
try {
$geoDbReader = new GeoIp2\Database\Reader(UserLoginProfile::GEOIP_DB_FILE);
$record = $geoDbReader->country($ip);
$country = $record->country->isoCode;
} catch (InvalidArgumentException $e) {
$this->logException("Could not get country code for IP address", $e);
$country = 'None';
}
} else {
$country = 'None';
}
$this->userProfile = [
'user_agent' => env('HTTP_USER_AGENT'),
'user_agent' => $_SERVER['HTTP_USER_AGENT'] ?? null,
'ip' => $ip,
'accept_lang' => env('HTTP_ACCEPT_LANGUAGE'),
'accept_lang' => $_SERVER['HTTP_ACCEPT_LANGUAGE'] ?? null,
'geoip' => $country,
'ua_pattern' => $browser->browser_name_pattern,
'ua_platform' => $browser->platform,
@ -114,62 +126,68 @@ class UserLoginProfile extends AppModel
return $this->userProfile;
}
public function _fromLog($logEntry) {
$data = json_decode('{"user_agent": "", "ip": "", "accept_lang":"", "geoip":"", "ua_pattern":"", "ua_platform":"", "ua_browser":""}', true);
$data = array_merge($data, json_decode($logEntry['change'], true) ?? []);
public function _fromLog($logEntry)
{
$data = ["user_agent" => "", "ip" => "", "accept_lang" => "", "geoip" => "", "ua_pattern" => "", "ua_platform" => "", "ua_browser" => ""];
$data = array_merge($data, JsonTool::decode($logEntry['change']) ?? []);
$data['ip'] = $logEntry['ip'];
$data['timestamp'] = $logEntry['created'];
if ($data['user_agent'] == "") return false;
if ($data['user_agent'] === "") {
return false;
}
return $data;
}
public function _isSimilar($a, $b) {
public function _isSimilar($a, $b)
{
// if one is not initialized
if (!$a || !$b) return false;
// transition for old logs where UA was not known
if (!$a['ua_browser'])
return false;
// really similar session, from same browser, region, but different IP
if ($a['ua_browser'] == $b['ua_browser'] &&
$a['ua_platform'] == $b['ua_platform'] &&
$a['accept_lang'] == $b['accept_lang'] &&
$a['geoip'] == $b['geoip']) {
if ($a['ua_browser'] === $b['ua_browser'] &&
$a['ua_platform'] === $b['ua_platform'] &&
$a['accept_lang'] === $b['accept_lang'] &&
$a['geoip'] === $b['geoip']) {
return true;
}
// similar browser pattern, OS and region
if ($a['ua_pattern'] == $b['ua_pattern'] &&
$a['ua_platform'] == $b['ua_platform'] &&
$a['accept_lang'] == $b['accept_lang'] &&
$a['geoip'] == $b['geoip']) {
if ($a['ua_pattern'] === $b['ua_pattern'] &&
$a['ua_platform'] === $b['ua_platform'] &&
$a['accept_lang'] === $b['accept_lang'] &&
$a['geoip'] === $b['geoip']) {
return true;
}
return false;
}
public function _isIdentical($a, $b) {
if ($a['ip'] == $b['ip'] &&
$a['ua_browser'] == $b['ua_browser'] &&
$a['ua_platform'] == $b['ua_platform'] &&
$a['accept_lang'] == $b['accept_lang'] &&
$a['geoip'] == $b['geoip']) {
public function _isIdentical(array $a, array $b)
{
if ($a['ip'] === $b['ip'] &&
$a['ua_browser'] === $b['ua_browser'] &&
$a['ua_platform'] === $b['ua_platform'] &&
$a['accept_lang'] === $b['accept_lang'] &&
$a['geoip'] === $b['geoip']) {
return true;
}
return false;
}
public function _getTrustStatus($userProfileToCheck, $user_id = null) {
if (!$user_id) {
$user_id = AuthComponent::user('id');
public function _getTrustStatus(array $userProfileToCheck, $userId = null)
{
if (!$userId) {
$userId = AuthComponent::user('id');
}
// load Singleton / caching
if (!isset($this->knownUserProfiles[$user_id])) {
$this->knownUserProfiles[$user_id] = $this->find('all', [
'conditions' => ['UserLoginProfile.user_id' => $user_id],
'recursive' => 0]
);
if (!isset($this->knownUserProfiles[$userId])) {
$this->knownUserProfiles[$userId] = $this->find('all', [
'conditions' => ['UserLoginProfile.user_id' => $userId],
'recursive' => 0
]);
}
// perform check on all entries, and stop when check OK
foreach ($this->knownUserProfiles[$user_id] as $knownUserProfile) {
foreach ($this->knownUserProfiles[$userId] as $knownUserProfile) {
// when it is the same
if ($this->_isIdentical($knownUserProfile['UserLoginProfile'], $userProfileToCheck)) {
return $knownUserProfile['UserLoginProfile']['status'];
@ -183,17 +201,19 @@ class UserLoginProfile extends AppModel
return 'unknown';
}
public function _isTrusted() {
public function _isTrusted()
{
if (strpos($this->_getTrustStatus($this->_getUserProfile()), 'trusted') !== false) {
return true;
}
return false;
}
public function _isSuspicious() {
public function _isSuspicious()
{
// previously marked loginuserprofile as malicious by the user
if (strpos($this->_getTrustStatus($this->_getUserProfile()), 'malicious') !== false) {
return _('A user reported a similar login profile as malicious.');
return __('A user reported a similar login profile as malicious.');
}
// same IP as previous malicious user
$maliciousWithSameIP = $this->find('first', [
@ -205,7 +225,7 @@ class UserLoginProfile extends AppModel
'fields' => array('UserLoginProfile.*')]
);
if ($maliciousWithSameIP) {
return _('The source IP was reported as as malicious by a user.');
return __('The source IP was reported as as malicious by a user.');
}
// LATER - use other data to identify suspicious logins, such as:
// - what with use-case where a user marks something as legitimate, but is marked by someone else as suspicious?
@ -214,7 +234,8 @@ class UserLoginProfile extends AppModel
return false;
}
public function email_newlogin($user) {
public function email_newlogin($user)
{
if (!Configure::read('MISP.disable_emailing')) {
$date_time = date('c');
@ -224,16 +245,12 @@ class UserLoginProfile extends AppModel
$body->set('misp_org', Configure::read('MISP.org'));
$body->set('date_time', $date_time);
// Fetch user that contains also PGP or S/MIME keys for e-mail encryption
$result = $this->User->sendEmail($user, $body, false, "[" . Configure::read('MISP.org') . " MISP] New sign in.");
if ($result) {
// all is well, email sent to user
} else {
// email flow system already logs errors
}
$this->User->sendEmail($user, $body, false, "[" . Configure::read('MISP.org') . " MISP] New sign in.");
}
}
public function email_report_malicious($user, $userLoginProfile) {
public function email_report_malicious($user, $userLoginProfile)
{
// inform the org admin
$date_time = $userLoginProfile['timestamp']; // LATER not ideal as timestamp is string without timezone info
$body = new SendEmailTemplate('userloginprofile_report_malicious');
@ -245,21 +262,17 @@ class UserLoginProfile extends AppModel
$org_admins = $this->User->getOrgAdminsForOrg($user['User']['org_id']);
$admins = $this->User->getSiteAdmins();
$all_admins = array_unique(array_merge($org_admins, $admins));
foreach($all_admins as $admin_email) {
foreach ($all_admins as $admin_email) {
$admin = $this->User->find('first', array(
'recursive' => -1,
'conditions' => ['User.email' => $admin_email]
));
$result = $this->User->sendEmail($admin, $body, false, "[" . Configure::read('MISP.org') . " MISP] Suspicious login reported.");
if ($result) {
// all is well, email sent to user
} else {
// email flow system already logs errors
}
$this->User->sendEmail($admin, $body, false, "[" . Configure::read('MISP.org') . " MISP] Suspicious login reported.");
}
}
public function email_suspicious($user, $suspiciousness_reason) {
public function email_suspicious($user, $suspiciousness_reason)
{
if (!Configure::read('MISP.disable_emailing')) {
$date_time = date('c');
// inform the user
@ -271,12 +284,8 @@ class UserLoginProfile extends AppModel
$body->set('date_time', $date_time);
$body->set('suspiciousness_reason', $suspiciousness_reason);
// inform the user
$result = $this->User->sendEmail($user, $body, false, "[" . Configure::read('MISP.org') . " MISP] Suspicious login with your account.");
if ($result) {
// all is well, email sent to user
} else {
// email flow system already logs errors
}
$this->User->sendEmail($user, $body, false, "[" . Configure::read('MISP.org') . " MISP] Suspicious login with your account.");
// inform the org admin
$body = new SendEmailTemplate('userloginprofile_suspicious_orgadmin');
$body->set('userLoginProfile', $this->_getUserProfile());
@ -285,21 +294,15 @@ class UserLoginProfile extends AppModel
$body->set('misp_org', Configure::read('MISP.org'));
$body->set('date_time', $date_time);
$body->set('suspiciousness_reason', $suspiciousness_reason);
$org_admins = $this->User->getOrgAdminsForOrg($user['User']['org_id']);
foreach($org_admins as $org_admin_email) {
foreach ($org_admins as $org_admin_email) {
$org_admin = $this->User->find('first', array(
'recursive' => -1,
'conditions' => ['User.email' => $org_admin_email]
));
$result = $this->User->sendEmail($org_admin, $body, false, "[" . Configure::read('MISP.org') . " MISP] Suspicious login detected.");
if ($result) {
// all is well, email sent to user
} else {
// email flow system already logs errors
}
$this->User->sendEmail($org_admin, $body, false, "[" . Configure::read('MISP.org') . " MISP] Suspicious login detected.");
}
}
}
}

17
app/Model/WorkflowModules/logic/Module_organisation_if.php
View File

@ -65,6 +65,7 @@ class Module_organisation_if extends WorkflowBaseLogicModule
$org_type = $params['org_type']['value'];
$operator = $params['condition']['value'];
$operator = $this->convertOperators($operator);
$selectedOrgs = !empty($params['org_id']['value']) ? $params['org_id']['value'] : [];
$selectedOrgs = is_array($selectedOrgs) ? $selectedOrgs : [$selectedOrgs]; // Backward compatibility for non-multiple `org_id`
$path = 'Event.org_id';
@ -75,4 +76,20 @@ class Module_organisation_if extends WorkflowBaseLogicModule
$eval = $this->evaluateCondition($selectedOrgs, $operator, $extracted_org);
return !empty($eval);
}
/**
* Make sure to convert version 0.1 of this module to the new operators
*
* @param string $operator
* @return string
*/
private function convertOperators($operator)
{
if ($operator == 'equals') {
$operator = 'in';
} else if ($operator == 'not_equals') {
$operator = 'not_in';
}
return $operator;
}
}

424
app/Plugin/EcsLog/Lib/Log/Engine/EcsLog.php Normal file
View File

@ -0,0 +1,424 @@
<?php
App::uses('JsonTool', 'Tools');
/**
* Logging class that sends logs in JSON format to UNIX socket in Elastic Common Schema (ECS) format
* Logs are separated by new line characters, so basically it is send as JSONL
*/
class EcsLog implements CakeLogInterface
{
const ECS_VERSION = '8.11';
/** @var string Unix socket path where logs will be send in JSONL format */
const SOCKET_PATH = '/run/vector';
/** @var false|resource */
private static $socket;
/** @var string[] */
private static $messageBuffer = [];
/** @var array[] */
private static $meta;
const LOG_LEVEL_STRING = [
LOG_EMERG => 'emergency',
LOG_ALERT => 'alert',
LOG_CRIT => 'critical',
LOG_ERR => 'error',
LOG_WARNING => 'warning',
LOG_NOTICE => 'notice',
LOG_INFO => 'info',
LOG_DEBUG => 'debug',
];
/**
* @param string $type The type of log you are making.
* @param string $message The message you want to log.
* @return void
*/
public function write($type, $message)
{
if (strpos($message, 'Could not convert ECS log message into JSON: ') !== false) {
return; // prevent recursion when saving logs
}
$message = [
'@timestamp' => self::now(),
'ecs' => [
'version' => self::ECS_VERSION,
],
'event' => [
'kind' => 'event',
'provider' => 'misp',
'module' => 'system',
'dataset' => 'system.logs',
],
'log' => [
'level' => $type,
],
'message' => $message,
];
static::writeMessage($message);
}
/**
* @param string $type
* @param string $action
* @param string $message
* @return void
*/
public static function writeApplicationLog($type, $action, $message)
{
$message = [
'@timestamp' => self::now(),
'ecs' => [
'version' => self::ECS_VERSION,
],
'event' => [
'kind' => 'event',
'provider' => 'misp',
'module' => 'application',
'dataset' => 'application.logs',
'action' => $action,
],
'log' => [
'level' => $type,
],
'message' => $message,
];
if (in_array($action, ['auth', 'auth_fail', 'auth_alert', 'change_pw', 'login', 'login_fail', 'logout', 'password_reset'], true)) {
$message['event']['category'] = 'authentication';
if (in_array($action, ['auth_fail', 'login_fail'], true)) {
$message['event']['outcome'] = 'failure';
}
}
static::writeMessage($message);
}
/**
* Include more meta information about email than would provide default `writeApplicationLog` log
* @param string $logTitle
* @param array $emailResult
* @param string|null $replyTo
* @return void
*/
public static function writeEmailLog($logTitle, array $emailResult, $replyTo = null)
{
$message = [
'@timestamp' => self::now(),
'ecs' => [
'version' => self::ECS_VERSION,
],
'event' => [
'kind' => 'event',
'provider' => 'misp',
'module' => 'application',
'dataset' => 'application.logs',
'category' => 'email',
'action' => 'email',
'type' => 'info',
],
'email' => [
'message_id' => $emailResult['message_id'],
'subject' => $emailResult['subject'],
'to' => [
'address' => $emailResult['to'],
],
],
'message' => $logTitle,
];
if ($replyTo) {
$message['email']['reply_to'] = ['address' => $replyTo];
}
static::writeMessage($message);
}
/**
* @param int $code
* @param string $description
* @param string|null $file
* @param int|null $line
* @return void
*/
public static function handleError($code, $description, $file = null, $line = null)
{
list($name, $log) = ErrorHandler::mapErrorCode($code);
$level = self::LOG_LEVEL_STRING[$log];
$message = [
'@timestamp' => self::now(),
'ecs' => [
'version' => self::ECS_VERSION,
],
'event' => [
'kind' => 'event',
'provider' => 'misp',
'module' => 'system',
'dataset' => 'system.logs',
'type' => 'error',
],
'error' => [
'code' => $code,
'message' => $description,
],
'log' => [
'level' => $level,
'origin' => [
'file' => [
'name' => $file,
'line' => $line,
],
],
],
];
static::writeMessage($message);
}
/**
* @param Exception $exception
* @return void
*/
public static function handleException(Exception $exception)
{
$code = $exception->getCode();
$code = ($code && is_int($code)) ? $code : 1;
$message = [
'@timestamp' => self::now(),
'ecs' => [
'version' => self::ECS_VERSION,
],
'event' => [
'kind' => 'event',
'provider' => 'misp',
'module' => 'system',
'dataset' => 'system.logs',
'type' => 'error',
],
'error' => [
'code' => $code,
'type' => get_class($exception),
'message' => $exception->getMessage(),
'stack_trace' => $exception->getTraceAsString(),
],
'log' => [
'level' => 'error',
'origin' => [
'file' => [
'name' => $exception->getFile(),
'line' => $exception->getLine(),
],
],
],
];
static::writeMessage($message);
}
/**
* @return array|null
*/
private static function clientIpFromHeaders()
{
$ipHeader = Configure::read('MISP.log_client_ip_header') ?: null;
if ($ipHeader && isset($_SERVER[$ipHeader])) {
return array_map('trim', explode(',', $_SERVER[$ipHeader]));
}
return null;
}
/**
* @return array[]
*/
private static function createLogMeta()
{
if (self::$meta) {
return self::$meta;
}
$meta = ['process' => ['pid' => getmypid()]];
// Add metadata if log was generated because of HTTP request
if (PHP_SAPI !== 'cli') {
if (isset($_SERVER['HTTP_X_REQUEST_ID'])) {
$meta['http'] = ['request' => ['id' => $_SERVER['HTTP_X_REQUEST_ID']]];
}
$meta['client'] = self::createClientMeta();
$meta['url'] = self::createUrlMeta();
} else {
$meta['process']['argv'] = $_SERVER['argv'];
}
$userMeta = self::createUserMeta();
if ($userMeta) {
$meta['user'] = $userMeta;
}
return self::$meta = $meta;
}
/**
* @return array
*/
private static function createClientMeta()
{
$client = [
'ip' => $_SERVER['REMOTE_ADDR'],
'port' => (int) $_SERVER['REMOTE_PORT'],
];
$clientIps = static::clientIpFromHeaders();
if ($clientIps) {
$clientIps[] = $_SERVER['REMOTE_ADDR'];
return [
'address' => $clientIps,
'ip' => $clientIps[0], // consider first IP as real client IP address
'nat' => $client,
];
}
$client['address'] = [$client['ip']];
return $client;
}
/**
* @return array
*/
private static function createUrlMeta()
{
if (strpos($_SERVER['REQUEST_URI'], '?') !== false) {
list($path, $query) = explode('?', $_SERVER['REQUEST_URI'], 2);
$url = [
'path' => $path,
'query' => $query,
];
} else {
$url = ['path' => $_SERVER['REQUEST_URI']];
}
if (strpos($_SERVER['HTTP_HOST'], ':') !== false) {
list($domain, $port) = explode(':', $_SERVER['HTTP_HOST'], 2);
$url['domain'] = $domain;
$url['port'] = (int) $port;
} else {
$url['domain'] = $_SERVER['HTTP_HOST'];
}
return $url;
}
/**
* Get user metadata (use unique id and email address)
* @return array|null
*/
private static function createUserMeta()
{
if (PHP_SAPI === 'cli') {
$currentUserId = Configure::read('CurrentUserId');
if (!empty($currentUserId)) {
/** @var User $userModel */
$userModel = ClassRegistry::init('User');
$user = $userModel->find('first', [
'recursive' => -1,
'conditions' => ['id' => $currentUserId],
'fields' => ['sub', 'email'],
]);
if (!empty($user)) {
return [
'id' => $user['User']['sub'] ?? $currentUserId,
'email' => $user['User']['email'],
];
}
}
} else {
App::uses('AuthComponent', 'Controller/Component');
$authUser = AuthComponent::user();
if (!empty($authUser)) {
return [
'id' => $authUser['sub'] ?? $authUser['id'],
'email' => $authUser['email'],
];
}
}
return null;
}
/**
* ISO 8601 timestamp with microsecond precision
* @return string
*/
private static function now()
{
return (new DateTime())->format('Y-m-d\TH:i:s.uP');
}
/**
* @param array $message
* @return bool True when message was successfully send to socket, false if message was saved to buffer
*/
private static function writeMessage(array $message)
{
$message = array_merge($message, self::createLogMeta());
try {
$data = JsonTool::encode($message) . "\n";
} catch (JsonException $e) {
CakeLog::error('Could not convert ECS log message into JSON: ' . $e->getMessage());
return null;
}
if (static::$socket === null) {
static::connect();
}
if (static::$socket) {
$bytesWritten = fwrite(static::$socket, $data);
if ($bytesWritten !== false) {
return true;
}
// In case of failure, try reconnect and send log again
static::connect();
if (static::$socket) {
$bytesWritten = fwrite(static::$socket, $data);
if ($bytesWritten !== false) {
return true;
}
}
}
// If sending message was not successful, save to buffer
self::$messageBuffer[] = $data;
if (count(self::$messageBuffer) > 100) {
array_shift(self::$messageBuffer); // remove oldest log
}
return false;
}
private static function connect()
{
static::$socket = null;
if (!file_exists(static::SOCKET_PATH)) {
return;
}
static::$socket = stream_socket_client('unix://' . static::SOCKET_PATH, $errorCode, $errorMessage);
if (static::$socket) {
foreach (self::$messageBuffer as $message) {
fwrite(static::$socket, $message);
}
self::$messageBuffer = [];
}
}
}

33
app/Plugin/OidcAuth/Lib/Oidc.php
View File

@ -44,7 +44,7 @@ class Oidc
if (!$user) { // User by sub not found, try to find by email
$user = $this->_findUser($settings, ['User.email' => $mispUsername]);
if ($user && $user['sub'] !== null && $user['sub'] !== $sub) {
$this->log($mispUsername, "User sub doesn't match ({$user['sub']} != $sub), could not login.");
$this->log($mispUsername, "User sub doesn't match ({$user['sub']} != $sub), could not login.", LOG_ERR);
return false;
}
}
@ -66,7 +66,7 @@ class Oidc
$roleProperty = $this->getConfig('roles_property', 'roles');
$roles = $claims->{$roleProperty} ?? $oidc->requestUserInfo($roleProperty);
if ($roles === null) {
$this->log($mispUsername, "Role property `$roleProperty` is missing in claims.");
$this->log($mispUsername, "Role property `$roleProperty` is missing in claims.", LOG_WARNING);
return false;
}
@ -79,6 +79,8 @@ class Oidc
return false;
}
$offlineAccessEnabled = $this->getConfig('offline_access', false);
if ($user) {
$this->log($mispUsername, "Found in database with ID {$user['id']}.");
@ -112,7 +114,10 @@ class Oidc
$user['disabled'] = false;
}
$refreshToken = $this->getConfig('offline_access', false) ? $oidc->getRefreshToken() : null;
$refreshToken = $offlineAccessEnabled ? $oidc->getRefreshToken() : null;
if ($offlineAccessEnabled && $refreshToken === null) {
$this->log($mispUsername, 'Refresh token requested, but not provided.', LOG_WARNING);
}
$this->storeMetadata($user['id'], $claims, $refreshToken);
$this->log($mispUsername, 'Logged in.');
@ -138,7 +143,10 @@ class Oidc
throw new RuntimeException("Could not create user `$mispUsername` in database.");
}
$refreshToken = $this->getConfig('offline_access', false) ? $oidc->getRefreshToken() : null;
$refreshToken = $offlineAccessEnabled ? $oidc->getRefreshToken() : null;
if ($offlineAccessEnabled && $refreshToken === null) {
$this->log($mispUsername, 'Refresh token requested, but not provided.', LOG_WARNING);
}
$this->storeMetadata($this->User->id, $claims, $refreshToken);
$this->log($mispUsername, "User created in database with ID {$this->User->id}");
@ -518,19 +526,20 @@ class Oidc
/**
* @param string|null $username
* @param string $message
* @param int $type
*/
private function log($username, $message)
private function log($username, $message, $type = LOG_INFO)
{
$sessionId = substr(session_id(), 0, 6);
$ipHeader = Configure::read('MISP.log_client_ip_header') ?: 'REMOTE_ADDR';
$ip = isset($_SERVER[$ipHeader]) ? trim($_SERVER[$ipHeader]) : $_SERVER['REMOTE_ADDR'];
$log = $username ? "OIDC user `$username`" : "OIDC";
if ($username) {
$message = "OIDC user `$username` [$ip;$sessionId] – $message";
if (PHP_SAPI !== 'cli') {
$sessionId = substr(session_id(), 0, 6);
$ip = $this->User->_remoteIp();
$log .= " [$ip;$sessionId] - $message";
} else {
$message = "OIDC [$ip;$sessionId] – $message";
$log .= " - $message";
}
CakeLog::info($message);
CakeLog::write($type, $log);
}
}

10
app/View/Elements/eventattributetoolbar.ctp
View File

@ -103,6 +103,16 @@
'popover-popup' => $baseurl . '/galaxies/selectGalaxyNamespace/selected/attribute/eventid:' . $eventId,
],
),
array(
'id' => 'multi-galaxy-button',
'title' => __('Add new local cluster to selected Attributes'),
'class' => 'mass-select hidden',
'fa-icon' => 'empire',
'fa-source' => 'fab',
'data' => [
'popover-popup' => $baseurl . '/galaxies/selectGalaxyNamespace/selected/attribute/local:1/eventid:' . $eventId,
],
),
array(
'id' => 'group-into-object-button',
'title' => __('Group selected Attributes into an Object'),

15
app/View/Events/upload_stix.ctp
View File

@ -27,9 +27,6 @@
'label' => __('Distribution ') . $distributionFormInfo,
'selected' => $initialDistribution,
));
?>
<div id="SGContainer" style="display:none;">
<?php
if (!empty($sharingGroups)) {
echo $this->Form->input('sharing_group_id', array(
'options' => array($sharingGroups),
@ -37,7 +34,6 @@
));
}
?>
</div>
<div class="input clear"></div>
<?php
echo $this->Form->input('publish', array(
@ -52,7 +48,7 @@
'label' => __('Include the original imported file as attachment')
));
if ($me['Role']['perm_site_admin'] || $me['Role']['perm_galaxy_editor']) {
$galaxiesFormInfo = $this-> element(
$galaxiesFormInfo = $this->element(
'genericElements/Form/formInfo',
[
'field' => [
@ -101,11 +97,8 @@
<script>
$(function(){
$('#EventDistribution').change(function() {
if ($(this).val() == 4) {
$('#SGContainer').show();
} else {
$('#SGContainer').hide();
}
}).change();
checkSharingGroup('Event');
});
checkSharingGroup('Event');
});
</script>

42
app/View/SharingGroupBlueprints/encode_sync_rule.ctp Normal file
View File

@ -0,0 +1,42 @@
<?php
$modelForForm = 'SharingGroupBlueprints';
$fields = [
[
'field' => 'type',
'type' => 'dropdown',
'options' => ['pull' => 'pull', 'push' => 'push'],
'class' => 'span6'
],
[
'field' => 'rule',
'type' => 'dropdown',
'options' => ['OR' => 'OR', 'NOT' => 'NOT'],
'class' => 'span6'
],
[
'field' => 'server_id',
'type' => 'dropdown',
'class' => 'span6',
'options' => $servers
]
];
$description = sprintf(
'%s<br />%s<br /><br />%s<br />%s',
__('Create a push or pull rule based '),
__('Simply create a JSON dictionary using a combination of filters and boolean operators.'),
'<span class="bold">Filters</span>: org_id, org_type, org_uuid, org_name, org_sector, org_nationality, sharing_group_id, , sharing_group_uuid',
'<span class="bold">Boolean operators</span>: OR, AND, NOT'
);
echo $this->element('genericElements/Form/genericForm', [
'data' => [
'description' => __('Create a push/pull org filter rule based on the organisations contained in a blueprint. The selected blueprint\'s rules will be transposed as either a push or a pull rule\'s OR or NOT list as per the selection.'),
'model' => 'SharingGroupBlueprint',
'title' => __('Create sync rules'),
'fields' => $fields,
'submit' => [
'action' => $this->request->params['action'],
'ajaxSubmit' => 'submitGenericFormInPlace();'
]
]
]);
?>

10
app/View/SharingGroupBlueprints/index.ctp
View File

@ -102,6 +102,16 @@
'icon' => 'recycle',
'title' => __('(Re)generate sharing group based on blueprint')
],
[
'onclick' => sprintf(
'openGenericModal(\'%s/SharingGroupBlueprints/encodeSyncRule/[onclick_params_data_path]\');',
$baseurl
),
'onclick_params_data_path' => 'SharingGroupBlueprint.id',
'icon' => 'filter',
'title' => __('Encode blueprint\'s contents as a sync rule'),
'requirements' => $me['Role']['perm_site_admin']
],
[
'onclick' => sprintf(
'openGenericModal(\'%s/SharingGroupBlueprints/delete/[onclick_params_data_path]\');',

0
app/files/certs/empty Normal file
View File

21
app/files/community-metadata/defaults.json
View File

@ -138,7 +138,7 @@
"url": "https://www.cssa.de/",
"sector": "Industry",
"nationality": "Germany",
"type": "Vetted Information Sharing Community",
"type": "Information Sharing Community",
"email": "undefined",
"pgp_key": null,
"misp_project_vetted": false,
@ -189,11 +189,28 @@
"url": "https://misp.beamteknoloji.com",
"sector": " Various",
"nationality": "Turkey",
"type": "Vetted Information Sharing Community",
"type": "Information Sharing Community",
"email": "contact@beamteknoloji.com",
"pgp_key": null,
"misp_project_vetted": false,
"scope_of_data_to_be_shared": "",
"self_registration": false
},
{
"name": "SecureGRID Alliance",
"logo": "https://misp-project.org/img/communities/445d6ff368486444a47b6e75bb67d2c9.png",
"uuid": "4dbce8ac-d36d-41b2-8a11-0e3acc813b77",
"org_uuid": "",
"org_name": "",
"description": "The SecureGRID Alliance is a cooperative framework for linking threat information between organizations that is free to join. Organizations participating in the alliance can mutually search MISP, which accumulates threat information provided by each alliance member organization, through a web portal site. By utilizing this framework, participating organizations will be able to have automatic analysis functions, strengthen information sharing and collaboration systems, and improve their own security levels and incident response capabilities.",
"url": "https://securegrid.lac.co.jp",
"sector": " Various",
"nationality": "Japan",
"type": "Information Sharing Community",
"email": "",
"pgp_key": null,
"misp_project_vetted": false,
"scope_of_data_to_be_shared": "",
"self_registration": false
}
]

30
app/files/feed-metadata/defaults.json
View File

@ -1737,5 +1737,35 @@
"lookup_visible": true,
"caching_enabled": true
}
},
{
"Feed": {
"name": "List of malicious hashes",
"provider": "Banco do Brasil S.A",
"url": "https://cti.bb.com.br:8443/hash-list.csv",
"rules": "{\"tags\":{\"OR\":[],\"NOT\":[]},\"orgs\":{\"OR\":[],\"NOT\":[]},\"type_attributes\":{\"NOT\":[]},\"type_objects\":{\"NOT\":[]},\"url_params\":\"\"}",
"enabled": true,
"distribution": "0",
"default": false,
"source_format": "csv",
"fixed_event": true,
"delta_merge": false,
"event_id": "0",
"publish": false,
"override_ids": false,
"settings": "{\"disable_correlation\":\"0\",\"csv\":{\"value\":\"\",\"delimiter\":\"\"},\"common\":{\"excluderegex\":\"\"}}",
"input_source": "network",
"delete_local_file": false,
"lookup_visible": false,
"headers": "",
"caching_enabled": true,
"force_to_ids": false
},
"Tag": {
"name": "osint:source-type=\"block-or-filter-list\"",
"colour": "#004577",
"exportable": true,
"hide_tag": false
}
}
]

2
app/files/misp-objects

@ -1 +1 @@
Subproject commit c18a240153cbe9ef68e46f05565d08653c2ad103
Subproject commit 587b298e1e7f87426182d55d44aa045a1522dc98

28
app/files/scripts/misp2stix.py
View File

@ -78,9 +78,13 @@ class StixExport:
if self._parser.errors:
self._handle_errors()
print(json.dumps(results))
except Exception as e:
print(json.dumps({'error': e.__str__()}))
error = type(e).__name__ + ': ' + e.__str__()
print(json.dumps({'error': error}))
traceback.print_tb(e.__traceback__)
print(error, file=sys.stderr)
sys.exit(1)
class StixAttributesExport(StixExport):
@ -157,19 +161,23 @@ class StixEventsExport(StixExport):
if __name__ == "__main__":
argparser = argparse.ArgumentParser(description='Export MISP into STIX1.')
argparser.add_argument('-s', '--scope', default='Event', choices=['Attribute', 'Event'], help='Scope: which kind of data is exported.')
argparser.add_argument('-v', '--version', default='1.1.1', choices=['1.1.1', '1.2'], help='STIX version (1.1.1 or 1.2).')
argparser.add_argument('-f', '--format', default='xml', choices=['json', 'xml'], help='Output format (xml or json).')
argparser.add_argument('-s', '--scope', default='Event', choices=('Attribute', 'Event'), help='Scope: which kind of data is exported.')
argparser.add_argument('-v', '--version', default='1.1.1', choices=('1.1.1', '1.2'), help='STIX version (1.1.1 or 1.2).')
argparser.add_argument('-f', '--format', default='xml', choices=('json', 'xml'), help='Output format (xml or json).')
argparser.add_argument('-i', '--input', nargs='+', help='Input file(s) containing MISP standard format.')
argparser.add_argument('-o', '--orgname', default='MISP', help='Default Org name to use if no Orgc value is provided.')
argparser.add_argument('-d', '--debug', action='store_true', help='Allow debug mode with warnings.')
try:
args = argparser.parse_args()
if args.input is None:
print(json.dumps({'error': 'No input file provided.'}))
else:
arguments = (args.orgname, args.format, args.version, args.debug)
exporter = globals()[f'Stix{args.scope}sExport'](*arguments)
exporter.parse_misp_files(args.input)
except SystemExit:
print(json.dumps({'error': 'Arguments error, please check you entered a valid version and provided input file names.'}))
sys.exit(1)
if args.input is None:
print(json.dumps({'error': 'No input file provided.'}))
sys.exit(1)
arguments = (args.orgname, args.format, args.version, args.debug)
exporter = globals()[f'Stix{args.scope}sExport'](*arguments)
exporter.parse_misp_files(args.input)
sys.exit(0)

14
app/files/scripts/stix2/misp2stix2.py
View File

@ -49,14 +49,14 @@ def _process_misp_files(
version: str, input_names: Union[list, None], debug: bool):
if input_names is None:
print(json.dumps({'error': 'No input file provided.'}))
return
sys.exit(1)
try:
parser = MISPtoSTIX20Parser() if version == '2.0' else MISPtoSTIX21Parser()
for name in input_names:
parser.parse_json_content(name)
with open(f'{name}.out', 'wt', encoding='utf-8') as f:
f.write(
f'{json.dumps(parser.stix_objects, cls=STIXJSONEncoder)}'
json.dumps(parser.stix_objects, cls=STIXJSONEncoder)
)
if parser.errors:
_handle_messages('Errors', parser.errors)
@ -64,8 +64,11 @@ def _process_misp_files(
_handle_messages('Warnings', parser.warnings)
print(json.dumps({'success': 1}))
except Exception as e:
print(json.dumps({'error': e.__str__()}))
error = type(e).__name__ + ': ' + e.__str__()
print(json.dumps({'error': error}))
traceback.print_tb(e.__traceback__)
print(error, file=sys.stderr)
sys.exit(1)
if __name__ == "__main__":
@ -84,7 +87,6 @@ if __name__ == "__main__":
)
try:
args = argparser.parse_args()
_process_misp_files(args.version, args.input, args.debug)
except SystemExit:
print(
json.dumps(
@ -94,3 +96,7 @@ if __name__ == "__main__":
}
)
)
sys.exit(1)
_process_misp_files(args.version, args.input, args.debug)
sys.exit(0)

12
app/files/scripts/stix2/stix2misp.py
View File

@ -44,7 +44,7 @@ def _handle_return_message(traceback):
return '\n - '.join(traceback)
def _process_stix_file(args: argparse.ArgumentParser):
def _process_stix_file(args: argparse.Namespace):
try:
with open(args.input, 'rt', encoding='utf-8') as f:
bundle = stix2_parser(
@ -81,8 +81,11 @@ def _process_stix_file(args: argparse.ArgumentParser):
file=sys.stderr
)
except Exception as e:
print(json.dumps({'error': e.__str__()}))
error = type(e).__name__ + ': ' + e.__str__()
print(json.dumps({'error': error}))
traceback.print_tb(e.__traceback__)
print(error, file=sys.stderr)
sys.exit(1)
if __name__ == '__main__':
@ -109,8 +112,7 @@ if __name__ == '__main__':
)
try:
args = argparser.parse_args()
_process_stix_file(args)
except SystemExit:
except SystemExit as e:
print(
json.dumps(
{
@ -118,4 +120,6 @@ if __name__ == '__main__':
}
)
)
sys.exit(1)
_process_stix_file(args)

20
app/files/scripts/stix2misp.py
View File

@ -96,7 +96,7 @@ class StixParser():
# Convert the MISP event we create from the STIX document into json format
# and write it in the output file
def saveFile(self):
def save_to_file(self):
for attribute in self.misp_event.attributes:
attribute_uuid = uuid.UUID(attribute.uuid) if isinstance(attribute.uuid, str) else attribute.uuid
if attribute_uuid.version not in _RFC_UUID_VERSIONS:
@ -1547,19 +1547,20 @@ def generate_event(filename, tries=0):
return STIXPackage.from_xml(filename)
except NamespaceNotFoundError:
if tries == 1:
print(json.dump({'error': 'Cannot handle STIX namespace'}))
sys.exit()
print(json.dumps({'error': 'Cannot handle STIX namespace'}))
sys.exit(1)
_update_namespaces()
return generate_event(filename, 1)
except NotImplementedError:
print(json.dumps({'error': 'Missing python library: stix_edh'}))
sys.exit(1)
except Exception as e:
try:
import maec
print(json.dumps({'error': f'Error while loading the STIX file: {e.__str__()}'}))
except ImportError:
print(json.dumps({'error': 'Missing python library: maec'}))
sys.exit(0)
sys.exit(1)
def is_from_misp(event):
@ -1567,7 +1568,7 @@ def is_from_misp(event):
title = event.stix_header.title
except AttributeError:
return False
return ('Export from ' in title and 'MISP' in title)
return 'Export from ' in title and 'MISP' in title
def main(args):
@ -1578,11 +1579,16 @@ def main(args):
stix_parser = StixFromMISPParser() if from_misp else ExternalStixParser()
stix_parser.load_event(args[2:], filename, from_misp, event.version)
stix_parser.build_misp_event(event)
stix_parser.saveFile()
stix_parser.save_to_file()
print(json.dumps({'success': 1}))
sys.exit(0)
except Exception as e:
print(json.dumps({'error': e.__str__()}))
error = type(e).__name__ + ': ' + e.__str__()
print(json.dumps({'error': error}))
traceback.print_tb(e.__traceback__)
print(error, file=sys.stderr)
sys.exit(1)
if __name__ == "__main__":
main(sys.argv)

2
app/files/warninglists

@ -1 +1 @@
Subproject commit fbf99aae8f054072d0d3097f04d9f81e12835bdc
Subproject commit b5dcb0b7b1a7347345c77bff70366248d577d878

2
app/webroot/doc/openapi.yaml
View File

@ -1,4 +1,4 @@
openapi: 3.1.0
openapi: 3.0.0
info:
title: MISP Automation API

144
db_schema.json
View File

@ -8625,6 +8625,140 @@
"extra": ""
}
],
"user_login_profiles": [
{
"column_name": "id",
"is_nullable": "NO",
"data_type": "int",
"character_maximum_length": null,
"numeric_precision": "10",
"collation_name": null,
"column_type": "int(10) unsigned",
"column_default": null,
"extra": "auto_increment"
},
{
"column_name": "created_at",
"is_nullable": "NO",
"data_type": "timestamp",
"character_maximum_length": null,
"numeric_precision": null,
"collation_name": null,
"column_type": "timestamp",
"column_default": "current_timestamp()",
"extra": ""
},
{
"column_name": "user_id",
"is_nullable": "NO",
"data_type": "int",
"character_maximum_length": null,
"numeric_precision": "10",
"collation_name": null,
"column_type": "int(11)",
"column_default": null,
"extra": ""
},
{
"column_name": "status",
"is_nullable": "YES",
"data_type": "varchar",
"character_maximum_length": "191",
"numeric_precision": null,
"collation_name": "utf8mb4_unicode_ci",
"column_type": "varchar(191)",
"column_default": "NULL",
"extra": ""
},
{
"column_name": "ip",
"is_nullable": "YES",
"data_type": "varchar",
"character_maximum_length": "191",
"numeric_precision": null,
"collation_name": "utf8mb4_unicode_ci",
"column_type": "varchar(191)",
"column_default": "NULL",
"extra": ""
},
{
"column_name": "user_agent",
"is_nullable": "YES",
"data_type": "varchar",
"character_maximum_length": "191",
"numeric_precision": null,
"collation_name": "utf8mb4_unicode_ci",
"column_type": "varchar(191)",
"column_default": "NULL",
"extra": ""
},
{
"column_name": "accept_lang",
"is_nullable": "YES",
"data_type": "varchar",
"character_maximum_length": "191",
"numeric_precision": null,
"collation_name": "utf8mb4_unicode_ci",
"column_type": "varchar(191)",
"column_default": "NULL",
"extra": ""
},
{
"column_name": "geoip",
"is_nullable": "YES",
"data_type": "varchar",
"character_maximum_length": "191",
"numeric_precision": null,
"collation_name": "utf8mb4_unicode_ci",
"column_type": "varchar(191)",
"column_default": "NULL",
"extra": ""
},
{
"column_name": "ua_platform",
"is_nullable": "YES",
"data_type": "varchar",
"character_maximum_length": "191",
"numeric_precision": null,
"collation_name": "utf8mb4_unicode_ci",
"column_type": "varchar(191)",
"column_default": "NULL",
"extra": ""
},
{
"column_name": "ua_browser",
"is_nullable": "YES",
"data_type": "varchar",
"character_maximum_length": "191",
"numeric_precision": null,
"collation_name": "utf8mb4_unicode_ci",
"column_type": "varchar(191)",
"column_default": "NULL",
"extra": ""
},
{
"column_name": "ua_pattern",
"is_nullable": "YES",
"data_type": "varchar",
"character_maximum_length": "191",
"numeric_precision": null,
"collation_name": "utf8mb4_unicode_ci",
"column_type": "varchar(191)",
"column_default": "NULL",
"extra": ""
},
{
"column_name": "hash",
"is_nullable": "NO",
"data_type": "varchar",
"character_maximum_length": "32",
"numeric_precision": null,
"collation_name": "utf8mb4_unicode_ci",
"column_type": "varchar(32)",
"column_default": null,
"extra": ""
}
],
"user_settings": [
{
"column_name": "id",
@ -9531,6 +9665,14 @@
"server_id": false,
"sub": true
},
"user_login_profiles": {
"geoip": false,
"hash": true,
"id": true,
"ip": false,
"status": false,
"user_id": false
},
"user_settings": {
"id": true,
"setting": false,
@ -9560,5 +9702,5 @@
"uuid": false
}
},
"db_version": "117"
"db_version": "118"
}

2
requirements.txt
View File

@ -6,7 +6,7 @@ misp-lib-stix2>=3.0.1.1
mixbox>=1.0.5
plyara>=2.1.1
pydeep2>=0.5.1
pymisp==2.4.182
pymisp==2.4.183
python-magic>=0.4.27
pyzmq>=25.1.1
redis>=5.0.1

2
tools/gen_website_communities.py
View File

@ -29,7 +29,7 @@ MISP is an open source software and it is also a large community of MISP users c
## Known Existing and Public MISP Communities
Each community might have specific rules to join them. Below is a brief overview of existing communities, feel free to contact the respective communities that fit your organization. Some of existing public communities might be interconnected and some might be in an island mode. By running MISP, these communities usually allow their members to connect using the MISP API, MISP user-interface or even to synchronize your MISP instance with their communities. If you want to add your MISP community to the list, don't hesitate to [contact us](mailto:info@misp-project.org).
Each community might have specific rules to join them. Below is a brief overview of existing communities, feel free to contact the respective communities that fit your organization. Some of existing public communities might be interconnected and some might be in an island mode. By running MISP, these communities usually allow their members to connect using the MISP API, MISP user-interface or even to synchronize your MISP instance with their communities. If you want to add your MISP community to the list, don't hesitate to [contact us](mailto:info@misp-project.org) or do a Pull Request on [this file](https://github.com/MISP/MISP/blob/develop/app/files/community-metadata/defaults.json).
The <i class="far fa-check-circle" style="color:green;"></i> sign indicates the community is vetted by the MISP Project.