MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #7851 from JakubOnderka/better-validation 

Better validation
pull/7861/head
Jakub Onderka 2021-10-18 09:47:41 +02:00 committed by GitHub
parent b3e9cf5bb9 9720a9772c
commit 934fa326f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 286 additions and 312 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
app/Controller/EventsController.php
View File

@ -3202,10 +3202,7 @@ class EventsController extends AppController
$iocData = FileAccessTool::readFromFile($this->data['Event']['submittedioc']['tmp_name'], $this->data['Event']['submittedioc']['size']);
// write
$attachments_dir = Configure::read('MISP.attachments_dir');
if (empty($attachments_dir)) {
$attachments_dir = $this->Event->getDefaultAttachments_dir();
}
$attachments_dir = Configure::read('MISP.attachments_dir') ?: (APP . 'files');
$rootDir = $attachments_dir . DS . $id . DS;
App::uses('Folder', 'Utility');
$dir = new Folder($rootDir . 'ioc', true);

3
app/Lib/Export/StixExport.php
View File

@ -1,6 +1,7 @@
<?php
App::uses('JSONConverterTool', 'Tools');
App::uses('TmpFileTool', 'Tools');
App::uses('JsonTool', 'Tools');
abstract class StixExport
{
@ -44,7 +45,7 @@ abstract class StixExport
}
$converter = new JSONConverterTool();
$event = json_encode($converter->convert($data, false, true)); // we don't need pretty printed JSON
$event = JsonTool::encode($converter->convert($data, false, true)); // we don't need pretty printed JSON
if ($this->__n_attributes + $attributesCount < $this->__attributes_limit) {
$this->__tmp_file->append($this->__n_attributes === 0 ? $event : ',' . $event);
$this->__n_attributes += $attributesCount;

15
app/Lib/Tools/FileAccessTool.php
View File

@ -2,6 +2,21 @@
class FileAccessTool
{
/**
* @param string $path
* @param int $permissions
* @throws Exception
*/
public static function createFile($path, $permissions = 0600)
{
if (!file_exists($path)) {
if (!touch($path)) {
throw new Exception("Could not create file `$path`.");
}
}
@chmod($path, $permissions); // hide error if current user is not file owner
}
/**
* Creates temporary file, but you have to delete it after use.
* @param string|null $dir

9
app/Lib/Tools/JSONConverterTool.php
View File

@ -48,7 +48,7 @@ class JSONConverterTool
}
}
if (isset($event['Event']['SharingGroup']) && empty($event['Event']['SharingGroup'])) {
if (empty($event['Event']['SharingGroup'])) {
unset($event['Event']['SharingGroup']);
}
@ -86,11 +86,6 @@ class JSONConverterTool
}
unset($tempSightings);
unset($event['Event']['RelatedAttribute']);
if (isset($event['Event']['RelatedEvent'])) {
foreach ($event['Event']['RelatedEvent'] as $key => $value) {
unset($event['Event']['RelatedEvent'][$key]['Event']['user_id']);
}
}
$result = array('Event' => $event['Event']);
if (isset($event['errors'])) {
$result = array_merge($result, array('errors' => $event['errors']));
@ -143,7 +138,7 @@ class JSONConverterTool
{
// remove value1 and value2 from the output and remove invalid utf8 characters for the xml parser
foreach ($attributes as $key => $attribute) {
if (isset($attribute['SharingGroup']) && empty($attribute['SharingGroup'])) {
if (empty($attribute['SharingGroup'])) {
unset($attributes[$key]['SharingGroup']);
}
unset($attributes[$key]['value1']);

61
app/Lib/Tools/PubSubTool.php
View File

@ -1,4 +1,7 @@
<?php
App::uses('FileAccessTool', 'Tools');
App::uses('JsonTool', 'Tools');
class PubSubTool
{
const SCRIPTS_TMP = APP . 'files' . DS . 'scripts' . DS . 'tmp' . DS;
@ -9,18 +12,12 @@ class PubSubTool
*/
private $redis;
/**
* @var array
*/
private $settings;
public function initTool()
{
if (!$this->redis) {
$settings = $this->getSetSettings();
$this->setupPubServer($settings);
$this->redis = $this->createRedisConnection($settings);
$this->settings = $settings;
}
}
@ -57,8 +54,8 @@ class PubSubTool
{
$settings = $this->getSetSettings();
$redis = $this->createRedisConnection($settings);
$redis->rPush($settings['redis_namespace'] . ':command', 'status');
$response = $redis->blPop($settings['redis_namespace'] . ':status', 5);
$redis->rPush( 'command', 'status');
$response = $redis->blPop('status', 5);
if ($response === null) {
throw new Exception("No response from status command returned after 5 seconds.");
}
@ -80,7 +77,7 @@ class PubSubTool
App::uses('JSONConverterTool', 'Tools');
$jsonTool = new JSONConverterTool();
$json = $jsonTool->convert($event);
return $this->pushToRedis(':data:misp_json', $json);
return $this->pushToRedis('data:misp_json', $json);
}
public function event_save(array $event, $action)
@ -88,7 +85,7 @@ class PubSubTool
if (!empty($action)) {
$event['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_event', $event);
return $this->pushToRedis('data:misp_json_event', $event);
}
public function object_save(array $object, $action)
@ -96,7 +93,7 @@ class PubSubTool
if (!empty($action)) {
$object['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_object', $object);
return $this->pushToRedis('data:misp_json_object', $object);
}
public function object_reference_save(array $object_reference, $action)
@ -104,12 +101,12 @@ class PubSubTool
if (!empty($action)) {
$object_reference['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_object_reference', $object_reference);
return $this->pushToRedis('data:misp_json_object_reference', $object_reference);
}
public function publishConversation(array $message)
{
return $this->pushToRedis(':data:misp_json_conversation', $message);
return $this->pushToRedis('data:misp_json_conversation', $message);
}
public function attribute_save(array $attribute, $action = false)
@ -117,7 +114,7 @@ class PubSubTool
if (!empty($action)) {
$attribute['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_attribute', $attribute);
return $this->pushToRedis('data:misp_json_attribute', $attribute);
}
public function tag_save(array $tag, $action = false)
@ -125,7 +122,7 @@ class PubSubTool
if (!empty($action)) {
$tag['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_tag', $tag);
return $this->pushToRedis('data:misp_json_tag', $tag);
}
public function sighting_save(array $sighting, $action = false)
@ -133,7 +130,7 @@ class PubSubTool
if (!empty($action)) {
$sighting['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_sighting', $sighting);
return $this->pushToRedis('data:misp_json_sighting', $sighting);
}
public function warninglist_save(array $warninglist, $action = false)
@ -141,7 +138,7 @@ class PubSubTool
if (!empty($action)) {
$warninglist['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_warninglist', $warninglist);
return $this->pushToRedis('data:misp_json_warninglist', $warninglist);
}
/**
@ -155,7 +152,7 @@ class PubSubTool
if (!empty($action)) {
$data['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_' . $type, $data);
return $this->pushToRedis('data:misp_json_' . $type, $data);
}
public function publish($data, $type, $action = false)
@ -163,7 +160,7 @@ class PubSubTool
if (!empty($action)) {
$data['action'] = $action;
}
return $this->pushToRedis(':data:misp_json_' . $type, $data);
return $this->pushToRedis('data:misp_json_' . $type, $data);
}
public function killService()
@ -171,7 +168,7 @@ class PubSubTool
if ($this->checkIfRunning()) {
$settings = $this->getSetSettings();
$redis = $this->createRedisConnection($settings);
$redis->rPush($settings['redis_namespace'] . ':command', 'kill');
$redis->rPush('command', 'kill');
sleep(1);
if ($this->checkIfRunning()) {
// Still running.
@ -194,7 +191,7 @@ class PubSubTool
if ($this->checkIfRunning()) {
$redis = $this->createRedisConnection($settings);
$redis->rPush($settings['redis_namespace'] . ':command', 'reload');
$redis->rPush( 'command', 'reload');
} else {
return 'Setting saved, but something is wrong with the ZeroMQ server. Please check the diagnostics page for more information.';
}
@ -226,7 +223,7 @@ class PubSubTool
if ($this->checkIfRunning(self::OLD_PID_LOCATION)) {
// Old version is running, kill it and start again new one.
$redis = $this->createRedisConnection($settings);
$redis->rPush($settings['redis_namespace'] . ':command', 'kill');
$redis->rPush('command', 'kill');
sleep(1);
}
@ -244,10 +241,10 @@ class PubSubTool
private function pushToRedis($ns, $data)
{
if (is_array($data)) {
$data = json_encode($data, JSON_UNESCAPED_UNICODE);
$data = JsonTool::encode($data);
}
$this->redis->rPush($this->settings['redis_namespace'] . $ns, $data);
$this->redis->rPush($ns, $data);
return true;
}
@ -264,6 +261,7 @@ class PubSubTool
$redis->auth($redisPassword);
}
$redis->select($settings['redis_database']);
$redis->setOption(Redis::OPT_PREFIX, $settings['redis_namespace'] . ':');
return $redis;
}
@ -274,18 +272,12 @@ class PubSubTool
private function saveSettingToFile(array $settings)
{
$settingFilePath = self::SCRIPTS_TMP . 'mispzmq_settings.json';
$settingsFile = new File($settingFilePath, true, 0644);
if (!$settingsFile->exists()) {
throw new Exception("Could not create zmq config file '$settingFilePath'.");
}
// Because setting file contains secrets, it should be readable just by owner. But because in Travis test,
// config file is created under one user and then changed under other user, file must be readable and writable
// also by group.
@chmod($settingsFile->pwd(), 0660); // hide error if current user is not file owner
if (!$settingsFile->write(json_encode($settings))) {
throw new Exception("Could not write zmq config file '$settingFilePath'.");
}
$settingsFile->close();
FileAccessTool::createFile($settingFilePath, 0660);
FileAccessTool::writeToFile($settingFilePath, JsonTool::encode($settings));
}
private function getSetSettings()
@ -302,8 +294,9 @@ class PubSubTool
'password' => null,
);
$pluginConfig = Configure::read('Plugin');
foreach ($settings as $key => $setting) {
$temp = Configure::read('Plugin.ZeroMQ_' . $key);
$temp = isset($pluginConfig['ZeroMQ_' . $key]) ? $pluginConfig['ZeroMQ_' . $key] : null;
if ($temp) {
$settings[$key] = $temp;
}

40
app/Model/AppModel.php
View File

@ -1931,10 +1931,9 @@ class AppModel extends Model
// alternative to the build in notempty/notblank validation functions, compatible with cakephp <= 2.6 and cakephp and cakephp >= 2.7
public function valueNotEmpty($value)
{
$field = array_keys($value);
$field = $field[0];
$value[$field] = trim($value[$field]);
if (!empty($value[$field])) {
$field = array_keys($value)[0];
$value = trim($value[$field]);
if (!empty($value)) {
return true;
}
return ucfirst($field) . ' cannot be empty.';
@ -1942,32 +1941,17 @@ class AppModel extends Model
public function valueIsJson($value)
{
$field = array_keys($value);
$field = $field[0];
$json_decoded = json_decode($value[$field]);
$value = array_values($value)[0];
$json_decoded = json_decode($value);
if ($json_decoded === null) {
return __('Invalid JSON.');
}
return true;
}
public function valueIsJsonOrNull($value)
{
$field = array_keys($value);
$field = $field[0];
if (!is_null($value[$field])) {
$json_decoded = json_decode($value[$field]);
if ($json_decoded === null) {
return __('Invalid JSON.');
}
}
return true;
}
public function valueIsID($value)
{
$field = array_keys($value);
$field = $field[0];
$field = array_keys($value)[0];
if (!is_numeric($value[$field]) || $value[$field] < 0) {
return 'Invalid ' . ucfirst($field) . ' ID';
}
@ -1976,10 +1960,9 @@ class AppModel extends Model
public function stringNotEmpty($value)
{
$field = array_keys($value);
$field = $field[0];
$value[$field] = trim($value[$field]);
if (!isset($value[$field]) || ($value[$field] == false && $value[$field] !== "0")) {
$field = array_keys($value)[0];
$value = trim($value[$field]);
if (!isset($value) || ($value == false && $value !== "0")) {
return ucfirst($field) . ' cannot be empty.';
}
return true;
@ -2923,11 +2906,6 @@ class AppModel extends Model
return $val / (1024 * 1024);
}
public function getDefaultAttachments_dir()
{
return APP . 'files';
}
private function __bumpReferences()
{
$this->Event = ClassRegistry::init('Event');

94
app/Model/Attribute.php
View File

@ -362,6 +362,9 @@ class Attribute extends AppModel
public function beforeSave($options = array())
{
if (empty($this->data['Attribute']['uuid'])) {
$this->data['Attribute']['uuid'] = CakeText::uuid();
}
if (!empty($this->data['Attribute']['id'])) {
$this->old = $this->find('first', array(
'recursive' => -1,
@ -563,81 +566,78 @@ class Attribute extends AppModel
public function beforeValidate($options = array())
{
if (empty($this->data['Attribute']['type'])) {
$attribute = &$this->data['Attribute'];
if (empty($attribute['type'])) {
$this->validationErrors['type'] = ['No type set.'];
return false;
}
$type = $this->data['Attribute']['type'];
if (is_array($this->data['Attribute']['value'])) {
$type = $attribute['type'];
if (is_array($attribute['value'])) {
$this->validationErrors['value'] = ['Value is an array.'];
return false;
}
if (!empty($this->data['Attribute']['object_id']) && empty($this->data['Attribute']['object_relation'])) {
if (!empty($attribute['object_id']) && empty($attribute['object_relation'])) {
$this->validationErrors['object_relation'] = ['Object attribute sent, but no object_relation set.'];
return false;
}
// If `value1` or `value2` provided and `value` is empty, merge them into `value` because of validation
if (empty($this->data['Attribute']['value'])) {
if (!empty($this->data['Attribute']['value1']) && !empty($this->data['Attribute']['value2'])) {
$this->data['Attribute']['value'] = "{$this->data['Attribute']['value1']}|{$this->data['Attribute']['value2']}";
} else if (!empty($this->data['Attribute']['value1'])) {
$this->data['Attribute']['value'] = $this->data['Attribute']['value1'];
if (empty($attribute['value'])) {
if (!empty($attribute['value1']) && !empty($attribute['value2'])) {
$attribute['value'] = "{$attribute['value1']}|{$attribute['value2']}";
} else if (!empty($attribute['value1'])) {
$attribute['value'] = $attribute['value1'];
}
}
// remove leading and trailing blanks and refang value and
$this->data['Attribute']['value'] = ComplexTypeTool::refangValue(trim($this->data['Attribute']['value']), $type);
$attribute['value'] = ComplexTypeTool::refangValue(trim($attribute['value']), $type);
// make some changes to the inserted value
$this->data['Attribute']['value'] = $this->modifyBeforeValidation($type, $this->data['Attribute']['value']);
$attribute['value'] = $this->modifyBeforeValidation($type, $attribute['value']);
// Run user defined regexp to attribute value
$result = $this->runRegexp($type, $this->data['Attribute']['value']);
$result = $this->runRegexp($type, $attribute['value']);
if ($result === false) {
$this->invalidate('value', 'This value is blocked by a regular expression in the import filters.');
} else {
$this->data['Attribute']['value'] = $result;
$attribute['value'] = $result;
}
if (empty($this->data['Attribute']['comment'])) {
$this->data['Attribute']['comment'] = "";
if (empty($attribute['comment'])) {
$attribute['comment'] = "";
}
// generate UUID if it doesn't exist
if (empty($this->data['Attribute']['uuid'])) {
$this->data['Attribute']['uuid'] = CakeText::uuid();
} else {
$this->data['Attribute']['uuid'] = strtolower($this->data['Attribute']['uuid']);
if (!empty($attribute['uuid'])) {
$attribute['uuid'] = strtolower($attribute['uuid']);
}
// generate timestamp if it doesn't exist
if (empty($this->data['Attribute']['timestamp'])) {
$this->data['Attribute']['timestamp'] = time();
if (empty($attribute['timestamp'])) {
$attribute['timestamp'] = time();
}
// parse first_seen different formats
if (isset($this->data['Attribute']['first_seen'])) {
$this->data['Attribute']['first_seen'] = $this->data['Attribute']['first_seen'] === '' ? null : $this->data['Attribute']['first_seen'];
if (isset($attribute['first_seen'])) {
$attribute['first_seen'] = $attribute['first_seen'] === '' ? null : $attribute['first_seen'];
}
// parse last_seen different formats
if (isset($this->data['Attribute']['last_seen'])) {
$this->data['Attribute']['last_seen'] = $this->data['Attribute']['last_seen'] === '' ? null : $this->data['Attribute']['last_seen'];
if (isset($attribute['last_seen'])) {
$attribute['last_seen'] = $attribute['last_seen'] === '' ? null : $attribute['last_seen'];
}
// Set defaults for when some of the mandatory fields don't have defaults
// These fields all have sane defaults either based on another field, or due to server settings
if (!isset($this->data['Attribute']['distribution'])) {
$this->data['Attribute']['distribution'] = $this->defaultDistribution();
if (!isset($attribute['distribution'])) {
$attribute['distribution'] = $this->defaultDistribution();
}
if ($attribute['distribution'] != 4) {
$attribute['sharing_group_id'] = 0;
}
// If category is not provided, assign default category by type
if (empty($this->data['Attribute']['category'])) {
$this->data['Attribute']['category'] = $this->typeDefinitions[$type]['default_category'];
if (empty($attribute['category'])) {
$attribute['category'] = $this->typeDefinitions[$type]['default_category'];
}
if (!isset($this->data['Attribute']['to_ids'])) {
$this->data['Attribute']['to_ids'] = $this->typeDefinitions[$type]['to_ids'];
}
if ($this->data['Attribute']['distribution'] != 4) {
$this->data['Attribute']['sharing_group_id'] = 0;
if (!isset($attribute['to_ids'])) {
$attribute['to_ids'] = $this->typeDefinitions[$type]['to_ids'];
}
// return true, otherwise the object cannot be saved
return true;
@ -742,21 +742,17 @@ class Attribute extends AppModel
// check whether the variable is null or datetime
public function datetimeOrNull($fields)
{
$k = array_keys($fields)[0];
$seen = $fields[$k];
try {
new DateTime($seen);
$returnValue = true;
} catch (Exception $e) {
$returnValue = false;
$seen = array_values($fields)[0];
if ($seen === null) {
return true;
}
return $returnValue || is_null($seen);
return strtotime($seen) !== false;
}
public function validateLastSeenValue($fields)
{
$ls = $fields['last_seen'];
if (!isset($this->data['Attribute']['first_seen']) || is_null($ls)) {
if (!isset($this->data['Attribute']['first_seen']) || $ls === null) {
return true;
}
$converted = $this->ISODatetimeToUTC(['Attribute' => [
@ -1410,7 +1406,7 @@ class Attribute extends AppModel
break;
case 'datetime':
try {
$value = (new DateTime($value))->setTimezone(new DateTimeZone('GMT'))->format('Y-m-d\TH:i:s.uO'); // ISO8601 formating with microseconds
$value = (new DateTime($value, new DateTimeZone('GMT')))->format('Y-m-d\TH:i:s.uO'); // ISO8601 formating with microseconds
} catch (Exception $e) {
// silently skip. Rejection will be done in runValidation()
}
@ -1698,8 +1694,7 @@ class Attribute extends AppModel
{
// convert into utc and micro sec
if (!empty($data[$alias]['first_seen'])) {
$d = new DateTime($data[$alias]['first_seen']);
$d->setTimezone(new DateTimeZone('GMT'));
$d = new DateTime($data[$alias]['first_seen'], new DateTimeZone('GMT'));
$fs_sec = $d->format('U');
$fs_micro = $d->format('u');
$fs_micro = str_pad($fs_micro, 6, "0", STR_PAD_LEFT);
@ -1707,8 +1702,7 @@ class Attribute extends AppModel
$data[$alias]['first_seen'] = $fs;
}
if (!empty($data[$alias]['last_seen'])) {
$d = new DateTime($data[$alias]['last_seen']);
$d->setTimezone(new DateTimeZone('GMT'));
$d = new DateTime($data[$alias]['last_seen'], new DateTimeZone('GMT'));
$ls_sec = $d->format('U');
$ls_micro = $d->format('u');
$ls_micro = str_pad($ls_micro, 6, "0", STR_PAD_LEFT);

15
app/Model/Behavior/AuditLogBehavior.php
View File

@ -81,11 +81,26 @@ class AuditLogBehavior extends ModelBehavior
if (!$this->enabled) {
return true;
}
// Do not fetch old version when just few fields will be fetched
$fieldToFetch = [];
if (!empty($options['fieldList'])) {
foreach ($options['fieldList'] as $field) {
if (!isset($this->skipFields[$field])) {
$fieldToFetch[] = $field;
}
}
if (empty($fieldToFetch)) {
$this->old = null;
return true;
}
}
if ($model->id) {
$this->old = $model->find('first', [
'conditions' => [$model->alias . '.' . $model->primaryKey => $model->id],
'recursive' => -1,
'callbacks' => false,
'fields' => $fieldToFetch,
]);
} else {
$this->old = null;

224
app/Model/Event.php
View File

@ -96,20 +96,14 @@ class Event extends AppModel
public $validate = array(
'org_id' => array(
'valueNotEmpty' => array(
'rule' => array('valueNotEmpty'),
),
'numeric' => array(
'rule' => array('numeric'),
),
'rule' => 'numeric',
'required' => true,
'allowEmpty' => false,
),
'orgc_id' => array(
'valueNotEmpty' => array(
'rule' => array('valueNotEmpty'),
),
'numeric' => array(
'rule' => array('numeric'),
),
'rule' => 'numeric',
'required' => true,
'allowEmpty' => false,
),
'date' => array(
'date' => array(
@ -134,22 +128,21 @@ class Event extends AppModel
'required' => true,
//'last' => false, // Stop validation after this rule
//'on' => 'create', // Limit validation to 'create' or 'update' operations
)
)
),
'sharing_group_id' => array(
'rule' => array('sharingGroupRequired'),
'message' => 'If the distribution is set to "Sharing Group", a sharing group has to be selected.',
//'required' => true,
//'allowEmpty' => true
'message' => 'If the distribution is set to "Sharing Group", a sharing group has to be selected.',
//'required' => true,
//'allowEmpty' => true
),
'analysis' => array(
'rule' => array('inList', array('0', '1', '2')),
'message' => 'Options : 0, 1, 2 (for Initial, Ongoing, Completed)',
//'allowEmpty' => false,
'required' => true,
//'last' => false, // Stop validation after this rule
//'on' => 'create', // Limit validation to 'create' or 'update' operations
'message' => 'Options : 0, 1, 2 (for Initial, Ongoing, Completed)',
//'allowEmpty' => false,
'required' => true,
//'last' => false, // Stop validation after this rule
//'on' => 'create', // Limit validation to 'create' or 'update' operations
),
'info' => array(
'valueNotEmpty' => array(
@ -442,91 +435,103 @@ class Event extends AppModel
public function beforeValidate($options = array())
{
$event = &$this->data['Event'];
// analysis - setting correct vars
if (isset($this->data['Event']['analysis'])) {
switch ($this->data['Event']['analysis']) {
if (isset($event['analysis'])) {
switch ($event['analysis']) {
case 'Initial':
$this->data['Event']['analysis'] = 0;
$event['analysis'] = 0;
break;
case 'Ongoing':
$this->data['Event']['analysis'] = 1;
$event['analysis'] = 1;
break;
case 'Completed':
$this->data['Event']['analysis'] = 2;
$event['analysis'] = 2;
break;
}
} else {
$this->data['Event']['analysis'] = 0;
$event['analysis'] = 0;
}
if (!isset($this->data['Event']['threat_level_id'])) {
$this->data['Event']['threat_level_id'] = Configure::read('MISP.default_event_threat_level') ?: 4;
if (!isset($event['threat_level_id'])) {
$event['threat_level_id'] = Configure::read('MISP.default_event_threat_level') ?: 4;
}
// generate UUID if it doesn't exist
if (empty($this->data['Event']['uuid'])) {
$this->data['Event']['uuid'] = CakeText::uuid();
} else {
$this->data['Event']['uuid'] = strtolower($this->data['Event']['uuid']);
if (!empty($event['uuid'])) {
$event['uuid'] = strtolower($event['uuid']);
}
// Convert event ID to uuid if needed
if (!empty($this->data['Event']['extends_uuid']) && is_numeric($this->data['Event']['extends_uuid'])) {
$extended_event = $this->find('first', array(
'recursive' => -1,
'conditions' => array('Event.id' => $this->data['Event']['extends_uuid']),
'fields' => array('Event.uuid')
));
if (empty($extended_event)) {
$this->data['Event']['extends_uuid'] = '';
if (!empty($event['extends_uuid'])) {
if (is_numeric($event['extends_uuid'])) {
$extended_event = $this->find('first', array(
'recursive' => -1,
'conditions' => array('Event.id' => $event['extends_uuid']),
'fields' => array('Event.uuid')
));
if (empty($extended_event)) {
$event['extends_uuid'] = '';
$this->invalidate('extends_uuid', 'Invalid event ID provided.');
} else {
$event['extends_uuid'] = $extended_event['Event']['uuid'];
}
} else {
$this->data['Event']['extends_uuid'] = $extended_event['Event']['uuid'];
$event['extends_uuid'] = strtolower($event['extends_uuid']);
}
} else if (!empty($this->data['Event']['extends_uuid'])) {
$this->data['Event']['extends_uuid'] = strtolower($this->data['Event']['extends_uuid']);
}
// generate timestamp if it doesn't exist
if (empty($this->data['Event']['timestamp'])) {
$this->data['Event']['timestamp'] = time();
if (empty($event['timestamp'])) {
$event['timestamp'] = time();
}
if (isset($this->data['Event']['publish_timestamp']) && empty($this->data['Event']['publish_timestamp'])) {
$this->data['Event']['publish_timestamp'] = 0;
if (isset($event['publish_timestamp']) && empty($event['publish_timestamp'])) {
$event['publish_timestamp'] = 0;
}
if (empty($this->data['Event']['date'])) {
$this->data['Event']['date'] = date('Y-m-d');
if (empty($event['date'])) {
$event['date'] = date('Y-m-d');
}
if (!isset($this->data['Event']['distribution']) || $this->data['Event']['distribution'] != 4) {
$this->data['Event']['sharing_group_id'] = 0;
if (!isset($event['distribution']) || $event['distribution'] != 4) {
$event['sharing_group_id'] = 0;
}
}
public function beforeSave($options = [])
{
// generate UUID if not provided
if (empty($this->data['Event']['uuid'])) {
$this->data['Event']['uuid'] = CakeText::uuid();
}
return true;
}
public function afterSave($created, $options = array())
{
$event = $this->data['Event'];
if (!Configure::read('MISP.completely_disable_correlation') && !$created) {
$updateCorrelation = [];
if (isset($this->data['Event']['distribution'])) {
$updateCorrelation['Correlation.distribution'] = (int)$this->data['Event']['distribution'];
if (isset($event['distribution']) && (empty($options['fieldList']) || in_array('distribution', $options['fieldList']))) {
$updateCorrelation['Correlation.distribution'] = (int)$event['distribution'];
}
if (isset($this->data['Event']['sharing_group_id'])) {
$updateCorrelation['Correlation.sharing_group_id'] = (int)$this->data['Event']['sharing_group_id'];
if (isset($event['sharing_group_id']) && (empty($options['fieldList']) || in_array('sharing_group_id', $options['fieldList']))) {
$updateCorrelation['Correlation.sharing_group_id'] = (int)$event['sharing_group_id'];
}
if (!empty($updateCorrelation)) {
$this->Attribute->Correlation->updateAll($updateCorrelation, ['Correlation.event_id' => (int)$this->data['Event']['id']]);
$this->Attribute->Correlation->updateAll($updateCorrelation, ['Correlation.event_id' => (int)$event['id']]);
}
}
if (empty($this->data['Event']['unpublishAction']) && empty($this->data['Event']['skip_zmq']) && Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) {
if (empty($event['unpublishAction']) && empty($event['skip_zmq']) && Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) {
$pubSubTool = $this->getPubSubTool();
$event = $this->quickFetchEvent($this->data['Event']['id']);
$eventForZmq = $this->quickFetchEvent($event['id']);
if (!empty($event)) {
$pubSubTool->event_save($event, $created ? 'add' : 'edit');
$pubSubTool->event_save($eventForZmq, $created ? 'add' : 'edit');
}
}
if (empty($this->data['Event']['unpublishAction']) && empty($this->data['Event']['skip_kafka'])) {
$this->publishKafkaNotification('event', $this->quickFetchEvent($this->data['Event']['id']), $created ? 'add' : 'edit');
if (empty($event['unpublishAction']) && empty($event['skip_kafka'])) {
$this->publishKafkaNotification('event', $this->quickFetchEvent($event['id']), $created ? 'add' : 'edit');
}
}
@ -546,10 +551,10 @@ class Event extends AppModel
'order' => false
));
$tags = array_column(array_column($tags, 'Tag'), null, 'id');
foreach ($events as $k => $event) {
foreach ($events as &$event) {
if (!empty($event['EventTag'])) {
foreach ($event['EventTag'] as $k2 => $et) {
$events[$k]['EventTag'][$k2]['Tag'] = $tags[$et['tag_id']];
foreach ($event['EventTag'] as &$et) {
$et['Tag'] = $tags[$et['tag_id']];
}
}
}
@ -772,51 +777,53 @@ class Event extends AppModel
public function getRelatedAttributes(array $user, $id, $shadowAttribute = false, $scope = 'event')
{
if ($shadowAttribute) {
$settings = array('correlationModel' => 'ShadowAttributeCorrelation', 'parentIdField' => '1_shadow_attribute_id');
$parentIdField = '1_shadow_attribute_id';
$correlationModelName = 'ShadowAttributeCorrelation';
} else {
$settings = array('correlationModel' => 'Correlation', 'parentIdField' => '1_attribute_id');
$parentIdField = '1_attribute_id';
$correlationModelName = 'Correlation';
}
if (!isset($this->{$settings['correlationModel']})) {
$this->{$settings['correlationModel']} = ClassRegistry::init($settings['correlationModel']);
if (!isset($this->{$correlationModelName})) {
$this->{$correlationModelName} = ClassRegistry::init($correlationModelName);
}
if (!$user['Role']['perm_site_admin']) {
$sgids = $this->cacheSgids($user, true);
$conditionsCorrelation = array(
'AND' => array(
$settings['correlationModel'] . '.1_' . $scope . '_id' => $id,
$correlationModelName . '.1_' . $scope . '_id' => $id,
array(
'OR' => array(
$settings['correlationModel'] . '.org_id' => $user['org_id'],
$correlationModelName . '.org_id' => $user['org_id'],
'AND' => array(
array(
'OR' => array(
array(
'AND' => array(
$settings['correlationModel'] . '.distribution >' => 0,
$settings['correlationModel'] . '.distribution <' => 4,
$correlationModelName . '.distribution >' => 0,
$correlationModelName . '.distribution <' => 4,
),
),
array(
'AND' => array(
$settings['correlationModel'] . '.distribution' => 4,
$settings['correlationModel'] . '.sharing_group_id' => $sgids
$correlationModelName . '.distribution' => 4,
$correlationModelName . '.sharing_group_id' => $sgids
),
),
),
),
array(
'OR' => array(
$settings['correlationModel'] . '.a_distribution' => 5,
$correlationModelName . '.a_distribution' => 5,
array(
'AND' => array(
$settings['correlationModel'] . '.a_distribution >' => 0,
$settings['correlationModel'] . '.a_distribution <' => 4,
$correlationModelName . '.a_distribution >' => 0,
$correlationModelName . '.a_distribution <' => 4,
),
),
array(
'AND' => array(
$settings['correlationModel'] . '.a_distribution' => 4,
$settings['correlationModel'] . '.a_sharing_group_id' => $sgids
$correlationModelName . '.a_distribution' => 4,
$correlationModelName . '.a_sharing_group_id' => $sgids
),
),
),
@ -828,11 +835,11 @@ class Event extends AppModel
)
);
} else {
$conditionsCorrelation = array($settings['correlationModel'] . '.1_' . $scope . '_id' => $id);
$conditionsCorrelation = array($correlationModelName . '.1_' . $scope . '_id' => $id);
}
$max_correlations = Configure::read('MISP.max_correlations_per_event') ?: 5000;
$correlations = $this->{$settings['correlationModel']}->find('all', array(
'fields' => ['event_id', 'attribute_id', 'value', $settings['parentIdField']],
$correlations = $this->{$correlationModelName}->find('all', array(
'fields' => ['event_id', 'attribute_id', 'value', $parentIdField],
'conditions' => $conditionsCorrelation,
'recursive' => -1,
'order' => false,
@ -842,11 +849,11 @@ class Event extends AppModel
return array();
}
$correlations = array_column($correlations, $settings['correlationModel']);
$correlations = array_column($correlations, $correlationModelName);
$eventIds = array_unique(array_column($correlations, 'event_id'));
$conditions = $this->createEventConditions($user);
$conditions['AND']['Event.id'] = $eventIds;
$conditions['Event.id'] = $eventIds;
$events = $this->find('all', array(
'recursive' => -1,
'conditions' => $conditions,
@ -858,20 +865,21 @@ class Event extends AppModel
$relatedAttributes = [];
foreach ($correlations as $correlation) {
// User don't have access to correlated attribute event, skip.
if (!isset($events[$correlation['event_id']])) {
$eventId = $correlation['event_id'];
if (!isset($events[$eventId])) {
continue;
}
$event = $events[$correlation['event_id']];
$event = $events[$eventId];
$current = array(
'id' => $correlation['event_id'],
'id' => $eventId,
'attribute_id' => $correlation['attribute_id'],
'value' => $correlation['value'],
'org_id' => $event['orgc_id'],
'info' => $event['info'],
'date' => $event['date'],
);
$parentId = $correlation[$settings['parentIdField']];
$parentId = $correlation[$parentIdField];
$relatedAttributes[$parentId][] = $current;
}
return $relatedAttributes;
@ -2124,13 +2132,6 @@ class Event extends AppModel
// Precache current user email
$userEmails = empty($user['id']) ? [] : [$user['id'] => $user['email']];
// Do some refactoring with the event
$fields = array(
'common' => array('distribution', 'sharing_group_id', 'uuid'),
'Attribute' => array('value', 'type', 'category', 'to_ids'),
'Object' => array('name', 'meta-category')
);
if (!$options['includeAllTags']) {
$justExportableTags = true;
} else {
@ -2162,7 +2163,7 @@ class Event extends AppModel
$this->Warninglist->attachWarninglistToAttributes($event['ShadowAttribute']);
$event['warnings'] = $eventWarnings;
}
$this->__attachReferences($event, $fields);
$this->__attachReferences($event);
$this->__attachTags($event, $justExportableTags);
$this->__attachGalaxies($event, $user, $options['excludeGalaxy'], $options['fetchFullClusters']);
$event = $this->Orgc->attachOrgs($event, $fieldsOrg);
@ -5846,19 +5847,21 @@ class Event extends AppModel
{
$event = $this->find('first', array(
'recursive' => -1,
'conditions' => array('Event.id' => $id)
'conditions' => array('Event.id' => $id),
'fields' => ['id', 'info'], // info is required because of SysLogLogableBehavior
));
if (empty($event)) {
return false;
}
$fields = ['published', 'timestamp'];
$event['Event']['published'] = 0;
$date = new DateTime();
$event['Event']['timestamp'] = $date->getTimestamp();
$event['Event']['timestamp'] = time();
if ($proposalLock) {
$event['Event']['proposal_email_lock'] = 0;
$fields[] = 'proposal_email_lock';
}
$event['Event']['unpublishAction'] = true;
return $this->save($event);
return $this->save($event, true, $fields);
}
/**
@ -6933,13 +6936,19 @@ class Event extends AppModel
* we just find proper element in event.
*
* @param array $event
* @param array $fields
*/
private function __attachReferences(array &$event, array $fields)
private function __attachReferences(array &$event)
{
if (!isset($event['Object'])) {
return;
}
$fieldsToCopy = array(
'common' => array('distribution', 'sharing_group_id', 'uuid'),
'Attribute' => array('value', 'type', 'category', 'to_ids'),
'Object' => array('name', 'meta-category')
);
foreach ($event['Object'] as $k => $object) {
foreach ($object['ObjectReference'] as $k2 => $reference) {
// find referenced object in current event
@ -6955,7 +6964,7 @@ class Event extends AppModel
if ($found) {
// copy requested fields
$reference = [];
foreach (array_merge($fields['common'], $fields[$type]) as $field) {
foreach (array_merge($fieldsToCopy['common'], $fieldsToCopy[$type]) as $field) {
$reference[$field] = $found[$field];
}
$event['Object'][$k]['ObjectReference'][$k2][$type] = $reference;
@ -7212,9 +7221,8 @@ class Event extends AppModel
'disable_correlation' => true
)
);
foreach ($attributes as $attribute) {
$this->Attribute->create();
$this->Attribute->save($attribute);
if (!$this->Attribute->saveMany($attributes)) {
throw new Exception("Could not save attributes for original file because of validation errors:" . json_encode($this->Attribute->validationErrors));
}
return true;
}

12
app/Model/EventGraph.php
View File

@ -30,7 +30,7 @@ class EventGraph extends AppModel
public $validate = array(
'network_json' => array(
'rule' => array('isValidJson'),
'rule' => 'valueIsJson',
'message' => 'The provided eventGraph is not a valid json format',
'required' => true,
),
@ -44,16 +44,6 @@ class EventGraph extends AppModel
return true;
}
public function isValidJson($fields)
{
$text = $fields['network_json'];
$check = json_decode($text);
if ($check === null) {
return false;
}
return true;
}
public function getPictureData($eventGraph)
{
$b64 = str_replace('data:image/png;base64,', '', $eventGraph['EventGraph']['preview_img']);

69
app/Model/MispObject.php
View File

@ -69,7 +69,6 @@ class MispObject extends AppModel
'unique' => array(
'rule' => 'isUnique',
'message' => 'The UUID provided is not unique',
'required' => true,
'on' => 'create'
),
),
@ -216,23 +215,19 @@ class MispObject extends AppModel
}
// check whether the variable is null or datetime
public function datetimeOrNull($fields)
{
$k = array_keys($fields)[0];
$seen = $fields[$k];
try {
new DateTime($seen);
$returnValue = true;
} catch (Exception $e) {
$returnValue = false;
}
return $returnValue || is_null($seen);
}
public function datetimeOrNull($fields)
{
$seen = array_values($fields)[0];
if ($seen === null) {
return true;
}
return strtotime($seen) !== false;
}
public function validateLastSeenValue($fields)
{
$ls = $fields['last_seen'];
if (!isset($this->data['Object']['first_seen']) || is_null($ls)) {
if (!isset($this->data['Object']['first_seen']) || $ls === null) {
return true;
}
$converted = $this->Attribute->ISODatetimeToUTC(['Object' => [
@ -253,44 +248,44 @@ class MispObject extends AppModel
return $results;
}
public function beforeSave($options = array()) {
public function beforeSave($options = array())
{
// generate UUID if it doesn't exist
if (empty($this->data['Object']['uuid'])) {
$this->data['Object']['uuid'] = CakeText::uuid();
}
$this->data = $this->Attribute->ISODatetimeToUTC($this->data, $this->alias);
}
public function beforeValidate($options = array())
{
parent::beforeValidate();
if (empty($this->data[$this->alias]['comment'])) {
$this->data[$this->alias]['comment'] = "";
}
// generate UUID if it doesn't exist
if (empty($this->data[$this->alias]['uuid'])) {
$this->data[$this->alias]['uuid'] = CakeText::uuid();
$object = &$this->data['Object'];
if (empty($object['comment'])) {
$object['comment'] = "";
}
// generate timestamp if it doesn't exist
if (empty($this->data[$this->alias]['timestamp'])) {
$date = new DateTime();
$this->data[$this->alias]['timestamp'] = $date->getTimestamp();
if (empty($object['timestamp'])) {
$object['timestamp'] = time();
}
// parse first_seen different formats
if (isset($this->data[$this->alias]['first_seen'])) {
$this->data[$this->alias]['first_seen'] = $this->data[$this->alias]['first_seen'] === '' ? null : $this->data[$this->alias]['first_seen'];
if (isset($object['first_seen'])) {
$object['first_seen'] = $object['first_seen'] === '' ? null : $object['first_seen'];
}
// parse last_seen different formats
if (isset($this->data[$this->alias]['last_seen'])) {
$this->data[$this->alias]['last_seen'] = $this->data[$this->alias]['last_seen'] === '' ? null : $this->data[$this->alias]['last_seen'];
if (isset($object['last_seen'])) {
$object['last_seen'] = $object['last_seen'] === '' ? null : $object['last_seen'];
}
if (empty($this->data[$this->alias]['template_version'])) {
$this->data[$this->alias]['template_version'] = 1;
if (empty($object['template_version'])) {
$object['template_version'] = 1;
}
if (isset($this->data[$this->alias]['deleted']) && empty($this->data[$this->alias]['deleted'])) {
$this->data[$this->alias]['deleted'] = 0;
if (isset($object['deleted']) && empty($object['deleted'])) {
$object['deleted'] = 0;
}
if (!isset($this->data[$this->alias]['distribution']) || $this->data['Object']['distribution'] != 4) {
$this->data['Object']['sharing_group_id'] = 0;
if (!isset($object['distribution']) || $object['distribution'] != 4) {
$object['sharing_group_id'] = 0;
}
if (!isset($this->data[$this->alias]['distribution'])) {
$this->data['Object']['distribution'] = 5;
if (!isset($object['distribution'])) {
$object['distribution'] = 5;
}
return true;
}

45
app/Model/Module.php
View File

@ -1,5 +1,6 @@
<?php
App::uses('AppModel', 'Model');
App::uses('JsonTool', 'Tools');
class Module extends AppModel
{
@ -234,28 +235,27 @@ class Module extends AppModel
* @param array|null $postData
* @param string $moduleFamily
* @return array
* @throws JsonException
* @throws HttpSocketJsonException
* @throws Exception
*/
public function sendRequest($uri, $timeout, $postData = null, $moduleFamily = 'Enrichment')
{
$url = $this->__getModuleServer($moduleFamily);
if (!$url) {
$serverUrl = $this->__getModuleServer($moduleFamily);
if (!$serverUrl) {
throw new Exception("Module type $moduleFamily is not enabled.");
}
App::uses('HttpSocket', 'Network/Http');
App::uses('HttpSocketExtended', 'Tools');
$httpSocketSetting = ['timeout' => $timeout];
$sslSettings = array('ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_verify_peer', 'ssl_cafile');
foreach ($sslSettings as $sslSetting) {
if (Configure::check('Plugin.' . $moduleFamily . '_' . $sslSetting) && Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting) !== '') {
$settings[$sslSetting] = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
$value = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
if ($value && $value !== '') {
$httpSocketSetting[$sslSetting] = $value;
}
}
$httpSocket = new HttpSocket(['timeout' => $timeout]);
$request = array(
'header' => array(
'Content-Type' => 'application/json',
)
);
if ($moduleFamily == 'Cortex') {
$httpSocket = new HttpSocketExtended($httpSocketSetting);
$request = [];
if ($moduleFamily === 'Cortex') {
if (!empty(Configure::read('Plugin.' . $moduleFamily . '_authkey'))) {
$request['header']['Authorization'] = 'Bearer ' . Configure::read('Plugin.' . $moduleFamily . '_authkey');
}
@ -264,26 +264,23 @@ class Module extends AppModel
if (!is_array($postData)) {
throw new InvalidArgumentException("Post data must be array, " . gettype($postData) . " given.");
}
$post = json_encode($postData);
$response = $httpSocket->post($url . $uri, $post, $request);
$post = JsonTool::encode($postData);
$request['header']['Content-Type'] = 'application/json';
$response = $httpSocket->post($serverUrl . $uri, $post, $request);
} else {
if ($moduleFamily == 'Cortex') {
unset($request['header']['Content-Type']);
}
$response = $httpSocket->get($url . $uri, false, $request);
$response = $httpSocket->get($serverUrl . $uri, false, $request);
}
if (!$response->isOk()) {
if ($httpSocket->lastError()) {
throw new Exception("Failed to get response from $moduleFamily module: " . $httpSocket->lastError['str']);
}
throw new Exception("Failed to get response from $moduleFamily module: HTTP $response->reasonPhrase", (int)$response->code);
$e = new HttpSocketHttpException($response, $serverUrl . $uri);
throw new Exception("Failed to get response from `$moduleFamily` module", 0, $e);
}
return $this->jsonDecode($response->body);
return $response->json();
}
/**
* @param string $moduleFamily
* @return array
* @throws JsonException
*/
public function getModuleSettings($moduleFamily = 'Enrichment')
{

6
app/Model/UserSetting.php
View File

@ -16,11 +16,7 @@ class UserSetting extends AppModel
);
public $validate = array(
'json' => array(
'isValidJson' => array(
'rule' => array('isValidJson'),
)
)
'value' => 'valueIsJson',
);
public $belongsTo = array(