Merge branch '2.4' of github.com:MISP/MISP into 2.4

pull/5417/head
mokaddem 2019-11-18 19:08:51 -05:00
commit 3a8d283e46
No known key found for this signature in database
GPG Key ID: 164C473F627A06FA
18 changed files with 285 additions and 72 deletions

View File

@ -46,7 +46,7 @@ class AppController extends Controller
public $helpers = array('Utility', 'OrgImg', 'FontAwesome', 'UserName');
private $__queryVersion = '91';
private $__queryVersion = '92';
public $pyMispVersion = '2.4.117';
public $phpmin = '7.0';
public $phprec = '7.2';

View File

@ -1662,7 +1662,7 @@ class AttributesController extends AppController
'request' => $this->request,
'named_params' => $this->params['named'],
'paramArray' => $paramArray,
'ordered_url_params' => compact($paramArray),
'ordered_url_params' => @compact($paramArray),
'additional_delimiters' => PHP_EOL
);
$exception = false;
@ -1877,7 +1877,7 @@ class AttributesController extends AppController
'request' => $this->request,
'named_params' => $this->params['named'],
'paramArray' => $paramArray,
'ordered_url_params' => compact($paramArray)
'ordered_url_params' => @compact($paramArray)
);
$validFormats = $this->Attribute->validFormats;
$exception = false;

View File

@ -370,6 +370,7 @@ class ACLComponent extends Component
'getGit' => array(),
'getInstanceUUID' => array('perm_sync'),
'getPyMISPVersion' => array('*'),
'getRemoteUser' => array(),
'getSetting' => array(),
'getSubmodulesStatus' => array(),
'getSubmoduleQuickUpdateForm' => array(),

View File

@ -22,13 +22,13 @@ class RestResponseComponent extends Component
'add' => array(
'description' => "POST a MISP Attribute JSON to this API to create an Attribute.",
'mandatory' => array('value', 'type'),
'optional' => array('category', 'to_ids', 'uuid', 'distribution', 'sharing_group_id', 'timestamp', 'comment'),
'optional' => array('category', 'to_ids', 'uuid', 'distribution', 'sharing_group_id', 'timestamp', 'comment', 'data', 'encrypt'),
'params' => array('event_id')
),
'edit' => array(
'description' => "POST a MISP Attribute JSON to this API to update an Attribute. If the timestamp is set, it has to be newer than the existing Attribute.",
'mandatory' => array(),
'optional' => array('value', 'type', 'category', 'to_ids', 'uuid', 'distribution', 'sharing_group_id', 'timestamp', 'comment'),
'optional' => array('value', 'type', 'category', 'to_ids', 'uuid', 'distribution', 'sharing_group_id', 'timestamp', 'comment', 'date', 'encrypt'),
'params' => array('attribute_id')
),
'deleteSelected' => array(
@ -721,6 +721,12 @@ class RestResponseComponent extends Component
'autoclose' => true
),
),
'data' => array(
'input' => 'text',
'type' => 'string',
'operators' => array('equal'),
'help' => __('Base64 encoded file contents')
),
'date' => array(
'type' => 'date',
'validation' => array( 'format' => 'YYYY-MM-DD' ),
@ -820,6 +826,12 @@ class RestResponseComponent extends Component
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' )
),
'encrypt' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' ),
'help' => __('When uploading malicious samples, set this flag to tell MISP to encrpyt the sample and extract the file hashes. This will create a MISP object with the appropriate attributes.')
),
//'enforceWarningList' => array(
// 'input' => 'radio',
// 'type' => 'integer',

View File

@ -3205,7 +3205,7 @@ class EventsController extends AppController
'request' => $this->request,
'named_params' => $this->params['named'],
'paramArray' => $paramArray,
'ordered_url_params' => compact($paramArray)
'ordered_url_params' => @compact($paramArray)
);
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception);
@ -3497,7 +3497,7 @@ class EventsController extends AppController
'request' => $this->request,
'named_params' => $this->params['named'],
'paramArray' => $paramArray,
'ordered_url_params' => compact($paramArray)
'ordered_url_params' => @compact($paramArray)
);
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception);
@ -3543,7 +3543,6 @@ class EventsController extends AppController
$filename .= '.' . $responseType;
return $this->RestResponse->viewData($final, $responseType, false, true, $filename, array('X-Result-Count' => $elementCounter, 'X-Export-Module-Used' => $returnFormat, 'X-Response-Format' => $responseType));
}
}
public function downloadOpenIOCEvent($key, $eventid, $enforceWarninglist = false)

View File

@ -198,6 +198,7 @@ class ObjectsController extends AppController
$this->MispObject->Event->insertLock($this->Auth->user(), $eventId);
}
$error = false;
$template = false;
if (!empty($templateId) || !$this->_isRest()) {
$templates = $this->MispObject->ObjectTemplate->find('all', array(
'conditions' => array('ObjectTemplate.id' => $templateId),
@ -247,10 +248,14 @@ class ObjectsController extends AppController
foreach ($object['Attribute'] as $k => $attribute) {
unset($object['Attribute'][$k]['id']);
$object['Attribute'][$k]['event_id'] = $eventId;
$this->MispObject->Event->Attribute->set($attribute);
$this->MispObject->Event->Attribute->set($object['Attribute'][$k]);
if (!$this->MispObject->Event->Attribute->validates()) {
if ($this->MispObject->Event->Attribute->validationErrors['value'][0] !== 'Composite type found but the value not in the composite (value1|value2) format.') {
$error = 'Could not save object as at least one attribute has failed validation (' . $attribute['object_relation'] . '). ' . json_encode($this->MispObject->Event->Attribute->validationErrors);
$error = sprintf(
'Could not save object as at least one attribute has failed validation (%s). %s',
isset($attribute['object_relation']) ? $attribute['object_relation'] : 'No object_relation',
json_encode($this->MispObject->Event->Attribute->validationErrors)
);
}
}
}

View File

@ -1470,6 +1470,20 @@ class ServersController extends AppController
}
}
public function getRemoteUser($id)
{
$this->Server->id = $id;
if (!$this->Server->exists()) {
throw new NotFoundException(__('Invalid server'));
}
$user = $this->Server->getRemoteUser($id);
if (empty($user)) {
throw new NotFoundException(__('Invalid user or user not found.'));
} else {
return $this->RestResponse->viewData($user);
}
}
public function testConnection($id = false)
{
if (!$this->Auth->user('Role')['perm_sync'] && !$this->Auth->user('Role')['perm_site_admin']) {

View File

@ -1297,6 +1297,10 @@ class AppModel extends Model
break;
case 43:
$sqlArray[] = "ALTER TABLE sightingdbs ADD namespace varchar(255) DEFAULT '';";
break;
case 44:
$sqlArray[] = "ALTER TABLE object_template_elements CHANGE `disable_correlation` `disable_correlation` tinyint(1);";
break;
case 'fixNonEmptySharingGroupID':
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
@ -1658,7 +1662,7 @@ class AppModel extends Model
'model' => 'Server',
'model_id' => 0,
'email' => 'SYSTEM',
'action' => 'update_database_worker',
'action' => 'update_db_worker',
'user_id' => 0,
'title' => __('Issues executing run_updates'),
'change' => __('Database updates are locked. Worker not spawned')
@ -1716,7 +1720,7 @@ class AppModel extends Model
'model' => 'Server',
'model_id' => 0,
'email' => 'SYSTEM',
'action' => 'update_database_worker',
'action' => 'update_db_worker',
'user_id' => 0,
'title' => __('Issues executing run_updates'),
'change' => __('Updates are locked. Stopping worker gracefully')

View File

@ -813,9 +813,11 @@ class Attribute extends AppModel
{
parent::beforeValidate();
if (!isset($this->data['Attribute']['type'])) {
$this->validationErrors['type'] = ['No type set.'];
return false;
}
if (is_array($this->data['Attribute']['value'])) {
$this->validationErrors['type'] = ['Value is an array.'];
return false;
}
App::uses('ComplexTypeTool', 'Tools');
@ -823,6 +825,7 @@ class Attribute extends AppModel
$this->data['Attribute']['value'] = $this->complexTypeTool->refangValue($this->data['Attribute']['value'], $this->data['Attribute']['type']);
if (!empty($this->data['Attribute']['object_id']) && empty($this->data['Attribute']['object_relation'])) {
$this->validationErrors['type'] = ['Object attribute sent, but no object_relation set.'];
return false;
}
// remove leading and trailing blanks

View File

@ -2157,7 +2157,7 @@ class Event extends AppModel
}
$event['Attribute'] = $this->Feed->attachFeedCorrelations($event['Attribute'], $user, $event['Event'], $overrideLimit);
}
if (!empty($options['includeServerCorrelations']) && $user['org_id'] == Configure::read('MISP.host_org_id')) {
if (!empty($options['includeServerCorrelations']) && ($user['Role']['perm_site_admin'] || $user['org_id'] == Configure::read('MISP.host_org_id'))) {
$this->Feed = ClassRegistry::init('Feed');
if (!empty($options['overrideLimit'])) {
$overrideLimit = true;

View File

@ -16,52 +16,55 @@ class Log extends AppModel
);
public $validate = array(
'action' => array(
'rule' => array('inList', array(
'accept',
'accept_delegation',
'add',
'admin_email',
'auth',
'auth_fail',
'blacklisted',
'change_pw',
'delete',
'disable',
'discard',
'edit',
'email',
'enable',
'error',
'export',
'file_upload',
'galaxy',
'include_formula',
'login',
'login_fail',
'logout',
'merge',
'pruneUpdateLogs',
'publish',
'publish alert',
'pull',
'purge_events',
'push',
'remove_dead_workers',
'request',
'request_delegation',
'reset_auth_key',
'security',
'serverSettingsEdit',
'tag',
'undelete',
'update',
'update_database',
'update_database_worker',
'upgrade_24',
'upload_sample',
'version_warning',
'warning'
)),
'rule' => array(
'inList',
array( // ensure that the length of the rules is < 20 in length
'accept',
'accept_delegation',
'add',
'admin_email',
'auth',
'auth_fail',
'blacklisted',
'change_pw',
'delete',
'disable',
'discard',
'edit',
'email',
'enable',
'error',
'export',
'file_upload',
'galaxy',
'include_formula',
'login',
'login_fail',
'logout',
'merge',
'pruneUpdateLogs',
'publish',
'publish alert',
'pull',
'purge_events',
'push',
'remove_dead_workers',
'request',
'request_delegation',
'reset_auth_key',
'security',
'serverSettingsEdit',
'tag',
'undelete',
'update',
'update_database',
'update_db_worker',
'upgrade_24',
'upload_sample',
'version_warning',
'warning'
)
),
'message' => 'Options : ...'
)
);

View File

@ -4293,9 +4293,69 @@ class Server extends AppModel
} else {
$schemaDiagnostic['error'] = sprintf('Diagnostic not available for DataSource `%s`', $dataSource);
}
if (!empty($schemaDiagnostic['diagnostic'])) {
foreach ($schemaDiagnostic['diagnostic'] as $table => &$fields) {
foreach ($fields as &$field) {
$field = $this->__attachRecoveryQuery($field, $table);
}
}
}
return $schemaDiagnostic;
}
/*
* Work in progress, still needs DEFAULT in the schema for it to work correctly
* Currently only works for missing_column and column_different
* Only currently supported field types are: int, tinyint, varchar, text
*/
private function __attachRecoveryQuery($field, $table)
{
if ($field['is_critical']) {
$length = false;
if (in_array($field['error_type'], array('missing_column', 'column_different'))) {
if ($field['expected']['data_type'] === 'int') {
$length = 11;
} elseif ($field['expected']['data_type'] === 'tinyint') {
$length = 1;
} elseif ($field['expected']['data_type'] === 'varchar') {
$length = $field['expected']['character_maximum_length'];
} elseif ($field['expected']['data_type'] === 'text') {
$length = null;
}
}
if ($length !== false) {
switch($field['error_type']) {
case 'missing_column':
$field['sql'] = sprintf(
'ALTER TABLE `%s` ADD COLUMN `%s` %s%s %s %s;',
$table,
$field['column_name'],
$field['expected']['data_type'],
$length !== null ? sprintf('(%d)', $length) : '',
isset($field['expected']['default']) ? 'DEFAULT "' . $field['expected']['default'] . '"' : '',
$field['expected']['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL',
empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name']
);
break;
case 'column_different':
$field['sql'] = sprintf(
'ALTER TABLE `%s` CHANGE `%s` `%s` %s%s %s %s;',
$table,
$field['column_name'],
$field['column_name'],
$field['expected']['data_type'],
$length !== null ? sprintf('(%d)', $length) : '',
isset($field['expected']['default']) ? 'DEFAULT "' . $field['expected']['default'] . '"' : '',
$field['expected']['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL',
empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name']
);
break;
}
}
}
return $field;
}
public function getExpectedDBSchema()
{
App::uses('Folder', 'Utility');
@ -4370,6 +4430,7 @@ class Server extends AppModel
if (!array_key_exists($tableName, $dbActualSchema)) {
$dbDiff[$tableName][] = array(
'description' => sprintf(__('Table `%s` does not exist'), $tableName),
'error_type' => 'missing_table',
'column_name' => $tableName,
'is_critical' => true
);
@ -4395,6 +4456,7 @@ class Server extends AppModel
}
$dbDiff[$tableName][] = array(
'description' => sprintf(__('Column `%s` exists but should not'), $additionalKeys),
'error_type' => 'additional_column',
'column_name' => $additionalKeys,
'is_critical' => false
);
@ -4417,6 +4479,7 @@ class Server extends AppModel
$dbDiff[$tableName][] = array(
'description' => sprintf(__('Column `%s` is different'), $columnName),
'column_name' => $column['column_name'],
'error_type' => 'column_different',
'actual' => $keyedActualColumn[$columnName],
'expected' => $column,
'is_critical' => $isCritical
@ -4426,6 +4489,7 @@ class Server extends AppModel
$dbDiff[$tableName][] = array(
'description' => sprintf(__('Column `%s` does not exist but should'), $columnName),
'column_name' => $columnName,
'error_type' => 'missing_column',
'actual' => array(),
'expected' => $column,
'is_critical' => true
@ -4438,6 +4502,7 @@ class Server extends AppModel
$dbDiff[$additionalTable][] = array(
'description' => sprintf(__('Table `%s` is an additional table'), $additionalTable),
'column_name' => $additionalTable,
'error_type' => 'additional_table',
'is_critical' => false
);
}
@ -5542,4 +5607,47 @@ class Server extends AppModel
}
return $success;
}
public function getRemoteUser($id)
{
$server = $this->find('first', array(
'conditions' => array('Server.id' => $id),
'recursive' => -1
));
$HttpSocket = $this->setupHttpSocket($server);
$request = $this->setupSyncRequest($server);
$uri = $server['Server']['url'] . '/users/view/me.json';
try {
$response = $HttpSocket->get($uri, false, $request);
} catch (Exception $e) {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$message = __('Could not reset fetch remote user account.');
$this->Log->save(array(
'org' => 'SYSTEM',
'model' => 'Server',
'model_id' => $id,
'email' => 'SYSTEM',
'action' => 'error',
'user_id' => 0,
'title' => 'Error: ' . $message,
));
return $message;
}
if ($response->isOk()) {
$user = json_decode($response->body, true);
if (!empty($user['User'])) {
$result = array(
'Email' => $user['User']['email'],
'Role name' => isset($user['Role']['name']) ? $user['Role']['name'] : 'Unknown, outdated instance',
'Sync flag' => isset($user['Role']['perm_sync']) ? ($user['Role']['perm_sync'] ? 1 : 0) : 'Unknown, outdated instance'
);
return $result;
} else {
return __('No user object received in response.');
}
} else {
return $response->code;
}
}
}

View File

@ -2,7 +2,13 @@
$data_elements = Hash::extract($row, $field['data_path']);
$links = array();
foreach ($data_elements as $data) {
if (strpos($field['url'], '%s') !== false) {
if (!empty($data['name'])) {
$field['title'] = $data['name'];
}
if (!empty($data['url'])) {
$data = $data['url'];
}
if (isset($field['url']) && strpos($field['url'], '%s') !== false) {
$url = sprintf(
$field['url'],
$data
@ -14,7 +20,7 @@
'<a href="%s" title="%s">%s</a>',
h($url),
empty($field['title']) ? h($data) : h($field['title']),
h($data)
empty($field['title']) ? h($data) : h($field['title'])
);
}
echo implode('<br />', $links);

View File

@ -22,6 +22,7 @@
<th><?php echo $this->Paginator->sort('name');?></th>
<th><?php echo __('Prio');?></th>
<th><?php echo __('Connection test');?></th>
<th><?php echo __('Sync user');?></th>
<th><?php echo __('Reset API key');?></th>
<th><?php echo $this->Paginator->sort('internal');?></th>
<th><?php echo $this->Paginator->sort('push');?></th>
@ -85,7 +86,8 @@ foreach ($servers as $row_pos => $server):
<td id="priority_<?php echo $server['Server']['id'];?>">
<?php echo $arrows;?>
</td>
<td id="connection_test_<?php echo $server['Server']['id'];?>"><span role="button" tabindex="0" aria-label="<?php echo __('Test the connection to the remote instance');?>" title="<?php echo __('Test the connection to the remote instance');?>" class="btn btn-primary" style="line-height:10px; padding: 4px 4px;" onClick="testConnection('<?php echo $server['Server']['id'];?>');"><?php echo __('Run');?></span></td>
<td class="short" id="connection_test_<?php echo $server['Server']['id'];?>"><span role="button" tabindex="0" aria-label="<?php echo __('Test the connection to the remote instance');?>" title="<?php echo __('Test the connection to the remote instance');?>" class="btn btn-primary" style="line-height:10px; padding: 4px 4px;" onClick="testConnection('<?php echo $server['Server']['id'];?>');"><?php echo __('Run');?></span></td>
<td class="short" id="sync_user_test_<?php echo $server['Server']['id'];?>"><span role="button" tabindex="0" aria-label="<?php echo __('View the sync user of the remote instance');?>" title="<?php echo __('View the sync user of the remote instance');?>" class="btn btn-primary" style="line-height:10px; padding: 4px 4px;" onClick="getRemoteSyncUser('<?php echo $server['Server']['id'];?>');"><?php echo __('View');?></span></td>
<td id="reset_api_key_<?php echo $server['Server']['id'];?>">
<?php
echo $this->Form->postLink(

View File

@ -1267,7 +1267,7 @@ class StixBuilder():
return pattern
@staticmethod
def resolve_stix2_pattern(attributes):
def resolve_stix2_pattern(attributes, _):
for attribute in attributes:
if attribute['object_relation'] == 'stix2-pattern':
return attribute['value']

View File

@ -305,6 +305,7 @@ objectsMapping = {'asn': {'to_call': 'handle_usual_object_name',
'registry-key': {'to_call': 'handle_usual_object_name',
'observable': {'0': {'type': 'windows-registry-key'}},
'pattern': "windows-registry-key:{0} = '{1}'"},
'stix2-pattern': {'to_call': 'handle_usual_object_name'},
'url': {'to_call': 'handle_usual_object_name',
'observable': {'0': {'type': 'url'}},
'pattern': "url:{0} = '{1}'"},

View File

@ -1149,6 +1149,7 @@ class ExternalStixParser(StixParser):
('windows-registry-key',): self.parse_regkey_pattern,
('x509-certificate',): self.parse_x509_pattern}
self.pattern_forbidden_relations = (' LIKE ', ' FOLLOWEDBY ', ' MATCHES ', ' ISSUBSET ', ' ISSUPERSET ', ' REPEATS ')
self.single_attribute_fields = ('type', 'value', 'to_ids')
def handler(self):
self.version_attribute = {'type': 'text', 'object_relation': 'version', 'value': self.stix_version}
@ -1180,16 +1181,15 @@ class ExternalStixParser(StixParser):
def parse_external_indicator(self, indicator):
pattern = indicator.pattern
# Deeper analyse of patterns coming when we get examples
attribute = {'type': 'stix2-pattern', 'object_relation': 'stix2-pattern', 'value': pattern}
misp_object = {'name': 'stix2-pattern', 'meta-category': 'stix2-pattern',
'Attribute': [self.version_attribute, attribute]}
self.misp_event.add_object(**misp_object)
indicator_id = indicator.id.split('--')[1]
if hasattr(indicator, 'object_marking_refs'):
self.parse_external_pattern(pattern, indicator_id, marking=indicator.object_marking_refs)
else:
self.parse_external_pattern(pattern, indicator_id)
try:
if hasattr(indicator, 'object_marking_refs'):
self.parse_external_pattern(pattern, indicator_id, marking=indicator.object_marking_refs)
else:
self.parse_external_pattern(pattern, indicator_id)
# Deeper analyse of patterns coming when we get examples
except Exception:
self.add_stix2_pattern_object(pattern, indicator_id)
def parse_external_observable(self, observable):
objects = observable.objects
@ -1225,6 +1225,7 @@ class ExternalStixParser(StixParser):
self.pattern_mapping[type_]([p.strip()], marking)
except KeyError:
print('{} not parsed at the moment'.format(type_), file=sys.stderr)
raise Exception
else:
pattern = [p.strip() for p in pattern.split(' AND ')]
types = self.parse_external_pattern_types(pattern)
@ -1232,6 +1233,9 @@ class ExternalStixParser(StixParser):
self.pattern_mapping[types](pattern, marking, uuid=uuid)
except KeyError:
print('{} not parsed at the moment'.format(types), file=sys.stderr)
raise Exception
else:
self.add_stix2_pattern_object(pattern, uuid)
@staticmethod
def parse_external_pattern_types(pattern):
@ -1460,6 +1464,13 @@ class ExternalStixParser(StixParser):
## UTILITY FUNCTIONS. ##
################################################################################
def add_stix2_pattern_object(self, pattern, uuid):
attribute = {'type': 'stix2-pattern', 'object_relation': 'stix2-pattern', 'value': pattern}
misp_object = {'name': 'stix2-pattern', 'meta-category': 'stix2-pattern',
'Attribute': [self.version_attribute, attribute],
'uuid': uuid}
self.misp_event.add_object(**misp_object)
@staticmethod
def create_misp_object(attributes, name, uuid=None):
misp_object = MISPObject(name, misp_objects_path_custom=_MISP_objects_path)
@ -1525,6 +1536,7 @@ class ExternalStixParser(StixParser):
def handle_import_case(self, attributes, name, marking=None, uuid=None):
if len(attributes) == 1:
attribute = attributes[0]
attribute = {field: attribute[field] for field in self.single_attribute_fields if attribute.get(field)}
attribute['uuid'] = uuid
if marking:
attribute = self.add_tag_in_attribute(attribute, marking)

View File

@ -3052,6 +3052,49 @@ function runOnDemandAction(element, url, target, postFormField) {
})
}
function getRemoteSyncUser(id) {
var resultContainer = $("#sync_user_test_" + id);
$.ajax({
url: '/servers/getRemoteUser/' + id,
type:'GET',
beforeSend: function (XMLHttpRequest) {
resultContainer.html('Running test...');
},
error: function(){
resultContainer.html('Internal error.');
},
success: function(response) {
if (typeof(response.message) != 'undefined') {
resultContainer.empty();
resultContainer.append(
$('<span>')
.attr('class', 'red bold')
.text('Error')
).append(
$('<span>')
.text(': #' + response.message)
);
} else {
resultContainer.empty();
Object.keys(response).forEach(function(key) {
var value = response[key];
resultContainer.append(
$('<span>')
.attr('class', 'blue bold')
.text(key)
).append(
$('<span>')
.text(': ' + value)
).append(
$('<br>')
);
});
}
var result = response;
}
});
}
function testConnection(id) {
$.ajax({
url: '/servers/testConnection/' + id,