Merge branch '2.4' of github.com:MISP/MISP into fix-mass-edit-form-with-post

pull/6147/head
mokaddem 2020-07-30 15:29:20 +02:00
commit 3d224cfdb8
No known key found for this signature in database
GPG Key ID: 164C473F627A06FA
21 changed files with 203 additions and 66 deletions

2
PyMISP

@ -1 +1 @@
Subproject commit 2fb61d4b32db8ac42c85cc3881f4441c2a7511e6
Subproject commit 706e553f5df9735204acb1aa8868ffec92f9aee9

View File

@ -53,4 +53,84 @@ class StatisticsShell extends AppShell {
}
return $results;
}
public function analyse_slow_logs()
{
$path = $this->args[0];
$raw = file_get_contents($path);
$raw = explode("\n", $raw);
$data = [
'users' => [],
'non_sync_action_users' => [],
'endpoints' => []
];
$this->User = ClassRegistry::init('User');
$users = $this->User->find('list', [
'fields' => ['id', 'email']
]);
foreach ($raw as $line) {
if (empty($line)) {
continue;
}
if ($line[0] === '/' && $line[1] === '*') {
$temp = preg_match('/\/\*\s\[User\:\s([0-9]+)\]/', $line, $matches);
if (!empty($matches[1])) {
$user = $matches[1];
if (isset($data['users'][$user])) {
$data['users'][$user] += 1;
} else {
$data['users'][$user] = 1;
}
}
$temp = preg_match('/\]\s([a-z\:\s]*)/', $line, $matches);
if (!empty($matches[1])) {
$endpoint = $matches[1];
$endpoint = trim($endpoint);
if (isset($data['endpoints'][$endpoint])) {
$data['endpoints'][$endpoint] += 1;
} else {
$data['endpoints'][$endpoint] = 1;
}
}
if (!in_array($endpoint, ['events :: add', 'events :: edit', 'events :: index'])) {
if (isset($data['non_sync_action_users'][$user])) {
$data['non_sync_action_users'][$user] += 1;
} else {
$data['non_sync_action_users'][$user] = 1;
}
}
}
}
arsort($data['endpoints']);
arsort($data['users']);
arsort($data['non_sync_action_users']);
echo "\n\n==================================\nCount | User\n";
echo "\n\n==================================\nSlow queries by user general\n==================================\nCount | User | Email\n";
foreach ($data['users'] as $user_id => $count) {
echo sprintf(
"%s | %s | %s\n",
str_pad($count, 5),
str_pad($user_id, 5),
!empty($users[$user_id]) ? $users[$user_id] : ''
);
}
echo "\n\n==================================\nSlow queries by user excluding sync\n==================================\nCount | User | Email\n";
foreach ($data['non_sync_action_users'] as $user_id => $count) {
echo sprintf(
"%s | %s | %s\n",
str_pad($count, 5),
str_pad($user_id, 5),
!empty($users[$user_id]) ? $users[$user_id] : ''
);
}
echo "\n\n==================================\nSlow queries by endpoint\n==================================\nCount | Endpoint\n";
foreach ($data['endpoints'] as $endpoint => $count) {
echo sprintf(
"%s | %s\n",
str_pad($count, 5),
$endpoint
);
}
echo "==================================\n\n";
}
}

View File

@ -72,7 +72,8 @@ class AttributesController extends AppController
'AttributeTag' => array('Tag'),
'Object' => array(
'fields' => array('Object.id', 'Object.distribution', 'Object.sharing_group_id')
)
),
'SharingGroup' => ['fields' => ['SharingGroup.name']],
);
$this->Attribute->contain(array('AttributeTag' => array('Tag')));
$this->set('isSearch', 0);
@ -83,20 +84,12 @@ class AttributesController extends AppController
}
return $this->RestResponse->viewData($attributes, $this->response->type());
}
$org_ids = array();
$orgs = $this->Attribute->Event->Orgc->find('list', array(
'conditions' => array('Orgc.id' => $org_ids),
'fields' => array('Orgc.id', 'Orgc.name')
list($attributes, $sightingsData) = $this->__searchUI($attributes);
$this->set('sightingsData', $sightingsData);
$orgTable = $this->Attribute->Event->Orgc->find('list', array(
'fields' => array('Orgc.id', 'Orgc.name')
));
if (!$this->_isRest()) {
$temp = $this->__searchUI($attributes);
$this->loadModel('Galaxy');
$this->set('mitreAttackGalaxyId', $this->Galaxy->getMitreAttackGalaxyId());
$attributes = $temp[0];
$sightingsData = $temp[1];
$this->set('sightingsData', $sightingsData);
}
$this->set('orgs', $orgs);
$this->set('orgTable', $orgTable);
$this->set('shortDist', $this->Attribute->shortDist);
$this->set('attributes', $attributes);
$this->set('attrDescriptions', $this->Attribute->fieldDescriptions);
@ -1752,7 +1745,8 @@ class AttributesController extends AppController
'AttributeTag' => array('Tag'),
'Object' => array(
'fields' => array('Object.id', 'Object.distribution', 'Object.sharing_group_id')
)
),
'SharingGroup' => ['fields' => ['SharingGroup.name']],
);
$attributes = $this->paginate();
if (!$this->_isRest()) {
@ -1813,13 +1807,17 @@ class AttributesController extends AppController
$attributes[$k] = $attribute;
}
if ($attribute['Attribute']['distribution'] == 4) {
$attributes[$k]['Attribute']['SharingGroup'] = $attribute['SharingGroup'];
}
$attributes[$k]['Attribute']['AttributeTag'] = $attributes[$k]['AttributeTag'];
$attributes[$k]['Attribute'] = $this->Attribute->Event->massageTags($attributes[$k]['Attribute'], 'Attribute', $excludeGalaxy = false, $cullGalaxyTags = true);
unset($attributes[$k]['AttributeTag']);
$sightingsData = array_merge(
$sightingsData,
$this->Sighting->attachToEvent($attribute, $user, $attributeId, $extraConditions = false)
$this->Sighting->attachToEvent($attribute, $user, $attribute, $extraConditions = false)
);
$correlations = $this->Attribute->Event->getRelatedAttributes($user, $attributeId, false, false, 'attribute');
if (!empty($correlations)) {

View File

@ -12,6 +12,9 @@ class ToolboxComponent extends Component
'recursive' => -1,
'fields' => array($model->alias . '.id')
));
if (empty($data)) {
throw new NotFoundException(__('Invalid %s.', $model->alias));
}
return $data[$model->alias]['id'];
} else {
if (!is_numeric($id)) {

View File

@ -1130,6 +1130,7 @@ class EventsController extends AppController
}
$conditions['includeAllTags'] = true;
$conditions['includeGranularCorrelations'] = 1;
$conditions['includeEventCorrelations'] = false;
if (!empty($filters['includeRelatedTags'])) {
$this->set('includeRelatedTags', 1);
$conditions['includeRelatedTags'] = 1;
@ -1266,9 +1267,12 @@ class EventsController extends AppController
$this->set('advancedFilteringActive', $advancedFiltering['active'] ? 1 : 0);
$this->set('advancedFilteringActiveRules', $advancedFiltering['activeRules']);
$this->set('defaultFilteringRules', $this->defaultFilteringRules);
$orgTable = $this->Event->Orgc->find('list', array(
'fields' => array('Orgc.id', 'Orgc.name')
));
$this->set('orgTable', $orgTable);
$this->disableCache();
$this->layout = 'ajax';
$this->loadModel('Sighting');
$uriArray = explode('/', $this->params->here);
foreach ($uriArray as $k => $v) {
if (strpos($v, ':')) {
@ -1282,7 +1286,6 @@ class EventsController extends AppController
if (!empty($filters['includeSightingdb']) && Configure::read('Plugin.Sightings_sighting_db_enable')) {
$this->set('sightingdbs', $this->Sightingdb->getSightingdbList($this->Auth->user()));
}
$this->set('sightingTypes', $this->Sighting->type);
$this->set('currentUri', $this->params->here);
$this->layout = false;
$this->render('/Elements/eventattribute');
@ -1525,8 +1528,6 @@ class EventsController extends AppController
}
$this->set('contributors', $contributors);
$this->set('typeGroups', array_keys($this->Event->Attribute->typeGroupings));
$this->loadModel('Sighting');
$this->set('sightingTypes', $this->Sighting->type);
$attributeUri = '/events/viewEventAttributes/' . $event['Event']['id'];
foreach ($this->params->named as $k => $v) {
if (!is_numeric($k)) {
@ -3679,8 +3680,13 @@ class EventsController extends AppController
foreach ($resultArray as $key => $result) {
if ($has_pipe = strpos($result['default_type'], '|') !== false || $result['default_type'] === 'malware-sample') {
$pieces = explode('|', $result['value']);
$or = array('Attribute.value1' => $pieces,
'Attribute.value2' => $pieces);
if (in_array($result['default_type'], $this->Event->Attribute->primaryOnlyCorrelatingTypes)) {
$or = array('Attribute.value1' => $pieces[0],
'Attribute.value2' => $pieces[0]);
} else {
$or = array('Attribute.value1' => $pieces,
'Attribute.value2' => $pieces);
}
} else {
$or = array('Attribute.value1' => $result['value'], 'Attribute.value2' => $result['value']);
}
@ -3688,6 +3694,7 @@ class EventsController extends AppController
'conditions' => array('OR' => $or),
'fields' => array('Attribute.type', 'Attribute.category', 'Attribute.value', 'Attribute.comment'),
'order' => false,
'limit' => 11,
'flatten' => 1
);
$resultArray[$key]['related'] = $this->Event->Attribute->fetchAttributes($this->Auth->user(), $options);

View File

@ -419,7 +419,7 @@ class ObjectsController extends AppController
$object = $revisedData['object'];
}
if (!empty($templateData['template'])) {
$template = $this->MispObject->prepareTemplate($templateData['template'], $object);
$template = $this->MispObject->prepareTemplate($templateData['template'], $object);
}
if ($this->request->is('post') || $this->request->is('put')) {
$this->Session->delete('object_being_created');

View File

@ -1,6 +1,9 @@
<?php
App::uses('AppController', 'Controller');
/**
* @property Sighting $Sighting
*/
class SightingsController extends AppController
{
public $components = array('Session', 'RequestHandler');
@ -326,13 +329,13 @@ class SightingsController extends AppController
foreach ($object as $k => $v) {
$eventIds[] = $v['Attribute']['event_id'];
}
$events = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $eventIds));
$events = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $eventIds, 'metadata' => true));
} else {
$attribute_id = false;
// let's set the context to event here, since we reuse the variable later on for some additional lookups.
// Passing $context = 'org' could have interesting results otherwise...
$context = 'event';
$events = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $id));
$events = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $id, 'metadata' => true));
}
if (empty($events)) {
throw new MethodNotAllowedException('Invalid object.');
@ -345,12 +348,9 @@ class SightingsController extends AppController
foreach ($raw as $sighting) {
$results[$sighting['type']][date('Ymd', $sighting['date_sighting'])][] = $sighting;
}
$tsv = 'date\tSighting\tFalse-positive\n';
$dataPoints = array();
$startDate = (date('Ymd'));
$details = array();
$range = (!empty(Configure::read('MISP.Sightings_range')) && is_numeric(Configure::read('MISP.Sightings_range'))) ? Configure::read('MISP.Sightings_range') : 365;
$range = date('Ymd', strtotime("-" . $range . " days", time()));
$startDate = date('Ymd');
$range = date('Ymd', $this->Sighting->getMaximumRange());
foreach ($results as $type => $data) {
foreach ($data as $date => $sighting) {
if ($date < $startDate) {
@ -371,11 +371,10 @@ class SightingsController extends AppController
}
}
$startDate = date('Ymd', strtotime("-3 days", strtotime($startDate)));
$tsv = 'date\tSighting\tFalse-positive\n';
for ($i = $startDate; $i < date('Ymd') + 1; $i++) {
if (checkdate(substr($i, 4, 2), substr($i, 6, 2), substr($i, 0, 4))) {
$tsv .= $i . '\t' . (isset($dataPoints[$i][0]['count']) ? $dataPoints[$i][0]['count'] : 0) . '\t' . (isset($dataPoints[$i][1]['count']) ? $dataPoints[$i][1]['count'] : 0) . '\n';
$details[$i][0] = isset($dataPoints[$i][0]['details']) ? $dataPoints[$i][0]['details'] : array();
$details[$i][1] = isset($dataPoints[$i][1]['details']) ? $dataPoints[$i][1]['details'] : array();
}
}
$this->set('tsv', $tsv);

View File

@ -2656,9 +2656,9 @@ class UsersController extends AppController
}
} else {
if (empty($results['fails']) && !empty($results['successes'])) {
return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'errors' => $message)), 'status'=>200, 'type' => 'json'));
return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => $message)), 'status'=>200, 'type' => 'json'));
} else {
return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'success' => $message)), 'status'=>200, 'type' => 'json'));
return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => $message)), 'status'=>200, 'type' => 'json'));
}
}
}

View File

@ -2914,7 +2914,23 @@ class AppModel extends Model
if (!is_array($decoded)) {
throw new UnexpectedValueException('JSON must be array type, get ' . gettype($decoded));
}
return $decoded;
}
/*
* Temporary solution for utf8 columns until we migrate to utf8mb4
* via https://stackoverflow.com/questions/16496554/can-php-detect-4-byte-encoded-utf8-chars
*/
public function handle4ByteUnicode($input)
{
return preg_replace(
'%(?:
\xF0[\x90-\xBF][\x80-\xBF]{2}
| [\xF1-\xF3][\x80-\xBF]{3}
| \xF4[\x80-\x8F][\x80-\xBF]{2}
)%xs',
'?',
$input
);
}
}

View File

@ -731,7 +731,7 @@ class Attribute extends AppModel
'perm_site_admin' => 1
)
);
$attribute['Attribute']['Sighting'] = $this->Sighting->attachToEvent($attribute, $user, $this->id);
$attribute['Attribute']['Sighting'] = $this->Sighting->attachToEvent($attribute, $user, $attribute);
if (empty($attribute['Object']['id'])) {
unset($attribute['Object']);
}
@ -1467,6 +1467,7 @@ class Attribute extends AppModel
// do some last second modifications before the validation
public function modifyBeforeValidation($type, $value)
{
$value = $this->handle4ByteUnicode($value);
switch ($type) {
case 'md5':
case 'sha1':

View File

@ -792,6 +792,9 @@ class MispObject extends AppModel
if (isset($objectToSave['Object']['comment'])) {
$object['Object']['comment'] = $objectToSave['Object']['comment'];
}
if (isset($objectToSave['Object']['template_version'])) {
$object['Object']['template_version'] = $objectToSave['Object']['template_version'];
}
if (isset($objectToSave['Object']['distribution'])) {
$object['Object']['distribution'] = $objectToSave['Object']['distribution'];
if ($object['Object']['distribution'] == 4) {

View File

@ -2714,7 +2714,7 @@ class Server extends AppModel
foreach ($filter_rules as $field => $rules) {
$temp = array();
if ($field === 'url_params') {
$url_params = $this->jsonDecode($rules);
$url_params = empty($rules) ? [] : $this->jsonDecode($rules);
} else {
foreach ($rules as $operator => $elements) {
foreach ($elements as $k => $element) {

View File

@ -3,6 +3,9 @@ App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
App::uses('TmpFileTool', 'Tools');
/**
* @property Attribute $Attribute
*/
class Sighting extends AppModel
{
public $useTable = 'sightings';
@ -201,32 +204,41 @@ class Sighting extends AppModel
return $result;
}
public function attachToEvent($event, $user = array(), $attribute_id = false, $extraConditions = false)
/**
* @param array $event
* @param array $user
* @param array|int|null $attribute Attribute array or attribute ID
* @param bool $extraConditions
* @return array|int
*/
public function attachToEvent(array $event, array $user, $attribute = null, $extraConditions = false)
{
if (empty($user)) {
$user = array(
'org_id' => -1,
'Role' => array(
'perm_site_admin' => 0
)
);
}
$ownEvent = false;
if ($user['Role']['perm_site_admin'] || $event['Event']['org_id'] == $user['org_id']) {
$ownEvent = true;
}
$contain = [];
$conditions = array('Sighting.event_id' => $event['Event']['id']);
if ($attribute_id) {
$attribute = $this->Attribute->fetchAttribute($attribute_id);
$conditions['Sighting.attribute_id'] = $attribute_id;
if (is_array($attribute)) {
$conditions['Sighting.attribute_id'] = $attribute['Attribute']['id'];
} elseif (is_numeric($attribute)) {
$conditions['Sighting.attribute_id'] = $attribute;
$attribute = $this->Attribute->find('first', [
'recursive' => -1,
'conditions' => ['Attribute.id' => $attribute],
'fields' => ['Attribute.uuid']
]);
} else {
$contain['Attribute'] = ['fields' => 'Attribute.uuid'];
}
if (!$ownEvent && (!Configure::read('Plugin.Sightings_policy') || Configure::read('Plugin.Sightings_policy') == 0)) {
$conditions['Sighting.org_id'] = $user['org_id'];
}
if ($extraConditions !== false) {
$conditions['AND'] = $extraConditions;
}
$contain = array();
if (Configure::read('MISP.showorg')) {
$contain['Organisation'] = array('fields' => array('Organisation.id', 'Organisation.uuid', 'Organisation.name'));
}
@ -265,10 +277,11 @@ class Sighting extends AppModel
$sightings[$k]['Sighting']['Organisation'] = $sightings[$k]['Organisation'];
}
// zeroq: add attribute UUID to sighting to make synchronization easier
if (!isset($attribute)) {
$attribute = $this->Attribute->fetchAttribute($sighting['Sighting']['attribute_id']);
if (isset($sighting['Attribute']['uuid'])) {
$sightings[$k]['Sighting']['attribute_uuid'] = $sighting['Attribute']['uuid'];
} else {
$sightings[$k]['Sighting']['attribute_uuid'] = $attribute['Attribute']['uuid'];
}
$sightings[$k]['Sighting']['attribute_uuid'] = $attribute['Attribute']['uuid'];
$sightings[$k] = $sightings[$k]['Sighting'] ;
}

View File

@ -439,8 +439,8 @@ class Tag extends AppModel
public function checkForOverride($tags)
{
$userId = Configure::read('CurrentUserId');
$this->UserSetting = ClassRegistry::init('UserSetting');
if ($this->tagOverrides === false && $userId > 0) {
$this->UserSetting = ClassRegistry::init('UserSetting');
$this->tagOverrides = $this->UserSetting->getTagNumericalValueOverride($userId);
}
foreach ($tags as $k => $tag) {

View File

@ -172,7 +172,7 @@
$fieldCount += 1;
}
if ($includeDecayScore) {
sprintf(
echo sprintf(
'<th class="decayingScoreField" title="%s">%s</th>',
__('Decaying Score'),
__('Score')

View File

@ -88,7 +88,7 @@
</td>
<td class="shortish">
<?php
foreach ($item['related'] as $relation):
foreach (array_slice($item['related'], 0, 10) as $relation):
$popover = array(
'Event ID' => $relation['Event']['id'],
'Event Info' => $relation['Event']['info'],
@ -105,7 +105,7 @@
<a href="<?php echo $baseurl; ?>/events/view/<?php echo h($relation['Event']['id']);?>" data-toggle="popover" title="Attribute details" data-content="<?php echo h($popoverHTML); ?>" data-trigger="hover"><?php echo h($relation['Event']['id']);?></a>
<?php
endforeach;
$correlationPopover = array('<span>', );
echo count($item['related']) > 10 ? sprintf('<div><i class="muted">%s</i></div>', __('10 +more')) : '';
?>
</td>
<td class="short">

View File

@ -14,7 +14,7 @@
if ($action == 'edit' && !$update_template_available && $newer_template_version !== false): ?>
<a class="btn btn-mini btn-primary useCursorPointer" title="<?php echo __('Update the template of this object to the newer version: ') . h($newer_template_version) ?>" href="<?php echo $baseurl . '/objects/edit/' . h($object['Object']['id']) . '/1'; ?>">
<span class="fa fa-arrow-circle-up"></span>
<?php echo __('Update template') ?>
<?php echo __('Update template to v%s', h($newer_template_version)) ?>
</a>
<?php endif; ?>
&nbsp;
@ -88,6 +88,14 @@
'div' => 'input hidden',
'required' => false,
));
if ($update_template_available && $newer_template_version !== false) {
echo $this->Form->input('template_version', array(
'type' => 'text',
'div' => 'input hidden',
'required' => false,
'value' => $newer_template_version
));
}
?>
<div id="bothSeenSliderContainer"></div>
</dl>
@ -193,7 +201,7 @@
<?php if ($update_template_available): ?>
<div class="fixedRightPanelHeader useCursorPointer" style="box-shadow: 0px 0px 6px #B2B2B2;margin-bottom: 2px;width: 100%;overflow: hidden; padding: 5px;">
<i class="fas fa-chevron-circle-down"></i>
<span style="margin-left: 5px; display: inline-block; font-size: large;"><?php echo __('Pre-update object\'s template'); ?></span>
<span style="margin-left: 5px; display: inline-block; font-size: large;"><?php echo __('Current Object state on older template version'); ?></span>
</div>
<div class="row" style="max-height: 800px; max-width: 800px; overflow: auto; padding: 15px;">
<div style="border: 1px solid #3465a4 ; border-radius: 5px; overflow: hidden;" class="span5">

View File

@ -31,6 +31,7 @@
<?php
echo $this->Form->input('enable_password', array('type' => 'checkbox', 'label' => __('Set password')));
?>
<a class="useCursorPointer" onclick="$('#resetAuthKeyForm').submit();"><?= __('Reset Auth Key') ?></a>
<div id="PasswordDiv">
<div class="clear"></div>
<?php
@ -70,8 +71,6 @@
echo $this->Form->input('change_pw', array('type' => 'checkbox', 'label' => __('Change Password')));
echo $this->Form->input('autoalert', array('label' => __('Receive alerts when events are published'), 'type' => 'checkbox'));
echo $this->Form->input('contactalert', array('label' => __('Receive alerts from "contact reporter" requests'), 'type' => 'checkbox'));
echo $this->Html->link(__('Reset Auth Key'), array('controller' => 'users', 'action' => 'resetauthkey', $currentId));
?>
<div class="clear"></div>
<?php
@ -89,7 +88,13 @@
</div>
<?php
echo $this->Form->button(__('Submit'), array('class' => 'btn btn-primary'));
echo $this->Form->end();?>
echo $this->Form->end();
echo $this->Form->create('User', array(
'url' => array('controller' => 'users', 'action' => 'resetauthkey', $id),
'id' => 'resetAuthKeyForm'
));
echo $this->Form->end();
?>
</div>
<?php
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'admin', 'menuItem' => 'editUser'));

@ -1 +1 @@
Subproject commit 1acdef286a3ac6f188f5028c6d1c0dd788dbadc4
Subproject commit 4636dc5640d6069fef23891734ece89a9c61dfa4

View File

@ -1329,6 +1329,9 @@ function handleAjaxModalResponse(response, context_id, url, referer, context, co
showMessage("fail", responseArray.errors);
}
} else {
if (responseArray.errors) {
showMessage("fail", responseArray.errors);
}
var savedArray = saveValuesForPersistance();
$.ajax({
async:true,
@ -3355,7 +3358,7 @@ function testConnection(id) {
$("#connection_test_" + id).html('<span class="red bold" title="The user account on the remote instance is not a sync user.">Remote user not a sync user</span>');
break;
case 8:
$("#connection_test_" + id).html('<span class="orange bold" title="The user account on the remote instance is only a sightings user.">Syncing sightings only</span>');
$("#connection_test_" + id).html('<span class="orange bold" title="The user account on the remote instance is only a sightings user.">Remote user not a sync user, syncing sightings only</span>');
break;
}
}

View File

@ -6876,7 +6876,8 @@
],
"feeds": [
"id",
"input_source"
"input_source",
"orgc_id"
],
"fuzzy_correlate_ssdeep": [
"id",