MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: Performance tuning 

- improved performance of inserting batch attributes / passing a large number of attributes to attributes/add
  - reworked algorithm to a two phase bulk insertion (validation -> mass insert) instead of looping through all attributes
  - removed the build in counter cache for incrementing attribute counts on events in favour of a more lightweight solution
  - performance gains on test data set: 50+ seconds -> 32 seconds

- Greatly improved attribute index / attribute search performance
  - fixed an issue that caused the lookup to avoid using indeces
  - performance gains on test data when paginating: 11 seconds -> 1 second
pull/2831/head
iglocska 2018-01-13 14:23:04 +01:00
parent 86713b32ac
commit a432f8358e
3 changed files with 96 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
app/Controller/AttributesController.php
View File

@ -9,10 +9,8 @@ class AttributesController extends AppController {
public $paginate = array(
'limit' => 60,
'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events
'conditions' => array('AND' => array('NOT' => array('Event.id' => 0), 'Attribute.deleted' => 0)),
// Don't sort by default. It's crazy expensive and not really needed in most cases.
// Users can still sort on demand
'maxLimit' => 9999,
'conditions' => array('AND' => array('Attribute.deleted' => 0)),
'order' => 'Attribute.event_id DESC'
);
@ -52,13 +50,12 @@ class AttributesController extends AppController {
}
// do not show private to other orgs
if (!$this->_isSiteAdmin()) {
// TEMP: change to passing an options array with the user!!
$this->paginate = Set::merge($this->paginate, array('conditions' => $this->Attribute->buildConditions($this->Auth->user())));
}
}
public function index() {
$this->Attribute->recursive = 2;
$this->Attribute->recursive = -1;
$this->paginate['contain'] = array(
'Event' => array(
'fields' => array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.info', 'Event.user_id')
@ -71,6 +68,10 @@ class AttributesController extends AppController {
$attributes = $this->paginate();
$org_ids = array();
foreach ($attributes as $k => $attribute) {
if (empty($attribute['Event']['id'])) {
unset($attribute[$k]);
continue;
}
if ($attribute['Attribute']['type'] == 'attachment' && preg_match('/.*\.(jpg|png|jpeg|gif)$/i', $attribute['Attribute']['value'])) {
$attributes[$k]['Attribute']['image'] = $this->Attribute->base64EncodeAttachment($attribute['Attribute']);
}
@ -174,7 +175,7 @@ class AttributesController extends AppController {
}
}
$fails = array();
$successes = array();
$successes = 0;
$attributeCount = count($attributes);
if (!empty($uuids)) {
$existingAttributes = $this->Attribute->find('list', array(
@ -192,27 +193,52 @@ class AttributesController extends AppController {
}
}
}
// deduplication
$duplicates = 0;
foreach ($attributes as $k => $attribute) {
foreach ($attributes as $k2 => $attribute2) {
if ($k == $k2) continue;
if (
(
!empty($attribute['uuid']) &&
!empty($attribute2['uuid']) &&
$attribute['uuid'] == $attribute2['uuid']
) || (
$attribute['value'] == $attribute2['value'] &&
$attribute['type'] == $attribute2['type'] &&
$attribute['category'] == $attribute2['category']
)
) {
$duplicates++;
unset($attributes[$k]);
break;
}
}
}
foreach ($attributes as $k => $attribute) {
if (empty($attribute['blocked'])) {
$this->Attribute->create();
$result = $this->Attribute->save($attribute);
$this->Attribute->set($attribute);
$result = $this->Attribute->validates();
if (!$result) {
$fails["attribute_$k"] = $this->Attribute->validationErrors;
unset($attributes[$k]);
} else {
$successes[$k] = $this->Attribute->id;
$successes++;
}
} else {
$fails["attribute_$k"] = 'Attirbute blocked due to warninglist';
$fails["attribute_$k"] = 'Attribute blocked due to warninglist';
unset($attributes[$k]);
}
}
if (!empty($successes)) {
$this->Event->unpublishEvent($eventId);
}
$result = $this->Attribute->saveMany($attributes);
if ($this->_isRest()) {
if (!empty($successes)) {
$attributes = $this->Attribute->find('all', array(
'recursive' => -1,
'conditions' => array('Attribute.id' => array_values($successes))
'conditions' => array('Attribute.id' => $this->Attribute->inserted_ids)
));
if (count($attributes) == 1) {
$attributes = $attributes[0];
@ -275,7 +301,7 @@ class AttributesController extends AppController {
$events = $this->Event->findById($eventId);
$this->set('event_id', $events['Event']['id']);
// combobox for distribution
$this->set('currentDist', $events['Event']['distribution']); // TODO default distribution
$this->set('currentDist', $events['Event']['distribution']);
// tooltip for distribution
$this->loadModel('SharingGroup');
@ -696,9 +722,6 @@ class AttributesController extends AppController {
if ('attachment' == $this->Attribute->data['Attribute']['type'] ||
'malware-sample' == $this->Attribute->data['Attribute']['type'] ) {
$this->set('attachment', true);
// TODO we should ensure 'value' cannot be changed here and not only on a view level (because of the associated file)
// $this->Session->setFlash(__('You cannot edit attachment attributes.', true), 'default', array(), 'error');
// $this->redirect(array('controller' => 'events', 'action' => 'view', $old_attribute['Event']['id']));
} else {
$this->set('attachment', false);
}
@ -1035,8 +1058,10 @@ class AttributesController extends AppController {
}
} else {
if (!$this->request->is('post') && !$this->_isRest()) throw new MethodNotAllowedException();
if ($this->Attribute->restore($id, $this->Auth->user())) $this->redirect(array('action' => 'view', $id));
else throw new NotFoundException('Could not restore the attribute');
if ($this->Attribute->restore($id, $this->Auth->user())) {
$this->Attribute->__alterAttributeCount($this->data['Attribute']['event_id']);
$this->redirect(array('action' => 'view', $id));
} else throw new NotFoundException('Could not restore the attribute');
}
}
@ -1296,7 +1321,13 @@ class AttributesController extends AppController {
// re-get pagination
$this->Attribute->recursive = 0;
$this->paginate = $this->Session->read('paginate_conditions');
$this->set('attributes', $this->paginate());
$attributes = $this->paginate();
foreach ($attributes as $k => $attribute) {
if (empty($attribute['Event']['id'])) {
unset($attribute[$k]);
}
}
$this->set('attributes', $attributes);
// set the same view as the index page
$this->render('index');
@ -1630,6 +1661,10 @@ class AttributesController extends AppController {
$attributes = $this->paginate();
$org_ids = array();
foreach ($attributes as $k => $attribute) {
if (empty($attribute['Event']['id'])) {
unset($attribute[$k]);
continue;
}
if ($attribute['Attribute']['type'] == 'attachment' && preg_match('/.*\.(jpg|png|jpeg|gif)$/i', $attribute['Attribute']['value'])) {
$attributes[$k]['Attribute']['image'] = $this->Attribute->base64EncodeAttachment($attribute['Attribute']);
}

16
app/Model/AppModel.php
View File

@ -28,6 +28,10 @@ class AppModel extends Model {
public $loadedPubSubTool = false;
public $start = 0;
public $inserted_ids = array();
private $__redisConnection = false;
public function __construct($id = false, $table = null, $ds = null) {
@ -48,11 +52,18 @@ class AppModel extends Model {
63 => false, 64 => false, 65 => false, 66 => false, 67 => true,
68 => false, 69 => false, 71 => false, 72 => false, 73 => false,
75 => false, 77 => false, 78 => false, 79 => false, 80 => false,
81 => false, 82 => false, 83 => false, 84 => false
81 => false, 82 => false, 83 => false, 84 => false, 85 => false
)
)
);
function afterSave($created, $options = array()) {
if ($created) {
$this->inserted_ids[] = $this->getInsertID();
}
return true;
}
// Generic update script
// add special cases where the upgrade does more than just update the DB
// this could become useful in the future
@ -832,6 +843,9 @@ class AppModel extends Model {
$sqlArray[] = "ALTER TABLE `tags` ADD `user_id` int(11) NOT NULL DEFAULT 0;";
$sqlArray[] = 'ALTER TABLE `tags` ADD INDEX `user_id` (`user_id`);';
break;
case '2.4.85':
$sqlArray[] = "ALTER TABLE `shadow_attributes` ADD `disable_correlation` tinyint(1) NOT NULL DEFAULT 0;";
break;
case 'fixNonEmptySharingGroupID':
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
$sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';

34
app/Model/Attribute.php
View File

@ -127,8 +127,8 @@ class Attribute extends AppModel {
'pattern-in-file' => array('desc' => 'Pattern in file that identifies the malware', 'default_category' => 'Payload installation', 'to_ids' => 1),
'pattern-in-traffic' => array('desc' => 'Pattern in network traffic that identifies the malware', 'default_category' => 'Network activity', 'to_ids' => 1),
'pattern-in-memory' => array('desc' => 'Pattern in memory dump that identifies the malware', 'default_category' => 'Payload installation', 'to_ids' => 1),
'yara' => array('desc' => 'Yara signature', 'default_category' => 'Payload installation', 'to_ids' => 1),
'stix2-pattern' => array('desc' => 'STIX 2 pattern', 'default_category' => 'Payload installation', 'to_ids' => 1),
'yara' => array('desc' => 'Yara signature', 'default_category' => 'Payload installation', 'to_ids' => 1),
'stix2-pattern' => array('desc' => 'STIX 2 pattern', 'default_category' => 'Payload installation', 'to_ids' => 1),
'sigma' => array('desc' => 'Sigma - Generic Signature Format for SIEM Systems', 'default_category' => 'Payload installation', 'to_ids' => 1),
'cookie' => array('desc' => 'HTTP cookie as often stored on the user web client. This can include authentication cookie or session cookie.', 'default_category' => 'Network activity', 'to_ids' => 0),
'vulnerability' => array('desc' => 'A reference to the vulnerability used in the exploit', 'default_category' => 'External analysis', 'to_ids' => 0),
@ -188,8 +188,8 @@ class Attribute extends AppModel {
'windows-service-displayname' => array('desc' => 'A windows service\'s displayname, not to be confused with the windows-service-name. This is the name that applications will generally display as the service\'s name in applications.', 'default_category' => 'Artifacts dropped', 'to_ids' => 0),
'whois-registrant-email' => array('desc' => 'The e-mail of a domain\'s registrant, obtained from the WHOIS information.', 'default_category' => 'Attribution', 'to_ids' => 0),
'whois-registrant-phone' => array('desc' => 'The phone number of a domain\'s registrant, obtained from the WHOIS information.', 'default_category' => 'Attribution', 'to_ids' => 0),
'whois-registrant-name' => array('desc' => 'The name of a domain\'s registrant, obtained from the WHOIS information.', 'default_category' => 'Attribution', 'to_ids' => 0),
'whois-registrant-org' => array('desc' => 'The org of a domain\'s registrant, obtained from the WHOIS information.', 'default_category' => 'Attribution', 'to_ids' => 0),
'whois-registrant-name' => array('desc' => 'The name of a domain\'s registrant, obtained from the WHOIS information.', 'default_category' => 'Attribution', 'to_ids' => 0),
'whois-registrant-org' => array('desc' => 'The org of a domain\'s registrant, obtained from the WHOIS information.', 'default_category' => 'Attribution', 'to_ids' => 0),
'whois-registrar' => array('desc' => 'The registrar of the domain, obtained from the WHOIS information.', 'default_category' => 'Attribution', 'to_ids' => 0),
'whois-creation-date' => array('desc' => 'The date of domain\'s creation, obtained from the WHOIS information.', 'default_category' => 'Attribution', 'to_ids' => 0),
// 'targeted-threat-index' => array('desc' => ''), // currently not mapped!
@ -489,9 +489,9 @@ class Attribute extends AppModel {
'foreignKey' => 'event_id',
'conditions' => '',
'fields' => '',
'order' => '',
'counterCache' => 'attribute_count',
'counterScope' => array('Attribute.deleted' => 0)
//'counterCache' => 'attribute_count',
//'counterScope' => array('Attribute.deleted' => 0),
'order' => ''
),
'SharingGroup' => array(
'className' => 'SharingGroup',
@ -567,10 +567,24 @@ class Attribute extends AppModel {
return true;
}
private function __alterAttributeCount($event_id, $increment = true) {
$event = $this->Event->find('first', array(
'recursive' => -1,
'conditions' => array('Event.id' => $event_id)
));
if (!empty($event)) {
if ($increment) $event['Event']['attribute_count'] = $event['Event']['attribute_count'] + 1;
else $event['Event']['attribute_count'] = $event['Event']['attribute_count'] - 1;
$this->Event->save($event, array('callbacks' => false));
}
}
public function afterSave($created, $options = array()) {
parent::afterSave($created, $options);
// update correlation...
if (isset($this->data['Attribute']['deleted']) && $this->data['Attribute']['deleted']) {
$this->__beforeSaveCorrelation($this->data['Attribute']);
if (isset($this->data['Attribute']['event_id'])) $this->__alterAttributeCount($this->data['Attribute']['event_id'], false);
} else {
$this->__afterSaveCorrelation($this->data['Attribute']);
}
@ -599,6 +613,9 @@ class Attribute extends AppModel {
if (isset($this->data['Attribute']['type']) && $this->typeIsAttachment($this->data['Attribute']['type']) && !empty($this->data['Attribute']['data'])) {
$result = $result && $this->saveBase64EncodedAttachment($this->data['Attribute']); // TODO : is this correct?
}
if ($created && isset($this->data['Attribute']['event_id']) && empty($this->data['Attribute']['skip_auto_increment'])) {
$this->__alterAttributeCount($this->data['Attribute']['event_id']);
}
return $result;
}
@ -634,6 +651,9 @@ class Attribute extends AppModel {
if (Configure::read('MISP.enable_advanced_correlations') && in_array($this->data['Attribute']['type'], array('ip-src', 'ip-dst', 'domain-ip')) && strpos($this->data['Attribute']['value'], '/')) {
$this->setCIDRList();
}
if (isset($this->data['Attribute']['event_id'])) {
if (empty($this->data['Attribute']['deleted'])) $this->__alterAttributeCount($this->data['Attribute']['event_id'], false);
}
if (!empty($this->data['Attribute']['id'])) {
$this->Object->ObjectReference->deleteAll(
array(