From e7f3d0d9dfa261f0e4a88d37db08033f1f9f459d Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 13 Jun 2019 09:16:34 +0200 Subject: [PATCH 01/73] new: [timeline/*-seen] Initial import of the timeline code from the zoidberg branch --- app/Controller/AttributesController.php | 17 +- app/Controller/Component/ACLComponent.php | 6 + .../Component/RestResponseComponent.php | 6 +- app/Controller/EventsController.php | 26 + app/Controller/ObjectsController.php | 301 ++++++++- app/Lib/Tools/EventGraphTool.php | 10 +- app/Lib/Tools/EventTimelineTool.php | 138 +++++ app/Model/AppModel.php | 92 +++ app/Model/Attribute.php | 81 ++- app/Model/Event.php | 2 +- app/Model/MispObject.php | 241 +++++-- app/View/Attributes/add.ctp | 3 +- .../ajax/attributeEditFirst_seenForm.ctp | 14 + .../ajax/attributeEditLast_seenForm.ctp | 14 + app/View/Elements/Events/View/row_object.ctp | 10 +- app/View/Elements/form_seen_input.ctp | 444 +++++++++++++ app/View/Elements/view_timeline.ctp | 33 + app/View/Events/view.ctp | 6 + app/View/Objects/add.ctp | 5 +- .../Objects/ajax/objectEditCommentForm.ctp | 22 + .../ajax/objectEditDistributionForm.ctp | 19 + .../Objects/ajax/objectEditFirst_seenForm.ctp | 14 + .../Objects/ajax/objectEditLast_seenForm.ctp | 14 + app/View/Objects/ajax/objectViewFieldForm.ctp | 2 + .../Objects/ajax/quickAddAttributeForm.ctp | 156 +++++ .../templateWithValidObjectAttributes.ctp | 22 + app/View/Objects/revise_object.ctp | 7 + app/webroot/css/event-timeline.css | 150 +++++ app/webroot/css/main.css | 55 ++ app/webroot/js/bootstrap-datepicker.js | 11 +- app/webroot/js/contextual_menu.js | 18 +- app/webroot/js/event-graph.js | 24 +- app/webroot/js/event-timeline.js | 586 ++++++++++++++++++ app/webroot/js/misp.js | 105 +++- 34 files changed, 2554 insertions(+), 100 deletions(-) create mode 100644 app/Lib/Tools/EventTimelineTool.php create mode 100644 app/View/Attributes/ajax/attributeEditFirst_seenForm.ctp create mode 100644 app/View/Attributes/ajax/attributeEditLast_seenForm.ctp create mode 100644 app/View/Elements/form_seen_input.ctp create mode 100644 app/View/Elements/view_timeline.ctp create mode 100644 app/View/Objects/ajax/objectEditCommentForm.ctp create mode 100644 app/View/Objects/ajax/objectEditDistributionForm.ctp create mode 100644 app/View/Objects/ajax/objectEditFirst_seenForm.ctp create mode 100644 app/View/Objects/ajax/objectEditLast_seenForm.ctp create mode 100644 app/View/Objects/ajax/objectViewFieldForm.ctp create mode 100644 app/View/Objects/ajax/quickAddAttributeForm.ctp create mode 100644 app/View/Objects/ajax/templateWithValidObjectAttributes.ctp create mode 100644 app/webroot/css/event-timeline.css create mode 100644 app/webroot/js/event-timeline.js diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 6abf54d16..6bc0cd8e8 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -919,7 +919,7 @@ class AttributesController extends AppController $skipTimeCheck = true; } if ($skipTimeCheck || $this->request->data['Attribute']['timestamp'] > $existingAttribute['Attribute']['timestamp']) { - $recoverFields = array('value', 'to_ids', 'distribution', 'category', 'type', 'comment'); + $recoverFields = array('value', 'to_ids', 'distribution', 'category', 'type', 'comment', 'first_seen', 'last_seen'); foreach ($recoverFields as $rF) { if (!isset($this->request->data['Attribute'][$rF])) { $this->request->data['Attribute'][$rF] = $existingAttribute['Attribute'][$rF]; @@ -958,6 +958,10 @@ class AttributesController extends AppController } else { return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Could not update attribute, reason: ' . json_encode($this->Attribute->validationErrors))),'status' => 200, 'type' => 'json')); } + } else { + if (!$result) { + return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Could not update attribute, reason: ' . json_encode($this->Attribute->validationErrors))),'status' => 200, 'type' => 'json')); + } } } if ($result) { @@ -975,10 +979,11 @@ class AttributesController extends AppController } } if ($this->_isRest() || $this->response->type() === 'application/json') { + $searchFields = array('id', 'type', 'to_ids', 'category', 'uuid', 'event_id', 'distribution', 'timestamp', 'comment', 'value', 'disable_correlation', 'first_seen', 'last_seen'); $saved_attribute = $this->Attribute->find('first', array( 'conditions' => array('id' => $this->Attribute->id), 'recursive' => -1, - 'fields' => array('id', 'type', 'to_ids', 'category', 'uuid', 'event_id', 'distribution', 'timestamp', 'comment', 'value', 'disable_correlation'), + 'fields' => $searchFields, )); $response = array('response' => array('Attribute' => $saved_attribute['Attribute'])); $this->set('response', $response); @@ -1113,7 +1118,7 @@ class AttributesController extends AppController if (!$this->_isRest()) { $this->Attribute->Event->insertLock($this->Auth->user(), $this->Attribute->data['Attribute']['event_id']); } - $validFields = array('value', 'category', 'type', 'comment', 'to_ids', 'distribution'); + $validFields = array('value', 'category', 'type', 'comment', 'to_ids', 'distribution', 'first_seen', 'last_seen'); $changed = false; if (empty($this->request->data['Attribute'])) { $this->request->data = array('Attribute' => $this->request->data); @@ -1721,7 +1726,7 @@ class AttributesController extends AppController unset($this->request->data['to_ids']); $this->request->data['ignore'] = 1; } - $paramArray = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags'); + $paramArray = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags', 'first_seen', 'last_seen'); $filterData = array( 'request' => $this->request, 'named_params' => $this->params['named'], @@ -1934,7 +1939,7 @@ class AttributesController extends AppController 'value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags', 'includeProposals', 'returnFormat', 'published', 'limit', 'page', 'requested_attributes', 'includeContext', 'headerless', - 'includeWarninglistHits', 'attackGalaxy', 'object_relation' + 'includeWarninglistHits', 'attackGalaxy', 'object_relation', 'first_seen', 'last_seen' ); $filterData = array( 'request' => $this->request, @@ -2389,7 +2394,7 @@ class AttributesController extends AppController public function fetchViewValue($id, $field = null) { - $validFields = array('value', 'comment', 'type', 'category', 'to_ids', 'distribution', 'timestamp'); + $validFields = array('value', 'comment', 'type', 'category', 'to_ids', 'distribution', 'timestamp', 'first_seen', 'last_seen'); if (!isset($field) || !in_array($field, $validFields)) { throw new MethodNotAllowedException(__('Invalid field requested.')); } diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php index 0df253fb7..f497fb8ac 100644 --- a/app/Controller/Component/ACLComponent.php +++ b/app/Controller/Component/ACLComponent.php @@ -117,6 +117,7 @@ class ACLComponent extends Component 'getEventGraphReferences' => array('*'), 'getEventGraphTags' => array('*'), 'getEventGraphGeneric' => array('*'), + 'getEventTimeline' => array('*'), 'genDistributionGraph' => array('*'), 'getDistributionGraph' => array('*'), 'getReferenceData' => array('*'), @@ -250,6 +251,11 @@ class ACLComponent extends Component 'edit' => array('perm_add'), 'get_row' => array('perm_add'), 'orphanedObjectDiagnostics' => array(), + 'editField' => array('perm_add'), + 'fetchEditForm' => array('perm_add'), + 'fetchViewValue' => array('*'), + 'quickAddAttributeForm' => array('perm_add'), + 'quickFetchTemplateWithValidObjectAttributes' => array('perm_add'), 'proposeObjectsFromAttributes' => array('*'), 'groupAttributesIntoObject' => array('perm_add'), 'revise_object' => array('perm_add'), diff --git a/app/Controller/Component/RestResponseComponent.php b/app/Controller/Component/RestResponseComponent.php index 301d1b3ea..397981ec4 100644 --- a/app/Controller/Component/RestResponseComponent.php +++ b/app/Controller/Component/RestResponseComponent.php @@ -20,13 +20,13 @@ class RestResponseComponent extends Component 'add' => array( 'description' => "POST a MISP Attribute JSON to this API to create an Attribute.", 'mandatory' => array('value', 'type'), - 'optional' => array('category', 'to_ids', 'uuid', 'distribution', 'sharing_group_id', 'timestamp', 'comment'), + 'optional' => array('category', 'to_ids', 'uuid', 'distribution', 'sharing_group_id', 'timestamp', 'comment', 'first_seen', 'last_seen'), 'params' => array('event_id') ), 'edit' => array( 'description' => "POST a MISP Attribute JSON to this API to update an Attribute. If the timestamp is set, it has to be newer than the existing Attribute.", 'mandatory' => array(), - 'optional' => array('value', 'type', 'category', 'to_ids', 'uuid', 'distribution', 'sharing_group_id', 'timestamp', 'comment'), + 'optional' => array('value', 'type', 'category', 'to_ids', 'uuid', 'distribution', 'sharing_group_id', 'timestamp', 'comment', 'first_seen', 'last_seen'), 'params' => array('attribute_id') ), 'deleteSelected' => array( @@ -382,6 +382,7 @@ class RestResponseComponent extends Component if (isset($this->__convertActionToMessage[$controller][$action['action']])) { $stringifiedAction = $this->__convertActionToMessage[$controller][$action['action']]; } + $response['saved'] = false; $response['name'] = 'Could not ' . $stringifiedAction . ' ' . Inflector::singularize($controller); $response['message'] = $response['name']; $response['url'] = $this->__generateURL($action, $controller, $id); @@ -395,6 +396,7 @@ class RestResponseComponent extends Component if (!$message) { $message = Inflector::singularize($controller) . ' ' . $action['action'] . ((substr($action['action'], -1) == 'e') ? 'd' : 'ed'); } + $response['saved'] = true; $response['name'] = $message; $response['message'] = $response['name']; $response['url'] = $this->__generateURL($action, $controller, $id); diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index e285e38ec..86b5e7e19 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -4752,6 +4752,32 @@ class EventsController extends AppController return $json; } + public function getEventTimeline($id, $type = 'event') + { + $validTools = array('event'); + if (!in_array($type, $validTools)) { + throw new MethodNotAllowedException('Invalid type.'); + } + + App::uses('EventTimelineTool', 'Tools'); + $grapher = new EventTimelineTool(); + $data = $this->request->is('post') ? $this->request->data : array(); + $dataFiltering = array_key_exists('filtering', $data) ? $data['filtering'] : array(); + + $extended = isset($this->params['named']['extended']) ? 1 : 0; + + $grapher->construct($this->Event, $this->Auth->user(), $dataFiltering, $extended); + $json = $grapher->get_timeline($id); + + array_walk_recursive($json, function (&$item, $key) { + if (!mb_detect_encoding($item, 'utf-8', true)) { + $item = utf8_encode($item); + } + }); + $this->response->type('json'); + return new CakeResponse(array('body' => json_encode($json), 'status' => 200, 'type' => 'json')); + } + public function getDistributionGraph($id, $type = 'event') { $extended = isset($this->params['named']['extended']) ? 1 : 0; diff --git a/app/Controller/ObjectsController.php b/app/Controller/ObjectsController.php index 2de1f2e64..1e1f78d8e 100644 --- a/app/Controller/ObjectsController.php +++ b/app/Controller/ObjectsController.php @@ -353,7 +353,7 @@ class ObjectsController extends AppController $this->set('element', $element); } - public function edit($id, $update_template_available=false) + public function edit($id, $update_template_available=false, $onlyAddNewAttribute=false) { if (Validation::uuid($id)) { $conditions = array('Object.uuid' => $id); @@ -532,7 +532,7 @@ class ObjectsController extends AppController $this->request->data = array('Attribute' => $this->request->data); } $objectToSave = $this->MispObject->attributeCleanup($this->request->data); - $objectToSave = $this->MispObject->deltaMerge($object, $objectToSave); + $objectToSave = $this->MispObject->deltaMerge($object, $objectToSave, $onlyAddNewAttribute); // we pre-validate the attributes before we create an object at this point // This allows us to stop the process and return an error (API) or return // to the add form @@ -595,6 +595,303 @@ class ObjectsController extends AppController $this->render('add'); } + // ajax edit - post a single edited field and this method will attempt to save it and return a json with the validation errors if they occur. + public function editField($id) + { + if (Validation::uuid($id)) { + $this->MispObject->recursive = -1; + $temp = $this->MispObject->findByUuid($id); + if ($temp == null) { + throw new NotFoundException('Invalid object'); + } + $id = $temp['Object']['id']; + } elseif (!is_numeric($id)) { + throw new NotFoundException(__('Invalid event id.')); + } + if ((!$this->request->is('post') && !$this->request->is('put'))) { + throw new MethodNotAllowedException(); + } + $this->MispObject->id = $id; + if (!$this->MispObject->exists()) { + return $this->RestResponse->saveFailResponse('Objects', 'edit', false, 'Invalid object', $this->response->type()); + } + $this->MispObject->recursive = -1; + $this->MispObject->contain('Event'); + $object = $this->MispObject->read(); + if (!$this->_isSiteAdmin()) { + if ($this->MispObject->data['Event']['orgc_id'] == $this->Auth->user('org_id') + && (($this->userRole['perm_modify'] && $this->MispObject->data['Event']['user_id'] != $this->Auth->user('id')) + || $this->userRole['perm_modify_org'])) { + // Allow the edit + } else { + return $this->RestResponse->saveFailResponse('Objects', 'edit', false, 'Invalid attribute', $this->response->type()); + } + } + $validFields = array('comment', 'distribution', 'first_seen', 'last_seen'); + $changed = false; + if (empty($this->request->data['Object'])) { + $this->request->data = array('Object' => $this->request->data); + if (empty($this->request->data['Object'])) { + throw new MethodNotAllowedException('Invalid input.'); + } + } + foreach ($this->request->data['Object'] as $changedKey => $changedField) { + if (!in_array($changedKey, $validFields)) { + throw new MethodNotAllowedException('Invalid field.'); + } + if ($object['Object'][$changedKey] == $changedField) { + $this->autoRender = false; + return $this->RestResponse->saveSuccessResponse('Objects', 'edit', $id, $this->response->type(), 'nochange'); + } + $object['Object'][$changedKey] = $changedField; + $changed = true; + } + if (!$changed) { + return $this->RestResponse->saveSuccessResponse('Objects', 'edit', $id, $this->response->type(), 'nochange'); + } + $date = new DateTime(); + $object['Object']['timestamp'] = $date->getTimestamp(); + $this->MispObject->setObjectSeenMetaFromAttribute($object, true); + if ($this->MispObject->save($object)) { + $event = $this->MispObject->Event->find('first', array( + 'recursive' => -1, + 'fields' => array('id', 'published', 'timestamp', 'info', 'uuid'), + 'conditions' => array( + 'id' => $object['Object']['event_id'], + ))); + $event['Event']['timestamp'] = $date->getTimestamp(); + $event['Event']['published'] = 0; + $this->MispObject->Event->save($event, array('fieldList' => array('published', 'timestamp', 'info'))); + $this->autoRender = false; + return $this->RestResponse->saveSuccessResponse('Objects', 'edit', $id, $this->response->type(), 'Field updated'); + } else { + $this->autoRender = false; + return $this->RestResponse->saveFailResponse('Objects', 'edit', false, $this->MispObject->validationErrors, $this->response->type()); + } + } + + public function fetchViewValue($id, $field = null) + { + $validFields = array('timestamp', 'comment', 'distribution', 'first_seen', 'last_seen'); + if (!isset($field) || !in_array($field, $validFields)) { + throw new MethodNotAllowedException('Invalid field requested.'); + } + if (!$this->request->is('ajax')) { + throw new MethodNotAllowedException('This function can only be accessed via AJAX.'); + } + $this->MispObject->id = $id; + if (!$this->MispObject->exists()) { + throw new NotFoundException(__('Invalid object')); + } + $params = array( + 'conditions' => array('Object.id' => $id), + 'fields' => array('id', 'distribution', 'event_id', $field), + 'contain' => array( + 'Event' => array( + 'fields' => array('distribution', 'id', 'org_id'), + ) + ), + 'flatten' => 1 + ); + $object = $this->MispObject->fetchObjectSimple($this->Auth->user(), $params); + if (empty($object)) { + throw new NotFoundException(__('Invalid object')); + } + $object = $object[0]; + $result = $object['Object'][$field]; + if ($field == 'distribution') { + $result=$this->MispObject->shortDist[$result]; + } + $this->set('value', $result); + $this->layout = 'ajax'; + $this->render('ajax/objectViewFieldForm'); + } + + public function fetchEditForm($id, $field = null) + { + $validFields = array('distribution', 'comment', 'first_seen', 'last_seen'); + if (!isset($field) || !in_array($field, $validFields)) { + throw new MethodNotAllowedException('Invalid field requested.'); + } + if (!$this->request->is('ajax')) { + throw new MethodNotAllowedException('This function can only be accessed via AJAX.'); + } + $this->MispObject->id = $id; + if (!$this->MispObject->exists()) { + throw new NotFoundException(__('Invalid object')); + } + $fields = array('id', 'distribution', 'event_id'); + $fields[] = $field; + $params = array( + 'conditions' => array('Object.id' => $id), + 'fields' => $fields, + 'flatten' => 1, + 'contain' => array( + 'Event' => array( + 'fields' => array('distribution', 'id', 'user_id', 'orgc_id', 'org_id'), + ) + ) + ); + $object = $this->MispObject->fetchObjectSimple($this->Auth->user(), $params); + if (empty($object)) { + throw new NotFoundException(__('Invalid attribute')); + } + $object = $object[0]; + if (!$this->_isSiteAdmin()) { + if ($object['Event']['orgc_id'] == $this->Auth->user('org_id') + && (($this->userRole['perm_modify'] && $object['Event']['user_id'] != $this->Auth->user('id')) + || $this->userRole['perm_modify_org'])) { + // Allow the edit + } else { + throw new NotFoundException(__('Invalid object')); + } + } + $this->layout = 'ajax'; + if ($field == 'distribution') { + $distributionLevels = $this->MispObject->shortDist; + unset($distributionLevels[4]); + $this->set('distributionLevels', $distributionLevels); + } + $this->set('object', $object['Object']); + $fieldURL = ucfirst($field); + $this->render('ajax/objectEdit' . $fieldURL . 'Form'); + } + + // Construct a template with valid object attributes to add to an object + public function quickFetchTemplateWithValidObjectAttributes($id) { + $this->MispObject->id = $id; + if (!$this->MispObject->exists()) { + if ($this->request->is('ajax')) { + return $this->RestResponse->saveFailResponse('Objects', 'add', false, 'Invalid object', $this->response->type()); + } else { + throw new NotFoundException(__('Invalid object')); + } + } + $fields = array('template_uuid', 'template_version', 'id'); + $params = array( + 'conditions' => array('Object.id' => $id), + 'fields' => $fields, + 'flatten' => 1, + ); + // fetchObjects restrict access based on user + $object = $this->MispObject->fetchObjects($this->Auth->user(), $params); + if (empty($object)) { + if ($this->request->is('ajax')) { + return $this->RestResponse->saveFailResponse('Objects', 'add', false, 'Invalid object', $this->response->type()); + } else { + throw new NotFoundException(__('Invalid object')); + } + } else { + $object = $object[0]; + } + // get object attributes already set + $objectRelation = array(); + foreach($object['Attribute'] as $attr) { + $objectRelation[$attr['object_relation']] = 1; + } + $objectRelation = array_keys($objectRelation); + // get object attribute defined in the object's template + $template = $this->MispObject->ObjectTemplate->find('first', array( + 'conditions' => array( + 'ObjectTemplate.uuid' => $object['Object']['template_uuid'], + 'ObjectTemplate.version' => $object['Object']['template_version'], + ), + 'recursive' => -1, + 'flatten' => 1, + 'contain' => 'ObjectTemplateElement' + )); + if (empty($template)) { + if ($this->request->is('ajax')) { + return $this->RestResponse->saveFailResponse('Objects', 'add', false, 'Invalid template', $this->response->type()); + } else { + throw new NotFoundException(__('Invalid template')); + } + } + // unset object invalid object attribute + foreach($template['ObjectTemplateElement'] as $i => $objAttr) { + if (in_array($objAttr['object_relation'], $objectRelation) && !$objAttr['multiple']) { + unset($template['ObjectTemplateElement'][$i]); + } + } + if ($this->request->is('get') || $this->request->is('post')) { + $this->set('template', $template); + $this->set('objectId', $object['Object']['id']); + $this->render('ajax/templateWithValidObjectAttributes'); + } else { + return $template; + } + } + + /** + * GET: Returns a form allowing to add a valid object attribute to an object + * POST/PUT: Add the attribute to the object + */ + public function quickAddAttributeForm($id, $fieldName = null) { + if ($this->request->is('GET')) { + if (!isset($fieldName)) { + throw new MethodNotAllowedException('No field requested.'); + } + $this->MispObject->id = $id; + if (!$this->MispObject->exists()) { + throw new NotFoundException(__('Invalid object')); + } + $fields = array('template_uuid', 'template_version', 'id', 'event_id'); + $params = array( + 'conditions' => array('Object.id' => $id), + 'fields' => $fields, + 'flatten' => 1, + ); + // fetchObjects restrict access based on user + $object = $this->MispObject->fetchObjects($this->Auth->user(), $params); + if (empty($object)) { + throw new NotFoundException(__('Invalid object')); + } else { + $object = $object[0]; + } + $template = $this->MispObject->ObjectTemplate->find('first', array( + 'conditions' => array( + 'ObjectTemplate.uuid' => $object['Object']['template_uuid'], + 'ObjectTemplate.version' => $object['Object']['template_version'], + ), + 'recursive' => -1, + 'flatten' => 1, + 'contain' => array( + 'ObjectTemplateElement' => array('conditions' => array( + 'object_relation' => $fieldName + )) + ) + )); + if (empty($template)) { + throw new NotFoundException(__('Invalid object')); + } + if (empty($template['ObjectTemplateElement'])) { + throw new NotFoundException(__('Invalid fields') . ' `' . h($fieldName) . '`'); + } + + // check if fields can be added + foreach($object['Attribute'] as $i => $objAttr) { + $objectAttrFromTemplate = $template['ObjectTemplateElement'][0]; + if ($objAttr['object_relation'] == $fieldName && !$objectAttrFromTemplate['multiple']) { + throw new NotFoundException(__('Invalid field')); + } + } + $template = $this->MispObject->prepareTemplate($template, $object); + $this->layout = 'ajax'; + $this->set('object', $object['Object']); + $this->set('template', $template); + $distributionData = $this->MispObject->Event->Attribute->fetchDistributionData($this->Auth->user()); + $this->set('distributionData', $distributionData); + $info = array(); + foreach ($distributionData['levels'] as $key => $value) { + $info['distribution'][$key] = array('key' => $value, 'desc' => $this->MispObject->Event->Attribute->distributionDescriptions[$key]['formdesc']); + } + $this->set('info', $info); + $this->render('ajax/quickAddAttributeForm'); + } else if ($this->request->is('post') || $this->request->is('put')) { + return $this->edit($this->request->data['Object']['id'], true); + } + } + public function delete($id, $hard = false) { if (!$this->userRole['perm_modify']) { diff --git a/app/Lib/Tools/EventGraphTool.php b/app/Lib/Tools/EventGraphTool.php index 8407b5adf..b9a809d71 100644 --- a/app/Lib/Tools/EventGraphTool.php +++ b/app/Lib/Tools/EventGraphTool.php @@ -40,7 +40,7 @@ private function __get_event($id) { - $this->__json['available_rotation_key'] = $this->__authorized_JSON_key; + $this->__json['available_pivot_key'] = $this->__authorized_JSON_key; $fullevent = $this->__eventModel->fetchEvent($this->__user, array('eventid' => $id, 'flatten' => 0, 'includeTagRelations' => 1, 'extended' => $this->__extended_view)); $event = array(); @@ -232,12 +232,12 @@ $event = $this->__get_filtered_event($id); $this->__json['items'] = array(); $this->__json['relations'] = array(); - + $this->__json['existing_object_relation'] = array(); if (empty($event)) { return $this->__json; } - + if (!empty($event['Object'])) { $object = $event['Object']; } else { @@ -312,7 +312,7 @@ if (empty($event)) { return $this->__json; } - + if (!empty($event['Object'])) { $object = $event['Object']; } else { @@ -419,7 +419,7 @@ if (empty($event)) { return $this->__json; } - + if (!empty($event['Object'])) { $object = $event['Object']; } else { diff --git a/app/Lib/Tools/EventTimelineTool.php b/app/Lib/Tools/EventTimelineTool.php new file mode 100644 index 000000000..bf94673e9 --- /dev/null +++ b/app/Lib/Tools/EventTimelineTool.php @@ -0,0 +1,138 @@ +__eventModel = $eventModel; + $this->__objectTemplateModel = $eventModel->Object->ObjectTemplate; + $this->__user = $user; + $this->__filterRules = $filterRules; + $this->__json = array(); + $this->__extended_view = $extended_view; + $this->__lookupTables = array( + 'analysisLevels' => $this->__eventModel->analysisLevels, + 'distributionLevels' => $this->__eventModel->Attribute->distributionLevels + ); + return true; + } + + public function construct_for_ref($refModel, $user) + { + $this->__refModel = $refModel; + $this->__user = $user; + $this->__json = array(); + return true; + } + + private function __get_event($id) + { + $fullevent = $this->__eventModel->fetchEvent($this->__user, array('eventid' => $id, 'flatten' => 0, 'includeTagRelations' => 1, 'extended' => $this->__extended_view)); + $event = array(); + if (empty($fullevent)) { + return $event; + } + + if (!empty($fullevent[0]['Object'])) { + $event['Object'] = $fullevent[0]['Object']; + } else { + $event['Object'] = array(); + } + + if (!empty($fullevent[0]['Attribute'])) { + $event['Attribute'] = $fullevent[0]['Attribute']; + } else { + $event['Attribute'] = array(); + } + + return $event; + } + + public function get_timeline($id) + { + $event = $this->__get_event($id); + $this->__json['items'] = array(); + + if (empty($event)) { + return $this->__json; + } + + if (!empty($event['Object'])) { + $object = $event['Object']; + } else { + $object = array(); + } + + if (!empty($event['Attribute'])) { + $attribute = $event['Attribute']; + } else { + $attribute = array(); + } + + // extract links and node type + foreach ($attribute as $attr) { + $toPush = array( + 'id' => $attr['id'], + 'uuid' => $attr['uuid'], + 'content' => $attr['value'], + 'event_id' => $attr['event_id'], + 'group' => 'attribute', + 'timestamp' => $attr['timestamp'], + 'first_seen' => $attr['first_seen'], + 'last_seen' => $attr['last_seen'], + ); + array_push($this->__json['items'], $toPush); + } + + foreach ($object as $obj) { + $toPush_obj = array( + 'id' => $obj['id'], + 'uuid' => $obj['uuid'], + 'content' => $obj['name'], + 'group' => 'object', + 'meta-category' => $obj['meta-category'], + 'template_uuid' => $obj['template_uuid'], + 'event_id' => $obj['event_id'], + 'timestamp' => $obj['timestamp'], + 'Attribute' => array(), + ); + + $toPush_obj['first_seen'] = $obj['first_seen']; + $toPush_obj['last_seen'] = $obj['last_seen']; + $toPush_obj['first_seen_overwrite'] = false; + $toPush_obj['last_seen_overwrite'] = false; + + foreach ($obj['Attribute'] as $obj_attr) { + // replaced *_seen based on object attribute + if ($obj_attr['object_relation'] == 'first-seen' && is_null($toPush_obj['first_seen'])) { + $toPush_obj['first_seen'] = $obj_attr['value']; // replace first_seen of the object to seen of the element + $toPush_obj['first_seen_overwrite'] = true; + } elseif ($obj_attr['object_relation'] == 'last-seen' && is_null($toPush_obj['last_seen'])) { + $toPush_obj['last_seen'] = $obj_attr['value']; // replace last_seen of the object to seen of the element + $toPush_obj['last_seen_overwrite'] = true; + } + $toPush_attr = array( + 'id' => $obj_attr['id'], + 'uuid' => $obj_attr['uuid'], + 'content' => $obj_attr['value'], + 'contentType' => $obj_attr['object_relation'], + 'event_id' => $obj_attr['event_id'], + 'group' => 'object_attribute', + 'timestamp' => $obj_attr['timestamp'], + ); + array_push($toPush_obj['Attribute'], $toPush_attr); + } + array_push($this->__json['items'], $toPush_obj); + } + + return $this->__json; + } + } diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index ba7ce9d02..aa794994f 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -79,6 +79,26 @@ class AppModel extends Model ); public $advanced_updates_description = array( + 'seenOnAttributeAndObject' => array( + 'title' => 'First seen/Last seen Attribute table', + 'description' => 'Update the Attribute table to support first_seen and last_seen feature, with a microsecond resolution.', + 'liveOff' => true, # should the instance be offline for users other than site_admin + 'recommendBackup' => true, # should the update recommend backup + 'exitOnError' => false, # should the update exit on error + 'requirements' => 'MySQL version must be >= 5.6', # message stating the requirements necessary for the update + 'record' => true, # should the update success be saved in the admin_table + 'preUpdate' => 'seenOnAttributeAndObjectPreUpdate', # Function to execute before the update. If it throws an error, it cancels the update + 'url' => '/servers/updateDatabase/seenOnAttributeAndObject/' # url pointing to the funcion performing the update + ), + 'testUpdate' => array( + 'title' => 'Test Update', + 'description' => 'Runs a test update', + 'liveOff' => true, + 'recommendBackup' => true, + 'exitOnError' => true, + 'preUpdate' => 'failingPreUpdate', # Function to execute before the update. If it returns false, cancel the update + 'url' => '/servers/updateDatabase/testUpdate/' + ) ); public $actions_description = array( 'verifyGnuPGkeys' => array( @@ -1210,6 +1230,55 @@ class AppModel extends Model $sqlArray[] = 'ALTER TABLE `threads` DROP `org`;'; $sqlArray[] = 'ALTER TABLE `users` DROP `org`;'; break; + case 'seenOnAttributeAndObject': + $sqlArray[] = + "ALTER TABLE `attributes` + DROP INDEX uuid, + DROP INDEX event_id, + DROP INDEX sharing_group_id, + DROP INDEX type, + DROP INDEX category, + DROP INDEX value1, + DROP INDEX value2, + DROP INDEX object_id, + DROP INDEX object_relation, + DROP INDEX deleted, + ;"; + $sqlArray[] = + "ALTER TABLE `attributes` + ADD COLUMN `first_seen` BIGINT(20) NULL DEFAULT NULL, + ADD COLUMN `last_seen` BIGINT(20) NULL DEFAULT NULL, + MODIFY comment TEXT COLLATE utf8_unicode_ci + ;"; + $sqlArray[] = " + ALTER TABLE `attributes` + ADD INDEX `uuid` (`uuid`), + ADD INDEX `event_id` (`event_id`), + ADD INDEX `sharing_group_id` (`sharing_group_id`), + ADD INDEX `type` (`type`), + ADD INDEX `category` (`category`), + ADD INDEX `value1` (`value1`(255)), + ADD INDEX `value2` (`value2`(255)), + ADD INDEX `object_id` (`object_id`), + ADD INDEX `object_relation` (`object_relation`), + ADD INDEX `deleted` (`deleted`), + ADD INDEX `first_seen` (`first_seen`), + ADD INDEX `last_seen` (`last_seen`), + ADD INDEX `comment` (`comment`(767)) + ;"; + $sqlArray[] = " + ALTER TABLE `objects` + ADD `first_seen` BIGINT(20) NULL DEFAULT NULL, + ADD `last_seen` BIGINT(20) NULL DEFAULT NULL + ;"; + $indexArray[] = array('objects', 'first_seen'); + $indexArray[] = array('objects', 'last_seen'); + break; + case 'testUpdate': + $sqlArray[] = "SELECT SLEEP(10);"; + $sqlArray[] = "SELECT SLEEPsdcfsac(4);"; + $sqlArray[] = "SELECT SLEEP(12);"; + break; default: return false; break; @@ -1481,6 +1550,29 @@ class AppModel extends Model return true; } + // Try to create a table with a BIGINT(20) + public function seenOnAttributeAndObjectPreUpdate() { + $sqlArray[] = "CREATE TABLE IF NOT EXISTS testtable ( + `testfield` BIGINT(6) NULL DEFAULT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;"; + try { + foreach($sqlArray as $i => $sql) { + $this->query($sql); + } + } catch (Exception $e) { + throw new Exception('Pre update test failed: ' . PHP_EOL . $sql . PHP_EOL . ' The returned error is: ' . $e->getMessage()); + } + // clean up + $sqlArray[] = "DROP TABLE testtable;"; + foreach($sqlArray as $i => $sql) { + $this->query($sql); + } + } + + public function failingPreUpdate() { + throw new Exception('Yolo fail'); + } + public function runUpdates($verbose = false) { $this->AdminSetting = ClassRegistry::init('AdminSetting'); diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 889ae0b60..d39dc42c8 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -360,7 +360,9 @@ class Attribute extends AppModel 'deleted', 'disable_correlation', 'object_id', - 'object_relation' + 'object_relation', + 'first_seen', + 'last_seen' ); public $searchResponseTypes = array( @@ -508,6 +510,16 @@ class Attribute extends AppModel 'rule' => array('inList', array('0', '1', '2', '3', '4', '5')), 'message' => 'Options: Your organisation only, This community only, Connected communities, All communities, Sharing group, Inherit event', 'required' => true + ), + 'first_seen' => array( + 'rule' => array('datetimeOrNull'), + 'required' => false, + 'message' => array('Invalid ISO 8601 format') + ), + 'last_seen' => array( + 'rule' => array('datetimeOrNull'), + 'required' => false, + 'message' => array('Invalid ISO 8601 format') ) ); @@ -586,6 +598,22 @@ class Attribute extends AppModel if (isset($v['Attribute']['object_relation']) && $v['Attribute']['object_relation'] === null) { $results[$k]['Attribute']['object_relation'] = ''; } + if (!empty($v['Attribute']['first_seen'])) { + $fs = $results[$k]['Attribute']['first_seen']; + $fs_sec = $fs / 1000000; // $fs is in micro (10^6) + $fs_micro = $fs % 1000000; + $fs_micro = str_pad($fs_micro, 6, "0", STR_PAD_LEFT); + $fs = $fs_sec . '.' . $fs_micro; + $results[$k]['Attribute']['first_seen'] = DateTime::createFromFormat('U.u', $fs)->format('Y-m-d\TH:i:s.uP'); + } + if (!empty($v['Attribute']['last_seen'])) { + $ls = $results[$k]['Attribute']['last_seen']; + $ls_sec = $ls / 1000000; // $ls is in micro (10^6) + $ls_micro = $ls % 1000000; + $ls_micro = str_pad($ls_micro, 6, "0", STR_PAD_LEFT); + $ls = $ls_sec . '.' . $ls_micro; + $results[$k]['Attribute']['last_seen'] = DateTime::createFromFormat('U.u', $ls)->format('Y-m-d\TH:i:s.uP'); + } } return $results; } @@ -610,6 +638,27 @@ class Attribute extends AppModel } } + // convert into utc and micro sec + if (!empty($this->data['Attribute']['first_seen'])) { + $d = new DateTime($this->data['Attribute']['first_seen']); + $d->setTimezone(new DateTimeZone('GMT')); + $fs_sec = $d->format('U'); + $fs_micro = $d->format('u'); + $fs_micro = str_pad($fs_micro, 6, "0", STR_PAD_LEFT); + $fs = $fs_sec . $fs_micro; + $this->data['Attribute']['first_seen'] = $fs; + } + if (!empty($this->data['Attribute']['last_seen'])) { + $d = new DateTime($this->data['Attribute']['last_seen']); + $d->setTimezone(new DateTimeZone('GMT')); + $ls_sec = $d->format('U'); + $ls_micro = $d->format('u'); + $ls_micro = str_pad($ls_micro, 6, "0", STR_PAD_LEFT); + $ls = $ls_sec . $ls_micro; + $this->data['Attribute']['last_seen'] = $ls; + } + + // update correlation... (only needed here if there's an update) if ($this->id || !empty($this->data['Attribute']['id'])) { $this->__beforeSaveCorrelation($this->data['Attribute']); @@ -807,6 +856,16 @@ class Attribute extends AppModel $date = new DateTime(); $this->data['Attribute']['timestamp'] = $date->getTimestamp(); } + + // parse first_seen different formats + if (isset($this->data['Attribute']['first_seen'])) { + $this->data['Attribute']['first_seen'] = $this->data['Attribute']['first_seen'] === '' ? null : $this->data['Attribute']['first_seen']; + } + // parse last_seen different formats + if (isset($this->data['Attribute']['last_seen'])) { + $this->data['Attribute']['last_seen'] = $this->data['Attribute']['last_seen'] === '' ? null : $this->data['Attribute']['last_seen']; + } + // TODO: add explanatory comment // TODO: i18n? $result = $this->runRegexp($this->data['Attribute']['type'], $this->data['Attribute']['value']); @@ -939,6 +998,20 @@ class Attribute extends AppModel return $this->runValidation($value, $this->data['Attribute']['type']); } + // check whether the variable is null or datetime + public function datetimeOrNull($fields) + { + $k = array_keys($fields)[0]; + $seen = $fields[$k]; + try { + new DateTime($seen); + $returnValue = true; + } catch (Exception $e) { + $returnValue = false; + } + return $returnValue || is_null($seen); + } + private $__hexHashLengths = array( 'authentihash' => 64, 'md5' => 32, @@ -3787,7 +3860,7 @@ class Attribute extends AppModel return true; } // If a field is not set in the request, just reuse the old value - $recoverFields = array('value', 'to_ids', 'distribution', 'category', 'type', 'comment', 'sharing_group_id', 'object_id', 'object_relation'); + $recoverFields = array('value', 'to_ids', 'distribution', 'category', 'type', 'comment', 'sharing_group_id', 'object_id', 'object_relation', 'first_seen', 'last_seen'); foreach ($recoverFields as $rF) { if (!isset($attribute[$rF])) { $attribute[$rF] = $existingAttribute['Attribute'][$rF]; @@ -3857,7 +3930,9 @@ class Attribute extends AppModel 'comment', 'sharing_group_id', 'deleted', - 'disable_correlation' + 'disable_correlation', + 'first_seen', + 'last_seen' ); if ($objectId) { $fieldList[] = 'object_id'; diff --git a/app/Model/Event.php b/app/Model/Event.php index 456b3e8ec..7fc0cc4b2 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -1993,7 +1993,7 @@ class Event extends AppModel // do not expose all the data ... $fields = array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.date', 'Event.threat_level_id', 'Event.info', 'Event.published', 'Event.uuid', 'Event.attribute_count', 'Event.analysis', 'Event.timestamp', 'Event.distribution', 'Event.proposal_email_lock', 'Event.user_id', 'Event.locked', 'Event.publish_timestamp', 'Event.sharing_group_id', 'Event.disable_correlation', 'Event.extends_uuid'); - $fieldsAtt = array('Attribute.id', 'Attribute.type', 'Attribute.category', 'Attribute.value', 'Attribute.to_ids', 'Attribute.uuid', 'Attribute.event_id', 'Attribute.distribution', 'Attribute.timestamp', 'Attribute.comment', 'Attribute.sharing_group_id', 'Attribute.deleted', 'Attribute.disable_correlation', 'Attribute.object_id', 'Attribute.object_relation'); + $fieldsAtt = array('Attribute.id', 'Attribute.type', 'Attribute.category', 'Attribute.value', 'Attribute.to_ids', 'Attribute.uuid', 'Attribute.event_id', 'Attribute.distribution', 'Attribute.timestamp', 'Attribute.comment', 'Attribute.sharing_group_id', 'Attribute.deleted', 'Attribute.disable_correlation', 'Attribute.object_id', 'Attribute.object_relation', 'Attribute.first_seen', 'Attribute.last_seen'); $fieldsObj = array('*'); $fieldsShadowAtt = array('ShadowAttribute.id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.value', 'ShadowAttribute.to_ids', 'ShadowAttribute.uuid', 'ShadowAttribute.event_uuid', 'ShadowAttribute.event_id', 'ShadowAttribute.old_id', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.proposal_to_delete', 'ShadowAttribute.timestamp'); $fieldsOrg = array('id', 'name', 'uuid'); diff --git a/app/Model/MispObject.php b/app/Model/MispObject.php index 1610a0e6c..5b651c098 100644 --- a/app/Model/MispObject.php +++ b/app/Model/MispObject.php @@ -46,6 +46,8 @@ class MispObject extends AppModel ), ); + public $shortDist = array(0 => 'Organisation', 1 => 'Community', 2 => 'Connected', 3 => 'All', 4 => ' Sharing Group', 5 => 'Inherit'); + public $validate = array( 'uuid' => array( 'uuid' => array( @@ -60,6 +62,51 @@ class MispObject extends AppModel ) ); + public function afterFind($results, $primary = false) + { + foreach ($results as $k => $v) { + if (!empty($v[$this->alias]['first_seen'])) { + $fs = $results[$k][$this->alias]['first_seen']; + $fs_sec = intval($fs / 1000000); // $fs is in micro (10^6) + $fs_micro = $fs % 1000000; + $fs_micro = str_pad($fs_micro, 6, "0", STR_PAD_LEFT); + $fs = $fs_sec . '.' . $fs_micro; + $results[$k][$this->alias]['first_seen'] = DateTime::createFromFormat('U.u', $fs)->format('Y-m-d\TH:i:s.uP'); + } + if (!empty($v[$this->alias]['last_seen'])) { + $ls = $results[$k][$this->alias]['last_seen']; + $ls_sec = intval($ls / 1000000); // $ls is in micro (10^6) + $ls_micro = $ls % 1000000; + $ls_micro = str_pad($ls_micro, 6, "0", STR_PAD_LEFT); + $ls = $ls_sec . '.' . $ls_micro; + $results[$k][$this->alias]['last_seen'] = DateTime::createFromFormat('U.u', $ls)->format('Y-m-d\TH:i:s.uP'); + } + } + return $results; + } + + public function beforeSave($options = array()) { + // convert into utc and micro sec + if (!empty($this->data[$this->alias]['first_seen'])) { + $d = new DateTime($this->data[$this->alias]['first_seen']); + $d->setTimezone(new DateTimeZone('GMT')); + $fs_sec = $d->format('U'); + $fs_micro = $d->format('u'); + $fs_micro = str_pad($fs_micro, 6, "0", STR_PAD_LEFT); + $fs = $fs_sec . $fs_micro; + $this->data[$this->alias]['first_seen'] = $fs; + } + if (!empty($this->data[$this->alias]['last_seen'])) { + $d = new DateTime($this->data[$this->alias]['last_seen']); + $d->setTimezone(new DateTimeZone('GMT')); + $ls_sec = $d->format('U'); + $ls_micro = $d->format('u'); + $ls_micro = str_pad($ls_micro, 6, "0", STR_PAD_LEFT); + $ls = $ls_sec . $ls_micro; + $this->data[$this->alias]['last_seen'] = $ls; + } + } + public function beforeValidate($options = array()) { parent::beforeValidate(); @@ -75,6 +122,14 @@ class MispObject extends AppModel $date = new DateTime(); $this->data[$this->alias]['timestamp'] = $date->getTimestamp(); } + // parse first_seen different formats + if (isset($this->data[$this->alias]['first_seen'])) { + $this->data[$this->alias]['first_seen'] = $this->data[$this->alias]['first_seen'] === '' ? null : $this->data[$this->alias]['first_seen']; + } + // parse last_seen different formats + if (isset($this->data[$this->alias]['last_seen'])) { + $this->data[$this->alias]['last_seen'] = $this->data[$this->alias]['last_seen'] === '' ? null : $this->data[$this->alias]['last_seen']; + } if (empty($this->data[$this->alias]['template_version'])) { $this->data[$this->alias]['template_version'] = 1; } @@ -254,6 +309,27 @@ class MispObject extends AppModel return $conditions; } + public function fetchObjectSimple($user, $options = array()) + { + $params = array( + 'conditions' => $this->buildConditions($user), + 'fields' => array(), + 'recursive' => -1 + ); + if (isset($options['conditions'])) { + $params['conditions']['AND'][] = $options['conditions']; + } + if (isset($options['fields'])) { + $params['fields'] = $options['fields']; + } + $results = $this->find('all', array( + 'conditions' => $params['conditions'], + 'recursive' => -1, + 'fields' => $params['fields'], + 'sort' => false + )); + return $results; + } // Method that fetches all objects // very flexible, it's basically a replacement for find, with the addition that it restricts access based on user @@ -493,7 +569,42 @@ class MispObject extends AppModel return $attributes; } - public function deltaMerge($object, $objectToSave) + // EUH????? No sure we should do that. Maybe the other way around??? #FIXME + // delete first/last-seen object attribute and set object's meta accordingly + // set object's meta (fs/ls) and potentially delete objectAttributes + public function setObjectSeenMetaFromAttribute($object, $delete=false) { + if ( !$delete && isset($object['Object']['first-seen']) && isset($object['Object']['last-seen'])) { + return; + } + if (isset($object['Attribute'])) { + foreach($object['Attribute'] as $i => $attribute) { + if ($attribute['object_relation'] == 'first-seen') { + // set object meta if unset + if (!isset($object['Object']['first-seen'])) { + $object['Object']['first-seen'] = $attribute['value']; + } + if ($delete) { + $object['Attribute'][$i]['deleted'] = 1; + $this->Event->Attribute->save($object['Attribute'][$i]); + } + continue; + } + if ($attribute['object_relation'] == 'last-seen') { + // set object meta if unset + if (!isset($object['Object']['last-seen'])) { + $object['Object']['last-seen'] = $attribute['value']; + } + if ($delete) { + $object['Attribute'][$i]['deleted'] = 1; + $this->Event->Attribute->save($object['Attribute'][$i]); + } + continue; + } + } + } + } + + public function deltaMerge($object, $objectToSave, $onlyAddNewAttribute=false) { if (!isset($objectToSave['Object'])) { $dataToBackup = array('ObjectReferences', 'Attribute', 'ShadowAttribute'); @@ -512,67 +623,95 @@ class MispObject extends AppModel } unset($dataToBackup); } - $object['Object']['comment'] = $objectToSave['Object']['comment']; - $object['Object']['distribution'] = $objectToSave['Object']['distribution']; - if ($object['Object']['distribution'] == 4) { - $object['Object']['sharing_group_id'] = $objectToSave['Object']['sharing_group_id']; + if (isset($objectToSave['Object']['comment'])) { + $object['Object']['comment'] = $objectToSave['Object']['comment']; + } + if (isset($objectToSave['Object']['distribution'])) { + $object['Object']['distribution'] = $objectToSave['Object']['distribution']; + if ($object['Object']['distribution'] == 4) { + $object['Object']['sharing_group_id'] = $objectToSave['Object']['sharing_group_id']; + } } $date = new DateTime(); $object['Object']['timestamp'] = $date->getTimestamp(); + if (isset($objectToSave['Object']['first_seen'])) { + $object['Object']['first_seen'] = $objectToSave['Object']['first_seen']; + } + if (isset($objectToSave['Object']['last_seen'])) { + $object['Object']['last_seen'] = $objectToSave['Object']['last_seen']; + } + $this->setObjectSeenMetaFromAttribute($object, true); $this->save($object); - $checkFields = array('category', 'value', 'to_ids', 'distribution', 'sharing_group_id', 'comment', 'disable_correlation'); - foreach ($objectToSave['Attribute'] as $newKey => $newAttribute) { - foreach ($object['Attribute'] as $origKey => $originalAttribute) { - if (!empty($newAttribute['uuid'])) { - if ($newAttribute['uuid'] == $originalAttribute['uuid']) { - $different = false; - foreach ($checkFields as $f) { - if ($f == 'sharing_group_id' && empty($newAttribute[$f])) { - $newAttribute[$f] = 0; + + if (!$onlyAddNewAttribute) { + $checkFields = array('category', 'value', 'to_ids', 'distribution', 'sharing_group_id', 'comment', 'disable_correlation'); + foreach ($objectToSave['Attribute'] as $newKey => $newAttribute) { + foreach ($object['Attribute'] as $origKey => $originalAttribute) { + if (!empty($newAttribute['uuid'])) { + if ($newAttribute['uuid'] == $originalAttribute['uuid']) { + $different = false; + foreach ($checkFields as $f) { + if ($f == 'sharing_group_id' && empty($newAttribute[$f])) { + $newAttribute[$f] = 0; + } + if ($newAttribute[$f] != $originalAttribute[$f]) { + $different = true; + } } - if ($newAttribute[$f] != $originalAttribute[$f]) { - $different = true; + if ($different) { + $newAttribute['id'] = $originalAttribute['id']; + $newAttribute['event_id'] = $object['Object']['event_id']; + $newAttribute['object_id'] = $object['Object']['id']; + $newAttribute['timestamp'] = $date->getTimestamp(); + $result = $this->Event->Attribute->save(array('Attribute' => $newAttribute), array( + 'category', + 'value', + 'to_ids', + 'distribution', + 'sharing_group_id', + 'comment', + 'timestamp', + 'object_id', + 'event_id', + 'disable_correlation' + )); } + unset($object['Attribute'][$origKey]); + continue 2; } - if ($different) { - $newAttribute['id'] = $originalAttribute['id']; - $newAttribute['event_id'] = $object['Object']['event_id']; - $newAttribute['object_id'] = $object['Object']['id']; - $newAttribute['timestamp'] = $date->getTimestamp(); - $result = $this->Event->Attribute->save(array('Attribute' => $newAttribute), array( - 'category', - 'value', - 'to_ids', - 'distribution', - 'sharing_group_id', - 'comment', - 'timestamp', - 'object_id', - 'event_id', - 'disable_correlation' - )); - } - unset($object['Attribute'][$origKey]); - continue 2; } } - } - $this->Event->Attribute->create(); - $newAttribute['event_id'] = $object['Object']['event_id']; - $newAttribute['object_id'] = $object['Object']['id']; - if (!isset($newAttribute['timestamp'])) { - $newAttribute['distribution'] = Configure::read('MISP.default_attribute_distribution'); - if ($newAttribute['distribution'] == 'event') { - $newAttribute['distribution'] = 5; + $this->Event->Attribute->create(); + $newAttribute['event_id'] = $object['Object']['event_id']; + $newAttribute['object_id'] = $object['Object']['id']; + if (!isset($newAttribute['timestamp'])) { + $newAttribute['distribution'] = Configure::read('MISP.default_attribute_distribution'); + if ($newAttribute['distribution'] == 'event') { + $newAttribute['distribution'] = 5; + } } + $this->Event->Attribute->save($newAttribute); + $attributeArrays['add'][] = $newAttribute; + unset($objectToSave['Attribute'][$newKey]); + } + foreach ($object['Attribute'] as $origKey => $originalAttribute) { + $originalAttribute['deleted'] = 1; + $this->Event->Attribute->save($originalAttribute); + } + } else { // we only add the new attribute + $newAttribute = $objectToSave['Attribute'][0]; + if ($newAttribute != 'last-seen' && $newAttribute != 'first-seen') { // fs/ls should be added in the object's meta + $this->Event->Attribute->create(); + $newAttribute['event_id'] = $object['Object']['event_id']; + $newAttribute['object_id'] = $object['Object']['id']; + if (!isset($newAttribute['timestamp'])) { + $newAttribute['distribution'] = Configure::read('MISP.default_attribute_distribution'); + if ($newAttribute['distribution'] == 'event') { + $newAttribute['distribution'] = 5; + } + } + $this->Attribute->saveAttributes(array($newAttribute)); } - $this->Event->Attribute->save($newAttribute); - $attributeArrays['add'][] = $newAttribute; - unset($objectToSave['Attribute'][$newKey]); - } - foreach ($object['Attribute'] as $origKey => $originalAttribute) { - $originalAttribute['deleted'] = 1; - $this->Event->Attribute->save($originalAttribute); } return $this->id; } diff --git a/app/View/Attributes/add.ctp b/app/View/Attributes/add.ctp index 5aaa256ce..1550d2d7c 100644 --- a/app/View/Attributes/add.ctp +++ b/app/View/Attributes/add.ctp @@ -1,7 +1,7 @@
Form->create('Attribute', array('id', 'url' => '/attributes/' . $url_params)); + echo $this->Form->create('Attribute', array('id'=>'AttributeForm', 'url' => '/attributes/' . $url_params)); ?>
@@ -78,6 +78,7 @@ echo $this->Form->input('batch_import', array( 'type' => 'checkbox' )); + echo $this->element('form_seen_input'); echo '
'; echo $this->Form->input('disable_correlation', array( 'type' => 'checkbox' diff --git a/app/View/Attributes/ajax/attributeEditFirst_seenForm.ctp b/app/View/Attributes/ajax/attributeEditFirst_seenForm.ctp new file mode 100644 index 000000000..8859c97e8 --- /dev/null +++ b/app/View/Attributes/ajax/attributeEditFirst_seenForm.ctp @@ -0,0 +1,14 @@ +Form->create('Attribute', array('id' => 'Attribute' . '_' . $object['id'] . '_first_seen_form', 'url' => '/attributes/editField/' . $object['id'])); +?> +Form->input('first_seen', array( + 'label' => false, + 'type' => 'text', + 'value' => 0, + 'id' => 'Attribute' . '_' . $object['id'] . '_first_seen_field', + 'div' => false + )); + echo $this->Form->end(); +?> +
diff --git a/app/View/Attributes/ajax/attributeEditLast_seenForm.ctp b/app/View/Attributes/ajax/attributeEditLast_seenForm.ctp new file mode 100644 index 000000000..2f97a1272 --- /dev/null +++ b/app/View/Attributes/ajax/attributeEditLast_seenForm.ctp @@ -0,0 +1,14 @@ +Form->create('Attribute', array('id' => 'Attribute' . '_' . $object['id'] . '_last_seen_form', 'url' => '/attributes/editField/' . $object['id'])); +?> +Form->input('last_seen', array( + 'label' => false, + 'type' => 'text', + 'value' => 0, + 'id' => 'Attribute' . '_' . $object['id'] . '_last_seen_field', + 'div' => false + )); + echo $this->Form->end(); +?> + diff --git a/app/View/Elements/Events/View/row_object.ctp b/app/View/Elements/Events/View/row_object.ctp index 27d85a240..bf0537654 100644 --- a/app/View/Elements/Events/View/row_object.ctp +++ b/app/View/Elements/Events/View/row_object.ctp @@ -1,7 +1,7 @@ - - + +
+
+   +
  @@ -147,5 +150,6 @@ 'child' => $attrKey == $lastElement ? 'last' : true )); } + echo ''; } ?> diff --git a/app/View/Elements/form_seen_input.ctp b/app/View/Elements/form_seen_input.ctp new file mode 100644 index 000000000..281e58189 --- /dev/null +++ b/app/View/Elements/form_seen_input.ctp @@ -0,0 +1,444 @@ +Html->script('moment-with-locales'); ?> + +
+Form->input('first_seen', array( + 'type' => 'text', + 'div' => 'input hidden', + 'required' => false, + )); + echo $this->Form->input('last_seen', array( + 'type' => 'text', + 'div' => 'input hidden', + 'required' => false, + )); +?> +
+ +
+ + + diff --git a/app/View/Elements/view_timeline.ctp b/app/View/Elements/view_timeline.ctp new file mode 100644 index 000000000..cde3ff8c1 --- /dev/null +++ b/app/View/Elements/view_timeline.ctp @@ -0,0 +1,33 @@ + + +
+
+ + + + +
+ + +
+
+ +
+
+
+ +
+ +Html->script('moment-with-locales'); + echo $this->Html->script('event-timeline'); + echo $this->Html->css('event-timeline'); +?> diff --git a/app/View/Events/view.ctp b/app/View/Events/view.ctp index fc9a62bf8..b5aeea590 100644 --- a/app/View/Events/view.ctp +++ b/app/View/Events/view.ctp @@ -467,6 +467,9 @@ + @@ -492,6 +495,9 @@ + diff --git a/app/View/Objects/ajax/objectViewFieldForm.ctp b/app/View/Objects/ajax/objectViewFieldForm.ctp new file mode 100644 index 000000000..236555c43 --- /dev/null +++ b/app/View/Objects/ajax/objectViewFieldForm.ctp @@ -0,0 +1,2 @@ +Form->create('Object', array( + 'id' => 'Object_' . $object['id'] . '_quick_add_attribute_form', + 'url' => $url, + 'class' => 'allDiv' + )); +?> + +
+ +
+ 'hidden', + 'value' => $element['object_relation'], + 'label' => false, + 'div' => false + ); + echo $this->Form->input('Attribute.' . $k . '.object_relation', $formSettings); + + $formSettings['value'] = $object['id']; + echo $this->Form->input('Object.id', $formSettings); + + $formSettings['value'] = $object['event_id']; + echo $this->Form->input('Object.event_id', $formSettings); + + $formSettings['value'] = $element['type']; + echo '
'; + echo $this->Form->input('Attribute.' . $k . '.type', $formSettings); + echo '' . Inflector::humanize(h($element['object_relation'])) . ''; + echo ' :: ' . h($element['type']) . ''; + echo '
'; + echo '' . h($element['description']) . ''; + echo '
'; + ?> + + + array_combine($element['categories'], $element['categories']), + 'default' => $element['default_category'], + 'div' => true + ); + echo $this->Form->input('Attribute.' . $k . '.category', $formSettings); + ?> + +
+ ' . __('Value') . ''; + echo $this->element( + 'Objects/object_value_field', + array( + 'element' => $element, + 'k' => $k, + 'action' => $action + ) + ); + ?> +
+ + Form->input('Attribute.' . $k . '.to_ids', array( + 'type' => 'checkbox', + 'checked' => $element['to_ids'], + )); + ?> + + Form->input('Attribute.' . $k . '.disable_correlation', array( + 'type' => 'checkbox', + 'checked' => $element['disable_correlation'], + )); + ?> + +
+ Form->input('Attribute.' . $k . '.distribution', array( + 'class' => 'Attribute_distribution_select', + 'options' => $distributionData['levels'], + 'default' => !empty($element['distribution']) ? $element['distribution'] : $distributionData['initial'], + 'div' => false, + 'label' => __('Distribution ') . $this->element('formInfo', array('type' => 'distribution')), + )); + ?> +
+
+ +
+ + Form->input('Attribute.' . $k . '.comment', array( + 'type' => 'textarea', + 'required' => false, + 'allowEmpty' => true, + 'div' => 'input clear', + 'class' => 'input-xxlarge' + )); + ?> + +
+
+ +
+ + + Form->button('Submit', array('class' => 'btn btn-primary')); + endif; + ?> + +
+ +Form->end(); +?> + + + diff --git a/app/View/Objects/ajax/templateWithValidObjectAttributes.ctp b/app/View/Objects/ajax/templateWithValidObjectAttributes.ctp new file mode 100644 index 000000000..aab0101e5 --- /dev/null +++ b/app/View/Objects/ajax/templateWithValidObjectAttributes.ctp @@ -0,0 +1,22 @@ +
+ +
+ + + + + + +
' . h($objectAttribute['object_relation']) . '' . ' :: ' . h($objectAttribute['type']) . '
' . h($objectAttribute['description']); ?>
+
+
+
+ diff --git a/app/View/Objects/revise_object.ctp b/app/View/Objects/revise_object.ctp index 70c19dd0d..6ef651c7c 100644 --- a/app/View/Objects/revise_object.ctp +++ b/app/View/Objects/revise_object.ctp @@ -49,6 +49,13 @@ + + + + + + + diff --git a/app/webroot/css/event-timeline.css b/app/webroot/css/event-timeline.css new file mode 100644 index 000000000..26799f823 --- /dev/null +++ b/app/webroot/css/event-timeline.css @@ -0,0 +1,150 @@ +.info_container_eventtimeline { + margin-top:10px; + border: 1px solid #0088cc; + border-radius: 7px; + box-shadow: 0px 0px 6px #B2B2B2; + width: 100%; + min-height: 100px; + padding: 1px; +} + +.vis-timeline { + position: relative; + border: none; + overflow: hidden; + padding: 0; + margin: 0; + box-sizing: border-box; +} + +.fullscreen-btn-timeline { + float: right; + display: inline-block; + bottom: 33px; + position: relative; + right: 3px; + opacity: 0.7; + z-index: 1; + cursor: nesw-resize; +} + +.loadingTimeline { + display: none; + position: absolute; + margin-top: 10px; + width: 100%; + text-align: center; +} + +.loadingTimeline > div.loadingText { + left: 0px; +} + +.timestamp-obj { + box-shadow: #ff0000 0px 0px 0px 5px; +} + +.timestamp-attr { + box-shadow: #ff0000 0px 0px 0px 5px; + padding-top: 4px; + padding-bottom: 4px; +} + +.vis-item.attribute { + background-color: orange; + border-color: black; +} +.vis-item.attribute.vis-selected { + box-shadow: 0 0 20px rgba(82, 168, 236, 1);` +} + +.vis-item.object { + background-color: #3465a4; + border-color: black; + color: white; +} +.vis-item.object.vis-selected { + box-shadow: 0 0 20px rgba(82, 168, 236, 1);` +} + +.vis-item.object_attribute { + background-color: greenyellow; + border-color: green; +} + +.timeline-objectName { + text-align:left; + border-bottom: white solid 1px; +} +.timeline-objectAttrType { + text-align:left; + line-height: 14px; + font-style: italic; +} +.timeline-objectAttrVal { + text-align:left; + line-height: 14px; + padding-left: 5px; +} + +.vis-delete { + top: -8px !important; + padding: 3px !important; + height: 22px !important; + width: 20px !important; +} + +.vis-expand-action { + position: absolute; + + padding: 3px 2px !important; + height: 22px !important; + width: 22px !important; + + + top: 15px; + right: -25px; + box-sizing: border-box; + cursor: pointer; + + -webkit-transition: background 0.2s linear; + -moz-transition: background 0.2s linear; + -ms-transition: background 0.2s linear; + -o-transition: background 0.2s linear; + transition: background 0.2s linear; +} +.vis-item .vis-expand-action:after { + color: green; + font-family: arial, sans-serif; + font-size: 22px; + font-weight: bold; + + -webkit-transition: color 0.2s linear; + -moz-transition: color 0.2s linear; + -ms-transition: color 0.2s linear; + -o-transition: color 0.2s linear; + transition: color 0.2s linear; +} +.expand-btn:after { + content: "\21d4"; +} +.collapse-btn:after { + content: "\21b5"; +} +.vis-item .vis-expand-action:hover { + background: green; +} +.vis-item .vis-expand-action:hover:after { + color: white; +} + +.vis-onUpdateTime-tooltip { + background: #7b7b7b !important; + padding: 0px !important; + border-radius: 5px !important; + border: solid 1px #000 !important; +} + +#timeline-typeahead { + width: 400px; +} diff --git a/app/webroot/css/main.css b/app/webroot/css/main.css index 364d49451..6553ba4b4 100644 --- a/app/webroot/css/main.css +++ b/app/webroot/css/main.css @@ -21,6 +21,10 @@ button .full-width { margin-bottom:10px; } +form button.btn[disabled] { + cursor: not-allowed; +} + /*.close{ float:none; }*/ @@ -289,6 +293,26 @@ td.highlight3 { } +table-striped tbody > tr > td.objectAddField { + padding: 2px 3px; + line-height: normal; + display: inline-block; + position: relative; + top: -13px; + left: -4px; + color: white; + background-color: #1369a0; + border: 0px; + border-radius: 15px; + height: 12px; + width: 10px; +} + +.table-striped tbody > tr > td.objectAddField:hover { + transform: scale(1.7); + cursor: pointer; +} + tr.highlightBlueSides { border-left:2px solid #0088cc; border-right:2px solid #0088cc; @@ -836,6 +860,20 @@ a.proposal_link_red:hover { border: 0; } +.immutableAttributeDescription:before { + background: rgba(255, 255, 255, 0); + content: ''; + margin-right: 5px; + margin-left: 2px; + border-left: 1px solid; + border-bottom: 1px solid; + height: 10px; + width: 5px; + position: relative; + display: block; + float: left; +} + .ajax_popover_form { display:none; width: 700px; @@ -2219,6 +2257,23 @@ table tr:hover .down-expand-button { position: relative; } +.nanosec-div { + font-weight:bold; + position: relative; + width: 100%; + text-align: center; + top: -5px; +} + +.larger-input-field { + width: 270px !important; +} + +.precision-tool { + margin-left: 10px; + display: inline-block; +} + .fa-as-icon { font-size: 20px; padding-left: 2px; diff --git a/app/webroot/js/bootstrap-datepicker.js b/app/webroot/js/bootstrap-datepicker.js index fb56652e1..42b9d9c2c 100644 --- a/app/webroot/js/bootstrap-datepicker.js +++ b/app/webroot/js/bootstrap-datepicker.js @@ -1461,6 +1461,7 @@ return i.jquery ? i[0] : i; }); delete options.inputs; + this.preventMultipleSet = options.preventMultipleSet; datepickerPlugin.call($(this.inputs), options) .on('changeDate', $.proxy(this.dateUpdated, this)); @@ -1507,10 +1508,12 @@ if (i === -1) return; - $.each(this.pickers, function(i, p){ - if (!p.getUTCDate()) - p.setUTCDate(new_date); - }); + if (!this.preventMultipleSet) { + $.each(this.pickers, function(i, p){ + if (!p.getUTCDate()) + p.setUTCDate(new_date); + }); + } if (new_date < this.dates[j]){ // Date being moved earlier/left diff --git a/app/webroot/js/contextual_menu.js b/app/webroot/js/contextual_menu.js index 455472d66..bc8ab94c9 100644 --- a/app/webroot/js/contextual_menu.js +++ b/app/webroot/js/contextual_menu.js @@ -143,6 +143,7 @@ class ContextualMenu { __create_menu_div_bootstrap_popover() { var div = document.createElement('div'); div.classList.add("contextual-menu"); + div.style.display = 'none'; this.container.appendChild(div); var that = this; this.trigger_container.tabIndex = 0; // required for the popover focus feature @@ -151,19 +152,20 @@ class ContextualMenu { container: 'body', html: true, placement: "bottom", - content: function () {return $(that.menu); }, // return contextual menu html + content: function () { var html=$(that.menu); html.css('display', 'inline-block'); return html;}, // return contextual menu html trigger: "manual", template: '' }) // Overwrite the default popover behavior: hidding cause the popover to be detached from the DOM, making impossible to fetch input values in the form $(this.trigger_container).click (function(e) { - if (that.has_been_shown_once) { - $('#popover-contextual-menu-'+this.id).toggle(); - } else { - that.has_been_shown_once = true; - $(this).popover('show'); - } + $(this).popover('toggle'); + // if (that.has_been_shown_once) { + // $('#popover-contextual-menu-'+this.id).toggle(); + // } else { + // that.has_been_shown_once = true; + // $(this).popover('show'); + // } }); return div; } @@ -371,7 +373,7 @@ class ContextualMenu { this.menu.appendChild(label); this.menu.appendChild(div); if(options.event !== undefined) { - button.addEventListener("click", function(evt) { + button.addEventListener("click", function(evt) { var corresponding_select_id = evt.target.dataset.correspondingId; var selected_value = $('#'+corresponding_select_id).val(); options.event(selected_value); diff --git a/app/webroot/js/event-graph.js b/app/webroot/js/event-graph.js index cb83f30b0..7c2578e14 100644 --- a/app/webroot/js/event-graph.js +++ b/app/webroot/js/event-graph.js @@ -155,7 +155,7 @@ class EventGraph { $("#select_graph_scope").val(value); } - if (value == "Rotation key") { + if (value == "Pivot key") { $("#network-scope-badge").text(value + ": " + eventGraph.scope_keyType); } else { $("#network-scope-badge").text(value); @@ -176,30 +176,30 @@ class EventGraph { label: "Scope", tooltip: "The scope represented by the network", event: function(value) { - if (value == "Rotation key" && $('#input_graph_scope_jsonkey').val() == "") { // no key selected for Rotation key scope + if (value == "Pivot key" && $('#input_graph_scope_jsonkey').val() == "") { // no key selected for Pivot key scope return; } else { eventGraph.update_scope(value); dataHandler.fetch_data_and_update(); } }, - options: ["Reference", "Tag", "Rotation key"], + options: ["Reference", "Tag", "Pivot key"], default: "Reference" }); menu_scope.add_select({ id: "input_graph_scope_jsonkey", - label: "Rotation key", + label: "Pivot key", tooltip: "The key around which the network will be constructed", event: function(value) { - if (value == "Rotation key" && $('#input_graph_scope_jsonkey').val() == "") { // no key selected for Rotation key scope + if (value == "Pivot key" && $('#input_graph_scope_jsonkey').val() == "") { // no key selected for Pivot key scope return; } else { eventGraph.scope_keyType = value; - eventGraph.update_scope("Rotation key"); + eventGraph.update_scope("Pivot key"); dataHandler.fetch_data_and_update(); } }, - options: dataHandler.available_rotation_key ? dataHandler.available_rotation_key : [], + options: dataHandler.available_pivot_key ? dataHandler.available_pivot_key : [], default: "" }); return menu_scope; @@ -298,7 +298,8 @@ class EventGraph { for(var nodeId of objectIds) { eventGraph.expand_node(nodeId); } - } + }, + title: "Expanding all nodes may takes some time" }); menu_display.add_button({ label: "Collapse all nodes", @@ -310,7 +311,8 @@ class EventGraph { for(var nodeId of objectIds) { eventGraph.collapse_node(nodeId); } - } + }, + title: "Collapsing all nodes may takes some time" }); menu_display.add_slider({ id: 'slider_display_max_char_num', @@ -1427,8 +1429,8 @@ class DataHandler { return [[index, value]]; }); dataHandler.update_filtering_selectors(available_object_references, available_tags); - dataHandler.available_rotation_key = data.available_rotation_key; - eventGraph.menu_scope.add_options("input_graph_scope_jsonkey", dataHandler.available_rotation_key); + dataHandler.available_pivot_key = data.available_pivot_key; + eventGraph.menu_scope.add_options("input_graph_scope_jsonkey", dataHandler.available_pivot_key); if (data.items.length < nodes_ask_threshold) { eventGraph.update_graph(data); } else if (data.items.length > nodes_ask_threshold && confirm("The network contains a lot of nodes, displaying it may slow down your browser. Continue?")) { diff --git a/app/webroot/js/event-timeline.js b/app/webroot/js/event-timeline.js new file mode 100644 index 000000000..ac03965e0 --- /dev/null +++ b/app/webroot/js/event-timeline.js @@ -0,0 +1,586 @@ +var max_displayed_char_timeline = 64; +var eventTimeline; +var items_timeline; +var items_backup; +var use_local_timezone = true; +var mapping_text_to_id = new Map(); +var user_manipulation = $('#event_timeline').data('user-manipulation'); +var extended_text = $('#event_timeline').data('extended') == 1 ? "extended:1/" : ""; +var container_timeline = document.getElementById('event_timeline'); +var hardThreshold = 50; +var softThreshold = 30; +var timeline_disabled = false; +var default_editable = { + add: false, // add new items by double tapping + updateTime: true, // drag items horizontally + remove: true +}; +var relationship_type_mapping = { + 'followed-by': 'after', + 'preceding-by': 'before', +} +var options = { + template: function (item, element, data) { + switch(item.group) { + case "attribute": + return build_attr_template(item); + + case "object": + return build_object_template(item); + + case "object_attribute": + console.log('Error'); + break; + + default: + break; + } + }, + moment: function(date) { + if (use_local_timezone) { + return vis.moment(date); + } else { + return vis.moment(date).utc(); + } + }, + verticalScroll: true, + zoomKey: 'altKey', + maxHeight: 400, + minHeight: 400, + multiselect: true, + editable: user_manipulation ? default_editable : false, + tooltipOnItemUpdateTime: true, + onRemove: function(item, callback) { // clear timestamps + update_seen(item, 'first', null, false, undefined); + update_seen(item, 'last', null, false, function() { reflect_change(true); }); + eventTimeline.setSelection([]); + $('.timelineSelectionTooltip').remove() + }, + onMove: function(item, callback) { + var newStart = moment(item.start.toISOString()); + var newEnd = (item.end !== undefined && item.end !== null) ? moment(item.end.toISOString()) : null; + var c1 = item.first_seen !== null ? !item.first_seen.isSame(newStart) : true; + var c2 = item.last_seen !== null ? !item.last_seen.isSame(newEnd) && item.seen_enabled : true; + if (c1) { + if (item.first_seen === null) { + if (!c2) { + update_seen(item, 'first', newStart, true, undefined); + } else { + update_seen(item, 'first', newStart, false, function() { reflect_change(true); }); + } + } else { + update_seen(item, 'first', newStart, !c2, undefined); + } + } + if (c2) { + update_seen(item, 'last', newEnd, true, undefined); + } + } +}; +var timeline_typeaheadDataSearch; +var timeline_typeaheadOption = { + source: function (query, process) { + if (timeline_typeaheadDataSearch === undefined) { // caching + timeline_typeaheadDataSearch = Array.from(mapping_text_to_id.keys()); + } + process(timeline_typeaheadDataSearch); + }, + updater: function(value) { + var id = mapping_text_to_id.get(value); + eventTimeline.focus(id); + $("#timeline-typeahead").blur(); + }, + autoSelect: true +} + +function isDefined(element) { + return element !== undefined && element !== null; +} + +function generate_timeline_tooltip(itemID, target) { + var item = items_timeline.get(itemID); + if ( + item.first_seen === undefined + || item.first_seen === null + || item.first_seen_overwrite + ) { // do not generate if first_seen not set + return; + } + var closest = $(target.closest(".vis-selected.vis-editable")); + var btn_type = item.last_seen !== null ? 'collapse-btn' : 'expand-btn'; + var fct_type = item.last_seen !== null ? 'collapseItem' : 'expandItem'; + var btn = $('
') + if (item.last_seen !== null) { + btn.click(collapseItem); + } else { + btn.click(expandItem); + } + closest.append(btn); +} + +/* UTIL */ +function collapseItem() { + var itemID = $(this).data('itemid'); + var item = items_timeline.get(itemID); + update_seen(item, 'last', null, true, undefined); +} +function expandItem() { + var itemID = $(this).data('itemid'); + var item = items_timeline.get(itemID); + var newEnd = get_next_step(item.first_seen); + update_seen(item, 'last', newEnd, true, undefined); +} + +function get_next_step(mom) { + var scale = eventTimeline.timeAxis.step.scale; + var momAhead = mom.clone(); + momAhead.add(1, scale); + return momAhead; +} + +function build_attr_template(attr) { + var span = $(''); + if (!attr.seen_enabled) { + span.addClass('timestamp-attr'); + } + span.text(attr.content); + span.data('seen_enabled', attr.seen_enabled); + var html = span[0].outerHTML; + return html; +} + +function build_object_template(obj) { + var table = $('
'); + table.data('seen_enabled', obj.seen_enabled); + if (!obj.seen_enabled) { + table.addClass('timestamp-obj'); + } + var bolt_html = obj.overwrite_enabled ? " " : ""; + table.append($('')); + for (var attr of obj.Attribute) { + var overwritten = obj.overwrite_enabled && (attr.contentType == "first-seen" || attr.contentType == "last-seen") ? " " : ""; + table.append( + $('').append( + $('' + +'' + ) + ) + ) + } + var html = table[0].outerHTML; + return html; +} + +function reflect_change(onIndex, itemType, itemId) { + if (onIndex) { + updateIndex(scope_id, 'event'); // MISP function + } else { // reflect change on item only + quick_fetch_seens(itemType, itemId, function(firstSeen, lastSeen) { + var updatedItem = items_timeline.get(itemId); + updatedItem.first_seen = firstSeen; + updatedItem.last_seen = lastSeen; + updatedItem.first_seen_overwrite = false; + updatedItem.last_seen_overwrite = false; + var e = $.extend({}, default_editable); + e.remove = true; + updatedItem.editable = e; + set_spanned_time(updatedItem); + items_timeline.remove(updatedItem.id); + items_timeline.add(updatedItem); + }); + } +} + +function quick_fetch_seens(itemType, itemId, callback) { + var url = "/" + itemType + "/" + "fetchViewValue" + "/" + itemId + "/"; + var dfs = $.ajax({ + dataType: "html", + cache: false, + success: function(data, textStatus) { + return data; + }, + url: url+"first_seen" + }); + var dls = $.ajax({ + dataType: "html", + cache: false, + success: function(data, textStatus) { + return data; + }, + url: url+"last_seen" + }); + + $.when( dfs, dls).done(function(a1, a2) { + firstSeen = a1[0].replace(' ', ''); + firstSeen = firstSeen == '' ? null : firstSeen; + lastSeen = a2[0].replace(' ', ''); + lastSeen = lastSeen == '' ? null : lastSeen; + callback(firstSeen, lastSeen); + }); +} + +function update_seen(item, seenType, value, reflect, callback) { + var itemType = item.group + 's'; + var momentISO = value !== null ? value.toISOString() : null; + fetch_form_and_submit(itemType, item, seenType, momentISO, reflect, callback); +} + +function fetch_form_and_submit(itemType, item, seenType, value, reflect, callback) { + var url = "/" + itemType + "/fetchEditForm/" + item.id + "/" + seenType+"_seen"; + $.ajax({ + beforeSend: function (XMLHttpRequest) { + $(".loadingTimeline").show(); + }, + dataType:"html", + cache: false, + success: function (data, textStatus) { + var form = $(data); + $(container_timeline).append(form); + form.css({display: 'none'}); + var field = form.find('input[name*="' + seenType + '_seen"]'); + field.val(value); + // submit the form + $.ajax({ + data: form.serialize(), + cache: false, + success:function (data, textStatus) { + if (reflect) { + reflect_change(false, itemType, item.id); + } + form.remove() + }, + error:function() { + console.log('fail', 'Request failed for an unknown reason.'); + }, + complete: function () { + $(".loadingTimeline").hide(); + if (callback !== undefined) { + callback(); + } + }, + type:"post", + url: form.attr('action') + }); + }, + error: function() { + console.log('Feature not supported.'); + }, + url: url, + }); +} + +function timestampToMoment(timestamp) { + var factor = 1000; + var d = moment(timestamp*factor); + return d; +} + +function set_spanned_time(item) { + var timestamp = item.timestamp; + var fs = item.first_seen == null ? null : moment(item.first_seen); + var ls = item.last_seen == null ? null : moment(item.last_seen); + item.first_seen = fs; + item.last_seen = ls; + + item.seen_enabled = false; + item.overwrite_enabled = false; + if (fs===null && ls===null) { + item.start = timestampToMoment(timestamp); + item.type = 'box'; + + } else if (fs===null && ls!==null) { + item.start = timestampToMoment(timestamp); + item.type = 'box'; + + } else if (ls===null && fs!==null) { + item.start = fs; + item.seen_enabled = true; + delete item.end; + item.type = 'box'; + + } else { // fs and ls are defined + item.start = fs; + item.end = ls; + item.seen_enabled = true; + if (fs == ls) { + item.type = 'box'; + } else { + item.type = 'range'; + } + } + + if (item.first_seen_overwrite === true || item.last_seen_overwrite === true) { + var e = $.extend({}, default_editable); + e.remove = false; + item.editable = e; + item.overwrite_enabled = true; + } +} + +function map_scope(val) { + switch(val) { + case 'First seen/Last seen': + return 'seen'; + case 'Object relationship': + return 'relationship'; + default: + return 'seen'; + } +} + +function timelinePopupCallback(state) { + if (eventTimeline === undefined) { + return; + } + reload_timeline(); +} + +function adjust_text_length(elem) { + var maxChar = $('#slider_timeline_display_max_char_num').val(); + elem.content = elem.content.substring(0, maxChar) + (elem.content.length < maxChar ? "" : "[...]"); +} + +function update_badge() { + if (use_local_timezone) { + $("#timeline-display-badge").text("Timezone: " + ": " + moment().format('Z')); + } else { + $("#timeline-display-badge").text("Timezone: " + ": " + moment().utc().format('Z (z)')); + } +} + +function reload_timeline() { + update_badge(); + var payload = {scope: map_scope($('#select_timeline_scope').val())}; + $.ajax({ + url: "/events/"+"getEventTimeline"+"/"+scope_id+"/"+extended_text+"event.json", + dataType: 'json', + type: 'post', + contentType: 'application/json', + data: JSON.stringify( payload ), + processData: false, + beforeSend: function (XMLHttpRequest) { + $(".loadingTimeline").show(); + }, + success: function( data, textStatus, jQxhr ){ + items_timeline.clear(); + for (var item of data.items) { + item.className = item.group; + set_spanned_time(item); + if (item.group == 'object') { + for (var attr of item.Attribute) { + mapping_text_to_id.set(attr.contentType+': '+attr.content+' ('+item.id+')', item.id); + adjust_text_length(attr); + } + } else { + mapping_text_to_id.set(item.content+' ('+item.id+')', item.id); + adjust_text_length(item); + } + } + items_timeline.add(data.items); + handle_not_seen_enabled($('#checkbox_timeline_display_hide_not_seen_enabled').val()) + }, + error: function( jqXhr, textStatus, errorThrown ){ + console.log( errorThrown ); + }, + complete: function() { + $(".loadingTimeline").hide(); + } + }); +} + +function enable_timeline() { + if (eventTimeline !== undefined) { + return; + } + + init_popover(); + + $('#timeline-typeahead').typeahead(timeline_typeaheadOption); + + var payload = {scope: map_scope($('#select_timeline_scope').val())}; + $.ajax({ + url: "/events/"+"getEventTimeline"+"/"+scope_id+"/"+extended_text+"event.json", + dataType: 'json', + type: 'post', + contentType: 'application/json', + data: JSON.stringify( payload ), + processData: false, + beforeSend: function (XMLHttpRequest) { + $(".loadingTimeline").show(); + }, + success: function( data, textStatus, jQxhr ){ + if (data.items.length > hardThreshold) { + $('#eventtimeline_div').html('
Timeline: To much data to show
'); + timeline_disabled = true; + return; + } else if (data.items.length > softThreshold) { + var res = confirm('You are about to draw a lot ('+data.items.length+') of items in the timeline. Do you wish to continue?'); + if (!res) { + $('#eventtimeline_div').html('
Timeline: To much data to show
'); + timeline_disabled = true; + return; + } + } + + for (var item of data.items) { + item.className = item.group; + set_spanned_time(item); + if (item.group == 'object') { + for (var attr of item.Attribute) { + mapping_text_to_id.set(attr.contentType+': '+attr.content+' ('+item.id+')', item.id); + adjust_text_length(attr); + } + } else { + mapping_text_to_id.set(item.content+' ('+item.id+')', item.id); + adjust_text_length(item); + } + } + items_timeline = new vis.DataSet(data.items); + eventTimeline = new vis.Timeline(container_timeline, items_timeline, options); + update_badge(); + + eventTimeline.on('select', handle_selection); + + eventTimeline.on('doubleClick', handle_doubleClick); + + items_timeline.on('update', function(eventname, data) { + handle_selection({ + event: { target: $('span[data-itemID="'+data.items[0]+'"]')}, + items: data.items + }); + }); + }, + error: function( jqXhr, textStatus, errorThrown ){ + console.log( errorThrown ); + }, + complete: function() { + $(".loadingTimeline").hide(); + } + }); +} + +function handle_selection(data) { + var event = data.event; + var target = event.target; + var items = data.items; + + if (items.length == 0) { + $('.timelineSelectionTooltip').remove() + } else { + for (var itemID of items) { + generate_timeline_tooltip(itemID, target); + } + } +} + +function edit_item(id, callback) { + var group = items_timeline.get(id).group; + if (group == 'attribute') { + simplePopup('/attributes/edit/'+id); + } else if (group == 'object') { + window.location = '/objects/edit/'+id; + } +} + +function handle_doubleClick(data) { + // should be replaced by keyboard shortcut: SHIFT+E ? + //edit_item(data.item); +} + +function handle_not_seen_enabled(hide) { + if (hide) { + var hidden = items_timeline.get({ + filter: function(item) { + return !item.seen_enabled; + } + }); + var hidden_ids = []; + items_timeline.forEach(function(item) { + hidden_ids.push(item.id); + }); + items_timeline.remove(hidden) + items_backup = hidden; + } else { + items_timeline.add(items_backup); + } +} + +$('#fullscreen-btn-timeline').click(function() { + var timeline_div = $('#eventtimeline_div'); + var fullscreen_enabled = !timeline_div.data('fullscreen'); + timeline_div.data('fullscreen', fullscreen_enabled); + var height_val = fullscreen_enabled == true ? "calc(100vh - 42px - 42px - 10px)" : "400px"; + + timeline_div.css("max-height", height_val); + setTimeout(function() { // timeline takes time to be drawn + timeline_div[0].scrollIntoView({ + behavior: "smooth", + }); + }, 1); + eventTimeline.setOptions({maxHeight: height_val}); +}); + +// init_scope_menu +var menu_scope_timeline, menu_display_timeline; +function init_popover() { + if (timeline_disabled) return; + menu_scope_timeline = new ContextualMenu({ + trigger_container: document.getElementById("timeline-scope"), + bootstrap_popover: true, + style: "z-index: 1", + container: document.getElementById("eventtimeline_div") + }); + menu_scope_timeline.add_select({ + id: "select_timeline_scope", + label: "Scope", + tooltip: "The time scope represented by the timeline", + event: function(value) { + if (value == "First seen/Last seen") { + reload_timeline(); + } + }, + options: ["First seen/Last seen"], + default: "First seen/Last seen" + }); + + menu_display_timeline = new ContextualMenu({ + trigger_container: document.getElementById("timeline-display"), + bootstrap_popover: true, + style: "z-index: 1", + container: document.getElementById("eventtimeline_div") + }); + menu_display_timeline.add_slider({ + id: 'slider_timeline_display_max_char_num', + label: "Charater to show", + title: "Maximum number of charater to display in the label", + min: 8, + max: 2048, + value: max_displayed_char_timeline, + step: 8, + applyButton: true, + event: function(value) { + $("#slider_timeline__display_max_char_num").parent().find("span").text(value); + }, + eventApply: function(value) { + reload_timeline(); + } + }); + menu_display_timeline.add_checkbox({ + id: 'checkbox_timeline_display_hide_not_seen_enabled', + label: "Hide first seen not set", + title: "Hide items that does not have first seen sets", + event: function(value) { + handle_not_seen_enabled(value) + } + }); + menu_display_timeline.add_checkbox({ + id: 'checkbox_timeline_display_gmt', + label: "Display with current timezone", + title: "Set the dates relative to the browser timezone. Otherwise, keep dates in GMT", + event: function(value) { + use_local_timezone = value; + reload_timeline() + }, + checked: true + }); +} diff --git a/app/webroot/js/misp.js b/app/webroot/js/misp.js index c3264b84a..88dcb7cec 100644 --- a/app/webroot/js/misp.js +++ b/app/webroot/js/misp.js @@ -409,6 +409,11 @@ function updateIndex(id, context, newPage) { } else { console.log("genericPopupCallback function not defined"); } + if (typeof timelinePopupCallback !== "undefined") { + timelinePopupCallback("success"); + } else { + console.log("timelinepopupcallback function not defined"); + } }, url: url, }); @@ -437,15 +442,42 @@ function updateAttributeFieldOnSuccess(name, type, id, field, event) { }); } +function updateObjectFieldOnSuccess(name, type, id, field, event) { + $.ajax({ + beforeSend: function (XMLHttpRequest) { + if (field != 'timestamp') { + $(".loading").show(); + } + }, + dataType:"html", + cache: false, + success:function (data, textStatus) { + if (field != 'timestamp') { + $(".loading").hide(); + $(name + '_solid').html(data); + $(name + '_placeholder').empty(); + $(name + '_solid').show(); + } else { + $('#' + type + '_' + id + '_' + 'timestamp_solid').html(data); + } + }, + url:"/objects/fetchViewValue/" + id + "/" + field, + }); +} + function activateField(type, id, field, event) { resetForms(); if (type == 'denyForm') return; var objectType = 'attributes'; + var containerName = 'Attribute'; if (type == 'ShadowAttribute') { objectType = 'shadow_attributes'; + } else if (type == 'Object') { + objectType = 'objects'; + containerName = 'Object'; } var name = '#' + type + '_' + id + '_' + field; - var container_name = '#Attribute_' + id + '_' + field; + var container_name = '#' + containerName + id + '_' + field; $.ajax({ beforeSend: function (XMLHttpRequest) { $(".loading").show(); @@ -583,6 +615,8 @@ function submitForm(type, id, field, context) { var name = '#' + type + '_' + id + '_' + field; if (type == 'ShadowAttribute') { object_type = 'shadow_attributes'; + } else if (type == 'Object') { + object_type = 'objects'; } $.ajax({ data: $(name + '_field').closest("form").serialize(), @@ -723,7 +757,8 @@ function handleAjaxEditResponse(data, name, type, id, field, event) { responseArray = data; if (type == 'Attribute') { if (responseArray.saved) { - showMessage('success', responseArray.success); + var msg = responseArray.success !== undefined ? responseArray.success : responseArray.message; + showMessage('success', msg); updateAttributeFieldOnSuccess(name, type, id, field, event); updateAttributeFieldOnSuccess(name, type, id, 'timestamp', event); eventUnpublish(); @@ -734,12 +769,75 @@ function handleAjaxEditResponse(data, name, type, id, field, event) { } if (type == 'ShadowAttribute') { updateIndex(event, 'event'); + } else if (type == 'Object') { + if (responseArray.saved) { + var msg = responseArray.success !== undefined ? responseArray.success : responseArray.message; + showMessage('success', msg); + updateObjectFieldOnSuccess(name, type, id, field, event); + updateObjectFieldOnSuccess(name, type, id, 'timestamp', event); + eventUnpublish(); + } else { + showMessage('fail', 'Validation failed: ' + responseArray.errors.value); + updateObjectFieldOnSuccess(name, type, id, field, event); + } } if (responseArray.hasOwnProperty('check_publish')) { checkAndSetPublishedInfo(); } } +function quickFetchValidObjectAttribute(objectId) { + var itemType = "objects"; + var formUrl= "quickFetchTemplateWithValidObjectAttributes"; + var compiledUrlForm = "/" + itemType + "/" + formUrl + "/" + objectId; + $.ajax({ + beforeSend: function (XMLHttpRequest) { + $(".loading").show(); + }, + success:function (data, textStatus) { + if (data.fail !== undefined && data.fail) { + showMessage('fail', data.errors); + } else { + $('#popover_form').html(data); + openPopup('#popover_form'); + } + }, + error:function() { + showMessage('fail', 'Could not fetch allowed attribute type.'); + }, + complete:function() { + $(".loading").hide(); + }, + type: "get", + url: compiledUrlForm + }); + return false; +} + +function fetchAddObjectAttributeForm(objectId, fieldName) { + var itemType = "objects"; + var formUrl= "quickAddAttributeForm"; + var compiledUrlForm = "/" + itemType + "/" + formUrl + "/" + objectId + "/" + fieldName; + $.ajax({ + beforeSend: function (XMLHttpRequest) { + $(".loading").show(); + }, + success:function (data, textStatus) { + $('#popover_form').html(data); + openPopup('#popover_form'); + }, + error:function() { + showMessage('fail', 'Could not fetch allowed attribute type.'); + }, + complete:function() { + $(".loading").hide(); + }, + type: "get", + url: compiledUrlForm + }); + return false; +} + function handleGenericAjaxResponse(data, skip_reload) { if (typeof skip_reload === "undefined") { skip_reload = false; @@ -1144,6 +1242,9 @@ function submitPopoverForm(context_id, referer, update_context_id) { case 'addObjectReference': url = "/objectReferences/add/" + context_id; break; + case 'quickAddAttributeForm': + url = "/objects/quickAddAttributeForm/" + context_id; + break; } if (url !== null) { $.ajax({ From 8e4011499751c89732d1e1f20e62ae9979a6dad7 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 13 Jun 2019 09:38:39 +0200 Subject: [PATCH 02/73] chg: [app] Improved and integrated *-seen database update --- app/Model/AppModel.php | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index aa794994f..760044fdf 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -75,7 +75,7 @@ class AppModel extends Model 13 => false, 14 => false, 15 => false, 18 => false, 19 => false, 20 => false, 21 => false, 22 => false, 23 => false, 24 => false, 25 => false, 26 => false, 27 => false, 28 => false, 29 => false, 30 => false, 31 => false, 32 => false, - 33 => false, 34 => false, 35 => false + 33 => false, 34 => false, 35 => false, 36 => false ); public $advanced_updates_description = array( @@ -209,6 +209,9 @@ class AppModel extends Model case 34: $this->__fixServerPullPushRules(); break; + case 36: + $this->updateDatabase('seenOnAttributeAndObject', true); + break; default: $this->updateDatabase($command); break; @@ -1241,9 +1244,11 @@ class AppModel extends Model DROP INDEX value1, DROP INDEX value2, DROP INDEX object_id, - DROP INDEX object_relation, - DROP INDEX deleted, - ;"; + DROP INDEX object_relation; + "; + $sqlArray[] = "ALTER TABLE `attributes` DROP INDEX deleted"; // deleted index may not be present + $sqlArray[] = "ALTER TABLE `attributes` DROP INDEX first_seen"; // for replayability + $sqlArray[] = "ALTER TABLE `attributes` DROP INDEX last_seen"; // for replayability $sqlArray[] = "ALTER TABLE `attributes` ADD COLUMN `first_seen` BIGINT(20) NULL DEFAULT NULL, @@ -1263,9 +1268,8 @@ class AppModel extends Model ADD INDEX `object_relation` (`object_relation`), ADD INDEX `deleted` (`deleted`), ADD INDEX `first_seen` (`first_seen`), - ADD INDEX `last_seen` (`last_seen`), - ADD INDEX `comment` (`comment`(767)) - ;"; + ADD INDEX `last_seen` (`last_seen`); + "; $sqlArray[] = " ALTER TABLE `objects` ADD `first_seen` BIGINT(20) NULL DEFAULT NULL, From 88eb7ee43349766984e1684b06ec4c762e82192a Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 13 Jun 2019 10:34:22 +0200 Subject: [PATCH 03/73] fix: [object:quickAttributeAdd] Fixed response to be of JSON type and improved layout --- .../Component/RestResponseComponent.php | 1 + app/Controller/ObjectsController.php | 25 +++++++++++++++---- app/webroot/css/main.css | 14 ++++++++--- app/webroot/js/misp.js | 1 + 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/app/Controller/Component/RestResponseComponent.php b/app/Controller/Component/RestResponseComponent.php index 397981ec4..e7526b6da 100644 --- a/app/Controller/Component/RestResponseComponent.php +++ b/app/Controller/Component/RestResponseComponent.php @@ -397,6 +397,7 @@ class RestResponseComponent extends Component $message = Inflector::singularize($controller) . ' ' . $action['action'] . ((substr($action['action'], -1) == 'e') ? 'd' : 'ed'); } $response['saved'] = true; + $response['success'] = true; $response['name'] = $message; $response['message'] = $response['name']; $response['url'] = $this->__generateURL($action, $controller, $id); diff --git a/app/Controller/ObjectsController.php b/app/Controller/ObjectsController.php index 1e1f78d8e..1674346b6 100644 --- a/app/Controller/ObjectsController.php +++ b/app/Controller/ObjectsController.php @@ -551,12 +551,27 @@ class ObjectsController extends AppController $this->MispObject->Event->unpublishEvent($object['Object']['event_id']); return $this->RestResponse->viewData($objectToSave, $this->response->type()); } else { - return $this->RestResponse->saveFailResponse('Objects', 'add', false, $id, $this->response->type()); + return $this->RestResponse->saveFailResponse('Objects', 'add', false, $id, false); } } else { - $this->MispObject->Event->unpublishEvent($object['Object']['event_id']); - $this->Flash->success('Object saved.'); - $this->redirect(array('controller' => 'events', 'action' => 'view', $object['Object']['event_id'])); + $message = __('Object attributes saved.'); + $error_message = __('Object attributes could not be saved.'); + if ($this->request->is('ajax')) { + $this->autoRender = false; + if (is_numeric($objectToSave)) { + return $this->RestResponse->saveSuccessResponse('Objects', 'edit', $id, $this->response->type('application/json'), $message); + } else { + return $this->RestResponse->saveFailResponse('Objects', 'edit', $id, $error_message, $this->response->type('application/json')); + } + } else { + if (is_numeric($objectToSave)) { + $this->MispObject->Event->unpublishEvent($object['Object']['event_id']); + $this->Flash->success($message); + } else { + $this->Flash->error($error_message); + } + $this->redirect(array('controller' => 'events', 'action' => 'view', $object['Object']['event_id'])); + } } } } else { @@ -888,7 +903,7 @@ class ObjectsController extends AppController $this->set('info', $info); $this->render('ajax/quickAddAttributeForm'); } else if ($this->request->is('post') || $this->request->is('put')) { - return $this->edit($this->request->data['Object']['id'], true); + return $this->edit($this->request->data['Object']['id'], false, true); } } diff --git a/app/webroot/css/main.css b/app/webroot/css/main.css index 6553ba4b4..2aaf728cf 100644 --- a/app/webroot/css/main.css +++ b/app/webroot/css/main.css @@ -293,8 +293,8 @@ td.highlight3 { } -table-striped tbody > tr > td.objectAddField { - padding: 2px 3px; +table.table-striped > tbody > tr > td.objectAddField { + padding: 0px; line-height: normal; display: inline-block; position: relative; @@ -304,8 +304,16 @@ table-striped tbody > tr > td.objectAddField { background-color: #1369a0; border: 0px; border-radius: 15px; + height: 16px; + width: 16px; + text-align: center; +} + +table.table-striped > tbody > tr > td.objectAddField:before { + left: 2px; height: 12px; - width: 10px; + width: 12px; + line-height: 16px; } .table-striped tbody > tr > td.objectAddField:hover { diff --git a/app/webroot/js/misp.js b/app/webroot/js/misp.js index 88dcb7cec..d5193d700 100644 --- a/app/webroot/js/misp.js +++ b/app/webroot/js/misp.js @@ -1286,6 +1286,7 @@ function handleAjaxPopoverResponse(response, context_id, url, referer, context, responseArray = response; var message = null; var result = "fail"; + console.log(responseArray); if (responseArray.saved) { updateIndex(context_id, context); if (responseArray.success) { From 553ea16783b294540aaa4c8a370fdaae3036ec87 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 13 Jun 2019 10:51:59 +0200 Subject: [PATCH 04/73] fix: [object:quickEdit] Fixed response to be of JSON type and improved layout --- app/Controller/ObjectsController.php | 12 ++++++------ app/View/Elements/Events/View/row_object.ctp | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/Controller/ObjectsController.php b/app/Controller/ObjectsController.php index 1674346b6..eb2c04c7e 100644 --- a/app/Controller/ObjectsController.php +++ b/app/Controller/ObjectsController.php @@ -628,7 +628,7 @@ class ObjectsController extends AppController } $this->MispObject->id = $id; if (!$this->MispObject->exists()) { - return $this->RestResponse->saveFailResponse('Objects', 'edit', false, 'Invalid object', $this->response->type()); + return $this->RestResponse->saveFailResponse('Objects', 'edit', false, 'Invalid object'); } $this->MispObject->recursive = -1; $this->MispObject->contain('Event'); @@ -639,7 +639,7 @@ class ObjectsController extends AppController || $this->userRole['perm_modify_org'])) { // Allow the edit } else { - return $this->RestResponse->saveFailResponse('Objects', 'edit', false, 'Invalid attribute', $this->response->type()); + return $this->RestResponse->saveFailResponse('Objects', 'edit', false, 'Invalid attribute'); } } $validFields = array('comment', 'distribution', 'first_seen', 'last_seen'); @@ -656,13 +656,13 @@ class ObjectsController extends AppController } if ($object['Object'][$changedKey] == $changedField) { $this->autoRender = false; - return $this->RestResponse->saveSuccessResponse('Objects', 'edit', $id, $this->response->type(), 'nochange'); + return $this->RestResponse->saveSuccessResponse('Objects', 'edit', $id, false, 'nochange'); } $object['Object'][$changedKey] = $changedField; $changed = true; } if (!$changed) { - return $this->RestResponse->saveSuccessResponse('Objects', 'edit', $id, $this->response->type(), 'nochange'); + return $this->RestResponse->saveSuccessResponse('Objects', 'edit', $id, false, 'nochange'); } $date = new DateTime(); $object['Object']['timestamp'] = $date->getTimestamp(); @@ -678,10 +678,10 @@ class ObjectsController extends AppController $event['Event']['published'] = 0; $this->MispObject->Event->save($event, array('fieldList' => array('published', 'timestamp', 'info'))); $this->autoRender = false; - return $this->RestResponse->saveSuccessResponse('Objects', 'edit', $id, $this->response->type(), 'Field updated'); + return $this->RestResponse->saveSuccessResponse('Objects', 'edit', $id, false, 'Field updated'); } else { $this->autoRender = false; - return $this->RestResponse->saveFailResponse('Objects', 'edit', false, $this->MispObject->validationErrors, $this->response->type()); + return $this->RestResponse->saveFailResponse('Objects', 'edit', false, $this->MispObject->validationErrors); } } diff --git a/app/View/Elements/Events/View/row_object.ctp b/app/View/Elements/Events/View/row_object.ctp index bf0537654..653b5ce25 100644 --- a/app/View/Elements/Events/View/row_object.ctp +++ b/app/View/Elements/Events/View/row_object.ctp @@ -93,13 +93,13 @@ - '; + echo ''; } ?> diff --git a/app/View/Objects/ajax/quickAddAttributeForm.ctp b/app/View/Objects/ajax/quickAddAttributeForm.ctp index b47804c7c..a94be2b03 100644 --- a/app/View/Objects/ajax/quickAddAttributeForm.ctp +++ b/app/View/Objects/ajax/quickAddAttributeForm.ctp @@ -1,132 +1,132 @@ -Form->create('Object', array( - 'id' => 'Object_' . $object['id'] . '_quick_add_attribute_form', - 'url' => $url, - 'class' => 'allDiv' - )); -?> - -
- -
+
'hidden', - 'value' => $element['object_relation'], - 'label' => false, - 'div' => false - ); - echo $this->Form->input('Attribute.' . $k . '.object_relation', $formSettings); - - $formSettings['value'] = $object['id']; - echo $this->Form->input('Object.id', $formSettings); - - $formSettings['value'] = $object['event_id']; - echo $this->Form->input('Object.event_id', $formSettings); - - $formSettings['value'] = $element['type']; - echo '
'; - echo $this->Form->input('Attribute.' . $k . '.type', $formSettings); - echo '' . Inflector::humanize(h($element['object_relation'])) . ''; - echo ' :: ' . h($element['type']) . ''; - echo '
'; - echo '' . h($element['description']) . ''; - echo '
'; - ?> - - - array_combine($element['categories'], $element['categories']), - 'default' => $element['default_category'], - 'div' => true - ); - echo $this->Form->input('Attribute.' . $k . '.category', $formSettings); - ?> - -
- ' . __('Value') . ''; - echo $this->element( - 'Objects/object_value_field', - array( - 'element' => $element, - 'k' => $k, - 'action' => $action - ) - ); - ?> -
- - Form->input('Attribute.' . $k . '.to_ids', array( - 'type' => 'checkbox', - 'checked' => $element['to_ids'], + $url = '/objects/quickAddAttributeForm/' . $object['id']; + $element = $template_element; + $k = 0; + $action = 'add'; + echo $this->Form->create('Object', array( + 'id' => 'Object_' . $object['id'] . '_quick_add_attribute_form', + 'url' => $url, + 'class' => 'allDiv' )); ?> - Form->input('Attribute.' . $k . '.disable_correlation', array( - 'type' => 'checkbox', - 'checked' => $element['disable_correlation'], - )); - ?> +
+ +
+ 'hidden', + 'value' => $element['object_relation'], + 'label' => false, + 'div' => false + ); + echo $this->Form->input('Attribute.' . $k . '.object_relation', $formSettings); -
- Form->input('Attribute.' . $k . '.distribution', array( - 'class' => 'Attribute_distribution_select', - 'options' => $distributionData['levels'], - 'default' => !empty($element['distribution']) ? $element['distribution'] : $distributionData['initial'], - 'div' => false, - 'label' => __('Distribution ') . $this->element('formInfo', array('type' => 'distribution')), - )); - ?> -
-
- +
+ +
+ + + Form->button('Submit', array('class' => 'btn btn-primary')); + endif; + ?>
-
Form->input('Attribute.' . $k . '.comment', array( - 'type' => 'textarea', - 'required' => false, - 'allowEmpty' => true, - 'div' => 'input clear', - 'class' => 'input-xxlarge' - )); + echo $this->Form->end(); ?> - -
-
- -
- - - Form->button('Submit', array('class' => 'btn btn-primary')); - endif; - ?> - -
- -Form->end(); -?> - + diff --git a/app/View/Objects/ajax/templateWithValidObjectAttributes.ctp b/app/View/Objects/ajax/templateWithValidObjectAttributes.ctp deleted file mode 100644 index aab0101e5..000000000 --- a/app/View/Objects/ajax/templateWithValidObjectAttributes.ctp +++ /dev/null @@ -1,22 +0,0 @@ -
- -
-
'+obj.content+bolt_html+'
' + attr.contentType + '' + attr.content+overwritten + '  +
-
class="inline-field-solid" ondblclick="activateField('', '', 'distribution', );"> +
class="inline-field-solid"> Date: Thu, 13 Jun 2019 10:53:34 +0200 Subject: [PATCH 05/73] fix: [object:quickEdit] fix input selector --- app/webroot/js/misp.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/webroot/js/misp.js b/app/webroot/js/misp.js index d5193d700..45c0f30ad 100644 --- a/app/webroot/js/misp.js +++ b/app/webroot/js/misp.js @@ -477,7 +477,7 @@ function activateField(type, id, field, event) { containerName = 'Object'; } var name = '#' + type + '_' + id + '_' + field; - var container_name = '#' + containerName + id + '_' + field; + var container_name = '#' + containerName + '_' + id + '_' + field; $.ajax({ beforeSend: function (XMLHttpRequest) { $(".loading").show(); @@ -1286,7 +1286,6 @@ function handleAjaxPopoverResponse(response, context_id, url, referer, context, responseArray = response; var message = null; var result = "fail"; - console.log(responseArray); if (responseArray.saved) { updateIndex(context_id, context); if (responseArray.success) { From 7d74408bdf5e87d4a43cd4c4ac405bc05ebbe0dd Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 13 Jun 2019 14:14:02 +0200 Subject: [PATCH 06/73] chg: [object:quickAttributeAdd] Replace popover selection by the generic picker --- app/Controller/ObjectsController.php | 23 +- app/View/Elements/Events/View/row_object.ctp | 2 +- .../Objects/ajax/quickAddAttributeForm.ctp | 244 +++++++++--------- .../templateWithValidObjectAttributes.ctp | 22 -- app/webroot/css/main.css | 10 + app/webroot/js/misp.js | 57 +--- 6 files changed, 156 insertions(+), 202 deletions(-) delete mode 100644 app/View/Objects/ajax/templateWithValidObjectAttributes.ctp diff --git a/app/Controller/ObjectsController.php b/app/Controller/ObjectsController.php index eb2c04c7e..3d1dca04b 100644 --- a/app/Controller/ObjectsController.php +++ b/app/Controller/ObjectsController.php @@ -831,7 +831,24 @@ class ObjectsController extends AppController if ($this->request->is('get') || $this->request->is('post')) { $this->set('template', $template); $this->set('objectId', $object['Object']['id']); - $this->render('ajax/templateWithValidObjectAttributes'); + + $items = array(); + foreach ($template['ObjectTemplateElement'] as $objectAttribute) { + $name = sprintf('%s :: %s', $objectAttribute['object_relation'], $objectAttribute['type']); + $items[] = array( + 'name' => $name, + 'value' => '/objects/quickAddAttributeForm/' . $object['Object']['id'] . '/' . $objectAttribute['object_relation'], + 'template' => array( + 'name' => $name, + 'infoExtra' => $objectAttribute['description'], + ) + ); + } + $this->set('options', array( + 'flag_redraw_chosen' => true + )); + $this->set('items', $items); + $this->render('/Elements/generic_picker'); } else { return $template; } @@ -893,7 +910,9 @@ class ObjectsController extends AppController $template = $this->MispObject->prepareTemplate($template, $object); $this->layout = 'ajax'; $this->set('object', $object['Object']); - $this->set('template', $template); + $template_element = $template['ObjectTemplateElement'][0]; + unset($template_element['value']); // avoid filling if multiple + $this->set('template_element', $template_element); $distributionData = $this->MispObject->Event->Attribute->fetchDistributionData($this->Auth->user()); $this->set('distributionData', $distributionData); $info = array(); diff --git a/app/View/Elements/Events/View/row_object.ctp b/app/View/Elements/Events/View/row_object.ctp index 653b5ce25..0fde98b93 100644 --- a/app/View/Elements/Events/View/row_object.ctp +++ b/app/View/Elements/Events/View/row_object.ctp @@ -150,6 +150,6 @@ 'child' => $attrKey == $lastElement ? 'last' : true )); } - echo '
- - - - - -
' . h($objectAttribute['object_relation']) . '' . ' :: ' . h($objectAttribute['type']) . '
' . h($objectAttribute['description']); ?>
- -
- - diff --git a/app/webroot/css/main.css b/app/webroot/css/main.css index 2aaf728cf..a29e71952 100644 --- a/app/webroot/css/main.css +++ b/app/webroot/css/main.css @@ -1033,6 +1033,16 @@ a.pill-pre-picker { position: absolute; } +div.generic-picker-wrapper > div.generic-picker-embeded-block { + margin-left: 15px; + margin-top: -12px; + display: inline-block; +} + +div.generic-picker-embeded-block legend { + margin-bottom: 0px; +} + .generic-picker-item-element-check { float: right; border: 1px solid #999; diff --git a/app/webroot/js/misp.js b/app/webroot/js/misp.js index 45c0f30ad..92e8d3008 100644 --- a/app/webroot/js/misp.js +++ b/app/webroot/js/misp.js @@ -786,58 +786,6 @@ function handleAjaxEditResponse(data, name, type, id, field, event) { } } -function quickFetchValidObjectAttribute(objectId) { - var itemType = "objects"; - var formUrl= "quickFetchTemplateWithValidObjectAttributes"; - var compiledUrlForm = "/" + itemType + "/" + formUrl + "/" + objectId; - $.ajax({ - beforeSend: function (XMLHttpRequest) { - $(".loading").show(); - }, - success:function (data, textStatus) { - if (data.fail !== undefined && data.fail) { - showMessage('fail', data.errors); - } else { - $('#popover_form').html(data); - openPopup('#popover_form'); - } - }, - error:function() { - showMessage('fail', 'Could not fetch allowed attribute type.'); - }, - complete:function() { - $(".loading").hide(); - }, - type: "get", - url: compiledUrlForm - }); - return false; -} - -function fetchAddObjectAttributeForm(objectId, fieldName) { - var itemType = "objects"; - var formUrl= "quickAddAttributeForm"; - var compiledUrlForm = "/" + itemType + "/" + formUrl + "/" + objectId + "/" + fieldName; - $.ajax({ - beforeSend: function (XMLHttpRequest) { - $(".loading").show(); - }, - success:function (data, textStatus) { - $('#popover_form').html(data); - openPopup('#popover_form'); - }, - error:function() { - showMessage('fail', 'Could not fetch allowed attribute type.'); - }, - complete:function() { - $(".loading").hide(); - }, - type: "get", - url: compiledUrlForm - }); - return false; -} - function handleGenericAjaxResponse(data, skip_reload) { if (typeof skip_reload === "undefined") { skip_reload = false; @@ -1181,7 +1129,7 @@ function clickCreateButton(event, type) { simplePopup("/" + destination + "/add/" + event); } -function submitPopoverForm(context_id, referer, update_context_id) { +function submitPopoverForm(context_id, referer, update_context_id, popover_dissmis_id_to_close) { var url = null; var context = 'event'; var contextNamingConvention = 'Attribute'; @@ -1253,6 +1201,9 @@ function submitPopoverForm(context_id, referer, update_context_id) { if (closePopover) { $("#gray_out").fadeOut(); $("#popover_form").fadeOut(); + if (popover_dissmis_id_to_close !== undefined) { + $('[data-dismissid="' + popover_dissmis_id_to_close + '"]').popover('destroy'); + } } }, data: $("#submitButton").closest("form").serialize(), From 93cea354bae406454e660874cb4ea95230bdfd4b Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 13 Jun 2019 14:36:38 +0200 Subject: [PATCH 07/73] fix: [event:timeline] Error when trying to restore non-existing backup entries --- app/webroot/css/contextual_menu.css | 1 + app/webroot/js/event-timeline.js | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/app/webroot/css/contextual_menu.css b/app/webroot/css/contextual_menu.css index 2488079b9..8981b7acf 100644 --- a/app/webroot/css/contextual_menu.css +++ b/app/webroot/css/contextual_menu.css @@ -26,6 +26,7 @@ .contextual-menu > label { font-weight: bold; margin-right: 3px; + user-select: none; } .contextual-menu > div > label { diff --git a/app/webroot/js/event-timeline.js b/app/webroot/js/event-timeline.js index ac03965e0..ea8e01bf8 100644 --- a/app/webroot/js/event-timeline.js +++ b/app/webroot/js/event-timeline.js @@ -377,7 +377,7 @@ function reload_timeline() { } } items_timeline.add(data.items); - handle_not_seen_enabled($('#checkbox_timeline_display_hide_not_seen_enabled').val()) + handle_not_seen_enabled($('#checkbox_timeline_display_hide_not_seen_enabled').prop('checked'), false) }, error: function( jqXhr, textStatus, errorThrown ){ console.log( errorThrown ); @@ -487,7 +487,8 @@ function handle_doubleClick(data) { //edit_item(data.item); } -function handle_not_seen_enabled(hide) { +function handle_not_seen_enabled(hide, include_hidden) { + include_hidden = include_hidden !== undefined ? include_hidden : true; if (hide) { var hidden = items_timeline.get({ filter: function(item) { @@ -500,8 +501,10 @@ function handle_not_seen_enabled(hide) { }); items_timeline.remove(hidden) items_backup = hidden; - } else { - items_timeline.add(items_backup); + } else if (include_hidden) { + if (items_backup !== undefined) { + items_timeline.add(items_backup); + } } } From 21332e2d824d313e9c2792373746d38517fd3515 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 13 Jun 2019 15:12:29 +0200 Subject: [PATCH 08/73] fix: [timeline] correctly adapt time scale when expanding items --- app/webroot/js/event-timeline.js | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/app/webroot/js/event-timeline.js b/app/webroot/js/event-timeline.js index ea8e01bf8..02141763b 100644 --- a/app/webroot/js/event-timeline.js +++ b/app/webroot/js/event-timeline.js @@ -132,12 +132,34 @@ function expandItem() { } function get_next_step(mom) { - var scale = eventTimeline.timeAxis.step.scale; + var scale = adapt_scale(eventTimeline.timeAxis.step.scale); var momAhead = mom.clone(); momAhead.add(1, scale); return momAhead; } +function adapt_scale(scale) { + first_letter = scale.charAt(0); + if (first_letter !== 'm' && first_letter !== 'w') { + return first_letter; + } else { + switch (scale) { + case 'millisecond': + return 'ms'; + case 'minute': + return 'm'; + case 'month': + return 'M'; + case 'week': + return 'w'; + case 'weekday': + return 'd'; + default: + return scale; + } + } +} + function build_attr_template(attr) { var span = $(''); if (!attr.seen_enabled) { From 0e209b610d7feffa4f72d431c12cd02cc7d47123 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 13 Jun 2019 15:12:59 +0200 Subject: [PATCH 09/73] fix: [attribute:*-seen] Force seconds to be integers and allows editForm for *-seen fields --- app/Controller/AttributesController.php | 2 +- app/Model/Attribute.php | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 6bc0cd8e8..7b124e946 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -2441,7 +2441,7 @@ class AttributesController extends AppController public function fetchEditForm($id, $field = null) { - $validFields = array('value', 'comment', 'type', 'category', 'to_ids', 'distribution'); + $validFields = array('value', 'comment', 'type', 'category', 'to_ids', 'distribution', 'first_seen', 'last_seen'); if (!isset($field) || !in_array($field, $validFields)) { throw new MethodNotAllowedException(__('Invalid field requested.')); } diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index d39dc42c8..3a4b0488a 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -600,7 +600,7 @@ class Attribute extends AppModel } if (!empty($v['Attribute']['first_seen'])) { $fs = $results[$k]['Attribute']['first_seen']; - $fs_sec = $fs / 1000000; // $fs is in micro (10^6) + $fs_sec = intval($fs / 1000000); // $fs is in micro (10^6) $fs_micro = $fs % 1000000; $fs_micro = str_pad($fs_micro, 6, "0", STR_PAD_LEFT); $fs = $fs_sec . '.' . $fs_micro; @@ -608,7 +608,7 @@ class Attribute extends AppModel } if (!empty($v['Attribute']['last_seen'])) { $ls = $results[$k]['Attribute']['last_seen']; - $ls_sec = $ls / 1000000; // $ls is in micro (10^6) + $ls_sec = intval($ls / 1000000); // $ls is in micro (10^6) $ls_micro = $ls % 1000000; $ls_micro = str_pad($ls_micro, 6, "0", STR_PAD_LEFT); $ls = $ls_sec . '.' . $ls_micro; From c24081563cdfa88861d08e6629387e3760bce4cf Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 13 Jun 2019 15:14:11 +0200 Subject: [PATCH 10/73] fix: [attribute:view] Correctly pick the matching form --- app/View/Elements/form_seen_input.ctp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/View/Elements/form_seen_input.ctp b/app/View/Elements/form_seen_input.ctp index 281e58189..3ab0cbf6f 100644 --- a/app/View/Elements/form_seen_input.ctp +++ b/app/View/Elements/form_seen_input.ctp @@ -292,7 +292,7 @@ function reflect_change_on_form() { $(document).ready(function() { - var sliders_container = "params->controller === 'attributes') { echo 'fieldset'; } else { echo '#meta-div'; } ?>"; + var sliders_container = "params->controller === 'attributes') { echo '#AttributeForm fieldset'; } else { echo '#meta-div'; } ?>"; var inputs_container = $('
'); // create separate date and time input var date_div_fs = $('
').append( From 4b9d69fe5811327edaaf3ff6953ad14835c8dd08 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 13 Jun 2019 15:23:34 +0200 Subject: [PATCH 11/73] fix: [time_precision_tool] Support of IE. Usage of prototypes instead of a class --- app/View/Elements/form_seen_input.ctp | 100 +++++++++++++------------- 1 file changed, 51 insertions(+), 49 deletions(-) diff --git a/app/View/Elements/form_seen_input.ctp b/app/View/Elements/form_seen_input.ctp index 3ab0cbf6f..06281293a 100644 --- a/app/View/Elements/form_seen_input.ctp +++ b/app/View/Elements/form_seen_input.ctp @@ -28,64 +28,66 @@ var time_vals = [ ['us', 999, 1], ]; -class MicroDatetime { - constructor(value) { - this.isoDatetimeMicroRegex = /(\d{4})-(\d{2})-(\d{2})T(\d{2})\:(\d{2})\:(\d{2})\.(\d{3})(\d*)(\D\S*)/g; - this.isotimeMicroRegex = /(\d{2})\:(\d{2})\:(\d{2})\.(\d{3})(\d*)(\D\S*)/g; - this.numberRegex = /^(\-|\+)?([0-9]+|Infinity)$/g; +var MicroDatetime = function(value) { + this.isoDatetimeMicroRegex = /(\d{4})-(\d{2})-(\d{2})T(\d{2})\:(\d{2})\:(\d{2})\.(\d{3})(\d*)(\D\S*)/g; + this.isotimeMicroRegex = /(\d{2})\:(\d{2})\:(\d{2})\.(\d{3})(\d*)(\D\S*)/g; + this.numberRegex = /^(\-|\+)?([0-9]+|Infinity)$/g; - if (value === undefined || value === "") { - this.moment = undefined; - this.micro = 0; - } else { - // check if timestamp UNIX timestamp (in sec) - if (this.numberRegex.test(value)) { - var timestamp = parseInt(value); - var timestamp_str = String(timestamp); - if (timestamp === '' || timestamp === undefined || timestamp === null || isNaN(timestamp)) { - this.moment = moment(0); - this.micro = 0; + if (value === undefined || value === "") { + this.moment = undefined; + this.micro = 0; + } else { + // check if timestamp UNIX timestamp (in sec) + if (this.numberRegex.test(value)) { + var timestamp = parseInt(value); + var timestamp_str = String(timestamp); + if (timestamp === '' || timestamp === undefined || timestamp === null || isNaN(timestamp)) { + this.moment = moment(0); + this.micro = 0; + } else { + var all_milli = parseInt(timestamp)*1000; + all_milli = isNaN(all_milli) || all_milli === undefined ? 0 : all_milli; + this.moment = moment(all_milli); + this.micro = 0; + } + // check if only a time + } else { // let moment parse the date + try { + value = String(value); + this.moment = new moment(value); + if (this.moment.isValid()) { + var res = this.isoDatetimeMicroRegex.exec(value); + var micro_str = res !== null ? res[8] : 0; + this.micro = parseInt(micro_str); + this.micro = isNaN(this.micro) ? 0 : this.micro; } else { - var all_milli = parseInt(timestamp)*1000; - all_milli = isNaN(all_milli) || all_milli === undefined ? 0 : all_milli; - this.moment = moment(all_milli); + this.moment = undefined; this.micro = 0; + showMessage('fail', 'Failed to parse the date: ' + value + ''); } - // check if only a time - } else { // let moment parse the date - try { - value = String(value); - this.moment = new moment(value); + } catch (err) { + if (this.isotimeMicroRegex.test(value)) { + this.moment = moment(value, "HH:mm:ss.SSSSSSZ"); if (this.moment.isValid()) { - var res = this.isoDatetimeMicroRegex.exec(value); + var res = this.isotimeMicroRegex.exec(value); var micro_str = res !== null ? res[8] : 0; this.micro = parseInt(micro_str); this.micro = isNaN(this.micro) ? 0 : this.micro; } else { this.moment = undefined; this.micro = 0; - showMessage('fail', 'Failed to parse the date: ' + value + ''); - } - } catch (err) { - if (this.isotimeMicroRegex.test(value)) { - this.moment = moment(value, "HH:mm:ss.SSSSSSZ"); - if (this.moment.isValid()) { - var res = this.isotimeMicroRegex.exec(value); - var micro_str = res !== null ? res[8] : 0; - this.micro = parseInt(micro_str); - this.micro = isNaN(this.micro) ? 0 : this.micro; - } else { - this.moment = undefined; - this.micro = 0; - showMessage('fail', 'Failed to parse the date: ' + value); - } + showMessage('fail', 'Failed to parse the date: ' + value); } } } } } +} - get_microISO() { +MicroDatetime.prototype = { + constructor: MicroDatetime, + + get_microISO: function() { if (this.moment === undefined || this.moment === null) { return ""; } @@ -93,29 +95,29 @@ class MicroDatetime { var str = this.moment.toISOString(true); str = str.replace(tz, pad_zero(this.micro, 3)+tz); return str; - } + }, - get_date() { + get_date: function() { if (this.moment === undefined || this.moment === null) { return ""; } return this.moment.format('YYYY/MM/DD'); - } + }, - get_time() { + get_time: function() { if (this.moment === undefined || this.moment === null) { return ""; } return this.moment.format('HH:mm:ss.SSS') + String(pad_zero(this.micro, 3)) + this.moment.format('Z'); - } + }, - has_date() { + has_date: function() { return this.moment !== undefined; - } + }, - has_time() { + has_time: function() { if (this.moment === undefined) { return false; } else { From ded0463498ff7c381153aed53a23559f71335c86 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Thu, 13 Jun 2019 16:11:00 +0200 Subject: [PATCH 12/73] fix: [restResponse] Added support of *-seen fields --- app/Controller/Component/RestResponseComponent.php | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/app/Controller/Component/RestResponseComponent.php b/app/Controller/Component/RestResponseComponent.php index e7526b6da..9f011cafe 100644 --- a/app/Controller/Component/RestResponseComponent.php +++ b/app/Controller/Component/RestResponseComponent.php @@ -824,6 +824,12 @@ class RestResponseComponent extends Component 'operators' => array('equal'), 'help' => 'A valid external auth key', ), + 'first_seen' => array( + 'input' => 'text', + 'type' => 'string', + 'operators' => array('equal'), + 'help' => 'A valid ISO 8601 datetime format, up to milli-seconds. i.e.: 2019-06-13T15:56:56.856074+02:00' + ), 'fixed_event' => array( 'input' => 'select', 'type' => 'integer', @@ -934,6 +940,12 @@ class RestResponseComponent extends Component 'operators' => array('equal', 'not_equal'), 'help' => 'Events published within the last x amount of time, where x can be defined in days, hours, minutes (for example 5d or 12h or 30m)' ), + 'last_seen' => array( + 'input' => 'text', + 'type' => 'string', + 'operators' => array('equal'), + 'help' => 'A valid ISO 8601 datetime format, up to milli-seconds. i.e.: 2019-06-13T15:56:56.856074+02:00' + ), 'limit' => array( 'input' => 'number', 'type' => 'integer', From 6af9719a4768f551df0422bb585774bc22641df6 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Mon, 24 Jun 2019 10:28:55 +0200 Subject: [PATCH 13/73] new: [shadowAttribute] first_seen and last_seen on shadowAttributes --- app/Controller/ShadowAttributesController.php | 18 ++--- app/Model/AppModel.php | 7 ++ app/Model/Event.php | 2 +- app/Model/ShadowAttribute.php | 67 +++++++++++++++++++ app/View/Elements/form_seen_input.ctp | 18 ++++- app/View/ShadowAttributes/add.ctp | 1 + app/View/ShadowAttributes/edit.ctp | 1 + 7 files changed, 103 insertions(+), 11 deletions(-) diff --git a/app/Controller/ShadowAttributesController.php b/app/Controller/ShadowAttributesController.php index 97f80cea9..305b6a0c4 100644 --- a/app/Controller/ShadowAttributesController.php +++ b/app/Controller/ShadowAttributesController.php @@ -433,7 +433,7 @@ class ShadowAttributesController extends AppController array( 'conditions' => array('ShadowAttribute.id' => $this->ShadowAttribute->id), 'recursive' => -1, - 'fields' => array('id', 'old_id', 'event_id', 'type', 'category', 'value', 'comment','to_ids', 'uuid', 'event_org_id', 'email', 'deleted', 'timestamp') + 'fields' => array('id', 'old_id', 'event_id', 'type', 'category', 'value', 'comment','to_ids', 'uuid', 'event_org_id', 'email', 'deleted', 'timestamp', 'first_seen', 'last_seen') ) ); $this->set('ShadowAttribute', $sa['ShadowAttribute']); @@ -703,12 +703,12 @@ class ShadowAttributesController extends AppController if ($attachment) { $fields = array( 'static' => array('old_id' => 'Attribute.id', 'uuid' => 'Attribute.uuid', 'event_id' => 'Attribute.event_id', 'event_uuid' => 'Event.uuid', 'event_org_id' => 'Event.orgc_id', 'category' => 'Attribute.category', 'type' => 'Attribute.type'), - 'optional' => array('value', 'to_ids', 'comment') + 'optional' => array('value', 'to_ids', 'comment', 'first_seen', 'last_seen') ); } else { $fields = array( 'static' => array('old_id' => 'Attribute.id', 'uuid' => 'Attribute.uuid', 'event_id' => 'Attribute.event_id', 'event_uuid' => 'Event.uuid', 'event_org_id' => 'Event.orgc_id'), - 'optional' => array('category', 'type', 'value', 'to_ids', 'comment') + 'optional' => array('category', 'type', 'value', 'to_ids', 'comment', 'first_seen', 'last_seen') ); if ($existingAttribute['Attribute']['object_id']) { unset($fields['optional']['type']); @@ -745,7 +745,7 @@ class ShadowAttributesController extends AppController array( 'conditions' => array('ShadowAttribute.id' => $this->ShadowAttribute->id), 'recursive' => -1, - 'fields' => array('id', 'old_id', 'event_id', 'type', 'category', 'value', 'comment','to_ids', 'uuid', 'event_org_id', 'email', 'deleted', 'timestamp') + 'fields' => array('id', 'old_id', 'event_id', 'type', 'category', 'value', 'comment','to_ids', 'uuid', 'event_org_id', 'email', 'deleted', 'timestamp', 'first_seen', 'last_seen') ) ); $this->set('ShadowAttribute', $sa['ShadowAttribute']); @@ -847,6 +847,8 @@ class ShadowAttributesController extends AppController 'type' => $existingAttribute['Attribute']['type'], 'to_ids' => $existingAttribute['Attribute']['to_ids'], 'value' => $existingAttribute['Attribute']['value'], + 'first_seen' => $existingAttribute['Attribute']['first_seen'], + 'last_seen' => $existingAttribute['Attribute']['last_seen'], 'email' => $this->Auth->user('email'), 'org_id' => $this->Auth->user('org_id'), 'proposal_to_delete' => true, @@ -887,7 +889,7 @@ class ShadowAttributesController extends AppController 'recursive' => -1, 'contain' => 'Event', 'fields' => array( - 'ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', + 'ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.first_seen', 'ShadowAttribute.last_seen', 'Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.distribution', 'Event.uuid' ), 'conditions' => array('AND' => array('ShadowAttribute.id' => $id, $distConditions, 'ShadowAttribute.deleted' => 0)) @@ -934,7 +936,7 @@ class ShadowAttributesController extends AppController if ($this->_isRest()) { $temp = $this->ShadowAttribute->find('all', array( 'conditions' => $conditions, - 'fields' => array('ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.timestamp', 'ShadowAttribute.proposal_to_delete'), + 'fields' => array('ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.timestamp', 'ShadowAttribute.proposal_to_delete', 'ShadowAttribute.first_seen', 'ShadowAttribute.last_seen'), 'contain' => array( 'Event' => array( 'fields' => array('id', 'org_id', 'info', 'orgc_id'), @@ -959,7 +961,7 @@ class ShadowAttributesController extends AppController } else { $this->paginate = array( 'conditions' => $conditions, - 'fields' => array('ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.timestamp'), + 'fields' => array('ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.timestamp', 'ShadowAttribute.first_seen', 'ShadowAttribute.last_seen'), 'contain' => array( 'Event' => array( 'fields' => array('id', 'org_id', 'info', 'orgc_id'), @@ -1137,7 +1139,7 @@ class ShadowAttributesController extends AppController } } - $keys = array_flip(array('uuid', 'event_id', 'value', 'type', 'category', 'to_ids')); + $keys = array_flip(array('uuid', 'event_id', 'value', 'type', 'category', 'to_ids', 'first_seen', 'last_seen')); $proposal = array_intersect_key($attribute['Attribute'], $keys); $proposal['email'] = $this->Auth->user('email'); diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 760044fdf..1e6a675cd 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -1277,6 +1277,13 @@ class AppModel extends Model ;"; $indexArray[] = array('objects', 'first_seen'); $indexArray[] = array('objects', 'last_seen'); + $sqlArray[] = " + ALTER TABLE `shadow_attributes` + ADD `first_seen` BIGINT(20) NULL DEFAULT NULL, + ADD `last_seen` BIGINT(20) NULL DEFAULT NULL + ;"; + $indexArray[] = array('shadow_attributes', 'first_seen'); + $indexArray[] = array('shadow_attributes', 'last_seen'); break; case 'testUpdate': $sqlArray[] = "SELECT SLEEP(10);"; diff --git a/app/Model/Event.php b/app/Model/Event.php index 7fc0cc4b2..02d0a5bfa 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -1995,7 +1995,7 @@ class Event extends AppModel $fields = array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.date', 'Event.threat_level_id', 'Event.info', 'Event.published', 'Event.uuid', 'Event.attribute_count', 'Event.analysis', 'Event.timestamp', 'Event.distribution', 'Event.proposal_email_lock', 'Event.user_id', 'Event.locked', 'Event.publish_timestamp', 'Event.sharing_group_id', 'Event.disable_correlation', 'Event.extends_uuid'); $fieldsAtt = array('Attribute.id', 'Attribute.type', 'Attribute.category', 'Attribute.value', 'Attribute.to_ids', 'Attribute.uuid', 'Attribute.event_id', 'Attribute.distribution', 'Attribute.timestamp', 'Attribute.comment', 'Attribute.sharing_group_id', 'Attribute.deleted', 'Attribute.disable_correlation', 'Attribute.object_id', 'Attribute.object_relation', 'Attribute.first_seen', 'Attribute.last_seen'); $fieldsObj = array('*'); - $fieldsShadowAtt = array('ShadowAttribute.id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.value', 'ShadowAttribute.to_ids', 'ShadowAttribute.uuid', 'ShadowAttribute.event_uuid', 'ShadowAttribute.event_id', 'ShadowAttribute.old_id', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.proposal_to_delete', 'ShadowAttribute.timestamp'); + $fieldsShadowAtt = array('ShadowAttribute.id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.value', 'ShadowAttribute.to_ids', 'ShadowAttribute.uuid', 'ShadowAttribute.event_uuid', 'ShadowAttribute.event_id', 'ShadowAttribute.old_id', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.proposal_to_delete', 'ShadowAttribute.timestamp', 'ShadowAttribute.first_seen', 'ShadowAttribute.last_seen'); $fieldsOrg = array('id', 'name', 'uuid'); $fieldsServer = array('id', 'url', 'name'); if (!$options['includeAllTags']) { diff --git a/app/Model/ShadowAttribute.php b/app/Model/ShadowAttribute.php index c08b79d3d..0efe38178 100644 --- a/app/Model/ShadowAttribute.php +++ b/app/Model/ShadowAttribute.php @@ -127,6 +127,16 @@ class ShadowAttribute extends AppModel 'rule' => array('boolean'), ), ), + 'first_seen' => array( + 'rule' => array('datetimeOrNull'), + 'required' => false, + 'message' => array('Invalid ISO 8601 format') + ), + 'last_seen' => array( + 'rule' => array('datetimeOrNull'), + 'required' => false, + 'message' => array('Invalid ISO 8601 format') + ) ); public function __construct($id = false, $table = null, $ds = null) @@ -171,6 +181,26 @@ class ShadowAttribute extends AppModel if ($this->data['ShadowAttribute']['deleted']) { $this->__beforeDeleteCorrelation($this->data['ShadowAttribute']); } + + // convert into utc and micro sec + if (!empty($this->data['ShadowAttribute']['first_seen'])) { + $d = new DateTime($this->data['ShadowAttribute']['first_seen']); + $d->setTimezone(new DateTimeZone('GMT')); + $fs_sec = $d->format('U'); + $fs_micro = $d->format('u'); + $fs_micro = str_pad($fs_micro, 6, "0", STR_PAD_LEFT); + $fs = $fs_sec . $fs_micro; + $this->data['ShadowAttribute']['first_seen'] = $fs; + } + if (!empty($this->data['ShadowAttribute']['last_seen'])) { + $d = new DateTime($this->data['ShadowAttribute']['last_seen']); + $d->setTimezone(new DateTimeZone('GMT')); + $ls_sec = $d->format('U'); + $ls_micro = $d->format('u'); + $ls_micro = str_pad($ls_micro, 6, "0", STR_PAD_LEFT); + $ls = $ls_sec . $ls_micro; + $this->data['ShadowAttribute']['last_seen'] = $ls; + } return true; } @@ -348,6 +378,29 @@ class ShadowAttribute extends AppModel return true; } + public function afterFind($results, $primary = false) + { + foreach ($results as $k => $v) { + if (!empty($v['ShadowAttribute']['first_seen'])) { + $fs = $results[$k]['ShadowAttribute']['first_seen']; + $fs_sec = intval($fs / 1000000); // $fs is in micro (10^6) + $fs_micro = $fs % 1000000; + $fs_micro = str_pad($fs_micro, 6, "0", STR_PAD_LEFT); + $fs = $fs_sec . '.' . $fs_micro; + $results[$k]['ShadowAttribute']['first_seen'] = DateTime::createFromFormat('U.u', $fs)->format('Y-m-d\TH:i:s.uP'); + } + if (!empty($v['ShadowAttribute']['last_seen'])) { + $ls = $results[$k]['ShadowAttribute']['last_seen']; + $ls_sec = intval($ls / 1000000); // $ls is in micro (10^6) + $ls_micro = $ls % 1000000; + $ls_micro = str_pad($ls_micro, 6, "0", STR_PAD_LEFT); + $ls = $ls_sec . '.' . $ls_micro; + $results[$k]['ShadowAttribute']['last_seen'] = DateTime::createFromFormat('U.u', $ls)->format('Y-m-d\TH:i:s.uP'); + } + } + return $results; + } + public function validateTypeValue($fields) { $category = $this->data['ShadowAttribute']['category']; @@ -464,6 +517,20 @@ class ShadowAttribute extends AppModel return $fails; } + // check whether the variable is null or datetime + public function datetimeOrNull($fields) + { + $k = array_keys($fields)[0]; + $seen = $fields[$k]; + try { + new DateTime($seen); + $returnValue = true; + } catch (Exception $e) { + $returnValue = false; + } + return $returnValue || is_null($seen); + } + public function setDeleted($id) { $this->Behaviors->detach('SysLogLogable.SysLogLogable'); diff --git a/app/View/Elements/form_seen_input.ctp b/app/View/Elements/form_seen_input.ctp index 06281293a..16496ba21 100644 --- a/app/View/Elements/form_seen_input.ctp +++ b/app/View/Elements/form_seen_input.ctp @@ -19,7 +19,14 @@ From 3d473ca29fc7c21a284d002c4e74f67cfd6ca759 Mon Sep 17 00:00:00 2001 From: iglocska Date: Mon, 20 Jan 2020 06:37:35 +0100 Subject: [PATCH 66/73] chg: [pymisp] bump --- PyMISP | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyMISP b/PyMISP index c4c05e43b..adf97dfef 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit c4c05e43b3c6468dd42c8921a1cea21c38b0798c +Subproject commit adf97dfeff25d28837ade5e8b47fc1cc28bbb082 From 43079f6dc189caf028a6d36a8981e396086d6878 Mon Sep 17 00:00:00 2001 From: iglocska Date: Mon, 20 Jan 2020 10:07:11 +0100 Subject: [PATCH 67/73] fix: [settings] purge previous setting, push new one --- app/Model/AdminSetting.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/Model/AdminSetting.php b/app/Model/AdminSetting.php index eea0f6736..b178ef80f 100644 --- a/app/Model/AdminSetting.php +++ b/app/Model/AdminSetting.php @@ -21,19 +21,19 @@ class AdminSetting extends AppModel $setting_object = $this->find('first', array( 'conditions' => array('setting' => $setting) )); - if (!empty($setting_object)) { - $setting_object['AdminSetting']['value'] = $value; - } else { - $this->create(); - $setting_object['AdminSetting'] = array('setting' => $setting, 'value' => $value); - } + $this->deleteAll('setting' => $setting); + $this->create(); + $setting_object['AdminSetting'] = array('setting' => $setting, 'value' => $value); if ($this->save($setting_object)) { return true; } else { + $this->create(); + $this->save($setting_object); return $this->validationErrors; } } + public function getSetting($setting) { $setting_object = $this->find('first', array( From 24d13b769c85a870ce17360b06b24fa0ad7cb21e Mon Sep 17 00:00:00 2001 From: iglocska Date: Mon, 20 Jan 2020 10:18:00 +0100 Subject: [PATCH 68/73] fix: [update] typo fixed --- app/Model/AdminSetting.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Model/AdminSetting.php b/app/Model/AdminSetting.php index b178ef80f..25102dbc3 100644 --- a/app/Model/AdminSetting.php +++ b/app/Model/AdminSetting.php @@ -21,7 +21,7 @@ class AdminSetting extends AppModel $setting_object = $this->find('first', array( 'conditions' => array('setting' => $setting) )); - $this->deleteAll('setting' => $setting); + $this->deleteAll(array('setting' => $setting)); $this->create(); $setting_object['AdminSetting'] = array('setting' => $setting, 'value' => $value); if ($this->save($setting_object)) { From 22b2e82ace35e8065034722fb9d436ff356282e8 Mon Sep 17 00:00:00 2001 From: iglocska Date: Mon, 20 Jan 2020 10:23:51 +0100 Subject: [PATCH 69/73] fix: [upgrade] Added a safety net for launching superfluous updates --- app/Model/AppModel.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index ad394cf65..fbcb8bd5b 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -146,6 +146,9 @@ class AppModel extends Model // this could become useful in the future public function updateMISP($command) { + if (!$this->_isSiteAdmin() && (!Configure::read('MISP.live') || $this->_isRest())) { + return false; + } $dbUpdateSuccess = false; switch ($command) { case '2.4.20': @@ -1490,7 +1493,7 @@ class AppModel extends Model if ($indexSuccess['success']) { $this->__setUpdateResMessages(count($sqlArray)+$i, __('Successfuly indexed ') . sprintf('%s -> %s', $iA[0], $iA[1])); } else { - $this->__setUpdateResMessages(count($sqlArray)+$i, sprintf('%s %s %s %s', + $this->__setUpdateResMessages(count($sqlArray)+$i, sprintf('%s %s %s %s', __('Failed to add index'), sprintf('%s -> %s', $iA[0], $iA[1]), __('The returned error is:') . PHP_EOL, @@ -1759,8 +1762,7 @@ class AppModel extends Model $job = $this->Job->find('first', array( 'conditions' => array('Job.id' => $processId) )); - - if (!empty($updates)) { + if (empty($updates)) { // Exit if updates are locked. // This is not as reliable as a real lock implementation // However, as all updates are re-playable, there is no harm if they From 379c6c3443051572cb5769b2ced9695080b8b6a3 Mon Sep 17 00:00:00 2001 From: iglocska Date: Mon, 20 Jan 2020 10:25:00 +0100 Subject: [PATCH 70/73] fix: [upgrade] removed test change --- app/Model/AppModel.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index fbcb8bd5b..3a2350b3a 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -1762,7 +1762,7 @@ class AppModel extends Model $job = $this->Job->find('first', array( 'conditions' => array('Job.id' => $processId) )); - if (empty($updates)) { + if (!empty($updates)) { // Exit if updates are locked. // This is not as reliable as a real lock implementation // However, as all updates are re-playable, there is no harm if they From 6dc79425dda08334761a843392c00ce8c4f90281 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Mon, 20 Jan 2020 10:39:50 +0100 Subject: [PATCH 71/73] chg: [queryVersion] Bumped version --- app/Controller/AppController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index ec06280b9..dcbb253af 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -46,7 +46,7 @@ class AppController extends Controller public $helpers = array('Utility', 'OrgImg', 'FontAwesome', 'UserName'); - private $__queryVersion = '95'; + private $__queryVersion = '96'; public $pyMispVersion = '2.4.120'; public $phpmin = '7.2'; public $phprec = '7.4'; From 887cfa1cc25864652ca403e330c5258003009c33 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 20 Jan 2020 11:26:05 +0100 Subject: [PATCH 72/73] chg: [misp-galaxy] updated to the latest version --- app/files/misp-galaxy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy index 5da0c7bd5..dbaab413b 160000 --- a/app/files/misp-galaxy +++ b/app/files/misp-galaxy @@ -1 +1 @@ -Subproject commit 5da0c7bd545ee93cf40786c1c535b9d4897943b1 +Subproject commit dbaab413b6b4680ae458a7d7a8ac6e1917fcc357 From 918b415486fae52c2c373c127d74d883d2bb1328 Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 20 Jan 2020 11:26:49 +0100 Subject: [PATCH 73/73] chg: [misp-objects] updated to the latest version --- app/files/misp-objects | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/misp-objects b/app/files/misp-objects index bce101832..fa6348039 160000 --- a/app/files/misp-objects +++ b/app/files/misp-objects @@ -1 +1 @@ -Subproject commit bce101832597b5f62679ac0299b2b4ef4681a145 +Subproject commit fa634803911d211f993049242d41eebaf342a9c4