From 9d83c840ec92cd8a04b8f28d82f7771b26c7d356 Mon Sep 17 00:00:00 2001 From: iglocska Date: Sun, 23 Sep 2018 17:44:23 +0200 Subject: [PATCH] new: [freetext] Freetext ingestion is now delegated to the background processing - no setup needed - data to be ingested dropped to file, background worker ingests and processes the file --- app/Console/Command/EventShell.php | 19 +++ app/Controller/EventsController.php | 138 +------------------ app/Model/Event.php | 197 ++++++++++++++++++++++++++++ app/tmp/cache/ingest/empty | 1 + 4 files changed, 220 insertions(+), 135 deletions(-) create mode 100644 app/tmp/cache/ingest/empty diff --git a/app/Console/Command/EventShell.php b/app/Console/Command/EventShell.php index f59bca641..59b416808 100644 --- a/app/Console/Command/EventShell.php +++ b/app/Console/Command/EventShell.php @@ -516,4 +516,23 @@ class EventShell extends AppShell ); $result = $this->Event->enrichment($options); } + + public function processfreetext() { + $inputFile = $this->args[0]; + $tempdir = new Folder(APP . 'tmp/cache/ingest', true, 0750); + $tempFile = new File(APP . 'tmp/cache/ingest' . DS . $inputFile); + $inputData = $tempFile->read(); + $inputData = json_decode($inputData, true); + $tempFile->delete(); + $this->Event->processFreeTextData( + $inputData['user'], + $inputData['attributes'], + $inputData['id'], + $inputData['default_comment'], + $inputData['force'], + $inputData['adhereToWarninglists'], + $inputData['jobId'] + ); + return true; + } } diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 0b7d9660e..85655df39 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -3641,142 +3641,12 @@ class EventsController extends AppController $attributes[$k] = $attribute; } // actually save the attribute now - $this->__processFreeTextData($attributes, $id, '', false, $adhereToWarninglists); + $this->Event->processFreeTextDataRouter($this->Auth->user(), $attributes, $id, '', false, $adhereToWarninglists); // FIXME $attributes does not contain the onteflyattributes $attributes = array_values($attributes); return $this->RestResponse->viewData($attributes, $this->response->type()); } - private function __processFreeTextData($attributes, $id, $default_comment = '', $force = false, $adhereToWarninglists = false) - { - $event = $this->Event->find('first', array( - 'conditions' => array('id' => $id), - 'recursive' => -1, - 'fields' => array('orgc_id', 'id', 'distribution', 'published', 'uuid'), - )); - if (!$this->_isSiteAdmin() && !empty($event) && $event['Event']['orgc_id'] != $this->Auth->user('org_id')) { - $objectType = 'ShadowAttribute'; - } elseif ($this->_isSiteAdmin() && isset($force) && $force) { - $objectType = 'ShadowAttribute'; - } else { - $objectType = 'Attribute'; - } - - if ($adhereToWarninglists) { - $this->Warninglist = ClassRegistry::init('Warninglist'); - $warninglists = $this->Warninglist->fetchForEventView(); - } - - $saved = 0; - $failed = 0; - $attributeSources = array('attributes', 'ontheflyattributes'); - $ontheflyattributes = array(); - foreach ($attributeSources as $source) { - foreach (${$source} as $k => $attribute) { - if ($attribute['type'] == 'ip-src/ip-dst') { - $types = array('ip-src', 'ip-dst'); - } elseif ($attribute['type'] == 'ip-src|port/ip-dst|port') { - $types = array('ip-src|port', 'ip-dst|port'); - } elseif ($attribute['type'] == 'malware-sample') { - if (!isset($attribute['data_is_handled']) || !$attribute['data_is_handled']) { - $result = $this->Event->Attribute->handleMaliciousBase64($id, $attribute['value'], $attribute['data'], array('md5', 'sha1', 'sha256'), $objectType == 'ShadowAttribute' ? true : false); - if (!$result['success']) { - $failed++; - continue; - } - $attribute['data'] = $result['data']; - $shortValue = $attribute['value']; - $attribute['value'] = $shortValue . '|' . $result['md5']; - $additionalHashes = array('sha1', 'sha256'); - foreach ($additionalHashes as $hash) { - $temp = $attribute; - $temp['type'] = 'filename|' . $hash; - $temp['value'] = $shortValue . '|' . $result[$hash]; - unset($temp['data']); - $ontheflyattributes[] = $temp; - } - } - $types = array($attribute['type']); - } else { - $types = array($attribute['type']); - } - foreach ($types as $type) { - $this->Event->$objectType->create(); - $attribute['type'] = $type; - if (empty($attribute['comment'])) { - $attribute['comment'] = $default_comment; - } - $attribute['event_id'] = $id; - if ($objectType == 'ShadowAttribute') { - $attribute['org_id'] = $this->Auth->user('org_id'); - $attribute['event_org_id'] = $event['Event']['orgc_id']; - $attribute['email'] = $this->Auth->user('email'); - $attribute['event_uuid'] = $event['Event']['uuid']; - } - // adhere to the warninglist - if ($adhereToWarninglists) { - if (!$this->Warninglist->filterWarninglistAttributes($warninglists, $attribute)) { - if ($adhereToWarninglists == 'soft') { - $attribute['to_ids'] = 0; - } else { - // just ignore the attribute - continue; - } - } - } - $AttributSave = $this->Event->$objectType->save($attribute); - if ($AttributSave) { - // If Tags, attache each tags to attribut - if (!empty($attribute['tags'])) { - foreach (explode(",", $attribute['tags']) as $tagName) { - $this->loadModel('Tag'); - $TagId = $this->Tag->captureTag(array('name' => $tagName), array('Role' => $this->userRole)); - $this->loadModel('AttributeTag'); - if (!$this->AttributeTag->attachTagToAttribute($AttributSave['Attribute']['id'], $id, $TagId)) { - throw new MethodNotAllowedException(__('Could not add tags.')); - } - } - } - $saved++; - } else { - $lastError = $this->Event->$objectType->validationErrors; - $failed++; - } - } - } - } - $emailResult = ''; - $messageScope = $objectType == 'ShadowAttribute' ? 'proposals' : 'attributes'; - if ($saved > 0) { - if ($objectType != 'ShadowAttribute') { - $event = $this->Event->find('first', array( - 'conditions' => array('Event.id' => $id), - 'recursive' => -1 - )); - if ($event['Event']['published'] == 1) { - $event['Event']['published'] = 0; - } - $date = new DateTime(); - $event['Event']['timestamp'] = $date->getTimestamp(); - $this->Event->save($event); - } else { - if (!$this->Event->ShadowAttribute->sendProposalAlertEmail($id)) { - $emailResult = " but sending out the alert e-mails has failed for at least one recipient"; - } - } - } - if ($failed > 0) { - if ($failed == 1) { - $flashMessage = $saved . ' ' . $messageScope . ' created' . $emailResult . '. ' . $failed . ' ' . $messageScope . ' could not be saved. Reason for the failure: ' . json_encode($lastError); - } else { - $flashMessage = $saved . ' ' . $messageScope . ' created' . $emailResult . '. ' . $failed . ' ' . $messageScope . ' could not be saved. This may be due to attributes with similar values already existing.'; - } - } else { - $flashMessage = $saved . ' ' . $messageScope . ' created' . $emailResult . '.'; - } - return $flashMessage; - } - public function saveFreeText($id) { if (!$this->userRole['perm_add']) { @@ -3790,13 +3660,11 @@ class EventsController extends AppController $attributes = json_decode($this->request->data['Attribute']['JsonObject'], true); $default_comment = $this->request->data['Attribute']['default_comment']; $force = $this->request->data['Attribute']['force']; - - $flashMessage = $this->__processFreeTextData($attributes, $id, $default_comment, $force); - + $flashMessage = $this->Event->processFreeTextDataRouter($this->Auth->user(), $attributes, $id, $default_comment, $force); $this->Flash->info($flashMessage); $this->redirect(array('controller' => 'events', 'action' => 'view', $id)); } else { - throw new MethodNotAllowedException(); + throw new MethodNotAllowedException('This endpoint requires a POST request.'); } } diff --git a/app/Model/Event.php b/app/Model/Event.php index f90903a6e..3da7732ff 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -5014,4 +5014,201 @@ class Event extends AppModel )); return false; } + + public function processFreeTextData($user, $attributes, $id, $default_comment = '', $force = false, $adhereToWarninglists = false, $jobId = false) + { + $event = $this->find('first', array( + 'conditions' => array('id' => $id), + 'recursive' => -1, + 'fields' => array('orgc_id', 'id', 'distribution', 'published', 'uuid'), + )); + if (!$user['Role']['perm_site_admin'] && !empty($event) && $event['Event']['orgc_id'] != $user['org_id']) { + $objectType = 'ShadowAttribute'; + } elseif ($user['Role']['perm_site_admin'] && isset($force) && $force) { + $objectType = 'ShadowAttribute'; + } else { + $objectType = 'Attribute'; + } + + if ($adhereToWarninglists) { + $this->Warninglist = ClassRegistry::init('Warninglist'); + $warninglists = $this->Warninglist->fetchForEventView(); + } + $saved = 0; + $failed = 0; + $attributeSources = array('attributes', 'ontheflyattributes'); + $ontheflyattributes = array(); + $i = 0; + $total = count($attributeSources); + if ($jobId) { + $this->Job = ClassRegistry::init('Job'); + $this->Job->id = $jobId; + } + foreach ($attributeSources as $sourceKey => $source) { + foreach (${$source} as $k => $attribute) { + if ($attribute['type'] == 'ip-src/ip-dst') { + $types = array('ip-src', 'ip-dst'); + } elseif ($attribute['type'] == 'ip-src|port/ip-dst|port') { + $types = array('ip-src|port', 'ip-dst|port'); + } elseif ($attribute['type'] == 'malware-sample') { + if (!isset($attribute['data_is_handled']) || !$attribute['data_is_handled']) { + $result = $this->Attribute->handleMaliciousBase64($id, $attribute['value'], $attribute['data'], array('md5', 'sha1', 'sha256'), $objectType == 'ShadowAttribute' ? true : false); + if (!$result['success']) { + $failed++; + continue; + } + $attribute['data'] = $result['data']; + $shortValue = $attribute['value']; + $attribute['value'] = $shortValue . '|' . $result['md5']; + $additionalHashes = array('sha1', 'sha256'); + foreach ($additionalHashes as $hash) { + $temp = $attribute; + $temp['type'] = 'filename|' . $hash; + $temp['value'] = $shortValue . '|' . $result[$hash]; + unset($temp['data']); + $ontheflyattributes[] = $temp; + } + } + $types = array($attribute['type']); + } else { + $types = array($attribute['type']); + } + foreach ($types as $type) { + $this->$objectType->create(); + $attribute['type'] = $type; + if (empty($attribute['comment'])) { + $attribute['comment'] = $default_comment; + } + $attribute['event_id'] = $id; + if ($objectType == 'ShadowAttribute') { + $attribute['org_id'] = $user['Role']['org_id']; + $attribute['event_org_id'] = $event['Event']['orgc_id']; + $attribute['email'] = $user['Role']['email']; + $attribute['event_uuid'] = $event['Event']['uuid']; + } + // adhere to the warninglist + if ($adhereToWarninglists) { + if (!$this->Warninglist->filterWarninglistAttributes($warninglists, $attribute)) { + if ($adhereToWarninglists == 'soft') { + $attribute['to_ids'] = 0; + } else { + // just ignore the attribute + continue; + } + } + } + $AttributSave = $this->$objectType->save($attribute); + if ($AttributSave) { + // If Tags, attache each tags to attribut + if (!empty($attribute['tags'])) { + foreach (explode(",", $attribute['tags']) as $tagName) { + $this->loadModel('Tag'); + $TagId = $this->Tag->captureTag(array('name' => $tagName), array('Role' => $user['Role'])); + $this->loadModel('AttributeTag'); + if (!$this->AttributeTag->attachTagToAttribute($AttributSave['Attribute']['id'], $id, $TagId)) { + throw new MethodNotAllowedException(__('Could not add tags.')); + } + } + } + $saved++; + } else { + $lastError = $this->$objectType->validationErrors; + $failed++; + } + } + if ($jobId) { + if ($i % 20 == 0) { + $this->Job->saveField('message', 'Attribute ' . $i . '/' . $total); + $this->Job->saveField('progress', $i * 80 / $total); + } + } + } + } + $emailResult = ''; + $messageScope = $objectType == 'ShadowAttribute' ? 'proposals' : 'attributes'; + if ($saved > 0) { + if ($objectType != 'ShadowAttribute') { + $event = $this->find('first', array( + 'conditions' => array('Event.id' => $id), + 'recursive' => -1 + )); + if ($event['Event']['published'] == 1) { + $event['Event']['published'] = 0; + } + $date = new DateTime(); + $event['Event']['timestamp'] = $date->getTimestamp(); + $this->save($event); + } else { + if (!$this->ShadowAttribute->sendProposalAlertEmail($id)) { + $emailResult = " but sending out the alert e-mails has failed for at least one recipient"; + } + } + } + if ($failed > 0) { + if ($failed == 1) { + $message = $saved . ' ' . $messageScope . ' created' . $emailResult . '. ' . $failed . ' ' . $messageScope . ' could not be saved. Reason for the failure: ' . json_encode($lastError); + } else { + $message = $saved . ' ' . $messageScope . ' created' . $emailResult . '. ' . $failed . ' ' . $messageScope . ' could not be saved. This may be due to attributes with similar values already existing.'; + } + } else { + $message = $saved . ' ' . $messageScope . ' created' . $emailResult . '.'; + } + if ($jobId) { + if ($i % 20 == 0) { + $this->Job->saveField('message', 'Processing complete. ' . $message); + $this->Job->saveField('progress', 100); + } + } + return $message; + } + + public function processFreeTextDataRouter($user, $attributes, $id, $default_comment = '', $force = false, $adhereToWarninglists = false) + { + if (Configure::read('MISP.background_jobs')) { + $job = ClassRegistry::init('Job'); + $job->create(); + $data = array( + 'worker' => 'default', + 'job_type' => 'process_freetext_data', + 'job_input' => 'Event: ' . $id, + 'status' => 0, + 'retries' => 0, + 'org_id' => $user['org_id'], + 'org' => $user['Organisation']['name'], + 'message' => 'Processing...', + ); + $job->save($data); + $randomFileName = $this->generateRandomFileName() . '.json'; + App::uses('Folder', 'Utility'); + App::uses('File', 'Utility'); + $tempdir = new Folder(APP . 'tmp/cache/ingest', true, 0755); + $tempFile = new File(APP . 'tmp/cache/ingest' . DS . $randomFileName, true, 0644); + $tempData = array( + 'user' => $user, + 'attributes' => $attributes, + 'id' => $id, + 'default_comment' => $default_comment, + 'force' => $force, + 'adhereToWarninglists' => $adhereToWarninglists, + 'jobId' => $job->id + ); + + $writeResult = $tempFile->write(json_encode($tempData)); + if (!$writeResult) { + return ($this->processFreeTextData($user, $attributes, $id, $default_comment = '', $force = false, $adhereToWarninglists = false)); + } + $tempFile->close(); + $jobId = $job->id; + $process_id = CakeResque::enqueue( + 'prio', + 'EventShell', + array('processfreetext', $randomFileName), + true + ); + $job->saveField('process_id', $process_id); + return 'Freetext ingestion queued for background processing. Attributes will be added to the event as they are being processed.'; + } else { + return ($this->processFreeTextData($user, $attributes, $id, $default_comment = '', $force = false, $adhereToWarninglists = false)); + } + } } diff --git a/app/tmp/cache/ingest/empty b/app/tmp/cache/ingest/empty new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/app/tmp/cache/ingest/empty @@ -0,0 +1 @@ +