mirror of https://github.com/MISP/MISP
new: [freetext] Freetext ingestion is now delegated to the background processing
- no setup needed - data to be ingested dropped to file, background worker ingests and processes the filepull/3711/head
iglocska
parent
072f85fe66
commit
9d83c840ec
|
|
@ -516,4 +516,23 @@ class EventShell extends AppShell
|
|||
);
|
||||
$result = $this->Event->enrichment($options);
|
||||
}
|
||||
|
||||
public function processfreetext() {
|
||||
$inputFile = $this->args[0];
|
||||
$tempdir = new Folder(APP . 'tmp/cache/ingest', true, 0750);
|
||||
$tempFile = new File(APP . 'tmp/cache/ingest' . DS . $inputFile);
|
||||
$inputData = $tempFile->read();
|
||||
$inputData = json_decode($inputData, true);
|
||||
$tempFile->delete();
|
||||
$this->Event->processFreeTextData(
|
||||
$inputData['user'],
|
||||
$inputData['attributes'],
|
||||
$inputData['id'],
|
||||
$inputData['default_comment'],
|
||||
$inputData['force'],
|
||||
$inputData['adhereToWarninglists'],
|
||||
$inputData['jobId']
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3641,142 +3641,12 @@ class EventsController extends AppController
|
|||
$attributes[$k] = $attribute;
|
||||
}
|
||||
// actually save the attribute now
|
||||
$this->__processFreeTextData($attributes, $id, '', false, $adhereToWarninglists);
|
||||
$this->Event->processFreeTextDataRouter($this->Auth->user(), $attributes, $id, '', false, $adhereToWarninglists);
|
||||
// FIXME $attributes does not contain the onteflyattributes
|
||||
$attributes = array_values($attributes);
|
||||
return $this->RestResponse->viewData($attributes, $this->response->type());
|
||||
}
|
||||
|
||||
private function __processFreeTextData($attributes, $id, $default_comment = '', $force = false, $adhereToWarninglists = false)
|
||||
{
|
||||
$event = $this->Event->find('first', array(
|
||||
'conditions' => array('id' => $id),
|
||||
'recursive' => -1,
|
||||
'fields' => array('orgc_id', 'id', 'distribution', 'published', 'uuid'),
|
||||
));
|
||||
if (!$this->_isSiteAdmin() && !empty($event) && $event['Event']['orgc_id'] != $this->Auth->user('org_id')) {
|
||||
$objectType = 'ShadowAttribute';
|
||||
} elseif ($this->_isSiteAdmin() && isset($force) && $force) {
|
||||
$objectType = 'ShadowAttribute';
|
||||
} else {
|
||||
$objectType = 'Attribute';
|
||||
}
|
||||
|
||||
if ($adhereToWarninglists) {
|
||||
$this->Warninglist = ClassRegistry::init('Warninglist');
|
||||
$warninglists = $this->Warninglist->fetchForEventView();
|
||||
}
|
||||
|
||||
$saved = 0;
|
||||
$failed = 0;
|
||||
$attributeSources = array('attributes', 'ontheflyattributes');
|
||||
$ontheflyattributes = array();
|
||||
foreach ($attributeSources as $source) {
|
||||
foreach (${$source} as $k => $attribute) {
|
||||
if ($attribute['type'] == 'ip-src/ip-dst') {
|
||||
$types = array('ip-src', 'ip-dst');
|
||||
} elseif ($attribute['type'] == 'ip-src|port/ip-dst|port') {
|
||||
$types = array('ip-src|port', 'ip-dst|port');
|
||||
} elseif ($attribute['type'] == 'malware-sample') {
|
||||
if (!isset($attribute['data_is_handled']) || !$attribute['data_is_handled']) {
|
||||
$result = $this->Event->Attribute->handleMaliciousBase64($id, $attribute['value'], $attribute['data'], array('md5', 'sha1', 'sha256'), $objectType == 'ShadowAttribute' ? true : false);
|
||||
if (!$result['success']) {
|
||||
$failed++;
|
||||
continue;
|
||||
}
|
||||
$attribute['data'] = $result['data'];
|
||||
$shortValue = $attribute['value'];
|
||||
$attribute['value'] = $shortValue . '|' . $result['md5'];
|
||||
$additionalHashes = array('sha1', 'sha256');
|
||||
foreach ($additionalHashes as $hash) {
|
||||
$temp = $attribute;
|
||||
$temp['type'] = 'filename|' . $hash;
|
||||
$temp['value'] = $shortValue . '|' . $result[$hash];
|
||||
unset($temp['data']);
|
||||
$ontheflyattributes[] = $temp;
|
||||
}
|
||||
}
|
||||
$types = array($attribute['type']);
|
||||
} else {
|
||||
$types = array($attribute['type']);
|
||||
}
|
||||
foreach ($types as $type) {
|
||||
$this->Event->$objectType->create();
|
||||
$attribute['type'] = $type;
|
||||
if (empty($attribute['comment'])) {
|
||||
$attribute['comment'] = $default_comment;
|
||||
}
|
||||
$attribute['event_id'] = $id;
|
||||
if ($objectType == 'ShadowAttribute') {
|
||||
$attribute['org_id'] = $this->Auth->user('org_id');
|
||||
$attribute['event_org_id'] = $event['Event']['orgc_id'];
|
||||
$attribute['email'] = $this->Auth->user('email');
|
||||
$attribute['event_uuid'] = $event['Event']['uuid'];
|
||||
}
|
||||
// adhere to the warninglist
|
||||
if ($adhereToWarninglists) {
|
||||
if (!$this->Warninglist->filterWarninglistAttributes($warninglists, $attribute)) {
|
||||
if ($adhereToWarninglists == 'soft') {
|
||||
$attribute['to_ids'] = 0;
|
||||
} else {
|
||||
// just ignore the attribute
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
$AttributSave = $this->Event->$objectType->save($attribute);
|
||||
if ($AttributSave) {
|
||||
// If Tags, attache each tags to attribut
|
||||
if (!empty($attribute['tags'])) {
|
||||
foreach (explode(",", $attribute['tags']) as $tagName) {
|
||||
$this->loadModel('Tag');
|
||||
$TagId = $this->Tag->captureTag(array('name' => $tagName), array('Role' => $this->userRole));
|
||||
$this->loadModel('AttributeTag');
|
||||
if (!$this->AttributeTag->attachTagToAttribute($AttributSave['Attribute']['id'], $id, $TagId)) {
|
||||
throw new MethodNotAllowedException(__('Could not add tags.'));
|
||||
}
|
||||
}
|
||||
}
|
||||
$saved++;
|
||||
} else {
|
||||
$lastError = $this->Event->$objectType->validationErrors;
|
||||
$failed++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
$emailResult = '';
|
||||
$messageScope = $objectType == 'ShadowAttribute' ? 'proposals' : 'attributes';
|
||||
if ($saved > 0) {
|
||||
if ($objectType != 'ShadowAttribute') {
|
||||
$event = $this->Event->find('first', array(
|
||||
'conditions' => array('Event.id' => $id),
|
||||
'recursive' => -1
|
||||
));
|
||||
if ($event['Event']['published'] == 1) {
|
||||
$event['Event']['published'] = 0;
|
||||
}
|
||||
$date = new DateTime();
|
||||
$event['Event']['timestamp'] = $date->getTimestamp();
|
||||
$this->Event->save($event);
|
||||
} else {
|
||||
if (!$this->Event->ShadowAttribute->sendProposalAlertEmail($id)) {
|
||||
$emailResult = " but sending out the alert e-mails has failed for at least one recipient";
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($failed > 0) {
|
||||
if ($failed == 1) {
|
||||
$flashMessage = $saved . ' ' . $messageScope . ' created' . $emailResult . '. ' . $failed . ' ' . $messageScope . ' could not be saved. Reason for the failure: ' . json_encode($lastError);
|
||||
} else {
|
||||
$flashMessage = $saved . ' ' . $messageScope . ' created' . $emailResult . '. ' . $failed . ' ' . $messageScope . ' could not be saved. This may be due to attributes with similar values already existing.';
|
||||
}
|
||||
} else {
|
||||
$flashMessage = $saved . ' ' . $messageScope . ' created' . $emailResult . '.';
|
||||
}
|
||||
return $flashMessage;
|
||||
}
|
||||
|
||||
public function saveFreeText($id)
|
||||
{
|
||||
if (!$this->userRole['perm_add']) {
|
||||
|
|
@ -3790,13 +3660,11 @@ class EventsController extends AppController
|
|||
$attributes = json_decode($this->request->data['Attribute']['JsonObject'], true);
|
||||
$default_comment = $this->request->data['Attribute']['default_comment'];
|
||||
$force = $this->request->data['Attribute']['force'];
|
||||
|
||||
$flashMessage = $this->__processFreeTextData($attributes, $id, $default_comment, $force);
|
||||
|
||||
$flashMessage = $this->Event->processFreeTextDataRouter($this->Auth->user(), $attributes, $id, $default_comment, $force);
|
||||
$this->Flash->info($flashMessage);
|
||||
$this->redirect(array('controller' => 'events', 'action' => 'view', $id));
|
||||
} else {
|
||||
throw new MethodNotAllowedException();
|
||||
throw new MethodNotAllowedException('This endpoint requires a POST request.');
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5014,4 +5014,201 @@ class Event extends AppModel
|
|||
));
|
||||
return false;
|
||||
}
|
||||
|
||||
public function processFreeTextData($user, $attributes, $id, $default_comment = '', $force = false, $adhereToWarninglists = false, $jobId = false)
|
||||
{
|
||||
$event = $this->find('first', array(
|
||||
'conditions' => array('id' => $id),
|
||||
'recursive' => -1,
|
||||
'fields' => array('orgc_id', 'id', 'distribution', 'published', 'uuid'),
|
||||
));
|
||||
if (!$user['Role']['perm_site_admin'] && !empty($event) && $event['Event']['orgc_id'] != $user['org_id']) {
|
||||
$objectType = 'ShadowAttribute';
|
||||
} elseif ($user['Role']['perm_site_admin'] && isset($force) && $force) {
|
||||
$objectType = 'ShadowAttribute';
|
||||
} else {
|
||||
$objectType = 'Attribute';
|
||||
}
|
||||
|
||||
if ($adhereToWarninglists) {
|
||||
$this->Warninglist = ClassRegistry::init('Warninglist');
|
||||
$warninglists = $this->Warninglist->fetchForEventView();
|
||||
}
|
||||
$saved = 0;
|
||||
$failed = 0;
|
||||
$attributeSources = array('attributes', 'ontheflyattributes');
|
||||
$ontheflyattributes = array();
|
||||
$i = 0;
|
||||
$total = count($attributeSources);
|
||||
if ($jobId) {
|
||||
$this->Job = ClassRegistry::init('Job');
|
||||
$this->Job->id = $jobId;
|
||||
}
|
||||
foreach ($attributeSources as $sourceKey => $source) {
|
||||
foreach (${$source} as $k => $attribute) {
|
||||
if ($attribute['type'] == 'ip-src/ip-dst') {
|
||||
$types = array('ip-src', 'ip-dst');
|
||||
} elseif ($attribute['type'] == 'ip-src|port/ip-dst|port') {
|
||||
$types = array('ip-src|port', 'ip-dst|port');
|
||||
} elseif ($attribute['type'] == 'malware-sample') {
|
||||
if (!isset($attribute['data_is_handled']) || !$attribute['data_is_handled']) {
|
||||
$result = $this->Attribute->handleMaliciousBase64($id, $attribute['value'], $attribute['data'], array('md5', 'sha1', 'sha256'), $objectType == 'ShadowAttribute' ? true : false);
|
||||
if (!$result['success']) {
|
||||
$failed++;
|
||||
continue;
|
||||
}
|
||||
$attribute['data'] = $result['data'];
|
||||
$shortValue = $attribute['value'];
|
||||
$attribute['value'] = $shortValue . '|' . $result['md5'];
|
||||
$additionalHashes = array('sha1', 'sha256');
|
||||
foreach ($additionalHashes as $hash) {
|
||||
$temp = $attribute;
|
||||
$temp['type'] = 'filename|' . $hash;
|
||||
$temp['value'] = $shortValue . '|' . $result[$hash];
|
||||
unset($temp['data']);
|
||||
$ontheflyattributes[] = $temp;
|
||||
}
|
||||
}
|
||||
$types = array($attribute['type']);
|
||||
} else {
|
||||
$types = array($attribute['type']);
|
||||
}
|
||||
foreach ($types as $type) {
|
||||
$this->$objectType->create();
|
||||
$attribute['type'] = $type;
|
||||
if (empty($attribute['comment'])) {
|
||||
$attribute['comment'] = $default_comment;
|
||||
}
|
||||
$attribute['event_id'] = $id;
|
||||
if ($objectType == 'ShadowAttribute') {
|
||||
$attribute['org_id'] = $user['Role']['org_id'];
|
||||
$attribute['event_org_id'] = $event['Event']['orgc_id'];
|
||||
$attribute['email'] = $user['Role']['email'];
|
||||
$attribute['event_uuid'] = $event['Event']['uuid'];
|
||||
}
|
||||
// adhere to the warninglist
|
||||
if ($adhereToWarninglists) {
|
||||
if (!$this->Warninglist->filterWarninglistAttributes($warninglists, $attribute)) {
|
||||
if ($adhereToWarninglists == 'soft') {
|
||||
$attribute['to_ids'] = 0;
|
||||
} else {
|
||||
// just ignore the attribute
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
$AttributSave = $this->$objectType->save($attribute);
|
||||
if ($AttributSave) {
|
||||
// If Tags, attache each tags to attribut
|
||||
if (!empty($attribute['tags'])) {
|
||||
foreach (explode(",", $attribute['tags']) as $tagName) {
|
||||
$this->loadModel('Tag');
|
||||
$TagId = $this->Tag->captureTag(array('name' => $tagName), array('Role' => $user['Role']));
|
||||
$this->loadModel('AttributeTag');
|
||||
if (!$this->AttributeTag->attachTagToAttribute($AttributSave['Attribute']['id'], $id, $TagId)) {
|
||||
throw new MethodNotAllowedException(__('Could not add tags.'));
|
||||
}
|
||||
}
|
||||
}
|
||||
$saved++;
|
||||
} else {
|
||||
$lastError = $this->$objectType->validationErrors;
|
||||
$failed++;
|
||||
}
|
||||
}
|
||||
if ($jobId) {
|
||||
if ($i % 20 == 0) {
|
||||
$this->Job->saveField('message', 'Attribute ' . $i . '/' . $total);
|
||||
$this->Job->saveField('progress', $i * 80 / $total);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
$emailResult = '';
|
||||
$messageScope = $objectType == 'ShadowAttribute' ? 'proposals' : 'attributes';
|
||||
if ($saved > 0) {
|
||||
if ($objectType != 'ShadowAttribute') {
|
||||
$event = $this->find('first', array(
|
||||
'conditions' => array('Event.id' => $id),
|
||||
'recursive' => -1
|
||||
));
|
||||
if ($event['Event']['published'] == 1) {
|
||||
$event['Event']['published'] = 0;
|
||||
}
|
||||
$date = new DateTime();
|
||||
$event['Event']['timestamp'] = $date->getTimestamp();
|
||||
$this->save($event);
|
||||
} else {
|
||||
if (!$this->ShadowAttribute->sendProposalAlertEmail($id)) {
|
||||
$emailResult = " but sending out the alert e-mails has failed for at least one recipient";
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($failed > 0) {
|
||||
if ($failed == 1) {
|
||||
$message = $saved . ' ' . $messageScope . ' created' . $emailResult . '. ' . $failed . ' ' . $messageScope . ' could not be saved. Reason for the failure: ' . json_encode($lastError);
|
||||
} else {
|
||||
$message = $saved . ' ' . $messageScope . ' created' . $emailResult . '. ' . $failed . ' ' . $messageScope . ' could not be saved. This may be due to attributes with similar values already existing.';
|
||||
}
|
||||
} else {
|
||||
$message = $saved . ' ' . $messageScope . ' created' . $emailResult . '.';
|
||||
}
|
||||
if ($jobId) {
|
||||
if ($i % 20 == 0) {
|
||||
$this->Job->saveField('message', 'Processing complete. ' . $message);
|
||||
$this->Job->saveField('progress', 100);
|
||||
}
|
||||
}
|
||||
return $message;
|
||||
}
|
||||
|
||||
public function processFreeTextDataRouter($user, $attributes, $id, $default_comment = '', $force = false, $adhereToWarninglists = false)
|
||||
{
|
||||
if (Configure::read('MISP.background_jobs')) {
|
||||
$job = ClassRegistry::init('Job');
|
||||
$job->create();
|
||||
$data = array(
|
||||
'worker' => 'default',
|
||||
'job_type' => 'process_freetext_data',
|
||||
'job_input' => 'Event: ' . $id,
|
||||
'status' => 0,
|
||||
'retries' => 0,
|
||||
'org_id' => $user['org_id'],
|
||||
'org' => $user['Organisation']['name'],
|
||||
'message' => 'Processing...',
|
||||
);
|
||||
$job->save($data);
|
||||
$randomFileName = $this->generateRandomFileName() . '.json';
|
||||
App::uses('Folder', 'Utility');
|
||||
App::uses('File', 'Utility');
|
||||
$tempdir = new Folder(APP . 'tmp/cache/ingest', true, 0755);
|
||||
$tempFile = new File(APP . 'tmp/cache/ingest' . DS . $randomFileName, true, 0644);
|
||||
$tempData = array(
|
||||
'user' => $user,
|
||||
'attributes' => $attributes,
|
||||
'id' => $id,
|
||||
'default_comment' => $default_comment,
|
||||
'force' => $force,
|
||||
'adhereToWarninglists' => $adhereToWarninglists,
|
||||
'jobId' => $job->id
|
||||
);
|
||||
|
||||
$writeResult = $tempFile->write(json_encode($tempData));
|
||||
if (!$writeResult) {
|
||||
return ($this->processFreeTextData($user, $attributes, $id, $default_comment = '', $force = false, $adhereToWarninglists = false));
|
||||
}
|
||||
$tempFile->close();
|
||||
$jobId = $job->id;
|
||||
$process_id = CakeResque::enqueue(
|
||||
'prio',
|
||||
'EventShell',
|
||||
array('processfreetext', $randomFileName),
|
||||
true
|
||||
);
|
||||
$job->saveField('process_id', $process_id);
|
||||
return 'Freetext ingestion queued for background processing. Attributes will be added to the event as they are being processed.';
|
||||
} else {
|
||||
return ($this->processFreeTextData($user, $attributes, $id, $default_comment = '', $force = false, $adhereToWarninglists = false));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
|
||||
Loading…
Reference in New Issue