diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php index 83d2121ed..5fc2d28f1 100644 --- a/app/Controller/AttributesController.php +++ b/app/Controller/AttributesController.php @@ -1769,7 +1769,7 @@ class AttributesController extends AppController } public function restSearch($returnFormat = 'json', $value = false, $type = false, $category = false, $org = false, $tags = false, $from = false, $to = false, $last = false, $eventid = false, $withAttachments = false, $uuid = false, $publish_timestamp = false, $published = false, $timestamp = false, $enforceWarninglist = false, $to_ids = false, $deleted = false, $includeEventUuid = false, $event_timestamp = false, $threat_level_id = false) { - $paramArray = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags'); + $paramArray = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags', 'includeProposals'); $filterData = array( 'request' => $this->request, 'named_params' => $this->params['named'], @@ -1794,9 +1794,10 @@ class AttributesController extends AppController if ($returnFormat === 'download') { $returnFormat = 'json'; } - $final = $this->Attribute->restSearch($user, $returnFormat, $filters); + $elementCounter = 0; + $final = $this->Attribute->restSearch($user, $returnFormat, $filters, false, false, $elementCounter); $responseType = $validFormats[$returnFormat][0]; - return $this->RestResponse->viewData($final, $responseType, false, true); + return $this->RestResponse->viewData($final, $responseType, false, true, false, array('X-result-count' => $elementCounter)); } // returns an XML with attributes that belong to an event. The type of attributes to be returned can be restricted by type using the 3rd parameter. diff --git a/app/Controller/Component/RestResponseComponent.php b/app/Controller/Component/RestResponseComponent.php index ae999ea1e..4d590db90 100644 --- a/app/Controller/Component/RestResponseComponent.php +++ b/app/Controller/Component/RestResponseComponent.php @@ -45,7 +45,7 @@ class RestResponseComponent extends Component Besides the parameters listed, other, format specific ones can be passed along (for example: requested_attributes and includeContext for the CSV export). This API allows pagination via the page and limit parameters.", 'mandatory' => array('returnFormat'), - 'optional' => array('page', 'limit', 'value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'includeEventTags', 'event_timestamp', 'threat_level_id', 'eventinfo'), + 'optional' => array('page', 'limit', 'value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'includeEventTags', 'event_timestamp', 'threat_level_id', 'eventinfo', 'includeProposals'), 'params' => array() ) ), @@ -352,7 +352,7 @@ class RestResponseComponent extends Component return $this->__sendResponse($response, 200, $format); } - private function __sendResponse($response, $code, $format = false, $raw = false, $download = false) + private function __sendResponse($response, $code, $format = false, $raw = false, $download = false, $headers = array()) { if (strtolower($format) === 'application/xml' || strtolower($format) === 'xml') { if (!$raw) { @@ -382,6 +382,11 @@ class RestResponseComponent extends Component $type = 'json'; } $cakeResponse = new CakeResponse(array('body'=> $response, 'status' => $code, 'type' => $type)); + if (!empty($headers)) { + foreach ($headers as $key => $value) { + $cakeResponse->header($key, $value); + } + } if ($download) { $cakeResponse->download($download); } @@ -404,12 +409,12 @@ class RestResponseComponent extends Component return array('action' => $action, 'admin' => $admin); } - public function viewData($data, $format = false, $errors = false, $raw = false, $download = false) + public function viewData($data, $format = false, $errors = false, $raw = false, $download = false, $headers = array()) { if (!empty($errors)) { $data['errors'] = $errors; } - return $this->__sendResponse($data, 200, $format, $raw, $download); + return $this->__sendResponse($data, 200, $format, $raw, $download, $headers); } public function sendFile($path, $format = false, $download = false, $name = 'download') { diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 29f37ad73..cbcb4583c 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -2976,9 +2976,17 @@ class EventsController extends AppController // the last 4 fields accept the following operators: // && - you can use && between two search values to put a logical OR between them. for value, 1.1.1.1&&2.2.2.2 would find attributes with the value being either of the two. // ! - you can negate a search term. For example: google.com&&!mail would search for all attributes with value google.com but not ones that include mail. www.google.com would get returned, mail.google.com wouldn't. - public function restSearch($returnFormat = 'json', $value = false, $type = false, $category = false, $org = false, $tags = false, $searchall = false, $from = false, $to = false, $last = false, $eventid = false, $withAttachments = false, $metadata = false, $uuid = false, $publish_timestamp = false, $timestamp = false, $published = false, $enforceWarninglist = false, $sgReferenceOnly = false) + public function restSearch( + $returnFormat = 'json', $value = false, $type = false, $category = false, $org = false, $tags = false, + $searchall = false, $from = false, $to = false, $last = false, $eventid = false, $withAttachments = false, + $metadata = false, $uuid = false, $publish_timestamp = false, $timestamp = false, $published = false, $enforceWarninglist = false, + $sgReferenceOnly = false + ) { - $paramArray = array('value', 'type', 'category', 'org', 'tag', 'tags', 'searchall', 'from', 'to', 'last', 'eventid', 'withAttachments', 'metadata', 'uuid', 'published', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'sgReferenceOnly'); + $paramArray = array( + 'value', 'type', 'category', 'org', 'tag', 'tags', 'searchall', 'from', 'to', 'last', 'eventid', 'withAttachments', + 'metadata', 'uuid', 'published', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'sgReferenceOnly' + ); $filterData = array( 'request' => $this->request, 'named_params' => $this->params['named'], @@ -3002,9 +3010,10 @@ class EventsController extends AppController if ($returnFormat === 'download') { $returnFormat = 'json'; } - $final = $this->Event->restSearch($user, $returnFormat, $filters); + $elementCounter = 0; + $final = $this->Event->restSearch($user, $returnFormat, $filters, false, false, $elementCounter); $responseType = $this->Event->validFormats[$returnFormat][0]; - return $this->RestResponse->viewData($final, $responseType, false, true); + return $this->RestResponse->viewData($final, $responseType, false, true, false, array('X-result-count' => $elementCounter)); } public function downloadOpenIOCEvent($key, $eventid, $enforceWarninglist = false) diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 946c49cfc..ff35cdb21 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -2796,6 +2796,38 @@ class Attribute extends AppModel ) ) ); + if ($options['includeProposals']) { + $this->bindModel( + array('hasMany' => array( + 'ShadowAttribute' => array( + 'className' => 'ShadowAttribute', + 'foreignKey' => 'old_id', + 'conditions' => array('ShadowAttribute.deleted' => 0) + ) + ) + ) + ); + $params['contain']['ShadowAttribute'] = array('fields' => array( + "id", + "old_id", + "event_id", + "type", + "category", + "value1", + "to_ids", + "uuid", + "value2", + "org_id", + "event_org_id", + "comment", + "event_uuid", + "deleted", + "timestamp", + "proposal_to_delete", + "disable_correlation", + "value" + )); + } if (empty($options['includeAllTags'])) { $params['contain']['AttributeTag']['Tag']['conditions']['exportable'] = 1; } @@ -3454,6 +3486,8 @@ class Attribute extends AppModel unset($attribute['id']); if (isset($attribute['encrypt'])) { $result = $this->handleMaliciousBase64($eventId, $attribute['value'], $attribute['data'], array('md5')); + $attribute['data'] = $result['data']; + $attribute['value'] = $attribute['value'] . '|' . $result['md5']; } $fieldList = $this->captureFields; $this->create(); @@ -3743,7 +3777,7 @@ class Attribute extends AppModel return $conditions; } - public function restSearch($user, $returnFormat, $filters, $paramsOnly = false, $jobId = false) + public function restSearch($user, $returnFormat, $filters, $paramsOnly = false, $jobId = false, &$elementCounter = 0) { if (!isset($this->validFormats[$returnFormat][1])) { throw new NotFoundException('Invalid output format.'); @@ -3781,7 +3815,8 @@ class Attribute extends AppModel 'includeAllTags' => !empty($filters['includeAllTags']) ? $filters['includeAllTags'] : 0, 'flatten' => 1, 'includeEventUuid' => !empty($filters['includeEventUuid']) ? $filters['includeEventUuid'] : 0, - 'includeEventTags' => !empty($filters['includeEventTags']) ? $filters['includeEventTags'] : 0 + 'includeEventTags' => !empty($filters['includeEventTags']) ? $filters['includeEventTags'] : 0, + 'includeProposals' => !empty($filters['includeProposals']) ? $filters['includeProposals'] : 0 ); if (isset($filters['include_event_uuid'])) { $params['includeEventUuid'] = $filters['include_event_uuid']; @@ -3829,7 +3864,7 @@ class Attribute extends AppModel $loop = true; $params['page'] = 1; } - $this->__iteratedFetch($user, $params, $loop, $tmpfile, $exportTool, $exportToolParams); + $this->__iteratedFetch($user, $params, $loop, $tmpfile, $exportTool, $exportToolParams, $elementCounter); fwrite($tmpfile, $exportTool->footer($exportToolParams)); fseek($tmpfile, 0); $final = fread($tmpfile, fstat($tmpfile)['size']); @@ -3837,7 +3872,7 @@ class Attribute extends AppModel return $final; } - private function __iteratedFetch($user, &$params, &$loop, &$tmpfile, $exportTool, $exportToolParams) { + private function __iteratedFetch($user, &$params, &$loop, &$tmpfile, $exportTool, $exportToolParams, &$elementCounter = 0) { $continue = true; while ($continue) { $this->Whitelist = ClassRegistry::init('Whitelist'); @@ -3848,6 +3883,7 @@ class Attribute extends AppModel $i = 0; $temp = ''; foreach ($results as $attribute) { + $elementCounter++; $temp .= $exportTool->handler($attribute, $exportToolParams); if ($temp !== '') { if ($i != count($results) -1) { diff --git a/app/Model/Event.php b/app/Model/Event.php index 2a1d98a51..1ba2acc9c 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -5454,7 +5454,7 @@ class Event extends AppModel } } - public function restSearch($user, $returnFormat, $filters, $paramsOnly = false, $jobId = false) + public function restSearch($user, $returnFormat, $filters, $paramsOnly = false, $jobId = false, &$elementCounter = 0) { if (!isset($this->validFormats[$returnFormat][1])) { throw new NotFoundException('Invalid output format.'); @@ -5534,6 +5534,7 @@ class Event extends AppModel ); if (!empty($result)) { foreach ($result as $event) { + $elementCounter++; if ($jobId && $i%10 == 0) { $this->Job->saveField('progress', intval((100 * $i) / $eventCount)); $this->Job->saveField('message', 'Converting Event ' . $i . '/' . $eventCount . '.');