MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

new: [API] object level restSearch added 

still WiP
pull/5635/head
iglocska 2020-02-29 08:57:32 +01:00
parent e480bd7113
commit 4bfcc3211b
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
6 changed files with 327 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
app/Controller/AppController.php
View File

@ -62,6 +62,7 @@ class AppController extends Controller
public $automationArray = array(
'events' => array('csv', 'nids', 'hids', 'xml', 'restSearch', 'stix', 'updateGraph', 'downloadOpenIOCEvent'),
'attributes' => array('text', 'downloadAttachment', 'returnAttributes', 'restSearch', 'rpz', 'bro'),
'objects' => array('restSearch')
);
protected $_legacyParams = array();
@ -1174,10 +1175,13 @@ class AppController extends Controller
public function restSearch()
{
if (empty($this->RestSearch->paramArray[$this->modelClass])) {
$scope = empty($this->scopeOverride) ? $this->modelClass : $this->scopeOverride;
if ($scope === 'MispObject') {
$scope = 'Object';
}
if (empty($this->RestSearch->paramArray[$scope])) {
throw new NotFoundException(__('RestSearch is not implemented (yet) for this scope.'));
}
$scope = empty($this->scopeOverride) ? $this->modelClass : $this->scopeOverride;
if (!isset($this->$scope)) {
$this->loadModel($scope);
}

6
app/Controller/Component/RestSearchComponent.php
View File

@ -18,6 +18,12 @@ class RestSearchComponent extends Component
'limit', 'page', 'requested_attributes', 'includeContext', 'headerless', 'includeWarninglistHits', 'attackGalaxy', 'deleted',
'excludeLocalTags', 'date', 'includeSightingdb', 'tag', 'object_relation'
),
'Object' => array(
'returnFormat', 'value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp',
'published', 'timestamp','enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags',
'includeProposals', 'returnFormat', 'limit', 'page', 'requested_attributes', 'includeContext', 'headerless',
'includeWarninglistHits', 'attackGalaxy', 'object_relation'
),
'Sighting' => array(
'context', 'returnFormat', 'id', 'type', 'from', 'to', 'last', 'org_id', 'source', 'includeAttribute', 'includeEvent'
)

12
app/Lib/Export/JsonExport.php
View File

@ -11,6 +11,8 @@ class JsonExport
return $this->__attributeHandler($data, $options);
} else if($options['scope'] === 'Event') {
return $this->__eventHandler($data, $options);
} else if($options['scope'] === 'Object') {
return $this->__eventHandler($data, $options);
} else if($options['scope'] === 'Sighting') {
return $this->__sightingsHandler($data, $options);
}
@ -21,9 +23,17 @@ class JsonExport
App::uses('JSONConverterTool', 'Tools');
$this->__converter = new JSONConverterTool();
}
return json_encode($this->__converter->convert($event, false, true));
return json_encode($this->__converter->convertObject($event, false, true));
}
private function __objectHandler($object, $options = array()) {
if ($this->__converter === false) {
App::uses('JSONConverterTool', 'Tools');
$this->__converter = new JSONConverterTool();
}
return json_encode($this->__converter->convert($object, false, true));
}
private function __attributeHandler($attribute, $options = array())
{
$attribute = array_merge($attribute['Attribute'], $attribute);

22
app/Lib/Tools/JSONConverterTool.php
View File

@ -11,6 +11,28 @@ class JSONConverterTool
return ']}' . PHP_EOL;
}
public function convertObject($object, $isSiteAdmin = false, $raw = false)
{
$toRearrange = array('SharingGroup', 'Attribute', 'ShadowAttribute', 'Event');
foreach ($toRearrange as $element) {
if (isset($object[$element])) {
$object['Object'][$element] = $object[$element];
unset($object[$element]);
}
if ($element == 'SharingGroup' && isset($object['Object']['SharingGroup']) && empty($object['Object']['SharingGroup'])) {
unset($object['Object']['SharingGroup']);
}
}
$result = array('Object' => $object['Object']);
if (isset($event['errors'])) {
$result = array_merge($result, array('errors' => $event['errors']));
}
if ($raw) {
return $result;
}
return json_encode($result, JSON_PRETTY_PRINT);
}
public function convert($event, $isSiteAdmin=false, $raw = false)
{
$toRearrange = array('Org', 'Orgc', 'SharingGroup', 'Attribute', 'ShadowAttribute', 'RelatedAttribute', 'RelatedEvent', 'Galaxy', 'Object');

32
app/Model/Event.php
View File

@ -1656,6 +1656,8 @@ class Event extends AppModel
),
'Object' => array(
'object_name' => array('function' => 'set_filter_object_name'),
'object_template_uuid' => array('function' => 'set_filter_object_template_uuid'),
'object_template_version' => array('function' => 'set_filter_object_template_version'),
'deleted' => array('function' => 'set_filter_deleted')
),
'Attribute' => array(
@ -2693,6 +2695,36 @@ class Event extends AppModel
return $conditions;
}
public function set_filter_object_name(&$params, $conditions, $options)
{
if (!empty($params['object_name'])) {
$params['object_name'] = $this->convert_filters($params['object_name']);
$conditions = $this->generic_add_filter($conditions, $params['object_name'], 'Object.name');
}
return $conditions;
}
public function set_filter_object_template_uuid(&$params, $conditions, $options)
{
if (!empty($params['object_template_uuid'])) {
$params['object_template_uuid'] = $this->convert_filters($params['object_template_uuid']);
$conditions = $this->generic_add_filter($conditions, $params['object_template_uuid'], 'Object.template_uuid');
}
return $conditions;
}
public function set_filter_object_template_version(&$params, $conditions, $options)
{
if (!empty($params['object_template_version'])) {
$params['object_template_version'] = $this->convert_filters($params['object_template_version']);
$conditions = $this->generic_add_filter($conditions, $params['object_template_version'], 'Object.template_version');
}
return $conditions;
}
public function set_filter_comment(&$params, $conditions, $options)
{
if (!empty($params['comment'])) {

254
app/Model/MispObject.php
View File

@ -46,6 +46,10 @@ class MispObject extends AppModel
),
);
public $validFormats = array(
'json' => array('json', 'JsonExport', 'json')
);
public $shortDist = array(0 => 'Organisation', 1 => 'Community', 2 => 'Connected', 3 => 'All', 4 => ' Sharing Group', 5 => 'Inherit');
public $validate = array(
@ -72,6 +76,81 @@ class MispObject extends AppModel
)
);
public function buildFilterConditions($user, &$params)
{
$conditions = $this->buildConditions($user);
if (isset($params['wildcard'])) {
$temp = array();
$options = array(
'filter' => 'wildcard',
'scope' => 'Object',
'pop' => false,
'context' => 'Event'
);
$conditions['AND'][] = array('OR' => $this->Event->set_filter_wildcard_attributes($params, $temp, $options));
} else {
$attribute_conditions = array();
$object_conditions = array();
if (isset($params['ignore'])) {
$params['to_ids'] = array(0, 1);
$params['published'] = array(0, 1);
}
$simple_params = array(
'Object' => array(
'object_name' => array('function' => 'set_filter_object_name'),
'object_template_uuid' => array('function' => 'set_filter_object_template_uuid'),
'object_template_version' => array('function' => 'set_filter_object_template_version'),
'deleted' => array('function' => 'set_filter_deleted')
),
'Event' => array(
'eventid' => array('function' => 'set_filter_eventid'),
'eventinfo' => array('function' => 'set_filter_eventinfo'),
'ignore' => array('function' => 'set_filter_ignore'),
'from' => array('function' => 'set_filter_timestamp'),
'to' => array('function' => 'set_filter_timestamp'),
'date' => array('function' => 'set_filter_date'),
'tags' => array('function' => 'set_filter_tags'),
'last' => array('function' => 'set_filter_timestamp', 'pop' => true),
'timestamp' => array('function' => 'set_filter_timestamp', 'pop' => true),
'event_timestamp' => array('function' => 'set_filter_timestamp', 'pop' => true),
'publish_timestamp' => array('function' => 'set_filter_timestamp'),
'org' => array('function' => 'set_filter_org'),
'uuid' => array('function' => 'set_filter_uuid'),
'published' => array('function' => 'set_filter_published')
),
'Attribute' => array(
'value' => array('function' => 'set_filter_value'),
'category' => array('function' => 'set_filter_simple_attribute'),
'type' => array('function' => 'set_filter_simple_attribute'),
'object_relation' => array('function' => 'set_filter_simple_attribute'),
'tags' => array('function' => 'set_filter_tags', 'pop' => true),
'uuid' => array('function' => 'set_filter_uuid'),
'deleted' => array('function' => 'set_filter_deleted'),
'timestamp' => array('function' => 'set_filter_timestamp'),
'attribute_timestamp' => array('function' => 'set_filter_timestamp'),
'first_seen' => array('function' => 'set_filter_seen'),
'last_seen' => array('function' => 'set_filter_seen'),
'to_ids' => array('function' => 'set_filter_to_ids'),
'comment' => array('function' => 'set_filter_comment')
)
);
foreach ($params as $param => $paramData) {
foreach ($simple_params as $scope => $simple_param_scoped) {
if (isset($simple_param_scoped[$param]) && isset($params[$param]) && $params[$param] !== false) {
$options = array(
'filter' => $param,
'scope' => $scope,
'pop' => !empty($simple_param_scoped[$param]['pop']),
'context' => 'Attribute'
);
$conditions = $this->Event->{$simple_param_scoped[$param]['function']}($params, $conditions, $options);
}
}
}
}
return $conditions;
}
// check whether the variable is null or datetime
public function datetimeOrNull($fields)
{
@ -425,7 +504,7 @@ class MispObject extends AppModel
'Tag'
)
)
)
),
);
if (empty($options['includeAllTags'])) {
$params['contain']['Attribute']['AttributeTag']['Tag']['conditions']['exportable'] = 1;
@ -453,9 +532,6 @@ class MispObject extends AppModel
);
$params['contain'] = array_merge($params['contain']['Attribute'], $proposalRestriction);
}
if (isset($options['fields'])) {
$params['fields'] = $options['fields'];
}
if (isset($options['conditions'])) {
$params['conditions']['AND'][] = $options['conditions'];
}
@ -474,6 +550,12 @@ class MispObject extends AppModel
if (isset($options['group'])) {
$params['group'] = array_merge(array('Object.id'), $options['group']);
}
if (isset($options['limit'])) {
$params['limit'] = $options['limit'];
if (isset($options['page'])) {
$params['page'] = $options['page'];
}
}
if (Configure::read('MISP.unpublishedprivate')) {
$params['conditions']['AND'][] = array('OR' => array('Event.published' => 1, 'Event.orgc_id' => $user['org_id']));
}
@ -1209,4 +1291,168 @@ class MispObject extends AppModel
'revised_object_both' => $revised_object_both
);
}
public function restSearch($user, $returnFormat, $filters, $paramsOnly = false, $jobId = false, &$elementCounter = 0, &$renderView = false)
{
if (!isset($this->validFormats[$returnFormat][1])) {
throw new NotFoundException('Invalid output format.');
}
App::uses($this->validFormats[$returnFormat][1], 'Export');
$exportTool = new $this->validFormats[$returnFormat][1]();
if (empty($exportTool->non_restrictive_export)) {
if (!isset($filters['to_ids'])) {
$filters['to_ids'] = 1;
}
if (!isset($filters['published'])) {
$filters['published'] = 1;
}
$filters['allow_proposal_blocking'] = 1;
}
if (!empty($filters['quickFilter'])) {
$filters['searchall'] = $filters['quickFilter'];
if (!empty($filters['value'])) {
unset($filters['value']);
}
}
if (!empty($exportTool->renderView)) {
$renderView = $exportTool->renderView;
}
if (isset($filters['searchall'])) {
if (!empty($filters['value'])) {
$filters['wildcard'] = $filters['value'];
unset($filters['value']);
} else {
$filters['wildcard'] = $filters['searchall'];
}
}
$subqueryElements = $this->Event->harvestSubqueryElements($filters);
$filters = $this->Event->addFiltersFromSubqueryElements($filters, $subqueryElements);
$conditions = $this->buildFilterConditions($user, $filters);
$params = array(
'conditions' => $conditions,
'fields' => array('Attribute.*', 'Event.org_id', 'Event.distribution', 'Object.*'),
'withAttachments' => !empty($filters['withAttachments']) ? $filters['withAttachments'] : 0,
'enforceWarninglist' => !empty($filters['enforceWarninglist']) ? $filters['enforceWarninglist'] : 0,
'includeAllTags' => !empty($filters['includeAllTags']) ? $filters['includeAllTags'] : 0,
'includeEventUuid' => !empty($filters['includeEventUuid']) ? $filters['includeEventUuid'] : 0,
'includeEventTags' => !empty($filters['includeEventTags']) ? $filters['includeEventTags'] : 0,
'includeProposals' => !empty($filters['includeProposals']) ? $filters['includeProposals'] : 0,
'includeWarninglistHits' => !empty($filters['includeWarninglistHits']) ? $filters['includeWarninglistHits'] : 0,
'includeContext' => !empty($filters['includeContext']) ? $filters['includeContext'] : 0,
'includeSightings' => !empty($filters['includeSightings']) ? $filters['includeSightings'] : 0,
'includeSightingdb' => !empty($filters['includeSightingdb']) ? $filters['includeSightingdb'] : 0,
'includeCorrelations' => !empty($filters['includeCorrelations']) ? $filters['includeCorrelations'] : 0,
'includeDecayScore' => !empty($filters['includeDecayScore']) ? $filters['includeDecayScore'] : 0,
'includeFullModel' => !empty($filters['includeFullModel']) ? $filters['includeFullModel'] : 0,
'allow_proposal_blocking' => !empty($filters['allow_proposal_blocking']) ? $filters['allow_proposal_blocking'] : 0
);
if (!empty($filters['attackGalaxy'])) {
$params['attackGalaxy'] = $filters['attackGalaxy'];
}
if (isset($filters['include_event_uuid'])) {
$params['includeEventUuid'] = $filters['include_event_uuid'];
}
if (isset($filters['limit'])) {
$params['limit'] = $filters['limit'];
if (!isset($filters['page'])) {
$filters['page'] = 1;
}
}
if (isset($filters['page'])) {
$params['page'] = $filters['page'];
}
if (!empty($filters['deleted'])) {
$params['deleted'] = $filters['deleted'];
}
if (!empty($filters['excludeDecayed'])) {
$params['excludeDecayed'] = $filters['excludeDecayed'];
$params['includeDecayScore'] = 1;
}
if (!empty($filters['decayingModel'])) {
$params['decayingModel'] = $filters['decayingModel'];
}
if (!empty($filters['modelOverrides'])) {
$params['modelOverrides'] = $filters['modelOverrides'];
}
if (!empty($filters['score'])) {
$params['score'] = $filters['score'];
}
if ($paramsOnly) {
return $params;
}
if (method_exists($exportTool, 'modify_params')) {
$params = $exportTool->modify_params($user, $params);
}
$exportToolParams = array(
'user' => $user,
'params' => $params,
'returnFormat' => $returnFormat,
'scope' => 'Object',
'filters' => $filters
);
if (!empty($exportTool->additional_params)) {
$params = array_merge_recursive(
$params,
$exportTool->additional_params
);
}
$tmpfile = tmpfile();
fwrite($tmpfile, $exportTool->header($exportToolParams));
$loop = false;
if (empty($params['limit'])) {
$memory_in_mb = $this->convert_to_memory_limit_to_mb(ini_get('memory_limit'));
$default_attribute_memory_coefficient = Configure::check('MISP.default_attribute_memory_coefficient') ? Configure::read('MISP.default_attribute_memory_coefficient') : 80;
$memory_scaling_factor = isset($exportTool->memory_scaling_factor) ? $exportTool->memory_scaling_factor : $default_attribute_memory_coefficient;
$params['limit'] = $memory_in_mb * $memory_scaling_factor / 10;
$loop = true;
$params['page'] = 1;
}
$this->__iteratedFetch($user, $params, $loop, $tmpfile, $exportTool, $exportToolParams, $elementCounter);
fwrite($tmpfile, $exportTool->footer($exportToolParams));
fseek($tmpfile, 0);
if (fstat($tmpfile)['size']) {
$final = fread($tmpfile, fstat($tmpfile)['size']);
} else {
$final = '';
}
fclose($tmpfile);
return $final;
}
private function __iteratedFetch($user, &$params, &$loop, &$tmpfile, $exportTool, $exportToolParams, &$elementCounter = 0)
{
$continue = true;
while ($continue) {
$this->Whitelist = ClassRegistry::init('Whitelist');
$results = $this->fetchObjects($user, $params, $continue);
if ($params['includeSightingdb']) {
$this->Sightingdb = ClassRegistry::init('Sightingdb');
$results = $this->Sightingdb->attachToObjects($results, $user);
}
$params['page'] += 1;
$results = $this->Whitelist->removeWhitelistedFromArray($results, true);
$results = array_values($results);
$i = 0;
$temp = '';
foreach ($results as $object) {
$elementCounter++;
$handlerResult = $exportTool->handler($object, $exportToolParams);
$temp .= $handlerResult;
if ($handlerResult !== '') {
if ($i != count($results) -1) {
$temp .= $exportTool->separator($exportToolParams);
}
}
$i++;
}
if (!$loop) {
$continue = false;
}
if ($continue) {
$temp .= $exportTool->separator($exportToolParams);
}
fwrite($tmpfile, $temp);
}
return true;
}
}