MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch '2.4' of github.com:MISP/MISP into chrisr3d_restSearch_tests

pull/3766/head
chrisr3d 2018-10-04 23:09:11 +02:00
parent 445bd0c84c 92eb8a91ad
commit 09a138fd38
12 changed files with 673 additions and 782 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

663
app/Controller/AttributesController.php
View File

@ -16,6 +16,17 @@ class AttributesController extends AppController
public $helpers = array('Js' => array('Jquery'));
public $validFormats = array(
'openioc' => array('xml', 'OpeniocExport'),
'json' => array('json', 'JsonExport'),
'xml' => array('xml', 'XmlExport'),
'suricata' => array('txt', 'NidsSuricataExport'),
'snort' => array('txt', 'NidsSnortExport'),
'text' => array('txt', 'TextExport'),
'rpz' => array('rpz', 'RPZExport'),
'csv' => array('csv', 'CsvExport')
);
public function beforeFilter()
{
parent::beforeFilter();
@ -118,7 +129,7 @@ class AttributesController extends AppController
foreach ($attribute['AttributeTag'] as $kat => $at) {
foreach ($tags as $ktag => $tag) {
if ($tag['Tag']['id'] == $at['tag_id']) {
$attributes[$k]['AttributeTag'][$kat]['Tag'] = $tag['Tag'];
$attributes[$k]['AttributeTag'][$kat]['Tag'] = $tag['Tag'];
}
}
}
@ -1565,453 +1576,85 @@ class AttributesController extends AppController
}
}
public function search()
public function search($continue = false)
{
$this->set('attrDescriptions', $this->Attribute->fieldDescriptions);
$this->set('typeDefinitions', $this->Attribute->typeDefinitions);
$this->set('categoryDefinitions', $this->Attribute->categoryDefinitions);
$fullAddress = '/attributes/search';
// if no search is given, show the search form
if ($this->request->here == $fullAddress && !$this->request->is('post')) {
// adding filtering by category and type
// combobox for types
$types = array('' => array('ALL' => 'ALL'), 'types' => array());
$types['types'] = array_merge($types['types'], $this->_arrayToValuesIndexArray(array_keys($this->Attribute->typeDefinitions)));
ksort($types['types']);
$this->set('types', $types);
// combobox for categories
$categories['categories'] = array_merge(array('ALL' => 'ALL'), $this->_arrayToValuesIndexArray(array_keys($this->Attribute->categoryDefinitions)));
$this->set('categories', $categories);
} else {
$this->set('isSearch', 1);
$attributeTagQuery = '/attributetag';
// check if the request is a GET request for attributes with a specific tag (usually after clicking on an attributetag)
if (substr($this->request->here, strlen($fullAddress), strlen($attributeTagQuery)) == $attributeTagQuery) {
$attributeTagId = substr($this->request->here, (strlen($fullAddress) + strlen($attributeTagQuery) + 1));
if (!is_numeric($attributeTagId)) {
// either pagination active or no correct id
unset($attributeTagId);
}
}
// if this is no new search, get parameters from session
if ($this->request->here != $fullAddress && !isset($attributeTagId)) {
$keyword = $this->Session->read('paginate_conditions_keyword');
$keyword2 = $this->Session->read('paginate_conditions_keyword2');
$attributeTags = $this->Session->read('paginate_conditions_attributetags');
$org = $this->Session->read('paginate_conditions_org');
$type = $this->Session->read('paginate_conditions_type');
$category = $this->Session->read('paginate_conditions_category');
$tags = $this->Session->read('paginate_conditions_tags');
$this->set('keywordSearch', $keyword);
$this->set('keywordSearch2', $keyword2);
$this->set('attributeTags', $attributeTags);
$this->set('orgSearch', $org);
$this->set('typeSearch', $type);
$this->set('tags', $tags);
$this->set('categorySearch', $category);
$this->Attribute->contain(array('AttributeTag' => array('Tag')));
// re-get pagination
$this->Attribute->recursive = 0;
$this->paginate = $this->Session->read('paginate_conditions');
$attributes = $this->paginate();
foreach ($attributes as $k => $attribute) {
if (empty($attribute['Event']['id'])) {
unset($attribute[$k]);
}
}
$this->set('attributes', $attributes);
// set the same view as the index page
$this->render('index');
} else {
// reset the paginate_conditions
$this->Session->write('paginate_conditions', array());
$conditions = array();
$alternateSearch = false;
if (isset($attributeTagId)) {
$this->loadModel('Tag');
$this->Tag->id = $attributeTagId;
if (!$this->Tag->exists()) {
throw new NotFoundException(__('Invalid tag'));
}
$attributeTags = $this->Tag->find('first', array(
'recursive' => -1,
'conditions' => array(
'id' => $attributeTagId
)
));
$attributeTags = $attributeTags['Tag']['name'];
$conditions['AND'][] = array('OR' => array('Attribute.id' => $this->Tag->findAttributeIdsByAttributeTagNames(array($attributeTags))));
$keyword = null;
$keyword2 = null;
$org = null;
$type = 'ALL';
$tags = null;
$category = 'ALL';
$ioc = false;
$this->set('keywordSearch', $keyword);
$this->set('keywordSearch2', $keyword2);
}
if ($this->request->is('post')) {
$keyword = $this->request->data['Attribute']['keyword'];
$keyword2 = $this->request->data['Attribute']['keyword2'];
$attributeTags = $this->request->data['Attribute']['attributetags'];
$tags = $this->request->data['Attribute']['tags'];
$org = $this->request->data['Attribute']['org'];
$type = $this->request->data['Attribute']['type'];
$ioc = $this->request->data['Attribute']['ioc'];
$this->set('ioc', $ioc);
$category = $this->request->data['Attribute']['category'];
$keyWordText = null;
$keyWordText2 = null;
$keyWordText3 = null;
// search the db
if ($ioc) {
$conditions['AND'][] = array('Attribute.to_ids =' => 1);
$conditions['AND'][] = array('Event.published =' => 1);
}
// search on the value field
if (isset($keyword)) {
$keywordArray = explode("\n", $keyword);
$this->set('keywordArray', $keywordArray);
$i = 1;
$temp = array();
$temp2 = array();
foreach ($keywordArray as $keywordArrayElement) {
$saveWord = trim(strtolower($keywordArrayElement));
if ($saveWord != '') {
$toInclude = true;
if ($saveWord[0] == '!') {
$toInclude = false;
$saveWord = substr($saveWord, 1);
}
// check for an IPv4 address and subnet in CIDR notation (e.g. 127.0.0.1/8)
if ($this->Cidr->checkCIDR($saveWord, 4)) {
$cidrresults = $this->Cidr->CIDR($saveWord);
foreach ($cidrresults as $result) {
$result = strtolower($result);
if (strpos($result, '|')) {
$resultParts = explode('|', $result);
if (!$toInclude) {
$temp2[] = array(
'AND' => array(
'LOWER(Attribute.value1) NOT LIKE' => $resultParts[0],
'LOWER(Attribute.value2) NOT LIKE' => $resultParts[1],
));
} else {
$temp[] = array(
'AND' => array(
'LOWER(Attribute.value1)' => $resultParts[0],
'LOWER(Attribute.value2)' => $resultParts[1],
));
}
} else {
if (!$toInclude) {
array_push($temp2, array('LOWER(Attribute.value1) NOT LIKE' => $result));
array_push($temp2, array('LOWER(Attribute.value2) NOT LIKE' => $result));
} else {
array_push($temp, array('LOWER(Attribute.value1) LIKE' => $result));
array_push($temp, array('LOWER(Attribute.value2) LIKE' => $result));
}
}
}
} else {
if (strpos($saveWord, '|')) {
$resultParts = explode('|', $saveWord);
if (!$toInclude) {
$temp2[] = array(
'AND' => array(
'LOWER(Attribute.value1) NOT LIKE' => $resultParts[0],
'LOWER(Attribute.value2) NOT LIKE' => $resultParts[1],
));
} else {
$temp2[] = array(
'AND' => array(
'LOWER(Attribute.value1)' => $resultParts[0],
'LOWER(Attribute.value2)' => $resultParts[1],
));
}
} else {
if (!$toInclude) {
array_push($temp2, array('LOWER(Attribute.value1) NOT LIKE' => $saveWord));
array_push($temp2, array('LOWER(Attribute.value2) NOT LIKE' => $saveWord));
} else {
array_push($temp, array('LOWER(Attribute.value1) LIKE' => $saveWord));
array_push($temp, array('LOWER(Attribute.value2) LIKE' => $saveWord));
}
}
}
if ($toInclude) {
array_push($temp, array('LOWER(Attribute.comment) LIKE' => $saveWord));
} else {
array_push($temp2, array('LOWER(Attribute.comment) NOT LIKE' => $saveWord));
}
}
if ($i == 1 && $saveWord != '') {
$keyWordText = $saveWord;
} elseif (($i > 1 && $i < 10) && $saveWord != '') {
$keyWordText = $keyWordText . ', ' . $saveWord;
} elseif ($i == 10 && $saveWord != '') {
$keyWordText = $keyWordText . ' and several other keywords';
}
$i++;
}
$this->set('keywordSearch', $keyWordText);
if (!empty($temp)) {
$conditions['AND']['OR'] = $temp;
}
if (!empty($temp2)) {
$conditions['AND'][] = $temp2;
}
}
// event IDs to be excluded
if (isset($keyword2)) {
$keywordArray2 = explode("\n", $keyword2);
$i = 1;
$temp = array();
foreach ($keywordArray2 as $keywordArrayElement) {
$saveWord = trim($keywordArrayElement);
if (empty($saveWord)) {
continue;
}
if ($saveWord[0] == '!') {
if (strlen(substr($saveWord, 1)) == 36) {
$temp[] = array('Event.uuid !=' => substr($saveWord, 1));
$temp[] = array('Attribute.uuid !=' => substr($saveWord, 1));
} else {
$temp[] = array('Attribute.event_id !=' => substr($saveWord, 1));
}
} else {
if (strlen($saveWord) == 36) {
$temp['OR'][] = array('Event.uuid =' => $saveWord);
$temp['OR'][] = array('Attribute.uuid' => $saveWord);
} else {
$temp['OR'][] = array('Attribute.event_id =' => $saveWord);
}
}
if ($i == 1 && $saveWord != '') {
$keyWordText2 = $saveWord;
} elseif (($i > 1 && $i < 10) && $saveWord != '') {
$keyWordText2 = $keyWordText2 . ', ' . $saveWord;
} elseif ($i == 10 && $saveWord != '') {
$keyWordText2 = $keyWordText2 . ' and several other events';
}
$i++;
}
$this->set('keywordSearch2', $keyWordText2);
if (!empty($temp)) {
$conditions['AND'][] = $temp;
}
}
if (!empty($attributeTags) || !empty($tags)) {
$this->loadModel('Tag');
}
if (!empty($attributeTags)) {
$includeAttributeTags = array();
$excludeAttributeTags = array();
$attributeTagsKeywordArray = explode("\n", $attributeTags);
foreach ($attributeTagsKeywordArray as $tagName) {
$tagName = trim($tagName);
if (empty($tagName)) {
continue;
}
if (substr($tagName, 0, 1) === '!') {
$excludeAttributeTags[] = substr($tagName, 1);
} else {
$includeAttributeTags[] = $tagName;
}
}
if (!empty($includeAttributeTags)) {
$conditions['AND'][] = array('OR' => array('Attribute.id' => $this->Tag->findAttributeIdsByAttributeTagNames($includeAttributeTags)));
}
if (!empty($excludeAttributeTags)) {
$conditions['AND'][] = array('Attribute.id !=' => $this->Tag->findAttributeIdsByAttributeTagNames($excludeAttributeTags));
}
}
if (!empty($tags)) {
$include = array();
$exclude = array();
$keywordArray = explode("\n", $tags);
foreach ($keywordArray as $tagname) {
$tagname = trim($tagname);
if (empty($tagname)) {
continue;
}
if (substr($tagname, 0, 1) === '!') {
$exclude[] = substr($tagname, 1);
} else {
$include[] = $tagname;
}
}
if (!empty($include)) {
$conditions['AND'][] = array('OR' => array('Attribute.event_id' => $this->Tag->findEventIdsByTagNames($include)));
}
if (!empty($exclude)) {
$conditions['AND'][] = array('Attribute.event_id !=' => $this->Tag->findEventIdsByTagNames($exclude));
}
}
if ($type != 'ALL') {
$conditions['Attribute.type ='] = $type;
}
if ($category != 'ALL') {
$conditions['Attribute.category ='] = $category;
}
// organisation search field
if (isset($org)) {
$temp = array();
$this->loadModel('Organisation');
$orgArray = explode("\n", $org);
foreach ($orgArray as $i => $orgArrayElement) {
$saveWord = trim($orgArrayElement);
if (empty($saveWord)) {
continue;
}
if ($saveWord[0] == '!') {
$org_names = $this->Organisation->find('all', array(
'fields' => array('id', 'name'),
'conditions' => array('lower(name) LIKE' => strtolower(substr($saveWord, 1))),
));
foreach ($org_names as $org_name) {
$temp['AND'][] = array('Event.orgc_id !=' => $org_name['Organisation']['id']);
}
} else {
$org_names = $this->Organisation->find('all', array(
'fields' => array('id', 'name'),
'conditions' => array('lower(name) LIKE' => strtolower($saveWord)),
));
if (empty($org_names)) {
$conditions['AND'][] = array('Event.orgc_id' => -1);
}
foreach ($org_names as $org_name) {
$temp['OR'][] = array('Event.orgc_id' => $org_name['Organisation']['id']);
}
}
if ($i == 0 && $saveWord != '') {
$keyWordText3 = $saveWord;
} elseif (($i > 0 && $i < 9) && $saveWord != '') {
$keyWordText3 = $keyWordText3 . ', ' . $saveWord;
} elseif ($i == 9 && $saveWord != '') {
$keyWordText3 = $keyWordText3 . ' and several other organisations';
}
}
$this->set('orgSearch', $keyWordText3);
if (!empty($temp)) {
$conditions['AND'][] = $temp;
}
}
if ($this->request->data['Attribute']['alternate']) {
$alternateSearch = true;
}
}
if (isset($attributeTags)) {
$this->set('attributeTags', $attributeTags);
}
$this->set('tags', $tags);
$this->set('typeSearch', $type);
$this->set('categorySearch', $category);
$conditions['AND'][] = array('Attribute.deleted' => 0);
if ($alternateSearch) {
$events = $this->searchAlternate($conditions);
$this->set('events', $events);
$this->render('alternate_search_result');
} else {
$this->Attribute->recursive = 0;
$this->paginate = array(
'limit' => 60,
'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 attributes?
'conditions' => $conditions,
'contain' => array(
'Event' => array(
'fields' => array(
'orgc_id', 'id', 'org_id', 'user_id', 'info'
),
'Orgc' => array('fields' => array('Orgc.name', 'Orgc.id'))
),
'Object' => array(
'fields' => array(
'id'
)
)
)
);
$this->Attribute->contain(array('AttributeTag' => array('Tag')));
if (!$this->_isSiteAdmin()) {
// merge in private conditions
$this->paginate['conditions'] = array('AND' => array($conditions, $this->Attribute->buildConditions($this->Auth->user())));
}
$idList = array();
$attributeIdList = array();
$attributes = $this->paginate();
$org_ids = array();
foreach ($attributes as $k => $attribute) {
if (empty($attribute['Event']['id'])) {
unset($attribute[$k]);
continue;
}
if ($attribute['Attribute']['type'] == 'attachment' && preg_match('/.*\.(jpg|png|jpeg|gif)$/i', $attribute['Attribute']['value'])) {
$attributes[$k]['Attribute']['image'] = $this->Attribute->base64EncodeAttachment($attribute['Attribute']);
}
$org_ids[$attribute['Event']['org_id']] = false;
$org_ids[$attribute['Event']['orgc_id']] = false;
}
$orgs = $this->Attribute->Event->Orgc->find('list', array(
'conditions' => array('Orgc.id' => array_keys($org_ids)),
'fields' => array('Orgc.id', 'Orgc.name')
));
$this->set('orgs', $orgs);
$this->set('attributes', $attributes);
// if we searched for IOCs only, apply the whitelist to the search result!
if ($ioc) {
$this->loadModel('Whitelist');
$attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true);
}
foreach ($attributes as $attribute) {
$attributeIdList[] = $attribute['Attribute']['id'];
if (!in_array($attribute['Attribute']['event_id'], $idList)) {
$idList[] = $attribute['Attribute']['event_id'];
}
}
$this->set('attributes', $attributes);
// and store into session
$this->Session->write('paginate_conditions', $this->paginate);
$this->Session->write('paginate_conditions_keyword', $keyword);
$this->Session->write('paginate_conditions_keyword2', $keyword2);
if (isset($attributeTags)) {
$this->Session->write('paginate_conditions_attributetags', $attributeTags);
}
$this->Session->write('paginate_conditions_org', $org);
$this->Session->write('paginate_conditions_type', $type);
$this->Session->write('paginate_conditions_ioc', $ioc);
$this->Session->write('paginate_conditions_tags', $tags);
$this->Session->write('paginate_conditions_category', $category);
$this->Session->write('search_find_idlist', $idList);
$this->Session->write('search_find_attributeidlist', $attributeIdList);
// set the same view as the index page
$this->render('index');
}
}
$this->set('attrDescriptions', $this->Attribute->fieldDescriptions);
$this->set('typeDefinitions', $this->Attribute->typeDefinitions);
$this->set('categoryDefinitions', $this->Attribute->categoryDefinitions);
if ($this->request->is('post')) {
if (isset($this->request->data['Attribute'])) {
$this->request->data = $this->request->data['Attribute'];
}
$checkForEmpty = array('value', 'tags', 'uuid', 'org', 'type', 'category');
foreach ($checkForEmpty as $field) {
if (empty($this->request->data[$field]) || $this->request->data[$field] === 'ALL') {
unset($this->request->data[$field]);
}
}
if (empty($this->request->data['to_ids'])) {
unset($this->request->data['to_ids']);
$this->request->data['ignore'] = 1;
}
$paramArray = array('value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags');
$filterData = array(
'request' => $this->request,
'named_params' => $this->params['named'],
'paramArray' => $paramArray,
'ordered_url_params' => compact($paramArray)
);
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception);
unset($filterData);
if ($filters === false) {
return $exception;
}
$this->Session->write('search_attributes_filters', json_encode($filters));
} else if ($continue === 'results') {
$filters = $this->Session->read('search_attributes_filters');
if (empty($filters)) {
$filters = array();
} else {
$filters = json_decode($filters, true);
}
} else {
$types = array('' => array('ALL' => 'ALL'), 'types' => array());
$types['types'] = array_merge($types['types'], $this->_arrayToValuesIndexArray(array_keys($this->Attribute->typeDefinitions)));
ksort($types['types']);
$this->set('types', $types);
// combobox for categories
$categories['categories'] = array_merge(array('ALL' => 'ALL'), $this->_arrayToValuesIndexArray(array_keys($this->Attribute->categoryDefinitions)));
$this->set('categories', $categories);
$this->Session->write('search_attributes_filters', null);
}
if (isset($filters)) {
$params = $this->Attribute->restSearch($this->Auth->user(), 'json', $filters, true);
$this->paginate = $params;
if (empty($this->paginate['limit'])) {
$this->paginate['limit'] = 60;
}
if (empty($this->paginate['page'])) {
$this->paginate['page'] = 1;
}
$this->paginate['recursive'] = -1;
$this->paginate['contain'] = array(
'Event' => array(
'fields' => array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.info', 'Event.user_id'),
'Orgc' => array('fields' => array('Orgc.id', 'Orgc.name')),
'Org' => array('fields' => array('Org.id', 'Org.name'))
),
'AttributeTag' => array('Tag'),
'Object' => array(
'fields' => array('Object.id', 'Object.distribution', 'Object.sharing_group_id')
)
);
$attributes = $this->paginate();
$this->set('filters', $filters);
$this->set('attributes', $attributes);
$this->set('isSearch', 1);
$this->render('index');
}
if (isset($attributeTags)) {
$this->set('attributeTags', $attributeTags);
}
}
@ -2090,15 +1733,7 @@ class AttributesController extends AppController
'paramArray' => $paramArray,
'ordered_url_params' => compact($paramArray)
);
$validFormats = array(
'openioc' => array('xml', 'OpeniocExport'),
'json' => array('json', 'JsonExport'),
'xml' => array('xml', 'XmlExport'),
'suricata' => array('txt', 'NidsSuricataExport'),
'snort' => array('txt', 'NidsSnortExport'),
'text' => array('txt', 'TextExport'),
'rpz' => array('rpz', 'RPZExport')
);
$validFormats = $this->validFormats;
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception);
unset($filterData);
@ -2116,109 +1751,11 @@ class AttributesController extends AppController
if ($returnFormat === 'download') {
$returnFormat = 'json';
}
if (!isset($validFormats[$returnFormat][1])) {
throw new NotFoundException('Invalid output format.');
}
App::uses($validFormats[$returnFormat][1], 'Export');
$exportTool = new $validFormats[$returnFormat][1]();
if (empty($exportTool->non_restrictive_export)) {
if (!isset($filters['to_ids'])) {
$filters['to_ids'] = 1;
}
if (!isset($filters['published'])) {
$filters['published'] = 1;
}
}
$conditions = $this->Attribute->buildFilterConditions($this->Auth->user(), $filters);
$params = array(
'conditions' => $conditions,
'fields' => array('Attribute.*', 'Event.org_id', 'Event.distribution'),
'withAttachments' => !empty($filters['withAttachments']) ? $filters['withAttachments'] : 0,
'enforceWarninglist' => !empty($filters['enforceWarninglist']) ? $filters['enforceWarninglist'] : 0,
'includeAllTags' => true,
'flatten' => 1,
'includeEventUuid' => !empty($filters['includeEventUuid']) ? $filters['includeEventUuid'] : 0,
'includeEventTags' => !empty($filters['includeEventTags']) ? $filters['includeEventTags'] : 0
);
if (isset($filters['include_event_uuid'])) {
$params['includeEventUuid'] = $filters['include_event_uuid'];
}
if (isset($filters['limit'])) {
$params['limit'] = $filters['limit'];
}
if (isset($filters['page'])) {
$params['page'] = $filters['page'];
}
if (!empty($filtes['deleted'])) {
$params['deleted'] = 1;
if ($params['deleted'] === 'only') {
$params['conditions']['AND'][] = array('Attribute.deleted' => 1);
$params['conditions']['AND'][] = array('Object.deleted' => 1);
}
}
if (!isset($validFormats[$returnFormat])) {
// this is where the new code path for the export modules will go
throw new MethodNotFoundException('Invalid export format.');
}
$exportToolParams = array(
'user' => $this->Auth->user(),
'params' => $params,
'returnFormat' => $returnFormat,
'scope' => 'Attribute',
'filters' => $filters
);
if (!empty($exportTool->additional_params)) {
$params = array_merge($params, $exportTool->additional_params);
}
$tmpfile = tmpfile();
fwrite($tmpfile, $exportTool->header($exportToolParams));
$loop = false;
if (empty($params['limit'])) {
$memory_in_mb = $this->Attribute->convert_to_memory_limit_to_mb(ini_get('memory_limit'));
$memory_scaling_factor = isset($exportTool->memory_scaling_factor) ? $exportTool->memory_scaling_factor : 100;
$params['limit'] = $memory_in_mb * $memory_scaling_factor;
$loop = true;
$params['page'] = 1;
}
$this->__iteratedFetch($params, $loop, $tmpfile, $exportTool, $exportToolParams);
fwrite($tmpfile, $exportTool->footer($exportToolParams));
fseek($tmpfile, 0);
$final = fread($tmpfile, fstat($tmpfile)['size']);
fclose($tmpfile);
$final = $this->Attribute->restSearch($user, $returnFormat, $filters);
$responseType = $validFormats[$returnFormat][0];
return $this->RestResponse->viewData($final, $responseType, false, true);
}
private function __iteratedFetch(&$params, &$loop, &$tmpfile, $exportTool, $exportToolParams) {
$continue = true;
while ($continue) {
$this->loadModel('Whitelist');
$results = $this->Attribute->fetchAttributes($this->Auth->user(), $params, $continue);
$params['page'] += 1;
$results = $this->Whitelist->removeWhitelistedFromArray($results, true);
$results = array_values($results);
$i = 0;
$temp = '';
foreach ($results as $attribute) {
$temp .= $exportTool->handler($attribute, $exportToolParams);
if ($temp !== '') {
if ($i != count($results) -1) {
$temp .= $exportTool->separator($exportToolParams);
}
}
$i++;
}
if (!$loop) {
$continue = false;
}
if ($continue) {
$temp .= $exportTool->separator($exportToolParams);
}
fwrite($tmpfile, $temp);
}
return true;
}
// returns an XML with attributes that belong to an event. The type of attributes to be returned can be restricted by type using the 3rd parameter.
// Similar to the restSearch, this parameter can be chained with '&&' and negations are accepted too. For example filename&&!filename|md5 would return all filenames that don't have an md5
// The usage of returnAttributes is the following: [MISP-url]/attributes/returnAttributes/<API-key>/<type>/<signature flag>
@ -3498,4 +3035,20 @@ class AttributesController extends AppController
}
return new CakeResponse(array('body'=>$counter, 'status'=>200));
}
public function exportSearch($type = false)
{
if (empty($type)) {
$exports = array_keys($this->validFormats);
$this->set('exports', $exports);
$this->render('ajax/exportSearch');
} else {
$filters = $this->Session->read('search_attributes_filters');
$filters = json_decode($filters, true);
$final = $this->Attribute->restSearch($this->Auth->user(), $type, $filters);
$responseType = $this->validFormats[$type][0];
return $this->RestResponse->viewData($final, $responseType, false, true, 'search.' . $type . '.' . $responseType);
}
}
}

124
app/Controller/EventsController.php
View File

@ -1147,6 +1147,7 @@ class EventsController extends AppController
}
$this->set('sightingTypes', $this->Sighting->type);
$this->set('currentUri', $this->params->here);
$this->layout = false;
$this->render('/Elements/eventattribute');
}
@ -2682,7 +2683,7 @@ class EventsController extends AppController
return new CakeResponse(array('body'=> implode(PHP_EOL, $rules), 'status' => 200, 'type' => 'txt'));
}
// csv function
// csv function ***DEPRECATED***
// Usage: csv($key, $eventid) - key can be a valid auth key or the string 'download'. Download requires the user to be logged in interactively and will generate a .csv file
// $eventid can be one of 3 options: left empty it will get all the visible to_ids attributes,
// $ignore is a flag that allows the export tool to ignore the ids flag. 0 = only IDS signatures, 1 = everything.
@ -2696,8 +2697,8 @@ class EventsController extends AppController
'ordered_url_params' => compact($paramArray)
);
$exception = false;
$params = $this->_harvestParameters($filterData, $exception);
if ($params === false) {
$filters = $this->_harvestParameters($filterData, $exception);
if ($filters === false) {
return $exception;
}
$list = array();
@ -2705,6 +2706,7 @@ class EventsController extends AppController
if ($user === false) {
return $exception;
}
$final = $this->Event->restSearch($user, 'csv', $filters);
// if it's a search, grab the attributeIDList from the session and get the IDs from it. Use those as the condition
// We don't need to look out for permissions since that's filtered by the search itself
// We just want all the attributes found by the search
@ -2733,75 +2735,8 @@ class EventsController extends AppController
$list[] = $attribute['Attribute']['id'];
}
}
$final = array();
$requested_attributes = array('uuid', 'event_id', 'category', 'type',
'value', 'comment', 'to_ids', 'timestamp', 'object_relation');
$requested_obj_attributes = array('uuid', 'name', 'meta-category');
if ($includeContext) {
$requested_attributes[] = 'attribute_tag';
}
if (isset($this->params['url']['attributes'])) {
if (!isset($this->params['url']['obj_attributes'])) {
$requested_obj_attributes = array();
}
$requested_attributes = explode(',', $this->params['url']['attributes']);
}
if (isset($this->params['url']['obj_attributes'])) {
$requested_obj_attributes = explode(',', $this->params['url']['obj_attributes']);
}
if (isset($data['request']['attributes'])) {
if (!isset($data['request']['obj_attributes'])) {
$requested_obj_attributes = array();
}
$requested_attributes = $data['request']['attributes'];
}
if (isset($data['request']['obj_attributes'])) {
$requested_obj_attributes = $data['request']['obj_attributes'];
}
$possibleParams = array(
'ignore', 'list', 'category', 'type', 'includeContext',
'enforceWarninglist', 'value', 'timestamp', 'tags',
'last', 'from', 'to'
);
if (isset($params['eventid']) && $params['eventid'] == 'all') {
unset($params['eventid']);
}
foreach ($possibleParams as $possibleParam) {
if (isset($params[$possibleParam])) {
$params[$possibleParam] = $params[$possibleParam];
}
}
$params['limit'] = 1000;
$params['page'] = 1;
$i = 0;
$continue = true;
$params = array_merge($params, array(
'requested_obj_attributes' => $requested_obj_attributes,
'requested_attributes' => $requested_attributes,
'includeContext' => $includeContext
));
App::uses('CsvExport', 'Export');
$export = new CsvExport();
$final = $export->header($params);
while ($continue) {
$attributes = $this->Event->csv($user, $params, false, $continue);
$params['page'] += 1;
$final .= $export->handler($attributes, $params);
$final .= $export->separator($attributes);
}
$export->footer();
$this->response->type('csv'); // set the content type
if (empty($params['eventid'])) {
$filename = "misp.filtered_attributes.csv";
} elseif ($params['eventid'] === 'search') {
$filename = "misp.search_result.csv";
} else {
if (is_array($params['eventid'])) {
$params['eventid'] = 'list';
}
$filename = "misp.event_" . $params['eventid'] . ".csv";
}
return $this->RestResponse->viewData($final, 'csv', false, true, $filename);
$responseType = 'csv';
return $this->RestResponse->viewData($final, $responseType, false, true, 'download.csv');
}
public function _addIOCFile($id)
@ -3006,43 +2941,6 @@ class EventsController extends AppController
return $this->response;
}
/*
* Receive a list of eventids in the id=>count format
* Chunk them by the attribute count to fit the memory limits
*
*/
private function __clusterEventIds($exportTool, $eventIds) {
$memory_in_mb = $this->Event->Attribute->convert_to_memory_limit_to_mb(ini_get('memory_limit'));
$memory_scaling_factor = isset($exportTool->memory_scaling_factor) ? $exportTool->memory_scaling_factor : 100;
$limit = $memory_in_mb * $memory_scaling_factor;
$eventIdList = array();
$continue = true;
$i = 0;
$current_chunk_size = 0;
while (!empty($eventIds)) {
foreach ($eventIds as $id => $count) {
if ($current_chunk_size == 0 && $count > $limit) {
$eventIdList[$i][] = $id;
$current_chunk_size = $count;
unset($eventIds[$id]);
$i++;
break;
} else {
if (($current_chunk_size + $count) > $limit) {
$i++;
$current_chunk_size = 0;
break;
} else {
$current_chunk_size += $count;
$eventIdList[$i][] = $id;
unset($eventIds[$id]);
}
}
}
}
return $eventIdList;
}
// Use the REST interface to search for attributes or events. Usage:
// MISP-base-url/events/restSearch/[api-key]/[value]/[type]/[category]/[orgc]
// value, type, category, orgc are optional
@ -3059,6 +2957,7 @@ class EventsController extends AppController
'paramArray' => $paramArray,
'ordered_url_params' => compact($paramArray)
);
<<<<<<< HEAD
$validFormats = array(
'openioc' => array('xml', 'OpeniocExport'),
'json' => array('json', 'JsonExport'),
@ -3070,6 +2969,8 @@ class EventsController extends AppController
'stix2' => array('json', 'Stix2Export'),
'text' => array('text', 'TextExport')
);
=======
>>>>>>> 92eb8a91ad0c507f97954350535ba87e16d73e23
$exception = false;
$filters = $this->_harvestParameters($filterData, $exception);
unset($filterData);
@ -3087,6 +2988,7 @@ class EventsController extends AppController
if ($returnFormat === 'download') {
$returnFormat = 'json';
}
<<<<<<< HEAD
if (!isset($validFormats[$returnFormat][1])) {
throw new NotFoundException('Invalid output format.');
}
@ -3174,6 +3076,10 @@ class EventsController extends AppController
$final = fread($tmpfile, fstat($tmpfile)['size']);
fclose($tmpfile);
$responseType = $validFormats[$returnFormat][0];
=======
$final = $this->Event->restSearch($user, $returnFormat, $filters);
$responseType = $this->Event->validFormats[$returnFormat][0];
>>>>>>> 92eb8a91ad0c507f97954350535ba87e16d73e23
return $this->RestResponse->viewData($final, $responseType, false, true);
}

1
app/Controller/ServersController.php
View File

@ -1651,6 +1651,7 @@ class ServersController extends AppController
if (!empty($request['skip_ssl_validation'])) {
$params['ssl_verify_peer'] = false;
}
$params['timeout'] = 300;
App::uses('HttpSocket', 'Network/Http');
$HttpSocket = new HttpSocket($params);
$view_data = array();

230
app/Lib/Export/CsvExport.php
View File

@ -2,66 +2,194 @@
class CsvExport
{
public $event_context_fields = array('event_info', 'event_member_org', 'event_source_org', 'event_distribution', 'event_threat_level_id', 'event_analysis', 'event_date', 'event_tag');
public $default_fields = array('uuid', 'event_id', 'category', 'type', 'value', 'comment', 'to_ids', 'timestamp', 'object_relation', 'attribute_tag');
public $default_obj_fields = array('object_uuid', 'object_name', 'object_meta-category');
public $requested_fields = array();
public $csv_event_context_fields_to_fetch = array(
'event_info' => array('object' => false, 'var' => 'info'),
'event_member_org' => array('object' => 'Org', 'var' => 'name'),
'event_source_org' => array('object' => 'Orgc', 'var' => 'name'),
'event_distribution' => array('object' => false, 'var' => 'distribution'),
'event_threat_level_id' => array('object' => 'ThreatLevel', 'var' => 'name'),
'event_analysis' => array('object' => false, 'var' => 'analysis'),
'event_date' => array('object' => false, 'var' => 'date'),
'event_tag' => array('object' => 'Tag', 'var' => 'name')
);
public function handler($attributes, $options = array())
public function handler($data, $options = array())
{
$result = array();
foreach ($attributes as $attribute) {
$line1 = '';
$line2 = '';
foreach ($options['requested_attributes'] as $requested_attribute) {
$line1 .= $attribute['Attribute'][$requested_attribute] . ',';
}
$line1 = rtrim($line1, ",");
foreach ($options['requested_obj_attributes'] as $requested_obj_attribute) {
$line2 .= $attribute['Object'][$requested_obj_attribute] . ',';
}
$line2 = rtrim($line2, ",");
$line = $line1 . ',' . $line2;
$line = rtrim($line, ",");
if (!empty($options['includeContext'])) {
foreach ($this->Event->csv_event_context_fields_to_fetch as $header => $field) {
if ($field['object']) {
$line .= ',' . $attribute['Event'][$field['object']][$field['var']];
} else {
$line .= ',' . str_replace(array("\n","\t","\r"), " ", $attribute['Event'][$field['var']]);
}
}
}
$result[] = $line;
}
$result = implode(PHP_EOL, $result);
return $result;
if ($options['scope'] === 'Attribute') {
$lines = $this->__attributesHandler($data, $options);
} else if($options['scope'] === 'Event') {
$lines = $this->__eventsHandler($data, $options);
}
return $lines;
}
public function header($options = array())
public function modify_params($user, $params)
{
if (empty($params['contain'])) {
$params['contain'] = array();
}
$params['contain'] = array_merge($params['contain'], array(
'Object' => array('fields' => array('Object.uuid', 'Object.name', 'Object.meta-category')),
'AttributeTag' => array('Tag'),
'Event' => array('fields' => array('Event.*'), 'EventTag' => 'Tag', 'Org.name', 'Orgc.name', 'ThreatLevel')
));
unset($params['fields']);
$params['includeEventUuid'] = 0;
$params['includeEventTags'] = 0;
$params['withAttachments'] = 0;
return $params;
}
private function __attributesHandler($attribute, $options)
{
$attribute = $this->__addMetadataToAttributeAtomic($attribute);
if (!empty($attribute['Object']['uuid'])) {
$attribute['object_uuid'] = $attribute['Object']['uuid'];
$attribute['object_name'] = $attribute['Object']['name'];
$attribute['object_meta-category'] = $attribute['Object']['meta-category'];
}
return $this->__addLine($attribute, $options);
}
private function __eventsHandler($event, $options)
{
$lines = '';
if (!empty($event['Attribute'])) {
foreach ($event['Attribute'] as $k => $attribute) {
$attribute = $this->__addMetadataToAttribute($event, $attribute);
$lines .= $this->__addLine($attribute, $options);
}
}
if (!empty($event['Object'])) {
foreach ($event['Object'] as $k => $object) {
if (!empty($object['Attribute'])) {
foreach ($object['Attribute'] as $attribute) {
$attribute = $this->__addMetadataToAttribute($event, $attribute);
$attribute['object_uuid'] = $object['uuid'];
$attribute['object_name'] = $object['name'];
$attribute['object_meta-category'] = $object['meta-category'];
$lines .= $this->__addLine($attribute, $options);
}
}
}
}
return $lines;
}
private function __addLine($attribute, $options = array()) {
$line = '';
foreach ($this->requested_fields as $req_att) {
if (empty($line)) {
$line = $this->__escapeCSVField($attribute[$req_att]);
} else {
$line .= ',' . $this->__escapeCSVField($attribute[$req_att]);
}
}
return $line . PHP_EOL;
}
private function __escapeCSVField(&$field)
{
if (is_bool($field)) {
return ($field ? 'true' : 'false');
}
if (is_numeric($field)) {
return $field;
}
$field = str_replace(array('"'), '""', $field);
$field = '"' . $field . '"';
return $field;
}
private function __addMetadataToAttributeAtomic($attribute_raw) {
$attribute = $attribute_raw['Attribute'];
if (!empty($attribute_raw['AttributeTag'])) {
$tags = array();
foreach ($attribute_raw['AttributeTag'] as $at) {
$tags[] = $at['Tag']['name'];
}
$tags = implode(',', $tags);
$attribute['attribute_tag'] = $tags;
}
$attribute['event_info'] = $attribute_raw['Event']['info'];
$attribute['event_member_org'] = $attribute_raw['Event']['Org']['name'];
$attribute['event_source_org'] = $attribute_raw['Event']['Orgc']['name'];
$attribute['event_distribution'] = $attribute_raw['Event']['distribution'];
$attribute['event_threat_level_id'] = $attribute_raw['Event']['ThreatLevel']['name'];
$attribute['event_analysis'] = $attribute_raw['Event']['analysis'];
$attribute['event_date'] = $attribute_raw['Event']['date'];
if (!empty($attribute_raw['EventTag'])) {
$tags = array();
foreach ($attribute_raw['EventTag'] as $et) {
$tags[] = $et['Tag']['name'];
}
$tags = implode(',', $tags);
$attribute['event_tag'] = $tags;
}
return $attribute;
}
private function __addMetadataToAttribute($event, $attribute) {
if (!empty($attribute['AttributeTag'])) {
$tags = array();
foreach ($attribute['AttributeTag'] as $at) {
$tags[] = $at['Tag']['name'];
}
$tags = implode(',', $tags);
$attribute['attribute_tag'] = $tags;
}
$attribute['event_info'] = $event['Event']['info'];
$attribute['event_member_org'] = $event['Org']['name'];
$attribute['event_source_org'] = $event['Orgc']['name'];
$attribute['event_distribution'] = $event['Event']['distribution'];
$attribute['event_threat_level_id'] = $event['ThreatLevel']['name'];
$attribute['event_analysis'] = $event['Event']['analysis'];
$attribute['event_date'] = $event['Event']['date'];
if (!empty($event['EventTag'])) {
$tags = array();
foreach ($event['EventTag'] as $et) {
$tags[] = $et['Tag']['name'];
}
$tags = implode(',', $tags);
$attribute['event_tag'] = $tags;
}
return $attribute;
}
public function header(&$options)
{
if (!empty($options['requested_obj_attributes'])) {
array_walk($options['requested_obj_attributes'], function (&$value, $key) {
$value = 'object-'.$value;
});
}
$headers = array_merge($options['requested_attributes'], $options['requested_obj_attributes']);
if (!empty($options['includeContext'])) {
$headers = array_merge($headers, array_keys($this->csv_event_context_fields_to_fetch));
}
foreach ($headers as $k => $v) {
if (isset($options['filters']['requested_attributes'])) {
$this->requested_fields = $options['filters']['requested_attributes'];
} else {
$this->requested_fields = $this->default_fields;
}
if (isset($options['filters']['requested_obj_attributes'])) {
$requested_obj_attributes = array();
foreach ($options['filters']['requested_obj_attributes'] as $roa) {
$requested_obj_attributes[] = 'object_' . $roa;
}
} else {
if (isset($options['filters']['requested_attributes'])) {
$requested_obj_attributes = array();
} else {
$requested_obj_attributes = $this->default_obj_fields;
}
}
foreach ($requested_obj_attributes as $obj_att) {
$this->requested_fields[] = $obj_att;
}
if (isset($options['filters']['includeContext'])) {
foreach ($this->event_context_fields as $event_context_field) {
$this->requested_fields[] = $event_context_field;
}
}
$object_level_search = false;
foreach ($this->requested_fields as $k => $v) {
if (in_array($v, $this->default_obj_fields)) {
$object_level_search = true;
}
$headers[$k] = str_replace('-', '_', $v);
if ($v == 'timestamp') {
$headers[$k] = 'date';
}
}
if (!$object_level_search) {
$options['flatten'] = 1;
}
$headers = implode(',', $headers) . PHP_EOL;
return $headers;
}
@ -73,7 +201,7 @@ class CsvExport
public function separator()
{
return PHP_EOL;
return '';
}
}

123
app/Model/Attribute.php
View File

@ -117,6 +117,17 @@ class Attribute extends AppModel
),
);
public $validFormats = array(
'openioc' => array('xml', 'OpeniocExport'),
'json' => array('json', 'JsonExport'),
'xml' => array('xml', 'XmlExport'),
'suricata' => array('txt', 'NidsSuricataExport'),
'snort' => array('txt', 'NidsSnortExport'),
'text' => array('txt', 'TextExport'),
'rpz' => array('rpz', 'RPZExport'),
'csv' => array('csv', 'CsvExport')
);
public $typeDefinitions = array(
'md5' => array('desc' => 'A checksum in md5 format', 'formdesc' => "You are encouraged to use filename|md5 instead. A checksum in md5 format, only use this if you don't know the correct filename", 'default_category' => 'Payload delivery', 'to_ids' => 1),
'sha1' => array('desc' => 'A checksum in sha1 format', 'formdesc' => "You are encouraged to use filename|sha1 instead. A checksum in sha1 format, only use this if you don't know the correct filename", 'default_category' => 'Payload delivery', 'to_ids' => 1),
@ -3166,7 +3177,7 @@ class Attribute extends AppModel
public function convertToOpenIOC($user, $attributes)
{
return $this->IOCExport->buildAll($this->Auth->user(), $event);
return $this->IOCExport->buildAll($user, $event);
}
private function __createTagSubQuery($tag_id, $blocked = false, $scope = 'Event', $limitAttributeHitsTo = 'event')
@ -3708,4 +3719,114 @@ class Attribute extends AppModel
}
return $conditions;
}
public function restSearch($user, $returnFormat, $filters, $paramsOnly = false)
{
if (!isset($this->validFormats[$returnFormat][1])) {
throw new NotFoundException('Invalid output format.');
}
App::uses($this->validFormats[$returnFormat][1], 'Export');
$exportTool = new $this->validFormats[$returnFormat][1]();
if (empty($exportTool->non_restrictive_export)) {
if (!isset($filters['to_ids'])) {
$filters['to_ids'] = 1;
}
if (!isset($filters['published'])) {
$filters['published'] = 1;
}
}
$conditions = $this->buildFilterConditions($user, $filters);
$params = array(
'conditions' => $conditions,
'fields' => array('Attribute.*', 'Event.org_id', 'Event.distribution'),
'withAttachments' => !empty($filters['withAttachments']) ? $filters['withAttachments'] : 0,
'enforceWarninglist' => !empty($filters['enforceWarninglist']) ? $filters['enforceWarninglist'] : 0,
'includeAllTags' => true,
'flatten' => 1,
'includeEventUuid' => !empty($filters['includeEventUuid']) ? $filters['includeEventUuid'] : 0,
'includeEventTags' => !empty($filters['includeEventTags']) ? $filters['includeEventTags'] : 0
);
if (isset($filters['include_event_uuid'])) {
$params['includeEventUuid'] = $filters['include_event_uuid'];
}
if (isset($filters['limit'])) {
$params['limit'] = $filters['limit'];
}
if (isset($filters['page'])) {
$params['page'] = $filters['page'];
}
if (!empty($filtes['deleted'])) {
$params['deleted'] = 1;
if ($params['deleted'] === 'only') {
$params['conditions']['AND'][] = array('Attribute.deleted' => 1);
$params['conditions']['AND'][] = array('Object.deleted' => 1);
}
}
if ($paramsOnly) {
return $params;
}
if (!isset($this->validFormats[$returnFormat])) {
// this is where the new code path for the export modules will go
throw new MethodNotFoundException('Invalid export format.');
}
if (method_exists($exportTool, 'modify_params')) {
$params = $exportTool->modify_params($user, $params);
}
$exportToolParams = array(
'user' => $user,
'params' => $params,
'returnFormat' => $returnFormat,
'scope' => 'Attribute',
'filters' => $filters
);
if (!empty($exportTool->additional_params)) {
$params = array_merge($params, $exportTool->additional_params);
}
$tmpfile = tmpfile();
fwrite($tmpfile, $exportTool->header($exportToolParams));
$loop = false;
if (empty($params['limit'])) {
$memory_in_mb = $this->convert_to_memory_limit_to_mb(ini_get('memory_limit'));
$memory_scaling_factor = isset($exportTool->memory_scaling_factor) ? $exportTool->memory_scaling_factor : 100;
$params['limit'] = $memory_in_mb * $memory_scaling_factor;
$loop = true;
$params['page'] = 1;
}
$this->__iteratedFetch($user, $params, $loop, $tmpfile, $exportTool, $exportToolParams);
fwrite($tmpfile, $exportTool->footer($exportToolParams));
fseek($tmpfile, 0);
$final = fread($tmpfile, fstat($tmpfile)['size']);
fclose($tmpfile);
return $final;
}
private function __iteratedFetch($user, &$params, &$loop, &$tmpfile, $exportTool, $exportToolParams) {
$continue = true;
while ($continue) {
$this->Whitelist = ClassRegistry::init('Whitelist');
$results = $this->fetchAttributes($user, $params, $continue);
$params['page'] += 1;
$results = $this->Whitelist->removeWhitelistedFromArray($results, true);
$results = array_values($results);
$i = 0;
$temp = '';
foreach ($results as $attribute) {
$temp .= $exportTool->handler($attribute, $exportToolParams);
if ($temp !== '') {
if ($i != count($results) -1) {
$temp .= $exportTool->separator($exportToolParams);
}
}
$i++;
}
if (!$loop) {
$continue = false;
}
if ($continue) {
$temp .= $exportTool->separator($exportToolParams);
}
fwrite($tmpfile, $temp);
}
return true;
}
}

142
app/Model/Event.php
View File

@ -152,6 +152,17 @@ class Event extends AppModel
),
);
public $validFormats = array(
'openioc' => array('xml', 'OpeniocExport', 'ioc'),
'json' => array('json', 'JsonExport', 'json'),
'xml' => array('xml', 'XmlExport', 'xml'),
'suricata' => array('txt', 'NidsSuricataExport', 'rules'),
'snort' => array('txt', 'NidsSnortExport', 'rules'),
'rpz' => array('rpz', 'RPZExport', 'rpz'),
'text' => array('text', 'TextExport', 'txt'),
'csv' => array('csv', 'CsvExport', 'csv')
);
public $csv_event_context_fields_to_fetch = array(
'event_info' => array('object' => false, 'var' => 'info'),
'event_member_org' => array('object' => 'Org', 'var' => 'name'),
@ -5256,4 +5267,135 @@ class Event extends AppModel
}
}
}
public function restSearch($user, $returnFormat, $filters)
{
if (!isset($this->validFormats[$returnFormat][1])) {
throw new NotFoundException('Invalid output format.');
}
App::uses($this->validFormats[$returnFormat][1], 'Export');
$exportTool = new $this->validFormats[$returnFormat][1]();
if (empty($exportTool->non_restrictive_export)) {
if (!isset($filters['to_ids'])) {
$filters['to_ids'] = 1;
}
if (!isset($filters['published'])) {
$filters['published'] = 1;
}
}
if (isset($filters['ignore'])) {
$filters['to_ids'] = array(0, 1);
$filters['published'] = array(0, 1);
}
if (isset($filters['searchall'])) {
$filters['tags'] = $filters['searchall'];
$filters['eventinfo'] = $filters['searchall'];
$filters['value'] = $filters['searchall'];
$filters['comment'] = $filters['searchall'];
}
if (!empty($filters['quickfilter']) && !empty($filters['value'])) {
$filters['tags'] = $filters['value'];
$filters['eventinfo'] = $filters['value'];
$filters['comment'] = $filters['value'];
}
$filters['include_attribute_count'] = 1;
$eventid = $this->filterEventIds($user, $filters);
$eventids_chunked = $this->__clusterEventIds($exportTool, $eventid);
if (!empty($exportTool->additional_params)) {
$filters = array_merge($filters, $exportTool->additional_params);
}
$exportToolParams = array(
'user' => $user,
'params' => array(),
'returnFormat' => $returnFormat,
'scope' => 'Event',
'filters' => $filters
);
if (empty($exportTool->non_restrictive_export)) {
if (!isset($filters['to_ids'])) {
$filters['to_ids'] = 1;
}
if (!isset($filters['published'])) {
$filters['published'] = 1;
}
}
$tmpfile = tmpfile();
fwrite($tmpfile, $exportTool->header($exportToolParams));
$eventCount = count($eventid);
$i = 0;
if (!empty($filters['withAttachments'])) {
$filters['includeAttachments'] = 1;
}
$this->Whitelist = ClassRegistry::init('Whitelist');
foreach ($eventids_chunked as $chunk_index => $chunk) {
$filters['eventid'] = $chunk;
if (!empty($filters['tags']['NOT'])) {
$filters['blockedAttributeTags'] = $filters['tags']['NOT'];
}
$result = $this->fetchEvent(
$user,
$filters,
true
);
if (!empty($result)) {
foreach ($result as $event) {
$result = $this->Whitelist->removeWhitelistedFromArray($result, false);
$temp = $exportTool->handler($event, $exportToolParams);
if ($temp !== '') {
if ($i !== 0) {
$temp = $exportTool->separator($exportToolParams) . $temp;
}
fwrite($tmpfile, $temp);
$i++;
}
}
}
}
unset($result);
unset($temp);
fwrite($tmpfile, $exportTool->footer($exportToolParams));
fseek($tmpfile, 0);
$final = fread($tmpfile, fstat($tmpfile)['size']);
fclose($tmpfile);
return $final;
}
/*
* Receive a list of eventids in the id=>count format
* Chunk them by the attribute count to fit the memory limits
*
*/
private function __clusterEventIds($exportTool, $eventIds)
{
$memory_in_mb = $this->Attribute->convert_to_memory_limit_to_mb(ini_get('memory_limit'));
$memory_scaling_factor = isset($exportTool->memory_scaling_factor) ? $exportTool->memory_scaling_factor : 100;
$limit = $memory_in_mb * $memory_scaling_factor;
$eventIdList = array();
$continue = true;
$i = 0;
$current_chunk_size = 0;
while (!empty($eventIds)) {
foreach ($eventIds as $id => $count) {
if ($current_chunk_size == 0 && $count > $limit) {
$eventIdList[$i][] = $id;
$current_chunk_size = $count;
unset($eventIds[$id]);
$i++;
break;
} else {
if (($current_chunk_size + $count) > $limit) {
$i++;
$current_chunk_size = 0;
break;
} else {
$current_chunk_size += $count;
$eventIdList[$i][] = $id;
unset($eventIds[$id]);
}
}
}
}
return $eventIdList;
}
}

18
app/Model/Feed.php
View File

@ -865,14 +865,16 @@ class Feed extends AppModel
}
$temp = $this->getFreetextFeed($this->data, $HttpSocket, $this->data['Feed']['source_format'], 'all');
$data = array();
foreach ($temp as $key => $value) {
$data[] = array(
'category' => $value['category'],
'type' => $value['default_type'],
'value' => $value['value'],
'to_ids' => $value['to_ids']
);
}
if (!empty($temp)) {
foreach ($temp as $key => $value) {
$data[] = array(
'category' => $value['category'],
'type' => $value['default_type'],
'value' => $value['value'],
'to_ids' => $value['to_ids']
);
}
}
if ($jobId) {
$job->saveField('progress', 50);
$job->saveField('message', 'Saving data.');

42
app/View/Attributes/ajax/exportSearch.ctp Normal file
View File

@ -0,0 +1,42 @@
<div class="popover_choice">
<legend><?php echo __('Choose the format that you wish to download the search results in'); ?></legend>
<div class="popover_choice_main" id ="popover_choice_main">
<table style="width:100%;">
<?php
foreach ($exports as $k => $export) {
$tr = 'style="border-bottom:1px solid black;" class="templateChoiceButton"';
$td = sprintf(
'class="" tabindex="0" title="%s" style="%s" data-type="%s"',
__('Export as %s', h($export)),
'padding-left:10px; text-align:center;width:100%;',
h($export)
);
$div = '<div style="height:100%;width:100%;">' . h($export) . '</div>';
$a = sprintf(
'<a href="%s" style="%s" download>%s</a>',
$baseurl . '/attributes/exportSearch/' . h($export),
'color: black; text-decoration: none;',
$div
);
$td = sprintf(
'<td class="export_choice_button" tabindex="0" title="%s", style="%s">%s</td>',
__('Export as %s', h($export)),
'padding-left:10px; text-align:center;width:100%;',
$a
);
echo sprintf('<tr %s>%s</tr>', $tr, $td);
}
?>
</table>
</div>
<div role="button" tabindex="0" aria-label="<?php echo __('Cancel');?>" title="<?php echo __('Cancel');?>" class="templateChoiceButton templateChoiceButtonLast" onClick="cancelPopoverForm();"><?php echo __('Cancel');?></div>
</div>
<script type="text/javascript">
$(document).ready(function() {
resizePopoverBody();
});
$(window).resize(function() {
resizePopoverBody();
});
</script>

44
app/View/Attributes/index.ctp
View File

@ -4,15 +4,22 @@
if ($isSearch == 1) {
// The following block should serve as an example and food
// for thought on how to optimize i18n & l10n (especially for languages that are not SOV)
echo "<h4>" . __("Results for all attributes");
if ($keywordSearch != null) echo __(" with the value containing "). "\"<b>" . h($keywordSearch) . "</b>\"";
if ($attributeTags != null) echo __(" being tagged with ") ."\"<b>" . h($attributeTags) . "</b>\"";
if ($keywordSearch2 != null) echo __(" from the events ") . "\"<b>" . h($keywordSearch2) . "</b>\"";
if ($tags != null) echo " from events tagged \"<b>" . h($tags) . "</b>\"";
if ($categorySearch != "ALL") echo __(" of category ") . "\"<b>" . h($categorySearch) . "</b>\"";
if ($typeSearch != "ALL") echo __(" of type ") . "\"<b>" . h($typeSearch) . "</b>\"";
if (isset($orgSearch) && $orgSearch != '' && $orgSearch != null) echo __(" created by the organisation ") . "\"<b>" . h($orgSearch) . "</b>\"";
echo ":</h4>";
$filterOptions = array(
'value' => __(" with the value containing "),
'tags' => __(" being tagged with "),
'id' => __(" from the events "),
'tag' => __(" carrying the tag(s) "),
'type' => __(" of type "),
'category' => __(" of category "),
'org' => __(" created by organisation ")
);
$temp = '';
foreach ($filterOptions as $fo => $text) {
if (!empty($filters[$fo])) {
$temp .= sprintf('%s <b>%s</b>', $text, h($filters[$fo]));
}
}
echo sprintf("<h4>%s%s</h4>", __("Results for all attributes"), $temp);
}
?>
<div class="pagination">
@ -32,10 +39,10 @@
</ul>
</div>
<table class="table table-striped table-hover table-condensed">
<tr>
<tr>
<th><?php echo $this->Paginator->sort('event_id');?></th>
<?php if (Configure::read('MISP.showorg') || $isAdmin): ?>
<th><?php echo $this->Paginator->sort('org_id', 'Org');?></th>
<th><?php echo $this->Paginator->sort('Event.orgc_id', 'Org');?></th>
<?php endif; ?>
<th><?php echo $this->Paginator->sort('category');?></th>
<th><?php echo $this->Paginator->sort('type');?></th>
@ -45,19 +52,20 @@
<th<?php echo ' title="' . $attrDescriptions['signature']['desc'] . '"';?>>
<?php echo $this->Paginator->sort('IDS');?></th>
<th class="actions">Actions</th>
</tr>
</tr>
<?php
$currentCount = 0;
if ($isSearch == 1) {
// sanitize data
if (isset($keywordArray)) {
foreach ($keywordArray as &$keywordArrayElement) {
$keywordArrayElement = h($keywordArrayElement);
}
$toHighlight = array('value', 'comment');
$keywordArray = array();
foreach ($toHighlight as $highlightedElement) {
if (!empty($filters[$highlightedElement])) {
$keywordArray[] = $filters[$highlightedElement];
}
}
// build the $replacePairs variable used to highlight the keywords
$replacePairs = $this->Highlight->build_replace_pairs($keywordArray);
}
}
foreach ($attributes as $attribute):

47
app/View/Attributes/search.ctp
View File

@ -1,5 +1,5 @@
<div class="attributes form">
<?php echo $this->Form->create('Attribute');?>
<?php echo $this->Form->create('Attribute', array('url' => array('controller' => 'attributes', 'action' => 'search', 'results')));?>
<fieldset>
<legend><?php echo __('Search Attribute'); ?></legend>
<?php echo __('You can search for attributes based on contained expression within the value, event ID, submitting organisation, category and type. <br />For the value, event ID and organisation, you can enter several search terms by entering each term as a new line. To exclude things from a result, use the NOT operator (!) in front of the term.'); ?>
@ -7,43 +7,36 @@
<?php echo __('For string searches (such as searching for an expression, tags, etc) - lookups are simple string matches. If you want a substring match encapsulate the lookup string between "%" characters.'); ?>
<br /><br />
<?php
echo $this->Form->input('keyword', array('type' => 'textarea', 'rows' => 2, 'label' => __('Containing the following expressions'), 'div' => 'clear', 'class' => 'input-xxlarge'));
echo $this->Form->input('attributetags', array('type' => 'textarea', 'rows' => 2, 'label' => __('Being an attribute matching the following tags'), 'div' => 'clear', 'class' => 'input-xxlarge'));
echo $this->Form->input('keyword2', array('type' => 'textarea', 'rows' => 2, 'label' => __('Being attributes of the following event IDs, event UUIDs or attribute UUIDs'), 'div' => 'clear', 'class' => 'input-xxlarge'));
echo $this->Form->input('tags', array('type' => 'textarea', 'rows' => 2, 'label' => __('Being an attribute of an event matching the following tags'), 'div' => 'clear', 'class' => 'input-xxlarge'));
?>
<?php
if (Configure::read('MISP.showorg') || $isAdmin)
echo $this->Form->input('org', array(
'type' => 'textarea',
'label' => __('From the following organisation(s)'),
'div' => 'input clear',
'rows' => 2,
'class' => 'input-xxlarge'));
?>
<?php
echo $this->Form->input('type', array(
echo $this->Form->input('value', array('type' => 'textarea', 'rows' => 2, 'label' => __('Containing the following expressions'), 'div' => 'clear', 'class' => 'input-xxlarge', 'required' => false));
echo $this->Form->input('tags', array('type' => 'textarea', 'rows' => 2, 'label' => __('Having tag or being an attribute of an event having the tag'), 'div' => 'clear', 'class' => 'input-xxlarge', 'required' => false));
echo $this->Form->input('uuid', array('type' => 'textarea', 'rows' => 2, 'maxlength' => false, 'label' => __('Being attributes of the following event IDs, event UUIDs or attribute UUIDs'), 'div' => 'clear', 'class' => 'input-xxlarge', 'required' => false));
echo $this->Form->input('org', array(
'type' => 'textarea',
'label' => __('From the following organisation(s)'),
'div' => 'input clear',
));
echo $this->Form->input('category', array(
));
'rows' => 2,
'class' => 'input-xxlarge'));
echo $this->Form->input('type', array(
'div' => 'input clear',
'required' => false
));
echo $this->Form->input('category', array('required' => false));
?>
<div class="input clear"></div>
<?php
echo $this->Form->input('ioc', array(
echo $this->Form->input('to_ids', array(
'type' => 'checkbox',
'label' => __('Only find IOCs to use in IDS'),
'label' => __('Only find IOCs flagged as to_ids')
));
echo $this->Form->input('alternate', array(
'type' => 'checkbox',
'label' => __('Alternate Search Result (Events)'),
'label' => __('Alternate Search Result (Events)')
));
?>
</fieldset>
<?php
echo $this->Form->button('Search', array('class' => 'btn btn-primary'));
echo $this->Form->end();
echo $this->Form->button('Search', array('class' => 'btn btn-primary'));
echo $this->Form->end();
?>
</div>
<script type="text/javascript">
@ -208,5 +201,5 @@ $('.input-xxlarge').keydown(function (e) {
</script>
<?php
echo $this->element('side_menu', array('menuList' => 'event-collection', 'menuItem' => 'searchAttributes'));
echo $this->Js->writeBuffer();
?>
<?php echo $this->Js->writeBuffer(); // Write cached scripts ?>

19
app/View/Elements/side_menu.ctp
View File

@ -219,18 +219,13 @@
));
if ($menuItem == 'searchAttributes2') {
echo $this->element('/side_menu_divider');
echo $this->element('/side_menu_link', array(
'url' => '/events/downloadSearchResult.json',
'text' => __('Download results as JSON')
));
echo $this->element('/side_menu_link', array(
'url' => '/events/downloadSearchResult.xml',
'text' => __('Download results as XML')
));
echo $this->element('/side_menu_link', array(
'url' => '/events/csv/download/search',
'text' => __('Download results as CSV')
));
echo $this->element('/side_menu_link', array(
'onClick' => array(
'function' => 'getPopup',
'params' => array(0, 'attributes', 'exportSearch')
),
'text' => __('Download as...')
));
}
echo $this->element('/side_menu_divider');
echo $this->element('/side_menu_link', array(

2
app/files/misp-galaxy

@ -1 +1 @@
Subproject commit 2402c7d98f0ab23f065ae00d3d34ab6610e9a3e9
Subproject commit ecba2dbdbf233d330c2bba7fecd569d31a3ca576