MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [rest] Faster attributes restSearch

pull/6529/head
Jakub Onderka 2020-10-30 16:46:03 +01:00
parent 56ffa335d6
commit 9cf7be802d
2 changed files with 44 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
app/Lib/Tools/TmpFileTool.php
View File

@ -1,11 +1,12 @@
<?php
class TmpFileTool
{
/**
* @var resource
*/
/** @var resource */
private $tmpfile;
/** @var string */
private $separator;
/**
* @param int $maxInMemory How many bytes should keep in memory before creating file on disk. By default is is 2 MB.
* @throws Exception
@ -21,6 +22,22 @@ class TmpFileTool
}
}
/**
* Write data to stream with separator. Separator will be prepend to content for next call.
* @param string $content
* @param string $separator
* @throws Exception
*/
public function writeWithSeparator($content, $separator)
{
if (isset($this->separator)) {
$this->write($this->separator . $content);
} else {
$this->write($content);
}
$this->separator = $separator;
}
/**
* @param string $content
* @throws Exception

46
app/Model/Attribute.php
View File

@ -3483,7 +3483,6 @@ class Attribute extends AppModel
$params['page'] = 0;
} else {
$loop = false;
$pagesToFetch = 1;
}
$attributes = array();
if (!empty($options['includeEventTags'])) {
@ -4648,46 +4647,49 @@ class Attribute extends AppModel
$params['page'] = 1;
}
if (empty($exportTool->mock_query_only)) {
$this->__iteratedFetch($user, $params, $loop, $tmpfile, $exportTool, $exportToolParams, $elementCounter);
$elementCounter = $this->__iteratedFetch($user, $params, $loop, $tmpfile, $exportTool, $exportToolParams);
}
$tmpfile->write($exportTool->footer($exportToolParams));
return $tmpfile->finish();
}
private function __iteratedFetch($user, &$params, &$loop, TmpFileTool $tmpfile, $exportTool, $exportToolParams, &$elementCounter = 0)
/**
* @param array $user
* @param array $params
* @param bool $loop If true, data are fetched in loop to keep memory usage low
* @param TmpFileTool $tmpfile
* @param $exportTool
* @param array $exportToolParams
* @return int
* @throws Exception
*/
private function __iteratedFetch(array $user, array $params, $loop, TmpFileTool $tmpfile, $exportTool, array $exportToolParams)
{
$this->Allowedlist = ClassRegistry::init('Allowedlist');
$separator = $exportTool->separator($exportToolParams);
$elementCounter = 0;
$continue = true;
while ($continue) {
do {
$results = $this->fetchAttributes($user, $params, $continue);
if (empty($results)) {
break; // nothing found, skip rest
}
if ($params['includeSightingdb']) {
$this->Sightingdb = ClassRegistry::init('Sightingdb');
$results = $this->Sightingdb->attachToAttributes($results, $user);
}
$params['page'] += 1;
$results = $this->Allowedlist->removeAllowedlistedFromArray($results, true);
$i = 0;
$temp = '';
$elementCounter += count($results);
foreach ($results as $attribute) {
$elementCounter++;
$handlerResult = $exportTool->handler($attribute, $exportToolParams);
$temp .= $handlerResult;
if ($handlerResult !== '') {
if ($i != count($results) -1) {
$temp .= $exportTool->separator($exportToolParams);
}
$tmpfile->writeWithSeparator($handlerResult, $separator);
}
$i++;
}
if (!$loop) {
$continue = false;
}
if ($continue) {
$temp .= $exportTool->separator($exportToolParams);
}
$tmpfile->write($temp);
}
return true;
$params['page'] += 1;
} while ($loop && $continue);
return $elementCounter;
}
public function set_filter_uuid(&$params, $conditions, $options)