MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [internal] Attribute REST search optimisations and error handling

pull/6035/head
Jakub Onderka 2020-06-19 16:21:53 +02:00
parent 4693929a5d
commit 3d12cb3f3c
5 changed files with 94 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
app/Lib/Tools/TmpFileTool.php Normal file
View File

@ -0,0 +1,64 @@
<?php
class TmpFileTool
{
/**
* @var resource
*/
private $tmpfile;
/**
* @param int $maxInMemory How many bytes should keep in memory before creating file on disk. By default is is 2 MB.
* @throws Exception
*/
public function __construct($maxInMemory = null)
{
if ($maxInMemory === null) {
$maxInMemory = 2 * 1024 * 1024;
}
$this->tmpfile = fopen("php://temp/maxmemory:$maxInMemory", "w+");
if ($this->tmpfile === false) {
throw new Exception('Could not create temporary file.');
}
}
/**
* @param string $content
* @throws Exception
*/
public function write($content)
{
if (fwrite($this->tmpfile, $content) === false) {
if ($this->tmpfile === null) {
throw new Exception('Could not write to finished temporary file.');
}
$tmpFolder = sys_get_temp_dir();
$freeSpace = disk_free_space($tmpFolder);
throw new Exception("Could not write to temporary file in $tmpFolder folder. Maybe not enough space? ($freeSpace bytes left)");
}
}
/**
* @return string
* @throws Exception
*/
public function finish()
{
fseek($this->tmpfile, 0);
$final = stream_get_contents($this->tmpfile);
if ($final === false) {
throw new Exception("Could not read from temporary file.");
}
fclose($this->tmpfile);
$this->tmpfile = null;
return $final;
}
/**
* @return string
* @throws Exception
*/
public function __toString()
{
return $this->finish();
}
}

24
app/Model/Attribute.php
View File

@ -6,6 +6,7 @@ App::uses('File', 'Utility');
App::uses('FinancialTool', 'Tools');
App::uses('RandomTool', 'Tools');
App::uses('MalwareTool', 'Tools');
App::uses('TmpFileTool', 'Tools');
class Attribute extends AppModel
{
@ -4582,8 +4583,9 @@ class Attribute extends AppModel
$exportTool->additional_params
);
}
$tmpfile = tmpfile();
fwrite($tmpfile, $exportTool->header($exportToolParams));
$tmpfile = new TmpFileTool();
$tmpfile->write($exportTool->header($exportToolParams));
$loop = false;
if (empty($params['limit'])) {
$memory_in_mb = $this->convert_to_memory_limit_to_mb(ini_get('memory_limit'));
@ -4596,22 +4598,15 @@ class Attribute extends AppModel
if (empty($exportTool->mock_query_only)) {
$this->__iteratedFetch($user, $params, $loop, $tmpfile, $exportTool, $exportToolParams, $elementCounter);
}
fwrite($tmpfile, $exportTool->footer($exportToolParams));
fseek($tmpfile, 0);
if (fstat($tmpfile)['size']) {
$final = fread($tmpfile, fstat($tmpfile)['size']);
} else {
$final = '';
}
fclose($tmpfile);
return $final;
$tmpfile->write($exportTool->footer($exportToolParams));
return $tmpfile->finish();
}
private function __iteratedFetch($user, &$params, &$loop, &$tmpfile, $exportTool, $exportToolParams, &$elementCounter = 0)
private function __iteratedFetch($user, &$params, &$loop, TmpFileTool $tmpfile, $exportTool, $exportToolParams, &$elementCounter = 0)
{
$this->Whitelist = ClassRegistry::init('Whitelist');
$continue = true;
while ($continue) {
$this->Whitelist = ClassRegistry::init('Whitelist');
$results = $this->fetchAttributes($user, $params, $continue);
if ($params['includeSightingdb']) {
$this->Sightingdb = ClassRegistry::init('Sightingdb');
@ -4619,7 +4614,6 @@ class Attribute extends AppModel
}
$params['page'] += 1;
$results = $this->Whitelist->removeWhitelistedFromArray($results, true);
$results = array_values($results);
$i = 0;
$temp = '';
foreach ($results as $attribute) {
@ -4639,7 +4633,7 @@ class Attribute extends AppModel
if ($continue) {
$temp .= $exportTool->separator($exportToolParams);
}
fwrite($tmpfile, $temp);
$tmpfile->write($temp);
}
return true;
}

18
app/Model/Event.php
View File

@ -2,6 +2,7 @@
App::uses('AppModel', 'Model');
App::uses('CakeEmail', 'Network/Email');
App::uses('RandomTool', 'Tools');
App::uses('TmpFileTool', 'Tools');
class Event extends AppModel
{
@ -6736,8 +6737,8 @@ class Event extends AppModel
$filters['published'] = 1;
}
}
$tmpfile = tmpfile();
fwrite($tmpfile, $exportTool->header($exportToolParams));
$tmpfile = new TmpFileTool();
$tmpfile->write($exportTool->header($exportToolParams));
$i = 0;
if (!empty($filters['withAttachments'])) {
$filters['includeAttachments'] = 1;
@ -6765,7 +6766,7 @@ class Event extends AppModel
if ($i !== 0) {
$temp = $exportTool->separator($exportToolParams) . $temp;
}
fwrite($tmpfile, $temp);
$tmpfile->write($temp);
$i++;
}
}
@ -6773,15 +6774,8 @@ class Event extends AppModel
}
unset($result);
unset($temp);
fwrite($tmpfile, $exportTool->footer($exportToolParams));
fseek($tmpfile, 0);
if (fstat($tmpfile)['size'] > 0) {
$final = fread($tmpfile, fstat($tmpfile)['size']);
} else {
$final = 0;
}
fclose($tmpfile);
return $final;
$tmpfile->write($exportTool->footer($exportToolParams));
return $tmpfile->finish();
}
/*

21
app/Model/MispObject.php
View File

@ -1,6 +1,6 @@
<?php
App::uses('AppModel', 'Model');
App::uses('TmpFileTool', 'Tools');
class MispObject extends AppModel
{
@ -1424,8 +1424,8 @@ class MispObject extends AppModel
$exportTool->additional_params
);
}
$tmpfile = tmpfile();
fwrite($tmpfile, $exportTool->header($exportToolParams));
$tmpfile = new TmpFileTool();
$tmpfile->write($exportTool->header($exportToolParams));
$loop = false;
if (empty($params['limit'])) {
$memory_in_mb = $this->convert_to_memory_limit_to_mb(ini_get('memory_limit'));
@ -1436,18 +1436,11 @@ class MispObject extends AppModel
$params['page'] = 1;
}
$this->__iteratedFetch($user, $params, $loop, $tmpfile, $exportTool, $exportToolParams, $elementCounter);
fwrite($tmpfile, $exportTool->footer($exportToolParams));
fseek($tmpfile, 0);
if (fstat($tmpfile)['size']) {
$final = fread($tmpfile, fstat($tmpfile)['size']);
} else {
$final = '';
}
fclose($tmpfile);
return $final;
$tmpfile->write($exportTool->footer($exportToolParams));
return $tmpfile->finish();
}
private function __iteratedFetch($user, &$params, &$loop, &$tmpfile, $exportTool, $exportToolParams, &$elementCounter = 0)
private function __iteratedFetch($user, &$params, &$loop, TmpFileTool $tmpfile, $exportTool, $exportToolParams, &$elementCounter = 0)
{
$continue = true;
while ($continue) {
@ -1483,7 +1476,7 @@ class MispObject extends AppModel
if (!$loop) {
$continue = false;
}
fwrite($tmpfile, $temp);
$tmpfile->write($temp);
}
return true;
}

19
app/Model/Sighting.php
View File

@ -1,6 +1,7 @@
<?php
App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
App::uses('TmpFileTool', 'Tools');
class Sighting extends AppModel
{
@ -632,11 +633,11 @@ class Sighting extends AppModel
$allowedContext = array('event', 'attribute');
// validate context
if (isset($filters['context']) && !in_array($filters['context'], $allowedContext, true)) {
throw new MethodNotAllowedException(_('Invalid context.'));
throw new MethodNotAllowedException(__('Invalid context.'));
}
// ensure that an id is provided if context is set
if (!empty($filters['context']) && !isset($filters['id'])) {
throw new MethodNotAllowedException(_('An id must be provided if the context is set.'));
throw new MethodNotAllowedException(__('An id must be provided if the context is set.'));
}
if (!isset($this->validFormats[$returnFormat][1])) {
@ -747,8 +748,8 @@ class Sighting extends AppModel
'filters' => $filters
);
$tmpfile = tmpfile();
fwrite($tmpfile, $exportTool->header($exportToolParams));
$tmpfile = new TmpFileTool();
$tmpfile->write($exportTool->header($exportToolParams));
$temp = '';
$i = 0;
@ -761,13 +762,9 @@ class Sighting extends AppModel
}
$i++;
}
fwrite($tmpfile, $temp);
fwrite($tmpfile, $exportTool->footer($exportToolParams));
fseek($tmpfile, 0);
$final = fread($tmpfile, fstat($tmpfile)['size']);
fclose($tmpfile);
return $final;
$tmpfile->write($temp);
$tmpfile->write($exportTool->footer($exportToolParams));
return $tmpfile->finish();
}
// Bulk save sightings