MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #6035 from JakubOnderka/rest-search-optim 

chg: [internal] Attribute REST search optimisations and error handling
pull/6104/head
Andras Iklody 2020-07-07 12:18:29 +02:00 committed by GitHub
parent aa40f67c8e f46ca1634a
commit e6995dab67
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 136 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
app/Lib/Tools/TmpFileTool.php Normal file
View File

@ -0,0 +1,94 @@
<?php
class TmpFileTool
{
/**
* @var resource
*/
private $tmpfile;
/**
* @param int $maxInMemory How many bytes should keep in memory before creating file on disk. By default is is 2 MB.
* @throws Exception
*/
public function __construct($maxInMemory = null)
{
if ($maxInMemory === null) {
$maxInMemory = 2 * 1024 * 1024;
}
$this->tmpfile = fopen("php://temp/maxmemory:$maxInMemory", "w+");
if ($this->tmpfile === false) {
throw new Exception('Could not create temporary file.');
}
}
/**
* @param string $content
* @throws Exception
*/
public function write($content)
{
if (fwrite($this->tmpfile, $content) === false) {
if ($this->tmpfile === null) {
throw new Exception('Could not write to finished temporary file.');
}
$tmpFolder = sys_get_temp_dir();
$freeSpace = disk_free_space($tmpFolder);
throw new Exception("Could not write to temporary file in $tmpFolder folder. Maybe not enough space? ($freeSpace bytes left)");
}
}
/**
* @return Generator
* @throws Exception
*/
public function lines()
{
$this->rewind();
while (!feof($this->tmpfile)) {
$result = fgets($this->tmpfile);
if ($result === false) {
throw new Exception('Could not read line from temporary file.');
}
yield $result;
}
fclose($this->tmpfile);
$this->tmpfile = null;
}
/**
* @return string
* @throws Exception
*/
public function finish()
{
$this->rewind();
$final = stream_get_contents($this->tmpfile);
if ($final === false) {
throw new Exception('Could not read from temporary file.');
}
fclose($this->tmpfile);
$this->tmpfile = null;
return $final;
}
/**
* @return string
* @throws Exception
*/
public function __toString()
{
return $this->finish();
}
/**
* Seek to start of file.
*
* @throws Exception
*/
private function rewind()
{
if (fseek($this->tmpfile, 0) === -1) {
throw new Exception('Could not seek to start of temporary file.');
}
}
}

24
app/Model/Attribute.php
View File

@ -6,6 +6,7 @@ App::uses('File', 'Utility');
App::uses('FinancialTool', 'Tools');
App::uses('RandomTool', 'Tools');
App::uses('MalwareTool', 'Tools');
App::uses('TmpFileTool', 'Tools');
class Attribute extends AppModel
{
@ -4588,8 +4589,9 @@ class Attribute extends AppModel
$exportTool->additional_params
);
}
$tmpfile = tmpfile();
fwrite($tmpfile, $exportTool->header($exportToolParams));
$tmpfile = new TmpFileTool();
$tmpfile->write($exportTool->header($exportToolParams));
$loop = false;
if (empty($params['limit'])) {
$memory_in_mb = $this->convert_to_memory_limit_to_mb(ini_get('memory_limit'));
@ -4602,22 +4604,15 @@ class Attribute extends AppModel
if (empty($exportTool->mock_query_only)) {
$this->__iteratedFetch($user, $params, $loop, $tmpfile, $exportTool, $exportToolParams, $elementCounter);
}
fwrite($tmpfile, $exportTool->footer($exportToolParams));
fseek($tmpfile, 0);
if (fstat($tmpfile)['size']) {
$final = fread($tmpfile, fstat($tmpfile)['size']);
} else {
$final = '';
}
fclose($tmpfile);
return $final;
$tmpfile->write($exportTool->footer($exportToolParams));
return $tmpfile->finish();
}
private function __iteratedFetch($user, &$params, &$loop, &$tmpfile, $exportTool, $exportToolParams, &$elementCounter = 0)
private function __iteratedFetch($user, &$params, &$loop, TmpFileTool $tmpfile, $exportTool, $exportToolParams, &$elementCounter = 0)
{
$this->Whitelist = ClassRegistry::init('Whitelist');
$continue = true;
while ($continue) {
$this->Whitelist = ClassRegistry::init('Whitelist');
$results = $this->fetchAttributes($user, $params, $continue);
if ($params['includeSightingdb']) {
$this->Sightingdb = ClassRegistry::init('Sightingdb');
@ -4625,7 +4620,6 @@ class Attribute extends AppModel
}
$params['page'] += 1;
$results = $this->Whitelist->removeWhitelistedFromArray($results, true);
$results = array_values($results);
$i = 0;
$temp = '';
foreach ($results as $attribute) {
@ -4645,7 +4639,7 @@ class Attribute extends AppModel
if ($continue) {
$temp .= $exportTool->separator($exportToolParams);
}
fwrite($tmpfile, $temp);
$tmpfile->write($temp);
}
return true;
}

18
app/Model/Event.php
View File

@ -2,6 +2,7 @@
App::uses('AppModel', 'Model');
App::uses('CakeEmail', 'Network/Email');
App::uses('RandomTool', 'Tools');
App::uses('TmpFileTool', 'Tools');
class Event extends AppModel
{
@ -6788,8 +6789,8 @@ class Event extends AppModel
$filters['published'] = 1;
}
}
$tmpfile = tmpfile();
fwrite($tmpfile, $exportTool->header($exportToolParams));
$tmpfile = new TmpFileTool();
$tmpfile->write($exportTool->header($exportToolParams));
$i = 0;
if (!empty($filters['withAttachments'])) {
$filters['includeAttachments'] = 1;
@ -6817,7 +6818,7 @@ class Event extends AppModel
if ($i !== 0) {
$temp = $exportTool->separator($exportToolParams) . $temp;
}
fwrite($tmpfile, $temp);
$tmpfile->write($temp);
$i++;
}
}
@ -6825,15 +6826,8 @@ class Event extends AppModel
}
unset($result);
unset($temp);
fwrite($tmpfile, $exportTool->footer($exportToolParams));
fseek($tmpfile, 0);
if (fstat($tmpfile)['size'] > 0) {
$final = fread($tmpfile, fstat($tmpfile)['size']);
} else {
$final = 0;
}
fclose($tmpfile);
return $final;
$tmpfile->write($exportTool->footer($exportToolParams));
return $tmpfile->finish();
}
/*

20
app/Model/Feed.php
View File

@ -1,6 +1,7 @@
<?php
App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
App::uses('TmpFileTool', 'Tools');
class Feed extends AppModel
{
@ -161,10 +162,10 @@ class Feed extends AppModel
/**
* @param array $feed
* @param HttpSocket $HttpSocket
* @return array
* @return Generator|array
* @throws Exception
*/
public function getCache($feed, $HttpSocket)
public function getCache(array $feed, HttpSocket $HttpSocket)
{
$uri = $feed['Feed']['url'] . '/hashes.csv';
$data = $this->feedGetUri($feed, $uri, $HttpSocket);
@ -173,13 +174,16 @@ class Feed extends AppModel
throw new Exception("File '$uri' with hashes for cache filling is empty.");
}
$data = trim($data);
$data = explode("\n", $data);
$result = array();
foreach ($data as $v) {
$result[] = explode(',', $v);
// CSV file can be pretty big to do operations in memory, so we save content to temp and iterate line by line.
$tmpFile = new TmpFileTool();
$tmpFile->write(trim($data));
unset($data);
foreach ($tmpFile->lines() as $line) {
yield explode(',', rtrim($line));
}
return $result;
return array();
}
/**

21
app/Model/MispObject.php
View File

@ -1,6 +1,6 @@
<?php
App::uses('AppModel', 'Model');
App::uses('TmpFileTool', 'Tools');
class MispObject extends AppModel
{
@ -1426,8 +1426,8 @@ class MispObject extends AppModel
$exportTool->additional_params
);
}
$tmpfile = tmpfile();
fwrite($tmpfile, $exportTool->header($exportToolParams));
$tmpfile = new TmpFileTool();
$tmpfile->write($exportTool->header($exportToolParams));
$loop = false;
if (empty($params['limit'])) {
$memory_in_mb = $this->convert_to_memory_limit_to_mb(ini_get('memory_limit'));
@ -1438,18 +1438,11 @@ class MispObject extends AppModel
$params['page'] = 1;
}
$this->__iteratedFetch($user, $params, $loop, $tmpfile, $exportTool, $exportToolParams, $elementCounter);
fwrite($tmpfile, $exportTool->footer($exportToolParams));
fseek($tmpfile, 0);
if (fstat($tmpfile)['size']) {
$final = fread($tmpfile, fstat($tmpfile)['size']);
} else {
$final = '';
}
fclose($tmpfile);
return $final;
$tmpfile->write($exportTool->footer($exportToolParams));
return $tmpfile->finish();
}
private function __iteratedFetch($user, &$params, &$loop, &$tmpfile, $exportTool, $exportToolParams, &$elementCounter = 0)
private function __iteratedFetch($user, &$params, &$loop, TmpFileTool $tmpfile, $exportTool, $exportToolParams, &$elementCounter = 0)
{
$continue = true;
while ($continue) {
@ -1485,7 +1478,7 @@ class MispObject extends AppModel
if (!$loop) {
$continue = false;
}
fwrite($tmpfile, $temp);
$tmpfile->write($temp);
}
return true;
}

19
app/Model/Sighting.php
View File

@ -1,6 +1,7 @@
<?php
App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
App::uses('TmpFileTool', 'Tools');
class Sighting extends AppModel
{
@ -634,11 +635,11 @@ class Sighting extends AppModel
$allowedContext = array('event', 'attribute');
// validate context
if (isset($filters['context']) && !in_array($filters['context'], $allowedContext, true)) {
throw new MethodNotAllowedException(_('Invalid context.'));
throw new MethodNotAllowedException(__('Invalid context.'));
}
// ensure that an id is provided if context is set
if (!empty($filters['context']) && !isset($filters['id'])) {
throw new MethodNotAllowedException(_('An id must be provided if the context is set.'));
throw new MethodNotAllowedException(__('An id must be provided if the context is set.'));
}
if (!isset($this->validFormats[$returnFormat][1])) {
@ -749,8 +750,8 @@ class Sighting extends AppModel
'filters' => $filters
);
$tmpfile = tmpfile();
fwrite($tmpfile, $exportTool->header($exportToolParams));
$tmpfile = new TmpFileTool();
$tmpfile->write($exportTool->header($exportToolParams));
$temp = '';
$i = 0;
@ -763,13 +764,9 @@ class Sighting extends AppModel
}
$i++;
}
fwrite($tmpfile, $temp);
fwrite($tmpfile, $exportTool->footer($exportToolParams));
fseek($tmpfile, 0);
$final = fread($tmpfile, fstat($tmpfile)['size']);
fclose($tmpfile);
return $final;
$tmpfile->write($temp);
$tmpfile->write($exportTool->footer($exportToolParams));
return $tmpfile->finish();
}
/**