diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index faaf68684..45e9c0c00 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -206,6 +206,7 @@ jobs: run: | ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/ ./app/Vendor/bin/phpunit app/Test/ComplexTypeToolTest.php + ./app/Vendor/bin/phpunit app/Test/JSONConverterToolTest.php # Ensure the perms USER=`id -u -n` sudo chown -R $USER:www-data `pwd`/app/Config diff --git a/.travis.yml b/.travis.yml index 167228311..b54fd6c3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -148,6 +148,7 @@ before_script: script: - ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/ - ./app/Vendor/bin/phpunit app/Test/ComplexTypeToolTest.php + - ./app/Vendor/bin/phpunit app/Test/JSONConverterToolTest.php # Ensure the perms - sudo chown -R $USER:www-data `pwd`/app/Config - sudo chmod -R 770 `pwd`/app/Config diff --git a/app/Lib/Export/JsonExport.php b/app/Lib/Export/JsonExport.php index 1b1fe0205..1d9b473ed 100644 --- a/app/Lib/Export/JsonExport.php +++ b/app/Lib/Export/JsonExport.php @@ -5,6 +5,11 @@ class JsonExport private $__converter = false; public $non_restrictive_export = true; + /** + * @param $data + * @param array $options + * @return false|Generator|string + */ public function handler($data, $options = array()) { if ($options['scope'] === 'Attribute') { @@ -18,12 +23,18 @@ class JsonExport } } - private function __eventHandler($event, $options = array()) { + /** + * @param array $event + * @param array $options + * @return Generator + */ + private function __eventHandler($event, $options = array()) + { if ($this->__converter === false) { App::uses('JSONConverterTool', 'Tools'); $this->__converter = new JSONConverterTool(); } - return json_encode($this->__converter->convert($event, false, true)); + return $this->__converter->streamConvert($event); } private function __objectHandler($object, $options = array()) { @@ -44,7 +55,6 @@ class JsonExport $tagTypes = array('AttributeTag', 'EventTag'); foreach($tagTypes as $tagType) { if (isset($attribute[$tagType])) { - $attributeTags = array(); foreach ($attribute[$tagType] as $tk => $tag) { if ($tagType === 'EventTag') { $attribute[$tagType][$tk]['Tag']['inherited'] = 1; @@ -59,10 +69,10 @@ class JsonExport return json_encode($attribute); } - private function __sightingsHandler($sighting, $options = array()) - { - return json_encode($sighting); - } + private function __sightingsHandler($sighting, $options = array()) + { + return json_encode($sighting); + } public function header($options = array()) { @@ -80,12 +90,10 @@ class JsonExport } else { return ']}' . PHP_EOL; } - } public function separator() { return ','; } - } diff --git a/app/Lib/Tools/JSONConverterTool.php b/app/Lib/Tools/JSONConverterTool.php index be691bba5..74eb5154f 100644 --- a/app/Lib/Tools/JSONConverterTool.php +++ b/app/Lib/Tools/JSONConverterTool.php @@ -1,16 +1,6 @@ + */ + public function streamConvert(array $event) + { + $event = $this->convert($event, false, true); + // Fast and inaccurate way how to check if event is too big for to convert in one call. This can be changed in future. + $isBigEvent = (isset($event['Event']['Attribute']) ? count($event['Event']['Attribute']) : 0) + + (isset($event['Event']['Object']) ? count($event['Event']['Object']) : 0) > 100; + if (!$isBigEvent) { + yield json_encode($event, JSON_PRETTY_PRINT); + return; + } + + yield '{"Event":{'; + $firstKey = key($event['Event']); + foreach ($event['Event'] as $key => $value) { + if ($key === 'Attribute' || $key === 'Object') { // Encode every object or attribute separately + yield ($firstKey === $key ? '' : ',') . json_encode($key) . ":["; + $firstInnerKey = key($value); + foreach ($value as $i => $attribute) { + yield ($firstInnerKey === $i ? '' : ',') . json_encode($attribute); + } + yield "]"; + } else { + yield ($firstKey === $key ? '' : ',') . json_encode($key) . ":" . json_encode($value); + } + } + if (isset($event['errors'])) { + yield '},"errors":' . json_encode($event['errors']) . '}'; + } else { + yield "}}"; + } + } + private function __cleanAttributes($attributes, $tempSightings = array()) { // remove value1 and value2 from the output and remove invalid utf8 characters for the xml parser @@ -177,23 +204,4 @@ class JSONConverterTool return $resultArray; } } - - public function eventCollection2Format($events, $isSiteAdmin=false) - { - $results = array(); - foreach ($events as $event) { - $results[] = $this->convert($event, $isSiteAdmin); - } - return implode(',' . PHP_EOL, $results); - } - - public function frameCollection($input, $mispVersion = false) - { - $result = '{"response":['; - $result .= $input; - if ($mispVersion) { - $result .= ',' . PHP_EOL . '{"xml_version":"' . $mispVersion . '"}' . PHP_EOL; - } - return $result . ']}'; - } } diff --git a/app/Lib/Tools/TmpFileTool.php b/app/Lib/Tools/TmpFileTool.php index c2f4c0a9c..9b85aebf1 100644 --- a/app/Lib/Tools/TmpFileTool.php +++ b/app/Lib/Tools/TmpFileTool.php @@ -24,16 +24,29 @@ class TmpFileTool /** * Write data to stream with separator. Separator will be prepend to content for next call. - * @param string $content + * @param string|Generator $content * @param string $separator * @throws Exception */ public function writeWithSeparator($content, $separator) { if (isset($this->separator)) { - $this->write($this->separator . $content); + if ($content instanceof Generator) { + $this->write($this->separator); + foreach ($content as $part) { + $this->write($part); + } + } else { + $this->write($this->separator . $content); + } } else { - $this->write($content); + if ($content instanceof Generator) { + foreach ($content as $part) { + $this->write($part); + } + } else { + $this->write($content); + } } $this->separator = $separator; } diff --git a/app/Lib/Tools/XMLConverterTool.php b/app/Lib/Tools/XMLConverterTool.php index a074bbcee..e70032437 100644 --- a/app/Lib/Tools/XMLConverterTool.php +++ b/app/Lib/Tools/XMLConverterTool.php @@ -198,15 +198,6 @@ class XMLConverterTool $field = str_replace($this->__toEscape, $this->__escapeWith, $field); } - public function eventCollection2Format($events, $isSiteAdmin=false) - { - $result = ""; - foreach ($events as $event) { - $result .= $this->convert($event) . PHP_EOL; - } - return $result; - } - public function frameCollection($input, $mispVersion = false) { $result = '' . PHP_EOL . '' . PHP_EOL; @@ -216,9 +207,4 @@ class XMLConverterTool } return $result . '' . PHP_EOL; } - - private function __prepareAttributes($attributes) - { - return $attributes; - } } diff --git a/app/Model/Event.php b/app/Model/Event.php index f08dc55fa..286fb16c4 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -7045,32 +7045,24 @@ class Event extends AppModel $filters['includeAttachments'] = 1; } $this->Allowedlist = ClassRegistry::init('Allowedlist'); - foreach ($eventids_chunked as $chunk_index => $chunk) { + $separator = $exportTool->separator($exportToolParams); + foreach ($eventids_chunked as $chunk) { $filters['eventid'] = $chunk; if (!empty($filters['tags']['NOT'])) { $filters['blockedAttributeTags'] = $filters['tags']['NOT']; unset($filters['tags']['NOT']); } - $result = $this->fetchEvent( - $user, - $filters, - true - ); - if (!empty($result)) { - foreach ($result as $event) { - if ($jobId && $i%10 == 0) { - $this->Job->saveField('progress', intval((100 * $i) / $eventCount)); - $this->Job->saveField('message', 'Converting Event ' . $i . '/' . $eventCount . '.'); - } - $result = $this->Allowedlist->removeAllowedlistedFromArray($result, false); - $temp = $exportTool->handler($event, $exportToolParams); - if ($temp !== '') { - if ($i !== 0) { - $temp = $exportTool->separator($exportToolParams) . $temp; - } - $tmpfile->write($temp); - $i++; - } + $result = $this->fetchEvent($user, $filters,true); + $result = $this->Allowedlist->removeAllowedlistedFromArray($result, false); + foreach ($result as $event) { + if ($jobId && $i % 10 == 0) { + $this->Job->saveField('progress', intval((100 * $i) / $eventCount)); + $this->Job->saveField('message', 'Converting Event ' . $i . '/' . $eventCount . '.'); + } + $temp = $exportTool->handler($event, $exportToolParams); + if ($temp !== '') { + $tmpfile->writeWithSeparator($temp, $separator); + $i++; } } } diff --git a/app/Test/JSONConverterToolTest.php b/app/Test/JSONConverterToolTest.php new file mode 100644 index 000000000..5298a0505 --- /dev/null +++ b/app/Test/JSONConverterToolTest.php @@ -0,0 +1,52 @@ + 1, 'event_id' => 2, 'type' => 'ip-src', 'value' => '1.1.1.1']; + $event = ['Event' => ['id' => 2, 'info' => 'Test event']]; + for ($i = 0; $i < 200; $i++) { + $event['Attribute'][] = $attribute; + } + $this->check($event); + } + + public function testCheckJsonIsValidWithError(): void + { + $attribute = ['id' => 1, 'event_id' => 2, 'type' => 'ip-src', 'value' => '1.1.1.1']; + $event = ['Event' => ['id' => 2, 'info' => 'Test event'], 'errors' => 'chyba']; + for ($i = 0; $i < 200; $i++) { + $event['Attribute'][] = $attribute; + } + $this->check($event); + } + + public function testCheckJsonIsValidSmall(): void + { + $attribute = ['id' => 1, 'event_id' => 2, 'type' => 'ip-src', 'value' => '1.1.1.1']; + $event = ['Event' => ['id' => 2, 'info' => 'Test event'], 'errors' => 'chyba']; + for ($i = 0; $i < 5; $i++) { + $event['Attribute'][] = $attribute; + } + $this->check($event); + } + + private function check(array $event): void + { + $complexTypeTool = new JSONConverterTool(); + $json = ''; + foreach ($complexTypeTool->streamConvert($event) as $part) { + $json .= $part; + } + if (defined('JSON_THROW_ON_ERROR')) { + json_decode($json, true, 512, JSON_THROW_ON_ERROR); + $this->assertTrue(true); + } else { + $this->assertNotNull(json_decode($json)); + } + } +} diff --git a/app/View/Events/json/view.ctp b/app/View/Events/json/view.ctp index 7f4155ae9..61feeac51 100644 --- a/app/View/Events/json/view.ctp +++ b/app/View/Events/json/view.ctp @@ -1,4 +1,6 @@ convert($event); +foreach ($converter->streamConvert($event) as $part) { + echo $part; +}