new: [internal] JSON stream convert

.github/workflows/main.yml

@@ -206,6 +206,7 @@ jobs:
         run: |
               ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/
               ./app/Vendor/bin/phpunit app/Test/ComplexTypeToolTest.php
+              ./app/Vendor/bin/phpunit app/Test/JSONConverterToolTest.php
               # Ensure the perms
               USER=`id -u -n`
               sudo chown -R $USER:www-data `pwd`/app/Config

.travis.yml

@@ -148,6 +148,7 @@ before_script:
 script:
     - ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/
     - ./app/Vendor/bin/phpunit app/Test/ComplexTypeToolTest.php
+    - ./app/Vendor/bin/phpunit app/Test/JSONConverterToolTest.php
     # Ensure the perms
     - sudo chown -R $USER:www-data `pwd`/app/Config
     - sudo chmod -R 770 `pwd`/app/Config

app/Lib/Export/JsonExport.php

@@ -5,6 +5,11 @@ class JsonExport
 	private $__converter = false;
 	public $non_restrictive_export = true;
 
+    /**
+     * @param $data
+     * @param array $options
+     * @return false|Generator|string
+     */
     public function handler($data, $options = array())
     {
 		if ($options['scope'] === 'Attribute') {
@@ -18,12 +23,18 @@ class JsonExport
 		}
     }
 
-	private function __eventHandler($event, $options = array()) {
+    /**
+     * @param array $event
+     * @param array $options
+     * @return Generator
+     */
+	private function __eventHandler($event, $options = array())
+    {
 		if ($this->__converter === false) {
 			App::uses('JSONConverterTool', 'Tools');
 			$this->__converter = new JSONConverterTool();
 		}
-		return json_encode($this->__converter->convert($event, false, true));
+		return $this->__converter->streamConvert($event);
 	}
 
     private function __objectHandler($object, $options = array()) {
@@ -44,7 +55,6 @@ class JsonExport
 		$tagTypes = array('AttributeTag', 'EventTag');
 		foreach($tagTypes as $tagType) {
 			if (isset($attribute[$tagType])) {
-				$attributeTags = array();
 				foreach ($attribute[$tagType] as $tk => $tag) {
 					if ($tagType === 'EventTag') {
 						$attribute[$tagType][$tk]['Tag']['inherited'] = 1;
@@ -59,10 +69,10 @@ class JsonExport
 		return json_encode($attribute);
 	}
 
-        private function __sightingsHandler($sighting, $options = array())
-        {
-            return json_encode($sighting);
-        }
+    private function __sightingsHandler($sighting, $options = array())
+    {
+        return json_encode($sighting);
+    }
 
     public function header($options = array())
     {
@@ -80,12 +90,10 @@ class JsonExport
 		} else {
 			return ']}' . PHP_EOL;
 		}
-
     }
 
     public function separator()
     {
         return ',';
     }
-
 }

app/Lib/Tools/JSONConverterTool.php

@@ -1,16 +1,6 @@
 <?php
 class JSONConverterTool
 {
-    public function generateTop()
-    {
-        return '{"response":[';
-    }
-
-    public function generateBottom()
-    {
-        return ']}' . PHP_EOL;
-    }
-
     public function convertAttribute($attribute, $raw = false)
     {
         $toRearrange = array('AttributeTag');
@@ -114,6 +104,43 @@ class JSONConverterTool
         return json_encode($result, JSON_PRETTY_PRINT);
     }
 
+    /**
+     * Event to JSON stream convertor.
+     * @param array $event
+     * @return Generator<string>
+     */
+    public function streamConvert(array $event)
+    {
+        $event = $this->convert($event, false, true);
+        // Fast and inaccurate way how to check if event is too big for to convert in one call. This can be changed in future.
+        $isBigEvent = (isset($event['Event']['Attribute']) ? count($event['Event']['Attribute']) : 0) +
+            (isset($event['Event']['Object']) ? count($event['Event']['Object']) : 0) > 100;
+        if (!$isBigEvent) {
+            yield json_encode($event, JSON_PRETTY_PRINT);
+            return;
+        }
+
+        yield '{"Event":{';
+        $firstKey = key($event['Event']);
+        foreach ($event['Event'] as $key => $value) {
+            if ($key === 'Attribute' || $key === 'Object') { // Encode every object or attribute separately
+                yield ($firstKey === $key ? '' : ',') . json_encode($key) . ":[";
+                $firstInnerKey = key($value);
+                foreach ($value as $i => $attribute) {
+                    yield ($firstInnerKey === $i ? '' : ',')  . json_encode($attribute);
+                }
+                yield "]";
+            } else {
+                yield ($firstKey === $key ? '' : ',') . json_encode($key) . ":" . json_encode($value);
+            }
+        }
+        if (isset($event['errors'])) {
+            yield '},"errors":' . json_encode($event['errors']) . '}';
+        } else {
+            yield "}}";
+        }
+    }
+
     private function __cleanAttributes($attributes, $tempSightings = array())
     {
         // remove value1 and value2 from the output and remove invalid utf8 characters for the xml parser
@@ -177,23 +204,4 @@ class JSONConverterTool
             return $resultArray;
         }
     }
-
-    public function eventCollection2Format($events, $isSiteAdmin=false)
-    {
-        $results = array();
-        foreach ($events as $event) {
-            $results[] = $this->convert($event, $isSiteAdmin);
-        }
-        return implode(',' . PHP_EOL, $results);
-    }
-
-    public function frameCollection($input, $mispVersion = false)
-    {
-        $result = '{"response":[';
-        $result .= $input;
-        if ($mispVersion) {
-            $result .= ',' . PHP_EOL . '{"xml_version":"' . $mispVersion . '"}' . PHP_EOL;
-        }
-        return $result . ']}';
-    }
 }

app/Lib/Tools/TmpFileTool.php

@@ -24,16 +24,29 @@ class TmpFileTool
 
     /**
      * Write data to stream with separator. Separator will be prepend to content for next call.
-     * @param string $content
+     * @param string|Generator $content
      * @param string $separator
      * @throws Exception
      */
     public function writeWithSeparator($content, $separator)
     {
         if (isset($this->separator)) {
-            $this->write($this->separator . $content);
+            if ($content instanceof Generator) {
+                $this->write($this->separator);
+                foreach ($content as $part) {
+                    $this->write($part);
+                }
+            } else {
+                $this->write($this->separator . $content);
+            }
         } else {
-            $this->write($content);
+            if ($content instanceof Generator) {
+                foreach ($content as $part) {
+                    $this->write($part);
+                }
+            } else {
+                $this->write($content);
+            }
         }
         $this->separator = $separator;
     }

app/Lib/Tools/XMLConverterTool.php

@@ -198,15 +198,6 @@ class XMLConverterTool
         $field = str_replace($this->__toEscape, $this->__escapeWith, $field);
     }
 
-    public function eventCollection2Format($events, $isSiteAdmin=false)
-    {
-        $result = "";
-        foreach ($events as $event) {
-            $result .= $this->convert($event) . PHP_EOL;
-        }
-        return $result;
-    }
-
     public function frameCollection($input, $mispVersion = false)
     {
         $result = '<?xml version="1.0" encoding="UTF-8"?>' . PHP_EOL . '<response>' . PHP_EOL;
@@ -216,9 +207,4 @@ class XMLConverterTool
         }
         return $result . '</response>' . PHP_EOL;
     }
-
-    private function __prepareAttributes($attributes)
-    {
-        return $attributes;
-    }
 }

app/Model/Event.php

@@ -7045,32 +7045,24 @@ class Event extends AppModel
             $filters['includeAttachments'] = 1;
         }
         $this->Allowedlist = ClassRegistry::init('Allowedlist');
-        foreach ($eventids_chunked as $chunk_index => $chunk) {
+        $separator = $exportTool->separator($exportToolParams);
+        foreach ($eventids_chunked as $chunk) {
             $filters['eventid'] = $chunk;
             if (!empty($filters['tags']['NOT'])) {
                 $filters['blockedAttributeTags'] = $filters['tags']['NOT'];
                 unset($filters['tags']['NOT']);
             }
-            $result = $this->fetchEvent(
-                $user,
-                $filters,
-                true
-            );
-            if (!empty($result)) {
-                foreach ($result as $event) {
-                    if ($jobId && $i%10 == 0) {
-                        $this->Job->saveField('progress', intval((100 * $i) / $eventCount));
-                        $this->Job->saveField('message', 'Converting Event ' . $i . '/' . $eventCount . '.');
-                    }
-                    $result = $this->Allowedlist->removeAllowedlistedFromArray($result, false);
-                    $temp = $exportTool->handler($event, $exportToolParams);
-                    if ($temp !== '') {
-                        if ($i !== 0) {
-                            $temp = $exportTool->separator($exportToolParams) . $temp;
-                        }
-                        $tmpfile->write($temp);
-                        $i++;
-                    }
+            $result = $this->fetchEvent($user, $filters,true);
+            $result = $this->Allowedlist->removeAllowedlistedFromArray($result, false);
+            foreach ($result as $event) {
+                if ($jobId && $i % 10 == 0) {
+                    $this->Job->saveField('progress', intval((100 * $i) / $eventCount));
+                    $this->Job->saveField('message', 'Converting Event ' . $i . '/' . $eventCount . '.');
+                }
+                $temp = $exportTool->handler($event, $exportToolParams);
+                if ($temp !== '') {
+                    $tmpfile->writeWithSeparator($temp, $separator);
+                    $i++;
                 }
             }
         }

app/Test/JSONConverterToolTest.php

@@ -0,0 +1,52 @@
+<?php
+require_once __DIR__ . '/../Lib/Tools/JSONConverterTool.php';
+
+use PHPUnit\Framework\TestCase;
+
+class JSONConverterToolTest extends TestCase
+{
+    public function testCheckJsonIsValid(): void
+    {
+        $attribute = ['id' => 1, 'event_id' => 2, 'type' => 'ip-src', 'value' => '1.1.1.1'];
+        $event = ['Event' => ['id' => 2, 'info' => 'Test event']];
+        for ($i = 0; $i < 200; $i++) {
+            $event['Attribute'][] = $attribute;
+        }
+        $this->check($event);
+    }
+
+    public function testCheckJsonIsValidWithError(): void
+    {
+        $attribute = ['id' => 1, 'event_id' => 2, 'type' => 'ip-src', 'value' => '1.1.1.1'];
+        $event = ['Event' => ['id' => 2, 'info' => 'Test event'], 'errors' => 'chyba'];
+        for ($i = 0; $i < 200; $i++) {
+            $event['Attribute'][] = $attribute;
+        }
+        $this->check($event);
+    }
+
+    public function testCheckJsonIsValidSmall(): void
+    {
+        $attribute = ['id' => 1, 'event_id' => 2, 'type' => 'ip-src', 'value' => '1.1.1.1'];
+        $event = ['Event' => ['id' => 2, 'info' => 'Test event'], 'errors' => 'chyba'];
+        for ($i = 0; $i < 5; $i++) {
+            $event['Attribute'][] = $attribute;
+        }
+        $this->check($event);
+    }
+
+    private function check(array $event): void
+    {
+        $complexTypeTool = new JSONConverterTool();
+        $json = '';
+        foreach ($complexTypeTool->streamConvert($event) as $part) {
+            $json .= $part;
+        }
+        if (defined('JSON_THROW_ON_ERROR')) {
+            json_decode($json, true, 512, JSON_THROW_ON_ERROR);
+            $this->assertTrue(true);
+        } else {
+            $this->assertNotNull(json_decode($json));
+        }
+    }
+}

app/View/Events/json/view.ctp

@@ -1,4 +1,6 @@
 <?php
 App::uses('JSONConverterTool', 'Tools');
 $converter = new JSONConverterTool();
-echo $converter->convert($event);
+foreach ($converter->streamConvert($event) as $part) {
+    echo $part;
+}