MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

new: [internal] JSON stream convert

pull/6496/head
Jakub Onderka 2020-10-24 19:09:20 +02:00
parent 83945e4885
commit 791dc9deab
9 changed files with 140 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/main.yml vendored
View File

@ -206,6 +206,7 @@ jobs:
run: |
./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/
./app/Vendor/bin/phpunit app/Test/ComplexTypeToolTest.php
./app/Vendor/bin/phpunit app/Test/JSONConverterToolTest.php
# Ensure the perms
USER=`id -u -n`
sudo chown -R $USER:www-data `pwd`/app/Config

1
.travis.yml
View File

@ -148,6 +148,7 @@ before_script:
script:
- ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/
- ./app/Vendor/bin/phpunit app/Test/ComplexTypeToolTest.php
- ./app/Vendor/bin/phpunit app/Test/JSONConverterToolTest.php
# Ensure the perms
- sudo chown -R $USER:www-data `pwd`/app/Config
- sudo chmod -R 770 `pwd`/app/Config

26
app/Lib/Export/JsonExport.php
View File

@ -5,6 +5,11 @@ class JsonExport
private $__converter = false;
public $non_restrictive_export = true;
/**
* @param $data
* @param array $options
* @return false|Generator|string
*/
public function handler($data, $options = array())
{
if ($options['scope'] === 'Attribute') {
@ -18,12 +23,18 @@ class JsonExport
}
}
private function __eventHandler($event, $options = array()) {
/**
* @param array $event
* @param array $options
* @return Generator
*/
private function __eventHandler($event, $options = array())
{
if ($this->__converter === false) {
App::uses('JSONConverterTool', 'Tools');
$this->__converter = new JSONConverterTool();
}
return json_encode($this->__converter->convert($event, false, true));
return $this->__converter->streamConvert($event);
}
private function __objectHandler($object, $options = array()) {
@ -44,7 +55,6 @@ class JsonExport
$tagTypes = array('AttributeTag', 'EventTag');
foreach($tagTypes as $tagType) {
if (isset($attribute[$tagType])) {
$attributeTags = array();
foreach ($attribute[$tagType] as $tk => $tag) {
if ($tagType === 'EventTag') {
$attribute[$tagType][$tk]['Tag']['inherited'] = 1;
@ -59,10 +69,10 @@ class JsonExport
return json_encode($attribute);
}
private function __sightingsHandler($sighting, $options = array())
{
return json_encode($sighting);
}
private function __sightingsHandler($sighting, $options = array())
{
return json_encode($sighting);
}
public function header($options = array())
{
@ -80,12 +90,10 @@ class JsonExport
} else {
return ']}' . PHP_EOL;
}
}
public function separator()
{
return ',';
}
}

66
app/Lib/Tools/JSONConverterTool.php
View File

@ -1,16 +1,6 @@
<?php
class JSONConverterTool
{
public function generateTop()
{
return '{"response":[';
}
public function generateBottom()
{
return ']}' . PHP_EOL;
}
public function convertAttribute($attribute, $raw = false)
{
$toRearrange = array('AttributeTag');
@ -114,6 +104,43 @@ class JSONConverterTool
return json_encode($result, JSON_PRETTY_PRINT);
}
/**
* Event to JSON stream convertor.
* @param array $event
* @return Generator<string>
*/
public function streamConvert(array $event)
{
$event = $this->convert($event, false, true);
// Fast and inaccurate way how to check if event is too big for to convert in one call. This can be changed in future.
$isBigEvent = (isset($event['Event']['Attribute']) ? count($event['Event']['Attribute']) : 0) +
(isset($event['Event']['Object']) ? count($event['Event']['Object']) : 0) > 100;
if (!$isBigEvent) {
yield json_encode($event, JSON_PRETTY_PRINT);
return;
}
yield '{"Event":{';
$firstKey = key($event['Event']);
foreach ($event['Event'] as $key => $value) {
if ($key === 'Attribute' || $key === 'Object') { // Encode every object or attribute separately
yield ($firstKey === $key ? '' : ',') . json_encode($key) . ":[";
$firstInnerKey = key($value);
foreach ($value as $i => $attribute) {
yield ($firstInnerKey === $i ? '' : ',') . json_encode($attribute);
}
yield "]";
} else {
yield ($firstKey === $key ? '' : ',') . json_encode($key) . ":" . json_encode($value);
}
}
if (isset($event['errors'])) {
yield '},"errors":' . json_encode($event['errors']) . '}';
} else {
yield "}}";
}
}
private function __cleanAttributes($attributes, $tempSightings = array())
{
// remove value1 and value2 from the output and remove invalid utf8 characters for the xml parser
@ -177,23 +204,4 @@ class JSONConverterTool
return $resultArray;
}
}
public function eventCollection2Format($events, $isSiteAdmin=false)
{
$results = array();
foreach ($events as $event) {
$results[] = $this->convert($event, $isSiteAdmin);
}
return implode(',' . PHP_EOL, $results);
}
public function frameCollection($input, $mispVersion = false)
{
$result = '{"response":[';
$result .= $input;
if ($mispVersion) {
$result .= ',' . PHP_EOL . '{"xml_version":"' . $mispVersion . '"}' . PHP_EOL;
}
return $result . ']}';
}
}

19
app/Lib/Tools/TmpFileTool.php
View File

@ -24,16 +24,29 @@ class TmpFileTool
/**
* Write data to stream with separator. Separator will be prepend to content for next call.
* @param string $content
* @param string|Generator $content
* @param string $separator
* @throws Exception
*/
public function writeWithSeparator($content, $separator)
{
if (isset($this->separator)) {
$this->write($this->separator . $content);
if ($content instanceof Generator) {
$this->write($this->separator);
foreach ($content as $part) {
$this->write($part);
}
} else {
$this->write($this->separator . $content);
}
} else {
$this->write($content);
if ($content instanceof Generator) {
foreach ($content as $part) {
$this->write($part);
}
} else {
$this->write($content);
}
}
$this->separator = $separator;
}

14
app/Lib/Tools/XMLConverterTool.php
View File

@ -198,15 +198,6 @@ class XMLConverterTool
$field = str_replace($this->__toEscape, $this->__escapeWith, $field);
}
public function eventCollection2Format($events, $isSiteAdmin=false)
{
$result = "";
foreach ($events as $event) {
$result .= $this->convert($event) . PHP_EOL;
}
return $result;
}
public function frameCollection($input, $mispVersion = false)
{
$result = '<?xml version="1.0" encoding="UTF-8"?>' . PHP_EOL . '<response>' . PHP_EOL;
@ -216,9 +207,4 @@ class XMLConverterTool
}
return $result . '</response>' . PHP_EOL;
}
private function __prepareAttributes($attributes)
{
return $attributes;
}
}

34
app/Model/Event.php
View File

@ -7045,32 +7045,24 @@ class Event extends AppModel
$filters['includeAttachments'] = 1;
}
$this->Allowedlist = ClassRegistry::init('Allowedlist');
foreach ($eventids_chunked as $chunk_index => $chunk) {
$separator = $exportTool->separator($exportToolParams);
foreach ($eventids_chunked as $chunk) {
$filters['eventid'] = $chunk;
if (!empty($filters['tags']['NOT'])) {
$filters['blockedAttributeTags'] = $filters['tags']['NOT'];
unset($filters['tags']['NOT']);
}
$result = $this->fetchEvent(
$user,
$filters,
true
);
if (!empty($result)) {
foreach ($result as $event) {
if ($jobId && $i%10 == 0) {
$this->Job->saveField('progress', intval((100 * $i) / $eventCount));
$this->Job->saveField('message', 'Converting Event ' . $i . '/' . $eventCount . '.');
}
$result = $this->Allowedlist->removeAllowedlistedFromArray($result, false);
$temp = $exportTool->handler($event, $exportToolParams);
if ($temp !== '') {
if ($i !== 0) {
$temp = $exportTool->separator($exportToolParams) . $temp;
}
$tmpfile->write($temp);
$i++;
}
$result = $this->fetchEvent($user, $filters,true);
$result = $this->Allowedlist->removeAllowedlistedFromArray($result, false);
foreach ($result as $event) {
if ($jobId && $i % 10 == 0) {
$this->Job->saveField('progress', intval((100 * $i) / $eventCount));
$this->Job->saveField('message', 'Converting Event ' . $i . '/' . $eventCount . '.');
}
$temp = $exportTool->handler($event, $exportToolParams);
if ($temp !== '') {
$tmpfile->writeWithSeparator($temp, $separator);
$i++;
}
}
}

52
app/Test/JSONConverterToolTest.php Normal file
View File

@ -0,0 +1,52 @@
<?php
require_once __DIR__ . '/../Lib/Tools/JSONConverterTool.php';
use PHPUnit\Framework\TestCase;
class JSONConverterToolTest extends TestCase
{
public function testCheckJsonIsValid(): void
{
$attribute = ['id' => 1, 'event_id' => 2, 'type' => 'ip-src', 'value' => '1.1.1.1'];
$event = ['Event' => ['id' => 2, 'info' => 'Test event']];
for ($i = 0; $i < 200; $i++) {
$event['Attribute'][] = $attribute;
}
$this->check($event);
}
public function testCheckJsonIsValidWithError(): void
{
$attribute = ['id' => 1, 'event_id' => 2, 'type' => 'ip-src', 'value' => '1.1.1.1'];
$event = ['Event' => ['id' => 2, 'info' => 'Test event'], 'errors' => 'chyba'];
for ($i = 0; $i < 200; $i++) {
$event['Attribute'][] = $attribute;
}
$this->check($event);
}
public function testCheckJsonIsValidSmall(): void
{
$attribute = ['id' => 1, 'event_id' => 2, 'type' => 'ip-src', 'value' => '1.1.1.1'];
$event = ['Event' => ['id' => 2, 'info' => 'Test event'], 'errors' => 'chyba'];
for ($i = 0; $i < 5; $i++) {
$event['Attribute'][] = $attribute;
}
$this->check($event);
}
private function check(array $event): void
{
$complexTypeTool = new JSONConverterTool();
$json = '';
foreach ($complexTypeTool->streamConvert($event) as $part) {
$json .= $part;
}
if (defined('JSON_THROW_ON_ERROR')) {
json_decode($json, true, 512, JSON_THROW_ON_ERROR);
$this->assertTrue(true);
} else {
$this->assertNotNull(json_decode($json));
}
}
}

4
app/View/Events/json/view.ctp
View File

@ -1,4 +1,6 @@
<?php
App::uses('JSONConverterTool', 'Tools');
$converter = new JSONConverterTool();
echo $converter->convert($event);
foreach ($converter->streamConvert($event) as $part) {
echo $part;
}