From bfff0f0320f09b0332817a523cd96a95e7e52874 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sun, 30 Oct 2022 17:07:54 +0100 Subject: [PATCH] chg: [internal] Store taxonomy in cache compressed --- app/Controller/TagsController.php | 2 +- app/Lib/Export/ContextExport.php | 2 +- app/Lib/Tools/CorrelationGraphTool.php | 3 +- app/Lib/Tools/FileAccessTool.php | 4 +- app/Lib/Tools/RedisTool.php | 8 +- app/Model/Taxonomy.php | 112 ++++++++++++++----------- 6 files changed, 74 insertions(+), 57 deletions(-) diff --git a/app/Controller/TagsController.php b/app/Controller/TagsController.php index 8bfa3dee6..89bec4f2d 100644 --- a/app/Controller/TagsController.php +++ b/app/Controller/TagsController.php @@ -987,7 +987,7 @@ class TagsController extends AppController $this->loadModel('Taxonomy'); foreach ($tags as $k => $t) { $dataFound = false; - $taxonomy = $this->Taxonomy->getTaxonomyForTag($t['Tag']['name'], false); + $taxonomy = $this->Taxonomy->getTaxonomyForTag($t['Tag']['name']); if (!empty($taxonomy) && !empty($taxonomy['TaxonomyPredicate'][0])) { $dataFound = true; $tags[$k]['Taxonomy'] = $taxonomy['Taxonomy']; diff --git a/app/Lib/Export/ContextExport.php b/app/Lib/Export/ContextExport.php index d6b288d28..b7961effb 100644 --- a/app/Lib/Export/ContextExport.php +++ b/app/Lib/Export/ContextExport.php @@ -113,7 +113,7 @@ class ContextExport return; // tag is not taxonomy tag } if (!isset($this->__taxonomyFetched[$splits['namespace']])) { - $fetchedTaxonomy = $this->Taxonomy->getTaxonomyForTag($tagName, false, true); + $fetchedTaxonomy = $this->Taxonomy->getTaxonomyForTag($tagName, true); if (!empty($fetchedTaxonomy)) { $fetched = [ 'Taxonomy' => $fetchedTaxonomy['Taxonomy'], diff --git a/app/Lib/Tools/CorrelationGraphTool.php b/app/Lib/Tools/CorrelationGraphTool.php index 0e18512be..d089a1010 100644 --- a/app/Lib/Tools/CorrelationGraphTool.php +++ b/app/Lib/Tools/CorrelationGraphTool.php @@ -6,7 +6,8 @@ private $__related_attributes = array(); /** @var Event */ private $__eventModel; - private $__taxonomyModel = false; + /** @var Taxonomy */ + private $__taxonomyModel; private $__galaxyClusterModel = false; private $__user = false; private $__json = array(); diff --git a/app/Lib/Tools/FileAccessTool.php b/app/Lib/Tools/FileAccessTool.php index e335b522a..3e3aa01cd 100644 --- a/app/Lib/Tools/FileAccessTool.php +++ b/app/Lib/Tools/FileAccessTool.php @@ -46,9 +46,9 @@ class FileAccessTool public static function readFromFile($file, $fileSize = -1) { if ($fileSize === -1) { - $content = file_get_contents($file); + $content = @file_get_contents($file); } else { - $content = file_get_contents($file, false, null, 0, $fileSize); + $content = @file_get_contents($file, false, null, 0, $fileSize); } if ($content === false) { if (!file_exists($file)) { diff --git a/app/Lib/Tools/RedisTool.php b/app/Lib/Tools/RedisTool.php index e1a0b537a..4159322fd 100644 --- a/app/Lib/Tools/RedisTool.php +++ b/app/Lib/Tools/RedisTool.php @@ -168,18 +168,22 @@ class RedisTool if (function_exists('zstd_compress')) { return zstd_compress($data, 1); } elseif (function_exists('brotli_compress')) { - return self::BROTLI_HEADER . brotli_compress($data, 4); + return self::BROTLI_HEADER . brotli_compress($data, 0); } } return $data; } /** - * @param string $data + * @param string|false $data * @return string */ public static function decompress($data) { + if ($data === false) { + return false; + } + $magic = substr($data, 0, 4); if ($magic === self::ZSTD_HEADER) { $data = zstd_uncompress($data); diff --git a/app/Model/Taxonomy.php b/app/Model/Taxonomy.php index 06a8af1d6..05db5bfb3 100644 --- a/app/Model/Taxonomy.php +++ b/app/Model/Taxonomy.php @@ -45,14 +45,11 @@ class Taxonomy extends AppModel $updated = array(); foreach ($directories as $dir) { $dir = basename($dir); - if ($dir === 'tools') { + if ($dir === 'tools' || $dir === 'mapping') { continue; } $machineTagPath = APP . 'files' . DS . 'taxonomies' . DS . $dir . DS . 'machinetag.json'; - if (!file_exists($machineTagPath)) { - continue; - } try { $vocab = FileAccessTool::readJsonFromFile($machineTagPath); @@ -77,7 +74,7 @@ class Taxonomy extends AppModel $vocab['version'] = 1; } if (!isset($existing[$vocab['namespace']]) || $vocab['version'] > $existing[$vocab['namespace']]['version']) { - $current = isset($existing[$vocab['namespace']]) ? $existing[$vocab['namespace']] : []; + $current = $existing[$vocab['namespace']] ?? []; $result = $this->__updateVocab($vocab, $current); if (is_numeric($result)) { $updated['success'][$result] = array('namespace' => $vocab['namespace'], 'new' => $vocab['version']); @@ -89,6 +86,11 @@ class Taxonomy extends AppModel } } } + + if (!empty($updated['success'])) { + $this->cleanupCache(); + } + return $updated; } @@ -123,6 +125,7 @@ class Taxonomy extends AppModel if (is_array($result)) { throw new Exception('Could not save taxonomy because of validation errors: ' . json_encode($result)); } + $this->cleanupCache(); return (int)$result; } @@ -592,60 +595,69 @@ class Taxonomy extends AppModel return $taxonomies; } - public function getTaxonomyForTag($tagName, $metaOnly = false, $fullTaxonomy = false) + private function cleanupCache() + { + RedisTool::deleteKeysByPattern(RedisTool::init(), "misp:taxonomies_cache:*"); + } + + + /** + * @param string $tagName + * @param bool $fullTaxonomy + * @return array|false + * @throws JsonException + * @throws RedisException + */ + public function getTaxonomyForTag($tagName, $fullTaxonomy = false) { $splits = $this->splitTagToComponents($tagName); if ($splits === null) { - return false; // not taxonomy tag + return false; // not a taxonomy tag } - $key = "taxonomies_cache:tagName=$tagName&metaOnly=$metaOnly&fullTaxonomy=$fullTaxonomy"; - $redis = $this->setupRedis(); - $taxonomy = $redis ? RedisTool::deserialize($redis->get($key)) : null; + $key = "misp:taxonomies_cache:tagName=$tagName&fullTaxonomy=$fullTaxonomy"; - if (!$taxonomy) { - if (isset($splits['value'])) { - $contain = array( - 'TaxonomyPredicate' => array( - 'TaxonomyEntry' => array() - ) + try { + $redis = RedisTool::init(); + $taxonomy = RedisTool::deserialize(RedisTool::decompress($redis->get($key))); + if (is_array($taxonomy)) { + return $taxonomy; + } + } catch (Exception $e) { + // ignore + } + + if (isset($splits['value'])) { + $contain = array( + 'TaxonomyPredicate' => array( + 'TaxonomyEntry' => array() + ) + ); + if (!$fullTaxonomy) { + $contain['TaxonomyPredicate']['conditions'] = array( + 'LOWER(TaxonomyPredicate.value)' => mb_strtolower($splits['predicate']), + ); + $contain['TaxonomyPredicate']['TaxonomyEntry']['conditions'] = array( + 'LOWER(TaxonomyEntry.value)' => mb_strtolower($splits['value']), ); - if (!$fullTaxonomy) { - $contain['TaxonomyPredicate']['conditions'] = array( - 'LOWER(TaxonomyPredicate.value)' => mb_strtolower($splits['predicate']), - ); - $contain['TaxonomyPredicate']['TaxonomyEntry']['conditions'] = array( - 'LOWER(TaxonomyEntry.value)' => mb_strtolower($splits['value']), - ); - } - $taxonomy = $this->find('first', array( - 'recursive' => -1, - 'conditions' => array('LOWER(Taxonomy.namespace)' => mb_strtolower($splits['namespace'])), - 'contain' => $contain - )); - if ($metaOnly && !empty($taxonomy)) { - $taxonomy = array('Taxonomy' => $taxonomy['Taxonomy']); - } - } else { - $contain = array('TaxonomyPredicate' => array()); - if (!$fullTaxonomy) { - $contain['TaxonomyPredicate']['conditions'] = array( - 'LOWER(TaxonomyPredicate.value)' => mb_strtolower($splits['predicate']) - ); - } - $taxonomy = $this->find('first', array( - 'recursive' => -1, - 'conditions' => array('LOWER(Taxonomy.namespace)' => mb_strtolower($splits['namespace'])), - 'contain' => $contain - )); - if ($metaOnly && !empty($taxonomy)) { - $taxonomy = array('Taxonomy' => $taxonomy['Taxonomy']); - } } + } else { + $contain = array('TaxonomyPredicate' => array()); + if (!$fullTaxonomy) { + $contain['TaxonomyPredicate']['conditions'] = array( + 'LOWER(TaxonomyPredicate.value)' => mb_strtolower($splits['predicate']) + ); + } + } - if ($redis) { - $redis->setex($key, 1800, RedisTool::serialize($taxonomy)); - } + $taxonomy = $this->find('first', array( + 'recursive' => -1, + 'conditions' => array('LOWER(Taxonomy.namespace)' => mb_strtolower($splits['namespace'])), + 'contain' => $contain + )); + + if (isset($redis)) { + $redis->setex($key, 1800, RedisTool::compress(RedisTool::serialize($taxonomy))); } return $taxonomy;