MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [internal] Store taxonomy in cache compressed

pull/8724/head
Jakub Onderka 2022-10-30 17:07:54 +01:00
parent 645b11e1b1
commit bfff0f0320
6 changed files with 74 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/Controller/TagsController.php
View File

@ -987,7 +987,7 @@ class TagsController extends AppController
$this->loadModel('Taxonomy');
foreach ($tags as $k => $t) {
$dataFound = false;
$taxonomy = $this->Taxonomy->getTaxonomyForTag($t['Tag']['name'], false);
$taxonomy = $this->Taxonomy->getTaxonomyForTag($t['Tag']['name']);
if (!empty($taxonomy) && !empty($taxonomy['TaxonomyPredicate'][0])) {
$dataFound = true;
$tags[$k]['Taxonomy'] = $taxonomy['Taxonomy'];

2
app/Lib/Export/ContextExport.php
View File

@ -113,7 +113,7 @@ class ContextExport
return; // tag is not taxonomy tag
}
if (!isset($this->__taxonomyFetched[$splits['namespace']])) {
$fetchedTaxonomy = $this->Taxonomy->getTaxonomyForTag($tagName, false, true);
$fetchedTaxonomy = $this->Taxonomy->getTaxonomyForTag($tagName, true);
if (!empty($fetchedTaxonomy)) {
$fetched = [
'Taxonomy' => $fetchedTaxonomy['Taxonomy'],

3
app/Lib/Tools/CorrelationGraphTool.php
View File

@ -6,7 +6,8 @@
private $__related_attributes = array();
/** @var Event */
private $__eventModel;
private $__taxonomyModel = false;
/** @var Taxonomy */
private $__taxonomyModel;
private $__galaxyClusterModel = false;
private $__user = false;
private $__json = array();

4
app/Lib/Tools/FileAccessTool.php
View File

@ -46,9 +46,9 @@ class FileAccessTool
public static function readFromFile($file, $fileSize = -1)
{
if ($fileSize === -1) {
$content = file_get_contents($file);
$content = @file_get_contents($file);
} else {
$content = file_get_contents($file, false, null, 0, $fileSize);
$content = @file_get_contents($file, false, null, 0, $fileSize);
}
if ($content === false) {
if (!file_exists($file)) {

8
app/Lib/Tools/RedisTool.php
View File

@ -168,18 +168,22 @@ class RedisTool
if (function_exists('zstd_compress')) {
return zstd_compress($data, 1);
} elseif (function_exists('brotli_compress')) {
return self::BROTLI_HEADER . brotli_compress($data, 4);
return self::BROTLI_HEADER . brotli_compress($data, 0);
}
}
return $data;
}
/**
* @param string $data
* @param string|false $data
* @return string
*/
public static function decompress($data)
{
if ($data === false) {
return false;
}
$magic = substr($data, 0, 4);
if ($magic === self::ZSTD_HEADER) {
$data = zstd_uncompress($data);

112
app/Model/Taxonomy.php
View File

@ -45,14 +45,11 @@ class Taxonomy extends AppModel
$updated = array();
foreach ($directories as $dir) {
$dir = basename($dir);
if ($dir === 'tools') {
if ($dir === 'tools' || $dir === 'mapping') {
continue;
}
$machineTagPath = APP . 'files' . DS . 'taxonomies' . DS . $dir . DS . 'machinetag.json';
if (!file_exists($machineTagPath)) {
continue;
}
try {
$vocab = FileAccessTool::readJsonFromFile($machineTagPath);
@ -77,7 +74,7 @@ class Taxonomy extends AppModel
$vocab['version'] = 1;
}
if (!isset($existing[$vocab['namespace']]) || $vocab['version'] > $existing[$vocab['namespace']]['version']) {
$current = isset($existing[$vocab['namespace']]) ? $existing[$vocab['namespace']] : [];
$current = $existing[$vocab['namespace']] ?? [];
$result = $this->__updateVocab($vocab, $current);
if (is_numeric($result)) {
$updated['success'][$result] = array('namespace' => $vocab['namespace'], 'new' => $vocab['version']);
@ -89,6 +86,11 @@ class Taxonomy extends AppModel
}
}
}
if (!empty($updated['success'])) {
$this->cleanupCache();
}
return $updated;
}
@ -123,6 +125,7 @@ class Taxonomy extends AppModel
if (is_array($result)) {
throw new Exception('Could not save taxonomy because of validation errors: ' . json_encode($result));
}
$this->cleanupCache();
return (int)$result;
}
@ -592,60 +595,69 @@ class Taxonomy extends AppModel
return $taxonomies;
}
public function getTaxonomyForTag($tagName, $metaOnly = false, $fullTaxonomy = false)
private function cleanupCache()
{
RedisTool::deleteKeysByPattern(RedisTool::init(), "misp:taxonomies_cache:*");
}
/**
* @param string $tagName
* @param bool $fullTaxonomy
* @return array|false
* @throws JsonException
* @throws RedisException
*/
public function getTaxonomyForTag($tagName, $fullTaxonomy = false)
{
$splits = $this->splitTagToComponents($tagName);
if ($splits === null) {
return false; // not taxonomy tag
return false; // not a taxonomy tag
}
$key = "taxonomies_cache:tagName=$tagName&metaOnly=$metaOnly&fullTaxonomy=$fullTaxonomy";
$redis = $this->setupRedis();
$taxonomy = $redis ? RedisTool::deserialize($redis->get($key)) : null;
$key = "misp:taxonomies_cache:tagName=$tagName&fullTaxonomy=$fullTaxonomy";
if (!$taxonomy) {
if (isset($splits['value'])) {
$contain = array(
'TaxonomyPredicate' => array(
'TaxonomyEntry' => array()
)
try {
$redis = RedisTool::init();
$taxonomy = RedisTool::deserialize(RedisTool::decompress($redis->get($key)));
if (is_array($taxonomy)) {
return $taxonomy;
}
} catch (Exception $e) {
// ignore
}
if (isset($splits['value'])) {
$contain = array(
'TaxonomyPredicate' => array(
'TaxonomyEntry' => array()
)
);
if (!$fullTaxonomy) {
$contain['TaxonomyPredicate']['conditions'] = array(
'LOWER(TaxonomyPredicate.value)' => mb_strtolower($splits['predicate']),
);
$contain['TaxonomyPredicate']['TaxonomyEntry']['conditions'] = array(
'LOWER(TaxonomyEntry.value)' => mb_strtolower($splits['value']),
);
if (!$fullTaxonomy) {
$contain['TaxonomyPredicate']['conditions'] = array(
'LOWER(TaxonomyPredicate.value)' => mb_strtolower($splits['predicate']),
);
$contain['TaxonomyPredicate']['TaxonomyEntry']['conditions'] = array(
'LOWER(TaxonomyEntry.value)' => mb_strtolower($splits['value']),
);
}
$taxonomy = $this->find('first', array(
'recursive' => -1,
'conditions' => array('LOWER(Taxonomy.namespace)' => mb_strtolower($splits['namespace'])),
'contain' => $contain
));
if ($metaOnly && !empty($taxonomy)) {
$taxonomy = array('Taxonomy' => $taxonomy['Taxonomy']);
}
} else {
$contain = array('TaxonomyPredicate' => array());
if (!$fullTaxonomy) {
$contain['TaxonomyPredicate']['conditions'] = array(
'LOWER(TaxonomyPredicate.value)' => mb_strtolower($splits['predicate'])
);
}
$taxonomy = $this->find('first', array(
'recursive' => -1,
'conditions' => array('LOWER(Taxonomy.namespace)' => mb_strtolower($splits['namespace'])),
'contain' => $contain
));
if ($metaOnly && !empty($taxonomy)) {
$taxonomy = array('Taxonomy' => $taxonomy['Taxonomy']);
}
}
} else {
$contain = array('TaxonomyPredicate' => array());
if (!$fullTaxonomy) {
$contain['TaxonomyPredicate']['conditions'] = array(
'LOWER(TaxonomyPredicate.value)' => mb_strtolower($splits['predicate'])
);
}
}
if ($redis) {
$redis->setex($key, 1800, RedisTool::serialize($taxonomy));
}
$taxonomy = $this->find('first', array(
'recursive' => -1,
'conditions' => array('LOWER(Taxonomy.namespace)' => mb_strtolower($splits['namespace'])),
'contain' => $contain
));
if (isset($redis)) {
$redis->setex($key, 1800, RedisTool::compress(RedisTool::serialize($taxonomy)));
}
return $taxonomy;