MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [sync] Change way how event index is cached in Redis to save memory

pull/9651/head
Jakub Onderka 2024-04-13 12:42:54 +02:00
parent d2176ab8bd
commit 47d35dae0b
1 changed files with 22 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
app/Model/Server.php
View File

@ -921,39 +921,41 @@ class Server extends AppModel
// Fetch event index from cache if exists and is not modified // Fetch event index from cache if exists and is not modified
$redis = RedisTool::init(); $redis = RedisTool::init();
$indexFromCache = $redis->get("misp:event_index:{$serverSync->serverId()}"); $indexFromCache = $redis->get("misp:event_index_cache:{$serverSync->serverId()}");
if ($indexFromCache) { if ($indexFromCache) {
$indexFromCache = RedisTool::decompress($indexFromCache); $etagPos = strpos($indexFromCache, "\n");
list($etag, $eventIndex) = RedisTool::deserialize($indexFromCache); if ($etagPos === false) {
unset($indexFromCache); throw new RuntimeException("Could not find etag in cache fro server {$serverSync->serverId()}");
$serverSync->debug("Event index loaded from Redis cache with etag $etag containing " . count($eventIndex) . ' events'); }
$etag = substr($indexFromCache, 0, $etagPos);
$serverSync->debug("Event index loaded from Redis cache with etag $etag containing");
} else { } else {
$etag = '""'; // Provide empty ETag, so MISP will compute ETag for returned data $etag = '""'; // Provide empty ETag, so MISP will compute ETag for returned data
} }
$response = $serverSync->eventIndex($filterRules, $etag); $response = $serverSync->eventIndex($filterRules, $etag);
if ($response->isNotModified() && isset($eventIndex)) { if ($response->isNotModified() && $indexFromCache) {
return $eventIndex; return JsonTool::decode(RedisTool::decompress(substr($indexFromCache, $etagPos + 1)));
}
unset($eventIndex);
$eventIndex = $response->json();
unset($response->body); // remove response that can take a lot of memory
// correct $eventArray if just one event, probably this response returns old MISP
if (isset($eventIndex['id'])) {
$eventIndex = [$eventIndex];
} }
// Save to cache for 24 hours if ETag provided // Save to cache for 24 hours if ETag provided
$etag = $response->getHeader('etag'); $etag = $response->getHeader('etag');
if ($etag) { if ($etag) {
$serverSync->debug("Event index from remote server has different etag $etag, saving to cache"); $serverSync->debug("Event index from remote server has different etag $etag, saving to cache");
$data = RedisTool::compress(RedisTool::serialize([$etag, $eventIndex])); $data = "$etag\n" . RedisTool::compress($response->body);
$redis->setex("misp:event_index:{$serverSync->serverId()}", 3600 * 24, $data); $redis->setex("misp:event_index_cache:{$serverSync->serverId()}", 3600 * 24, $data);
} elseif (isset($eventIndex)) { } elseif ($indexFromCache) {
RedisTool::unlink($redis, "misp:event_index:{$serverSync->serverId()}"); RedisTool::unlink($redis, "misp:event_index_cache:{$serverSync->serverId()}");
}
unset($indexFromCache); // clean up memory
$eventIndex = $response->json();
// correct $eventArray if just one event, probably this response returns old MISP
if (isset($eventIndex['id'])) {
$eventIndex = [$eventIndex];
} }
return $eventIndex; return $eventIndex;