mirror of https://github.com/MISP/MISP
new: [redis] Store some data in Redis compressed to save memory
parent
3deec3e3be
commit
645b11e1b1
|
@ -124,7 +124,7 @@ class AppController extends Controller
|
||||||
$this->__cors();
|
$this->__cors();
|
||||||
if (Configure::read('Security.check_sec_fetch_site_header')) {
|
if (Configure::read('Security.check_sec_fetch_site_header')) {
|
||||||
$secFetchSite = $this->request->header('Sec-Fetch-Site');
|
$secFetchSite = $this->request->header('Sec-Fetch-Site');
|
||||||
if ($secFetchSite !== false && $secFetchSite !== 'same-origin' && ($this->request->is('post') || $this->request->is('put') || $this->request->is('ajax'))) {
|
if ($secFetchSite !== false && $secFetchSite !== 'same-origin' && $this->request->is(['post', 'put', 'ajax'])) {
|
||||||
throw new MethodNotAllowedException("POST, PUT and AJAX requests are allowed just from same origin.");
|
throw new MethodNotAllowedException("POST, PUT and AJAX requests are allowed just from same origin.");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1396,8 +1396,7 @@ class AppController extends Controller
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$redis = $this->User->setupRedisWithException();
|
return RedisTool::init()->get('misp:live') !== '0';
|
||||||
return $redis->get('misp:live') !== '0';
|
|
||||||
} catch (Exception $e) {
|
} catch (Exception $e) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,10 @@
|
||||||
<?php
|
<?php
|
||||||
class RedisTool
|
class RedisTool
|
||||||
{
|
{
|
||||||
|
const COMPRESS_MIN_LENGTH = 200,
|
||||||
|
BROTLI_HEADER = "\xce\xb2\xcf\x81",
|
||||||
|
ZSTD_HEADER = "\x28\xb5\x2f\xfd";
|
||||||
|
|
||||||
/** @var Redis|null */
|
/** @var Redis|null */
|
||||||
private static $connection;
|
private static $connection;
|
||||||
|
|
||||||
|
@ -153,4 +157,41 @@ class RedisTool
|
||||||
return JsonTool::decode($string);
|
return JsonTool::decode($string);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $data
|
||||||
|
* @return string
|
||||||
|
*/
|
||||||
|
public static function compress($data)
|
||||||
|
{
|
||||||
|
if (strlen($data) >= self::COMPRESS_MIN_LENGTH) {
|
||||||
|
if (function_exists('zstd_compress')) {
|
||||||
|
return zstd_compress($data, 1);
|
||||||
|
} elseif (function_exists('brotli_compress')) {
|
||||||
|
return self::BROTLI_HEADER . brotli_compress($data, 4);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return $data;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $data
|
||||||
|
* @return string
|
||||||
|
*/
|
||||||
|
public static function decompress($data)
|
||||||
|
{
|
||||||
|
$magic = substr($data, 0, 4);
|
||||||
|
if ($magic === self::ZSTD_HEADER) {
|
||||||
|
$data = zstd_uncompress($data);
|
||||||
|
if ($data === false) {
|
||||||
|
throw new RuntimeException('Could not decompress');
|
||||||
|
}
|
||||||
|
} elseif ($magic === self::BROTLI_HEADER) {
|
||||||
|
$data = brotli_uncompress(substr($data, 4));
|
||||||
|
if ($data === false) {
|
||||||
|
throw new RuntimeException('Could not decompress');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return $data;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,9 +1,11 @@
|
||||||
<?php
|
<?php
|
||||||
App::uses('AppModel', 'Model');
|
App::uses('AppModel', 'Model');
|
||||||
|
|
||||||
// Table `event_locks` is not used anymore
|
|
||||||
class EventLock extends AppModel
|
class EventLock extends AppModel
|
||||||
{
|
{
|
||||||
|
// Table `event_locks` is not used anymore
|
||||||
|
public $useTable = false;
|
||||||
|
|
||||||
// In seconds
|
// In seconds
|
||||||
const DEFAULT_TTL = 900,
|
const DEFAULT_TTL = 900,
|
||||||
PREFIX = 'misp:event_lock:';
|
PREFIX = 'misp:event_lock:';
|
||||||
|
@ -126,6 +128,8 @@ class EventLock extends AppModel
|
||||||
* @param string $lockId
|
* @param string $lockId
|
||||||
* @param array $data
|
* @param array $data
|
||||||
* @return bool
|
* @return bool
|
||||||
|
* @throws JsonException
|
||||||
|
* @throws RedisException
|
||||||
*/
|
*/
|
||||||
private function insertLockToRedis($eventId, $lockId, array $data)
|
private function insertLockToRedis($eventId, $lockId, array $data)
|
||||||
{
|
{
|
||||||
|
|
|
@ -816,7 +816,7 @@ class Server extends AppModel
|
||||||
$redis = RedisTool::init();
|
$redis = RedisTool::init();
|
||||||
$indexFromCache = $redis->get("misp:event_index:{$serverSync->serverId()}");
|
$indexFromCache = $redis->get("misp:event_index:{$serverSync->serverId()}");
|
||||||
if ($indexFromCache) {
|
if ($indexFromCache) {
|
||||||
list($etag, $eventIndex) = RedisTool::deserialize($indexFromCache);
|
list($etag, $eventIndex) = RedisTool::deserialize(RedisTool::decompress($indexFromCache));
|
||||||
} else {
|
} else {
|
||||||
$etag = '""'; // Provide empty ETag, so MISP will compute ETag for returned data
|
$etag = '""'; // Provide empty ETag, so MISP will compute ETag for returned data
|
||||||
}
|
}
|
||||||
|
@ -835,10 +835,11 @@ class Server extends AppModel
|
||||||
}
|
}
|
||||||
|
|
||||||
// Save to cache for 24 hours if ETag provided
|
// Save to cache for 24 hours if ETag provided
|
||||||
if (isset($response->headers["ETag"])) {
|
$etag = $response->getHeader('etag');
|
||||||
$data = RedisTool::serialize([$response->headers["ETag"], $eventIndex]);
|
if ($etag) {
|
||||||
|
$data = RedisTool::compress(RedisTool::serialize([$etag, $eventIndex]));
|
||||||
$redis->setex("misp:event_index:{$serverSync->serverId()}", 3600 * 24, $data);
|
$redis->setex("misp:event_index:{$serverSync->serverId()}", 3600 * 24, $data);
|
||||||
} else if ($indexFromCache) {
|
} elseif ($indexFromCache) {
|
||||||
RedisTool::unlink($redis, "misp:event_index:{$serverSync->serverId()}");
|
RedisTool::unlink($redis, "misp:event_index:{$serverSync->serverId()}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue