MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

fix: Fixes a feed caching issue introduced by the performance tweaks 

- moved the combined feed generation for the fast lookups to the feed caching algorigthms as opposed to an on an on-the-fly merge
pull/2260/head
iglocska 2017-06-12 15:25:08 +02:00
parent 0b9d55c7b5
commit cffc8a4f8f
1 changed files with 10 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
app/Model/Feed.php
View File

@ -128,7 +128,11 @@ class Feed extends AppModel {
} }
} else { } else {
$uri = $feed['Feed']['url'] . '/manifest.json'; $uri = $feed['Feed']['url'] . '/manifest.json';
$response = $HttpSocket->get($uri, '', $request); try {
$response = $HttpSocket->get($uri, '', $request);
} catch (Exception $e) {
return false;
}
if ($response->code != 200) { if ($response->code != 200) {
return false; return false;
} }
@ -245,6 +249,7 @@ class Feed extends AppModel {
foreach ($feeds as $k => $v) { foreach ($feeds as $k => $v) {
if ($redis->sismember('misp:feed_cache:' . $v['Feed']['id'], md5($value['value']))) { if ($redis->sismember('misp:feed_cache:' . $v['Feed']['id'], md5($value['value']))) {
$data[$key]['feed_correlations'][] = array($v); $data[$key]['feed_correlations'][] = array($v);
} else {
} }
} }
} }
@ -266,10 +271,6 @@ class Feed extends AppModel {
$counter = 0; $counter = 0;
$hashTable = array(); $hashTable = array();
$feedList = array(); $feedList = array();
foreach ($feeds as $feed) {
$feedList[] = $feed['Feed']['id'];
}
$redis->sunionstore('misp:feed_cache:combined', implode(' ', $feedList));
foreach ($objects as $k => $object) { foreach ($objects as $k => $object) {
$hashTable[$object['value']] = md5($object['value']); $hashTable[$object['value']] = md5($object['value']);
if ($redis->sismember('misp:feed_cache:combined', $hashTable[$object['value']])) { if ($redis->sismember('misp:feed_cache:combined', $hashTable[$object['value']])) {
@ -833,6 +834,7 @@ class Feed extends AppModel {
if (!empty($values)) { if (!empty($values)) {
foreach ($values as $k => $value) { foreach ($values as $k => $value) {
$redis->sAdd('misp:feed_cache:' . $feed['Feed']['id'], md5($value['value'])); $redis->sAdd('misp:feed_cache:' . $feed['Feed']['id'], md5($value['value']));
$redis->sAdd('misp:feed_cache:combined', md5($value['value']));
if ($jobId && ($k % 1000 == 0)) { if ($jobId && ($k % 1000 == 0)) {
$job->saveField('message', 'Feed ' . $feed['Feed']['id'] . ': ' . $k . ' values cached.'); $job->saveField('message', 'Feed ' . $feed['Feed']['id'] . ': ' . $k . ' values cached.');
} }
@ -889,8 +891,11 @@ class Feed extends AppModel {
$value = explode('|', $attribute['value']); $value = explode('|', $attribute['value']);
$redis->sAdd('misp:feed_cache:' . $feed['Feed']['id'], md5($value[0])); $redis->sAdd('misp:feed_cache:' . $feed['Feed']['id'], md5($value[0]));
$redis->sAdd('misp:feed_cache:' . $feed['Feed']['id'], md5($value[1])); $redis->sAdd('misp:feed_cache:' . $feed['Feed']['id'], md5($value[1]));
$redis->sAdd('misp:feed_cache:combined', md5($value[0]));
$redis->sAdd('misp:feed_cache:combined', md5($value[1]));
} else { } else {
$redis->sAdd('misp:feed_cache:' . $feed['Feed']['id'], md5($attribute['value'])); $redis->sAdd('misp:feed_cache:' . $feed['Feed']['id'], md5($attribute['value']));
$redis->sAdd('misp:feed_cache:combined', md5($value['value']));
} }
} }
} }