chg: [remote cache] Further progress on caching remote instances

pull/4033/head
iglocska 2019-01-18 15:33:04 +01:00
parent 9c5fbdd7e9
commit eb56fe4e01
5 changed files with 296 additions and 87 deletions

View File

@ -1881,6 +1881,9 @@ class Event extends AppModel
$overrideLimit = false;
}
$event['Attribute'] = $this->Feed->attachFeedCorrelations($event['Attribute'], $user, $event['Event'], $overrideLimit);
if ($user['org_id'] == Configure::read('MISP.host_org_id')) {
$event['Attribute'] = $this->Feed->attachFeedCorrelations($event['Attribute'], $user, $event['Event'], $overrideLimit, 'Server');
}
}
$event = $this->__filterBlockedAttributesByTags($event, $options, $user);
$event['Attribute'] = $this->__attachSharingGroups(!$options['sgReferenceOnly'], $event['Attribute'], $sharingGroupData);
@ -1964,6 +1967,9 @@ class Event extends AppModel
$overrideLimit = false;
}
$event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user, $event['Event'], $overrideLimit);
if ($user['org_id'] == Configure::read('MISP.host_org_id')) {
$event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user, $event['Event'], $overrideLimit, 'Server');
}
}
}
$event['Sighting'] = $this->Sighting->attachToEvent($event, $user);

View File

@ -323,21 +323,34 @@ class Feed extends AppModel
return $data;
}
public function attachFeedCorrelations($objects, $user, &$event, $overrideLimit = false)
public function attachFeedCorrelations($objects, $user, &$event, $overrideLimit = false, $scope = 'Feed')
{
$redis = $this->setupRedis();
if ($redis !== false) {
$params = array(
'recursive' => -1,
'fields' => array('id', 'name', 'url', 'provider', 'source_format')
);
if (!$user['Role']['perm_site_admin']) {
$params['conditions'] = array('Feed.lookup_visible' => 1);
if ($scope === 'Feed') {
$params = array(
'recursive' => -1,
'fields' => array('id', 'name', 'url', 'provider', 'source_format')
);
if (!$user['Role']['perm_site_admin']) {
$params['conditions'] = array('Feed.lookup_visible' => 1);
}
$sources = $this->find('all', $params);
} else {
$params = array(
'recursive' => -1,
'fields' => array('id', 'name', 'url', 'caching_enabled')
);
if (!$user['Role']['perm_site_admin']) {
$params['conditions'] = array('Server.caching_enabled' => 1);
}
$this->Server = ClassRegistry::init('Server');
$sources = $this->Server->find('all', $params);
}
$feeds = $this->find('all', $params);
$counter = 0;
$hashTable = array();
$feedList = array();
$sourceList = array();
$pipe = $redis->multi(Redis::PIPELINE);
$objectsWithFeedHits = array();
$hashTable = array();
@ -352,7 +365,7 @@ class Feed extends AppModel
} else {
$hashTable[$k] = md5($object['value']);
}
$redis->sismember('misp:feed_cache:combined', $hashTable[$k]);
$redis->sismember('misp:' . strtolower($scope) . '_cache:combined', $hashTable[$k]);
$objectKeys[] = $k;
}
$results = array();
@ -374,53 +387,53 @@ class Feed extends AppModel
$hitIds[] = $k;
}
}
foreach ($feeds as $k3 => $feed) {
foreach ($sources as $k3 => $source) {
$pipe = $redis->multi(Redis::PIPELINE);
foreach ($hitIds as $k2 => $k) {
$redis->sismember('misp:feed_cache:' . $feed['Feed']['id'], $hashTable[$k]);
$redis->sismember('misp:' . strtolower($scope) . '_cache:' . $source[$scope]['id'], $hashTable[$k]);
}
$feedHits = $pipe->exec();
foreach ($feedHits as $k4 => $hit) {
$sourceHits = $pipe->exec();
foreach ($sourceHits as $k4 => $hit) {
if ($hit) {
if (!isset($event['Feed'][$feeds[$k3]['Feed']['id']]['id'])) {
if (!isset($event['Feed'][$feeds[$k3]['Feed']['id']])) {
$event['Feed'][$feeds[$k3]['Feed']['id']] = array();
if (!isset($event[$scope][$sources[$k3][$scope]['id']]['id'])) {
if (!isset($event[$scope][$sources[$k3][$scope]['id']])) {
$event[$scope][$sources[$k3][$scope]['id']] = array();
}
$event['Feed'][$feeds[$k3]['Feed']['id']] = array_merge($event['Feed'][$feeds[$k3]['Feed']['id']], $feed['Feed']);
$event[$scope][$sources[$k3][$scope]['id']] = array_merge($event[$scope][$sources[$k3][$scope]['id']], $source[$scope]);
}
$objects[$hitIds[$k4]]['Feed'][] = $feed['Feed'];
$objects[$hitIds[$k4]][$scope][] = $source[$scope];
}
}
if ($feed['Feed']['source_format'] == 'misp') {
if ($scope === 'Server' || $source[$scope]['source_format'] == 'misp') {
$pipe = $redis->multi(Redis::PIPELINE);
$eventUuidHitPosition = array();
$i = 0;
foreach ($objects as $k => $object) {
if (isset($object['Feed'])) {
foreach ($object['Feed'] as $currentFeed) {
if ($feed['Feed']['id'] == $currentFeed['id']) {
if (isset($object[$scope])) {
foreach ($object[$scope] as $currentFeed) {
if ($source[$scope]['id'] == $currentFeed['id']) {
$eventUuidHitPosition[$i] = $k;
$i++;
if (in_array($object['type'], $this->Event->Attribute->getCompositeTypes())) {
$value = explode('|', $object['value']);
$redis->smembers('misp:feed_cache:event_uuid_lookup:' . md5($value[0]));
$redis->smembers('misp:' . strtolower($scope) . '_cache:event_uuid_lookup:' . md5($value[0]));
} else {
$redis->smembers('misp:feed_cache:event_uuid_lookup:' . md5($object['value']));
$redis->smembers('misp:' . strtolower($scope) . '_cache:event_uuid_lookup:' . md5($object['value']));
}
}
}
}
}
$mispFeedHits = $pipe->exec();
foreach ($mispFeedHits as $feedhitPos => $f) {
foreach ($mispFeedHits as $sourcehitPos => $f) {
foreach ($f as $url) {
$urlParts = explode('/', $url);
if (empty($event['Feed'][$urlParts[0]]['event_uuids']) || !in_array($urlParts[1], $event['Feed'][$urlParts[0]]['event_uuids'])) {
$event['Feed'][$urlParts[0]]['event_uuids'][] = $urlParts[1];
if (empty($event[$scope][$urlParts[0]]['event_uuids']) || !in_array($urlParts[1], $event[$scope][$urlParts[0]]['event_uuids'])) {
$event[$scope][$urlParts[0]]['event_uuids'][] = $urlParts[1];
}
foreach ($objects[$eventUuidHitPosition[$feedhitPos]]['Feed'] as $tempKey => $tempFeed) {
foreach ($objects[$eventUuidHitPosition[$sourcehitPos]][$scope] as $tempKey => $tempFeed) {
if ($tempFeed['id'] == $urlParts[0]) {
$objects[$eventUuidHitPosition[$feedhitPos]]['Feed'][$tempKey]['event_uuids'][] = $urlParts[1];
$objects[$eventUuidHitPosition[$sourcehitPos]][$scope][$tempKey]['event_uuids'][] = $urlParts[1];
}
}
}
@ -429,8 +442,8 @@ class Feed extends AppModel
}
}
}
if (!empty($event['Feed'])) {
$event['Feed'] = array_values($event['Feed']);
if (!empty($event[$scope])) {
$event[$scope] = array_values($event[$scope]);
}
return $objects;
}

View File

@ -4296,19 +4296,22 @@ class Server extends AppModel
while ($continue) {
$i++;
$pipe = $redis->multi(Redis::PIPELINE);
$chunk_size = 10000;
$chunk_size = 100000;
$data = $this->__getCachedAttributes($server, $HttpSocket, $chunk_size, $i);
$data = explode(PHP_EOL, trim($data));
if (empty($data) || count($data) < $chunk_size) {
return true;
$continue = false;
}
foreach ($data as $entry) {
list($value, $uuid) = explode(',', $entry);
$redis->sAdd('misp:server_cache:' . $server['Server']['id'], $value);
$redis->sAdd('misp:server_cache:combined', $value);
$redis->sAdd('misp:server_cache:event_uuid_lookup:' . $value, $server['Server']['id'] . '/' . $uuid);
if (!empty($data)) {
foreach ($data as $entry) {
list($value, $uuid) = explode(',', $entry);
$redis->sAdd('misp:server_cache:' . $server['Server']['id'], $value);
$redis->sAdd('misp:server_cache:combined', $value);
$redis->sAdd('misp:server_cache:event_uuid_lookup:' . $value, $server['Server']['id'] . '/' . $uuid);
}
}
if ($jobId) {
$job->saveField('message', 'Server ' . $server['Server']['id'] . ': ' . $i * $chunk_size . ' attributes cached.');
$job->saveField('message', 'Server ' . $server['Server']['id'] . ': ' . ((($i -1) * $chunk_size) + count($data)) . ' attributes cached.');
}
$pipe->exec();
}
@ -4319,7 +4322,7 @@ class Server extends AppModel
{
$filter_rules = array(
'returnFormat' => 'cache',
'includeEventUuid',
'includeEventUuid' => 1,
'page' => $i,
'limit' => $chunk_size
);
@ -4331,4 +4334,116 @@ class Server extends AppModel
}
return $response->body;
}
public function attachFeedCorrelations($objects, $user, &$event, $overrideLimit = false)
{
$redis = $this->setupRedis();
if ($redis !== false) {
$params = array(
'recursive' => -1,
'fields' => array('id', 'name', 'url', 'provider', 'source_format')
);
if (!$user['Role']['perm_site_admin']) {
$params['conditions'] = array('Server.caching_enabled' => 1);
}
$servers = $this->find('all', $params);
$counter = 0;
$hashTable = array();
$serverList = array();
$pipe = $redis->multi(Redis::PIPELINE);
$objectsWithServerHits = array();
$hashTable = array();
$hitIds = array();
$this->Event = ClassRegistry::init('Event');
$objectKeys = array();
foreach ($objects as $k => $object) {
if (in_array($object['type'], $this->Event->Attribute->getCompositeTypes())) {
$value = explode('|', $object['value']);
$hashTable[$k] = md5($value[0]);
$objectKeys[] = $k;
} else {
$hashTable[$k] = md5($object['value']);
}
$redis->sismember('misp:server_cache:combined', $hashTable[$k]);
$objectKeys[] = $k;
}
$results = array();
$results = $pipe->exec();
if (!$overrideLimit && count($objects) > 10000) {
foreach ($results as $k => $result) {
if ($result) {
if (isset($event['ServerCount'])) {
$event['ServerCount']++;
} else {
$event['ServerCount'] = 1;
}
$objects[$k]['ServerHit'] = true;
}
}
} else {
foreach ($results as $k => $result) {
if ($result) {
$hitIds[] = $k;
}
}
foreach ($feeds as $k3 => $feed) {
$pipe = $redis->multi(Redis::PIPELINE);
foreach ($hitIds as $k2 => $k) {
$redis->sismember('misp:feed_cache:' . $feed['Feed']['id'], $hashTable[$k]);
}
$feedHits = $pipe->exec();
foreach ($feedHits as $k4 => $hit) {
if ($hit) {
if (!isset($event['Feed'][$feeds[$k3]['Feed']['id']]['id'])) {
if (!isset($event['Feed'][$feeds[$k3]['Feed']['id']])) {
$event['Feed'][$feeds[$k3]['Feed']['id']] = array();
}
$event['Feed'][$feeds[$k3]['Feed']['id']] = array_merge($event['Feed'][$feeds[$k3]['Feed']['id']], $feed['Feed']);
}
$objects[$hitIds[$k4]]['Feed'][] = $feed['Feed'];
}
}
if ($feed['Feed']['source_format'] == 'misp') {
$pipe = $redis->multi(Redis::PIPELINE);
$eventUuidHitPosition = array();
$i = 0;
foreach ($objects as $k => $object) {
if (isset($object['Feed'])) {
foreach ($object['Feed'] as $currentFeed) {
if ($feed['Feed']['id'] == $currentFeed['id']) {
$eventUuidHitPosition[$i] = $k;
$i++;
if (in_array($object['type'], $this->Event->Attribute->getCompositeTypes())) {
$value = explode('|', $object['value']);
$redis->smembers('misp:feed_cache:event_uuid_lookup:' . md5($value[0]));
} else {
$redis->smembers('misp:feed_cache:event_uuid_lookup:' . md5($object['value']));
}
}
}
}
}
$mispFeedHits = $pipe->exec();
foreach ($mispFeedHits as $feedhitPos => $f) {
foreach ($f as $url) {
$urlParts = explode('/', $url);
if (empty($event['Feed'][$urlParts[0]]['event_uuids']) || !in_array($urlParts[1], $event['Feed'][$urlParts[0]]['event_uuids'])) {
$event['Feed'][$urlParts[0]]['event_uuids'][] = $urlParts[1];
}
foreach ($objects[$eventUuidHitPosition[$feedhitPos]]['Feed'] as $tempKey => $tempFeed) {
if ($tempFeed['id'] == $urlParts[0]) {
$objects[$eventUuidHitPosition[$feedhitPos]]['Feed'][$tempKey]['event_uuids'][] = $urlParts[1];
}
}
}
}
}
}
}
}
if (!empty($event['Feed'])) {
$event['Feed'] = array_values($event['Feed']);
}
return $objects;
}
}

View File

@ -208,53 +208,99 @@
<td class="shortish">
<ul class="inline" style="margin:0px;">
<?php
if (!empty($object['Feed'])):
foreach ($object['Feed'] as $feed):
$popover = '';
foreach ($feed as $k => $v):
if ($k == 'id') continue;
if (is_array($v)) {
foreach ($v as $k2 => $v2) {
$v[$k2] = h($v2);
}
$v = implode('<br />', $v);
} else {
$v = h($v);
if (!empty($object['Feed'])) {
foreach ($object['Feed'] as $feed) {
$popover = '';
foreach ($feed as $k => $v) {
if ($k == 'id') continue;
if (is_array($v)) {
foreach ($v as $k2 => $v2) {
$v[$k2] = h($v2);
}
$v = implode('<br />', $v);
} else {
$v = h($v);
}
$popover .= '<span class=\'bold black\'>' . Inflector::humanize(h($k)) . '</span>: <span class="blue">' . $v . '</span><br />';
}
$liContents = '';
if ($isSiteAdmin) {
if ($feed['source_format'] == 'misp') {
$liContents .= sprintf(
'<form action="%s/feeds/previewIndex/1" method="post" style="margin:0px;line-height:auto;">%s%s</form>',
$baseurl,
sprintf(
'<input type="hidden" name="data[Feed][eventid]" value="%s">',
h(json_encode($feed['event_uuids'], true))
),
sprintf(
'<input type="submit" class="linkButton useCursorPointer" value="%s" data-toggle="popover" data-content="%s" data-trigger="hover" style="margin-right:3px;line-height:normal;vertical-align: text-top;" />',
h($feed['id']),
h($popover)
)
);
} else {
$liContents .= sprintf(
'<form>%s</form>',
sprintf(
'<a href="%s/feeds/previewIndex/%s" style="margin-right:3px;">%s</a>',
$baseurl,
h($feed['id']),
h($feed['id'])
)
);
}
} else {
$liContents .= sprintf(
'<span style="margin-right:3px;">%s</span>',
h($feed['id'])
);
}
echo sprintf(
'<li style="padding-right: 0px; padding-left:0px;">%s</li>',
$liContents
);
}
$popover .= '<span class=\'bold black\'>' . Inflector::humanize(h($k)) . '</span>: <span class="blue">' . $v . '</span><br />';
endforeach;
?>
<li style="padding-right: 0px; padding-left:0px;"><span>
<?php
if ($isSiteAdmin):
if ($feed['source_format'] == 'misp'):
?>
<form action="<?php echo $baseurl; ?>/feeds/previewIndex/1" method="post" style="margin:0px;line-height:auto;">
<input type="hidden" name="data[Feed][eventid]" value="<?php echo h(json_encode($feed['event_uuids'], true)); ?>">
<input type="submit" class="linkButton useCursorPointer" value="<?php echo h($feed['id']); ?>" data-toggle="popover" data-content="<?php echo h($popover);?>" data-trigger="hover" style="margin-right:3px;line-height:normal;vertical-align: text-top;" />
</form>
<?php
else:
?>
<form>
<a href="<?php echo $baseurl; ?>/feeds/previewIndex/<?php echo h($feed['id']); ?>" style="margin-right:3px;"><?php echo h($feed['id']); ?></a>
</form>
<?php
endif;
else:
?>
<span style="margin-right:3px;"><?php echo h($feed['id']);?></span>
<?php
endif;
endforeach;
?>
</li>
<?php
elseif (!empty($object['FeedHit'])):
?>
<span class="icon-ok"></span>
<?php
endif;
}
if (!empty($object['Server'])) {
$liContents = '';
foreach ($object['Server'] as $server) {
$popover = '';
foreach ($server as $k => $v) {
if ($k == 'id') continue;
if (is_array($v)) {
foreach ($v as $k2 => $v2) {
$v[$k2] = h($v2);
}
$v = implode('<br />', $v);
} else {
$v = h($v);
}
$popover .= '<span class=\'bold black\'>' . Inflector::humanize(h($k)) . '</span>: <span class="blue">' . $v . '</span><br />';
}
foreach ($server['event_uuids'] as $k => $event_uuid) {
if ($isSiteAdmin) {
$liContents .= sprintf(
'<a href="%s/servers/previewEvent/%s/%s" data-toggle="popover" data-content="%s" data-trigger="hover">%s</a>',
$baseurl,
h($server['id']),
h($event_uuid),
h($popover),
'S' . h($server['id']) . ':' . ($k + 1)
);
} else {
$liContents .= sprintf(
'<span style="margin-right:3px;">%s</span>',
'S' . h($server['id']) . ':' . ($k + 1)
);
}
}
echo sprintf(
'<li style="padding-right: 0px; padding-left:0px;">%s</li>',
$liContents
);
}
}
?>
</ul>
</td>

View File

@ -347,6 +347,35 @@
<?php
endif;
endif;
if (!empty($event['Server']) || !empty($event['Event']['ServerCount'])):
?>
<h3>Related Server</h3>
<?php
if (!empty($event['Server'])):
foreach ($event['Server'] as $relatedServer):
$relatedData = array('Name' => $relatedServer['name'], 'URL' => $relatedServer['url']);
$popover = '';
foreach ($relatedData as $k => $v) {
$popover .= '<span class=\'bold\'>' . h($k) . '</span>: <span class="blue">' . h($v) . '</span><br />';
}
?>
<span style="white-space: nowrap;">
<form action="<?php echo $baseurl; ?>/servers/previewIndex/<?php echo h($relatedServer['id']); ?>" method="post" style="margin:0px;">
<input type="hidden" name="data[Feed][eventid]" value="<?php echo h(json_encode($relatedServer['event_uuids'], true)); ?>">
<input type="submit" class="linkButton useCursorPointer" value="<?php echo h($relatedServer['name']) . ' (' . $relatedServer['id'] . ')'; ?>" data-toggle="popover" data-content="<?php echo h($popover); ?>" data-trigger="hover" />
</form>
</span>
<?php
endforeach;
elseif (!empty($event['Event']['FeedCount'])):
?>
<span>
<?php echo __('This event has ');?><span class="bold"><?php echo h($event['Event']['FeedCount']); ?></span>
<?php echo __('correlations with data contained within the various feeds, however, due to the large number of attributes the actual feed correlations are not shown. Click <a href="%s\/overrideLimit:1">here</a> to refresh the page with the feed data loaded.', h($this->here));?>
</span>
<?php
endif;
endif;
?>
<?php if (!empty($event['Event']['warnings'])): ?>
<div class="warning_container" style="width:80%;">