mirror of https://github.com/MISP/MISP
new: [server caching] Initial version WIP
iglocska
parent
a9a47fb46c
commit
9c5fbdd7e9
|
|
@ -179,6 +179,54 @@ class ServerShell extends AppShell
|
|||
echo $outcome['message'] . PHP_EOL;
|
||||
}
|
||||
|
||||
public function cacheServer() {
|
||||
if (empty($this->args[0]) || empty($this->args[1])) {
|
||||
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['cacheServer'] . PHP_EOL);
|
||||
}
|
||||
$userId = $this->args[0];
|
||||
$user = $this->User->getAuthUser($userId);
|
||||
if (empty($user)) die('Invalid user.' . PHP_EOL);
|
||||
$scope = $this->args[1];
|
||||
if (!empty($this->args[2])) {
|
||||
$jobId = $this->args[2];
|
||||
} else {
|
||||
$this->Job->create();
|
||||
$data = array(
|
||||
'worker' => 'default',
|
||||
'job_type' => 'cache_servers',
|
||||
'job_input' => 'Server: ' . $id,
|
||||
'status' => 0,
|
||||
'retries' => 0,
|
||||
'org' => $user['Organisation']['name'],
|
||||
'message' => 'Starting server caching.',
|
||||
);
|
||||
$this->Job->save($data);
|
||||
$jobId = $this->Job->id;
|
||||
}
|
||||
$this->Job->read(null, $jobId);
|
||||
$result = $this->Server->cacheServerInitiator($user, $id, $jobId);
|
||||
$this->Job->id = $jobId;
|
||||
if ($result !== true) {
|
||||
$message = 'Job Failed. Reason: ';
|
||||
$this->Job->save(array(
|
||||
'id' => $jobId,
|
||||
'message' => $message . $result,
|
||||
'progress' => 0,
|
||||
'status' => 3
|
||||
));
|
||||
} else {
|
||||
$message = 'Job done.';
|
||||
$this->Job->save(array(
|
||||
'id' => $jobId,
|
||||
'message' => $message,
|
||||
'progress' => 100,
|
||||
'status' => 4
|
||||
));
|
||||
}
|
||||
echo $message . PHP_EOL;
|
||||
}
|
||||
|
||||
|
||||
public function cacheFeed() {
|
||||
if (empty($this->args[0]) || empty($this->args[1])) {
|
||||
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['cacheFeed'] . PHP_EOL);
|
||||
|
|
|
|||
|
|
@ -236,6 +236,7 @@ class ServersController extends AppController
|
|||
$defaults = array(
|
||||
'push' => 0,
|
||||
'pull' => 0,
|
||||
'caching_enabled' => 0,
|
||||
'json' => '[]',
|
||||
'push_rules' => '[]',
|
||||
'pull_rules' => '[]',
|
||||
|
|
@ -422,7 +423,7 @@ class ServersController extends AppController
|
|||
}
|
||||
if (!$fail) {
|
||||
// say what fields are to be updated
|
||||
$fieldList = array('id', 'url', 'push', 'pull', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
|
||||
$fieldList = array('id', 'url', 'push', 'pull', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
|
||||
$this->request->data['Server']['id'] = $id;
|
||||
if (isset($this->request->data['Server']['authkey']) && "" != $this->request->data['Server']['authkey']) {
|
||||
$fieldList[] = 'authkey';
|
||||
|
|
@ -1830,4 +1831,43 @@ misp.direct_call(relative_path, body)
|
|||
}
|
||||
}
|
||||
|
||||
public function cache($id = 'all')
|
||||
{
|
||||
if (Configure::read('MISP.background_jobs')) {
|
||||
$this->loadModel('Job');
|
||||
$this->Job->create();
|
||||
$data = array(
|
||||
'worker' => 'default',
|
||||
'job_type' => 'cache_servers',
|
||||
'job_input' => intval($id) ? $id : 'all',
|
||||
'status' => 0,
|
||||
'retries' => 0,
|
||||
'org' => $this->Auth->user('Organisation')['name'],
|
||||
'message' => __('Starting server caching.'),
|
||||
);
|
||||
$this->Job->save($data);
|
||||
$jobId = $this->Job->id;
|
||||
$process_id = CakeResque::enqueue(
|
||||
'default',
|
||||
'ServerShell',
|
||||
array('cacheServer', $this->Auth->user('id'), $id, $jobId),
|
||||
true
|
||||
);
|
||||
$this->Job->saveField('process_id', $process_id);
|
||||
$message = 'Server caching job initiated.';
|
||||
} else {
|
||||
$result = $this->Server->cacheServerInitiator($this->Auth->user(), $id);
|
||||
if (!$result) {
|
||||
$this->Flash->error(__('Caching the servers has failed.'));
|
||||
$this->redirect(array('action' => 'index'));
|
||||
}
|
||||
$message = __('Caching the servers has successfully completed.');
|
||||
}
|
||||
if ($this->_isRest()) {
|
||||
return $this->RestResponse->saveSuccessResponse('Server', 'cache', false, $this->response->type(), $message);
|
||||
} else {
|
||||
$this->Flash->error($message);
|
||||
$this->redirect(array('action' => 'index'));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -72,7 +72,7 @@ class AppModel extends Model
|
|||
7 => false, 8 => false, 9 => false, 10 => false, 11 => false, 12 => false,
|
||||
13 => false, 14 => false, 15 => false, 18 => false, 19 => false, 20 => false,
|
||||
21 => false, 22 => false, 23 => false, 24 => false, 25 => false, 26 => false,
|
||||
27 => false
|
||||
27 => false, 28 => false
|
||||
);
|
||||
|
||||
public function afterSave($created, $options = array())
|
||||
|
|
@ -1086,6 +1086,9 @@ class AppModel extends Model
|
|||
case 27:
|
||||
$sqlArray[] = 'ALTER TABLE `tags` CHANGE `org_id` `org_id` int(11) NOT NULL DEFAULT 0;';
|
||||
break;
|
||||
case 28:
|
||||
$sqlArray[] = "ALTER TABLE `servers` ADD `caching_enabled` tinyint(1) NOT NULL DEFAULT 0;";
|
||||
break;
|
||||
case 'fixNonEmptySharingGroupID':
|
||||
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
|
||||
$sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
|
||||
|
|
|
|||
|
|
@ -4247,4 +4247,88 @@ class Server extends AppModel
|
|||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public function cacheServerInitiator($user, $id = 'all', $jobId = false)
|
||||
{
|
||||
$params = array(
|
||||
'conditions' => array('caching_enabled' => 1),
|
||||
'recursive' => -1
|
||||
);
|
||||
$redis = $this->setupRedis();
|
||||
if ($redis === false) {
|
||||
return 'Redis not reachable.';
|
||||
}
|
||||
if ($id !== 'all') {
|
||||
$params['conditions']['Server.id'] = $id;
|
||||
} else {
|
||||
$redis->del('misp:server_cache:combined');
|
||||
$redis->del('misp:server_cache:event_uuid_lookup:');
|
||||
}
|
||||
$servers = $this->find('all', $params);
|
||||
if ($jobId) {
|
||||
$job = ClassRegistry::init('Job');
|
||||
$job->id = $jobId;
|
||||
if (!$job->exists()) {
|
||||
$jobId = false;
|
||||
}
|
||||
}
|
||||
foreach ($servers as $k => $server) {
|
||||
$this->__cacheInstance($server, $redis, $jobId);
|
||||
if ($jobId) {
|
||||
$job->saveField('progress', 100 * $k / count($servers));
|
||||
$job->saveField('message', 'Server ' . $server['Server']['id'] . ' cached.');
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private function __cacheInstance($server, $redis, $jobId = false)
|
||||
{
|
||||
$continue = true;
|
||||
$i = 0;
|
||||
if ($jobId) {
|
||||
$job = ClassRegistry::init('Job');
|
||||
$job->id = $jobId;
|
||||
}
|
||||
$redis->del('misp:server_cache:' . $server['Server']['id']);
|
||||
$HttpSocket = null;
|
||||
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
|
||||
while ($continue) {
|
||||
$i++;
|
||||
$pipe = $redis->multi(Redis::PIPELINE);
|
||||
$chunk_size = 10000;
|
||||
$data = $this->__getCachedAttributes($server, $HttpSocket, $chunk_size, $i);
|
||||
if (empty($data) || count($data) < $chunk_size) {
|
||||
return true;
|
||||
}
|
||||
foreach ($data as $entry) {
|
||||
list($value, $uuid) = explode(',', $entry);
|
||||
$redis->sAdd('misp:server_cache:' . $server['Server']['id'], $value);
|
||||
$redis->sAdd('misp:server_cache:combined', $value);
|
||||
$redis->sAdd('misp:server_cache:event_uuid_lookup:' . $value, $server['Server']['id'] . '/' . $uuid);
|
||||
}
|
||||
if ($jobId) {
|
||||
$job->saveField('message', 'Server ' . $server['Server']['id'] . ': ' . $i * $chunk_size . ' attributes cached.');
|
||||
}
|
||||
$pipe->exec();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private function __getCachedAttributes($server, $HttpSocket, $chunk_size, $i)
|
||||
{
|
||||
$filter_rules = array(
|
||||
'returnFormat' => 'cache',
|
||||
'includeEventUuid',
|
||||
'page' => $i,
|
||||
'limit' => $chunk_size
|
||||
);
|
||||
$request = $this->setupSyncRequest($server);
|
||||
try {
|
||||
$response = $HttpSocket->post($server['Server']['url'] . '/attributes/restSearch.json', json_encode($filter_rules), $request);
|
||||
} catch (SocketException $e) {
|
||||
return $e->getMessage();
|
||||
}
|
||||
return $response->body;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
<fieldset>
|
||||
<legend><?php echo __('Add Server');?></legend>
|
||||
<?php
|
||||
echo '<h4 class="input clear">' . __('Instance identification') . '</h4>';
|
||||
echo $this->Form->input('url', array(
|
||||
'label' => __('Base URL'),
|
||||
));
|
||||
|
|
@ -25,8 +26,9 @@
|
|||
endif;
|
||||
?>
|
||||
<div class="input clear" style="width:100%;">
|
||||
<hr />
|
||||
<p class="red"><?php echo __('Information about the organisation that will receive the events, typically the remote instance\'s host organisation.');?></p>
|
||||
<hr />
|
||||
<h4><?php echo __('Instance ownership and credentials'); ?></h4>
|
||||
<p class="red"><?php echo __('Information about the organisation that will receive the events, typically the remote instance\'s host organisation.');?></p>
|
||||
</div>
|
||||
<div class = "input clear"></div>
|
||||
<?php
|
||||
|
|
@ -61,38 +63,25 @@
|
|||
endif;
|
||||
echo $this->Form->input('authkey', array(
|
||||
));
|
||||
?>
|
||||
<div class = "input clear" style="width:100%;"><hr /></div>
|
||||
<div class = "input clear"></div>
|
||||
<?php
|
||||
echo $this->Form->input('push', array(
|
||||
));
|
||||
|
||||
echo $this->Form->input('pull', array(
|
||||
));
|
||||
?>
|
||||
<div class = "input clear"></div>
|
||||
<?php
|
||||
echo '<div class = "input clear" style="width:100%;"><hr /></div>';
|
||||
echo '<h4 class="input clear">' . __('Enabled synchronisation methods') . '</h4>';
|
||||
echo $this->Form->input('push', array());
|
||||
echo $this->Form->input('pull', array());
|
||||
echo $this->Form->input('caching_enabled', array());
|
||||
echo '<div class = "input clear" style="width:100%;"><hr /></div>';
|
||||
echo $this->Form->input('unpublish_event', array(
|
||||
'type' => 'checkbox',
|
||||
));
|
||||
?>
|
||||
<div class = "input clear"></div>
|
||||
<?php
|
||||
echo '<div class="input clear"></div>';
|
||||
echo $this->Form->input('publish_without_email', array(
|
||||
'type' => 'checkbox',
|
||||
));
|
||||
?>
|
||||
<div class = "input clear"></div>
|
||||
<?php
|
||||
echo '<div class="input clear"></div>';
|
||||
echo $this->Form->input('self_signed', array(
|
||||
'type' => 'checkbox',
|
||||
));
|
||||
?>
|
||||
<div class = "input clear"></div>
|
||||
<?php
|
||||
echo '<div class="input clear"></div>';
|
||||
echo $this->Form->input('skip_proxy', array('type' => 'checkbox', 'label' => 'Skip proxy (if applicable)'));
|
||||
|
||||
echo $this->Form->input('Server.submitted_cert', array(
|
||||
'label' => '<b>' . __('Server certificate file') . '</b>',
|
||||
'type' => 'file',
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
<fieldset>
|
||||
<legend><?php echo __('Edit Server');?></legend>
|
||||
<?php
|
||||
echo '<h4 class="input clear">' . __('Instance identification') . '</h4>';
|
||||
echo $this->Form->input('url', array(
|
||||
'label' => __('Base URL'),
|
||||
));
|
||||
|
|
@ -25,7 +26,12 @@
|
|||
endif;
|
||||
?>
|
||||
<div class="input clear"></div>
|
||||
|
||||
<div class="input clear" style="width:100%;">
|
||||
<hr />
|
||||
<h4><?php echo __('Instance ownership and credentials'); ?></h4>
|
||||
<p class="red"><?php echo __('Information about the organisation that will receive the events, typically the remote instance\'s host organisation.');?></p>
|
||||
</div>
|
||||
<div class = "input clear"></div>
|
||||
<?php
|
||||
echo $this->Form->input('organisation_type', array(
|
||||
'label' => __('Organisation Type'),
|
||||
|
|
@ -63,40 +69,29 @@
|
|||
<label for="ServerExternalUuid"><?php echo __('Remote Organisation\'s Uuid');?></label>
|
||||
<input type="text" id="ServerExternalUuid" <?php if (isset($this->request->data['Server']['external_uuid'])) echo 'value="' . $this->request->data['Server']['external_uuid'] . '"';?>>
|
||||
</div>
|
||||
<div class = "input clear"></div>
|
||||
<?php
|
||||
echo '<div class="input clear"></div>';
|
||||
echo $this->Form->input('authkey', array(
|
||||
'placeholder' => __('Leave empty to use current key')
|
||||
));
|
||||
?>
|
||||
<div class = "input clear"></div>
|
||||
<?php
|
||||
echo $this->Form->input('push', array(
|
||||
));
|
||||
|
||||
echo $this->Form->input('pull', array(
|
||||
));
|
||||
?>
|
||||
<div class = "input clear"></div>
|
||||
<?php
|
||||
echo '<div class = "input clear" style="width:100%;"><hr /></div>';
|
||||
echo '<h4 class="input clear">' . __('Enabled synchronisation methods') . '</h4>';
|
||||
echo $this->Form->input('push', array());
|
||||
echo $this->Form->input('pull', array());
|
||||
echo $this->Form->input('caching_enabled', array());
|
||||
echo '<div class = "input clear" style="width:100%;"><hr /><h4>' . __('Misc settings') . '</h4></div>';
|
||||
echo $this->Form->input('unpublish_event', array(
|
||||
'type' => 'checkbox',
|
||||
));
|
||||
?>
|
||||
<div class = "input clear"></div>
|
||||
<?php
|
||||
echo '<div class="input clear"></div>';
|
||||
echo $this->Form->input('publish_without_email', array(
|
||||
'type' => 'checkbox',
|
||||
));
|
||||
?>
|
||||
<div class = "input clear"></div>
|
||||
<?php
|
||||
echo '<div class="input clear"></div>';
|
||||
echo $this->Form->input('self_signed', array(
|
||||
'type' => 'checkbox',
|
||||
));
|
||||
?>
|
||||
<div class = "input clear"></div>
|
||||
<?php
|
||||
echo '<div class="input clear"></div>';
|
||||
echo $this->Form->input('skip_proxy', array('type' => 'checkbox', 'label' => 'Skip proxy (if applicable)'));
|
||||
?>
|
||||
<div class="clear">
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@
|
|||
<th><?php echo $this->Paginator->sort('internal');?></th>
|
||||
<th><?php echo $this->Paginator->sort('push');?></th>
|
||||
<th><?php echo $this->Paginator->sort('pull');?></th>
|
||||
<th><?php echo $this->Paginator->sort('caching_enabled', 'Cache');?></th>
|
||||
<th><?php echo $this->Paginator->sort('unpublish_event (push event)');?></th>
|
||||
<th><?php echo $this->Paginator->sort('publish_without_email (pull event)');?></th>
|
||||
<th><?php echo $this->Paginator->sort('url');?></th>
|
||||
|
|
@ -71,7 +72,8 @@ foreach ($servers as $server):
|
|||
<td id="connection_test_<?php echo $server['Server']['id'];?>"><span role="button" tabindex="0" aria-label="<?php echo __('Test the connection to the remote instance');?>" title="<?php echo __('Test the connection to the remote instance');?>" class="btn btn-primary" style="line-height:10px; padding: 4px 4px;" onClick="testConnection('<?php echo $server['Server']['id'];?>');"><?php echo __('Run');?></span></td>
|
||||
<td><span class="<?php echo ($server['Server']['internal']? 'icon-ok' : 'icon-remove'); ?>" title="<?php echo ($server['Server']['internal']? __('Internal instance that ignores distribution level degradation *WARNING: Only use this setting if you have several internal instances and the sync link is to an internal extension of the current MISP community*') : __('Normal sync link to an external MISP instance. Distribution degradation will follow the normal rules.')); ?>"></span></td>
|
||||
<td><span class="<?php echo ($server['Server']['push']? 'icon-ok' : 'icon-remove'); ?>"></span><span class="short <?php if (!$server['Server']['push'] || empty($ruleDescription['push'])) echo "hidden"; ?>" data-toggle="popover" title="Distribution List" data-content="<?php echo $ruleDescription['push']; ?>"> (<?php echo __('Rules');?>)</span></td>
|
||||
<td><span class="<?php echo ($server['Server']['pull']? 'icon-ok' : 'icon-remove'); ?>"></span><span class="short <?php if (!$server['Server']['pull'] || empty($ruleDescription['pull'])) echo "hidden"; ?>" data-toggle="popover" title="Distribution List" data-content="<?php echo $ruleDescription['pull']; ?>"> (<?php echo __('Rules');?>)</span>
|
||||
<td><span class="<?php echo ($server['Server']['pull']? 'icon-ok' : 'icon-remove'); ?>"></span><span class="short <?php if (!$server['Server']['pull'] || empty($ruleDescription['pull'])) echo "hidden"; ?>" data-toggle="popover" title="Distribution List" data-content="<?php echo $ruleDescription['pull']; ?>"> (<?php echo __('Rules');?>)</span></td>
|
||||
<td><span class="<?php echo ($server['Server']['caching_enabled']? 'icon-ok' : 'icon-remove'); ?>"></span></td>
|
||||
<td class="short"><span class="<?php echo ($server['Server']['unpublish_event'] ? 'icon-ok' : 'icon-remove'); ?>"></span></td>
|
||||
<td class="short"><span class="<?php echo ($server['Server']['publish_without_email'] ? 'icon-ok' : 'icon-remove'); ?>"></span></td>
|
||||
<td><?php echo h($server['Server']['url']); ?> </td>
|
||||
|
|
@ -91,6 +93,9 @@ foreach ($servers as $server):
|
|||
if ($server['Server']['push']) {
|
||||
echo $this->Html->link('', array('action' => 'push', $server['Server']['id'], 'full'), array('class' => 'icon-upload', 'title' => __('Push all')));
|
||||
}
|
||||
if ($server['Server']['caching_enabled']) {
|
||||
echo $this->Html->link('', array('action' => 'cache', $server['Server']['id']), array('class' => 'icon-download-alt', 'title' => __('Cache instance')));
|
||||
}
|
||||
?>
|
||||
|
||||
<?php
|
||||
|
|
|
|||
Loading…
Reference in New Issue