MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Changes to the logging and scheduling 

- Scheduled tasks for pull / push now working as intended
- Rescheduling of all tasks fixed
- protection against the rescheduled task ending up in the past

- further event history fixes
- fixed lots of erroneous logging
- performance improvement with logging (no longer loading controllers for no reason)
- logging extra actions that weren't logged before (proposal accept / discard, server pull / push)
pull/217/head
iglocska 2014-01-29 15:52:09 +01:00
parent 2740da9094
commit 70c7f650f6
11 changed files with 241 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
app/Console/Command/EventShell.php
View File

@ -139,17 +139,21 @@ class EventShell extends AppShell
$extra = $this->args[3];
$eventIds = $this->Event->fetchEventIds($org, $isSiteAdmin);
$eventCount = count($eventIds);
$attributes[] = array();
foreach ($eventIds as $k => $eventId) {
$attributes = $this->Event->csv($org, $isSiteAdmin, $eventId['Event']['id'], $extra);
$attributes = array_merge($this->Event->csv($org, $isSiteAdmin, $eventId['Event']['id'], $extra), $attributes);
if ($k % 10 == 0) {
$this->Job->saveField('progress', $k / $eventCount * 80);
}
}
$this->loadModel('Whitelist');
$final = array();
$final[] = 'uuid,event_id,category,type,value';
$attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true);
foreach ($attributes as $attribute) {
$final[] = $attribute['Attribute']['uuid'] . ',' . $attribute['Attribute']['event_id'] . ',' . $attribute['Attribute']['category'] . ',' . $attribute['Attribute']['type'] . ',' . $attribute['Attribute']['value'];
if (!empty($attribute)) {
$final[] = $attribute['Attribute']['uuid'] . ',' . $attribute['Attribute']['event_id'] . ',' . $attribute['Attribute']['category'] . ',' . $attribute['Attribute']['type'] . ',' . $attribute['Attribute']['value'];
}
}
$dir = new Folder(APP . DS . '/tmp/cached_exports/' . $extra);
if ($isSiteAdmin) {
@ -256,7 +260,8 @@ class EventShell extends AppShell
// If the next execution time and the timestamp don't match, it means that this task is no longer valid as the time for the execution has since being scheduled
// been updated.
if ($task['Task']['next_execution_time'] != $timestamp) return;
$task['Task']['scheduled_time'] = date('H:i', $task['Task']['next_execution_time']);
$this->Task->save($task);
$orgs = $this->User->getOrgs();
// Queue a set of exports for admins. This "ADMIN" organisation. The organisation of the admin users doesn't actually matter, it is only used to indentify
@ -272,10 +277,14 @@ class EventShell extends AppShell
}
$task['Task']['message'] = $i . ' jobs started at ' . date('d/m/Y - H:i:s') . '.';
if ($task['Task']['timer'] > 0) {
$task['Task']['next_execution_time'] = strtotime('+' . $task['Task']['timer'] . ' hours', $task['Task']['next_execution_time']);
$time = time();
// Keep adding the timer's time interval until we get a date that is in the future! We don't want to keep queuing tasks in the past since they will execute until it catches up.
while ($task['Task']['next_execution_time'] < $time) {
$task['Task']['next_execution_time'] = strtotime('+' . $task['Task']['timer'] . ' hours', $task['Task']['next_execution_time']);
}
$task['Task']['scheduled_time'] = $this->Task->breakTime($task['Task']['scheduled_time'], $task['Task']['timer']);
$this->Task->save($task);
}
$this->Task->save($task);
}
public function publish() {

137
app/Console/Command/ServerShell.php
View File

@ -53,17 +53,148 @@ class ServerShell extends AppShell
$technique = $this->args[1];
$jobId = $this->args[2];
$this->Job->read(null, $jobId);
$server = $this->Server->find(null, $serverId);
$server = $this->Server->read(null, $serverId);
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
$result = $this->Server->push($id, 'full', $jobId, $HttpSocket);
$result = $this->Server->push($serverId, 'full', $jobId, $HttpSocket);
$this->Job->save(array(
'id' => $jobId,
'message' => 'Job done.',
'progress' => 100,
'status' => 4
));
if (isset($this->args[3])) {
$this->Task->id = $this->args[4];
$this->Task->saveField('message', 'Job(s) started at ' . date('d/m/Y - H:i:s') . '.');
}
}
public function enqueuePull() {
$timestamp = $this->args[0];
$userId = $this->args[1];
$taskId = $this->args[2];
$task = $this->Task->read(null, $taskId);
if ($timestamp != $task['Task']['next_execution_time']) {
return;
}
$this->User->recursive = -1;
$user = $this->User->read(array('id', 'org', 'email'), $userId);
$servers = $this->Server->find('all', array('recursive' => -1, 'conditions' => array('push' => 1)));
$count = count($servers);
foreach ($servers as $k => $server) {
$this->Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'push',
'job_input' => 'Server: ' . $server['Server']['id'],
'retries' => 0,
'org' => $user['User']['org'],
'process_id' => $this->Task->data['Task']['job_id'],
'message' => 'Pushing.',
);
$this->Job->save($data);
$jobId = $this->Job->id;
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$result = $this->Server->pull($user['User'], null, 'full', $server, $jobId);
$this->Job->save(array(
'id' => $jobId,
'message' => 'Job done.',
'progress' => 100,
'status' => 4
));
if (is_numeric($result[0])) {
switch ($result[0]) {
case '1' :
$this->Job->saveField('message', 'Not authorised. This is either due to an invalid auth key, or due to the sync user not having authentication permissions enabled on the remote server.');
break;
case '2' :
$this->Job->saveField('message', $result[1]);
break;
case '3' :
$this->Job->saveField('message', 'Sorry, incremental pushes are not yet implemented.');
break;
case '4' :
$this->Job->saveField('message', 'Invalid technique chosen.');
break;
}
}
}
$task['Task']['message'] = count($servers) . ' job(s) completed at ' . date('d/m/Y - H:i:s') . '.';
if ($task['Task']['timer'] > 0) {
$time = time();
// Keep adding the timer's time interval until we get a date that is in the future! We don't want to keep queuing tasks in the past since they will execute until it catches up.
while ($task['Task']['next_execution_time'] < $time) {
$task['Task']['next_execution_time'] = strtotime('+' . $task['Task']['timer'] . ' hours', $task['Task']['next_execution_time']);
}
$task['Task']['scheduled_time'] = $this->Task->breakTime($task['Task']['scheduled_time'], $task['Task']['timer']);
$task['Task']['scheduled_time'] = date('H:i', $task['Task']['next_execution_time']);
// Now that we have figured out when the next execution should happen, it's time to enqueue it.
$process_id = CakeResque::enqueueAt(
$task['Task']['next_execution_time'],
'default',
'ServerShell',
array('enqueuePull', $task['Task']['next_execution_time'],$userId, $taskId),
true
);
$task['Task']['job_id'] = $process_id;
$this->Task->save($task);
}
}
public function enqueuePush() {
$timestamp = $this->args[0];
$taskId = $this->args[1];
$org = $this->args[2];
$this->Task->id = $taskId;
$task = $this->Task->read(null, $taskId);
if ($timestamp != $task['Task']['next_execution_time']) {
return;
}
$servers = $this->Server->find('all', array('recursive' => -1, 'conditions' => array('push' => 1)));
$count = count($servers);
foreach ($servers as $k => $server) {
$this->Job->create();
$data = array(
'worker' => 'default',
'job_type' => 'push',
'job_input' => 'Server: ' . $server['Server']['id'],
'retries' => 0,
'org' => $org,
'process_id' => $this->Task->data['Task']['job_id'],
'message' => 'Pushing.',
);
$this->Job->save($data);
$jobId = $this->Job->id;
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
$result = $this->Server->push($server['Server']['id'], 'full', $jobId, $HttpSocket);
}
$task['Task']['message'] = count($servers) . ' job(s) completed at ' . date('d/m/Y - H:i:s') . '.';
if ($task['Task']['timer'] > 0) {
$time = time();
// Keep adding the timer's time interval until we get a date that is in the future! We don't want to keep queuing tasks in the past since they will execute until it catches up.
while ($task['Task']['next_execution_time'] < $time) {
$task['Task']['next_execution_time'] = strtotime('+' . $task['Task']['timer'] . ' hours', $task['Task']['next_execution_time']);
}
$task['Task']['scheduled_time'] = $this->Task->breakTime($task['Task']['scheduled_time'], $task['Task']['timer']);
$task['Task']['scheduled_time'] = date('H:i', $task['Task']['next_execution_time']);
// Now that we have figured out when the next execution should happen, it's time to enqueue it.
$process_id = CakeResque::enqueueAt(
$task['Task']['next_execution_time'],
'default',
'ServerShell',
array('enqueuePush', $task['Task']['next_execution_time'], $taskId, $org),
true
);
$task['Task']['job_id'] = $process_id;
$this->Task->save($task);
}
}
}

2
app/Controller/ServersController.php
View File

@ -103,7 +103,7 @@ class ServersController extends AppController {
$fieldList[] = 'authkey';
// Save the data
if ($this->Server->save($this->request->data, true, $fieldList)) {
if (isset($this->request->data['Server']['submitted_cert'])) {
if (isset($this->request->data['Server']['submitted_cert']) && $this->request->data['Server']['submitted_cert']['size'] != 0) {
$this->__saveCert($this->request->data, $this->Server->id);
}
$this->Session->setFlash(__('The server has been saved'));

2
app/Controller/ShadowAttributesController.php
View File

@ -117,7 +117,7 @@ class ShadowAttributesController extends AppController {
'model_id' => $id,
'email' => $this->Auth->user('email'),
'action' => 'accept',
'title' => 'Proposal (' . $shadow['id'] . ') of ' . $shadow['org'] . ' to Attribute (' . $shadow['old_id'] . ') of Event (' . $shadow['event_id'] . ') accepted',
'title' => 'Proposal (' . $shadow['id'] . ') of ' . $shadow['org'] . ' to Attribute (' . $shadow['old_id'] . ') of Event (' . $shadow['event_id'] . ') accepted - ' . $shadow['category'] . '/' . $shadow['type'] . ' ' . $shadow['value'],
));
$this->Session->setFlash(__('Proposed change accepted', true), 'default', array());
$this->redirect(array('controller' => 'events', 'action' => 'view', $activeAttribute['Attribute']['event_id']));

47
app/Controller/TasksController.php
View File

@ -3,9 +3,9 @@
App::uses('AppController', 'Controller');
/**
* Jobs Controller
* Tasks Controller
*
* @property Job $Job
* @property Task $Task
*/
class TasksController extends AppController {
public $components = array('Security' ,'RequestHandler', 'Session');
@ -50,11 +50,9 @@ class TasksController extends AppController {
if ($this->request->is('post') || $this->request->is('put')) {
$tasks = $this->Task->find('all', array('fields' => array('id', 'timer', 'scheduled_time', 'type', 'next_execution_time')));
foreach ($tasks as $k => $task) {
if ($this->request->data['Task'][$task['Task']['id']]['timer'] == $task['Task']['timer']) unset($this->request->data['Task'][$task['Task']['id']]['timer']);
if ($this->request->data['Task'][$task['Task']['id']]['scheduled_time'] == $task['Task']['scheduled_time']) unset($this->request->data['Task'][$task['Task']['id']]['scheduled_time']);
if (empty($this->request->data['Task'][$task['Task']['id']])) {
unset($this->request->data['Task'][$task['Task']['id']]);
} else {
if ($this->request->data['Task'][$task['Task']['id']]['timer'] !== $task['Task']['timer'] ||
$this->request->data['Task'][$task['Task']['id']]['scheduled_time'] !== $task['Task']['scheduled_time'] ||
$this->request->data['Task'][$task['Task']['id']]['next_execution_time'] !== date("Y-m-d", $task['Task']['next_execution_time'])) {
$this->request->data['Task'][$task['Task']['id']]['id'] = $task['Task']['id'];
if (isset($this->request->data['Task'][$task['Task']['id']]['next_execution_time'])) {
$temp = $this->request->data['Task'][$task['Task']['id']]['next_execution_time'];
@ -67,7 +65,7 @@ class TasksController extends AppController {
$this->request->data['Task'][$task['Task']['id']]['next_execution_time'] = strtotime($temp . ' ' . $task['Task']['scheduled_time']);
}
// schedule task
$this->_jobScheduler($task['Task']['type'], $this->request->data['Task'][$task['Task']['id']]['next_execution_time']);
$this->_jobScheduler($task['Task']['type'], $this->request->data['Task'][$task['Task']['id']]['next_execution_time'], $task['Task']['id']);
$this->Task->save($this->request->data['Task'][$task['Task']['id']]);
}
}
@ -80,11 +78,13 @@ class TasksController extends AppController {
return strtotime(date("d/m/Y") . ' 00:00:00');
}
private function _jobScheduler($type, $timestamp) {
if ($type === 'cache_exports') $this->_cacheScheduler($timestamp);
private function _jobScheduler($type, $timestamp, $id) {
if ($type === 'cache_exports') $this->_cacheScheduler($timestamp, $id);
if ($type === 'pull_all') $this->_pullScheduler($timestamp, $id);
if ($type === 'push_all') $this->_pushScheduler($timestamp, $id);
}
private function _cacheScheduler($timestamp) {
private function _cacheScheduler($timestamp, $id) {
CakeResque::enqueueAt(
$timestamp,
'cache',
@ -93,4 +93,29 @@ class TasksController extends AppController {
true
);
}
private function _pushScheduler($timestamp, $id) {
$process_id = CakeResque::enqueueAt(
$timestamp,
'default',
'ServerShell',
array('enqueuePush', $timestamp, $id, $this->Auth->user('org')),
true
);
$this->Task->id = $id;
$this->Task->saveField('job_id', $process_id);
}
private function _pullScheduler($timestamp, $id) {
$process_id = CakeResque::enqueueAt(
$timestamp,
'default',
'ServerShell',
array('enqueuePull', $timestamp, $this->Auth->user('id'), $id),
true
);
$this->Task->id = $id;
$this->Task->saveField('job_id', $process_id);
}
}

4
app/Model/Event.php
View File

@ -771,7 +771,7 @@ class Event extends AppModel {
//'Connection' => 'keep-alive' // LATER followup cakephp ticket 2854 about this problem http://cakephp.lighthouseapp.com/projects/42648-cakephp/tickets/2854
)
);
$uri = $url . '/events/index/sort:id/direction:desc/limit:999'; // LATER verify if events are missing because we only selected the last 999
$uri = $url . '/events/index/sort:id/direction:desc/limit:9999'; // LATER verify if events are missing because we only selected the last 999
try {
$response = $HttpSocket->get($uri, $data = '', $request);
if ($response->isOk()) {
@ -1114,7 +1114,7 @@ class Event extends AppModel {
)
);
// encrypt the mail for each user and send it separately
foreach ($alertUsers as &$user) {
foreach ($alertUsers as $k => &$user) {
// send the email
$Email = new CakeEmail();
$Email->from(Configure::read('CyDefSIG.email'));

4
app/Model/Log.php
View File

@ -19,7 +19,9 @@ class Log extends AppModel {
'delete',
'publish',
'accept',
'discard'
'discard',
'pull',
'push'
)),
'message' => 'Options : ...'
)

50
app/Model/Server.php
View File

@ -108,7 +108,7 @@ class Server extends AppModel {
return $this->field('id', array('id' => $serverid, 'org' => $org)) === $serverid;
}
public function pull($user, $id = null, $technique=false, $server, $jobId = false) {
public function pull($user, $id = null, $technique=false, $server, $jobId = false, $percent = 100, $current = 0) {
$eventModel = ClassRegistry::init('Event');
if ($jobId) {
$job = ClassRegistry::init('Job');
@ -237,9 +237,6 @@ class Server extends AppModel {
// error
$fails[$eventId] = 'failed downloading the event';
}
if ($jobId && $k%10 == 0) {
$job->saveField('progress', $k * 100 / $eventCount);
}
}
if (count($fails) > 0) {
// there are fails, take the lowest fail
@ -280,11 +277,22 @@ class Server extends AppModel {
}
}
}
if ($jobId) {
$job->saveField('progress', $k * 100 / $eventCount);
if ($jobId && $k%10 == 0) {
$job->saveField('progress', $k / $eventCount);
}
}
}
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org' => $user['org'],
'model' => 'Server',
'model_id' => $id,
'email' => $user['email'],
'action' => 'pull',
'title' => 'Pull from ' . $server['Server']['url'] . ' initiated by ' . $user['email'],
'change' => count($successes) . ' events and ' . count($pulledProposals) . ' proposals pulled or updated. ' . count($fails) . ' events failed or didn\'t need an update.'
));
return array($successes, $fails, $pulledProposals, $lastpulledid);
}
@ -295,6 +303,7 @@ class Server extends AppModel {
}
$eventModel = ClassRegistry::init('Event');
$this->read(null, $id);
$url = $this->data['Server']['url'];
if ("full" == $technique) {
$eventid_conditions_key = 'Event.id >';
$eventid_conditions_value = 0;
@ -328,9 +337,9 @@ class Server extends AppModel {
$lowestfailedid = null;
foreach ($eventIds as $k => $eventId) {
$eventModel->recursive=1;
$eventModel->contain(array('Attribute'));
$event = $eventModel->findById($eventId['Event']['id']);
$event['Event']['locked'] = true;
unset($event['User']);
$result = $eventModel->uploadEventToServer(
$event,
$this->data,
@ -341,7 +350,7 @@ class Server extends AppModel {
$fails[$event['Event']['id']] = $result;
}
if ($jobId && $k%10 == 0) {
$job->saveField('progress', $k * 100 / $eventCount);
$job->saveField('progress', 100 * $k / $eventCount);
}
}
if (count($fails) > 0) {
@ -358,18 +367,21 @@ class Server extends AppModel {
}
if (!isset($successes)) $successes = null;
if (!isset($fails)) $fails = null;
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'model' => 'Server',
'model_id' => $id,
'action' => 'push',
'title' => 'Push to ' . $url . '.',
'change' => count($successes) . ' events pushed or updated. ' . count($fails) . ' events failed or didn\'t need an update.'
));
if ($jobId) {
$temp = 'Fails: ';
$failCount = count($fails);
foreach ($fails as $k => $fail) {
if ($k < $failCount) {
$temp .= $fail . ', ';
} else {
$temp .= $fail;
}
}
$job->saveField('message', $temp);
return array($temp);
$job->id = $jobId;
$job->saveField('progress', 100);
$job->saveField('message', 'Push to server ' . $id . ' complete.');
$job->saveField('status', 4);
return;
} else {
return array($successes, $fails);
}

14
app/Plugin/SysLogLogable/Model/Behavior/SysLogLogableBehavior.php
View File

@ -64,7 +64,7 @@ class SysLogLogableBehavior extends LogableBehavior {
$old = '';
}
// TODO Audit, removed 'revision' as well
if ($key != 'revision' && $key != 'modified' && !in_array($key, $this->settings[$Model->alias]['ignore']) && $value != $old && in_array($key, $db_fields)) {
if ($key != 'lastpushedid' && $key!= 'timestamp' && $key != 'revision' && $key != 'modified' && !in_array($key, $this->settings[$Model->alias]['ignore']) && $value != $old && in_array($key, $db_fields)) {
if ($this->settings[$Model->alias]['change'] == 'full') {
if (($key != 'published') || (($key == 'published') && ($value == '1'))) { // remove (un-)published from edit
$changed_fields[] = $key . ' (' . $old . ') => (' . $value . ')';
@ -183,10 +183,6 @@ class SysLogLogableBehavior extends LogableBehavior {
$title = 'User ('. $Model->data[$Model->alias]['id'].') '. $Model->data[$Model->alias]['email'];
break;
case "Event":
App::uses('EventsController', 'Controller');
App::build(array('Controller' => array(APP . DS . 'Controller'), 'EventsController'));
$this->Events = new EventsController();
$this->Events->constructClasses();
$title = 'Event ('. $Model->data[$Model->alias]['id'] .'): '. $Model->data[$Model->alias]['info'];
$logData['Log']['title'] = $title;
break;
@ -207,26 +203,18 @@ class SysLogLogableBehavior extends LogableBehavior {
}
break;
case "Server":
$this->Servers = new ServersController();
$this->Servers->constructClasses();
$title = 'Server ('. $Model->data[$Model->alias]['id'].'): '. $Model->data[$Model->alias]['url'];
$logData['Log']['title'] = $title;
break;
case "Role":
$this->Roles = new RolesController();
$this->Roles->constructClasses();
$title = 'Role ('. $Model->data[$Model->alias]['id'] .'): '. $Model->data[$Model->alias]['name'];
$logData['Log']['title'] = $title;
break;
case "Whitelist":
$this->Whitelists = new WhitelistsController();
$this->Whitelists->constructClasses();
$title = 'Whitelist ('. $Model->data[$Model->alias]['id'] .'): '. $Model->data[$Model->alias]['name'];
$logData['Log']['title'] = $title;
break;
case "Regexp":
$this->Regexp = new RegexpController();
$this->Regexp->constructClasses();
$title = 'Regexp ('. $Model->data[$Model->alias]['id'] .'): '. $Model->data[$Model->alias]['regexp'];
$logData['Log']['title'] = $title;
break;

2
app/View/Events/proposal_event_index.ctp
View File

@ -18,7 +18,7 @@
<table class="table table-striped table-hover table-condensed">
<tr>
<th class="filter">
<?php echo $this->Paginator->sort('published', 'Valid.');?>
<?php echo $this->Paginator->sort('published');?>
</th>
<th><?php echo $this->Paginator->sort('id');?></th>
<th><?php echo $this->Paginator->sort('attribute_count', 'Proposals');?></th>

21
app/View/Logs/event_index.ctp
View File

@ -22,17 +22,30 @@ $mayPublish = ($isAclPublish && $event['Event']['orgc'] == $me['org']);
</div>
<table class="table table-striped table-hover table-condensed">
<tr>
<th><?php echo $this->Paginator->sort('model');?></th>
<th><?php echo $this->Paginator->sort('org');?></th>
<th><?php echo $this->Paginator->sort('action');?></th>
<th><?php echo $this->Paginator->sort('created');?></th>
<th><?php echo $this->Paginator->sort('model');?></th>
<th><?php echo $this->Paginator->sort('title');?></th>
<th><?php echo $this->Paginator->sort('created');?></th>
</tr>
<?php foreach ($list as $item): ?>
<tr>
<td class="short"><?php echo (h($item['Log']['model']) . '(' . h($item['Log']['model_id']) . ')'); ?>&nbsp;</td>
<td class="short">
<?php
$imgRelativePath = 'orgs' . DS . h($item['Log']['org']) . '.png';
$imgAbsolutePath = APP . WEBROOT_DIR . DS . 'img' . DS . $imgRelativePath;
if (file_exists($imgAbsolutePath)) echo $this->Html->image('orgs/' . h($item['Log']['org']) . '.png', array('alt' => h($item['Log']['org']), 'title' => h($item['Log']['org']), 'style' => 'width:24px; height:24px'));
else echo $this->Html->tag('span', h($item['Log']['org']), array('class' => 'welcome', 'style' => 'float:left;'));
?>
&nbsp;
</td>
<td class="short"><?php echo h($item['Log']['action']); ?>&nbsp;</td>
<td class="short"><?php echo (h($item['Log']['created'])); ?>&nbsp;</td>
<td class="short"><?php
if ($item['Log']['model'] !== 'ShadowAttribute') echo h($item['Log']['model']);
else echo 'Proposal';
?>&nbsp;</td>
<td><?php echo h($item['Log']['title']); ?>&nbsp;</td>
<td class="short"><?php echo (h($item['Log']['created'])); ?>&nbsp;</td>
</tr>
<?php endforeach; ?>
</table>