MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Further work on the scheduled tasks 

- Also some changes left off from the previous commit
pull/217/head
iglocska 2013-12-17 11:38:06 +01:00
parent bc42fb99a8
commit 0e744871bd
12 changed files with 200 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/Console/Command/AdminShell.php
View File

@ -28,7 +28,6 @@ class AdminShell extends AppShell
foreach ($attributes as $k => $attribute) {
$this->Job->saveField('progress', $k/$total*100);
$this->Attribute->__afterSaveCorrelation($attribute['Attribute']);
sleep(1);
}
$this->Job->saveField('progress', 100);
$this->Job->saveField('message', 'Job done.');

13
app/Controller/AppController.php
View File

@ -163,6 +163,8 @@ class AppController extends Controller {
$this->loadModel('ShadowAttribute');
$this->ShadowAttribute->recursive = -1;
$shadowAttributes = $this->ShadowAttribute->find('all', array(
'recursive' => -1,
'fields' => array('event_id', 'event_org'),
'conditions' => array(
'ShadowAttribute.event_org' => $this->Auth->user('org')
)));
@ -219,16 +221,23 @@ class AppController extends Controller {
$this->Auth->login($user['User']);
}
public function queuegenerateCorrelation() {
if (!$this->_isSiteAdmin()) throw new NotFoundException();
CakeResque::enqueue(
$process_id = CakeResque::enqueue(
'default',
'AdminShell',
array('jobGenerateCorrelation')
array('jobGenerateCorrelation'),
true
);
debug($process_id);
debug(CakeResque::getJobStatus($process_id));
debug(CakeResque::getJobStatus('f80f51ee76dd22194a0dd6cd28c15f46'));
throw new Exception();
$this->Session->setFlash('Job queued.');
$this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
}
public function generateCorrelation() {
$this->loadModel('Correlation');
$this->Correlation->deleteAll(array('id !=' => ''), false);

54
app/Controller/AttributesController.php
View File

@ -10,7 +10,7 @@ App::uses('File', 'Utility');
*/
class AttributesController extends AppController {
public $components = array('Security', 'RequestHandler');
public $components = array('Security', 'RequestHandler', 'Cidr');
public $paginate = array(
'limit' => 60,
@ -873,9 +873,23 @@ class AttributesController extends AppController {
$keywordArrayElement = '%' . trim($keywordArrayElement) . '%';
if ($keywordArrayElement != '%%') {
if ($keywordArrayElement[1] == '!') {
array_push($temp2, array('Attribute.value NOT LIKE' => '%' . substr($keywordArrayElement, 2)));
if (preg_match('@^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(\d|[1-2]\d|3[0-2]))$@', substr($saveWord, 2))) {
$cidrresults = $this->Cidr->CIDR($saveWord);
foreach ($cidrresults as $result) {
array_push($temp2, array('Attribute.value NOT LIKE' => $result));
}
} else {
array_push($temp2, array('Attribute.value NOT LIKE' => '%' . substr($keywordArrayElement, 2)));
}
} else {
array_push($temp, array('Attribute.value LIKE' => $keywordArrayElement));
if (preg_match('@^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(\d|[1-2]\d|3[0-2]))$@', $saveWord)) {
$cidrresults = $this->Cidr->CIDR($saveWord);
foreach ($cidrresults as $result) {
array_push($temp, array('Attribute.value LIKE' => $result));
}
} else {
array_push($temp, array('Attribute.value LIKE' => $keywordArrayElement));
}
}
}
if ($i == 1 && $saveWord != '') $keyWordText = $saveWord;
@ -1086,6 +1100,7 @@ class AttributesController extends AppController {
if (!$user) {
throw new UnauthorizedException('This authentication key is not authorized to be used for exports. Contact your administrator.');
}
$value = str_replace('|', '/', $value);
$this->response->type('xml'); // set the content type
$this->layout = 'xml/default';
$this->header('Content-Disposition: download; filename="misp.search.attribute.results.xml"');
@ -1095,15 +1110,37 @@ class AttributesController extends AppController {
// add the values as specified in the 2nd parameter to the conditions
$values = explode('&&', $value);
$parameters = array('value', 'type', 'category', 'org');
foreach ($parameters as $k => $param) {
if (isset(${$parameters[$k]})) {
$elements = explode('&&', ${$parameters[$k]});
foreach($elements as $v) {
if (substr($v, 0, 1) == '!') {
$subcondition['AND'][] = array('Attribute.' . $parameters[$k] . ' NOT LIKE' => '%'.substr($v, 1).'%');
if ($parameters[$k] === 'value' && preg_match('@^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(\d|[1-2]\d|3[0-2]))$@', substr($v, 1))) {
$cidrresults = $this->Cidr->CIDR(substr($v, 1));
foreach ($cidrresults as $result) {
$subcondition['AND'][] = array('Attribute.value NOT LIKE' => $result);
}
} else {
if ($parameters[$k] === 'org') {
$subcondition['AND'][] = array('Event.' . $parameters[$k] . ' NOT LIKE' => '%'.substr($v, 1).'%');
} else {
$subcondition['AND'][] = array('Attribute.' . $parameters[$k] . ' NOT LIKE' => '%'.substr($v, 1).'%');
}
}
} else {
$subcondition['OR'][] = array('Attribute.' . $parameters[$k] . ' LIKE' => '%'.$v.'%');
if ($parameters[$k] === 'value' && preg_match('@^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(\d|[1-2]\d|3[0-2]))$@', substr($v, 1))) {
$cidrresults = $this->Cidr->CIDR($v);
foreach ($cidrresults as $result) {
$subcondition['OR'][] = array('Attribute.value LIKE' => $result);
}
} else {
if ($parameters[$k] === 'org') {
$subcondition['OR'][] = array('Event.' . $parameters[$k] . ' LIKE' => '%'.$v.'%');
} else {
$subcondition['OR'][] = array('Attribute.' . $parameters[$k] . ' LIKE' => '%'.$v.'%');
}
}
}
}
array_push ($conditions['AND'], $subcondition);
@ -1266,9 +1303,4 @@ class AttributesController extends AppController {
$attributes = $this->Whitelist->removeWhitelistedFromArray($attributes, true);
$this->set('attributes', $attributes);
}
public function tester () {
debug($this->Attribute->nids(true, 'ADMIN', 'suricata', 1111));
}
}

3
app/Controller/EventsController.php
View File

@ -1055,11 +1055,12 @@ class EventsController extends AppController {
);
$this->Job->save($data);
$jobId = $this->Job->id;
$result = CakeResque::enqueue(
$process_id = CakeResque::enqueue(
'default',
'EventShell',
array('alertemail', $this->Auth->user('org'), $this->_isSiteAdmin(), $jobId, $id)
);
$this->Job->saveField('process_id', $process_id);
return true;
} else {
return ($this->Event->sendAlertEmail($id, $this->Auth->user('org'), $this->_isSiteAdmin()));

45
app/Controller/JobsController.php
View File

@ -25,7 +25,32 @@ class JobsController extends AppController {
if (!$this->_isSiteAdmin()) throw new MethodNotAllowedException();
if (!Configure::read('MISP.background_jobs')) throw new NotFoundException('Background jobs are not enabled on this instance.');
$this->recursive = 0;
$this->set('list', $this->paginate());
$jobs = $this->paginate();
foreach($jobs as &$job) {
if ($job['Job']['process_id']) {
$job['Job']['status'] = $this->__jobStatusConverter(CakeResque::getJobStatus($job['Job']['process_id']));
} else {
$job['Job']['status'] = '???';
}
}
$this->set('list', $jobs);
}
private function __jobStatusConverter($status) {
switch ($status) {
case 1:
return 'In progress...';
break;
case 2:
return 'Unknown';
break;
case 3:
return 'Unknown';
break;
case 4:
return 'Completed';
break;
}
}
public function getGenerateCorrelationProgress($id) {
@ -71,7 +96,7 @@ class JobsController extends AppController {
$shell = 'Event';
$this->Job->create();
$data = array(
'worker' => 'default',
'worker' => 'cache',
'job_type' => 'cache_' . $type,
'job_input' => $target,
'status' => 0,
@ -94,11 +119,21 @@ class JobsController extends AppController {
}
$this->Job->save($data);
$id = $this->Job->id;
CakeResque::enqueue(
'default',
$process_id = CakeResque::enqueue(
'cache',
$shell . 'Shell',
array('cache' . $type, $this->Auth->user('org'), $this->_isSiteAdmin(), $id, $extra, $extra2)
array('cache' . $type, $this->Auth->user('org'), $this->_isSiteAdmin(), $id, $extra, $extra2),
true
);
$this->Job->saveField('process_id', $process_id);
return new CakeResponse(array('body' => json_encode($id)));
}
public function jobScheduler() {
if (!$this->_isSiteAdmin()) throw new MethodNotAllowedException();
}
public function stopWorker() {
debug(CakeResque::getFailedJobLog('323c143da4e12cf682554c4a8742f412'));
}
}

14
app/Controller/TasksController.php
View File

@ -7,7 +7,7 @@ App::uses('AppController', 'Controller');
*
* @property Job $Job
*/
class JobsController extends AppController {
class TasksController extends AppController {
public $components = array('Security' ,'RequestHandler', 'Session');
public $paginate = array(
@ -24,19 +24,25 @@ class JobsController extends AppController {
public function index() {
if (!$this->_isSiteAdmin()) throw new MethodNotAllowedException();
if (!Configure::read('MISP.background_jobs')) throw new NotFoundException('Background jobs are not enabled on this instance.');
$this->__checkTasks();
$this->recursive = 0;
$tasks = $this->paginate();
$this->set('list', $tasks);
}
// checks if all the mandatory tasks exist, and if not, creates them
// default tasks are:
// 'cache_exports'
private function __checkTasks() {
foreach ($this->Task->tasks as $default_task) {
if (!$this->Task->findByType($default_task['type'], array('id', 'type'))) {
$this->Task->save($default_task);
}
}
}
private function __setTask($id) {
public function setTask($id) {
}
}

17
app/Model/Task.php Normal file
View File

@ -0,0 +1,17 @@
<?php
App::uses('AppModel', 'Model');
/**
* Task Model
*
* @property Task $Task
*/
class Task extends AppModel {
public $tasks = array(
'cache_exports' => array(
'type' => 'cache_exports',
'timer' => 0,
'scheduled_time' => 0,
'recurring' => false,
'description' => 'Generates export caches for every export type and for every organisation. This process is heavy, schedule so it might be a good idea to schedule this outside of working hours and before your daily automatic imports on connected services are scheduled.'
));
}

2
app/Plugin/CakeResque

@ -1 +1 @@
Subproject commit a88ba3183df87dd8dfe30bac896c41cd62923ed4
Subproject commit 247c06cb8647b812eea4cf868a2b1f134159b48b

6
app/View/Events/automation.ctp
View File

@ -67,6 +67,12 @@ For example, in order to search for all attributes created by your organisation
echo Configure::read('CyDefSIG.baseurl').'/attributes/restSearch/'.$me['authkey'].'/192.168&&127.0&&!0.1/ip-src/null/' . $me['org'];
?>
</pre>
<p>You can also use search for IP addresses using CIDR. Make sure that you use '|' (pipe) instead of '/' (slashes). See below for an example: </p>
<pre>
<?php
echo Configure::read('CyDefSIG.baseurl').'/attributes/restSearch/'.$me['authkey'].'/192.168.1.1|16/ip-src/null/' . $me['org'];
?>
</pre>
<h3>Export attributes of event with specified type as XML</h3>
<p>If you want to export all attributes of a pre-defined type that belong to an event, use the following syntax:</p>

1
app/View/Helper/PivotHelper.php
View File

@ -7,6 +7,7 @@ App::uses('AppHelper', 'View/Helper');
$data = null;
$text = $pivot['id'] . ': ';
$active = '';
$pivot['info'] = h($pivot['info']);
// Truncate string if longer than (11 - length of event id) chars to fit the pivot bubble
if (strlen($pivot['info']) > (11 - strlen((string)$pivot['id']))) {
$text .= substr($pivot['info'], 0, 7) . '...';

2
app/View/Jobs/index.ctp
View File

@ -38,6 +38,7 @@
<table class="table table-striped table-hover table-condensed">
<tr>
<th><?php echo $this->Paginator->sort('id');?></th>
<th><?php echo $this->Paginator->sort('process_id');?></th>
<th><?php echo $this->Paginator->sort('worker');?></th>
<th><?php echo $this->Paginator->sort('job_type');?></th>
<th><?php echo $this->Paginator->sort('job_input');?></th>
@ -49,6 +50,7 @@
foreach ($list as $k => $item): ?>
<tr>
<td class="short"><?php echo h($item['Job']['id']); ?>&nbsp;</td>
<td class="short"><?php echo h($item['Job']['process_id']); ?>&nbsp;</td>
<td class="short"><?php echo h($item['Job']['worker']); ?>&nbsp;</td>
<td class="short"><?php echo h($item['Job']['job_type']); ?>&nbsp;</td>
<td><?php echo h($item['Job']['job_input']); ?>&nbsp;</td>

67
app/View/Tasks/index.ctp Normal file
View File

@ -0,0 +1,67 @@
<div class="task index">
<h2>Scheduled Tasks</h2>
<div class="pagination">
<ul>
<?php
$this->Paginator->options(array(
'update' => '.span12',
'evalScripts' => true,
'before' => '$(".progress").show()',
'complete' => '$(".progress").hide()',
));
echo $this->Paginator->prev('&laquo; ' . __('previous'), array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'prev disabled', 'escape' => false, 'disabledTag' => 'span'));
echo $this->Paginator->numbers(array('modulus' => 20, 'separator' => '', 'tag' => 'li', 'currentClass' => 'active', 'currentTag' => 'span'));
echo $this->Paginator->next(__('next') . ' &raquo;', array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'next disabled', 'escape' => false, 'disabledTag' => 'span'));
?>
</ul>
</div>
<table class="table table-striped table-hover table-condensed">
<tr>
<th><?php echo $this->Paginator->sort('id');?></th>
<th><?php echo $this->Paginator->sort('type');?></th>
<th><?php echo $this->Paginator->sort('timer');?></th>
<th><?php echo $this->Paginator->sort('scheduled_time');?></th>
<th><?php echo $this->Paginator->sort('recurring');?></th>
<th><?php echo $this->Paginator->sort('description');?></th>
<th class="actions"><?php echo __('Actions');?></th>
</tr><?php
foreach ($list as $item):?>
<tr>
<td class="short"><?php echo h($item['Task']['id']);?>&nbsp;</td>
<td class="short"><?php echo h($item['Task']['type']);?>&nbsp;</td>
<td class="short"><?php echo h($item['Task']['timer']);?>&nbsp;</td>
<td class="short"><?php echo h($item['Task']['scheduled_time']);?>&nbsp;</td>
<td class="short">
<?php
if (!$item['Task']['recurring']) echo 'No';
else echo 'Yes';
?>
&nbsp;</td>
<td><?php echo h($item['Task']['description']);?>&nbsp;</td>
<td class="short action-links">
<?php echo $this->Html->link('', array('action' => 'setTask', $item['Task']['id']), array('class' => 'icon-edit', 'title' => 'Edit'));?>
</td>
</tr><?php
endforeach;?>
</table>
<p>
<?php
echo $this->Paginator->counter(array(
'format' => __('Page {:page} of {:pages}, showing {:current} records out of {:count} total, starting on record {:start}, ending on {:end}')
));
?>
</p>
<div class="pagination">
<ul>
<?php
echo $this->Paginator->prev('&laquo; ' . __('previous'), array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'prev disabled', 'escape' => false, 'disabledTag' => 'span'));
echo $this->Paginator->numbers(array('modulus' => 20, 'separator' => '', 'tag' => 'li', 'currentClass' => 'active', 'currentTag' => 'span'));
echo $this->Paginator->next(__('next') . ' &raquo;', array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'next disabled', 'escape' => false, 'disabledTag' => 'span'));
?>
</ul>
</div>
</div>
<?php
echo $this->element('side_menu', array('menuList' => 'task', 'menuItem' => 'index'));
?>