MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Further progress 

- removed some junk
- more work on the background workers
- rewrote the correlation background job - should work correctly now and be a lot more memory efficient
pull/762/head
Iglocska 2015-04-20 13:50:14 +02:00
parent 93c2194e5f
commit c44bee0e96
3 changed files with 22 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
app/Console/Command/AdminShell.php
View File

@ -11,16 +11,18 @@ class AdminShell extends AppShell
$this->loadModel('Correlation');
$this->Correlation->deleteAll(array('id !=' => ''), false);
$this->loadModel('Attribute');
$fields = array('Attribute.id', 'Attribute.event_id', 'Attribute.distribution', 'Attribute.cluster', 'Event.date', 'Event.org');
// get all attributes..
$attributes = $this->Attribute->find('all', array('recursive' => -1));
// for all attributes..
$total = count($attributes);
foreach ($attributes as $k => $attribute) {
if ($k > 0 && $k % 1000 == 0) {
$this->Job->saveField('progress', $k/$total*100);
$attribute_ids = array_keys($this->Attribute->find('list'));
$attributeCount = count($attribute_ids);
for ($i = 0; $i < ceil($attributeCount / 1000); $i++) {
$currentIds = array_slice($attribute_ids, ($i * 1000), (($i+1) * 1000));
$fields = array('Attribute.id', 'Attribute.event_id', 'Attribute.distribution', 'Attribute.sharing_group_id', 'Event.date', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id');
// get all attributes..
$attributes = $this->Attribute->find('all', array('recursive' => -1, 'conditions' => array('Attribute.id' => $currentIds)));
foreach ($attributes as $k => $attribute) {
$this->Attribute->__afterSaveCorrelation($attribute['Attribute']);
}
$this->Attribute->__afterSaveCorrelation($attribute['Attribute']);
$this->Job->saveField('progress', $i/$attributeCount*100);
}
$this->Job->saveField('progress', 100);
$this->Job->saveField('message', 'Job done.');

21
app/Console/Command/ServerShell.php
View File

@ -15,8 +15,7 @@ class ServerShell extends AppShell
$jobId = $this->args[3];
$this->Job->read(null, $jobId);
$this->Server->id = $serverId;
$this->User->recursive = -1;
$user = $this->User->read(null, $userId);
$user = $this->User->getAuthUser($userId);
$server = $this->Server->read(null, $serverId);
$result = $this->Server->pull($user['User'], $serverId, $technique, $server, $jobId);
$this->Job->id = $jobId;
@ -59,9 +58,8 @@ class ServerShell extends AppShell
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
$this->User->recursive = -1;
$user = $this->User->read(array('id', 'org', 'email'), $userId);
$result = $this->Server->push($serverId, 'full', $jobId, $HttpSocket, $user['User']['email']);
$user = $this->User->getAuthUser($userId);
$result = $this->Server->push($serverId, 'full', $jobId, $HttpSocket, $user['email']);
$this->Job->save(array(
'id' => $jobId,
'message' => 'Job done.',
@ -82,8 +80,7 @@ class ServerShell extends AppShell
if ($timestamp != $task['Task']['next_execution_time']) {
return;
}
$this->User->recursive = -1;
$user = $this->User->read(array('id', 'org', 'email'), $userId);
$user = $this->User->getAuthUser($userId);
$servers = $this->Server->find('all', array('recursive' => -1, 'conditions' => array('push' => 1)));
$count = count($servers);
foreach ($servers as $k => $server) {
@ -93,7 +90,8 @@ class ServerShell extends AppShell
'job_type' => 'pull',
'job_input' => 'Server: ' . $server['Server']['id'],
'retries' => 0,
'org' => $user['User']['org'],
'org' => $user['Organisation']['name'],
'org_id' => $user['org_id'],
'process_id' => 'Part of scheduled pull',
'message' => 'Pulling.',
);
@ -160,7 +158,7 @@ class ServerShell extends AppShell
return;
}
$this->User->recursive = -1;
$user = $this->User->read(array('id', 'org', 'email'), $userId);
$user = $this->User->getAuthUser($userId);
$servers = $this->Server->find('all', array('recursive' => -1, 'conditions' => array('push' => 1)));
$count = count($servers);
foreach ($servers as $k => $server) {
@ -170,7 +168,8 @@ class ServerShell extends AppShell
'job_type' => 'push',
'job_input' => 'Server: ' . $server['Server']['id'],
'retries' => 0,
'org' => $org,
'org' => $user['Organisation']['name'],
'org_id' => $user['org_id'],
'process_id' => 'Part of scheduled push',
'message' => 'Pushing.',
);
@ -179,7 +178,7 @@ class ServerShell extends AppShell
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
$result = $this->Server->push($server['Server']['id'], 'full', $jobId, $HttpSocket, $user['User']['email']);
$result = $this->Server->push($server['Server']['id'], 'full', $jobId, $HttpSocket, $user['email']);
}
$task['Task']['message'] = count($servers) . ' job(s) completed at ' . date('d/m/Y - H:i:s') . '.';
if ($task['Task']['timer'] > 0) {

7
app/Controller/AttributesController.php
View File

@ -52,7 +52,7 @@ class AttributesController extends AppController {
$this->paginate = Set::merge($this->paginate, array('conditions' => $this->Attribute->buildConditions($this->Auth->user())));
}
}
/**
* index method
*
@ -2058,9 +2058,4 @@ class AttributesController extends AppController {
if ($number != 1) return 's';
return '';
}
public function test() {
$this->Attribute->fetchAttributes($this->Auth->user());
}
}