MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [internal] Top correlations cleanup

pull/8355/head
Jakub Onderka 2022-05-09 14:34:38 +02:00
parent 5b5f9f42db
commit f2580dda7a
5 changed files with 41 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
app/Console/Command/EventShell.php
View File

@ -10,6 +10,7 @@ require_once 'AppShell.php';
* @property Job $Job
* @property Tag $Tag
* @property Server $Server
* @property Correlation $Correlation
*/
class EventShell extends AppShell
{
@ -44,6 +45,9 @@ class EventShell extends AppShell
$parser->addSubcommand('duplicateTags', [
'help' => __('Show duplicate tags'),
]);
$parser->addSubcommand('generateTopCorrelations', [
'help' => __('Generate top correlations'),
]);
$parser->addSubcommand('mergeTags', [
'help' => __('Merge tags'),
'parser' => [
@ -644,17 +648,20 @@ class EventShell extends AppShell
public function generateTopCorrelations()
{
$this->ConfigLoad->execute();
$jobId = $this->args[0];
$job = $this->Job->read(null, $jobId);
$job['Job']['progress'] = 1;
$job['Job']['date_modified'] = date("Y-m-d H:i:s");
$job['Job']['message'] = __('Generating top correlations list.');
$this->Job->save($job);
$result = $this->Correlation->generateTopCorrelations($jobId);
$job['Job']['progress'] = 100;
$job['Job']['date_modified'] = date("Y-m-d H:i:s");
$job['Job']['message'] = __('Job done.');
$this->Job->save($job);
$jobId = $this->args[0] ?? null;
if ($jobId) {
$job = $this->Job->read(null, $jobId);
$job['Job']['progress'] = 1;
$job['Job']['date_modified'] = date("Y-m-d H:i:s");
$job['Job']['message'] = __('Generating top correlations list.');
$this->Job->save($job);
}
$this->Correlation->generateTopCorrelations($jobId);
if ($jobId) {
$job['Job']['progress'] = 100;
$job['Job']['date_modified'] = date("Y-m-d H:i:s");
$job['Job']['message'] = __('Job done.');
$this->Job->save($job);
}
}
}

2
app/Controller/CorrelationExclusionsController.php
View File

@ -122,7 +122,7 @@ class CorrelationExclusionsController extends AppController
} else {
$this->set('title', __('Clean up correlations'));
$this->set('question', __('Execute the cleaning of all correlations that are at odds with the exclusion rules? This will delete all matching correlations.'));
$this->set('actionName', 'clean');
$this->set('actionName', __('Clean'));
$this->layout = false;
$this->render('/genericTemplates/confirm');
}

20
app/Model/Correlation.php
View File

@ -624,11 +624,11 @@ class Correlation extends AppModel
} catch (Exception $e) {
throw new NotFoundException(__('No redis connection found.'));
}
$max_id = $this->find('first', [
$maxId = $this->find('first', [
'fields' => ['MAX(id) AS max_id'],
'recursive' => -1
'recursive' => -1,
]);
if (empty($max_id)) {
if (empty($maxId)) {
return false;
}
if ($jobId) {
@ -643,18 +643,18 @@ class Correlation extends AppModel
$jobId = false;
}
}
$max_id = $max_id[0]['max_id'];
$maxId = $maxId[0]['max_id'];
$redis->del(self::CACHE_NAME);
$redis->set(self::CACHE_AGE, time());
$chunk_size = 1000000;
$max = ceil($max_id / $chunk_size);
for ($i = 0; $i < $max; $i++) {
$chunkSize = 1000000;
$maxPage = ceil($maxId / $chunkSize);
for ($page = 0; $page < $maxPage; $page++) {
$correlations = $this->find('column', [
'fields' => ['value'],
'conditions' => [
'id >' => $i * $chunk_size,
'id <=' => (($i + 1) * $chunk_size)
'id >' => $page * $chunkSize,
'id <=' => ($page + 1) * $chunkSize
]
]);
$newElements = count($correlations);
@ -665,7 +665,7 @@ class Correlation extends AppModel
}
$pipeline->exec();
if ($jobId) {
$this->Job->saveProgress($jobId, __('Generating top correlations. Processed %s IDs.', ($i * $chunk_size) + $newElements), floor(100 * $i / $max));
$this->Job->saveProgress($jobId, __('Generating top correlations. Processed %s IDs.', ($page * $chunkSize) + $newElements), floor(100 * $page / $maxPage));
return $jobId;
}
}

1
app/Model/CorrelationExclusion.php
View File

@ -1,6 +1,5 @@
<?php
App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
class CorrelationExclusion extends AppModel
{

28
app/View/CorrelationExclusions/index.ctp
View File

@ -11,15 +11,10 @@
'children' => [
'data' => [
'type' => 'simple',
'fa-icon' => 'plus',
'text' => __('Add correlation exclusion entry'),
'class' => 'btn btn-primary',
'onClick' => 'openGenericModal',
'onClickParams' => [
sprintf(
'%s/correlation_exclusions/add',
$baseurl
)
]
'class' => 'btn btn-primary modal-open',
'url' => "$baseurl/correlation_exclusions/add",
]
]
],
@ -29,14 +24,8 @@
'data' => [
'type' => 'simple',
'text' => __('Clean up correlations'),
'class' => 'btn btn-primary',
'onClick' => 'openGenericModal',
'onClickParams' => [
sprintf(
'%s/correlation_exclusions/clean',
$baseurl
)
]
'class' => 'btn btn-primary modal-open',
'url' => "$baseurl/correlation_exclusions/clean",
]
]
],
@ -45,6 +34,11 @@
'button' => __('Filter'),
'placeholder' => __('Enter value to search'),
'searchKey' => 'quickFilter',
'cancel' => [
'fa-icon' => 'times',
'title' => __('Remove filters'),
'onClick' => 'cancelSearch',
],
]
]
],
@ -102,7 +96,7 @@
echo $this->element('/genericElements/SideMenu/side_menu', $menuData);
}
?>
<script type="text/javascript">
<script>
var passedArgsArray = <?php echo $passedArgs; ?>;
$(function() {
$('#quickFilterButton').click(function() {