MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [internal] Top correlations cleanup

pull/8355/head
Jakub Onderka 2022-05-09 14:34:38 +02:00
parent 5b5f9f42db
commit f2580dda7a
5 changed files with 41 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
app/Console/Command/EventShell.php
View File

@ -10,6 +10,7 @@ require_once 'AppShell.php';
* @property Job $Job * @property Job $Job
* @property Tag $Tag * @property Tag $Tag
* @property Server $Server * @property Server $Server
* @property Correlation $Correlation
*/ */
class EventShell extends AppShell class EventShell extends AppShell
{ {
@ -44,6 +45,9 @@ class EventShell extends AppShell
$parser->addSubcommand('duplicateTags', [ $parser->addSubcommand('duplicateTags', [
'help' => __('Show duplicate tags'), 'help' => __('Show duplicate tags'),
]); ]);
$parser->addSubcommand('generateTopCorrelations', [
'help' => __('Generate top correlations'),
]);
$parser->addSubcommand('mergeTags', [ $parser->addSubcommand('mergeTags', [
'help' => __('Merge tags'), 'help' => __('Merge tags'),
'parser' => [ 'parser' => [
@ -644,17 +648,20 @@ class EventShell extends AppShell
public function generateTopCorrelations() public function generateTopCorrelations()
{ {
$this->ConfigLoad->execute(); $jobId = $this->args[0] ?? null;
$jobId = $this->args[0]; if ($jobId) {
$job = $this->Job->read(null, $jobId); $job = $this->Job->read(null, $jobId);
$job['Job']['progress'] = 1; $job['Job']['progress'] = 1;
$job['Job']['date_modified'] = date("Y-m-d H:i:s"); $job['Job']['date_modified'] = date("Y-m-d H:i:s");
$job['Job']['message'] = __('Generating top correlations list.'); $job['Job']['message'] = __('Generating top correlations list.');
$this->Job->save($job); $this->Job->save($job);
$result = $this->Correlation->generateTopCorrelations($jobId); }
$job['Job']['progress'] = 100; $this->Correlation->generateTopCorrelations($jobId);
$job['Job']['date_modified'] = date("Y-m-d H:i:s"); if ($jobId) {
$job['Job']['message'] = __('Job done.'); $job['Job']['progress'] = 100;
$this->Job->save($job); $job['Job']['date_modified'] = date("Y-m-d H:i:s");
$job['Job']['message'] = __('Job done.');
$this->Job->save($job);
}
} }
} }

2
app/Controller/CorrelationExclusionsController.php
View File

@ -122,7 +122,7 @@ class CorrelationExclusionsController extends AppController
} else { } else {
$this->set('title', __('Clean up correlations')); $this->set('title', __('Clean up correlations'));
$this->set('question', __('Execute the cleaning of all correlations that are at odds with the exclusion rules? This will delete all matching correlations.')); $this->set('question', __('Execute the cleaning of all correlations that are at odds with the exclusion rules? This will delete all matching correlations.'));
$this->set('actionName', 'clean'); $this->set('actionName', __('Clean'));
$this->layout = false; $this->layout = false;
$this->render('/genericTemplates/confirm'); $this->render('/genericTemplates/confirm');
} }

20
app/Model/Correlation.php
View File

@ -624,11 +624,11 @@ class Correlation extends AppModel
} catch (Exception $e) { } catch (Exception $e) {
throw new NotFoundException(__('No redis connection found.')); throw new NotFoundException(__('No redis connection found.'));
} }
$max_id = $this->find('first', [ $maxId = $this->find('first', [
'fields' => ['MAX(id) AS max_id'], 'fields' => ['MAX(id) AS max_id'],
'recursive' => -1 'recursive' => -1,
]); ]);
if (empty($max_id)) { if (empty($maxId)) {
return false; return false;
} }
if ($jobId) { if ($jobId) {
@ -643,18 +643,18 @@ class Correlation extends AppModel
$jobId = false; $jobId = false;
} }
} }
$max_id = $max_id[0]['max_id']; $maxId = $maxId[0]['max_id'];
$redis->del(self::CACHE_NAME); $redis->del(self::CACHE_NAME);
$redis->set(self::CACHE_AGE, time()); $redis->set(self::CACHE_AGE, time());
$chunk_size = 1000000; $chunkSize = 1000000;
$max = ceil($max_id / $chunk_size); $maxPage = ceil($maxId / $chunkSize);
for ($i = 0; $i < $max; $i++) { for ($page = 0; $page < $maxPage; $page++) {
$correlations = $this->find('column', [ $correlations = $this->find('column', [
'fields' => ['value'], 'fields' => ['value'],
'conditions' => [ 'conditions' => [
'id >' => $i * $chunk_size, 'id >' => $page * $chunkSize,
'id <=' => (($i + 1) * $chunk_size) 'id <=' => ($page + 1) * $chunkSize
] ]
]); ]);
$newElements = count($correlations); $newElements = count($correlations);
@ -665,7 +665,7 @@ class Correlation extends AppModel
} }
$pipeline->exec(); $pipeline->exec();
if ($jobId) { if ($jobId) {
$this->Job->saveProgress($jobId, __('Generating top correlations. Processed %s IDs.', ($i * $chunk_size) + $newElements), floor(100 * $i / $max)); $this->Job->saveProgress($jobId, __('Generating top correlations. Processed %s IDs.', ($page * $chunkSize) + $newElements), floor(100 * $page / $maxPage));
return $jobId; return $jobId;
} }
} }

1
app/Model/CorrelationExclusion.php
View File

@ -1,6 +1,5 @@
<?php <?php
App::uses('AppModel', 'Model'); App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
class CorrelationExclusion extends AppModel class CorrelationExclusion extends AppModel
{ {

28
app/View/CorrelationExclusions/index.ctp
View File

@ -11,15 +11,10 @@
'children' => [ 'children' => [
'data' => [ 'data' => [
'type' => 'simple', 'type' => 'simple',
'fa-icon' => 'plus',
'text' => __('Add correlation exclusion entry'), 'text' => __('Add correlation exclusion entry'),
'class' => 'btn btn-primary', 'class' => 'btn btn-primary modal-open',
'onClick' => 'openGenericModal', 'url' => "$baseurl/correlation_exclusions/add",
'onClickParams' => [
sprintf(
'%s/correlation_exclusions/add',
$baseurl
)
]
] ]
] ]
], ],
@ -29,14 +24,8 @@
'data' => [ 'data' => [
'type' => 'simple', 'type' => 'simple',
'text' => __('Clean up correlations'), 'text' => __('Clean up correlations'),
'class' => 'btn btn-primary', 'class' => 'btn btn-primary modal-open',
'onClick' => 'openGenericModal', 'url' => "$baseurl/correlation_exclusions/clean",
'onClickParams' => [
sprintf(
'%s/correlation_exclusions/clean',
$baseurl
)
]
] ]
] ]
], ],
@ -45,6 +34,11 @@
'button' => __('Filter'), 'button' => __('Filter'),
'placeholder' => __('Enter value to search'), 'placeholder' => __('Enter value to search'),
'searchKey' => 'quickFilter', 'searchKey' => 'quickFilter',
'cancel' => [
'fa-icon' => 'times',
'title' => __('Remove filters'),
'onClick' => 'cancelSearch',
],
] ]
] ]
], ],
@ -102,7 +96,7 @@
echo $this->element('/genericElements/SideMenu/side_menu', $menuData); echo $this->element('/genericElements/SideMenu/side_menu', $menuData);
} }
?> ?>
<script type="text/javascript"> <script>
var passedArgsArray = <?php echo $passedArgs; ?>; var passedArgsArray = <?php echo $passedArgs; ?>;
$(function() { $(function() {
$('#quickFilterButton').click(function() { $('#quickFilterButton').click(function() {