MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [internal] Optimise reportValidationIssuesAttributes

pull/9431/head
Jakub Onderka 2023-12-06 11:28:57 +01:00
parent bf51c9ebde
commit 8678da10d8
4 changed files with 51 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
app/Console/Command/EventShell.php
View File

@ -636,6 +636,13 @@ class EventShell extends AppShell
}
}
public function reportValidationIssuesAttributes()
{
foreach ($this->Event->Attribute->reportValidationIssuesAttributes() as $validationIssue) {
echo $this->json($validationIssue) . "\n";
}
}
/**
* @param int $userId
* @return array

2
app/Controller/AttributesController.php
View File

@ -1917,7 +1917,7 @@ class AttributesController extends AppController
public function reportValidationIssuesAttributes($eventId = false)
{
// search for validation problems in the attributes
$this->set('result', $this->Attribute->reportValidationIssuesAttributes($eventId));
$this->set('result', iterator_to_array($this->Attribute->reportValidationIssuesAttributes($eventId)));
}
public function generateCorrelation()

65
app/Model/Attribute.php
View File

@ -1224,38 +1224,58 @@ class Attribute extends AppModel
$this->Correlation->purgeCorrelations($eventId);
}
public function reportValidationIssuesAttributes($eventId)
/**
* @param array $conditions
* @return Generator|void
*/
private function fetchAttributesInChunks(array $conditions = [])
{
while (true) {
$attributes = $this->find('all', [
'recursive' => -1,
'conditions' => $conditions,
'limit' => 500,
'order' => 'Attribute.id',
]);
if (empty($attributes)) {
return;
}
foreach ($attributes as $attribute) {
yield $attribute;
}
$count = count($attributes);
$lastAttribute = $attributes[$count - 1];
$conditions['Attribute.id >'] = $lastAttribute['Attribute']['id'];
}
}
/**
* @param int|null $eventId
* @return Generator
*/
public function reportValidationIssuesAttributes($eventId = null)
{
$conditions = array();
if ($eventId && is_numeric($eventId)) {
$conditions = array('event_id' => $eventId);
}
$attributeIds = $this->find('column', array(
'fields' => array('id'),
'conditions' => $conditions
));
$chunks = array_chunk($attributeIds, 500);
$attributes = $this->fetchAttributesInChunks($conditions);
$result = array();
foreach ($chunks as $chunk) {
$attributes = $this->find('all', array('recursive' => -1, 'conditions' => array('id' => $chunk)));
foreach ($attributes as $attribute) {
$this->set($attribute);
if (!$this->validates()) {
$resultErrors = array();
foreach ($this->validationErrors as $field => $error) {
$resultErrors[$field] = array('value' => $attribute['Attribute'][$field], 'error' => $error[0]);
}
$result[] = [
'id' => $attribute['Attribute']['id'],
'error' => $resultErrors,
'details' => 'Event ID: [' . $attribute['Attribute']['event_id'] . "] - Category: [" . $attribute['Attribute']['category'] . "] - Type: [" . $attribute['Attribute']['type'] . "] - Value: [" . $attribute['Attribute']['value'] . ']',
];
foreach ($attributes as $attribute) {
$this->set($attribute);
if (!$this->validates()) {
$resultErrors = [];
foreach ($this->validationErrors as $field => $error) {
$resultErrors[$field] = ['value' => $attribute['Attribute'][$field], 'error' => $error[0]];
}
yield [
'id' => $attribute['Attribute']['id'],
'error' => $resultErrors,
'details' => 'Event ID: [' . $attribute['Attribute']['event_id'] . "] - Category: [" . $attribute['Attribute']['category'] . "] - Type: [" . $attribute['Attribute']['type'] . "] - Value: [" . $attribute['Attribute']['value'] . ']',
];
}
}
return $result;
}
/**
@ -1610,6 +1630,7 @@ class Attribute extends AppModel
* @param array $user
* @param array $options
* @param int|false $result_count If false, count is not fetched
* @param bool $real_count
* @return array
* @throws Exception
*/

8
app/View/Pages/administration.ctp
View File

@ -1,11 +1,3 @@
<?php
if (!$isSiteAdmin) exit();
?>
<div class="actions">
<ol class="nav nav-list">
</ol>
</div>
<div class="index">
<h2><?php echo __('Administrative actions');?></h2>
<ul>