MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [analyst-data:pull] Change in pull strategy + few improvements

notes
Sami Mokaddem 2024-02-05 10:59:20 +01:00
parent ea88d5c7bb
commit f649814afb
No known key found for this signature in database
GPG Key ID: 164C473F627A06FA
4 changed files with 57 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
app/Controller/AnalystDataController.php
View File

@ -150,8 +150,8 @@ class AnalystDataController extends AppController
'conditions' => $conditions,
'contain' => ['Org', 'Orgc'],
'afterFind' => function(array $analystData) {
$canEdit = $this->ACL->canEditAnalystData($this->Auth->user(), $analystData, $this->modelSelection);
if (!$this->IndexFilter->isRest()) {
$canEdit = $this->ACL->canEditAnalystData($this->Auth->user(), $analystData, $this->modelSelection);
$analystData[$this->modelSelection]['_canEdit'] = $canEdit;
}
return $analystData;
@ -175,14 +175,16 @@ class AnalystDataController extends AppController
$conditions = $this->AnalystData->buildConditions($this->Auth->user());
$params = [
'filters' => ['uuid', 'target_object', 'uuid'],
'filters' => ['uuid', 'target_object'],
'quickFilters' => ['name'],
'conditions' => $conditions,
'afterFind' => function(array $data) {
foreach ($data as $i => $analystData) {
$canEdit = $this->ACL->canEditAnalystData($this->Auth->user(), $analystData, $this->modelSelection);
if (!$this->IndexFilter->isRest()) {
$data[$i][$this->modelSelection]['_canEdit'] = $canEdit;
$canEdit = $this->ACL->canEditAnalystData($this->Auth->user(), $analystData, $this->modelSelection);
if (!$this->IndexFilter->isRest()) {
$data[$i][$this->modelSelection]['_canEdit'] = $canEdit;
}
}
}
return $data;

3
app/Controller/ServersController.php
View File

@ -776,7 +776,7 @@ class ServersController extends AppController
if (!Configure::read('MISP.background_jobs')) {
$result = $this->Server->pull($this->Auth->user(), $technique, $s);
if (is_array($result)) {
$success = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4]);
$success = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled, %s analyst data pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4], $result[5]);
} else {
$error = $result;
}
@ -784,6 +784,7 @@ class ServersController extends AppController
$this->set('fails', $result[1]);
$this->set('pulledProposals', $result[2]);
$this->set('pulledSightings', $result[3]);
$this->set('pulledAnalystData', $result[5]);
} else {
$this->loadModel('Job');
$jobId = $this->Job->createJob(

6
app/Lib/Tools/ServerSyncTool.php
View File

@ -250,17 +250,17 @@ class ServerSyncTool
* @throws HttpSocketJsonException
* @throws HttpSocketHttpException
*/
public function fetchAnalystData(array $uuids)
public function fetchAnalystData($type, array $uuids)
{
if (!$this->isSupported(self::PERM_ANALYST_DATA)) {
return [];
}
$params = [
'uuids' => $uuids,
'uuid' => $uuids,
];
$url = '/analyst_data/index';
$url = '/analyst_data/index/' . $type;
$url .= $this->createParams($params);
$url .= '.json';
return $this->get($url);

72
app/Model/AnalystData.php
View File

@ -343,7 +343,7 @@ class AnalystData extends AppModel
return $results;
}
if (!isset($analystData[$type]['orgc_uuid']) && !isset($cluster['Orgc'])) {
if (!isset($analystData[$type]['orgc_uuid']) && !isset($analystData[$type]['Orgc'])) {
$analystData[$type]['orgc_uuid'] = $analystData[$type]['org_uuid'];
} else {
if (!isset($analystData[$type]['Orgc'])) {
@ -405,10 +405,6 @@ class AnalystData extends AppModel
}
if ($saveSuccess) {
$results['imported']++;
$analystModel->find('first', [
'conditions' => ['uuid' => $analystData[$type]['uuid']],
'recursive' => -1
]);
} else {
$results['failed']++;
foreach ($analystModel->validationErrors as $validationError) {
@ -657,6 +653,13 @@ class AnalystData extends AppModel
return $analystData;
}
/**
* Collect all UUIDs with their modified time on the remote side, then filter the list based on what we have locally.
* Afterward, iteratively pull what should be pulled.
*
* @param array $user
* @param ServerSyncTool $serverSync
*/
public function pull(array $user, ServerSyncTool $serverSync)
{
$this->Server = ClassRegistry::init('Server');
@ -668,30 +671,34 @@ class AnalystData extends AppModel
return 0;
}
// Downloads new analyst data and the ones newer than local.
$allRemoteUUIDs = [];
foreach (self::ANALYST_DATA_TYPES as $type) {
$allRemoteUUIDs = array_merge($allRemoteUUIDs, array_keys($remoteData[$type]));
}
$localAnalystData = $this->getAllAnalystData('list', [
'Event.uuid' => array_column($remoteData, 'uuid')
'conditions' => ['uuid' => $allRemoteUUIDs],
'fields' => ['uuid', 'modified'],
]);
$remoteDataUuids = [];
$remoteUUIDsToFetch = [];
foreach ($remoteData as $type => $remoteAnalystData) {
foreach ($remoteAnalystData as $i => $remoteEntry) {
if (
isset($localAnalystData[$remoteEntry['uuid']]) &&
strtotime($localAnalystData[$type][$remoteEntry['uuid']]) < strtotime($remoteEntry['modified'])
) {
$remoteDataUuids[$remoteEntry['uuid']] = $remoteEntry['modified'];
foreach ($remoteAnalystData as $remoteUUID => $remoteModified) {
if (!isset($localAnalystData[$type][$remoteUUID])) {
$remoteUUIDsToFetch[$type][$remoteUUID] = $remoteModified;
} elseif (strtotime($localAnalystData[$type][$remoteUUID]) < strtotime($remoteModified)) {
$remoteUUIDsToFetch[$type][$remoteUUID] = $remoteModified;
}
}
}
unset($remoteData, $localAnalystData);
unset($remoteData, $allRemoteUUIDs, $localAnalystData);
if (empty($remoteDataUuids)) {
if (empty($remoteUUIDsToFetch)) {
return 0;
}
if ($serverSync->isSupported(ServerSyncTool::PERM_ANALYST_DATA)) {
return $this->pullAnalystData($user, $remoteDataUuids, $serverSync);
return $this->pullAnalystData($user, $remoteUUIDsToFetch, $serverSync);
}
}
@ -699,18 +706,31 @@ class AnalystData extends AppModel
{
$uuids = array_keys($analystDataUuids);
$saved = 0;
foreach (array_chunk($uuids, 100) as $uuidChunk) {
try {
$chunkedAnalystData = $serverSync->fetchAnalystData($uuidChunk);
} catch (Exception $e) {
$this->logException("Failed downloading the chunked analyst data from {$serverSync->server()['Server']['name']}.", $e);
$serverOrgUUID = $this->Org->find('first', [
'recursive' => -1,
'conditions' => ['id' => $serverSync->server()['Server']['org_id']],
'fields' => ['id', 'uuid']
])['Organisation']['uuid'];
foreach ($analystDataUuids as $type => $entries) {
$uuids = array_keys($entries);
if (empty($uuids)) {
continue;
}
foreach ($chunkedAnalystData as $analystData) {
$savedAmount = $this->captureAnalystData($user, $analystData, true, $serverSync->server()['Server']['org_id'], $serverSync->server());
if ($savedAmount) {
$saved += $savedAmount;
foreach (array_chunk($uuids, 100) as $uuidChunk) {
try {
$chunkedAnalystData = $serverSync->fetchAnalystData($type, $uuidChunk)->json();
} catch (Exception $e) {
$this->logException("Failed downloading the chunked analyst data from {$serverSync->server()['Server']['name']}.", $e);
continue;
}
foreach ($chunkedAnalystData as $analystData) {
$savedResult = $this->captureAnalystData($user, $analystData, true, $serverOrgUUID, $serverSync->server());
if ($savedResult['success']) {
$saved += $savedResult['imported'];
}
}
}
}