MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [internal] Server sync debug messages

pull/9651/head
Jakub Onderka 2024-03-30 15:06:42 +01:00
parent d861ff2b2d
commit 2b38de942b
7 changed files with 33 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
app/Lib/Tools/ServerSyncTool.php
View File

@ -506,6 +506,16 @@ class ServerSyncTool
return $this->socket->getMetaData();
}
/**
* @param string $message
* @return void
*/
public function debug($message)
{
$memoryUsage = round(memory_get_usage() / 1024 / 1024, 2);
CakeLog::debug("[Server sync #{$this->serverId()}]: $message. Memory: $memoryUsage MB");
}
/**
* @params string $url Relative URL
* @return HttpSocketResponseExtended
@ -556,6 +566,7 @@ class ServerSyncTool
if ($etag) {
// Remove compression marks that adds Apache for compressed content
// This can be removed in future as this is already checked by MISP itself since 2024-03
$etagWithoutQuotes = trim($etag, '"');
$dashPos = strrpos($etagWithoutQuotes, '-');
if ($dashPos && in_array(substr($etagWithoutQuotes, $dashPos + 1), ['br', 'gzip'], true)) {

1
app/Lib/Tools/SyncTool.php
View File

@ -1,7 +1,6 @@
<?php
class SyncTool
{
const ALLOWED_CERT_FILE_EXTENSIONS = ['pem', 'crt'];
/**

4
app/Model/AnalystData.php
View File

@ -641,9 +641,8 @@ class AnalystData extends AppModel
return [];
}
$this->Server = ClassRegistry::init('Server');
$this->AnalystData = ClassRegistry::init('AnalystData');
$this->log("Starting Analyst Data sync with server #{$server['Server']['id']}", LOG_INFO);
$serverSync->debug("Starting Analyst Data sync");
$analystData = $this->collectDataForPush($serverSync->server());
$keyedAnalystData = [];
@ -1018,7 +1017,6 @@ class AnalystData extends AppModel
}
$this->Server = ClassRegistry::init('Server');
$this->AnalystData = ClassRegistry::init('AnalystData');
try {
$filterRules = $this->buildPullFilterRules($serverSync->server());
$remoteData = $serverSync->fetchIndexMinimal($filterRules)->json();

3
app/Model/GalaxyCluster.php
View File

@ -1845,6 +1845,9 @@ class GalaxyCluster extends AppModel
if (!$compatible) {
return 0;
}
$serverSync->debug("Pulling galaxy clusters with technique $technique");
$clusterIds = $this->getClusterIdListBasedOnPullTechnique($user, $technique, $serverSync);
$successes = 0;
// now process the $clusterIds to pull each of the events sequentially

18
app/Model/Server.php
View File

@ -604,6 +604,7 @@ class Server extends AppModel
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
* @throws JsonException
* @throws Exception
*/
public function pull(array $user, $technique, array $server, $jobId = false, $force = false)
{
@ -619,7 +620,7 @@ class Server extends AppModel
try {
$server['Server']['version'] = $serverSync->info()['version'];
} catch (Exception $e) {
$this->logException("Could not get remote server `{$server['Server']['name']}` version.", $e);
$this->logException("Could not get remote server `{$serverSync->serverName()}` version.", $e);
if ($e instanceof HttpSocketHttpException && $e->getCode() === 403) {
$message = __('Not authorised. This is either due to an invalid auth key, or due to the sync user not having authentication permissions enabled on the remote server. Another reason could be an incorrect sync server setting.');
} else {
@ -648,6 +649,8 @@ class Server extends AppModel
}
}
$serverSync->debug("Pulling event list with technique $technique");
try {
$eventIds = $this->__getEventIdListBasedOnPullTechnique($technique, $serverSync, $force);
} catch (Exception $e) {
@ -673,26 +676,29 @@ class Server extends AppModel
$job->saveProgress($jobId, __n('Pulling %s event.', 'Pulling %s events.', count($eventIds), count($eventIds)));
}
foreach ($eventIds as $k => $eventId) {
$serverSync->debug("Pulling event $eventId");
$this->__pullEvent($eventId, $successes, $fails, $eventModel, $serverSync, $user, $jobId, $force);
if ($jobId && $k % 10 === 0) {
$job->saveProgress($jobId, null, 10 + 40 * (($k + 1) / count($eventIds)));
}
}
foreach ($fails as $eventid => $message) {
$this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['Server']['id'], "Failed to pull event #$eventid.", 'Reason: ' . $message);
$this->loadLog()->createLogEntry($user, 'pull', 'Server', $serverSync->serverId(), "Failed to pull event #$eventid.", 'Reason: ' . $message);
}
}
if ($jobId) {
$job->saveProgress($jobId, 'Pulling proposals.', 50);
}
$pulledProposals = $pulledSightings = 0;
$pulledProposals = $pulledSightings = $pulledAnalystData = 0;
if ($technique === 'full' || $technique === 'update') {
$pulledProposals = $eventModel->ShadowAttribute->pullProposals($user, $serverSync);
if ($jobId) {
$job->saveProgress($jobId, 'Pulling sightings.', 75);
}
$pulledSightings = $eventModel->Sighting->pullSightings($user, $serverSync);
$this->AnalystData = ClassRegistry::init('AnalystData');
$pulledAnalystData = $this->AnalystData->pull($user, $serverSync);
}
@ -819,7 +825,7 @@ class Server extends AppModel
*/
public function getElligibleClusterIdsFromServerForPull(ServerSyncTool $serverSync, $onlyUpdateLocalCluster=true, array $eligibleClusters=array(), array $conditions=array())
{
$this->log("Fetching eligible clusters from server #{$serverSync->serverId()} for pull: " . JsonTool::encode($conditions), LOG_INFO);
$serverSync->debug("Fetching eligible clusters for pull: " . JsonTool::encode($conditions));
if ($onlyUpdateLocalCluster && empty($eligibleClusters)) {
return []; // no clusters for update
@ -875,7 +881,7 @@ class Server extends AppModel
*/
private function getElligibleClusterIdsFromServerForPush(ServerSyncTool $serverSync, array $localClusters=array(), array $conditions=array())
{
$this->log("Fetching eligible clusters from server #{$serverSync->serverId()} for push: " . JsonTool::encode($conditions), LOG_INFO);
$serverSync->debug("Fetching eligible clusters for push: " . JsonTool::encode($conditions));
$clusterArray = $this->fetchCustomClusterIdsFromServer($serverSync, $conditions=$conditions);
$keyedClusterArray = Hash::combine($clusterArray, '{n}.GalaxyCluster.uuid', '{n}.GalaxyCluster.version');
if (!empty($localClusters)) {
@ -1372,7 +1378,7 @@ class Server extends AppModel
return []; // pushing clusters is not enabled
}
$this->log("Starting $technique clusters sync with server #{$serverSync->serverId()}", LOG_INFO);
$serverSync->debug("Starting $technique clusters sync");
$this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
$this->Event = ClassRegistry::init('Event');

2
app/Model/ShadowAttribute.php
View File

@ -706,6 +706,8 @@ class ShadowAttribute extends AppModel
return 0;
}
$serverSync->debug("Pulling proposals");
$i = 1;
$fetchedCount = 0;
$chunkSize = 1000;

4
app/Model/Sighting.php
View File

@ -1418,6 +1418,8 @@ class Sighting extends AppModel
*/
public function pullSightings(array $user, ServerSyncTool $serverSync)
{
$serverSync->debug("Fetching event index for pulling sightings");
$this->Server = ClassRegistry::init('Server');
try {
$remoteEvents = $this->Server->getEventIndexFromServer($serverSync);
@ -1452,6 +1454,8 @@ class Sighting extends AppModel
return 0;
}
$serverSync->debug("Pulling sightings for " . count($eventUuids) . " events");
if ($serverSync->isSupported(ServerSyncTool::FEATURE_SIGHTING_REST_SEARCH)) {
return $this->pullSightingNewWay($user, $eventUuids, $serverSync);
} else {