mirror of https://github.com/MISP/MISP
chg: [internal] Server sync debug messages
parent
d861ff2b2d
commit
2b38de942b
|
@ -506,6 +506,16 @@ class ServerSyncTool
|
||||||
return $this->socket->getMetaData();
|
return $this->socket->getMetaData();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $message
|
||||||
|
* @return void
|
||||||
|
*/
|
||||||
|
public function debug($message)
|
||||||
|
{
|
||||||
|
$memoryUsage = round(memory_get_usage() / 1024 / 1024, 2);
|
||||||
|
CakeLog::debug("[Server sync #{$this->serverId()}]: $message. Memory: $memoryUsage MB");
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @params string $url Relative URL
|
* @params string $url Relative URL
|
||||||
* @return HttpSocketResponseExtended
|
* @return HttpSocketResponseExtended
|
||||||
|
@ -556,6 +566,7 @@ class ServerSyncTool
|
||||||
|
|
||||||
if ($etag) {
|
if ($etag) {
|
||||||
// Remove compression marks that adds Apache for compressed content
|
// Remove compression marks that adds Apache for compressed content
|
||||||
|
// This can be removed in future as this is already checked by MISP itself since 2024-03
|
||||||
$etagWithoutQuotes = trim($etag, '"');
|
$etagWithoutQuotes = trim($etag, '"');
|
||||||
$dashPos = strrpos($etagWithoutQuotes, '-');
|
$dashPos = strrpos($etagWithoutQuotes, '-');
|
||||||
if ($dashPos && in_array(substr($etagWithoutQuotes, $dashPos + 1), ['br', 'gzip'], true)) {
|
if ($dashPos && in_array(substr($etagWithoutQuotes, $dashPos + 1), ['br', 'gzip'], true)) {
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
<?php
|
<?php
|
||||||
class SyncTool
|
class SyncTool
|
||||||
{
|
{
|
||||||
|
|
||||||
const ALLOWED_CERT_FILE_EXTENSIONS = ['pem', 'crt'];
|
const ALLOWED_CERT_FILE_EXTENSIONS = ['pem', 'crt'];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -641,9 +641,8 @@ class AnalystData extends AppModel
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
$this->Server = ClassRegistry::init('Server');
|
$this->Server = ClassRegistry::init('Server');
|
||||||
$this->AnalystData = ClassRegistry::init('AnalystData');
|
|
||||||
|
|
||||||
$this->log("Starting Analyst Data sync with server #{$server['Server']['id']}", LOG_INFO);
|
$serverSync->debug("Starting Analyst Data sync");
|
||||||
|
|
||||||
$analystData = $this->collectDataForPush($serverSync->server());
|
$analystData = $this->collectDataForPush($serverSync->server());
|
||||||
$keyedAnalystData = [];
|
$keyedAnalystData = [];
|
||||||
|
@ -1018,7 +1017,6 @@ class AnalystData extends AppModel
|
||||||
}
|
}
|
||||||
|
|
||||||
$this->Server = ClassRegistry::init('Server');
|
$this->Server = ClassRegistry::init('Server');
|
||||||
$this->AnalystData = ClassRegistry::init('AnalystData');
|
|
||||||
try {
|
try {
|
||||||
$filterRules = $this->buildPullFilterRules($serverSync->server());
|
$filterRules = $this->buildPullFilterRules($serverSync->server());
|
||||||
$remoteData = $serverSync->fetchIndexMinimal($filterRules)->json();
|
$remoteData = $serverSync->fetchIndexMinimal($filterRules)->json();
|
||||||
|
|
|
@ -1845,6 +1845,9 @@ class GalaxyCluster extends AppModel
|
||||||
if (!$compatible) {
|
if (!$compatible) {
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$serverSync->debug("Pulling galaxy clusters with technique $technique");
|
||||||
|
|
||||||
$clusterIds = $this->getClusterIdListBasedOnPullTechnique($user, $technique, $serverSync);
|
$clusterIds = $this->getClusterIdListBasedOnPullTechnique($user, $technique, $serverSync);
|
||||||
$successes = 0;
|
$successes = 0;
|
||||||
// now process the $clusterIds to pull each of the events sequentially
|
// now process the $clusterIds to pull each of the events sequentially
|
||||||
|
|
|
@ -604,6 +604,7 @@ class Server extends AppModel
|
||||||
* @throws HttpSocketHttpException
|
* @throws HttpSocketHttpException
|
||||||
* @throws HttpSocketJsonException
|
* @throws HttpSocketJsonException
|
||||||
* @throws JsonException
|
* @throws JsonException
|
||||||
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
public function pull(array $user, $technique, array $server, $jobId = false, $force = false)
|
public function pull(array $user, $technique, array $server, $jobId = false, $force = false)
|
||||||
{
|
{
|
||||||
|
@ -619,7 +620,7 @@ class Server extends AppModel
|
||||||
try {
|
try {
|
||||||
$server['Server']['version'] = $serverSync->info()['version'];
|
$server['Server']['version'] = $serverSync->info()['version'];
|
||||||
} catch (Exception $e) {
|
} catch (Exception $e) {
|
||||||
$this->logException("Could not get remote server `{$server['Server']['name']}` version.", $e);
|
$this->logException("Could not get remote server `{$serverSync->serverName()}` version.", $e);
|
||||||
if ($e instanceof HttpSocketHttpException && $e->getCode() === 403) {
|
if ($e instanceof HttpSocketHttpException && $e->getCode() === 403) {
|
||||||
$message = __('Not authorised. This is either due to an invalid auth key, or due to the sync user not having authentication permissions enabled on the remote server. Another reason could be an incorrect sync server setting.');
|
$message = __('Not authorised. This is either due to an invalid auth key, or due to the sync user not having authentication permissions enabled on the remote server. Another reason could be an incorrect sync server setting.');
|
||||||
} else {
|
} else {
|
||||||
|
@ -648,6 +649,8 @@ class Server extends AppModel
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$serverSync->debug("Pulling event list with technique $technique");
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$eventIds = $this->__getEventIdListBasedOnPullTechnique($technique, $serverSync, $force);
|
$eventIds = $this->__getEventIdListBasedOnPullTechnique($technique, $serverSync, $force);
|
||||||
} catch (Exception $e) {
|
} catch (Exception $e) {
|
||||||
|
@ -673,26 +676,29 @@ class Server extends AppModel
|
||||||
$job->saveProgress($jobId, __n('Pulling %s event.', 'Pulling %s events.', count($eventIds), count($eventIds)));
|
$job->saveProgress($jobId, __n('Pulling %s event.', 'Pulling %s events.', count($eventIds), count($eventIds)));
|
||||||
}
|
}
|
||||||
foreach ($eventIds as $k => $eventId) {
|
foreach ($eventIds as $k => $eventId) {
|
||||||
|
$serverSync->debug("Pulling event $eventId");
|
||||||
$this->__pullEvent($eventId, $successes, $fails, $eventModel, $serverSync, $user, $jobId, $force);
|
$this->__pullEvent($eventId, $successes, $fails, $eventModel, $serverSync, $user, $jobId, $force);
|
||||||
if ($jobId && $k % 10 === 0) {
|
if ($jobId && $k % 10 === 0) {
|
||||||
$job->saveProgress($jobId, null, 10 + 40 * (($k + 1) / count($eventIds)));
|
$job->saveProgress($jobId, null, 10 + 40 * (($k + 1) / count($eventIds)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
foreach ($fails as $eventid => $message) {
|
foreach ($fails as $eventid => $message) {
|
||||||
$this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['Server']['id'], "Failed to pull event #$eventid.", 'Reason: ' . $message);
|
$this->loadLog()->createLogEntry($user, 'pull', 'Server', $serverSync->serverId(), "Failed to pull event #$eventid.", 'Reason: ' . $message);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ($jobId) {
|
if ($jobId) {
|
||||||
$job->saveProgress($jobId, 'Pulling proposals.', 50);
|
$job->saveProgress($jobId, 'Pulling proposals.', 50);
|
||||||
}
|
}
|
||||||
$pulledProposals = $pulledSightings = 0;
|
$pulledProposals = $pulledSightings = $pulledAnalystData = 0;
|
||||||
if ($technique === 'full' || $technique === 'update') {
|
if ($technique === 'full' || $technique === 'update') {
|
||||||
$pulledProposals = $eventModel->ShadowAttribute->pullProposals($user, $serverSync);
|
$pulledProposals = $eventModel->ShadowAttribute->pullProposals($user, $serverSync);
|
||||||
|
|
||||||
if ($jobId) {
|
if ($jobId) {
|
||||||
$job->saveProgress($jobId, 'Pulling sightings.', 75);
|
$job->saveProgress($jobId, 'Pulling sightings.', 75);
|
||||||
}
|
}
|
||||||
|
|
||||||
$pulledSightings = $eventModel->Sighting->pullSightings($user, $serverSync);
|
$pulledSightings = $eventModel->Sighting->pullSightings($user, $serverSync);
|
||||||
|
|
||||||
$this->AnalystData = ClassRegistry::init('AnalystData');
|
$this->AnalystData = ClassRegistry::init('AnalystData');
|
||||||
$pulledAnalystData = $this->AnalystData->pull($user, $serverSync);
|
$pulledAnalystData = $this->AnalystData->pull($user, $serverSync);
|
||||||
}
|
}
|
||||||
|
@ -819,7 +825,7 @@ class Server extends AppModel
|
||||||
*/
|
*/
|
||||||
public function getElligibleClusterIdsFromServerForPull(ServerSyncTool $serverSync, $onlyUpdateLocalCluster=true, array $eligibleClusters=array(), array $conditions=array())
|
public function getElligibleClusterIdsFromServerForPull(ServerSyncTool $serverSync, $onlyUpdateLocalCluster=true, array $eligibleClusters=array(), array $conditions=array())
|
||||||
{
|
{
|
||||||
$this->log("Fetching eligible clusters from server #{$serverSync->serverId()} for pull: " . JsonTool::encode($conditions), LOG_INFO);
|
$serverSync->debug("Fetching eligible clusters for pull: " . JsonTool::encode($conditions));
|
||||||
|
|
||||||
if ($onlyUpdateLocalCluster && empty($eligibleClusters)) {
|
if ($onlyUpdateLocalCluster && empty($eligibleClusters)) {
|
||||||
return []; // no clusters for update
|
return []; // no clusters for update
|
||||||
|
@ -875,7 +881,7 @@ class Server extends AppModel
|
||||||
*/
|
*/
|
||||||
private function getElligibleClusterIdsFromServerForPush(ServerSyncTool $serverSync, array $localClusters=array(), array $conditions=array())
|
private function getElligibleClusterIdsFromServerForPush(ServerSyncTool $serverSync, array $localClusters=array(), array $conditions=array())
|
||||||
{
|
{
|
||||||
$this->log("Fetching eligible clusters from server #{$serverSync->serverId()} for push: " . JsonTool::encode($conditions), LOG_INFO);
|
$serverSync->debug("Fetching eligible clusters for push: " . JsonTool::encode($conditions));
|
||||||
$clusterArray = $this->fetchCustomClusterIdsFromServer($serverSync, $conditions=$conditions);
|
$clusterArray = $this->fetchCustomClusterIdsFromServer($serverSync, $conditions=$conditions);
|
||||||
$keyedClusterArray = Hash::combine($clusterArray, '{n}.GalaxyCluster.uuid', '{n}.GalaxyCluster.version');
|
$keyedClusterArray = Hash::combine($clusterArray, '{n}.GalaxyCluster.uuid', '{n}.GalaxyCluster.version');
|
||||||
if (!empty($localClusters)) {
|
if (!empty($localClusters)) {
|
||||||
|
@ -1372,7 +1378,7 @@ class Server extends AppModel
|
||||||
return []; // pushing clusters is not enabled
|
return []; // pushing clusters is not enabled
|
||||||
}
|
}
|
||||||
|
|
||||||
$this->log("Starting $technique clusters sync with server #{$serverSync->serverId()}", LOG_INFO);
|
$serverSync->debug("Starting $technique clusters sync");
|
||||||
|
|
||||||
$this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
|
$this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
|
||||||
$this->Event = ClassRegistry::init('Event');
|
$this->Event = ClassRegistry::init('Event');
|
||||||
|
|
|
@ -706,6 +706,8 @@ class ShadowAttribute extends AppModel
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$serverSync->debug("Pulling proposals");
|
||||||
|
|
||||||
$i = 1;
|
$i = 1;
|
||||||
$fetchedCount = 0;
|
$fetchedCount = 0;
|
||||||
$chunkSize = 1000;
|
$chunkSize = 1000;
|
||||||
|
|
|
@ -1418,6 +1418,8 @@ class Sighting extends AppModel
|
||||||
*/
|
*/
|
||||||
public function pullSightings(array $user, ServerSyncTool $serverSync)
|
public function pullSightings(array $user, ServerSyncTool $serverSync)
|
||||||
{
|
{
|
||||||
|
$serverSync->debug("Fetching event index for pulling sightings");
|
||||||
|
|
||||||
$this->Server = ClassRegistry::init('Server');
|
$this->Server = ClassRegistry::init('Server');
|
||||||
try {
|
try {
|
||||||
$remoteEvents = $this->Server->getEventIndexFromServer($serverSync);
|
$remoteEvents = $this->Server->getEventIndexFromServer($serverSync);
|
||||||
|
@ -1452,6 +1454,8 @@ class Sighting extends AppModel
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$serverSync->debug("Pulling sightings for " . count($eventUuids) . " events");
|
||||||
|
|
||||||
if ($serverSync->isSupported(ServerSyncTool::FEATURE_SIGHTING_REST_SEARCH)) {
|
if ($serverSync->isSupported(ServerSyncTool::FEATURE_SIGHTING_REST_SEARCH)) {
|
||||||
return $this->pullSightingNewWay($user, $eventUuids, $serverSync);
|
return $this->pullSightingNewWay($user, $eventUuids, $serverSync);
|
||||||
} else {
|
} else {
|
||||||
|
|
Loading…
Reference in New Issue