mirror of https://github.com/MISP/MISP
chg: [sync] Pull just necessary data when pulling sightings
Jakub Onderka
parent
1f10a88504
commit
0174336156
|
|
@ -1322,20 +1322,20 @@ class ServersController extends AppController
|
|||
$remote_server = $this->Server->find('first', array('conditions' => $conditions));
|
||||
if (!empty($remote_server)) {
|
||||
try {
|
||||
$remote_event = $this->Event->downloadEventFromServer($this->request->data['Event']['uuid'], $remote_server, null, true);
|
||||
$remote_event = $this->Event->downloadEventMetadataFromServer($this->request->data['Event']['uuid'], $remote_server);
|
||||
} catch (Exception $e) {
|
||||
$this->Flash->error(__("Issue while contacting the remote server to retrieve event information"));
|
||||
return;
|
||||
}
|
||||
|
||||
$local_event = $this->Event->fetchSimpleEvent($this->Auth->user(), $remote_event[0]['uuid']);
|
||||
$local_event = $this->Event->fetchSimpleEvent($this->Auth->user(), $remote_event['uuid']);
|
||||
// we record it to avoid re-querying the same server in the 2nd phase
|
||||
if (!empty($local_event)) {
|
||||
$remote_events[] = array(
|
||||
"server_id" => $remote_server['Server']['id'],
|
||||
"server_name" => $remote_server['Server']['name'],
|
||||
"url" => $remote_server['Server']['url']."/events/view/".$remote_event[0]['id'],
|
||||
"remote_id" => $remote_event[0]['id']
|
||||
"url" => $remote_server['Server']['url']."/events/view/".$remote_event['id'],
|
||||
"remote_id" => $remote_event['id']
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -1356,12 +1356,12 @@ class ServersController extends AppController
|
|||
|
||||
$exception = null;
|
||||
try {
|
||||
$remoteEvent = $this->Event->downloadEventFromServer($local_event['Event']['uuid'], $server, null, true);
|
||||
$remoteEvent = $this->Event->downloadEventMetadataFromServer($local_event['Event']['uuid'], $server);
|
||||
} catch (Exception $e) {
|
||||
$remoteEvent = null;
|
||||
$exception = $e->getMessage();
|
||||
}
|
||||
$remoteEventId = isset($remoteEvent[0]['id']) ? $remoteEvent[0]['id'] : null;
|
||||
$remoteEventId = isset($remoteEvent['id']) ? $remoteEvent['id'] : null;
|
||||
$remote_events[] = array(
|
||||
"server_id" => $server['Server']['id'],
|
||||
"server_name" => $server['Server']['name'],
|
||||
|
|
|
|||
|
|
@ -1327,37 +1327,22 @@ class Event extends AppModel
|
|||
}
|
||||
|
||||
/**
|
||||
* Download event from remote server.
|
||||
* Download event metadata from remote server.
|
||||
*
|
||||
* @param int $eventId
|
||||
* @param array $server
|
||||
* @param null|HttpSocket $HttpSocket
|
||||
* @param boolean $metadataOnly, if True, we only retrieve the metadata, without attributes and attachments which is much faster
|
||||
* @return array
|
||||
* @param bool $minimal Return just minimal event response
|
||||
* @return array|null Null when event doesn't exists on remote server
|
||||
* @throws Exception
|
||||
*/
|
||||
public function downloadEventFromServer($eventId, $server, HttpSocket $HttpSocket=null, $metadataOnly=false)
|
||||
public function downloadEventMetadataFromServer($eventId, $server, $minimal = false)
|
||||
{
|
||||
$url = $server['Server']['url'];
|
||||
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
|
||||
$request = $this->setupSyncRequest($server);
|
||||
if ($metadataOnly) {
|
||||
$uri = $url . '/events/index';
|
||||
$data = json_encode(['eventid' => $eventId]);
|
||||
$response = $HttpSocket->post($uri, $data, $request);
|
||||
} else {
|
||||
$uri = $url . '/events/view/' . $eventId . '/deleted[]:0/deleted[]:1/excludeGalaxy:1';
|
||||
if (empty($server['Server']['internal'])) {
|
||||
$uri = $uri . '/excludeLocalTags:1';
|
||||
}
|
||||
$response = $HttpSocket->get($uri, [], $request);
|
||||
$serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server));
|
||||
$data = $serverSync->eventIndex(['eventid' => $eventId, 'minimal' => $minimal ? '1' : '0'])->json();
|
||||
if (empty($data)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!$response->isOk()) {
|
||||
throw new Exception("Fetching the '$uri' failed with HTTP error {$response->code}: {$response->reasonPhrase}");
|
||||
}
|
||||
|
||||
return $this->jsonDecode($response->body);
|
||||
return $data;
|
||||
}
|
||||
|
||||
public function quickDelete($event)
|
||||
|
|
|
|||
|
|
@ -588,7 +588,7 @@ class Server extends AppModel
|
|||
if ($jobId) {
|
||||
$job->saveProgress($jobId, 'Pulling sightings.', 75);
|
||||
}
|
||||
$pulledSightings = $eventModel->Sighting->pullSightings($user, $server);
|
||||
$pulledSightings = $eventModel->Sighting->pullSightings($user, $serverSync);
|
||||
}
|
||||
if ($jobId) {
|
||||
$job->saveProgress($jobId, 'Pull completed.', 100);
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
<?php
|
||||
App::uses('AppModel', 'Model');
|
||||
App::uses('TmpFileTool', 'Tools');
|
||||
App::uses('ServerSyncTool', 'Tools');
|
||||
|
||||
/**
|
||||
* @property Attribute $Attribute
|
||||
|
|
@ -1101,36 +1102,47 @@ class Sighting extends AppModel
|
|||
}
|
||||
}
|
||||
|
||||
public function pullSightings($user, $server)
|
||||
public function pullSightings(array $user, ServerSyncTool $serverSync)
|
||||
{
|
||||
$HttpSocket = $this->setupHttpSocket($server);
|
||||
$this->Server = ClassRegistry::init('Server');
|
||||
try {
|
||||
$eventIds = $this->Server->getEventIdsFromServer($server, false, $HttpSocket, false, 'sightings');
|
||||
$eventUuids = $this->Server->getEventIdsFromServer($serverSync->server(), false, null, false, 'sightings');
|
||||
} catch (Exception $e) {
|
||||
$this->logException("Could not fetch event IDs from server {$server['Server']['name']}", $e);
|
||||
$this->logException("Could not fetch event IDs from server {$serverSync->server()['Server']['name']}", $e);
|
||||
return 0;
|
||||
}
|
||||
$saved = 0;
|
||||
// now process the $eventIds to pull each of the events sequentially
|
||||
// We don't need some of the event data, like correlations and event reports
|
||||
$params = [
|
||||
'deleted' => [0, 1],
|
||||
'excludeGalaxy' => 1,
|
||||
'excludeLocalTags' => 1,
|
||||
'includeAttachments' => 0,
|
||||
'includeEventCorrelations' => 0,
|
||||
'includeFeedCorrelations' => 0,
|
||||
'includeWarninglistHits' => 0,
|
||||
'noEventReports' => 1,
|
||||
'noShadowAttributes' => 1,
|
||||
];
|
||||
// now process the $eventUuids to pull each of the events sequentially
|
||||
// download each event and save sightings
|
||||
foreach ($eventIds as $k => $eventId) {
|
||||
foreach ($eventUuids as $eventUuid) {
|
||||
try {
|
||||
$event = $this->Event->downloadEventFromServer($eventId, $server);
|
||||
$event = $serverSync->fetchEvent($eventUuid, $params)->json();
|
||||
} catch (Exception $e) {
|
||||
$this->logException("Failed downloading the event $eventId from {$server['Server']['name']}.", $e);
|
||||
$this->logException("Failed downloading the event $eventUuid from {$serverSync->server()['Server']['name']}.", $e);
|
||||
continue;
|
||||
}
|
||||
$sightings = array();
|
||||
if (!empty($event) && !empty($event['Event']['Attribute'])) {
|
||||
if (!empty($event['Event']['Attribute'])) {
|
||||
foreach ($event['Event']['Attribute'] as $attribute) {
|
||||
if (!empty($attribute['Sighting'])) {
|
||||
$sightings = array_merge($sightings, $attribute['Sighting']);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!empty($event) && !empty($sightings)) {
|
||||
$result = $this->bulkSaveSightings($event['Event']['uuid'], $sightings, $user, $server['Server']['id']);
|
||||
if (!empty($sightings)) {
|
||||
$result = $this->bulkSaveSightings($event['Event']['uuid'], $sightings, $user, $serverSync->server()['Server']['id']);
|
||||
if (is_numeric($result)) {
|
||||
$saved += $result;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue