mirror of https://github.com/MISP/MISP
chg: [sync] Simplify code for sighting pushing
Jakub Onderka
parent
625032e58b
commit
90cd99685f
|
|
@ -439,7 +439,7 @@ class EventShell extends AppShell
|
|||
}
|
||||
|
||||
$this->Event->Behaviors->unload('SysLogLogable.SysLogLogable');
|
||||
$result = $this->Event->publish_sightings($id, $passAlong, $sightingsUuidsToPush);
|
||||
$result = $this->Event->publishSightings($id, $passAlong, $sightingsUuidsToPush);
|
||||
|
||||
$count = count($sightingsUuidsToPush);
|
||||
$message = $count === 0 ? "All sightings published" : "$count sightings published";
|
||||
|
|
|
|||
|
|
@ -4548,7 +4548,7 @@ class Event extends AppModel
|
|||
);
|
||||
}
|
||||
|
||||
return $this->publish_sightings($id, $passAlong, $sightingUuids);
|
||||
return $this->publishSightings($id, $passAlong, $sightingUuids);
|
||||
}
|
||||
|
||||
public function publishRouter($id, $passAlong = null, $user)
|
||||
|
|
@ -4577,7 +4577,14 @@ class Event extends AppModel
|
|||
return $this->publish($id, $passAlong);
|
||||
}
|
||||
|
||||
public function publish_sightings($id, $passAlong = null, array $sightingsUuidsToPush = [])
|
||||
/**
|
||||
* @param int|string $id Event ID or UUID
|
||||
* @param $passAlong
|
||||
* @param array $sightingsUuidsToPush
|
||||
* @return array|bool
|
||||
* @throws Exception
|
||||
*/
|
||||
public function publishSightings($id, $passAlong = null, array $sightingsUuidsToPush = [])
|
||||
{
|
||||
if (is_numeric($id)) {
|
||||
$condition = array('Event.id' => $id);
|
||||
|
|
|
|||
|
|
@ -204,11 +204,11 @@ class Server extends AppModel
|
|||
{
|
||||
if ("full" === $technique) {
|
||||
// get a list of the event_ids on the server
|
||||
$eventIds = $this->getEventIdsFromServer($serverSync, false, false, 'events', $force);
|
||||
$eventIds = $this->getEventIdsFromServer($serverSync, false, false, $force);
|
||||
// reverse array of events, to first get the old ones, and then the new ones
|
||||
return array_reverse($eventIds);
|
||||
} elseif ("update" === $technique) {
|
||||
$eventIds = $this->getEventIdsFromServer($serverSync, false, true, 'events', $force);
|
||||
$eventIds = $this->getEventIdsFromServer($serverSync, false, true, $force);
|
||||
$eventModel = ClassRegistry::init('Event');
|
||||
$localEventUuids = $eventModel->find('column', array(
|
||||
'fields' => array('Event.uuid'),
|
||||
|
|
@ -762,7 +762,13 @@ class Server extends AppModel
|
|||
}
|
||||
$filterRules['minimal'] = 1;
|
||||
$filterRules['published'] = 1;
|
||||
return $serverSync->eventIndex($filterRules)->json();
|
||||
$eventIndex = $serverSync->eventIndex($filterRules)->json();
|
||||
|
||||
// correct $eventArray if just one event, probably this response returns old MISP
|
||||
if (isset($eventIndex['id'])) {
|
||||
$eventIndex = [$eventIndex];
|
||||
}
|
||||
return $eventIndex;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -771,65 +777,40 @@ class Server extends AppModel
|
|||
* @param ServerSyncTool $serverSync
|
||||
* @param bool $all
|
||||
* @param bool $ignoreFilterRules
|
||||
* @param string $scope 'events' or 'sightings'
|
||||
* @param bool $force
|
||||
* @return array Array of event UUIDs.
|
||||
* @throws HttpSocketHttpException
|
||||
* @throws HttpSocketJsonException
|
||||
* @throws InvalidArgumentException
|
||||
*/
|
||||
private function getEventIdsFromServer(ServerSyncTool $serverSync, $all = false, $ignoreFilterRules = false, $scope = 'events', $force = false)
|
||||
private function getEventIdsFromServer(ServerSyncTool $serverSync, $all = false, $ignoreFilterRules = false, $force = false)
|
||||
{
|
||||
if (!in_array($scope, ['events', 'sightings'], true)) {
|
||||
throw new InvalidArgumentException("Scope must be 'events' or 'sightings', '$scope' given.");
|
||||
}
|
||||
|
||||
$eventArray = $this->getEventIndexFromServer($serverSync, $ignoreFilterRules);
|
||||
// correct $eventArray if just one event
|
||||
if (isset($eventArray['id'])) {
|
||||
$eventArray = array($eventArray);
|
||||
}
|
||||
|
||||
if ($all) {
|
||||
if ($scope === 'sightings') {
|
||||
// Used when pushing: return just eventUuids that has sightings newer than remote server
|
||||
$this->Event = ClassRegistry::init('Event');
|
||||
$localEvents = $this->Event->find('list', array(
|
||||
'fields' => array('Event.uuid', 'Event.sighting_timestamp'),
|
||||
'conditions' => array('Event.uuid' => array_column($eventArray, 'uuid'))
|
||||
));
|
||||
|
||||
$eventUuids = [];
|
||||
foreach ($eventArray as $event) {
|
||||
if (isset($localEvents[$event['uuid']]) && $localEvents[$event['uuid']] > $event['sighting_timestamp']) {
|
||||
$eventUuids[] = $event['uuid'];
|
||||
}
|
||||
}
|
||||
} else {
|
||||
$eventUuids = array_column($eventArray, 'uuid');
|
||||
}
|
||||
} else {
|
||||
if (Configure::read('MISP.enableEventBlocklisting') !== false) {
|
||||
$this->EventBlocklist = ClassRegistry::init('EventBlocklist');
|
||||
$this->EventBlocklist->removeBlockedEvents($eventArray);
|
||||
}
|
||||
|
||||
if (Configure::read('MISP.enableOrgBlocklisting') !== false) {
|
||||
$this->OrgBlocklist = ClassRegistry::init('OrgBlocklist');
|
||||
$this->OrgBlocklist->removeBlockedEvents($eventArray);
|
||||
}
|
||||
|
||||
foreach ($eventArray as $k => $event) {
|
||||
if (1 != $event['published']) {
|
||||
unset($eventArray[$k]); // do not keep non-published events
|
||||
}
|
||||
}
|
||||
if (!$force) {
|
||||
$this->Event = ClassRegistry::init('Event');
|
||||
$this->Event->removeOlder($eventArray, $scope);
|
||||
}
|
||||
$eventUuids = array_column($eventArray, 'uuid');
|
||||
return array_column($eventArray, 'uuid');
|
||||
}
|
||||
return $eventUuids;
|
||||
|
||||
if (Configure::read('MISP.enableEventBlocklisting') !== false) {
|
||||
$this->EventBlocklist = ClassRegistry::init('EventBlocklist');
|
||||
$this->EventBlocklist->removeBlockedEvents($eventArray);
|
||||
}
|
||||
|
||||
if (Configure::read('MISP.enableOrgBlocklisting') !== false) {
|
||||
$this->OrgBlocklist = ClassRegistry::init('OrgBlocklist');
|
||||
$this->OrgBlocklist->removeBlockedEvents($eventArray);
|
||||
}
|
||||
|
||||
foreach ($eventArray as $k => $event) {
|
||||
if (1 != $event['published']) {
|
||||
unset($eventArray[$k]); // do not keep non-published events
|
||||
}
|
||||
}
|
||||
if (!$force) {
|
||||
$this->Event = ClassRegistry::init('Event');
|
||||
$this->Event->removeOlder($eventArray);
|
||||
}
|
||||
return array_column($eventArray, 'uuid');
|
||||
}
|
||||
|
||||
public function serverEventsOverlap()
|
||||
|
|
@ -900,9 +881,11 @@ class Server extends AppModel
|
|||
if (!$server) {
|
||||
throw new NotFoundException('Server not found');
|
||||
}
|
||||
$serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server));
|
||||
|
||||
$this->Event = ClassRegistry::init('Event');
|
||||
$url = $server['Server']['url'];
|
||||
$push = $this->checkVersionCompatibility($server, $user);
|
||||
$push = $this->checkVersionCompatibility($server, $user, $serverSync);
|
||||
if (is_array($push) && !$push['canPush'] && !$push['canSight']) {
|
||||
$push = 'Remote instance is outdated or no permission to push.';
|
||||
}
|
||||
|
|
@ -911,14 +894,14 @@ class Server extends AppModel
|
|||
$this->Log = ClassRegistry::init('Log');
|
||||
$this->Log->create();
|
||||
$this->Log->save(array(
|
||||
'org' => $user['Organisation']['name'],
|
||||
'model' => 'Server',
|
||||
'model_id' => $id,
|
||||
'email' => $user['email'],
|
||||
'action' => 'error',
|
||||
'user_id' => $user['id'],
|
||||
'title' => 'Failed: Push to ' . $url . ' initiated by ' . $user['email'],
|
||||
'change' => $message
|
||||
'org' => $user['Organisation']['name'],
|
||||
'model' => 'Server',
|
||||
'model_id' => $id,
|
||||
'email' => $user['email'],
|
||||
'action' => 'error',
|
||||
'user_id' => $user['id'],
|
||||
'title' => 'Failed: Push to ' . $url . ' initiated by ' . $user['email'],
|
||||
'change' => $message
|
||||
));
|
||||
if ($jobId) {
|
||||
$job->saveStatus($jobId, false, $message);
|
||||
|
|
@ -1052,7 +1035,8 @@ class Server extends AppModel
|
|||
}
|
||||
|
||||
if ($push['canPush'] || $push['canSight']) {
|
||||
$sightingSuccesses = $this->syncSightings($HttpSocket, $server, $user, $this->Event);
|
||||
$this->Sighting = ClassRegistry::init('Sighting');
|
||||
$sightingSuccesses =$this->Sighting->pushSightings($user, $serverSync);
|
||||
} else {
|
||||
$sightingSuccesses = array();
|
||||
}
|
||||
|
|
@ -1069,14 +1053,14 @@ class Server extends AppModel
|
|||
$this->Log = ClassRegistry::init('Log');
|
||||
$this->Log->create();
|
||||
$this->Log->save(array(
|
||||
'org' => $user['Organisation']['name'],
|
||||
'model' => 'Server',
|
||||
'model_id' => $id,
|
||||
'email' => $user['email'],
|
||||
'action' => 'push',
|
||||
'user_id' => $user['id'],
|
||||
'title' => 'Push to ' . $url . ' initiated by ' . $user['email'],
|
||||
'change' => count($successes) . ' events pushed or updated. ' . count($fails) . ' events failed or didn\'t need an update.'
|
||||
'org' => $user['Organisation']['name'],
|
||||
'model' => 'Server',
|
||||
'model_id' => $id,
|
||||
'email' => $user['email'],
|
||||
'action' => 'push',
|
||||
'user_id' => $user['id'],
|
||||
'title' => 'Push to ' . $url . ' initiated by ' . $user['email'],
|
||||
'change' => count($successes) . ' events pushed or updated. ' . count($fails) . ' events failed or didn\'t need an update.'
|
||||
));
|
||||
if ($jobId) {
|
||||
$job->saveStatus($jobId, true, __('Push to server %s complete.', $id));
|
||||
|
|
@ -1152,70 +1136,6 @@ class Server extends AppModel
|
|||
return $successes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Push sightings to remote server.
|
||||
* @param HttpSocket $HttpSocket
|
||||
* @param array $server
|
||||
* @param array $user
|
||||
* @param Event $eventModel
|
||||
* @return array
|
||||
* @throws Exception
|
||||
*/
|
||||
private function syncSightings($HttpSocket, array $server, array $user, Event $eventModel)
|
||||
{
|
||||
$successes = array();
|
||||
if (!$server['Server']['push_sightings']) {
|
||||
return $successes;
|
||||
}
|
||||
$serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server));
|
||||
$this->Sighting = ClassRegistry::init('Sighting');
|
||||
try {
|
||||
$eventUuids = $this->getEventIdsFromServer($serverSync, true, true, 'sightings');
|
||||
} catch (Exception $e) {
|
||||
$this->logException("Could not fetch event IDs from server {$server['Server']['name']}", $e);
|
||||
return $successes;
|
||||
}
|
||||
// now process the $eventIds to push each of the events sequentially
|
||||
// check each event and push sightings when needed
|
||||
$fakeSyncUser = [
|
||||
'org_id' => $server['Server']['remote_org_id'],
|
||||
'Role' => [
|
||||
'perm_site_admin' => 0,
|
||||
],
|
||||
];
|
||||
|
||||
foreach ($eventUuids as $eventUuid) {
|
||||
$event = $eventModel->fetchEvent($user, ['event_uuid' => $eventUuid, 'metadata' => true]);
|
||||
if (!empty($event)) {
|
||||
$event = $event[0];
|
||||
|
||||
if (empty($this->eventFilterPushableServers($event, [$server]))) {
|
||||
continue;
|
||||
}
|
||||
if (!$eventModel->checkDistributionForPush($event, $server)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Process sightings in batch to keep memory requirements low
|
||||
foreach ($this->Sighting->fetchUuidsForEventToPush($event, $fakeSyncUser) as $batch) {
|
||||
// Filter out sightings that already exists on remote server
|
||||
$existingSightings = $serverSync->filterSightingUuidsForPush($event, $batch);
|
||||
$newSightings = array_diff($batch, $existingSightings);
|
||||
if (empty($newSightings)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$conditions = ['Sighting.uuid' => $newSightings];
|
||||
$sightings = $this->Sighting->attachToEvent($event, $fakeSyncUser, null, $conditions, true);
|
||||
$serverSync->uploadSightings($sightings, $event['Event']['uuid']);
|
||||
}
|
||||
|
||||
$successes[] = 'Sightings for event ' . $event['Event']['id'];
|
||||
}
|
||||
}
|
||||
return $successes;
|
||||
}
|
||||
|
||||
public function syncProposals($HttpSocket, array $server, $sa_id = null, $event_id = null, $eventModel)
|
||||
{
|
||||
$saModel = ClassRegistry::init('ShadowAttribute');
|
||||
|
|
|
|||
|
|
@ -1107,6 +1107,90 @@ class Sighting extends AppModel
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Push sightings to remote server.
|
||||
* @param array $user
|
||||
* @param ServerSyncTool $serverSync
|
||||
* @return array
|
||||
* @throws Exception
|
||||
*/
|
||||
public function pushSightings(array $user, ServerSyncTool $serverSync)
|
||||
{
|
||||
$server = $serverSync->server();
|
||||
|
||||
if (!$serverSync->server()['Server']['push_sightings']) {
|
||||
return [];
|
||||
}
|
||||
$this->Server = ClassRegistry::init('Server');
|
||||
|
||||
try {
|
||||
$eventArray = $this->Server->getEventIndexFromServer($serverSync);
|
||||
} catch (Exception $e) {
|
||||
$this->logException("Could not fetch event IDs from server {$server['Server']['name']}", $e);
|
||||
return [];
|
||||
}
|
||||
|
||||
// Fetch local events that has sightings
|
||||
$localEvents = $this->Event->find('list', [
|
||||
'fields' => ['Event.uuid', 'Event.sighting_timestamp'],
|
||||
'conditions' => [
|
||||
'Event.uuid' => array_column($eventArray, 'uuid'),
|
||||
'Event.sighting_timestamp >' => 0,
|
||||
],
|
||||
]);
|
||||
|
||||
// Filter just local events that has sighting_timestamp newer than remote event
|
||||
$eventUuids = [];
|
||||
foreach ($eventArray as $event) {
|
||||
if (isset($localEvents[$event['uuid']]) && $localEvents[$event['uuid']] > $event['sighting_timestamp']) {
|
||||
$eventUuids[] = $event['uuid'];
|
||||
}
|
||||
}
|
||||
unset($localEvents, $eventArray);
|
||||
|
||||
$fakeSyncUser = [
|
||||
'org_id' => $server['Server']['remote_org_id'],
|
||||
'Role' => [
|
||||
'perm_site_admin' => 0,
|
||||
],
|
||||
];
|
||||
|
||||
$successes = [];
|
||||
// now process the $eventUuids to push each of the events sequentially
|
||||
// check each event and push sightings when needed
|
||||
foreach ($eventUuids as $eventUuid) {
|
||||
$event = $this->Event->fetchEvent($user, ['event_uuid' => $eventUuid, 'metadata' => true]);
|
||||
if (empty($event)) {
|
||||
continue;
|
||||
}
|
||||
$event = $event[0];
|
||||
|
||||
if (empty($this->Server->eventFilterPushableServers($event, [$server]))) {
|
||||
continue;
|
||||
}
|
||||
if (!$this->Event->checkDistributionForPush($event, $server)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Process sightings in batch to keep memory requirements low
|
||||
foreach ($this->fetchUuidsForEventToPush($event, $fakeSyncUser) as $batch) {
|
||||
// Filter out sightings that already exists on remote server
|
||||
$existingSightings = $serverSync->filterSightingUuidsForPush($event, $batch);
|
||||
$newSightings = array_diff($batch, $existingSightings);
|
||||
if (empty($newSightings)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$conditions = ['Sighting.uuid' => $newSightings];
|
||||
$sightings = $this->attachToEvent($event, $fakeSyncUser, null, $conditions, true);
|
||||
$serverSync->uploadSightings($sightings, $event['Event']['uuid']);
|
||||
}
|
||||
|
||||
$successes[] = 'Sightings for event ' . $event['Event']['id'];
|
||||
}
|
||||
return $successes;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $user
|
||||
* @param ServerSyncTool $serverSync
|
||||
|
|
|
|||
Loading…
Reference in New Issue