mirror of https://github.com/MISP/MISP
new: [sync] When pushing event, upload sightings by another call
Jakub Onderka
parent
6af83b5d5a
commit
d82a95b903
|
|
@ -123,7 +123,7 @@ class ServerSyncTool
|
|||
*/
|
||||
private function post($url, $data, $logMessage = null)
|
||||
{
|
||||
$data = json_encode($data);
|
||||
$data = json_encode($data, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
|
||||
|
||||
if ($logMessage && !empty(Configure::read('Security.sync_audit'))) {
|
||||
$pushLogEntry = sprintf(
|
||||
|
|
|
|||
|
|
@ -4344,38 +4344,10 @@ class Event extends AppModel
|
|||
return true;
|
||||
}
|
||||
|
||||
$this->Sighting = ClassRegistry::init('Sighting');
|
||||
App::uses('ServerSyncTool', 'Tools');
|
||||
|
||||
$failedServers = [];
|
||||
foreach ($servers as $server) {
|
||||
$serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server));
|
||||
try {
|
||||
if ($serverSync->eventExists($event) === false) {
|
||||
continue;
|
||||
}
|
||||
} catch (Exception $e) {}
|
||||
|
||||
$fakeSyncUser = [
|
||||
'org_id' => $server['Server']['remote_org_id'],
|
||||
'Role' => [
|
||||
'perm_site_admin' => 0,
|
||||
],
|
||||
];
|
||||
|
||||
$sightingsUuids = $this->Sighting->fetchUuidsForEvent($event, $fakeSyncUser);
|
||||
if (empty($sightingsUuids)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
// Filter out sightings that already exists on remote server
|
||||
$existingSightings = $serverSync->filterSightingUuidsForPush($event, $sightingsUuids);
|
||||
$sightings = $this->Sighting->attachToEvent($event, $fakeSyncUser, null, ['NOT' => ['Sighting.uuid' => $existingSightings]], true);
|
||||
if (empty($sightings)) {
|
||||
continue;
|
||||
}
|
||||
$serverSync->uploadSightings($sightings, $event['Event']['uuid']);
|
||||
$this->pushSightingsToServer($server, $event);
|
||||
} catch (Exception $e) {
|
||||
$this->logException("Uploading sightings to server {$server['Server']['id']} failed.", $e);
|
||||
$failedServers[] = $server['Server']['url'];
|
||||
|
|
@ -4387,6 +4359,48 @@ class Event extends AppModel
|
|||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $server
|
||||
* @param array $event
|
||||
* @throws HttpClientJsonException
|
||||
* @throws JsonException
|
||||
*/
|
||||
private function pushSightingsToServer(array $server, array $event)
|
||||
{
|
||||
App::uses('ServerSyncTool', 'Tools');
|
||||
if (!isset($this->Sighting)) {
|
||||
$this->Sighting = ClassRegistry::init('Sighting');
|
||||
}
|
||||
$serverSync = new ServerSyncTool($server, $this->setupSyncRequest($server));
|
||||
try {
|
||||
if ($serverSync->eventExists($event) === false) {
|
||||
return;
|
||||
}
|
||||
} catch (Exception $e) {}
|
||||
|
||||
$fakeSyncUser = [
|
||||
'org_id' => $server['Server']['remote_org_id'],
|
||||
'Role' => [
|
||||
'perm_site_admin' => 0,
|
||||
],
|
||||
];
|
||||
|
||||
foreach ($this->Sighting->fetchUuidsForEventToPush($event, $fakeSyncUser) as $sightingsUuids) {
|
||||
// Filter out sightings that already exists on remote server
|
||||
$existingSightings = $serverSync->filterSightingUuidsForPush($event, $sightingsUuids);
|
||||
$sightingToPush = array_diff($sightingsUuids, $existingSightings);
|
||||
if (empty($sightingToPush)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$query = [
|
||||
'conditions' => ['Sighting.uuid' => $sightingToPush],
|
||||
];
|
||||
$sightings = $this->Sighting->attachToEvent($event, $fakeSyncUser, null, $query, true);
|
||||
$serverSync->uploadSightings($sightings, $event['Event']['uuid']);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param int $id
|
||||
* @param int|null $passAlong
|
||||
|
|
@ -4458,7 +4472,8 @@ class Event extends AppModel
|
|||
'includeAttachments' => true,
|
||||
'includeAllTags' => true,
|
||||
'deleted' => array(0,1),
|
||||
'excludeGalaxy' => 1
|
||||
'excludeGalaxy' => 1,
|
||||
'noSightings' => true, // sightings are pushed separately
|
||||
));
|
||||
if (!empty($server['Server']['internal'])) {
|
||||
$params['excludeLocalTags'] = 0;
|
||||
|
|
@ -4475,6 +4490,13 @@ class Event extends AppModel
|
|||
);
|
||||
$this->Server->syncGalaxyClusters($HttpSocket, $server, $fakeSyncUser, $technique=$event['Event']['id'], $event=$event);
|
||||
$thisUploaded = $this->uploadEventToServer($event, $server, $HttpSocket);
|
||||
if ($thisUploaded === 'Success') {
|
||||
try {
|
||||
$this->pushSightingsToServer($server, $event); // push sighting by method that check for duplicates
|
||||
} catch (Exception $e) {
|
||||
$this->logException("Uploading sightings to server {$server['Server']['id']} failed.", $e);
|
||||
}
|
||||
}
|
||||
if (isset($this->data['ShadowAttribute'])) {
|
||||
$this->Server->syncProposals($HttpSocket, $server, null, $id, $this);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -550,38 +550,50 @@ class Sighting extends AppModel
|
|||
/**
|
||||
* @param array $event
|
||||
* @param array $user
|
||||
* @return array
|
||||
* @return Generator<array>
|
||||
*/
|
||||
public function fetchUuidsForEvent(array $event, array $user)
|
||||
public function fetchUuidsForEventToPush(array $event, array $user)
|
||||
{
|
||||
$conditions = $this->createConditions($user, $event);
|
||||
if ($conditions === false) {
|
||||
return [];
|
||||
return null;
|
||||
}
|
||||
$conditions['Sighting.event_id'] = $event['Event']['id'];
|
||||
|
||||
return $this->find('column', [
|
||||
'conditions' => $conditions,
|
||||
'fields' => ['Sighting.uuid'],
|
||||
]);
|
||||
while (true) {
|
||||
$uuids = $this->find('column', [
|
||||
'conditions' => $conditions,
|
||||
'fields' => ['Sighting.uuid'],
|
||||
'limit' => 250000,
|
||||
'order' => ['Sighting.uuid'],
|
||||
]);
|
||||
$count = count($uuids);
|
||||
if ($count === 0) {
|
||||
return null;
|
||||
}
|
||||
yield $uuids;
|
||||
if ($count !== 250000) {
|
||||
return;
|
||||
}
|
||||
$conditions['Sighting.uuid >'] = $uuids[$count - 1];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array $event Just 'Event' object is enough
|
||||
* @param array $user
|
||||
* @param array|int|null $attribute Attribute model or attribute ID
|
||||
* @param array|bool $extraConditions
|
||||
* @param array|bool $extraQuery
|
||||
* @param bool $forSync
|
||||
* @return array|int
|
||||
*/
|
||||
public function attachToEvent(array $event, array $user, $attribute = null, $extraConditions = false, $forSync = false)
|
||||
public function attachToEvent(array $event, array $user, $attribute = null, $extraQuery = false, $forSync = false)
|
||||
{
|
||||
$conditions = $this->createConditions($user, $event);
|
||||
if ($conditions === false) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$contain = [];
|
||||
$conditions['Sighting.event_id'] = $event['Event']['id'];
|
||||
if (isset($attribute['Attribute']['id'])) {
|
||||
$conditions['Sighting.attribute_id'] = $attribute['Attribute']['id'];
|
||||
|
|
@ -594,13 +606,21 @@ class Sighting extends AppModel
|
|||
]);
|
||||
}
|
||||
|
||||
if ($extraConditions !== false) {
|
||||
$conditions['AND'] = $extraConditions;
|
||||
}
|
||||
$sightings = $this->find('all', array(
|
||||
$query = [
|
||||
'conditions' => $conditions,
|
||||
'recursive' => -1,
|
||||
));
|
||||
];
|
||||
if (isset($extraQuery['conditions'])) {
|
||||
$query['conditions']['AND'] = $extraQuery['conditions'];
|
||||
}
|
||||
if (isset($extraQuery['limit'])) {
|
||||
$query['limit'] = $extraQuery['limit'];
|
||||
}
|
||||
if (isset($extraQuery['order'])) {
|
||||
$query['order'] = $extraQuery['order'];
|
||||
}
|
||||
|
||||
$sightings = $this->find('all', $query);
|
||||
if (empty($sightings)) {
|
||||
return [];
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue