mirror of https://github.com/MISP/MISP
Push now also only does a differential push
- send uuids of events to be pushed together with timestamps to the other instance - other instance removes events that are already up to date or locally created from the array - sends the remaining uuids back - first instance initiates the push of events that were not filtered outpull/304/merge
iglocska
parent
e6c28fe69a
commit
8b16f0cf18
|
|
@ -977,7 +977,7 @@ class EventsController extends AppController {
|
|||
$fieldList = array(
|
||||
'Event' => array('date', 'threat_level_id', 'analysis', 'info', 'published', 'uuid', 'from', 'distribution', 'timestamp'),
|
||||
'Attribute' => array('event_id', 'category', 'type', 'value', 'value1', 'value2', 'to_ids', 'uuid', 'revision', 'distribution', 'timestamp', 'comment'),
|
||||
'ShadowAttribute' => array('event_id', 'category', 'type', 'value', 'value1', 'value2', 'org', 'event_org', 'comment', 'event_uuid', 'deleted', 'to_ids')
|
||||
'ShadowAttribute' => array('event_id', 'category', 'type', 'value', 'value1', 'value2', 'org', 'event_org', 'comment', 'event_uuid', 'deleted', 'to_ids', 'uuid')
|
||||
);
|
||||
|
||||
$c = 0;
|
||||
|
|
@ -1003,7 +1003,7 @@ class EventsController extends AppController {
|
|||
// check if the exact proposal exists, if yes check if the incoming one is deleted or not. If it is deleted, remove the old proposal and replace it with the one marked for being deleted
|
||||
// otherwise throw the new one away.
|
||||
if (isset($this->request->data['ShadowAttribute'])) {
|
||||
foreach ($this->request->data['ShadowAttribute'] as &$proposal) {
|
||||
foreach ($this->request->data['ShadowAttribute'] as $k => &$proposal) {
|
||||
$existingProposal = $this->Event->ShadowAttribute->find('first', array(
|
||||
'recursive' => -1,
|
||||
'conditions' => array(
|
||||
|
|
@ -1011,14 +1011,14 @@ class EventsController extends AppController {
|
|||
'category' => $proposal['category'],
|
||||
'to_ids' => $proposal['to_ids'],
|
||||
'type' => $proposal['type'],
|
||||
'event_uuid' => $event_uuid,
|
||||
'uuid' => $uuid
|
||||
'event_uuid' => $proposal['event_uuid'],
|
||||
'uuid' => $proposal['uuid']
|
||||
)
|
||||
));
|
||||
if ($existingProposal['ShadowAttribute']['deleted'] == 1) {
|
||||
$this->Event->ShadowAttribute->delete($existingProposal['ShadowAttribute']['id'], false);
|
||||
} else {
|
||||
unset($proposal);
|
||||
unset($this->request->data['ShadowAttribute'][$k]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1029,6 +1029,7 @@ class EventsController extends AppController {
|
|||
} else {
|
||||
throw new MethodNotAllowedException();
|
||||
}
|
||||
|
||||
if ($saveResult) {
|
||||
// TODO RESTfull: we now need to compare attributes, to see if we need to do a RESTfull attribute delete
|
||||
$message = 'Saved';
|
||||
|
|
@ -2561,4 +2562,31 @@ class EventsController extends AppController {
|
|||
throw new Exception(h($result['message']));
|
||||
}
|
||||
}
|
||||
|
||||
public function filterEventIdsForPush() {
|
||||
if (!$this->userRole['perm_sync']) throw new MethodNotAllowedException('You do not have the permission to do that.');
|
||||
if ($this->request->is('post')) {
|
||||
$incomingIDs = array();
|
||||
$incomingEvents = array();
|
||||
foreach ($this->request->data as $event) {
|
||||
$incomingIDs[] = $event['Event']['uuid'];
|
||||
$incomingEvents[$event['Event']['uuid']] = $event['Event']['timestamp'];
|
||||
}
|
||||
$events = $this->Event->find('all', array(
|
||||
'conditions' => array('Event.uuid' => $incomingIDs),
|
||||
'recursive' => -1,
|
||||
'fields' => array('Event.uuid', 'Event.timestamp', 'Event.locked'),
|
||||
));
|
||||
foreach ($events as $k => $v) {
|
||||
if (!$v['Event']['timestamp'] < $incomingEvents[$v['Event']['uuid']]) {
|
||||
unset($incomingEvents[$v['Event']['uuid']]);
|
||||
continue;
|
||||
}
|
||||
if ($v['Event']['locked'] == 0) {
|
||||
unset($incomingEvents[$v['Event']['uuid']]);
|
||||
}
|
||||
}
|
||||
$this->set('result', array_keys($incomingEvents));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -456,18 +456,21 @@ class Event extends AppModel {
|
|||
* @throws InternalErrorException
|
||||
*/
|
||||
public function cleanupEventArrayFromXML(&$data) {
|
||||
// Workaround for different structure in XML/array than what CakePHP expects
|
||||
if (isset($data['Event']['Attribute']) && is_array($data['Event']['Attribute']) && count($data['Event']['Attribute'])) {
|
||||
if (is_numeric(implode(array_keys($data['Event']['Attribute']), ''))) {
|
||||
// normal array of multiple Attributes
|
||||
$data['Attribute'] = $data['Event']['Attribute'];
|
||||
} else {
|
||||
// single attribute
|
||||
$data['Attribute'][0] = $data['Event']['Attribute'];
|
||||
$objects = array('Attribute', 'ShadowAttribute');
|
||||
|
||||
foreach ($objects as $object) {
|
||||
// Workaround for different structure in XML/array than what CakePHP expects
|
||||
if (isset($data['Event'][$object]) && is_array($data['Event'][$object]) && count($data['Event'][$object])) {
|
||||
if (is_numeric(implode(array_keys($data['Event'][$object]), ''))) {
|
||||
// normal array of multiple Attributes
|
||||
$data[$object] = $data['Event'][$object];
|
||||
} else {
|
||||
// single attribute
|
||||
$data[$object][0] = $data['Event'][$object];
|
||||
}
|
||||
}
|
||||
unset($data['Event'][$object]);
|
||||
}
|
||||
unset($data['Event']['Attribute']);
|
||||
|
||||
return $data;
|
||||
}
|
||||
|
||||
|
|
@ -587,49 +590,41 @@ class Event extends AppModel {
|
|||
// LATER try to do this using a separate EventsController and renderAs() function
|
||||
$xmlArray = array();
|
||||
// rearrange things to be compatible with the Xml::fromArray()
|
||||
$event['Event']['Attribute'] = $event['Attribute'];
|
||||
unset($event['Attribute']);
|
||||
$event['Event']['ShadowAttribute'] = $event['ShadowAttribute'];
|
||||
unset($event['ShadowAttribute']);
|
||||
if (isset($event['Attribute'])) {
|
||||
$event['Event']['Attribute'] = $event['Attribute'];
|
||||
unset($event['Attribute']);
|
||||
}
|
||||
|
||||
// cleanup the array from things we do not want to expose
|
||||
//unset($event['Event']['org']);
|
||||
// remove value1 and value2 from the output
|
||||
foreach ($event['Event']['Attribute'] as $key => &$attribute) {
|
||||
// do not keep attributes that are private, nor cluster
|
||||
if ($attribute['distribution'] < 2) {
|
||||
unset($event['Event']['Attribute'][$key]);
|
||||
continue; // stop processing this
|
||||
if (isset($event['Event']['Attribute'])) {
|
||||
foreach ($event['Event']['Attribute'] as $key => &$attribute) {
|
||||
// do not keep attributes that are private, nor cluster
|
||||
if ($attribute['distribution'] < 2) {
|
||||
unset($event['Event']['Attribute'][$key]);
|
||||
continue; // stop processing this
|
||||
}
|
||||
// Distribution, correct Connected Community to Community in Attribute
|
||||
if ($attribute['distribution'] == 2) {
|
||||
$attribute['distribution'] = 1;
|
||||
}
|
||||
// remove value1 and value2 from the output
|
||||
unset($attribute['value1']);
|
||||
unset($attribute['value2']);
|
||||
// also add the encoded attachment
|
||||
if ($this->Attribute->typeIsAttachment($attribute['type'])) {
|
||||
$encodedFile = $this->Attribute->base64EncodeAttachment($attribute);
|
||||
$attribute['data'] = $encodedFile;
|
||||
}
|
||||
// Passing the attribute ID together with the attribute could cause the deletion of attributes after a publish/push
|
||||
// Basically, if the attribute count differed between two instances, and the instance with the lower attribute
|
||||
// count pushed, the old attributes with the same ID got overwritten. Unsetting the ID before pushing it
|
||||
// solves the issue and a new attribute is always created.
|
||||
unset($attribute['id']);
|
||||
}
|
||||
// Distribution, correct Connected Community to Community in Attribute
|
||||
if ($attribute['distribution'] == 2) {
|
||||
$attribute['distribution'] = 1;
|
||||
}
|
||||
// remove value1 and value2 from the output
|
||||
unset($attribute['value1']);
|
||||
unset($attribute['value2']);
|
||||
// also add the encoded attachment
|
||||
if ($this->Attribute->typeIsAttachment($attribute['type'])) {
|
||||
$encodedFile = $this->Attribute->base64EncodeAttachment($attribute);
|
||||
$attribute['data'] = $encodedFile;
|
||||
}
|
||||
// Passing the attribute ID together with the attribute could cause the deletion of attributes after a publish/push
|
||||
// Basically, if the attribute count differed between two instances, and the instance with the lower attribute
|
||||
// count pushed, the old attributes with the same ID got overwritten. Unsetting the ID before pushing it
|
||||
// solves the issue and a new attribute is always created.
|
||||
unset($attribute['id']);
|
||||
}
|
||||
|
||||
foreach ($event['Event']['ShadowAttribute'] as $k => &$v) {
|
||||
if ($this->ShadowAttribute->typeIsAttachment($v['type'])) {
|
||||
$encodedFile = $this->ShadowAttribute->base64EncodeAttachment($v);
|
||||
$v['data'] = $encodedFile;
|
||||
}
|
||||
unset($v['id']);
|
||||
unset($v['value1']);
|
||||
unset($v['value2']);
|
||||
}
|
||||
|
||||
|
||||
// Distribution, correct All to Community in Event
|
||||
if ($event['Event']['distribution'] == 2) {
|
||||
$event['Event']['distribution'] = 1;
|
||||
|
|
@ -640,6 +635,7 @@ class Event extends AppModel {
|
|||
$eventsXml = $xmlObject->asXML();
|
||||
// do a REST POST request with the server
|
||||
$data = $eventsXml;
|
||||
|
||||
// LATER validate HTTPS SSL certificate
|
||||
$this->Dns = ClassRegistry::init('Dns');
|
||||
if ($this->Dns->testipaddress(parse_url($uri, PHP_URL_HOST))) {
|
||||
|
|
|
|||
|
|
@ -335,46 +335,51 @@ class Server extends AppModel {
|
|||
'Event.attribute_count >' => 0
|
||||
), //array of conditions
|
||||
'recursive' => -1, //int
|
||||
'fields' => array('Event.id'), //array of field names
|
||||
'fields' => array('Event.id', 'Event.timestamp', 'Event.uuid'), //array of field names
|
||||
);
|
||||
$eventIds = $eventModel->find('all', $findParams);
|
||||
}
|
||||
$eventCount = count($eventIds);
|
||||
//debug($eventIds);
|
||||
// now process the $eventIds to pull each of the events sequentially
|
||||
if (!empty($eventIds)) {
|
||||
$successes = array();
|
||||
$fails = array();
|
||||
$lowestfailedid = null;
|
||||
foreach ($eventIds as $k => $eventId) {
|
||||
$eventModel->recursive=1;
|
||||
$eventModel->contain(array('Attribute'));
|
||||
$event = $eventModel->findById($eventId['Event']['id']);
|
||||
$event['Event']['locked'] = true;
|
||||
$result = $eventModel->uploadEventToServer(
|
||||
$event,
|
||||
$this->data,
|
||||
$HttpSocket);
|
||||
if ('Success' === $result) {
|
||||
$successes[] = $event['Event']['id'];
|
||||
|
||||
$eventUUIDsFiltered = $this->filterEventIdsForPush($id, $HttpSocket, $eventIds);
|
||||
if (!empty($eventUUIDsFiltered)) {
|
||||
|
||||
$eventCount = count($eventUUIDsFiltered);
|
||||
//debug($eventIds);
|
||||
// now process the $eventIds to pull each of the events sequentially
|
||||
if (!empty($eventIds)) {
|
||||
$successes = array();
|
||||
$fails = array();
|
||||
$lowestfailedid = null;
|
||||
foreach ($eventUUIDsFiltered as $k => $eventUuid) {
|
||||
$eventModel->recursive=1;
|
||||
$eventModel->contain(array('Attribute'));
|
||||
$event = $eventModel->findByUuid($eventUuid);
|
||||
$event['Event']['locked'] = true;
|
||||
$result = $eventModel->uploadEventToServer(
|
||||
$event,
|
||||
$this->data,
|
||||
$HttpSocket);
|
||||
if ('Success' === $result) {
|
||||
$successes[] = $event['Event']['id'];
|
||||
} else {
|
||||
$fails[$event['Event']['id']] = $result;
|
||||
}
|
||||
if ($jobId && $k%10 == 0) {
|
||||
$job->saveField('progress', 100 * $k / $eventCount);
|
||||
}
|
||||
}
|
||||
if (count($fails) > 0) {
|
||||
// there are fails, take the lowest fail
|
||||
$lastpushedid = min(array_keys($fails));
|
||||
} else {
|
||||
$fails[$event['Event']['id']] = $result;
|
||||
}
|
||||
if ($jobId && $k%10 == 0) {
|
||||
$job->saveField('progress', 100 * $k / $eventCount);
|
||||
// no fails, take the highest success
|
||||
$lastpushedid = max($successes);
|
||||
}
|
||||
// increment lastid based on the highest ID seen
|
||||
// Save the entire Server data instead of just a single field, so that the logger can be fed with the extra fields.
|
||||
$this->data['Server']['lastpushedid'] = $lastpushedid;
|
||||
$this->save($this->data);
|
||||
}
|
||||
if (count($fails) > 0) {
|
||||
// there are fails, take the lowest fail
|
||||
$lastpushedid = min(array_keys($fails));
|
||||
} else {
|
||||
// no fails, take the highest success
|
||||
$lastpushedid = max($successes);
|
||||
}
|
||||
// increment lastid based on the highest ID seen
|
||||
// Save the entire Server data instead of just a single field, so that the logger can be fed with the extra fields.
|
||||
$this->data['Server']['lastpushedid'] = $lastpushedid;
|
||||
$this->save($this->data);
|
||||
}
|
||||
if (!isset($successes)) $successes = null;
|
||||
if (!isset($fails)) $fails = null;
|
||||
|
|
@ -397,4 +402,32 @@ class Server extends AppModel {
|
|||
return array($successes, $fails);
|
||||
}
|
||||
}
|
||||
|
||||
public function filterEventIdsForPush($id, $HttpSocket, $eventIds) {
|
||||
foreach ($eventIds as $k => $event) {
|
||||
unset($eventIds[$k]['Event']['id']);
|
||||
}
|
||||
$server = $this->read(null, $id);
|
||||
if (null == $HttpSocket) {
|
||||
App::uses('SyncTool', 'Tools');
|
||||
$syncTool = new SyncTool();
|
||||
$HttpSocket = $syncTool->setupHttpSocket($server);
|
||||
}
|
||||
$data = json_encode($eventIds);
|
||||
$request = array(
|
||||
'header' => array(
|
||||
'Authorization' => $server['Server']['authkey'],
|
||||
'Accept' => 'application/json',
|
||||
'Content-Type' => 'application/json',
|
||||
)
|
||||
);
|
||||
$uri = $server['Server']['url'] . '/events/filterEventIdsForPush';
|
||||
$response = $HttpSocket->post($uri, $data, $request);
|
||||
if ($response->code == '200') {
|
||||
$uuidList = json_decode($response->body());
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
return $uuidList;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
<?php
|
||||
echo json_encode($result);
|
||||
Loading…
Reference in New Issue