MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Sync sightings on push, pull and push on add

pull/4917/head
Richard van den Berg 2019-11-22 21:53:51 +01:00
parent 815af0d2a5
commit dd963c2e21
18 changed files with 356 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
INSTALL/MYSQL.sql
View File

@ -199,6 +199,7 @@ CREATE TABLE IF NOT EXISTS `events` (
`locked` tinyint(1) NOT NULL DEFAULT 0,
`threat_level_id` int(11) NOT NULL,
`publish_timestamp` int(11) NOT NULL DEFAULT 0,
`sighting_timestamp` int(11) NOT NULL DEFAULT 0,
`disable_correlation` tinyint(1) NOT NULL DEFAULT 0,
`extends_uuid` varchar(40) COLLATE utf8_bin DEFAULT '',
PRIMARY KEY (`id`),
@ -821,6 +822,7 @@ CREATE TABLE IF NOT EXISTS `servers` (
`org_id` int(11) NOT NULL,
`push` tinyint(1) NOT NULL,
`pull` tinyint(1) NOT NULL,
`push_sightings` tinyint(1) NOT NULL DEFAULT 0,
`lastpulledid` int(11) DEFAULT NULL,
`lastpushedid` int(11) DEFAULT NULL,
`organization` varchar(10) COLLATE utf8_bin DEFAULT NULL,

4
INSTALL/POSTGRESQL-data-initial.sql
View File

@ -100,7 +100,7 @@ COPY public.event_tags (id, event_id, tag_id) FROM stdin;
-- Data for Name: events; Type: TABLE DATA; Schema: public; Owner: -
--
COPY public.events (id, org_id, date, info, user_id, uuid, published, analysis, attribute_count, orgc_id, "timestamp", distribution, sharing_group_id, proposal_email_lock, locked, threat_level_id, publish_timestamp, disable_correlation, extends_uuid) FROM stdin;
COPY public.events (id, org_id, date, info, user_id, uuid, published, analysis, attribute_count, orgc_id, "timestamp", distribution, sharing_group_id, proposal_email_lock, locked, threat_level_id, publish_timestamp, sighting_timestamp, disable_correlation, extends_uuid) FROM stdin;
\.
@ -323,7 +323,7 @@ COPY public.roles (id, name, created, modified, perm_add, perm_modify, perm_modi
-- Data for Name: servers; Type: TABLE DATA; Schema: public; Owner: -
--
COPY public.servers (id, name, url, authkey, org_id, push, pull, lastpulledid, lastpushedid, organization, remote_org_id, publish_without_email, unpublish_event, self_signed, pull_rules, push_rules, cert_file, client_cert_file, internal) FROM stdin;
COPY public.servers (id, name, url, authkey, org_id, push, pull, push_sightings, lastpulledid, lastpushedid, organization, remote_org_id, publish_without_email, unpublish_event, self_signed, pull_rules, push_rules, cert_file, client_cert_file, internal) FROM stdin;
\.

2
INSTALL/POSTGRESQL-structure.sql
View File

@ -335,6 +335,7 @@ CREATE TABLE public.events (
locked boolean DEFAULT false NOT NULL,
threat_level_id bigint NOT NULL,
publish_timestamp bigint DEFAULT '0'::bigint NOT NULL,
sighting_timestamp bigint DEFAULT '0'::bigint NOT NULL,
disable_correlation boolean DEFAULT false NOT NULL,
extends_uuid character varying(40) DEFAULT ''::character varying
);
@ -1171,6 +1172,7 @@ CREATE TABLE public.servers (
org_id bigint NOT NULL,
push boolean NOT NULL,
pull boolean NOT NULL,
push_sightings boolean DEFAULT false NOT NULL,
lastpulledid bigint,
lastpushedid bigint,
organization character varying(10),

22
app/Console/Command/EventShell.php
View File

@ -508,6 +508,28 @@ class EventShell extends AppShell
$log->createLogEntry($user, 'publish', 'Event', $id, 'Event (' . $id . '): published.', 'published () => (1)');
}
public function publish_sightings() {
$id = $this->args[0];
$passAlong = $this->args[1];
$jobId = $this->args[2];
$userId = $this->args[3];
$user = $this->User->getAuthUser($userId);
$job = $this->Job->read(null, $jobId);
$this->Event->Behaviors->unload('SysLogLogable.SysLogLogable');
$result = $this->Event->publish_sightings($id, $passAlong);
$job['Job']['progress'] = 100;
$job['Job']['date_modified'] = date("Y-m-d H:i:s");
if ($result) {
$job['Job']['message'] = 'Sightings published.';
} else {
$job['Job']['message'] = 'Sightings published, but the upload to other instances may have failed.';
}
$this->Job->save($job);
$log = ClassRegistry::init('Log');
$log->create();
$log->createLogEntry($user, 'publish_sightings', 'Event', $id, 'Sightings for event (' . $id . '): published.', 'publish_sightings updated');
}
public function enrichment() {
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
die('Usage: ' . $this->Server->command_line_functions['enrichment'] . PHP_EOL);

2
app/Console/Command/ServerShell.php
View File

@ -72,7 +72,7 @@ class ServerShell extends AppShell
'status' => 4
));
if (is_array($result)) {
$message = sprintf(__('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled.', count($result[0]), count($result[1]), $result[2]));
$message = sprintf(__('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled.', count($result[0]), count($result[1]), $result[2], $result[3]));
} else {
$message = sprintf(__('ERROR: %s'), $result);
}

1
app/Controller/Component/ACLComponent.php
View File

@ -444,6 +444,7 @@ class ACLComponent extends Component
'listSightings' => array('*'),
'quickDelete' => array('perm_sighting'),
'viewSightings' => array('*'),
'bulkSaveSightings' => array('OR' => array('perm_sync', 'perm_sighting')),
'quickAdd' => array('perm_sighting')
),
'sightingdb' => array(

10
app/Controller/Component/RestResponseComponent.php
View File

@ -171,11 +171,11 @@ class RestResponseComponent extends Component
'add' => array(
'description' => "POST an Server object in JSON format to this API to add a server.",
'mandatory' => array('url', 'name', 'remote_org_id', 'authkey'),
'optional' => array('push', 'pull', 'push_rules', 'pull_rules', 'submitted_cert', 'submitted_client_cert', 'json')
'optional' => array('push', 'pull', 'push_sightings', 'push_rules', 'pull_rules', 'submitted_cert', 'submitted_client_cert', 'json')
),
'edit' => array(
'description' => "POST an Server object in JSON format to this API to edit a server.",
'optional' => array('url', 'name', 'authkey', 'json', 'push', 'pull', 'push_rules', 'pull_rules', 'submitted_cert', 'submitted_client_cert', 'remote_org_id')
'optional' => array('url', 'name', 'authkey', 'json', 'push', 'pull', 'push_sightings', 'push_rules', 'pull_rules', 'submitted_cert', 'submitted_client_cert', 'remote_org_id')
),
'serverSettings' => array(
'description' => "Send a GET request to this endpoint to get a full diagnostic along with all currently set settings of the current instance. This will also include the worker status"
@ -1291,6 +1291,12 @@ class RestResponseComponent extends Component
'values' => array(1 => 'True', 0 => 'False' ),
'help' => __('Allow the upload of events and their attribute to the server')
),
'push_sightings' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' ),
'help' => __('Allow the upload of sightings to the server')
),
'releasability' => array(
'input' => 'text',
'type' => 'string',

55
app/Controller/EventsController.php
View File

@ -733,7 +733,7 @@ class EventsController extends AppController
if (!empty($passedArgs['searchminimal'])) {
unset($rules['contain']);
$rules['recursive'] = -1;
$rules['fields'] = array('id', 'timestamp', 'published', 'uuid');
$rules['fields'] = array('id', 'timestamp', 'sighting_timestamp', 'published', 'uuid');
$rules['contain'] = array('Orgc.uuid');
}
$paginationRules = array('page', 'limit', 'sort', 'direction', 'order');
@ -2572,6 +2572,59 @@ class EventsController extends AppController
}
}
public function publishSightings($id = null)
{
$id = $this->Toolbox->findIdByUuid($this->Event, $id);
$event = fetchEvent(
$this->Auth->user(),
array(
'eventid' => $id,
'metadata' => 1
)
);
if (empty($event)) {
throw new NotFoundException(__('Invalid event'));
}
if ($this->request->is('post') || $this->request->is('put')) {
$result = $this->Event->publishRouter($id, null, $this->Auth->user(), 'sightings');
if (!Configure::read('MISP.background_jobs')) {
if (!is_array($result)) {
// redirect to the view event page
$message = 'Sightings published';
} else {
$lastResult = array_pop($result);
$resultString = (count($result) > 0) ? implode(', ', $result) . ' and ' . $lastResult : $lastResult;
$errors['failed_servers'] = $result;
$message = sprintf('Sightings published but not pushed to %s, re-try later. If the issue persists, make sure that the correct sync user credentials are used for the server link and that the sync user on the remote server has authentication privileges.', $resultString);
}
} else {
// update the DB to set the published flag
// for background jobs, this should be done already
$fieldList = array('id', 'info', 'sighting_timestamp');
$event['Event']['sighting_timestamp'] = time();
$this->Event->save($event, array('fieldList' => $fieldList));
$message = 'Job queued';
}
if ($this->_isRest()) {
$this->set('name', 'Publish Sightings');
$this->set('message', $message);
if (!empty($errors)) {
$this->set('errors', $errors);
}
$this->set('url', '/events/publishSightings/' . $id);
$this->set('id', $id);
$this->set('_serialize', array('name', 'message', 'url', 'id', 'errors'));
} else {
$this->Flash->success($message);
$this->redirect(array('action' => 'view', $id));
}
} else {
$this->set('id', $id);
$this->set('type', 'publish_sightings');
$this->render('ajax/eventPublishConfirmationForm');
}
}
// Publishes the event without sending an alert email
public function publish($id = null)
{

6
app/Controller/ServersController.php
View File

@ -250,6 +250,7 @@ class ServersController extends AppController
$defaults = array(
'push' => 0,
'pull' => 0,
'push_sightings' => 0,
'caching_enabled' => 0,
'json' => '[]',
'push_rules' => '[]',
@ -444,7 +445,7 @@ class ServersController extends AppController
}
if (!$fail) {
// say what fields are to be updated
$fieldList = array('id', 'url', 'push', 'pull', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
$fieldList = array('id', 'url', 'push', 'pull', 'push_sightings', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
$this->request->data['Server']['id'] = $id;
if (isset($this->request->data['Server']['authkey']) && "" != $this->request->data['Server']['authkey']) {
$fieldList[] = 'authkey';
@ -663,13 +664,14 @@ class ServersController extends AppController
if (!Configure::read('MISP.background_jobs')) {
$result = $this->Server->pull($this->Auth->user(), $id, $technique, $s);
if (is_array($result)) {
$success = sprintf(__('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled.', count($result[0]), count($result[1]), $result[2]));
$success = sprintf(__('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled.', count($result[0]), count($result[1]), $result[2], $result[3]));
} else {
$error = $result;
}
$this->set('successes', $result[0]);
$this->set('fails', $result[1]);
$this->set('pulledProposals', $result[2]);
$this->set('pulledSightings', $result[3]);
} else {
$this->loadModel('Job');
$this->Job->create();

26
app/Controller/SightingsController.php
View File

@ -66,7 +66,7 @@ class SightingsController extends AppController
$source = isset($this->request->data['source']) ? trim($this->request->data['source']) : '';
}
if (!$error) {
$result = $this->Sighting->saveSightings($id, $values, $timestamp, $this->Auth->user(), $type, $source);
$result = $this->Sighting->saveSightings($id, $values, $timestamp, $this->Auth->user(), $type, $source, false, true);
}
if (!is_numeric($result)) {
$error = $result;
@ -422,4 +422,28 @@ class SightingsController extends AppController
$this->layout = 'ajax';
$this->render('ajax/view_sightings');
}
// Save sightings synced over, restricted to sync users
public function bulkSaveSightings($eventId = false)
{
if ($this->request->is('post')) {
if (empty($this->request->data['Sighting'])) {
$sightings = $this->request->data;
} else {
$sightings = $this->request->data['Sighting'];
}
$saved = $this->Sighting->bulkSaveSightings($eventId, $sightings, $this->Auth->user());
if (is_numeric($saved)) {
if ($saved > 0) {
return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => $saved . ' sightings added.')), 'status' => 200, 'type' => 'json'));
} else {
return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'success' => 'No sightings added.')), 'status' => 200, 'type' => 'json'));
}
} else {
throw new MethodNotAllowedException($saved);
}
} else {
throw new MethodNotAllowedException('This method is only accessible via POST requests.');
}
}
}

7
app/Model/AppModel.php
View File

@ -76,7 +76,8 @@ class AppModel extends Model
21 => false, 22 => false, 23 => false, 24 => false, 25 => false, 26 => false,
27 => false, 28 => false, 29 => false, 30 => false, 31 => false, 32 => false,
33 => false, 34 => false, 35 => false, 36 => false, 37 => false, 38 => false,
39 => false, 40 => false, 41 => false, 42 => false, 43 => false
39 => false, 40 => false, 41 => false, 42 => false, 43 => false, 44 => false,
45 => false
);
public $advanced_updates_description = array(
@ -1301,6 +1302,10 @@ class AppModel extends Model
case 44:
$sqlArray[] = "ALTER TABLE object_template_elements CHANGE `disable_correlation` `disable_correlation` tinyint(1);";
break;
case 45:
$sqlArray[] = "ALTER TABLE `events` ADD `sighting_timestamp` int(11) NOT NULL DEFAULT 0 AFTER `publish_timestamp`;";
$sqlArray[] = "ALTER TABLE `servers` ADD `push_sightings` tinyint(1) NOT NULL DEFAULT 0 AFTER `pull`;";
break;
case 'fixNonEmptySharingGroupID':
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';

104
app/Model/Event.php
View File

@ -1063,9 +1063,9 @@ class Event extends AppModel
return $error;
}
private function __executeRestfulEventToServer($event, $server, $resourceId, &$newLocation, &$newTextBody, $HttpSocket)
private function __executeRestfulEventToServer($event, $server, $resourceId, &$newLocation, &$newTextBody, $HttpSocket, $scope)
{
$result = $this->restfulEventToServer($event, $server, $resourceId, $newLocation, $newTextBody, $HttpSocket);
$result = $this->restfulEventToServer($event, $server, $resourceId, $newLocation, $newTextBody, $HttpSocket, $scope);
if (is_numeric($result)) {
$error = $this->__resolveErrorCode($result, $event, $server);
if ($error) {
@ -1075,24 +1075,26 @@ class Event extends AppModel
return true;
}
public function uploadEventToServer($event, $server, $HttpSocket = null)
public function uploadEventToServer($event, $server, $HttpSocket = null, $scope = 'events')
{
$this->Server = ClassRegistry::init('Server');
$push = $this->Server->checkVersionCompatibility($server['Server']['id'], false, $HttpSocket);
if (empty($push['canPush'])) {
if ($scope === 'events' && empty($push['canPush'])) {
return 'The remote user is not a sync user - the upload of the event has been blocked.';
} elseif ($scope === 'sightings' && empty($push['canPush']) && empty($push['canSight'])) {
return 'The remote user is not a sightings user - the upload of the sightings has been blocked.';
}
if (!empty($server['Server']['unpublish_event'])) {
$event['Event']['published'] = 0;
}
$updated = null;
$newLocation = $newTextBody = '';
$result = $this->__executeRestfulEventToServer($event, $server, null, $newLocation, $newTextBody, $HttpSocket);
$result = $this->__executeRestfulEventToServer($event, $server, null, $newLocation, $newTextBody, $HttpSocket, $scope);
if ($result !== true) {
return $result;
}
if (strlen($newLocation)) { // HTTP/1.1 302 Found and Location: http://<newLocation>
$result = $this->__executeRestfulEventToServer($event, $server, $newLocation, $newLocation, $newTextBody, $HttpSocket);
$result = $this->__executeRestfulEventToServer($event, $server, $newLocation, $newLocation, $newTextBody, $HttpSocket, $scope);
if ($result !== true) {
return $result;
}
@ -1181,7 +1183,7 @@ class Event extends AppModel
}
// Uploads the event and the associated Attributes to another Server
public function restfulEventToServer($event, $server, $urlPath, &$newLocation, &$newTextBody, $HttpSocket = null)
public function restfulEventToServer($event, $server, $urlPath, &$newLocation, &$newTextBody, $HttpSocket = null, $scope)
{
$event = $this->__prepareForPushToServer($event, $server);
if (is_numeric($event)) {
@ -1190,7 +1192,11 @@ class Event extends AppModel
$url = $server['Server']['url'];
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$request = $this->setupSyncRequest($server);
$uri = $url . '/events' . $this->__getLastUrlPathComponent($urlPath);
if ($scope === 'sightings') {
$scope .= '/bulkSaveSightings';
$urlPath = $event['Event']['uuid'];
}
$uri = $url . '/' . $scope . $this->__getLastUrlPathComponent($urlPath);
$data = json_encode($event);
if (!empty(Configure::read('Security.sync_audit'))) {
$pushLogEntry = sprintf(
@ -2095,6 +2101,7 @@ class Event extends AppModel
if ($options['metadata']) {
unset($params['contain']['Attribute']);
unset($params['contain']['ShadowAttribute']);
unset($params['contain']['Object']);
}
if ($user['Role']['perm_site_admin']) {
$params['contain']['User'] = array('fields' => 'email');
@ -2104,7 +2111,6 @@ class Event extends AppModel
return array();
}
// Do some refactoring with the event
$this->Sighting = ClassRegistry::init('Sighting');
$userEmails = array();
$fields = array(
'common' => array('distribution', 'sharing_group_id', 'uuid'),
@ -2269,7 +2275,10 @@ class Event extends AppModel
}
$event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user, $event['Event'], $overrideLimit, 'Server');
}
$event['Sighting'] = $this->Sighting->attachToEvent($event, $user);
if (empty($options['metadata'])) {
$this->Sighting = ClassRegistry::init('Sighting');
$event['Sighting'] = $this->Sighting->attachToEvent($event, $user);
}
if ($options['includeSightingdb']) {
$this->Sightingdb = ClassRegistry::init('Sightingdb');
$event = $this->Sightingdb->attachToEvent($event, $user);
@ -4047,8 +4056,8 @@ class Event extends AppModel
return true;
}
// Uploads this specific event to all remote servers
public function uploadEventToServersRouter($id, $passAlong = null)
// Uploads this specific event or sightings to all remote servers
public function uploadEventToServersRouter($id, $passAlong = null, $scope = 'events')
{
$eventOrgcId = $this->find('first', array(
'conditions' => array('Event.id' => $id),
@ -4074,6 +4083,11 @@ class Event extends AppModel
}
$event = $event[0];
$event['Event']['locked'] = 1;
// attach sightings if needed
if ($scope === 'sightings') {
$this->Sighting = ClassRegistry::init('Sighting');
$event['Sighting'] = $this->Sighting->attachToEvent($event, $elevatedUser);
}
// get a list of the servers
$this->Server = ClassRegistry::init('Server');
$conditions = array('push' => 1);
@ -4092,7 +4106,8 @@ class Event extends AppModel
$failedServers = array();
App::uses('SyncTool', 'Tools');
foreach ($servers as &$server) {
if ((!isset($server['Server']['internal']) || !$server['Server']['internal']) && $event['Event']['distribution'] < 2) {
if (((!isset($server['Server']['internal']) || !$server['Server']['internal']) && $event['Event']['distribution'] < 2) ||
((!isset($server['Server']['push_sightings']) || !$server['Server']['push_sightings'])) && $scope === 'sightings') {
continue;
}
$syncTool = new SyncTool();
@ -4116,7 +4131,7 @@ class Event extends AppModel
$event = $this->fetchEvent($elevatedUser, $params);
$event = $event[0];
$event['Event']['locked'] = 1;
$thisUploaded = $this->uploadEventToServer($event, $server, $HttpSocket);
$thisUploaded = $this->uploadEventToServer($event, $server, $HttpSocket, $scope);
if (!$thisUploaded) {
$uploaded = !$uploaded ? $uploaded : $thisUploaded;
$failedServers[] = $server['Server']['url'];
@ -4149,9 +4164,16 @@ class Event extends AppModel
return $workerType;
}
public function publishRouter($id, $passAlong = null, $user)
public function publishRouter($id, $passAlong = null, $user, $scope = 'events')
{
if (Configure::read('MISP.background_jobs')) {
$job_type = 'publish_' . $scope;
$function = 'publish';
$message = 'Publishing.';
if ($scope === 'sightings') {
$message = 'Publishing sightings.';
$function = 'publish_sightings';
}
$job = ClassRegistry::init('Job');
$job->create();
$data = array(
@ -4162,24 +4184,56 @@ class Event extends AppModel
'retries' => 0,
'org_id' => $user['org_id'],
'org' => $user['Organisation']['name'],
'message' => 'Publishing.',
'message' => $message
);
$job->save($data);
$jobId = $job->id;
$process_id = CakeResque::enqueue(
'prio',
'EventShell',
array('publish', $id, $passAlong, $jobId, $user['id']),
array($function, $id, $passAlong, $jobId, $user['id']),
true
);
$job->saveField('process_id', $process_id);
return $process_id;
} elseif ($scope === 'sightings') {
$result = $this->publish_sightings($id, $passAlong);
return $result;
} else {
$result = $this->publish($id, $passAlong);
return $result;
}
}
public function publish_sightings($id, $passAlong = null, $jobId = null)
{
if (is_numeric($id)) {
$condition = array('Event.id' => $id);
} else {
$condition = array('Event.uuid' => $id);
}
$event = $this->find('first', array(
'recursive' => -1,
'conditions' => $condition
));
if (empty($event)) {
return false;
}
if ($jobId) {
$this->Behaviors->unload('SysLogLogable.SysLogLogable');
} else {
// update the DB to set the sightings timestamp
// for background jobs, this should be done already
$fieldList = array('id', 'info', 'sighting_timestamp');
$event['Event']['sighting_timestamp'] = time();
$event['Event']['skip_zmq'] = 1;
$event['Event']['skip_kafka'] = 1;
$this->save($event, array('fieldList' => $fieldList));
}
$uploaded = $this->uploadEventToServersRouter($id, $passAlong, 'sightings');
return $uploaded;
}
// Performs all the actions required to publish an event
public function publish($id, $passAlong = null, $jobId = null)
{
@ -4450,19 +4504,27 @@ class Event extends AppModel
return false;
}
public function removeOlder(&$eventArray)
public function removeOlder(&$eventArray, $scope = 'events')
{
if ($scope === 'sightings' ) {
$field = 'sighting_timestamp';
} else {
$field = 'timestamp';
}
$uuidsToCheck = array();
foreach ($eventArray as $k => &$event) {
$uuidsToCheck[$event['uuid']] = $k;
}
$localEvents = array();
$temp = $this->find('all', array('recursive' => -1, 'fields' => array('Event.uuid', 'Event.timestamp', 'Event.locked')));
$temp = $this->find('all', array('recursive' => -1, 'fields' => array('Event.uuid', 'Event.' . $field, 'Event.locked')));
foreach ($temp as $e) {
$localEvents[$e['Event']['uuid']] = array('timestamp' => $e['Event']['timestamp'], 'locked' => $e['Event']['locked']);
$localEvents[$e['Event']['uuid']] = array($field => $e['Event'][$field], 'locked' => $e['Event']['locked']);
}
foreach ($uuidsToCheck as $uuid => $eventArrayId) {
if (isset($localEvents[$uuid]) && ($localEvents[$uuid]['timestamp'] >= $eventArray[$eventArrayId]['timestamp'] || !$localEvents[$uuid]['locked'])) {
if (isset($localEvents[$uuid])
&& ($localEvents[$uuid][$field] >= $eventArray[$eventArrayId][$field]
|| ($scope === 'events' && !$localEvents[$uuid]['locked'])))
{
unset($eventArray[$eventArrayId]);
}
}

80
app/Model/Server.php
View File

@ -81,6 +81,16 @@ class Server extends AppModel
//'on' => 'create', // Limit validation to 'create' or 'update' operations
),
),
'push_sightings' => array(
'boolean' => array(
'rule' => array('boolean'),
//'message' => 'Your custom message here',
'allowEmpty' => true,
'required' => false,
//'last' => false, // Stop validation after this rule
//'on' => 'create', // Limit validation to 'create' or 'update' operations
),
),
'lastpushedid' => array(
'numeric' => array(
'rule' => array('numeric'),
@ -2495,6 +2505,11 @@ class Server extends AppModel
$job->saveField('message', 'Pulling proposals.');
}
$pulledProposals = $eventModel->ShadowAttribute->pullProposals($user, $server);
if ($jobId) {
$job->saveField('progress', 75);
$job->saveField('message', 'Pulling sightings.');
}
$pulledSightings = $eventModel->Sighting->pullSightings($user, $server);
if ($jobId) {
$job->saveField('progress', 100);
$job->saveField('message', 'Pull completed.');
@ -2511,13 +2526,14 @@ class Server extends AppModel
'user_id' => $user['id'],
'title' => 'Pull from ' . $server['Server']['url'] . ' initiated by ' . $email,
'change' => sprintf(
'%s events and %s proposals pulled or updated. %s events failed or didn\'t need an update.',
'%s events, %s proposals and %s sightings pulled or updated. %s events failed or didn\'t need an update.',
count($successes),
$pulledProposals,
$pulledSightings,
count($fails)
)
));
return array($successes, $fails, $pulledProposals);
return array($successes, $fails, $pulledProposals, $pulledSightings);
}
public function filterRuleToParameter($filter_rules)
@ -2549,7 +2565,7 @@ class Server extends AppModel
// Get an array of event_ids that are present on the remote server
public function getEventIdsFromServer($server, $all = false, $HttpSocket=null, $force_uuid=false, $ignoreFilterRules = false)
public function getEventIdsFromServer($server, $all = false, $HttpSocket=null, $force_uuid=false, $ignoreFilterRules = false, $scope = 'events')
{
$url = $server['Server']['url'];
if ($ignoreFilterRules) {
@ -2576,8 +2592,21 @@ class Server extends AppModel
$eventIds = array();
if ($all) {
if (!empty($eventArray)) {
foreach ($eventArray as $event) {
$eventIds[] = $event['uuid'];
if ($scope === 'sightings') {
foreach ($eventArray as $event) {
$localEvent = $this->Event->find('first', array(
'recursive' => -1,
'fields' => array('Event.uuid', 'Event.sighting_timestamp'),
'conditions' => array('Event.uuid' => $event['uuid'])
));
if (!empty($localEvent) && $localEvent['Event']['sighting_timestamp'] > $event['sighting_timestamp']) {
$eventIds[] = $event['uuid'];
}
}
} else {
foreach ($eventArray as $event) {
$eventIds[] = $event['uuid'];
}
}
}
} else {
@ -2617,7 +2646,7 @@ class Server extends AppModel
}
}
}
$this->Event->removeOlder($eventArray);
$this->Event->removeOlder($eventArray, $scope);
if (!empty($eventArray)) {
foreach ($eventArray as $event) {
if ($force_uuid) {
@ -2722,7 +2751,7 @@ class Server extends AppModel
), // array of conditions
'recursive' => -1, //int
'contain' => array('EventTag' => array('fields' => array('EventTag.tag_id'))),
'fields' => array('Event.id', 'Event.timestamp', 'Event.uuid', 'Event.orgc_id'), // array of field names
'fields' => array('Event.id', 'Event.timestamp', 'Event.sighting_timestamp', 'Event.uuid', 'Event.orgc_id'), // array of field names
);
$eventIds = $this->Event->find('all', $findParams);
$eventUUIDsFiltered = $this->getEventIdsForPush($id, $HttpSocket, $eventIds, $user);
@ -2731,7 +2760,7 @@ class Server extends AppModel
}
if (!empty($eventUUIDsFiltered)) {
$eventCount = count($eventUUIDsFiltered);
// now process the $eventIds to pull each of the events sequentially
// now process the $eventIds to push each of the events sequentially
if (!empty($eventUUIDsFiltered)) {
$successes = array();
$fails = array();
@ -2779,9 +2808,12 @@ class Server extends AppModel
}
$this->syncProposals($HttpSocket, $this->data, null, null, $this->Event);
$sightingSuccesses = $this->syncSightings($HttpSocket, $this->data, $user, $this->Event);
if (!isset($successes)) {
$successes = array();
$successes = $sightingSuccesses;
} else {
$successes = array_merge($successes, $sightingSuccesses);
}
if (!isset($fails)) {
$fails = array();
@ -2834,6 +2866,33 @@ class Server extends AppModel
return $uuidList;
}
public function syncSightings($HttpSocket, $server, $user, $eventModel)
{
$successes = array();
if (!$server['Server']['push_sightings']) {
return $successes;
}
$this->Sighting = ClassRegistry::init('Sighting');
$HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
$eventIds = $this->getEventIdsFromServer($server, true, $HttpSocket, false, true, 'sightings');
// now process the $eventIds to push each of the events sequentially
if (!empty($eventIds)) {
// check each event and push sightings when needed
foreach ($eventIds as $k => $eventId) {
$event = $eventModel->fetchEvent($user, $options = array('event_uuid' => $eventId, 'metadata' => true));
if (!empty($event)) {
$event = $event[0];
$event['Sighting'] = $this->Sighting->attachToEvent($event, $user);
$result = $eventModel->uploadEventToServer($event, $server, $HttpSocket, 'sightings');
if ($result === 'Success') {
$successes[] = 'Sightings for event ' . $event['Event']['id'];
}
}
}
}
return $successes;
}
public function syncProposals($HttpSocket, $server, $sa_id = null, $event_id = null, $eventModel)
{
$saModel = ClassRegistry::init('ShadowAttribute');
@ -4114,6 +4173,7 @@ class Server extends AppModel
}
$remoteVersion = json_decode($response->body, true);
$canPush = isset($remoteVersion['perm_sync']) ? $remoteVersion['perm_sync'] : false;
$canSight = isset($remoteVersion['perm_sighting']) ? $remoteVersion['perm_sighting'] : false;
$remoteVersion = explode('.', $remoteVersion['version']);
if (!isset($remoteVersion[0])) {
$this->Log = ClassRegistry::init('Log');
@ -4175,7 +4235,7 @@ class Server extends AppModel
'title' => ucfirst($issueLevel) . ': ' . $response,
));
}
return array('success' => $success, 'response' => $response, 'canPush' => $canPush, 'version' => $remoteVersion);
return array('success' => $success, 'response' => $response, 'canPush' => $canPush, 'canSight' => $canSight, 'version' => $remoteVersion);
}
public function isJson($string)

62
app/Model/Sighting.php
View File

@ -337,7 +337,7 @@ class Sighting extends AppModel
return $attributes;
}
public function saveSightings($id, $values, $timestamp, $user, $type = false, $source = false, $sighting_uuid = false)
public function saveSightings($id, $values, $timestamp, $user, $type = false, $source = false, $sighting_uuid = false, $publish = false)
{
$conditions = array();
if ($id && $id !== 'stix') {
@ -402,6 +402,9 @@ class Sighting extends AppModel
return json_encode($this->validationErrors);
}
$sightingsAdded += $result ? 1 : 0;
if ($publish) {
$this->Event->publishRouter($sighting['event_id'], null, $user, 'sightings');
}
}
if ($sightingsAdded == 0) {
return 'There was nothing to add.';
@ -756,4 +759,61 @@ class Sighting extends AppModel
fclose($tmpfile);
return $final;
}
// Bulk save sightings
public function bulkSaveSightings($eventId, $sightings, $user, $passAlong = null)
{
if (!is_numeric($eventId)) {
$eventId = $this->Event->field('id', array('uuid' => $eventId));
}
$event = $this->Event->fetchEvent($user, array(
'eventid' => $eventId,
'metadata' => 1,
'flatten' => true
));
if (empty($event)) {
return 'Event not found or not accesible by this user.';
}
$saved = 0;
foreach ($sightings as $s) {
$result = $this->saveSightings($s['attribute_uuid'], false, $s['date_sighting'], $user, $s['type'], $s['source'], $s['uuid']);
if (is_numeric($result)) {
$saved += $result;
}
}
if ($saved > 0) {
$this->Event->publishRouter($eventId, $passAlong, $user, 'sightings');
}
return $saved;
}
public function pullSightings($user, $server)
{
$HttpSocket = $this->setupHttpSocket($server);
$this->Server = ClassRegistry::init('Server');
$eventIds = $this->Server->getEventIdsFromServer($server, false, $HttpSocket, false, false, 'sightings');
$saved = 0;
// now process the $eventIds to pull each of the events sequentially
if (!empty($eventIds)) {
// download each event and save sightings
foreach ($eventIds as $k => $eventId) {
$event = $this->Event->downloadEventFromServer($eventId, $server);
$sightings = array();
if(!empty($event) && !empty($event['Event']['Attribute'])) {
foreach($event['Event']['Attribute'] as $attribute) {
if(!empty($attribute['Sighting'])) {
$sightings = array_merge($sightings, $attribute['Sighting']);
}
}
}
if(!empty($event) && !empty($sightings)) {
$result = $this->bulkSaveSightings($event['Event']['uuid'], $sightings, $user, $server['Server']['id']);
if (is_numeric($result)) {
$saved += $result;
}
}
}
}
return $saved;
}
}

1
app/View/Servers/add.ctp
View File

@ -80,6 +80,7 @@
echo '<h4 class="input clear">' . __('Enabled synchronisation methods') . '</h4>';
echo $this->Form->input('push', array());
echo $this->Form->input('pull', array());
echo $this->Form->input('push_sightings', array());
echo $this->Form->input('caching_enabled', array());
echo '<div class = "input clear" style="width:100%;"><hr /></div>';
echo $this->Form->input('unpublish_event', array(

1
app/View/Servers/edit.ctp
View File

@ -80,6 +80,7 @@
echo '<h4 class="input clear">' . __('Enabled synchronisation methods') . '</h4>';
echo $this->Form->input('push', array());
echo $this->Form->input('pull', array());
echo $this->Form->input('push_sightings', array());
echo $this->Form->input('caching_enabled', array());
echo '<div class = "input clear" style="width:100%;"><hr /><h4>' . __('Misc settings') . '</h4></div>';
echo $this->Form->input('unpublish_event', array(

2
app/View/Servers/index.ctp
View File

@ -27,6 +27,7 @@
<th><?php echo $this->Paginator->sort('internal');?></th>
<th><?php echo $this->Paginator->sort('push');?></th>
<th><?php echo $this->Paginator->sort('pull');?></th>
<th><?php echo $this->Paginator->sort('push_sightings', 'Push Sightings');?></th>
<th><?php echo $this->Paginator->sort('caching_enabled', 'Cache');?></th>
<th><?php echo $this->Paginator->sort('unpublish_event (push event)');?></th>
<th><?php echo $this->Paginator->sort('publish_without_email (pull event)');?></th>
@ -106,6 +107,7 @@ foreach ($servers as $row_pos => $server):
<td><span class="<?php echo ($server['Server']['internal']? 'icon-ok' : 'icon-remove'); ?>" role="img" aria-label="<?php echo ($server['Server']['internal']? __('Yes') : __('No')); ?>" title="<?php echo ($server['Server']['internal']? __('Internal instance that ignores distribution level degradation *WARNING: Only use this setting if you have several internal instances and the sync link is to an internal extension of the current MISP community*') : __('Normal sync link to an external MISP instance. Distribution degradation will follow the normal rules.')); ?>"></span></td>
<td><span class="<?php echo ($server['Server']['push']? 'icon-ok' : 'icon-remove'); ?>" role="img" aria-label="<?php echo ($server['Server']['push']? __('Yes') : __('No')); ?>"></span><span class="short <?php if (!$server['Server']['push'] || empty($ruleDescription['push'])) echo "hidden"; ?>" data-toggle="popover" title="Distribution List" data-content="<?php echo $ruleDescription['push']; ?>"> (<?php echo __('Rules');?>)</span></td>
<td><span class="<?php echo ($server['Server']['pull']? 'icon-ok' : 'icon-remove'); ?>" role="img" aria-label="<?php echo ($server['Server']['pull']? __('Yes') : __('No')); ?>"></span><span class="short <?php if (!$server['Server']['pull'] || empty($ruleDescription['pull'])) echo "hidden"; ?>" data-toggle="popover" title="Distribution List" data-content="<?php echo $ruleDescription['pull']; ?>"> (<?php echo __('Rules');?>)</span></td>
<td class="short"><span class="<?php echo ($server['Server']['push_sightings'] ? 'icon-ok' : 'icon-remove'); ?>" role="img" aria-label="<?php echo ($server['Server']['push_sightings'] ? __('Yes') : __('No')); ?>"></span></td>
<td>
<?php
if ($server['Server']['caching_enabled']) {

11
app/View/Servers/pull.ctp
View File

@ -32,6 +32,17 @@ else:?>
</ul>
<?php
endif;?>
<h2><?php echo __('Sightings pulled');?></h2>
<?php
if (0 == count($pulledSightings)):?>
<p><?php echo __('No sightings pulled');?></p>
<?php
else:?>
<ul>
<?php foreach ($pulledSightins as $e => $p) echo '<li>Event ' . $e . ' : ' . $p . ' sighting(s).</li>'; ?>
</ul>
<?php
endif;?>
</div>
<?php