mirror of https://github.com/MISP/MISP
Merge branch 'hotfix/sync_rework' into 2.4
commit
e626936191
|
@ -46,7 +46,7 @@ class AppController extends Controller
|
|||
|
||||
public $helpers = array('Utility', 'OrgImg', 'FontAwesome');
|
||||
|
||||
private $__queryVersion = '79';
|
||||
private $__queryVersion = '80';
|
||||
public $pyMispVersion = '2.4.106';
|
||||
public $phpmin = '7.0';
|
||||
public $phprec = '7.2';
|
||||
|
|
|
@ -633,7 +633,7 @@ class ServersController extends AppController
|
|||
if (!Configure::read('MISP.background_jobs')) {
|
||||
$result = $this->Server->pull($this->Auth->user(), $id, $technique, $s);
|
||||
if (is_array($result)) {
|
||||
$success = sprintf(__('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled.', count($result[0]), count($result[1]), count($result[2])));
|
||||
$success = sprintf(__('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled.', count($result[0]), count($result[1]), $result[2]));
|
||||
} else {
|
||||
$error = $result;
|
||||
}
|
||||
|
@ -1433,6 +1433,9 @@ class ServersController extends AppController
|
|||
}
|
||||
}
|
||||
}
|
||||
if (!$mismatch && $version[2] < 111) {
|
||||
$mismatch = 'proposal';
|
||||
}
|
||||
if (!$perm_sync) {
|
||||
$result['status'] = 7;
|
||||
return new CakeResponse(array('body'=> json_encode($result), 'type' => 'json'));
|
||||
|
|
|
@ -942,11 +942,19 @@ class ShadowAttributesController extends AppController
|
|||
'LOWER(Event.uuid) LIKE' => '%' . strtolower(trim($this->request['named']['searchall'])) . '%'
|
||||
));
|
||||
}
|
||||
if (isset($this->request['named']['deleted'])) {
|
||||
$conditions['AND'][] = array(
|
||||
'ShadowAttribute.deleted' => $this->request['named']['deleted']
|
||||
);
|
||||
}
|
||||
if (!empty($this->request['named']['timestamp'])) {
|
||||
$conditions['AND'][] = array(
|
||||
'ShadowAttribute.timestamp >=' => $this->request['named']['timestamp']
|
||||
);
|
||||
}
|
||||
if (!$this->_isRest() && !isset($this->request['named']['deleted'])) {
|
||||
$conditions['AND'][] = array('ShadowAttribute.deleted' => 0);
|
||||
}
|
||||
$params = array(
|
||||
'conditions' => $conditions,
|
||||
'fields' => array('ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.timestamp'),
|
||||
|
@ -1043,52 +1051,10 @@ class ShadowAttributesController extends AppController
|
|||
}
|
||||
}
|
||||
|
||||
// deprecated function, returns empty array - proposal sync on more modern versions (>=2.4.111) happens via the shadow_attributes/index endpoint
|
||||
public function getProposalsByUuidList()
|
||||
{
|
||||
if (!$this->_isRest() || !$this->userRole['perm_sync']) {
|
||||
throw new MethodNotAllowedException(__('This feature is only available using the API to Sync users'));
|
||||
}
|
||||
if (!$this->request->is('Post')) {
|
||||
throw new MethodNotAllowedException(__('This feature is only available using POST requests'));
|
||||
}
|
||||
$result = array();
|
||||
if (!empty($this->request->data)) {
|
||||
foreach ($this->request->data as $eventUuid) {
|
||||
$temp = $this->ShadowAttribute->find('all', array(
|
||||
'conditions' => array(
|
||||
'event_uuid' => $eventUuid,
|
||||
'timestamp >' => strtotime("-14 day")
|
||||
),
|
||||
'recursive' => -1,
|
||||
'contain' => array(
|
||||
'Org' => array('fields' => array('uuid', 'name')),
|
||||
'EventOrg' => array('fields' => array('uuid', 'name')),
|
||||
),
|
||||
));
|
||||
if (empty($temp)) {
|
||||
continue;
|
||||
}
|
||||
foreach ($temp as $key => $t) {
|
||||
if ($this->ShadowAttribute->typeIsAttachment($t['ShadowAttribute']['type'])) {
|
||||
$temp[$key]['ShadowAttribute']['data'] = $this->ShadowAttribute->base64EncodeAttachment($t['ShadowAttribute']);
|
||||
}
|
||||
}
|
||||
$result = array_merge($result, $temp);
|
||||
}
|
||||
}
|
||||
if (empty($result)) {
|
||||
$this->response->statusCode(404);
|
||||
$this->set('name', 'No proposals found.');
|
||||
$this->set('message', 'No proposals found');
|
||||
$this->set('errors', 'No proposals found');
|
||||
$this->set('url', '/shadow_attributes/getProposalsByUuidList');
|
||||
$this->set('_serialize', array('name', 'message', 'url', 'errors'));
|
||||
$this->response->send();
|
||||
return false;
|
||||
} else {
|
||||
$this->set('result', $result);
|
||||
$this->render('get_proposals_by_uuid_list');
|
||||
}
|
||||
return $this->RestResponse->viewData(array());
|
||||
}
|
||||
|
||||
public function fetchEditForm($id, $field = null)
|
||||
|
|
|
@ -2364,6 +2364,7 @@ class Server extends AppModel
|
|||
// if we are downloading a single event, don't fetch all proposals
|
||||
$conditions = is_numeric($technique) ? array('Event.id' => $technique) : array();
|
||||
$eventIds = $this->__getEventIdListBasedOnPullTechnique($technique, $server);
|
||||
$server['Server']['version'] = $this->getRemoteVersion($id);
|
||||
if (!empty($eventIds['error'])) {
|
||||
$errors = array(
|
||||
'1' => __('Not authorised. This is either due to an invalid auth key, or due to the sync user not having authentication permissions enabled on the remote server. Another reason could be an incorrect sync server setting.'),
|
||||
|
@ -2416,18 +2417,10 @@ class Server extends AppModel
|
|||
}
|
||||
}
|
||||
if ($jobId) {
|
||||
$job->saveField('progress', 50);
|
||||
$job->saveField('message', 'Pulling proposals.');
|
||||
}
|
||||
$events = $eventModel->find('list', array(
|
||||
'fields' => array('uuid'),
|
||||
'recursive' => -1,
|
||||
'conditions' => $conditions
|
||||
));
|
||||
$pulledProposals = array();
|
||||
if (!empty($events)) {
|
||||
$proposals = $eventModel->downloadProposalsFromServer($events, $server);
|
||||
$pulledProposals = $this->__handlePulledProposals($proposals, $events, $job, $jobId, $eventModel, $user);
|
||||
}
|
||||
$pulledProposals = $eventModel->ShadowAttribute->pullProposals($user, $server);
|
||||
if ($jobId) {
|
||||
$job->saveField('progress', 100);
|
||||
$job->saveField('message', 'Pull completed.');
|
||||
|
@ -2443,7 +2436,12 @@ class Server extends AppModel
|
|||
'action' => 'pull',
|
||||
'user_id' => $user['id'],
|
||||
'title' => 'Pull from ' . $server['Server']['url'] . ' initiated by ' . $email,
|
||||
'change' => count($successes) . ' events and ' . array_sum($pulledProposals) . ' proposals pulled or updated. ' . count($fails) . ' events failed or didn\'t need an update.'
|
||||
'change' => sprintf(
|
||||
'%s events and %s proposals pulled or updated. %s events failed or didn\'t need an update.',
|
||||
count($successes),
|
||||
$pulledProposals,
|
||||
count($fails)
|
||||
)
|
||||
));
|
||||
return array($successes, $fails, $pulledProposals);
|
||||
}
|
||||
|
@ -4051,6 +4049,9 @@ class Server extends AppModel
|
|||
if ($response === false && $localVersion['hotfix'] < $remoteVersion[2]) {
|
||||
$response = "Sync to Server ('" . $id . "') initiated, but the remote instance is a few hotfixes ahead. Make sure you keep your instance up to date!";
|
||||
}
|
||||
if (empty($response) && $remoteVersion[2] < 111) {
|
||||
$response = "Sync to Server ('" . $id . "') initiated, but version 2.4.111 is required in order to be able to pull proposals from the remote side.";
|
||||
}
|
||||
|
||||
if ($response !== false) {
|
||||
$this->Log = ClassRegistry::init('Log');
|
||||
|
@ -4506,6 +4507,7 @@ class Server extends AppModel
|
|||
App::uses('SyncTool', 'Tools');
|
||||
$syncTool = new SyncTool();
|
||||
$HttpSocket = $syncTool->setupHttpSocket($server);
|
||||
$request = $this->setupSyncRequest($server);
|
||||
$response = $HttpSocket->get($server['Server']['url'] . '/servers/getVersion', $data = '', $request);
|
||||
if ($response->code == 200) {
|
||||
try {
|
||||
|
|
|
@ -487,6 +487,7 @@ class ShadowAttribute extends AppModel
|
|||
$oldsa = $this->find('first', array(
|
||||
'conditions' => array(
|
||||
'event_uuid' => $sa['event_uuid'],
|
||||
'uuid' => $sa['uuid'],
|
||||
'value' => $sa['value'],
|
||||
'type' => $sa['type'],
|
||||
'category' => $sa['category'],
|
||||
|
@ -599,7 +600,7 @@ class ShadowAttribute extends AppModel
|
|||
if (empty($proposal['event_uuid']) || empty($proposal['Org'])) {
|
||||
return false;
|
||||
}
|
||||
if (!empty($proposal['id'])) {
|
||||
if (isset($proposal['id'])) {
|
||||
unset($proposal['id']);
|
||||
}
|
||||
$event = $this->Event->find('first', array(
|
||||
|
@ -624,31 +625,34 @@ class ShadowAttribute extends AppModel
|
|||
$oldsa = $this->findOldProposal($proposal);
|
||||
if (!$oldsa || $oldsa['timestamp'] < $proposal['timestamp']) {
|
||||
if ($oldsa) {
|
||||
$shadowAttribute->delete($oldsa['id']);
|
||||
$this->delete($oldsa['id']);
|
||||
}
|
||||
if (isset($proposal['old_id'])) {
|
||||
$oldAttribute = $eventModel->Attribute->find('first', array('recursive' => -1, 'conditions' => array('uuid' => $proposal['uuid'])));
|
||||
$oldAttribute = $this->Attribute->find('first', array('recursive' => -1, 'conditions' => array('uuid' => $proposal['uuid'])));
|
||||
if ($oldAttribute) {
|
||||
$proposal['old_id'] = $oldAttribute['Attribute']['id'];
|
||||
} else {
|
||||
$proposal['old_id'] = 0;
|
||||
}
|
||||
} else {
|
||||
$proposal['old_id'] = 0;
|
||||
}
|
||||
$proposal['org_id'] = $this->Organisation->captureOrg($proposal['Org'], $user);
|
||||
$proposal['org_id'] = $this->Event->Orgc->captureOrg($proposal['Org'], $user);
|
||||
unset($proposal['Org']);
|
||||
$this->create();
|
||||
if ($this->save($proposal)) {
|
||||
if (!isset($proposal['deleted']) || !$proposal['deleted']) {
|
||||
$this->sendProposalAlertEmail($proposal['event_id']);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
public function pullProposals($user, $server, $HttpSocket = null)
|
||||
{
|
||||
$version = explode($server['Server']['version']);
|
||||
$version = explode('.', $server['Server']['version']);
|
||||
if (
|
||||
($version[0] == 2 && $version[1] == 4 && $version[2] < 111)
|
||||
) {
|
||||
|
@ -660,9 +664,10 @@ class ShadowAttribute extends AppModel
|
|||
$i = 1;
|
||||
$fetchedCount = 0;
|
||||
$chunk_size = 1000;
|
||||
$timestamp = strtotime("-90 day");
|
||||
while(true) {
|
||||
$uri = sprintf(
|
||||
'%s/shadow_attributes/index/all:1/timestamp:%s/limit:%s/page:%s.json',
|
||||
'%s/shadow_attributes/index/all:1/timestamp:%s/limit:%s/page:%s/deleted[]:0/deleted[]:1.json',
|
||||
$url,
|
||||
$timestamp,
|
||||
$chunk_size,
|
||||
|
@ -670,14 +675,14 @@ class ShadowAttribute extends AppModel
|
|||
);
|
||||
$i += 1;
|
||||
$response = $HttpSocket->get($uri, false, $request);
|
||||
if ($response->isOk()) {
|
||||
if ($response->code == 200) {
|
||||
$data = json_decode($response->body, true);
|
||||
if (empty($data)) {
|
||||
return $fetchedCount;
|
||||
}
|
||||
$returnSize = count($data);
|
||||
foreach ($data as $k => $proposal) {
|
||||
$result = $this->capture($proposal, $user);
|
||||
$result = $this->capture($proposal['ShadowAttribute'], $user);
|
||||
if ($result) {
|
||||
$fetchedCount += 1;
|
||||
}
|
||||
|
|
|
@ -23,7 +23,7 @@
|
|||
array(
|
||||
'text' => __('My Org\'s Events'),
|
||||
'active' => !$all,
|
||||
'url' => '/shadow_attributes/index'
|
||||
'url' => '/shadow_attributes/index/all:0'
|
||||
),
|
||||
array(
|
||||
'text' => __('All Events'),
|
||||
|
@ -31,6 +31,12 @@
|
|||
'url' => '/shadow_attributes/index/all:1'
|
||||
)
|
||||
)
|
||||
),
|
||||
array(
|
||||
'type' => 'search',
|
||||
'button' => __('Filter'),
|
||||
'placeholder' => __('Enter value to search'),
|
||||
'data' => '',
|
||||
)
|
||||
)
|
||||
);
|
||||
|
@ -38,6 +44,7 @@
|
|||
?>
|
||||
<table class="table table-striped table-hover table-condensed">
|
||||
<tr>
|
||||
<th><?php echo $this->Paginator->sort('id');?></th>
|
||||
<th><?php echo __('Event');?></th>
|
||||
<th>
|
||||
<?php echo $this->Paginator->sort('org', __('Proposal by'));?>
|
||||
|
@ -66,6 +73,9 @@
|
|||
</tr>
|
||||
<?php foreach ($shadowAttributes as $event):?>
|
||||
<tr>
|
||||
<td class="short">
|
||||
<?php echo h($event['ShadowAttribute']['id']);?>
|
||||
</td>
|
||||
<td class="short" onclick="document.location.href ='<?php echo $baseurl."/events/view/".$event['Event']['id'];?>'">
|
||||
<?php echo h($event['Event']['id']);?>
|
||||
</td>
|
||||
|
@ -125,3 +135,15 @@
|
|||
<?php
|
||||
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event-collection', 'menuItem' => 'viewProposals'));
|
||||
?>
|
||||
<script type="text/javascript">
|
||||
$(document).ready(function() {
|
||||
$('#quickFilterButton').click(function() {
|
||||
runIndexQuickFilter('/all:<?php echo h($all); ?>');
|
||||
});
|
||||
$('#quickFilterField').on('keypress', function (e) {
|
||||
if(e.which === 13) {
|
||||
runIndexQuickFilter('/all:<?php echo h($all); ?>');
|
||||
}
|
||||
});
|
||||
});
|
||||
</script>
|
||||
|
|
|
@ -2975,8 +2975,11 @@ function testConnection(id) {
|
|||
compatibility = "Incompatible";
|
||||
compatibility_colour = "red";
|
||||
}
|
||||
} else if (result.mismatch == "proposal") {
|
||||
compatibility_colour = "orange";
|
||||
compatibility = "Proposal pull disabled (remote version < v2.4.111)";
|
||||
}
|
||||
if (result.mismatch != false) {
|
||||
if (result.mismatch != false && result.mismatch != "proposal") {
|
||||
if (result.newer == "remote") status_message = "Local instance outdated, update!";
|
||||
else status_message = "Remote outdated, notify admin!"
|
||||
colours.status = 'class="' + issue_colour + '"';
|
||||
|
|
Loading…
Reference in New Issue