MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

new: [API] Proposal sync rework done

pull/4877/head
iglocska 2019-07-12 16:03:08 +02:00
parent 1e5e407ea0
commit c8018d7daa
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
7 changed files with 69 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/Controller/AppController.php
View File

@ -46,7 +46,7 @@ class AppController extends Controller
public $helpers = array('Utility', 'OrgImg', 'FontAwesome');
private $__queryVersion = '79';
private $__queryVersion = '80';
public $pyMispVersion = '2.4.106';
public $phpmin = '7.0';
public $phprec = '7.2';

5
app/Controller/ServersController.php
View File

@ -633,7 +633,7 @@ class ServersController extends AppController
if (!Configure::read('MISP.background_jobs')) {
$result = $this->Server->pull($this->Auth->user(), $id, $technique, $s);
if (is_array($result)) {
$success = sprintf(__('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled.', count($result[0]), count($result[1]), count($result[2])));
$success = sprintf(__('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled.', count($result[0]), count($result[1]), $result[2]));
} else {
$error = $result;
}
@ -1433,6 +1433,9 @@ class ServersController extends AppController
}
}
}
if (!$mismatch && $version[2] < 111) {
$mismatch = 'proposal';
}
if (!$perm_sync) {
$result['status'] = 7;
return new CakeResponse(array('body'=> json_encode($result), 'type' => 'json'));

54
app/Controller/ShadowAttributesController.php
View File

@ -942,11 +942,19 @@ class ShadowAttributesController extends AppController
'LOWER(Event.uuid) LIKE' => '%' . strtolower(trim($this->request['named']['searchall'])) . '%'
));
}
if (isset($this->request['named']['deleted'])) {
$conditions['AND'][] = array(
'ShadowAttribute.deleted' => $this->request['named']['deleted']
);
}
if (!empty($this->request['named']['timestamp'])) {
$conditions['AND'][] = array(
'ShadowAttribute.timestamp >=' => $this->request['named']['timestamp']
);
}
if (!$this->_isRest() && !isset($this->request['named']['deleted'])) {
$conditions['AND'][] = array('ShadowAttribute.deleted' => 0);
}
$params = array(
'conditions' => $conditions,
'fields' => array('ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.timestamp'),
@ -1043,52 +1051,10 @@ class ShadowAttributesController extends AppController
}
}
// deprecated function, returns empty array - proposal sync on more modern versions (>=2.4.111) happens via the shadow_attributes/index endpoint
public function getProposalsByUuidList()
{
if (!$this->_isRest() || !$this->userRole['perm_sync']) {
throw new MethodNotAllowedException(__('This feature is only available using the API to Sync users'));
}
if (!$this->request->is('Post')) {
throw new MethodNotAllowedException(__('This feature is only available using POST requests'));
}
$result = array();
if (!empty($this->request->data)) {
foreach ($this->request->data as $eventUuid) {
$temp = $this->ShadowAttribute->find('all', array(
'conditions' => array(
'event_uuid' => $eventUuid,
'timestamp >' => strtotime("-14 day")
),
'recursive' => -1,
'contain' => array(
'Org' => array('fields' => array('uuid', 'name')),
'EventOrg' => array('fields' => array('uuid', 'name')),
),
));
if (empty($temp)) {
continue;
}
foreach ($temp as $key => $t) {
if ($this->ShadowAttribute->typeIsAttachment($t['ShadowAttribute']['type'])) {
$temp[$key]['ShadowAttribute']['data'] = $this->ShadowAttribute->base64EncodeAttachment($t['ShadowAttribute']);
}
}
$result = array_merge($result, $temp);
}
}
if (empty($result)) {
$this->response->statusCode(404);
$this->set('name', 'No proposals found.');
$this->set('message', 'No proposals found');
$this->set('errors', 'No proposals found');
$this->set('url', '/shadow_attributes/getProposalsByUuidList');
$this->set('_serialize', array('name', 'message', 'url', 'errors'));
$this->response->send();
return false;
} else {
$this->set('result', $result);
$this->render('get_proposals_by_uuid_list');
}
return $this->RestResponse->viewData(array());
}
public function fetchEditForm($id, $field = null)

24
app/Model/Server.php
View File

@ -2364,6 +2364,7 @@ class Server extends AppModel
// if we are downloading a single event, don't fetch all proposals
$conditions = is_numeric($technique) ? array('Event.id' => $technique) : array();
$eventIds = $this->__getEventIdListBasedOnPullTechnique($technique, $server);
$server['Server']['version'] = $this->getRemoteVersion($id);
if (!empty($eventIds['error'])) {
$errors = array(
'1' => __('Not authorised. This is either due to an invalid auth key, or due to the sync user not having authentication permissions enabled on the remote server. Another reason could be an incorrect sync server setting.'),
@ -2416,18 +2417,10 @@ class Server extends AppModel
}
}
if ($jobId) {
$job->saveField('progress', 50);
$job->saveField('message', 'Pulling proposals.');
}
$events = $eventModel->find('list', array(
'fields' => array('uuid'),
'recursive' => -1,
'conditions' => $conditions
));
$pulledProposals = array();
if (!empty($events)) {
$proposals = $eventModel->downloadProposalsFromServer($events, $server);
$pulledProposals = $this->__handlePulledProposals($proposals, $events, $job, $jobId, $eventModel, $user);
}
$pulledProposals = $eventModel->ShadowAttribute->pullProposals($user, $server);
if ($jobId) {
$job->saveField('progress', 100);
$job->saveField('message', 'Pull completed.');
@ -2443,7 +2436,12 @@ class Server extends AppModel
'action' => 'pull',
'user_id' => $user['id'],
'title' => 'Pull from ' . $server['Server']['url'] . ' initiated by ' . $email,
'change' => count($successes) . ' events and ' . array_sum($pulledProposals) . ' proposals pulled or updated. ' . count($fails) . ' events failed or didn\'t need an update.'
'change' => sprintf(
'%s events and %s proposals pulled or updated. %s events failed or didn\'t need an update.',
count($successes),
$pulledProposals,
count($fails)
)
));
return array($successes, $fails, $pulledProposals);
}
@ -4051,6 +4049,9 @@ class Server extends AppModel
if ($response === false && $localVersion['hotfix'] < $remoteVersion[2]) {
$response = "Sync to Server ('" . $id . "') initiated, but the remote instance is a few hotfixes ahead. Make sure you keep your instance up to date!";
}
if (empty($response) && $remoteVersion[2] < 111) {
$response = "Sync to Server ('" . $id . "') initiated, but version 2.4.111 is required in order to be able to pull proposals from the remote side.";
}
if ($response !== false) {
$this->Log = ClassRegistry::init('Log');
@ -4506,6 +4507,7 @@ class Server extends AppModel
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket($server);
$request = $this->setupSyncRequest($server);
$response = $HttpSocket->get($server['Server']['url'] . '/servers/getVersion', $data = '', $request);
if ($response->code == 200) {
try {

23
app/Model/ShadowAttribute.php
View File

@ -487,6 +487,7 @@ class ShadowAttribute extends AppModel
$oldsa = $this->find('first', array(
'conditions' => array(
'event_uuid' => $sa['event_uuid'],
'uuid' => $sa['uuid'],
'value' => $sa['value'],
'type' => $sa['type'],
'category' => $sa['category'],
@ -599,7 +600,7 @@ class ShadowAttribute extends AppModel
if (empty($proposal['event_uuid']) || empty($proposal['Org'])) {
return false;
}
if (!empty($proposal['id'])) {
if (isset($proposal['id'])) {
unset($proposal['id']);
}
$event = $this->Event->find('first', array(
@ -624,31 +625,34 @@ class ShadowAttribute extends AppModel
$oldsa = $this->findOldProposal($proposal);
if (!$oldsa || $oldsa['timestamp'] < $proposal['timestamp']) {
if ($oldsa) {
$shadowAttribute->delete($oldsa['id']);
$this->delete($oldsa['id']);
}
if (isset($proposal['old_id'])) {
$oldAttribute = $eventModel->Attribute->find('first', array('recursive' => -1, 'conditions' => array('uuid' => $proposal['uuid'])));
$oldAttribute = $this->Attribute->find('first', array('recursive' => -1, 'conditions' => array('uuid' => $proposal['uuid'])));
if ($oldAttribute) {
$proposal['old_id'] = $oldAttribute['Attribute']['id'];
} else {
$proposal['old_id'] = 0;
}
} else {
$proposal['old_id'] = 0;
}
$proposal['org_id'] = $this->Organisation->captureOrg($proposal['Org'], $user);
$proposal['org_id'] = $this->Event->Orgc->captureOrg($proposal['Org'], $user);
unset($proposal['Org']);
$this->create();
if ($this->save($proposal)) {
if (!isset($proposal['deleted']) || !$proposal['deleted']) {
$this->sendProposalAlertEmail($proposal['event_id']);
}
return true;
}
}
return true;
return false;
}
public function pullProposals($user, $server, $HttpSocket = null)
{
$version = explode($server['Server']['version']);
$version = explode('.', $server['Server']['version']);
if (
($version[0] == 2 && $version[1] == 4 && $version[2] < 111)
) {
@ -660,9 +664,10 @@ class ShadowAttribute extends AppModel
$i = 1;
$fetchedCount = 0;
$chunk_size = 1000;
$timestamp = strtotime("-90 day");
while(true) {
$uri = sprintf(
'%s/shadow_attributes/index/all:1/timestamp:%s/limit:%s/page:%s.json',
'%s/shadow_attributes/index/all:1/timestamp:%s/limit:%s/page:%s/deleted[]:0/deleted[]:1.json',
$url,
$timestamp,
$chunk_size,
@ -670,14 +675,14 @@ class ShadowAttribute extends AppModel
);
$i += 1;
$response = $HttpSocket->get($uri, false, $request);
if ($response->isOk()) {
if ($response->code == 200) {
$data = json_decode($response->body, true);
if (empty($data)) {
return $fetchedCount;
}
$returnSize = count($data);
foreach ($data as $k => $proposal) {
$result = $this->capture($proposal, $user);
$result = $this->capture($proposal['ShadowAttribute'], $user);
if ($result) {
$fetchedCount += 1;
}

24
app/View/ShadowAttributes/index.ctp
View File

@ -23,7 +23,7 @@
array(
'text' => __('My Org\'s Events'),
'active' => !$all,
'url' => '/shadow_attributes/index'
'url' => '/shadow_attributes/index/all:0'
),
array(
'text' => __('All Events'),
@ -31,6 +31,12 @@
'url' => '/shadow_attributes/index/all:1'
)
)
),
array(
'type' => 'search',
'button' => __('Filter'),
'placeholder' => __('Enter value to search'),
'data' => '',
)
)
);
@ -38,6 +44,7 @@
?>
<table class="table table-striped table-hover table-condensed">
<tr>
<th><?php echo $this->Paginator->sort('id');?></th>
<th><?php echo __('Event');?></th>
<th>
<?php echo $this->Paginator->sort('org', __('Proposal by'));?>
@ -66,6 +73,9 @@
</tr>
<?php foreach ($shadowAttributes as $event):?>
<tr>
<td class="short">
<?php echo h($event['ShadowAttribute']['id']);?>
</td>
<td class="short" onclick="document.location.href ='<?php echo $baseurl."/events/view/".$event['Event']['id'];?>'">
<?php echo h($event['Event']['id']);?>
</td>
@ -125,3 +135,15 @@
<?php
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event-collection', 'menuItem' => 'viewProposals'));
?>
<script type="text/javascript">
$(document).ready(function() {
$('#quickFilterButton').click(function() {
runIndexQuickFilter('/all:<?php echo h($all); ?>');
});
$('#quickFilterField').on('keypress', function (e) {
if(e.which === 13) {
runIndexQuickFilter('/all:<?php echo h($all); ?>');
}
});
});
</script>

5
app/webroot/js/misp.js
View File

@ -2975,8 +2975,11 @@ function testConnection(id) {
compatibility = "Incompatible";
compatibility_colour = "red";
}
} else if (result.mismatch == "proposal") {
compatibility_colour = "orange";
compatibility = "Proposal pull disabled (remote version < v2.4.111)";
}
if (result.mismatch != false) {
if (result.mismatch != false && result.mismatch != "proposal") {
if (result.newer == "remote") status_message = "Local instance outdated, update!";
else status_message = "Remote outdated, notify admin!"
colours.status = 'class="' + issue_colour + '"';