MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch '2.4' of github.com:MISP/MISP into 2.4

pull/5615/head
iglocska 2020-02-10 14:33:39 +01:00
parent 4ea3612dfc 8e2da13e0e
commit 8803f47a9e
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
20 changed files with 270 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
PyMISP

@ -1 +1 @@
Subproject commit db9c54bb0883296707023cd30f3a23c0c47937bb
Subproject commit cb718b97f1e36e11a06870adb5368309e1c14912

4
app/Controller/AppController.php
View File

@ -47,7 +47,7 @@ class AppController extends Controller
public $helpers = array('Utility', 'OrgImg', 'FontAwesome', 'UserName');
private $__queryVersion = '97';
public $pyMispVersion = '2.4.120';
public $pyMispVersion = '2.4.121';
public $phpmin = '7.2';
public $phprec = '7.4';
public $pythonmin = '3.6';
@ -1162,7 +1162,6 @@ class AppController extends Controller
public function restSearch()
{
$ordered_url_params = func_get_args();
if (empty($this->RestSearch->paramArray[$this->modelClass])) {
throw new NotFoundException(__('RestSearch is not implemented (yet) for this scope.'));
}
@ -1185,7 +1184,6 @@ class AppController extends Controller
if ($filters === false) {
return $exception;
}
$list = array();
$key = empty($filters['key']) ? $filters['returnFormat'] : $filters['key'];
$user = $this->_getApiAuthUser($key, $exception);
if ($user === false) {

3
app/Controller/Component/ACLComponent.php
View File

@ -552,7 +552,8 @@ class ACLComponent extends Component
'delete' => array('perm_admin'),
'downloadTerms' => array('*'),
'edit' => array('*'),
'fetchPGPKey' => array('*'),
'searchGpgKey' => array('*'),
'fetchGpgKey' => array('*'),
'histogram' => array('*'),
'initiatePasswordReset' => array('perm_admin'),
'login' => array('*'),

22
app/Controller/UsersController.php
View File

@ -2129,14 +2129,14 @@ class UsersController extends AppController
$this->Auth->login($newUser['User']);
}
public function fetchPGPKey($email = false)
public function searchGpgKey($email = false)
{
if ($email == false) {
if (!$email) {
throw new NotFoundException('No email provided.');
}
$keys = $this->User->fetchPGPKey($email);
if (is_numeric($keys)) {
throw new NotFoundException('Could not retrieved any keys from the key server.');
$keys = $this->User->searchGpgKey($email);
if (empty($keys)) {
throw new NotFoundException('No keys found for given email at keyserver.');
}
$this->set('keys', $keys);
$this->autorender = false;
@ -2144,6 +2144,18 @@ class UsersController extends AppController
$this->render('ajax/fetchpgpkey');
}
public function fetchGpgKey($fingerprint = null)
{
if (!$fingerprint) {
throw new NotFoundException('No fingerprint provided.');
}
$key = $this->User->fetchGpgKey($fingerprint);
if (!$key) {
throw new NotFoundException('No key with given fingerprint found.');
}
return new CakeResponse(array('body' => $key));
}
public function dashboard()
{
$events = array();

96
app/Lib/Tools/GpgTool.php Normal file
View File

@ -0,0 +1,96 @@
<?php
class GpgTool
{
/**
* @param string $search
* @return array
* @throws Exception
*/
public function searchGpgKey($search)
{
$uri = 'https://pgp.circl.lu/pks/lookup?search=' . urlencode($search) . '&op=index&fingerprint=on&options=mr';
$response = $this->keyServerLookup($uri);
if ($response->code == 404) {
return array(); // no keys found
} else if ($response->code != 200) {
throw new Exception("Fetching the '$uri' failed with HTTP error {$response->code}: {$response->reasonPhrase}");
}
return $this->extractKeySearch($response->body);
}
/**
* @param string $fingerprint
* @return string|null
* @throws Exception
*/
public function fetchGpgKey($fingerprint)
{
$uri = 'https://pgp.circl.lu/pks/lookup?search=0x' . urlencode($fingerprint) . '&op=get&options=mr';
$response = $this->keyServerLookup($uri);
if ($response->code == 404) {
return null; // key with given fingerprint not found
} else if ($response->code != 200) {
throw new Exception("Fetching the '$uri' failed with HTTP error {$response->code}: {$response->reasonPhrase}");
}
$key = $response->body;
return $key;
}
/**
* @param string $body
* @return array
*/
private function extractKeySearch($body)
{
$final = array();
$lines = explode("\n", $body);
foreach ($lines as $line) {
$parts = explode(":", $line);
if ($parts[0] === 'pub') {
if (!empty($temp)) {
$final[] = $temp;
$temp = array();
}
if (strpos($parts[6], 'r') !== false || strpos($parts[6], 'd') !== false || strpos($parts[6], 'e') !== false) {
continue; // skip if key is expired, revoked or disabled
}
$temp = array(
'fingerprint' => $parts[1],
'key_id' => substr($parts[1], -8),
'date' => date('Y-m-d', $parts[4]),
);
} else if ($parts[0] === 'uid' && !empty($temp)) {
$temp['address'] = urldecode($parts[1]);
}
}
if (!empty($temp)) {
$final[] = $temp;
}
return $final;
}
/**
* @param string $uri
* @return HttpSocketResponse
* @throws Exception
*/
private function keyServerLookup($uri)
{
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket();
$response = $HttpSocket->get($uri);
if ($response === false) {
throw new Exception("Could not fetch '$uri'.");
}
return $response;
}
}

7
app/Model/AppModel.php
View File

@ -77,7 +77,7 @@ class AppModel extends Model
27 => false, 28 => false, 29 => false, 30 => false, 31 => false, 32 => false,
33 => false, 34 => false, 35 => false, 36 => false, 37 => false, 38 => false,
39 => false, 40 => false, 41 => false, 42 => false, 43 => false, 44 => false,
45 => false, 46 => false
45 => false, 46 => false, 47 => false
);
public $advanced_updates_description = array(
@ -1319,6 +1319,11 @@ class AppModel extends Model
$sqlArray[] = "ALTER TABLE `events` ADD `sighting_timestamp` int(11) NOT NULL DEFAULT 0 AFTER `publish_timestamp`;";
$sqlArray[] = "ALTER TABLE `servers` ADD `push_sightings` tinyint(1) NOT NULL DEFAULT 0 AFTER `pull`;";
break;
case 47:
$this->__addIndex('tags', 'numerical_value');
$this->__addIndex('taxonomy_predicates', 'numerical_value');
$this->__addIndex('taxonomy_entries', 'numerical_value');
break;
case 'fixNonEmptySharingGroupID':
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
$sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';

7
app/Model/Attribute.php
View File

@ -1594,6 +1594,13 @@ class Attribute extends AppModel
}
$value = ($value) ? '1' : '0';
break;
case 'datetime':
try {
$value = (new DateTime($value))->setTimezone(new DateTimeZone('GMT'))->format('Y-m-d\TH:i:s.uO'); // ISO8601 formating with microseconds
} catch (Exception $e) {
// silently skip. Rejection will be done in runValidation()
}
break;
}
return $value;
}

48
app/Model/Server.php
View File

@ -4390,7 +4390,7 @@ class Server extends AppModel
$dbExpectedSchema = $this->getExpectedDBSchema();
if ($dbExpectedSchema !== false) {
$db_schema_comparison = $this->compareDBSchema($dbActualSchema['schema'], $dbExpectedSchema['schema']);
$db_indexes_comparison = $this->compareDBIndexes($dbActualSchema['indexes'], $dbExpectedSchema['indexes']);
$db_indexes_comparison = $this->compareDBIndexes($dbActualSchema['indexes'], $dbExpectedSchema['indexes'], $dbExpectedSchema);
$schemaDiagnostic['checked_table_column'] = $dbActualSchema['column'];
$schemaDiagnostic['diagnostic'] = $db_schema_comparison;
$schemaDiagnostic['diagnostic_index'] = $db_indexes_comparison;
@ -4665,23 +4665,53 @@ class Server extends AppModel
return $dbDiff;
}
public function compareDBIndexes($actualIndex, $expectedIndex)
public function compareDBIndexes($actualIndex, $expectedIndex, $dbExpectedSchema)
{
$defaultIndexKeylength = 255;
$whitelistTables = array();
$indexDiff = array();
foreach($expectedIndex as $tableName => $indexes) {
if (!array_key_exists($tableName, $actualIndex)) {
// If table does not exists, it is covered by the schema diagnostic
continue; // If table does not exists, it is covered by the schema diagnostic
} elseif(in_array($tableName, $whitelistTables)) {
continue; // Ignore whitelisted tables
} else {
$tableIndexDiff = array_diff($indexes, $actualIndex[$tableName]); // check for missing indexes
if (count($tableIndexDiff) > 0) {
foreach($tableIndexDiff as $columnDiff) {
$indexDiff[$tableName][$columnDiff] = sprintf(__('Column `%s` should be indexed'), $columnDiff);
$columnData = Hash::extract($dbExpectedSchema['schema'][$tableName], sprintf('{n}[column_name=%s]', $columnDiff))[0];
$message = sprintf(__('Column `%s` should be indexed'), $columnDiff);
if ($columnData['data_type'] == 'varchar') {
$keyLength = sprintf('(%s)', $columnData['character_maximum_length'] < $defaultIndexKeylength ? $columnData['character_maximum_length'] : $defaultIndexKeylength);
} elseif ($columnData['data_type'] == 'text') {
$keyLength = sprintf('(%s)', $defaultIndexKeylength);
} else {
$keyLength = '';
}
$sql = sprintf('CREATE INDEX `%s` ON `%s` (%s%s);',
$columnDiff,
$tableName,
$columnDiff,
$keyLength
);
$indexDiff[$tableName][$columnDiff] = array(
'message' => $message,
'sql' => $sql
);
}
}
$tableIndexDiff = array_diff($actualIndex[$tableName], $indexes); // check for additional indexes
if (count($tableIndexDiff) > 0) {
foreach($tableIndexDiff as $columnDiff) {
$indexDiff[$tableName][$columnDiff] = sprintf(__('Column `%s` is indexed but should not'), $columnDiff);
$message = sprintf(__('Column `%s` is indexed but should not'), $columnDiff);
$sql = sprintf('DROP INDEX `%s` ON %s;',
$columnDiff,
$tableName
);
$indexDiff[$tableName][$columnDiff] = array(
'message' => $message,
'sql' => $sql
);
}
}
}
@ -4784,7 +4814,7 @@ class Server extends AppModel
public function stixDiagnostics(&$diagnostic_errors, &$stixVersion, &$cyboxVersion, &$mixboxVersion, &$maecVersion, &$stix2Version, &$pymispVersion)
{
$result = array();
$expected = array('stix' => '>1.2.0.6', 'cybox' => '>2.1.0.18.dev0', 'mixbox' => '1.0.3', 'maec' => '>4.1.0.14', 'stix2' => '>1.2.0', 'pymisp' => '>2.4.93');
$expected = array('stix' => '>1.2.0.6', 'cybox' => '>2.1.0.18.dev0', 'mixbox' => '1.0.3', 'maec' => '>4.1.0.14', 'stix2' => '>1.2.0', 'pymisp' => '>2.4.120');
// check if the STIX and Cybox libraries are working using the test script stixtest.py
$scriptResult = shell_exec($this->getPythonVersion() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'stixtest.py');
$scriptResult = json_decode($scriptResult, true);
@ -5261,15 +5291,15 @@ class Server extends AppModel
public function extensionDiagnostics()
{
$results = array();
$extensions = array('redis', 'gd');
$extensions = array('redis', 'gd', 'ssdeep');
foreach ($extensions as $extension) {
$results['web']['extensions'][$extension] = extension_loaded($extension);
}
if (!is_readable(APP . '/files/scripts/selftest.php')) {
$results['cli'] = false;
} else {
$results['cli'] = exec('php ' . APP . '/files/scripts/selftest.php');
$results['cli'] = json_decode($results['cli'], true);
$execResult = exec('php ' . APP . '/files/scripts/selftest.php');
$results['cli'] = json_decode($execResult, true);
}
return $results;
}

59
app/Model/User.php
View File

@ -2,6 +2,7 @@
App::uses('AppModel', 'Model');
App::uses('AuthComponent', 'Controller/Component');
App::uses('RandomTool', 'Tools');
App::uses('GpgTool', 'Tools');
/**
* @property Log $Log
@ -1038,52 +1039,26 @@ class User extends AppModel
return $message;
}
public function fetchPGPKey($email)
/**
* @param string $email
* @return array
* @throws Exception
*/
public function searchGpgKey($email)
{
App::uses('SyncTool', 'Tools');
$syncTool = new SyncTool();
$HttpSocket = $syncTool->setupHttpSocket();
$response = $HttpSocket->get('https://pgp.circl.lu/pks/lookup?search=' . urlencode($email) . '&op=index&fingerprint=on&options=mr');
if ($response->code != 200) {
return $response->code;
}
return $this->__extractPGPInfo($response->body);
$gpgTool = new GpgTool();
return $gpgTool->searchGpgKey($email);
}
private function __extractPGPInfo($body)
/**
* @param string $fingerprint
* @return string|null
* @throws Exception
*/
public function fetchGpgKey($fingerprint)
{
$final = array();
$lines = explode("\n", $body);
foreach ($lines as $line) {
$parts = explode(":", $line);
if ($parts[0] === 'pub') {
if (!empty($temp)) {
$final[] = $temp;
$temp = array();
}
if (strpos($parts[6], 'r') !== false || strpos($parts[6], 'd') !== false || strpos($parts[6], 'e') !== false) {
continue; // skip if key is expired, revoked or disabled
}
$temp = array(
'fingerprint' => chunk_split($parts[1], 4, ' '),
'key_id' => substr($parts[1], -8),
'date' => date('Y-m-d', $parts[4]),
'uri' => '/pks/lookup?op=get&search=0x' . $parts[1],
);
} else if ($parts[0] === 'uid' && !empty($temp)) {
$temp['address'] = urldecode($parts[1]);
}
}
if (!empty($temp)) {
$final[] = $temp;
}
return $final;
$gpgTool = new GpgTool();
return $gpgTool->fetchGpgKey($fingerprint);
}
public function describeAuthFields()

25
app/View/Attributes/add.ctp
View File

@ -141,6 +141,31 @@
endif;
?>
checkSharingGroup('Attribute');
var $form = $('#AttributeType').closest('form').submit(function( event ) {
if ($('#AttributeType').val() === 'datetime') {
// add timezone of the browser if not set
var allowLocalTZ = true;
var $valueInput = $('#AttributeValue')
var dateValue = moment($valueInput.val())
if (dateValue.isValid()) {
if (dateValue.creationData().format !== "YYYY-MM-DDTHH:mm:ssZ" && dateValue.creationData().format !== "YYYY-MM-DDTHH:mm:ss.SSSSZ") {
// Missing timezone data
var confirm_message = '<?php echo __('Timezone missing, auto-detected as: ') ?>' + dateValue.format('Z')
confirm_message += '<?php echo '\r\n' . __('The following value will be submited instead: '); ?>' + dateValue.toISOString(allowLocalTZ)
if (confirm(confirm_message)) {
$valueInput.val(dateValue.toISOString(allowLocalTZ));
} else {
return false;
}
}
} else {
textStatus = '<?php echo __('Value is not a valid datetime. Excpected format YYYY-MM-DDTHH:mm:ssZ') ?>'
showMessage('fail', textStatus);
return false;
}
}
});
});
</script>
<?php echo $this->element('form_seen_input'); ?>

24
app/View/Elements/healthElements/db_indexes_diagnostic.ctp
View File

@ -19,6 +19,7 @@
<th>Column name</th>
<th>Indexed</th>
<th>Description</th>
<th>Action</th>
</tr>
</thead>
<tbody>
@ -30,15 +31,23 @@
<?php foreach($columnArray as $columnName): ?>
<?php
$columnIndexed = !empty($indexes[$tableName]) && in_array($columnName, $indexes[$tableName]);
$warning = isset($diagnostic[$tableName][$columnName]);
if ($warning) {
$warningArray = isset($diagnostic[$tableName][$columnName]);
if ($warningArray) {
$columnCount++;
}
$rowHtml .= sprintf('%s%s%s%s%s',
sprintf('<tr class="%s">', $warning ? 'error' : 'indexInfo hidden'),
$rowHtml .= sprintf('%s%s%s%s%s%s',
sprintf('<tr class="%s">', $warningArray ? 'error' : 'indexInfo hidden'),
sprintf('<td>%s</td>', h($columnName)),
sprintf('<td><i class="bold fa %s"></i></td>', $columnIndexed ? 'green fa-check' : 'red fa-times'),
sprintf('<td>%s</td>', $warning ? h($diagnostic[$tableName][$columnName]) : ''),
sprintf('<td>%s</td>', $warningArray ? h($diagnostic[$tableName][$columnName]['message']) : ''),
sprintf('<td>%s</td>', $warningArray ?
sprintf(
'<i class="fa fa-wrench useCursorPointer" onclick="quickFixIndexSchema(this, \'%s\')" title="%s" data-query="%s"></i>',
h($diagnostic[$tableName][$columnName]['sql']),
__('Fix Database Index Schema'),
h($diagnostic[$tableName][$columnName]['sql'])
) : ''
),
'</tr>'
);
?>
@ -59,4 +68,9 @@
$('#containerDBIndexes').toggle();
})
})
function quickFixIndexSchema(clicked, sqlQuery) {
var message = "<?php echo sprintf('<div class=\"alert alert-error\" style=\"margin-bottom: 5px;\"><h5>%s</h5> %s</div>', __('Warning'), __('Executing this query might take some time and may harm your database. Please review the query below or backup your database in case of doubt.')) ?>"
message += "<div class=\"well\"><kbd>" + sqlQuery + "</kbd></div>"
openPopover(clicked, message, undefined, 'left');
}
</script>

2
app/View/Servers/add.ctp
View File

@ -74,7 +74,7 @@
echo sprintf(
'<div id="AuthkeyContainer"><p class="red clear" style="width:50%%;">%s</p>%s</div>',
__('Ask the owner of the remote instance for a sync account on their instance, log into their MISP using the sync user\'s credentials and retrieve your API key by navigating to Global actions -> My profile. This key is used to authenticate with the remote instance.'),
$this->Form->input('authkey', array())
$this->Form->input('authkey', array('autocomplete' => 'off'))
);
echo '<div class = "input clear" style="width:100%;"><hr /></div>';
echo '<h4 class="input clear">' . __('Enabled synchronisation methods') . '</h4>';

25
app/View/Users/ajax/fetchpgpkey.ctp
View File

@ -1,5 +1,9 @@
<div class="popover_choice">
<legend><?php echo __('Choose the key that you would like to use'); ?></legend>
<p style="padding:0.3em 10px">
<?php echo __("Do not blindly trust fetched keys and check the fingerprint from other source.") ?>
<a href="https://evil32.com" target="_blank"><?php echo __("And do not check just Key ID, but whole fingerprint.") ?></a>
</p>
<div class="popover_choice_main" id ="popover_choice_main">
<table style="width:100%;">
<tr>
@ -7,15 +11,13 @@
<th style="text-align:left;"><?php echo __('Creation date');?></th>
<th style="padding-right:10px; text-align:left;"><?php echo __('Associated E-mail addresses');?></th>
</tr>
<?php foreach ($keys as $k => $key): ?>
<tr style="border-bottom:1px solid black;" class="templateChoiceButton">
<td role="button" tabindex="0" aria-label="<?php echo __('Select GnuPG key');?>" style="padding-left:10px; text-align:left;width:20%;" title="<?php echo h($key['fingerprint']); ?>" onClick="pgpChoiceSelect('<?php echo h($key['uri']); ?>')"><?php echo h($key['key_id']); ?></td>
<td style="text-align:left;width:20%;" title="<?php echo h($key['fingerprint']); ?>" onClick="pgpChoiceSelect('<?php echo h($key['uri']); ?>')"><?php echo h($key['date']); ?></td>
<td style="padding-right:10px; text-align:left;width:60%;" title="<?php echo h($key['fingerprint']); ?>" onClick="pgpChoiceSelect('<?php echo h($key['uri']); ?>')">
<span class="bold">
<?php echo h($key['fingerprint']); ?>
</span><br />
<?php echo nl2br(h($key['address'])); ?>
<?php foreach ($keys as $key): ?>
<tr style="border-bottom:1px solid black;cursor:pointer;" class="templateChoiceButton" data-fingerprint="<?php echo h($key['fingerprint']); ?>">
<td role="button" tabindex="0" aria-label="<?php echo __('Select GnuPG key');?>" style="padding-left:10px; text-align:left;width:20%;" title="<?php echo h($key['fingerprint']); ?>"><?php echo h($key['key_id']); ?></td>
<td style="text-align:left;width:20%;" title="<?php echo h($key['fingerprint']); ?>"><?php echo h($key['date']); ?></td>
<td style="padding-right:10px; text-align:left;width:60%;" title="<?php echo h($key['fingerprint']); ?>">
<b><?php echo h(chunk_split($key['fingerprint'], 4, ' ')); ?></b><br />
<?php echo nl2br(h($key['address'])); ?>
</td>
</tr>
<?php endforeach; ?>
@ -26,6 +28,11 @@
<script type="text/javascript">
$(document).ready(function() {
resizePopoverBody();
$('tr[data-fingerprint]').click(function () {
var fingerprint = $(this).data('fingerprint');
gpgSelect(fingerprint);
});
});
$(window).resize(function() {

2
app/files/misp-galaxy

@ -1 +1 @@
Subproject commit 6d078a88dd9f715ba90ccda10365fab585ec9c0f
Subproject commit 33aa1c8f3f25a70b9ab393c48db0b9c6a1776971

2
app/files/misp-objects

@ -1 +1 @@
Subproject commit 195fc46a139dba2fdbdc46cb0d6e510738508366
Subproject commit 6c7a8f4524c7926ed722ccddc5d1d0a21c89791d

16
app/files/scripts/selftest.php
View File

@ -1,10 +1,8 @@
<?php
$extensions = array('redis', 'gd');
$results = array();
$results['phpversion'] = phpversion();
foreach ($extensions as $extension) {
$results['extensions'][$extension] = extension_loaded($extension);
}
echo json_encode($results);
?>
$extensions = array('redis', 'gd', 'ssdeep');
$results = array();
$results['phpversion'] = phpversion();
foreach ($extensions as $extension) {
$results['extensions'][$extension] = extension_loaded($extension);
}
echo json_encode($results);

2
app/files/taxonomies

@ -1 +1 @@
Subproject commit d5cc5db3d736e5acede93d514070636834f385d4
Subproject commit 90f77df5792109de3b810006616dcbc68e239a3b

2
app/files/warninglists

@ -1 +1 @@
Subproject commit 5a299129122558bcebefcbc95f827a63f1d99097
Subproject commit 659264240a82893e22df6dccdf8fbdced6398b43

18
app/webroot/js/misp.js
View File

@ -3361,27 +3361,31 @@ function getTextColour(hex) {
}
}
function pgpChoiceSelect(uri) {
function gpgSelect(fingerprint) {
$("#popover_form").fadeOut();
$("#gray_out").fadeOut();
$.ajax({
type: "get",
url: "https://pgp.circl.lu" + uri,
url: "/users/fetchGpgKey/" + fingerprint,
beforeSend: function () {
$(".loading").show();
},
success: function (data) {
var result = data.split("<pre>")[1].split("</pre>")[0];
$("#UserGpgkey").val(result);
$("#UserGpgkey").val(data);
showMessage('success', "Key found!");
},
error: function (data, textStatus, errorThrown) {
showMessage('fail', textStatus + ": " + errorThrown);
},
complete: function () {
$(".loading").hide();
$("#gray_out").fadeOut();
}
},
});
}
function lookupPGPKey(emailFieldName) {
simplePopup("/users/fetchPGPKey/" + $('#' + emailFieldName).val());
var email = $('#' + emailFieldName).val();
simplePopup("/users/searchGpgKey/" + email);
}
function zeroMQServerAction(action) {

2
db_schema.json
View File

@ -5718,5 +5718,5 @@
"id"
]
},
"db_version": "46"
"db_version": "47"
}