MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'develop' of github.com:MISP/MISP into develop

pull/9491/head
Sami Mokaddem 2024-01-11 15:47:54 +01:00
parent 75fc8936c2 e965203484
commit 0e0eed218f
No known key found for this signature in database
GPG Key ID: 164C473F627A06FA
9 changed files with 160 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
app/Console/Command/EventShell.php
View File

@ -53,10 +53,21 @@ class EventShell extends AppShell
$parser->addSubcommand('mergeTags', [
'help' => __('Merge tags'),
'parser' => [
'arguments' => array(
'arguments' => [
'source' => ['help' => __('Source tag ID or name. Source tag will be deleted.'), 'required' => true],
'destination' => ['help' => __('Destination tag ID or name.'), 'required' => true],
)
],
],
]);
$parser->addSubcommand('reportValidationIssuesAttributes', [
'help' => __('Report validation issues on attributes'),
]);
$parser->addSubcommand('normalizeIpAddress', [
'help' => __('Normalize IP address format in old events'),
'parser' => [
'options' => [
'dry-run' => ['help' => __('Just show what changes will be made.'), 'boolean' => true],
],
],
]);
return $parser;
@ -636,18 +647,28 @@ class EventShell extends AppShell
}
}
/**
* @param int $userId
* @return array
*/
private function getUser($userId)
public function reportValidationIssuesAttributes()
{
$user = $this->User->getAuthUser($userId, true);
if (empty($user)) {
$this->error("User with ID $userId does not exist.");
foreach ($this->Event->Attribute->reportValidationIssuesAttributes() as $validationIssue) {
echo $this->json($validationIssue) . "\n";
}
}
public function normalizeIpAddress()
{
$dryRun = $this->param('dry-run');
$count = 0;
foreach ($this->Event->Attribute->normalizeIpAddress($dryRun) as $attribute) {
$count++;
echo JsonTool::encode($attribute) . "\n";
}
if ($dryRun) {
$this->err(__n("%s attribute to fix", "%s attributes to fix", $count, $count));
} else {
$this->err(__n("%s attribute fixed", "%s attributes fixed", $count, $count));
}
Configure::write('CurrentUserId', $user['id']); // for audit logging purposes
return $user;
}
public function generateTopCorrelations()
@ -668,4 +689,18 @@ class EventShell extends AppShell
$this->Job->save($job);
}
}
/**
* @param int $userId
* @return array
*/
private function getUser($userId)
{
$user = $this->User->getAuthUser($userId, true);
if (empty($user)) {
$this->error("User with ID $userId does not exist.");
}
Configure::write('CurrentUserId', $user['id']); // for audit logging purposes
return $user;
}
}

2
app/Controller/AttributesController.php
View File

@ -1917,7 +1917,7 @@ class AttributesController extends AppController
public function reportValidationIssuesAttributes($eventId = false)
{
// search for validation problems in the attributes
$this->set('result', $this->Attribute->reportValidationIssuesAttributes($eventId));
$this->set('result', iterator_to_array($this->Attribute->reportValidationIssuesAttributes($eventId)));
}
public function generateCorrelation()

2
app/Controller/EventsController.php
View File

@ -2383,7 +2383,7 @@ class EventsController extends AppController
$results = $this->Event->addMISPExportFile($this->Auth->user(), $data, $isXml, $takeOwnership, $publish);
} catch (Exception $e) {
$this->log("Exception during processing MISP file import: {$e->getMessage()}");
$this->Flash->error(__('Could not process MISP export file. %s.', $e->getMessage()));
$this->Flash->error(__('Could not process MISP export file. %s', $e->getMessage()));
$this->redirect(['controller' => 'events', 'action' => 'add_misp_export']);
}
}

27
app/Lib/Tools/AttributeValidationTool.php
View File

@ -41,7 +41,7 @@ class AttributeValidationTool
switch ($type) {
case 'ip-src':
case 'ip-dst':
return self::compressIpv6($value);
return self::normalizeIp($value);
case 'md5':
case 'sha1':
case 'sha224':
@ -98,7 +98,7 @@ class AttributeValidationTool
$parts[0] = $punyCode;
}
}
$parts[1] = self::compressIpv6($parts[1]);
$parts[1] = self::normalizeIp($parts[1]);
return "$parts[0]|$parts[1]";
case 'filename|md5':
case 'filename|sha1':
@ -175,7 +175,7 @@ class AttributeValidationTool
} else {
return $value;
}
return self::compressIpv6($parts[0]) . '|' . $parts[1];
return self::normalizeIp($parts[0]) . '|' . $parts[1];
case 'mac-address':
case 'mac-eui-64':
$value = str_replace(array('.', ':', '-', ' '), '', strtolower($value));
@ -700,11 +700,30 @@ class AttributeValidationTool
* @param string $value
* @return string
*/
private static function compressIpv6($value)
private static function normalizeIp($value)
{
// If IP is a CIDR
if (strpos($value, '/')) {
list($ip, $range) = explode('/', $value, 2);
// Compress IPv6
if (strpos($ip, ':') && $converted = inet_pton($ip)) {
$ip = inet_ntop($converted);
}
// If IP is in CIDR format, but the network is 32 for IPv4 or 128 for IPv6, normalize to non CIDR type
if (($range === '32' && strpos($value, '.')) || ($range === '128' && strpos($value, ':'))) {
return $ip;
}
return "$ip/$range";
}
// Compress IPv6
if (strpos($value, ':') && $converted = inet_pton($value)) {
return inet_ntop($converted);
}
return $value;
}

98
app/Model/Attribute.php
View File

@ -1224,38 +1224,89 @@ class Attribute extends AppModel
$this->Correlation->purgeCorrelations($eventId);
}
public function reportValidationIssuesAttributes($eventId)
/**
* @param array $conditions
* @return Generator|void
*/
private function fetchAttributesInChunks(array $conditions = [])
{
while (true) {
$attributes = $this->find('all', [
'recursive' => -1,
'conditions' => $conditions,
'limit' => 500,
'order' => 'Attribute.id',
]);
if (empty($attributes)) {
return;
}
foreach ($attributes as $attribute) {
yield $attribute;
}
$count = count($attributes);
$lastAttribute = $attributes[$count - 1];
$conditions['Attribute.id >'] = $lastAttribute['Attribute']['id'];
}
}
/**
* @param int|null $eventId
* @return Generator
*/
public function reportValidationIssuesAttributes($eventId = null)
{
$conditions = array();
if ($eventId && is_numeric($eventId)) {
$conditions = array('event_id' => $eventId);
}
$attributeIds = $this->find('column', array(
'fields' => array('id'),
'conditions' => $conditions
));
$chunks = array_chunk($attributeIds, 500);
$attributes = $this->fetchAttributesInChunks($conditions);
$result = array();
foreach ($chunks as $chunk) {
$attributes = $this->find('all', array('recursive' => -1, 'conditions' => array('id' => $chunk)));
foreach ($attributes as $attribute) {
$this->set($attribute);
if (!$this->validates()) {
$resultErrors = array();
foreach ($this->validationErrors as $field => $error) {
$resultErrors[$field] = array('value' => $attribute['Attribute'][$field], 'error' => $error[0]);
}
$result[] = [
'id' => $attribute['Attribute']['id'],
'error' => $resultErrors,
'details' => 'Event ID: [' . $attribute['Attribute']['event_id'] . "] - Category: [" . $attribute['Attribute']['category'] . "] - Type: [" . $attribute['Attribute']['type'] . "] - Value: [" . $attribute['Attribute']['value'] . ']',
];
foreach ($attributes as $attribute) {
$this->set($attribute);
if (!$this->validates()) {
$resultErrors = [];
foreach ($this->validationErrors as $field => $error) {
$resultErrors[$field] = ['value' => $attribute['Attribute'][$field], 'error' => $error[0]];
}
yield [
'id' => $attribute['Attribute']['id'],
'error' => $resultErrors,
'details' => 'Event ID: [' . $attribute['Attribute']['event_id'] . "] - Category: [" . $attribute['Attribute']['category'] . "] - Type: [" . $attribute['Attribute']['type'] . "] - Value: [" . $attribute['Attribute']['value'] . ']',
];
}
}
}
/**
* @param bool $dryRun If true, no changes will be made to
* @return Generator
* @throws Exception
*/
public function normalizeIpAddress($dryRun = false)
{
$attributes = $this->fetchAttributesInChunks([
'Attribute.type' => ['ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'domain|ip'],
]);
foreach ($attributes as $attribute) {
$value = $attribute['Attribute']['value'];
$normalizedValue = AttributeValidationTool::modifyBeforeValidation($attribute['Attribute']['type'], $value);
if ($value !== $normalizedValue) {
if (!$dryRun) {
$attribute['Attribute']['value'] = $normalizedValue;
$this->save($attribute, true, ['value1', 'value2']);
}
yield [
'id' => (int) $attribute['Attribute']['id'],
'event_id' => (int) $attribute['Attribute']['event_id'],
'type' => $attribute['Attribute']['type'],
'value' => $value,
'normalized_value' => $normalizedValue,
];
}
}
return $result;
}
/**
@ -1610,6 +1661,7 @@ class Attribute extends AppModel
* @param array $user
* @param array $options
* @param int|false $result_count If false, count is not fetched
* @param bool $real_count
* @return array
* @throws Exception
*/
@ -3673,7 +3725,7 @@ class Attribute extends AppModel
);
}
private function findAttributeByValue($attribute)
private function findAttributeByValue(array $attribute)
{
$type = $attribute['type'];
$conditions = [

10
app/Test/AttributeValidationToolTest.php
View File

@ -124,6 +124,16 @@ class AttributeValidationToolTest extends TestCase
]);
}
public function testRemoveCidrFromIp(): void
{
$this->assertEquals('127.0.0.1', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1/32'));
$this->assertEquals('127.0.0.1/31', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1/31'));
$this->assertEquals('example.com|1234:fd2:5621:1:89::4500', AttributeValidationTool::modifyBeforeValidation('domain|ip', 'example.com|1234:0fd2:5621:0001:0089:0000:0000:4500/128'));
$this->assertEquals('1234:fd2:5621:1:89::4500|80', AttributeValidationTool::modifyBeforeValidation('ip-src|port', '1234:0fd2:5621:0001:0089:0000:0000:4500/128|80'));
$this->assertEquals('1234:fd2:5621:1:89::4500/127|80', AttributeValidationTool::modifyBeforeValidation('ip-src|port', '1234:0fd2:5621:0001:0089:0000:0000:4500/127|80'));
$this->assertEquals('127.0.0.1', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1'));
}
public function testCompressIpv6(): void
{
$this->assertEquals('1234:fd2:5621:1:89::4500', AttributeValidationTool::modifyBeforeValidation('ip-src', '1234:0fd2:5621:0001:0089:0000:0000:4500'));

12
app/View/Elements/global_menu.ctp
View File

@ -277,11 +277,6 @@
'url' => $baseurl . '/servers/createSync',
'requirement' => $isAclSync && !$isSiteAdmin
),
array(
'text' => __('Import Server Settings'),
'url' => $baseurl . '/servers/import',
'requirement' => $this->Acl->canAccess('servers', 'import'),
),
array(
'text' => __('Remote Servers'),
'url' => $baseurl . '/servers/index',
@ -292,11 +287,6 @@
'url' => $baseurl . '/feeds/index',
'requirement' => $this->Acl->canAccess('feeds', 'index'),
),
array(
'text' => __('Search Feed Caches'),
'url' => $baseurl . '/feeds/searchCaches',
'requirement' => $this->Acl->canAccess('feeds', 'searchCaches'),
),
array(
'text' => __('SightingDB'),
'url' => $baseurl . '/sightingdb/index',
@ -313,7 +303,7 @@
'requirement' => $this->Acl->canAccess('cerebrates', 'index'),
),
array(
'text' => __('List Taxii Servers'),
'text' => __('TAXII Servers'),
'url' => $baseurl . '/TaxiiServers/index',
'requirement' => $this->Acl->canAccess('taxiiServers', 'index'),
),

8
app/View/Pages/administration.ctp
View File

@ -1,11 +1,3 @@
<?php
if (!$isSiteAdmin) exit();
?>
<div class="actions">
<ol class="nav nav-list">
</ol>
</div>
<div class="index">
<h2><?php echo __('Administrative actions');?></h2>
<ul>

4
app/View/Users/totp_new.ctp
View File

@ -1,7 +1,7 @@
<?php echo $this->Flash->render(); ?>
<?php
$detailsHtml = __("To enable TOTP for your account, scan the following QR code with your TOTP application and validate the token.");;
$secretHtml = __("Alternatively you can enter the following secret in your TOTP application: ") . "<pre>" . $secret . "</pre>";
$detailsHtml = __("To enable TOTP for your account, scan the following QR code with your TOTP application (for example Google authenticator or KeepassXC) and validate the token.");;
$secretHtml = __("Alternatively you can enter the following secret in your TOTP application. This can be particularly handy in case you don't have a supported application in your working environment. Once the verification is done you'll also get 50 \"paper-based\" login tokens so you don't have to use a TOTP application each time: ") . "<pre>" . $secret . "</pre>";
echo $this->element('/genericElements/Form/genericForm', array(
"form" => $this->Form,