mirror of https://github.com/MISP/MISP
Merge pull request #9431 from JakubOnderka/remove-ip-cidr
chg: [validation] Remove CIDR from /32 IPv4 and /128 IPv6 to normalizepull/9493/head
commit
fe0097e5c6
|
@ -53,10 +53,21 @@ class EventShell extends AppShell
|
|||
$parser->addSubcommand('mergeTags', [
|
||||
'help' => __('Merge tags'),
|
||||
'parser' => [
|
||||
'arguments' => array(
|
||||
'arguments' => [
|
||||
'source' => ['help' => __('Source tag ID or name. Source tag will be deleted.'), 'required' => true],
|
||||
'destination' => ['help' => __('Destination tag ID or name.'), 'required' => true],
|
||||
)
|
||||
],
|
||||
],
|
||||
]);
|
||||
$parser->addSubcommand('reportValidationIssuesAttributes', [
|
||||
'help' => __('Report validation issues on attributes'),
|
||||
]);
|
||||
$parser->addSubcommand('normalizeIpAddress', [
|
||||
'help' => __('Normalize IP address format in old events'),
|
||||
'parser' => [
|
||||
'options' => [
|
||||
'dry-run' => ['help' => __('Just show what changes will be made.'), 'boolean' => true],
|
||||
],
|
||||
],
|
||||
]);
|
||||
return $parser;
|
||||
|
@ -636,18 +647,28 @@ class EventShell extends AppShell
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param int $userId
|
||||
* @return array
|
||||
*/
|
||||
private function getUser($userId)
|
||||
public function reportValidationIssuesAttributes()
|
||||
{
|
||||
$user = $this->User->getAuthUser($userId, true);
|
||||
if (empty($user)) {
|
||||
$this->error("User with ID $userId does not exist.");
|
||||
foreach ($this->Event->Attribute->reportValidationIssuesAttributes() as $validationIssue) {
|
||||
echo $this->json($validationIssue) . "\n";
|
||||
}
|
||||
}
|
||||
|
||||
public function normalizeIpAddress()
|
||||
{
|
||||
$dryRun = $this->param('dry-run');
|
||||
|
||||
$count = 0;
|
||||
foreach ($this->Event->Attribute->normalizeIpAddress($dryRun) as $attribute) {
|
||||
$count++;
|
||||
echo JsonTool::encode($attribute) . "\n";
|
||||
}
|
||||
|
||||
if ($dryRun) {
|
||||
$this->err(__n("%s attribute to fix", "%s attributes to fix", $count, $count));
|
||||
} else {
|
||||
$this->err(__n("%s attribute fixed", "%s attributes fixed", $count, $count));
|
||||
}
|
||||
Configure::write('CurrentUserId', $user['id']); // for audit logging purposes
|
||||
return $user;
|
||||
}
|
||||
|
||||
public function generateTopCorrelations()
|
||||
|
@ -668,4 +689,18 @@ class EventShell extends AppShell
|
|||
$this->Job->save($job);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param int $userId
|
||||
* @return array
|
||||
*/
|
||||
private function getUser($userId)
|
||||
{
|
||||
$user = $this->User->getAuthUser($userId, true);
|
||||
if (empty($user)) {
|
||||
$this->error("User with ID $userId does not exist.");
|
||||
}
|
||||
Configure::write('CurrentUserId', $user['id']); // for audit logging purposes
|
||||
return $user;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1917,7 +1917,7 @@ class AttributesController extends AppController
|
|||
public function reportValidationIssuesAttributes($eventId = false)
|
||||
{
|
||||
// search for validation problems in the attributes
|
||||
$this->set('result', $this->Attribute->reportValidationIssuesAttributes($eventId));
|
||||
$this->set('result', iterator_to_array($this->Attribute->reportValidationIssuesAttributes($eventId)));
|
||||
}
|
||||
|
||||
public function generateCorrelation()
|
||||
|
|
|
@ -41,7 +41,7 @@ class AttributeValidationTool
|
|||
switch ($type) {
|
||||
case 'ip-src':
|
||||
case 'ip-dst':
|
||||
return self::compressIpv6($value);
|
||||
return self::normalizeIp($value);
|
||||
case 'md5':
|
||||
case 'sha1':
|
||||
case 'sha224':
|
||||
|
@ -98,7 +98,7 @@ class AttributeValidationTool
|
|||
$parts[0] = $punyCode;
|
||||
}
|
||||
}
|
||||
$parts[1] = self::compressIpv6($parts[1]);
|
||||
$parts[1] = self::normalizeIp($parts[1]);
|
||||
return "$parts[0]|$parts[1]";
|
||||
case 'filename|md5':
|
||||
case 'filename|sha1':
|
||||
|
@ -175,7 +175,7 @@ class AttributeValidationTool
|
|||
} else {
|
||||
return $value;
|
||||
}
|
||||
return self::compressIpv6($parts[0]) . '|' . $parts[1];
|
||||
return self::normalizeIp($parts[0]) . '|' . $parts[1];
|
||||
case 'mac-address':
|
||||
case 'mac-eui-64':
|
||||
$value = str_replace(array('.', ':', '-', ' '), '', strtolower($value));
|
||||
|
@ -700,11 +700,30 @@ class AttributeValidationTool
|
|||
* @param string $value
|
||||
* @return string
|
||||
*/
|
||||
private static function compressIpv6($value)
|
||||
private static function normalizeIp($value)
|
||||
{
|
||||
// If IP is a CIDR
|
||||
if (strpos($value, '/')) {
|
||||
list($ip, $range) = explode('/', $value, 2);
|
||||
|
||||
// Compress IPv6
|
||||
if (strpos($ip, ':') && $converted = inet_pton($ip)) {
|
||||
$ip = inet_ntop($converted);
|
||||
}
|
||||
|
||||
// If IP is in CIDR format, but the network is 32 for IPv4 or 128 for IPv6, normalize to non CIDR type
|
||||
if (($range === '32' && strpos($value, '.')) || ($range === '128' && strpos($value, ':'))) {
|
||||
return $ip;
|
||||
}
|
||||
|
||||
return "$ip/$range";
|
||||
}
|
||||
|
||||
// Compress IPv6
|
||||
if (strpos($value, ':') && $converted = inet_pton($value)) {
|
||||
return inet_ntop($converted);
|
||||
}
|
||||
|
||||
return $value;
|
||||
}
|
||||
|
||||
|
|
|
@ -1224,38 +1224,89 @@ class Attribute extends AppModel
|
|||
$this->Correlation->purgeCorrelations($eventId);
|
||||
}
|
||||
|
||||
public function reportValidationIssuesAttributes($eventId)
|
||||
/**
|
||||
* @param array $conditions
|
||||
* @return Generator|void
|
||||
*/
|
||||
private function fetchAttributesInChunks(array $conditions = [])
|
||||
{
|
||||
while (true) {
|
||||
$attributes = $this->find('all', [
|
||||
'recursive' => -1,
|
||||
'conditions' => $conditions,
|
||||
'limit' => 500,
|
||||
'order' => 'Attribute.id',
|
||||
]);
|
||||
if (empty($attributes)) {
|
||||
return;
|
||||
}
|
||||
foreach ($attributes as $attribute) {
|
||||
yield $attribute;
|
||||
}
|
||||
$count = count($attributes);
|
||||
$lastAttribute = $attributes[$count - 1];
|
||||
$conditions['Attribute.id >'] = $lastAttribute['Attribute']['id'];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param int|null $eventId
|
||||
* @return Generator
|
||||
*/
|
||||
public function reportValidationIssuesAttributes($eventId = null)
|
||||
{
|
||||
$conditions = array();
|
||||
if ($eventId && is_numeric($eventId)) {
|
||||
$conditions = array('event_id' => $eventId);
|
||||
}
|
||||
|
||||
$attributeIds = $this->find('column', array(
|
||||
'fields' => array('id'),
|
||||
'conditions' => $conditions
|
||||
));
|
||||
$chunks = array_chunk($attributeIds, 500);
|
||||
$attributes = $this->fetchAttributesInChunks($conditions);
|
||||
|
||||
$result = array();
|
||||
foreach ($chunks as $chunk) {
|
||||
$attributes = $this->find('all', array('recursive' => -1, 'conditions' => array('id' => $chunk)));
|
||||
foreach ($attributes as $attribute) {
|
||||
$this->set($attribute);
|
||||
if (!$this->validates()) {
|
||||
$resultErrors = array();
|
||||
foreach ($this->validationErrors as $field => $error) {
|
||||
$resultErrors[$field] = array('value' => $attribute['Attribute'][$field], 'error' => $error[0]);
|
||||
}
|
||||
$result[] = [
|
||||
'id' => $attribute['Attribute']['id'],
|
||||
'error' => $resultErrors,
|
||||
'details' => 'Event ID: [' . $attribute['Attribute']['event_id'] . "] - Category: [" . $attribute['Attribute']['category'] . "] - Type: [" . $attribute['Attribute']['type'] . "] - Value: [" . $attribute['Attribute']['value'] . ']',
|
||||
];
|
||||
foreach ($attributes as $attribute) {
|
||||
$this->set($attribute);
|
||||
if (!$this->validates()) {
|
||||
$resultErrors = [];
|
||||
foreach ($this->validationErrors as $field => $error) {
|
||||
$resultErrors[$field] = ['value' => $attribute['Attribute'][$field], 'error' => $error[0]];
|
||||
}
|
||||
yield [
|
||||
'id' => $attribute['Attribute']['id'],
|
||||
'error' => $resultErrors,
|
||||
'details' => 'Event ID: [' . $attribute['Attribute']['event_id'] . "] - Category: [" . $attribute['Attribute']['category'] . "] - Type: [" . $attribute['Attribute']['type'] . "] - Value: [" . $attribute['Attribute']['value'] . ']',
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param bool $dryRun If true, no changes will be made to
|
||||
* @return Generator
|
||||
* @throws Exception
|
||||
*/
|
||||
public function normalizeIpAddress($dryRun = false)
|
||||
{
|
||||
$attributes = $this->fetchAttributesInChunks([
|
||||
'Attribute.type' => ['ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'domain|ip'],
|
||||
]);
|
||||
|
||||
foreach ($attributes as $attribute) {
|
||||
$value = $attribute['Attribute']['value'];
|
||||
$normalizedValue = AttributeValidationTool::modifyBeforeValidation($attribute['Attribute']['type'], $value);
|
||||
if ($value !== $normalizedValue) {
|
||||
if (!$dryRun) {
|
||||
$attribute['Attribute']['value'] = $normalizedValue;
|
||||
$this->save($attribute, true, ['value1', 'value2']);
|
||||
}
|
||||
|
||||
yield [
|
||||
'id' => (int) $attribute['Attribute']['id'],
|
||||
'event_id' => (int) $attribute['Attribute']['event_id'],
|
||||
'type' => $attribute['Attribute']['type'],
|
||||
'value' => $value,
|
||||
'normalized_value' => $normalizedValue,
|
||||
];
|
||||
}
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -1610,6 +1661,7 @@ class Attribute extends AppModel
|
|||
* @param array $user
|
||||
* @param array $options
|
||||
* @param int|false $result_count If false, count is not fetched
|
||||
* @param bool $real_count
|
||||
* @return array
|
||||
* @throws Exception
|
||||
*/
|
||||
|
@ -3673,7 +3725,7 @@ class Attribute extends AppModel
|
|||
);
|
||||
}
|
||||
|
||||
private function findAttributeByValue($attribute)
|
||||
private function findAttributeByValue(array $attribute)
|
||||
{
|
||||
$type = $attribute['type'];
|
||||
$conditions = [
|
||||
|
|
|
@ -124,6 +124,16 @@ class AttributeValidationToolTest extends TestCase
|
|||
]);
|
||||
}
|
||||
|
||||
public function testRemoveCidrFromIp(): void
|
||||
{
|
||||
$this->assertEquals('127.0.0.1', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1/32'));
|
||||
$this->assertEquals('127.0.0.1/31', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1/31'));
|
||||
$this->assertEquals('example.com|1234:fd2:5621:1:89::4500', AttributeValidationTool::modifyBeforeValidation('domain|ip', 'example.com|1234:0fd2:5621:0001:0089:0000:0000:4500/128'));
|
||||
$this->assertEquals('1234:fd2:5621:1:89::4500|80', AttributeValidationTool::modifyBeforeValidation('ip-src|port', '1234:0fd2:5621:0001:0089:0000:0000:4500/128|80'));
|
||||
$this->assertEquals('1234:fd2:5621:1:89::4500/127|80', AttributeValidationTool::modifyBeforeValidation('ip-src|port', '1234:0fd2:5621:0001:0089:0000:0000:4500/127|80'));
|
||||
$this->assertEquals('127.0.0.1', AttributeValidationTool::modifyBeforeValidation('ip-src', '127.0.0.1'));
|
||||
}
|
||||
|
||||
public function testCompressIpv6(): void
|
||||
{
|
||||
$this->assertEquals('1234:fd2:5621:1:89::4500', AttributeValidationTool::modifyBeforeValidation('ip-src', '1234:0fd2:5621:0001:0089:0000:0000:4500'));
|
||||
|
|
|
@ -1,11 +1,3 @@
|
|||
<?php
|
||||
if (!$isSiteAdmin) exit();
|
||||
?>
|
||||
<div class="actions">
|
||||
<ol class="nav nav-list">
|
||||
|
||||
</ol>
|
||||
</div>
|
||||
<div class="index">
|
||||
<h2><?php echo __('Administrative actions');?></h2>
|
||||
<ul>
|
||||
|
|
Loading…
Reference in New Issue