Merge branch 'master' into 2.4-beta

Conflicts:
	VERSION.json
	app/Controller/Component/IOCImportComponent.php
pull/762/head
iglocska 2015-11-22 17:53:08 +01:00
commit ba8728abb9
3 changed files with 266 additions and 86 deletions

View File

@ -1,2 +1 @@
{"major":2, "minor":4, "hotfix":0}

View File

@ -1,19 +1,63 @@
<?php
class IOCImportComponent extends Component {
// predefined attribute pairs that should be saved together - these are the exceptions to AND operators that will not be omitted
// The format is: attribute1, attribute2, new type, category, behaviour.
// Behaviour can be one of the following: 'first', 'second', 'both' -> this explains what the attribute that is to be created should carry as a value
// first means that the value will be equal to the first attribute, second means that it will equal that of the second. Both will add both separated by '|'
private $attributePairs = array(
array('filename', 'md5', 'filename|md5', 'Payload installation', 'both', '|'),
array('filename', 'sha1', 'filename|sha1', 'Payload installation', 'both', '|'),
array('filename', 'sha256', 'filename|sha256', 'Payload installation', 'both', '|'),
array('regkey', 'tempRegValue', 'regkey|value', 'Persistence mechanism', 'both', '|'),
array('filename', 'tempCertificateSubject', 'filename', 'Payload installation', 'first', ''),
array('filename', 'tempExtension', 'filename', 'Payload installation', 'both', '.'),
);
// rework of handling composite attributes
// each entry is only triggered if an "AND" branch contains the exact list of parsed attribute types that are set in the components field
// Note that some indicators are not immediately discarded by the parser even if they cannot be turned into an attribute
// The reason for this is that they might not be convertable to an attribute, but they might be converted into a part of a composite attribute
// For example: There is no Registry value type in MISP, but there is a regkey|value type. Meaning that a Registry value can be turned into an attribute
// as long as a registry key is "AND"ed with the value in the OpenIOC file.
// notes about the format for the composition:
// - components have to be in alphabetical order
// - returnFormat has to be a valid MISP type
// - returnCategory has to be a valid MISP category
// - replace: passed attribute values will replace the $[component position] substring to form the final attribute value
private $attributeComposition = array(
array(
'components' => array('filename', 'md5'),
'returnFormat' => 'filename|md5',
'returnCategory' => 'Payload installation',
'replace'=> '$0|$1'
),
array(
'components' => array('filename', 'sha1'),
'returnFormat' => 'filename|sha1',
'returnCategory' => 'Payload installation',
'replace'=> '$0|$1'
),
array(
'components' => array('filename', 'sha256'),
'returnFormat' => 'filename|sha256',
'returnCategory' => 'Payload installation',
'replace'=> '$0|$1'
),
array(
'components' => array('regkey', 'tempRegValue'),
'returnFormat' => 'regkey|value',
'returnCategory' => 'Persistence mechanism',
'replace'=> '$0|$1'
),
array(
'components' => array('filename', 'tempCertificateSubject'),
'returnFormat' => 'filename',
'returnCategory' => 'Payload installation',
'replace'=> '$0'
),
array(
'components' => array('filename', 'tempExtension'),
'returnFormat' => 'filename',
'returnCategory' => 'Payload installation',
'replace'=> '$0.$1'
),
array(
'components' => array('regkey', 'tempRegName', 'tempRegValue'),
'returnFormat' => 'regkey|value',
'returnCategory' => 'Persistence mechanism',
'replace'=> '$0$1|$2'
),
);
// Indicators that we can safely remove if they pop up within an AND branch
private $discardableIndicators = array(
@ -83,6 +127,7 @@ class IOCImportComponent extends Component {
private $filename = "";
public function readXML($data, $id, $dist, $filename) {
$this->Attribute = ClassRegistry::init('Attribute');
$this->filename = $filename;
$event = array();
$attributes = array();
@ -133,11 +178,12 @@ class IOCImportComponent extends Component {
$duplicateFilter = array();
// check if we have any attributes, if yes, add their UUIDs to our list of success-array
if (count ($event['Attribute']) > 0) {
foreach ($event['Attribute'] as $k => $attribute) {
foreach ($event['Attribute'] as $k => &$attribute) {
$condensed = strtolower($attribute['value']) . $attribute['category'] . $attribute['type'];
if (!in_array($condensed, $duplicateFilter)) {
$this->saved_uuids[] = $attribute['uuid'];
$duplicateFilter[] = $condensed;
$attribute['uuid'] = $this->Attribute->generateUuid();
} else unset($event['Attribute'][$k]);
}
}
@ -148,7 +194,6 @@ class IOCImportComponent extends Component {
// Add a special attribute that captures the basic data about the .ioc such as the ioc-s uuid, info, long info, author, etc.
// Define the fields used in the global iocinfo variable.
$this->Attribute = ClassRegistry::init('Attribute');
foreach ($this->iocinfo as $k => $v) {
if (isset($event[$v])) $event['Attribute'][] = array('uuid' => $this->Attribute->generateUuid(), 'category' => 'Other', 'type' => 'comment', 'event_id' => $id, 'value' => $v . ': ' . $event[$v], 'to_ids' => $this->typeToIdsSettings['comment'], 'distribution' => $this->distribution, 'comment' => 'OpenIOC import from file ' . $filename);
}
@ -208,7 +253,7 @@ class IOCImportComponent extends Component {
if ($temp[0] == 'Other' && $temp[1] == 'other') {
$attribute['value'] = $attribute['search'] . ': ' . $attribute['value'];
}
$attribute['comment'] = 'OpenIOC import from file ' . $this->filename;
$attribute['comment'] = 'OpenIOC import from file ' . $this->filename . PHP_EOL . 'Original UUID: ' . $attribute['uuid'];
return $attribute;
}
@ -318,6 +363,8 @@ class IOCImportComponent extends Component {
case 'RegistryItem/Path':
return array('Persistence mechanism', 'regkey', true);
break;
case 'RegistryItem/ValueName':
return array('Persistence mechanism', 'tempRegName', false);
case 'Snort/Snort':
return array('Network activity', 'snort', true);
break;
@ -333,6 +380,7 @@ class IOCImportComponent extends Component {
return array('Network activity', 'hostname', true);
break;
case 'RegistryItem/Text':
case 'RegistryItem/Value':
return array('Persistence mechanism', 'tempRegValue', false);
break;
// We don't keep the following, they are often used with AND and a filename. We'll only keep the filename in those cases.
@ -365,14 +413,9 @@ class IOCImportComponent extends Component {
private function __resolveBranch($branch, $uuid, $type, &$leaves, $root = false) {
$toBeOmitted = $branch;
$toReindex = false;
// Resolve any deeper branching before we attempt to resolve this, as we might be able to turn it into a single attribute
foreach ($branch['branches'] as $key => $value) {
$r = $this->__resolveBranch($value, $branch['uuid'], $branch['type'], $branch['leaves']);
// If one of the child branch became empty, mark the whole branch for reindexing
if ($r == null) {
$toReindex = true;
}
if ($r === 'getFromTemp') {
unset ($branch['branches'][$key]);
foreach ($this->tempLeaves as $tempLeaf) {
@ -446,10 +489,9 @@ class IOCImportComponent extends Component {
$this->saved_uuids[] = $branch['uuid'];
}
}
if (isset($branch['leaves']) && count($branch['leaves']) == 1 && count($branch['branches']) == 0) {
$leaves[] = $branch['leaves'][0];
$branch['leaves'] = array();
$branch['leaves'] = $leaves;
}
if (($branch['type'] == 'OR') && count($branch['branches']) == 0 && count($branch['leaves']) != 0) {
@ -459,7 +501,6 @@ class IOCImportComponent extends Component {
return 'getFromTemp';
}
}
// If we have no branches and no leaves left after all of this, return nothing and unset this branch
if ((!isset($branch['leaves']) || count($branch['leaves']) == 0) && count($branch['branches']) == 0 && !isset($branch['long_info'])) {
return;
@ -504,30 +545,21 @@ class IOCImportComponent extends Component {
private function __resolveAndBranch($array, $id) {
// Let's see how many indicators we have left and take action accordingly
switch (count($array)) {
case 0 :
// We are left with 0 eligible indicators, just return nothing
return;
break;
case 1 :
// If the eliminations have gotten us down to just one indicator left, we can return it - Unless it's of type other, we don't want to keep that and interfere with further resolution
if ($array[0]['category'] == 'Other' && $array[0]['type'] == 'other') return;
$count = count($array);
if ($count == 0) return;
else if ($count == 1) {
return $array[0];
} else {
for ($i=0; $i < $count; $i++) $att[$i] = $this->__analyseIndicator($array[$i], $id);
$attempt = $this->__convertToCompositeAttribute($att, $id);
if ($attempt) {
$attempt['uuid'] = $att[0]['uuid'];
$this->saved_uuids[] = $id;
return $array[0];
break;
case 2 :
// We are left with more than one indicator, check whether they can be turned into a single attribute, if it's not possible, return false
$att1 = $this->__analyseIndicator($array[0], $id);
$att2 = $this->__analyseIndicator($array[1], $id);
$attempt = $this->__convertToCompositeAttribute($att1, $att2, $id);
if ($attempt) {
$attempt['to_ids'] = $this->typeToIdsSettings[$attempt['type']];
$this->saved_uuids[] = $id;
return $attempt;
}
break;
foreach ($att as &$temp) $this->saved_uuids[] = $temp['uuid'];
return $attempt;
}
return;
}
return;
}
// We have a list of attributes that we can omit in nested logical branches - the idea is to always make sure that we don't insert attributes
@ -543,46 +575,34 @@ class IOCImportComponent extends Component {
}
// Attempt to convert the two attributes retrieved from an AND indicator into a single attribute, if they are eligible to be converted. If not, add it to the array of failures.
private function __convertToCompositeAttribute($att1, $att2, $uuid) {
$this->Attribute = ClassRegistry::init('Attribute');
private function __convertToCompositeAttribute($att, $uuid) {
// check if the current attribute is one of the known pairs saved in the array $attributePairs
foreach ($this->attributePairs as $pair) {
// if attribute 1's type = the first type of the pair and attribute 2's type is the type of the second attribute of the pair, return a new joint attribute with the new type-name (usually type1|type2) and its predefined category
if($att1['type'] == $pair[0] && $att2['type'] == $pair[1]) {
$this->saved_uuids[] = $uuid;
$this->saved_uuids[] = $att1['uuid'];
$this->saved_uuids[] = $att2['uuid'];
if ($pair[4] == 'both') $value = $att1['value'] . $pair[5] . $att2['value'];
// switch to see which value to keep and which to get rid of
switch ($pair[4]) {
case 'first':
$value = $att1['value'];
break;
case 'second':
$value = $att2['value'];
break;
default:
$value = $att1['value'] . '|' . $att2['value'];
$componentCount = count($att);
$tempArray = $values = $uuids = array();
foreach ($att as &$temp) $tempArray[$temp['type']] = $temp;
ksort($tempArray);
$keys = array_keys($tempArray);
$att = array_values($tempArray);
foreach ($att as &$temp) {
$values[] = $temp['value'];
$uuids[] = $temp['uuid'];
}
foreach ($this->attributeComposition as $composition) {
if (count($composition['components']) != count($att)) continue;
if ($keys === $composition['components']) {
$value = $composition['replace'];
foreach ($values as $k => &$v) {
$value = str_replace('$' . $k, $v, $value);
}
return array('type' => $pair[2], 'value' => $value, 'uuid' => $this->Attribute->generateUuid(), 'category' => $pair[3], 'to_ids' => $this->typeToIdsSettings[$pair[2]], 'distribution' => $this->distribution);
}
// Try the same thing above with the attributes reversed
if ($att2['type'] == $pair[0] && $att1['type'] == $pair[1]) {
$this->saved_uuids[] = $uuid;
$this->saved_uuids[] = $att1['uuid'];
$this->saved_uuids[] = $att2['uuid'];
// switch to see which value to keep and which to get rid of
switch ($pair[4]) {
case 'first':
$value = $att2['value'];
break;
case 'second':
$value = $att1['value'];
break;
default:
$value = $att2['value'] . '|' . $att1['value'];
}
return array('type' => $pair[2], 'value' => $value, 'uuid' => $this->Attribute->generateUuid(), 'category' => $pair[3], 'to_ids' => $this->typeToIdsSettings[$pair[2]], 'distribution' => $this->distribution);
return array(
'type' => $composition['returnFormat'],
'category' => $composition['returnCategory'],
'value' => $value,
'to_ids' => $this->typeToIdsSettings[$composition['returnFormat']],
'distribution' => $this->distribution,
'comment' => 'OpenIOC import from file ' . $this->filename . ' - Original UUIDs:' . PHP_EOL . implode(PHP_EOL, $uuids),
);
}
}
// If no match found, return false, it's not a valid composite attribute for MISP
@ -592,7 +612,7 @@ class IOCImportComponent extends Component {
// used to save the value of attributes of type other (attributes that could not be mapped) and convert temporary attributes to type other.
private function __convertToOther(&$attribute) {
$attribute['category'] = 'Other';
$attribute['type'] = 'other';
$attribute['type'] = 'comment';
$attribute['value'] = $attribute['search'] . ': ' . $attribute['value'];
}
}

161
format/2.3/schema.json Normal file
View File

@ -0,0 +1,161 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json",
"type": "object",
"properties": {
"Event": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event",
"type": "object",
"properties": {
"id": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/id",
"type": "string"
},
"org": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/org",
"type": "string"
},
"date": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/date",
"type": "string"
},
"threat_level_id": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/threat_level_id",
"type": "string"
},
"info": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/info",
"type": "string"
},
"published": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/published",
"type": "boolean"
},
"uuid": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/uuid",
"type": "string"
},
"attribute_count": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/attribute_count",
"type": "string"
},
"analysis": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/analysis",
"type": "string"
},
"timestamp": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/timestamp",
"type": "string"
},
"distribution": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/distribution",
"type": "string"
},
"proposal_email_lock": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/proposal_email_lock",
"type": "boolean"
},
"orgc": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/orgc",
"type": "string"
},
"locked": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/locked",
"type": "boolean"
},
"publish_timestamp": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/publish_timestamp",
"type": "string"
},
"Attribute": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/Attribute",
"type": "array",
"items": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/Attribute/0",
"type": "object",
"properties": {
"id": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/Attribute/0/id",
"type": "string"
},
"type": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/Attribute/0/type",
"type": "string"
},
"category": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/Attribute/0/category",
"type": "string"
},
"to_ids": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/Attribute/0/to_ids",
"type": "boolean"
},
"uuid": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/Attribute/0/uuid",
"type": "string"
},
"event_id": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/Attribute/0/event_id",
"type": "string"
},
"distribution": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/Attribute/0/distribution",
"type": "string"
},
"timestamp": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/Attribute/0/timestamp",
"type": "string"
},
"comment": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/Attribute/0/comment",
"type": "string"
},
"value": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/Attribute/0/value",
"type": "string"
},
"ShadowAttribute": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/Attribute/0/ShadowAttribute",
"type": "array",
"items": []
}
}
}
},
"ShadowAttribute": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/ShadowAttribute",
"type": "array",
"items": []
},
"RelatedEvent": {
"id": "https://www.github.com/MISP/MISP/format/2.3/schema.json/Event/RelatedEvent",
"type": "array",
"items": []
}
},
"required": [
"id",
"org",
"date",
"threat_level_id",
"info",
"published",
"uuid",
"attribute_count",
"analysis",
"timestamp",
"distribution",
"proposal_email_lock",
"orgc",
"locked",
"publish_timestamp",
"Attribute",
"ShadowAttribute",
"RelatedEvent"
]
}
},
"required": [
"Event"
]
}