add: api tests for freetext feeds

pull/9489/head
Luciano Righetti 2024-01-18 15:38:52 +01:00
parent e72f664319
commit 26720ddd8d
11 changed files with 636 additions and 344 deletions

View File

@ -0,0 +1,97 @@
<?php
namespace App\Command;
use App\Model\Entity\Server;
use Cake\Command\Command;
use Cake\Console\Arguments;
use Cake\Console\ConsoleIo;
use Cake\ORM\Locator\LocatorAwareTrait;
use Cake\Core\Configure;
use App\Model\Entity\Job;
class ServerCommand extends Command
{
use LocatorAwareTrait;
/** @var ConsoleIo */
private $io;
public function execute(Arguments $args, ConsoleIo $io)
{
$this->io = $io;
$arguments = $args->getArguments();
$action = array_shift($arguments);
switch ($action) {
case 'fetchFeed':
$this->fetchFeed(...$arguments);
break;
default:
$this->io->err('Invalid action.');
}
$this->io->out("Bye.");
}
public function fetchFeed($userId, $feedId, $jobId = null)
{
if (empty($userId) || empty($feedId)) {
$this->io->err('Usage: ' . (new Server())->command_line_functions['console_automation_tasks']['data']['Fetch feeds as local data'] . PHP_EOL);
die();
}
$UsersTable = $this->fetchTable('Users');
$user = $UsersTable->getAuthUser($userId, true);
Configure::write('CurrentUserId', $userId);
$FeedsTable = $this->fetchTable('Feeds');
$JobsTable = $this->fetchTable('Jobs');
if (!empty($jobId)) {
$jobId = $this->args[2];
} else {
$jobId = $JobsTable->createJob($user->toArray(), Job::WORKER_DEFAULT, 'fetch_feeds', 'Feed: ' . $feedId, 'Starting fetch from Feed.');
}
if ($feedId === 'all') {
$feedIds = $FeedsTable->find('column', array(
'fields' => array('id'),
'conditions' => array('enabled' => 1)
))->toArray();
$successes = 0;
$fails = 0;
foreach ($feedIds as $k => $feedId) {
$JobsTable->saveProgress($jobId, 'Fetching feed: ' . $feedId, 100 * $k / count($feedIds));
$result = $FeedsTable->downloadFromFeedInitiator($feedId, $user);
if ($result) {
$successes++;
} else {
$fails++;
}
}
$message = 'Job done. ' . $successes . ' feeds pulled successfully, ' . $fails . ' feeds could not be pulled.';
$JobsTable->saveStatus($jobId, true, $message);
$this->io->out($message);
} else {
$feedEnabled = $FeedsTable->exists([
'enabled' => 1,
'id' => $feedId,
]);
if ($feedEnabled) {
$result = $FeedsTable->downloadFromFeedInitiator($feedId, $user, $jobId);
if (!$result) {
$JobsTable->saveStatus($jobId, false, 'Job failed. See error log for more details.');
$this->io->err('Job failed.');
} else {
$JobsTable->saveStatus($jobId, true);
$this->io->out('Job done.');
}
} else {
$message = "Feed with ID $feedId not found or not enabled.";
$JobsTable->saveStatus($jobId, false, $message);
$this->io->err($message);
}
}
}
}

View File

@ -511,7 +511,7 @@ class FeedsController extends AppController
$this->redirect(['action' => 'index']);
}
}
if (Configure::read('MISP.BackgroundJobs')) {
if (Configure::read('BackgroundJobs.enabled')) {
/** @var JobsTable $JobsTable */
$JobsTable = $this->fetchTable('Jobs');
@ -528,7 +528,7 @@ class FeedsController extends AppController
BackgroundJobsTool::CMD_SERVER,
[
'fetchFeed',
$this->Auth->user('id'),
$this->ACL->getUser()->id,
$feedId,
$jobId
],
@ -581,7 +581,7 @@ class FeedsController extends AppController
if (!$this->Feeds->data['Feed']['enabled']) {
continue;
}
if (Configure::read('MISP.BackgroundJobs')) {
if (Configure::read('BackgroundJobs.enabled')) {
/** @var Job $job */
$JobsTable = $this->fetchTable('Jobs');
@ -958,7 +958,7 @@ class FeedsController extends AppController
public function cacheFeeds($scope = 'freetext')
{
if (Configure::read('MISP.BackgroundJobs')) {
if (Configure::read('BackgroundJobs.enabled')) {
/** @var JobsTable $JobsTable */
$JobsTable = $this->fetchTable('Jobs');

View File

@ -6,33 +6,33 @@ use Exception;
class ComplexTypeTool
{
const REFANG_REGEX_TABLE = array(
array(
const REFANG_REGEX_TABLE = [
[
'from' => '/^(hxxp|hxtp|htxp|meow|h\[tt\]p)/i',
'to' => 'http',
'types' => array('link', 'url')
),
array(
'types' => ['link', 'url']
],
[
'from' => '/(\[\.\]|\[dot\]|\(dot\))/',
'to' => '.',
'types' => array('link', 'url', 'ip-dst', 'ip-src', 'domain|ip', 'domain', 'hostname')
),
array(
'types' => ['link', 'url', 'ip-dst', 'ip-src', 'domain|ip', 'domain', 'hostname']
],
[
'from' => '/\[hxxp:\/\/\]/',
'to' => 'http://',
'types' => array('link', 'url')
),
array(
'types' => ['link', 'url']
],
[
'from' => '/[\@]|\[at\]/',
'to' => '@',
'types' => array('email', 'email-src', 'email-dst')
),
array(
'types' => ['email', 'email-src', 'email-dst']
],
[
'from' => '/\[:\]/',
'to' => ':',
'types' => array('url', 'link')
)
);
'types' => ['url', 'link']
]
];
const HEX_HASH_TYPES = [
32 => ['single' => ['md5', 'imphash', 'x509-fingerprint-md5', 'ja3-fingerprint-md5'], 'composite' => ['filename|md5', 'filename|imphash']],
@ -61,7 +61,7 @@ class ComplexTypeTool
return $value;
}
public function setTLDs($tlds = array())
public function setTLDs($tlds = [])
{
$this->__tlds = [];
foreach ($tlds as $tld) {
@ -77,7 +77,7 @@ class ComplexTypeTool
$this->securityVendorDomains = $securityVendorDomains;
}
public function checkComplexRouter($input, $type, $settings = array())
public function checkComplexRouter($input, $type, $settings = [])
{
switch ($type) {
case 'File':
@ -125,31 +125,31 @@ class ComplexTypeTool
if ($type == '') {
$type = 'other';
}
return array('type' => $type, 'value' => $original);
return ['type' => $type, 'value' => $original];
}
public function checkComplexCnC($input)
{
$toReturn = array();
$toReturn = [];
// check if it's an IP address
if (filter_var($input, FILTER_VALIDATE_IP)) {
return array('type' => 'ip-dst', 'value' => $input);
return ['type' => 'ip-dst', 'value' => $input];
}
if (preg_match("#^[A-Z0-9.-]+\.[A-Z]{2,4}$#i", $input)) {
$result = explode('.', $input);
if (count($result) > 2) {
$toReturn['multi'][] = array('type' => 'hostname', 'value' => $input);
$toReturn['multi'][] = ['type' => 'hostname', 'value' => $input];
$pos = strpos($input, '.');
$toReturn['multi'][] = array('type' => 'domain', 'value' => substr($input, (1 + $pos)));
$toReturn['multi'][] = ['type' => 'domain', 'value' => substr($input, (1 + $pos))];
return $toReturn;
}
return array('type' => 'domain', 'value' => $input);
return ['type' => 'domain', 'value' => $input];
}
if (!preg_match("#\n#", $input)) {
return array('type' => 'url', 'value' => $input);
return ['type' => 'url', 'value' => $input];
}
return array('type' => 'other', 'value' => $input);
return ['type' => 'other', 'value' => $input];
}
/**
@ -164,7 +164,7 @@ class ComplexTypeTool
* @return array
* @throws Exception
*/
public function checkCSV($input, $settings = array())
public function checkCSV($input, $settings = [])
{
if (empty($input)) {
return [];
@ -174,7 +174,7 @@ class ComplexTypeTool
if ($delimiter === '\t') {
$delimiter = "\t";
}
$values = !empty($settings['value']) ? $settings['value'] : array();
$values = !empty($settings['value']) ? $settings['value'] : [];
if (!is_array($values)) {
$values = explode(',', $values);
}
@ -223,6 +223,10 @@ class ComplexTypeTool
*/
public function checkFreeText($input, array $settings = [])
{
if (is_resource($input)) {
$input = stream_get_contents($input);
}
if (empty($input)) {
return [];
}
@ -269,18 +273,21 @@ class ComplexTypeTool
$parsed = JsonTool::decode($input);
$values = [];
array_walk_recursive($parsed, function ($value) use (&$values) {
if (is_bool($value) || is_int($value) || empty($value)) {
return; // skip boolean, integer or empty values
}
array_walk_recursive(
$parsed,
function ($value) use (&$values) {
if (is_bool($value) || is_int($value) || empty($value)) {
return; // skip boolean, integer or empty values
}
$values[] = $value;
foreach ($this->parseFreetext($value) as $v) {
if ($v !== $value) {
$values[] = $v;
$values[] = $value;
foreach ($this->parseFreetext($value) as $v) {
if ($v !== $value) {
$values[] = $v;
}
}
}
});
);
unset($parsed);
$resultArray = [];
@ -391,7 +398,7 @@ class ComplexTypeTool
if (strpos($input['refanged'], '@') !== false) {
if (filter_var($input['refanged'], FILTER_VALIDATE_EMAIL)) {
return [
'types' => array('email', 'email-src', 'email-dst', 'target-email', 'whois-registrant-email'),
'types' => ['email', 'email-src', 'email-dst', 'target-email', 'whois-registrant-email'],
'default_type' => 'email-src',
'value' => $input['refanged'],
];
@ -404,7 +411,7 @@ class ComplexTypeTool
{
if (preg_match('#^as[0-9]+$#i', $input['raw'])) {
$input['raw'] = strtoupper($input['raw']);
return array('types' => array('AS'), 'default_type' => 'AS', 'value' => $input['raw']);
return ['types' => ['AS'], 'default_type' => 'AS', 'value' => $input['raw']];
}
return false;
}
@ -418,10 +425,10 @@ class ComplexTypeTool
if ($this->__resolveFilename($compositeParts[0])) {
$hash = $this->__resolveHash($compositeParts[1]);
if ($hash) {
return array('types' => $hash['composite'], 'default_type' => $hash['composite'][0], 'value' => $input['raw']);
return ['types' => $hash['composite'], 'default_type' => $hash['composite'][0], 'value' => $input['raw']];
}
if ($this->__resolveSsdeep($compositeParts[1])) {
return array('types' => array('filename|ssdeep'), 'default_type' => 'filename|ssdeep', 'value' => $input['raw']);
return ['types' => ['filename|ssdeep'], 'default_type' => 'filename|ssdeep', 'value' => $input['raw']];
}
}
}
@ -434,11 +441,11 @@ class ComplexTypeTool
if ($this->__checkForBTC($input)) {
$types[] = 'btc';
}
return array('types' => $types, 'default_type' => $types[0], 'value' => $input['raw']);
return ['types' => $types, 'default_type' => $types[0], 'value' => $input['raw']];
}
// ssdeep has a different pattern
if ($this->__resolveSsdeep($input['raw'])) {
return array('types' => array('ssdeep'), 'default_type' => 'ssdeep', 'value' => $input['raw']);
return ['types' => ['ssdeep'], 'default_type' => 'ssdeep', 'value' => $input['raw']];
}
return false;
}
@ -489,7 +496,7 @@ class ComplexTypeTool
// Phone numbers - for automatic recognition, needs to start with + or include dashes
if ($input['raw'][0] === '+' || strpos($input['raw'], '-')) {
if (!preg_match('#^[0-9]{4}-[0-9]{2}-[0-9]{2}$#i', $input['raw']) && preg_match("#^(\+)?([0-9]{1,3}(\(0\))?)?[0-9\/\-]{5,}[0-9]$#i", $input['raw'])) {
return array('types' => array('phone-number', 'prtn', 'whois-registrant-phone'), 'default_type' => 'phone-number', 'value' => $input['raw']);
return ['types' => ['phone-number', 'prtn', 'whois-registrant-phone'], 'default_type' => 'phone-number', 'value' => $input['raw']];
}
}
return false;
@ -499,9 +506,9 @@ class ComplexTypeTool
{
if (filter_var($input['refanged_no_port'], FILTER_VALIDATE_IP)) {
if (isset($input['port'])) {
return array('types' => array('ip-dst|port', 'ip-src|port', 'ip-src|port/ip-dst|port'), 'default_type' => 'ip-dst|port', 'comment' => $input['comment'], 'value' => $input['refanged_no_port'] . '|' . $input['port']);
return ['types' => ['ip-dst|port', 'ip-src|port', 'ip-src|port/ip-dst|port'], 'default_type' => 'ip-dst|port', 'comment' => $input['comment'], 'value' => $input['refanged_no_port'] . '|' . $input['port']];
} else {
return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
return ['types' => ['ip-dst', 'ip-src', 'ip-src/ip-dst'], 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']];
}
}
// IPv6 address that is considered as IP address with port
@ -532,7 +539,7 @@ class ComplexTypeTool
if (strpos($input['refanged_no_port'], '/')) {
$temp = explode('/', $input['refanged_no_port']);
if (count($temp) === 2 && filter_var($temp[0], FILTER_VALIDATE_IP) && is_numeric($temp[1])) {
return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
return ['types' => ['ip-dst', 'ip-src', 'ip-src/ip-dst'], 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']];
}
}
return false;
@ -552,9 +559,9 @@ class ComplexTypeTool
}
if ($domainDetection) {
if (count($temp) > 2) {
return array('types' => array('hostname', 'domain', 'url', 'filename'), 'default_type' => 'hostname', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
return ['types' => ['hostname', 'domain', 'url', 'filename'], 'default_type' => 'hostname', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']];
} else {
return array('types' => array('domain', 'filename'), 'default_type' => 'domain', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
return ['types' => ['domain', 'filename'], 'default_type' => 'domain', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']];
}
} else {
// check if it is a URL
@ -562,14 +569,14 @@ class ComplexTypeTool
if (count($temp) > 1 && (filter_var($input['refanged_no_port'], FILTER_VALIDATE_URL) || filter_var('http://' . $input['refanged_no_port'], FILTER_VALIDATE_URL))) {
// Even though some domains are valid, we want to exclude them as they are known security vendors / etc
if ($this->isLink($input['refanged_no_port'])) {
return array('types' => array('link'), 'default_type' => 'link', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
return ['types' => ['link'], 'default_type' => 'link', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']];
}
if (strpos($input['refanged_no_port'], '/')) {
return array('types' => array('url'), 'default_type' => 'url', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
return ['types' => ['url'], 'default_type' => 'url', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']];
}
}
if ($this->__resolveFilename($input['raw'])) {
return array('types' => array('filename'), 'default_type' => 'filename', 'value' => $input['raw']);
return ['types' => ['filename'], 'default_type' => 'filename', 'value' => $input['raw']];
}
}
}
@ -577,10 +584,10 @@ class ComplexTypeTool
$temp = explode('\\', $input['raw']);
if (strpos(end($temp), '.') || preg_match('/^.:/i', $temp[0])) {
if ($this->__resolveFilename(end($temp))) {
return array('types' => array('filename'), 'default_type' => 'filename', 'value' => $input['raw']);
return ['types' => ['filename'], 'default_type' => 'filename', 'value' => $input['raw']];
}
} else if (!empty($temp[0])) {
return array('types' => array('regkey'), 'default_type' => 'regkey', 'value' => $input['raw']);
return ['types' => ['regkey'], 'default_type' => 'regkey', 'value' => $input['raw']];
}
}
return false;
@ -656,7 +663,7 @@ class ComplexTypeTool
private function __generateTLDList()
{
$tlds = array('biz', 'cat', 'com', 'edu', 'gov', 'int', 'mil', 'net', 'org', 'pro', 'tel', 'aero', 'arpa', 'asia', 'coop', 'info', 'jobs', 'mobi', 'name', 'museum', 'travel', 'onion');
$tlds = ['biz', 'cat', 'com', 'edu', 'gov', 'int', 'mil', 'net', 'org', 'pro', 'tel', 'aero', 'arpa', 'asia', 'coop', 'info', 'jobs', 'mobi', 'name', 'museum', 'travel', 'onion'];
$char1 = $char2 = 'a';
for ($i = 0; $i < 26; $i++) {
for ($j = 0; $j < 26; $j++) {

View File

@ -2,9 +2,11 @@
namespace App\Model\Table;
use App\Lib\Tools\BackgroundJobsTool;
use App\Lib\Tools\FileAccessTool;
use App\Lib\Tools\GitTool;
use Cake\Collection\CollectionInterface;
use Cake\Core\Configure;
use Cake\Database\Expression\QueryExpression;
use Cake\Http\Exception\MethodNotAllowedException;
use Cake\I18n\FrozenTime;
@ -23,6 +25,9 @@ class AppTable extends Table
/** @var LogsTable */
public $Log = null;
/** @var BackgroundJobsTool */
private static $loadedBackgroundJobsTool;
public function initialize(array $config): void
{
}
@ -340,4 +345,17 @@ class AppTable extends Table
}
return $commit;
}
/**
* @return BackgroundJobsTool
*/
public function getBackgroundJobsTool(): BackgroundJobsTool
{
if (!self::$loadedBackgroundJobsTool) {
self::$loadedBackgroundJobsTool = new BackgroundJobsTool(Configure::read('BackgroundJobs'));
;
}
return self::$loadedBackgroundJobsTool;
}
}

View File

@ -3,12 +3,76 @@
namespace App\Model\Table;
use App\Model\Table\AppTable;
use ArrayObject;
use Cake\Datasource\EntityInterface;
use Cake\Event\EventInterface;
use Cake\Http\Exception\InternalErrorException;
use Cake\Utility\Text;
class AttributesTable extends AppTable
{
public $_typeDefinitions = null;
public $_categoryDefinitions = null;
public function initialize(array $config): void
{
parent::initialize($config);
$this->belongsTo(
'Events',
[
'propertyName' => 'Event'
]
);
}
public function beforeSave(EventInterface $event, EntityInterface $attribute, ArrayObject $options)
{
// TODO: [3.x-MIGRATION] this is a simplified version of the old beforeSave, fix it when moving Attributes to 3.x see #9383
$attribute->sharing_group_id ??= $attribute->Events->sharing_group_id ?? 0;
if ($attribute->uuid === null) {
$attribute->uuid = Text::uuid();
}
$attribute->event_id ??= $attribute->Events->id;
if (!empty($attribute['type'])) {
// explode composite types in value1 and value2
if (in_array($attribute['type'], $this->getCompositeTypes(), true)) {
$pieces = explode('|', $attribute['value']);
if (2 !== count($pieces)) {
throw new InternalErrorException(__('Composite type, but value not explodable'));
}
$attribute['value1'] = $pieces[0];
$attribute['value2'] = $pieces[1];
} else {
$attribute['value1'] = $attribute['value'];
$attribute['value2'] = '';
}
}
return true;
}
public function getCompositeTypes()
{
static $compositeTypes;
if ($compositeTypes === null) {
// build the list of composite Attribute.type dynamically by checking if type contains a |
// default composite types
$compositeTypes = ['malware-sample']; // TODO hardcoded composite
// dynamically generated list
foreach ($this->typeDefinitions as $type => $foo) {
if (strpos($type, '|') !== false) {
$compositeTypes[] = $type;
}
}
}
return $compositeTypes;
}
public function __get($name)
{
if ($name === 'typeDefinitions') {
@ -28,81 +92,81 @@ class AttributesTable extends AppTable
*/
private function generateCategoryDefinitions()
{
return array(
'Internal reference' => array(
return [
'Internal reference' => [
'desc' => __('Reference used by the publishing party (e.g. ticket number)'),
'types' => array('text', 'link', 'comment', 'other', 'hex', 'anonymised', 'git-commit-id')
),
'Targeting data' => array(
'types' => ['text', 'link', 'comment', 'other', 'hex', 'anonymised', 'git-commit-id']
],
'Targeting data' => [
'desc' => __('Internal Attack Targeting and Compromise Information'),
'formdesc' => __('Targeting information to include recipient email, infected machines, department, and or locations.'),
'types' => array('target-user', 'target-email', 'target-machine', 'target-org', 'target-location', 'target-external', 'comment', 'anonymised')
),
'Antivirus detection' => array(
'types' => ['target-user', 'target-email', 'target-machine', 'target-org', 'target-location', 'target-external', 'comment', 'anonymised']
],
'Antivirus detection' => [
'desc' => __('All the info about how the malware is detected by the antivirus products'),
'formdesc' => __('List of anti-virus vendors detecting the malware or information on detection performance (e.g. 13/43 or 67%). Attachment with list of detection or link to VirusTotal could be placed here as well.'),
'types' => array('link', 'comment', 'text', 'hex', 'attachment', 'other', 'anonymised')
),
'Payload delivery' => array(
'types' => ['link', 'comment', 'text', 'hex', 'attachment', 'other', 'anonymised']
],
'Payload delivery' => [
'desc' => __('Information about how the malware is delivered'),
'formdesc' => __('Information about the way the malware payload is initially delivered, for example information about the email or web-site, vulnerability used, originating IP etc. Malware sample itself should be attached here.'),
'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy', 'filename|pehash', 'mac-address', 'mac-eui-64', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'hostname', 'domain', 'email', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'email-body', 'url', 'user-agent', 'AS', 'pattern-in-file', 'pattern-in-traffic', 'filename-pattern', 'stix2-pattern', 'yara', 'sigma', 'mime-type', 'attachment', 'malware-sample', 'link', 'malware-type', 'comment', 'text', 'hex', 'vulnerability', 'cpe', 'weakness', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'jarm-fingerprint', 'hassh-md5', 'hasshserver-md5', 'other', 'hostname|port', 'email-dst-display-name', 'email-src-display-name', 'email-header', 'email-reply-to', 'email-x-mailer', 'email-mime-boundary', 'email-thread-index', 'email-message-id', 'azure-application-id', 'mobile-application-id', 'chrome-extension-id', 'whois-registrant-email', 'anonymised')
),
'Artifacts dropped' => array(
'types' => ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy', 'filename|pehash', 'mac-address', 'mac-eui-64', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'hostname', 'domain', 'email', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'email-body', 'url', 'user-agent', 'AS', 'pattern-in-file', 'pattern-in-traffic', 'filename-pattern', 'stix2-pattern', 'yara', 'sigma', 'mime-type', 'attachment', 'malware-sample', 'link', 'malware-type', 'comment', 'text', 'hex', 'vulnerability', 'cpe', 'weakness', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'jarm-fingerprint', 'hassh-md5', 'hasshserver-md5', 'other', 'hostname|port', 'email-dst-display-name', 'email-src-display-name', 'email-header', 'email-reply-to', 'email-x-mailer', 'email-mime-boundary', 'email-thread-index', 'email-message-id', 'azure-application-id', 'mobile-application-id', 'chrome-extension-id', 'whois-registrant-email', 'anonymised']
],
'Artifacts dropped' => [
'desc' => __('Any artifact (files, registry keys etc.) dropped by the malware or other modifications to the system'),
'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy', 'filename|pehash', 'regkey', 'regkey|value', 'pattern-in-file', 'pattern-in-memory', 'filename-pattern', 'pdb', 'stix2-pattern', 'yara', 'sigma', 'attachment', 'malware-sample', 'named pipe', 'mutex', 'process-state', 'windows-scheduled-task', 'windows-service-name', 'windows-service-displayname', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'cookie', 'gene', 'kusto-query', 'mime-type', 'anonymised', 'pgp-public-key', 'pgp-private-key')
),
'Payload installation' => array(
'types' => ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy', 'filename|pehash', 'regkey', 'regkey|value', 'pattern-in-file', 'pattern-in-memory', 'filename-pattern', 'pdb', 'stix2-pattern', 'yara', 'sigma', 'attachment', 'malware-sample', 'named pipe', 'mutex', 'process-state', 'windows-scheduled-task', 'windows-service-name', 'windows-service-displayname', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'cookie', 'gene', 'kusto-query', 'mime-type', 'anonymised', 'pgp-public-key', 'pgp-private-key']
],
'Payload installation' => [
'desc' => __('Info on where the malware gets installed in the system'),
'formdesc' => __('Location where the payload was placed in the system and the way it was installed. For example, a filename|md5 type attribute can be added here like this: c:\\windows\\system32\\malicious.exe|41d8cd98f00b204e9800998ecf8427e.'),
'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy', 'filename|pehash', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'filename-pattern', 'stix2-pattern', 'yara', 'sigma', 'vulnerability', 'cpe', 'weakness', 'attachment', 'malware-sample', 'malware-type', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'azure-application-id', 'azure-application-id', 'mobile-application-id', 'chrome-extension-id', 'other', 'mime-type', 'anonymised')
),
'Persistence mechanism' => array(
'types' => ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy', 'filename|pehash', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'filename-pattern', 'stix2-pattern', 'yara', 'sigma', 'vulnerability', 'cpe', 'weakness', 'attachment', 'malware-sample', 'malware-type', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'azure-application-id', 'azure-application-id', 'mobile-application-id', 'chrome-extension-id', 'other', 'mime-type', 'anonymised']
],
'Persistence mechanism' => [
'desc' => __('Mechanisms used by the malware to start at boot'),
'formdesc' => __('Mechanisms used by the malware to start at boot. This could be a registry key, legitimate driver modification, LNK file in startup'),
'types' => array('filename', 'regkey', 'regkey|value', 'comment', 'text', 'other', 'hex', 'anonymised')
),
'Network activity' => array(
'types' => ['filename', 'regkey', 'regkey|value', 'comment', 'text', 'other', 'hex', 'anonymised']
],
'Network activity' => [
'desc' => __('Information about network traffic generated by the malware'),
'types' => array('ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'port', 'hostname', 'domain', 'domain|ip', 'mac-address', 'mac-eui-64', 'email', 'email-dst', 'email-src', 'eppn', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'pattern-in-file', 'filename-pattern', 'stix2-pattern', 'pattern-in-traffic', 'attachment', 'comment', 'text', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'jarm-fingerprint', 'hassh-md5', 'hasshserver-md5', 'other', 'hex', 'cookie', 'hostname|port', 'bro', 'zeek', 'anonymised', 'community-id', 'email-subject', 'favicon-mmh3', 'dkim', 'dkim-signature', 'ssh-fingerprint')
),
'Payload type' => array(
'types' => ['ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'port', 'hostname', 'domain', 'domain|ip', 'mac-address', 'mac-eui-64', 'email', 'email-dst', 'email-src', 'eppn', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'pattern-in-file', 'filename-pattern', 'stix2-pattern', 'pattern-in-traffic', 'attachment', 'comment', 'text', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'jarm-fingerprint', 'hassh-md5', 'hasshserver-md5', 'other', 'hex', 'cookie', 'hostname|port', 'bro', 'zeek', 'anonymised', 'community-id', 'email-subject', 'favicon-mmh3', 'dkim', 'dkim-signature', 'ssh-fingerprint']
],
'Payload type' => [
'desc' => __('Information about the final payload(s)'),
'formdesc' => __('Information about the final payload(s). Can contain a function of the payload, e.g. keylogger, RAT, or a name if identified, such as Poison Ivy.'),
'types' => array('comment', 'text', 'other', 'anonymised')
),
'Attribution' => array(
'types' => ['comment', 'text', 'other', 'anonymised']
],
'Attribution' => [
'desc' => __('Identification of the group, organisation, or country behind the attack'),
'types' => array('threat-actor', 'campaign-name', 'campaign-id', 'whois-registrant-phone', 'whois-registrant-email', 'whois-registrant-name', 'whois-registrant-org', 'whois-registrar', 'whois-creation-date', 'comment', 'text', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'dns-soa-email', 'anonymised', 'email')
),
'External analysis' => array(
'types' => ['threat-actor', 'campaign-name', 'campaign-id', 'whois-registrant-phone', 'whois-registrant-email', 'whois-registrant-name', 'whois-registrant-org', 'whois-registrar', 'whois-creation-date', 'comment', 'text', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'dns-soa-email', 'anonymised', 'email']
],
'External analysis' => [
'desc' => __('Any other result from additional analysis of the malware like tools output'),
'formdesc' => __('Any other result from additional analysis of the malware like tools output Examples: pdf-parser output, automated sandbox analysis, reverse engineering report.'),
'types' => array('md5', 'sha1', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'mac-address', 'mac-eui-64', 'hostname', 'domain', 'domain|ip', 'url', 'user-agent', 'regkey', 'regkey|value', 'AS', 'snort', 'bro', 'zeek', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'filename-pattern', 'vulnerability', 'cpe', 'weakness', 'attachment', 'malware-sample', 'link', 'comment', 'text', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'jarm-fingerprint', 'hassh-md5', 'hasshserver-md5', 'github-repository', 'other', 'cortex', 'anonymised', 'community-id')
),
'Financial fraud' => array(
'types' => ['md5', 'sha1', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'mac-address', 'mac-eui-64', 'hostname', 'domain', 'domain|ip', 'url', 'user-agent', 'regkey', 'regkey|value', 'AS', 'snort', 'bro', 'zeek', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'filename-pattern', 'vulnerability', 'cpe', 'weakness', 'attachment', 'malware-sample', 'link', 'comment', 'text', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'jarm-fingerprint', 'hassh-md5', 'hasshserver-md5', 'github-repository', 'other', 'cortex', 'anonymised', 'community-id']
],
'Financial fraud' => [
'desc' => __('Financial Fraud indicators'),
'formdesc' => __('Financial Fraud indicators, for example: IBAN Numbers, BIC codes, Credit card numbers, etc.'),
'types' => array('btc', 'dash', 'xmr', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn', 'phone-number', 'comment', 'text', 'other', 'hex', 'anonymised'),
),
'Support Tool' => array(
'types' => ['btc', 'dash', 'xmr', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn', 'phone-number', 'comment', 'text', 'other', 'hex', 'anonymised'],
],
'Support Tool' => [
'desc' => __('Tools supporting analysis or detection of the event'),
'types' => array('link', 'text', 'attachment', 'comment', 'other', 'hex', 'anonymised')
),
'Social network' => array(
'types' => ['link', 'text', 'attachment', 'comment', 'other', 'hex', 'anonymised']
],
'Social network' => [
'desc' => __('Social networks and platforms'),
// email-src and email-dst or should we go with a new email type that is neither / both?
'types' => array('github-username', 'github-repository', 'github-organisation', 'jabber-id', 'twitter-id', 'email', 'email-src', 'email-dst', 'eppn', 'comment', 'text', 'other', 'whois-registrant-email', 'anonymised', 'pgp-public-key', 'pgp-private-key')
),
'Person' => array(
'types' => ['github-username', 'github-repository', 'github-organisation', 'jabber-id', 'twitter-id', 'email', 'email-src', 'email-dst', 'eppn', 'comment', 'text', 'other', 'whois-registrant-email', 'anonymised', 'pgp-public-key', 'pgp-private-key']
],
'Person' => [
'desc' => __('A human being - natural person'),
'types' => array('first-name', 'middle-name', 'last-name', 'full-name', 'date-of-birth', 'place-of-birth', 'gender', 'passport-number', 'passport-country', 'passport-expiration', 'redress-number', 'nationality', 'visa-number', 'issue-date-of-the-visa', 'primary-residence', 'country-of-residence', 'special-service-request', 'frequent-flyer-number', 'travel-details', 'payment-details', 'place-port-of-original-embarkation', 'place-port-of-clearance', 'place-port-of-onward-foreign-destination', 'passenger-name-record-locator-number', 'comment', 'text', 'other', 'phone-number', 'identity-card-number', 'anonymised', 'email', 'pgp-public-key', 'pgp-private-key')
),
'Other' => array(
'types' => ['first-name', 'middle-name', 'last-name', 'full-name', 'date-of-birth', 'place-of-birth', 'gender', 'passport-number', 'passport-country', 'passport-expiration', 'redress-number', 'nationality', 'visa-number', 'issue-date-of-the-visa', 'primary-residence', 'country-of-residence', 'special-service-request', 'frequent-flyer-number', 'travel-details', 'payment-details', 'place-port-of-original-embarkation', 'place-port-of-clearance', 'place-port-of-onward-foreign-destination', 'passenger-name-record-locator-number', 'comment', 'text', 'other', 'phone-number', 'identity-card-number', 'anonymised', 'email', 'pgp-public-key', 'pgp-private-key']
],
'Other' => [
'desc' => __('Attributes that are not part of any other category or are meant to be used as a component in MISP objects in the future'),
'types' => array('comment', 'text', 'other', 'size-in-bytes', 'counter', 'datetime', 'cpe', 'port', 'float', 'hex', 'phone-number', 'boolean', 'anonymised', 'pgp-public-key', 'pgp-private-key')
)
);
'types' => ['comment', 'text', 'other', 'size-in-bytes', 'counter', 'datetime', 'cpe', 'port', 'float', 'hex', 'phone-number', 'boolean', 'anonymised', 'pgp-public-key', 'pgp-private-key']
]
];
}
/**
@ -112,204 +176,204 @@ class AttributesTable extends AppTable
*/
private function generateTypeDefinitions()
{
return array(
'md5' => array('desc' => __('A checksum in MD5 format'), 'formdesc' => __("You are encouraged to use filename|md5 instead. A checksum in md5 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'sha1' => array('desc' => __('A checksum in SHA1 format'), 'formdesc' => __("You are encouraged to use filename|sha1 instead. A checksum in sha1 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'sha256' => array('desc' => __('A checksum in SHA256 format'), 'formdesc' => __("You are encouraged to use filename|sha256 instead. A checksum in sha256 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'filename' => array('desc' => __('Filename'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'pdb' => array('desc' => __('Microsoft Program database (PDB) path information'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0),
'filename|md5' => array('desc' => __('A filename and an MD5 hash separated by a |'), 'formdesc' => __("A filename and an md5 hash separated by a | (no spaces)"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'filename|sha1' => array('desc' => __('A filename and an SHA1 hash separated by a |'), 'formdesc' => __("A filename and an sha1 hash separated by a | (no spaces)"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'filename|sha256' => array('desc' => __('A filename and an SHA256 hash separated by a |'), 'formdesc' => __("A filename and an sha256 hash separated by a | (no spaces)"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'ip-src' => array('desc' => __("A source IP address of the attacker"), 'default_category' => 'Network activity', 'to_ids' => 1),
'ip-dst' => array('desc' => __('A destination IP address of the attacker or C&C server'), 'formdesc' => __("A destination IP address of the attacker or C&C server. Also set the IDS flag on when this IP is hardcoded in malware"), 'default_category' => 'Network activity', 'to_ids' => 1),
'hostname' => array('desc' => __('A full host/dnsname of an attacker'), 'formdesc' => __("A full host/dnsname of an attacker. Also set the IDS flag on when this hostname is hardcoded in malware"), 'default_category' => 'Network activity', 'to_ids' => 1),
'domain' => array('desc' => __('A domain name used in the malware'), 'formdesc' => __("A domain name used in the malware. Use this instead of hostname when the upper domain is important or can be used to create links between events."), 'default_category' => 'Network activity', 'to_ids' => 1),
'domain|ip' => array('desc' => __('A domain name and its IP address (as found in DNS lookup) separated by a |'), 'formdesc' => __("A domain name and its IP address (as found in DNS lookup) separated by a | (no spaces)"), 'default_category' => 'Network activity', 'to_ids' => 1),
'email' => array('desc' => ('An email address'), 'default_category' => 'Social network', 'to_ids' => 1),
'email-src' => array('desc' => __("The source email address. Used to describe the sender when describing an e-mail."), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'eppn' => array('desc' => __("eduPersonPrincipalName - eppn - the NetId of the person for the purposes of inter-institutional authentication. Should be stored in the form of user@univ.edu, where univ.edu is the name of the local security domain."), 'default_category' => 'Network activity', 'to_ids' => 1),
'email-dst' => array('desc' => __("The destination email address. Used to describe the recipient when describing an e-mail."), 'default_category' => 'Network activity', 'to_ids' => 1),
'email-subject' => array('desc' => __("The subject of the email"), 'default_category' => 'Payload delivery', 'to_ids' => 0),
'email-attachment' => array('desc' => __("File name of the email attachment."), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'email-body' => array('desc' => __('Email body'), 'default_category' => 'Payload delivery', 'to_ids' => 0),
'float' => array('desc' => __("A floating point value."), 'default_category' => 'Other', 'to_ids' => 0),
'git-commit-id' => array('desc' => __("A Git commit ID."), 'default_category' => 'Internal reference', 'to_ids' => 0),
'url' => array('desc' => __('Uniform Resource Locator'), 'default_category' => 'Network activity', 'to_ids' => 1),
'http-method' => array('desc' => __("HTTP method used by the malware (e.g. POST, GET, ...)."), 'default_category' => 'Network activity', 'to_ids' => 0),
'user-agent' => array('desc' => __("The user-agent used by the malware in the HTTP request."), 'default_category' => 'Network activity', 'to_ids' => 0),
'ja3-fingerprint-md5' => array('desc' => __("JA3 is a method for creating SSL/TLS client fingerprints that should be easy to produce on any platform and can be easily shared for threat intelligence."), 'default_category' => 'Network activity', 'to_ids' => 1),
'jarm-fingerprint' => array('desc' => __("JARM is a method for creating SSL/TLS server fingerprints."), 'default_category' => 'Network activity', 'to_ids' => 1),
'favicon-mmh3' => array('desc' => __("favicon-mmh3 is the murmur3 hash of a favicon as used in Shodan."), 'default_category' => 'Network activity', 'to_ids' => 1),
'hassh-md5' => array('desc' => __("hassh is a network fingerprinting standard which can be used to identify specific Client SSH implementations. The fingerprints can be easily stored, searched and shared in the form of an MD5 fingerprint."), 'default_category' => 'Network activity', 'to_ids' => 1),
'hasshserver-md5' => array('desc' => __("hasshServer is a network fingerprinting standard which can be used to identify specific Server SSH implementations. The fingerprints can be easily stored, searched and shared in the form of an MD5 fingerprint."), 'default_category' => 'Network activity', 'to_ids' => 1),
'regkey' => array('desc' => __("Registry key or value"), 'default_category' => 'Persistence mechanism', 'to_ids' => 1),
'regkey|value' => array('desc' => __("Registry value + data separated by |"), 'default_category' => 'Persistence mechanism', 'to_ids' => 1),
'AS' => array('desc' => __('Autonomous system'), 'default_category' => 'Network activity', 'to_ids' => 0),
'snort' => array('desc' => __('An IDS rule in Snort rule-format'), 'formdesc' => __("An IDS rule in Snort rule-format. This rule will be automatically rewritten in the NIDS exports."), 'default_category' => 'Network activity', 'to_ids' => 1),
'bro' => array('desc' => __('An NIDS rule in the Bro rule-format'), 'formdesc' => __("An NIDS rule in the Bro rule-format."), 'default_category' => 'Network activity', 'to_ids' => 1),
'zeek' => array('desc' => __('An NIDS rule in the Zeek rule-format'), 'formdesc' => __("An NIDS rule in the Zeek rule-format."), 'default_category' => 'Network activity', 'to_ids' => 1),
'community-id' => array('desc' => __('A community ID flow hashing algorithm to map multiple traffic monitors into common flow id'), 'formdesc' => __("a community ID flow hashing algorithm to map multiple traffic monitors into common flow id"), 'default_category' => 'Network activity', 'to_ids' => 1),
'pattern-in-file' => array('desc' => __('Pattern in file that identifies the malware'), 'default_category' => 'Payload installation', 'to_ids' => 1),
'pattern-in-traffic' => array('desc' => __('Pattern in network traffic that identifies the malware'), 'default_category' => 'Network activity', 'to_ids' => 1),
'pattern-in-memory' => array('desc' => __('Pattern in memory dump that identifies the malware'), 'default_category' => 'Payload installation', 'to_ids' => 1),
'filename-pattern' => array('desc' => __('A pattern in the name of a file'), 'default_category' => 'Payload installation', 'to_ids' => 1),
'pgp-public-key' => array('desc' => __('A PGP public key'), 'default_category' => 'Person', 'to_ids' => 0),
'pgp-private-key' => array('desc' => __('A PGP private key'), 'default_category' => 'Person', 'to_ids' => 0),
'ssh-fingerprint' => array('desc' => __('A fingerprint of SSH key material'), 'default_category' => 'Network activity', 'to_ids' => 0),
'yara' => array('desc' => __('YARA signature'), 'default_category' => 'Payload installation', 'to_ids' => 1),
'stix2-pattern' => array('desc' => __('STIX 2 pattern'), 'default_category' => 'Payload installation', 'to_ids' => 1),
'sigma' => array('desc' => __('Sigma - Generic Signature Format for SIEM Systems'), 'default_category' => 'Payload installation', 'to_ids' => 1),
'gene' => array('desc' => __('GENE - Go Evtx sigNature Engine'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0),
'kusto-query' => array('desc' => __('Kusto query - Kusto from Microsoft Azure is a service for storing and running interactive analytics over Big Data.'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0),
'mime-type' => array('desc' => __('A media type (also MIME type and content type) is a two-part identifier for file formats and format contents transmitted on the Internet'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0),
'identity-card-number' => array('desc' => __('Identity card number'), 'default_category' => 'Person', 'to_ids' => 0),
'cookie' => array('desc' => __('HTTP cookie as often stored on the user web client. This can include authentication cookie or session cookie.'), 'default_category' => 'Network activity', 'to_ids' => 0),
'vulnerability' => array('desc' => __('A reference to the vulnerability used in the exploit'), 'default_category' => 'External analysis', 'to_ids' => 0),
'cpe' => array('desc' => __('Common Platform Enumeration - structured naming scheme for information technology systems, software, and packages.'), 'default_category' => 'External analysis', 'to_ids' => 0),
'weakness' => array('desc' => __('A reference to the weakness (CWE) used in the exploit'), 'default_category' => 'External analysis', 'to_ids' => 0),
'attachment' => array('desc' => __('Attachment with external information'), 'formdesc' => __("Please upload files using the <em>Upload Attachment</em> button."), 'default_category' => 'External analysis', 'to_ids' => 0),
'malware-sample' => array('desc' => __('Attachment containing encrypted malware sample'), 'formdesc' => __("Please upload files using the <em>Upload Attachment</em> button."), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'link' => array('desc' => __('Link to an external information'), 'default_category' => 'External analysis', 'to_ids' => 0),
'comment' => array('desc' => __('Comment or description in a human language'), 'formdesc' => __('Comment or description in a human language. This will not be correlated with other attributes'), 'default_category' => 'Other', 'to_ids' => 0),
'text' => array('desc' => __('Name, ID or a reference'), 'default_category' => 'Other', 'to_ids' => 0),
'hex' => array('desc' => __('A value in hexadecimal format'), 'default_category' => 'Other', 'to_ids' => 0),
'other' => array('desc' => __('Other attribute'), 'default_category' => 'Other', 'to_ids' => 0),
'named pipe' => array('desc' => __('Named pipe, use the format \\.\pipe\<PipeName>'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0),
'mutex' => array('desc' => __('Mutex, use the format \BaseNamedObjects\<Mutex>'), 'default_category' => 'Artifacts dropped', 'to_ids' => 1),
'process-state' => array('desc' => __('State of a process'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0),
'target-user' => array('desc' => __('Attack Targets Username(s)'), 'default_category' => 'Targeting data', 'to_ids' => 0),
'target-email' => array('desc' => __('Attack Targets Email(s)'), 'default_category' => 'Targeting data', 'to_ids' => 0),
'target-machine' => array('desc' => __('Attack Targets Machine Name(s)'), 'default_category' => 'Targeting data', 'to_ids' => 0),
'target-org' => array('desc' => __('Attack Targets Department or Organization(s)'), 'default_category' => 'Targeting data', 'to_ids' => 0),
'target-location' => array('desc' => __('Attack Targets Physical Location(s)'), 'default_category' => 'Targeting data', 'to_ids' => 0),
'target-external' => array('desc' => __('External Target Organizations Affected by this Attack'), 'default_category' => 'Targeting data', 'to_ids' => 0),
'btc' => array('desc' => __('Bitcoin Address'), 'default_category' => 'Financial fraud', 'to_ids' => 1),
'dash' => array('desc' => __('Dash Address'), 'default_category' => 'Financial fraud', 'to_ids' => 1),
'xmr' => array('desc' => __('Monero Address'), 'default_category' => 'Financial fraud', 'to_ids' => 1),
'iban' => array('desc' => __('International Bank Account Number'), 'default_category' => 'Financial fraud', 'to_ids' => 1),
'bic' => array('desc' => __('Bank Identifier Code Number also known as SWIFT-BIC, SWIFT code or ISO 9362 code'), 'default_category' => 'Financial fraud', 'to_ids' => 1),
'bank-account-nr' => array('desc' => __('Bank account number without any routing number'), 'default_category' => 'Financial fraud', 'to_ids' => 1),
'aba-rtn' => array('desc' => __('ABA routing transit number'), 'default_category' => 'Financial fraud', 'to_ids' => 1),
'bin' => array('desc' => __('Bank Identification Number'), 'default_category' => 'Financial fraud', 'to_ids' => 1),
'cc-number' => array('desc' => __('Credit-Card Number'), 'default_category' => 'Financial fraud', 'to_ids' => 1),
'prtn' => array('desc' => __('Premium-Rate Telephone Number'), 'default_category' => 'Financial fraud', 'to_ids' => 1),
'phone-number' => array('desc' => __('Telephone Number'), 'default_category' => 'Person', 'to_ids' => 0),
'threat-actor' => array('desc' => __('A string identifying the threat actor'), 'default_category' => 'Attribution', 'to_ids' => 0),
'campaign-name' => array('desc' => __('Associated campaign name'), 'default_category' => 'Attribution', 'to_ids' => 0),
'campaign-id' => array('desc' => __('Associated campaign ID'), 'default_category' => 'Attribution', 'to_ids' => 0),
'malware-type' => array('desc' => '', 'default_category' => 'Payload delivery', 'to_ids' => 0),
'uri' => array('desc' => __('Uniform Resource Identifier'), 'default_category' => 'Network activity', 'to_ids' => 1),
'authentihash' => array('desc' => __('Authenticode executable signature hash'), 'formdesc' => __("You are encouraged to use filename|authentihash instead. Authenticode executable signature hash, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'vhash' => array('desc' => __('A VirusTotal checksum'), 'formdesc' => __("You are encouraged to use filename|vhash instead. A checksum from VirusTotal, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'ssdeep' => array('desc' => __('A checksum in ssdeep format'), 'formdesc' => __("You are encouraged to use filename|ssdeep instead. A checksum in the SSDeep format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'imphash' => array('desc' => __('Import hash - a hash created based on the imports in the sample.'), 'formdesc' => __("You are encouraged to use filename|imphash instead. A hash created based on the imports in the sample, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'telfhash' => array('desc' => __('telfhash is symbol hash for ELF files, just like imphash is imports hash for PE files.'), 'formdesc' => __("You are encouraged to use a file object with telfash"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'pehash' => array('desc' => __('peHash - a hash calculated based of certain pieces of a PE executable file'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'impfuzzy' => array('desc' => __('A fuzzy hash of import table of Portable Executable format'), 'formdesc' => __("You are encouraged to use filename|impfuzzy instead. A fuzzy hash created based on the imports in the sample, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'sha224' => array('desc' => __('A checksum in SHA-224 format'), 'formdesc' => __("You are encouraged to use filename|sha224 instead. A checksum in sha224 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'sha384' => array('desc' => __('A checksum in SHA-384 format'), 'formdesc' => __("You are encouraged to use filename|sha384 instead. A checksum in sha384 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'sha512' => array('desc' => __('A checksum in SHA-512 format'), 'formdesc' => __("You are encouraged to use filename|sha512 instead. A checksum in sha512 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'sha512/224' => array('desc' => __('A checksum in the SHA-512/224 format'), 'formdesc' => __("You are encouraged to use filename|sha512/224 instead. A checksum in sha512/224 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'sha512/256' => array('desc' => __('A checksum in the SHA-512/256 format'), 'formdesc' => __("You are encouraged to use filename|sha512/256 instead. A checksum in sha512/256 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'sha3-224' => array('desc' => __('A checksum in SHA3-224 format'), 'formdesc' => __("You are encouraged to use filename|sha3-224 instead. A checksum in sha3-224 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'sha3-256' => array('desc' => __('A checksum in SHA3-256 format'), 'formdesc' => __("You are encouraged to use filename|sha3-256 instead. A checksum in sha3-256 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'sha3-384' => array('desc' => __('A checksum in SHA3-384 format'), 'formdesc' => __("You are encouraged to use filename|sha3-384 instead. A checksum in sha3-384 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'sha3-512' => array('desc' => __('A checksum in SHA3-512 format'), 'formdesc' => __("You are encouraged to use filename|sha3-512 instead. A checksum in sha3-512 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'tlsh' => array('desc' => __('A checksum in the Trend Micro Locality Sensitive Hash format'), 'formdesc' => __("You are encouraged to use filename|tlsh instead. A checksum in the Trend Micro Locality Sensitive Hash format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'cdhash' => array('desc' => __('An Apple Code Directory Hash, identifying a code-signed Mach-O executable file'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'filename|authentihash' => array('desc' => __('A filename and Authenticode executable signature hash'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'filename|vhash' => array('desc' => __('A filename and a VirusTotal hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'filename|ssdeep' => array('desc' => __('A checksum in ssdeep format'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'filename|imphash' => array('desc' => __('Import hash - a hash created based on the imports in the sample.'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'filename|impfuzzy' => array('desc' => __('Import fuzzy hash - a fuzzy hash created based on the imports in the sample.'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'filename|pehash' => array('desc' => __('A filename and a peHash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'filename|sha224' => array('desc' => __('A filename and a SHA-224 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'filename|sha384' => array('desc' => __('A filename and a SHA-384 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'filename|sha512' => array('desc' => __('A filename and a SHA-512 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'filename|sha512/224' => array('desc' => __('A filename and a SHa-512/224 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'filename|sha512/256' => array('desc' => __('A filename and a SHA-512/256 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'filename|sha3-224' => array('desc' => __('A filename and an SHA3-224 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'filename|sha3-256' => array('desc' => __('A filename and an SHA3-256 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'filename|sha3-384' => array('desc' => __('A filename and an SHA3-384 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'filename|sha3-512' => array('desc' => __('A filename and an SHA3-512 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'filename|tlsh' => array('desc' => __('A filename and a Trend Micro Locality Sensitive Hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'windows-scheduled-task' => array('desc' => __('A scheduled task in windows'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0),
'windows-service-name' => array('desc' => __('A windows service name. This is the name used internally by windows. Not to be confused with the windows-service-displayname.'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0),
'windows-service-displayname' => array('desc' => __('A windows service\'s displayname, not to be confused with the windows-service-name. This is the name that applications will generally display as the service\'s name in applications.'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0),
'whois-registrant-email' => array('desc' => __('The e-mail of a domain\'s registrant, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0),
'whois-registrant-phone' => array('desc' => __('The phone number of a domain\'s registrant, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0),
'whois-registrant-name' => array('desc' => __('The name of a domain\'s registrant, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0),
'whois-registrant-org' => array('desc' => __('The org of a domain\'s registrant, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0),
'whois-registrar' => array('desc' => __('The registrar of the domain, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0),
'whois-creation-date' => array('desc' => __('The date of domain\'s creation, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0),
return [
'md5' => ['desc' => __('A checksum in MD5 format'), 'formdesc' => __("You are encouraged to use filename|md5 instead. A checksum in md5 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'sha1' => ['desc' => __('A checksum in SHA1 format'), 'formdesc' => __("You are encouraged to use filename|sha1 instead. A checksum in sha1 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'sha256' => ['desc' => __('A checksum in SHA256 format'), 'formdesc' => __("You are encouraged to use filename|sha256 instead. A checksum in sha256 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'filename' => ['desc' => __('Filename'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'pdb' => ['desc' => __('Microsoft Program database (PDB) path information'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0],
'filename|md5' => ['desc' => __('A filename and an MD5 hash separated by a |'), 'formdesc' => __("A filename and an md5 hash separated by a | (no spaces)"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'filename|sha1' => ['desc' => __('A filename and an SHA1 hash separated by a |'), 'formdesc' => __("A filename and an sha1 hash separated by a | (no spaces)"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'filename|sha256' => ['desc' => __('A filename and an SHA256 hash separated by a |'), 'formdesc' => __("A filename and an sha256 hash separated by a | (no spaces)"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'ip-src' => ['desc' => __("A source IP address of the attacker"), 'default_category' => 'Network activity', 'to_ids' => 1],
'ip-dst' => ['desc' => __('A destination IP address of the attacker or C&C server'), 'formdesc' => __("A destination IP address of the attacker or C&C server. Also set the IDS flag on when this IP is hardcoded in malware"), 'default_category' => 'Network activity', 'to_ids' => 1],
'hostname' => ['desc' => __('A full host/dnsname of an attacker'), 'formdesc' => __("A full host/dnsname of an attacker. Also set the IDS flag on when this hostname is hardcoded in malware"), 'default_category' => 'Network activity', 'to_ids' => 1],
'domain' => ['desc' => __('A domain name used in the malware'), 'formdesc' => __("A domain name used in the malware. Use this instead of hostname when the upper domain is important or can be used to create links between events."), 'default_category' => 'Network activity', 'to_ids' => 1],
'domain|ip' => ['desc' => __('A domain name and its IP address (as found in DNS lookup) separated by a |'), 'formdesc' => __("A domain name and its IP address (as found in DNS lookup) separated by a | (no spaces)"), 'default_category' => 'Network activity', 'to_ids' => 1],
'email' => ['desc' => ('An email address'), 'default_category' => 'Social network', 'to_ids' => 1],
'email-src' => ['desc' => __("The source email address. Used to describe the sender when describing an e-mail."), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'eppn' => ['desc' => __("eduPersonPrincipalName - eppn - the NetId of the person for the purposes of inter-institutional authentication. Should be stored in the form of user@univ.edu, where univ.edu is the name of the local security domain."), 'default_category' => 'Network activity', 'to_ids' => 1],
'email-dst' => ['desc' => __("The destination email address. Used to describe the recipient when describing an e-mail."), 'default_category' => 'Network activity', 'to_ids' => 1],
'email-subject' => ['desc' => __("The subject of the email"), 'default_category' => 'Payload delivery', 'to_ids' => 0],
'email-attachment' => ['desc' => __("File name of the email attachment."), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'email-body' => ['desc' => __('Email body'), 'default_category' => 'Payload delivery', 'to_ids' => 0],
'float' => ['desc' => __("A floating point value."), 'default_category' => 'Other', 'to_ids' => 0],
'git-commit-id' => ['desc' => __("A Git commit ID."), 'default_category' => 'Internal reference', 'to_ids' => 0],
'url' => ['desc' => __('Uniform Resource Locator'), 'default_category' => 'Network activity', 'to_ids' => 1],
'http-method' => ['desc' => __("HTTP method used by the malware (e.g. POST, GET, ...)."), 'default_category' => 'Network activity', 'to_ids' => 0],
'user-agent' => ['desc' => __("The user-agent used by the malware in the HTTP request."), 'default_category' => 'Network activity', 'to_ids' => 0],
'ja3-fingerprint-md5' => ['desc' => __("JA3 is a method for creating SSL/TLS client fingerprints that should be easy to produce on any platform and can be easily shared for threat intelligence."), 'default_category' => 'Network activity', 'to_ids' => 1],
'jarm-fingerprint' => ['desc' => __("JARM is a method for creating SSL/TLS server fingerprints."), 'default_category' => 'Network activity', 'to_ids' => 1],
'favicon-mmh3' => ['desc' => __("favicon-mmh3 is the murmur3 hash of a favicon as used in Shodan."), 'default_category' => 'Network activity', 'to_ids' => 1],
'hassh-md5' => ['desc' => __("hassh is a network fingerprinting standard which can be used to identify specific Client SSH implementations. The fingerprints can be easily stored, searched and shared in the form of an MD5 fingerprint."), 'default_category' => 'Network activity', 'to_ids' => 1],
'hasshserver-md5' => ['desc' => __("hasshServer is a network fingerprinting standard which can be used to identify specific Server SSH implementations. The fingerprints can be easily stored, searched and shared in the form of an MD5 fingerprint."), 'default_category' => 'Network activity', 'to_ids' => 1],
'regkey' => ['desc' => __("Registry key or value"), 'default_category' => 'Persistence mechanism', 'to_ids' => 1],
'regkey|value' => ['desc' => __("Registry value + data separated by |"), 'default_category' => 'Persistence mechanism', 'to_ids' => 1],
'AS' => ['desc' => __('Autonomous system'), 'default_category' => 'Network activity', 'to_ids' => 0],
'snort' => ['desc' => __('An IDS rule in Snort rule-format'), 'formdesc' => __("An IDS rule in Snort rule-format. This rule will be automatically rewritten in the NIDS exports."), 'default_category' => 'Network activity', 'to_ids' => 1],
'bro' => ['desc' => __('An NIDS rule in the Bro rule-format'), 'formdesc' => __("An NIDS rule in the Bro rule-format."), 'default_category' => 'Network activity', 'to_ids' => 1],
'zeek' => ['desc' => __('An NIDS rule in the Zeek rule-format'), 'formdesc' => __("An NIDS rule in the Zeek rule-format."), 'default_category' => 'Network activity', 'to_ids' => 1],
'community-id' => ['desc' => __('A community ID flow hashing algorithm to map multiple traffic monitors into common flow id'), 'formdesc' => __("a community ID flow hashing algorithm to map multiple traffic monitors into common flow id"), 'default_category' => 'Network activity', 'to_ids' => 1],
'pattern-in-file' => ['desc' => __('Pattern in file that identifies the malware'), 'default_category' => 'Payload installation', 'to_ids' => 1],
'pattern-in-traffic' => ['desc' => __('Pattern in network traffic that identifies the malware'), 'default_category' => 'Network activity', 'to_ids' => 1],
'pattern-in-memory' => ['desc' => __('Pattern in memory dump that identifies the malware'), 'default_category' => 'Payload installation', 'to_ids' => 1],
'filename-pattern' => ['desc' => __('A pattern in the name of a file'), 'default_category' => 'Payload installation', 'to_ids' => 1],
'pgp-public-key' => ['desc' => __('A PGP public key'), 'default_category' => 'Person', 'to_ids' => 0],
'pgp-private-key' => ['desc' => __('A PGP private key'), 'default_category' => 'Person', 'to_ids' => 0],
'ssh-fingerprint' => ['desc' => __('A fingerprint of SSH key material'), 'default_category' => 'Network activity', 'to_ids' => 0],
'yara' => ['desc' => __('YARA signature'), 'default_category' => 'Payload installation', 'to_ids' => 1],
'stix2-pattern' => ['desc' => __('STIX 2 pattern'), 'default_category' => 'Payload installation', 'to_ids' => 1],
'sigma' => ['desc' => __('Sigma - Generic Signature Format for SIEM Systems'), 'default_category' => 'Payload installation', 'to_ids' => 1],
'gene' => ['desc' => __('GENE - Go Evtx sigNature Engine'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0],
'kusto-query' => ['desc' => __('Kusto query - Kusto from Microsoft Azure is a service for storing and running interactive analytics over Big Data.'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0],
'mime-type' => ['desc' => __('A media type (also MIME type and content type) is a two-part identifier for file formats and format contents transmitted on the Internet'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0],
'identity-card-number' => ['desc' => __('Identity card number'), 'default_category' => 'Person', 'to_ids' => 0],
'cookie' => ['desc' => __('HTTP cookie as often stored on the user web client. This can include authentication cookie or session cookie.'), 'default_category' => 'Network activity', 'to_ids' => 0],
'vulnerability' => ['desc' => __('A reference to the vulnerability used in the exploit'), 'default_category' => 'External analysis', 'to_ids' => 0],
'cpe' => ['desc' => __('Common Platform Enumeration - structured naming scheme for information technology systems, software, and packages.'), 'default_category' => 'External analysis', 'to_ids' => 0],
'weakness' => ['desc' => __('A reference to the weakness (CWE) used in the exploit'), 'default_category' => 'External analysis', 'to_ids' => 0],
'attachment' => ['desc' => __('Attachment with external information'), 'formdesc' => __("Please upload files using the <em>Upload Attachment</em> button."), 'default_category' => 'External analysis', 'to_ids' => 0],
'malware-sample' => ['desc' => __('Attachment containing encrypted malware sample'), 'formdesc' => __("Please upload files using the <em>Upload Attachment</em> button."), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'link' => ['desc' => __('Link to an external information'), 'default_category' => 'External analysis', 'to_ids' => 0],
'comment' => ['desc' => __('Comment or description in a human language'), 'formdesc' => __('Comment or description in a human language. This will not be correlated with other attributes'), 'default_category' => 'Other', 'to_ids' => 0],
'text' => ['desc' => __('Name, ID or a reference'), 'default_category' => 'Other', 'to_ids' => 0],
'hex' => ['desc' => __('A value in hexadecimal format'), 'default_category' => 'Other', 'to_ids' => 0],
'other' => ['desc' => __('Other attribute'), 'default_category' => 'Other', 'to_ids' => 0],
'named pipe' => ['desc' => __('Named pipe, use the format \\.\pipe\<PipeName>'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0],
'mutex' => ['desc' => __('Mutex, use the format \BaseNamedObjects\<Mutex>'), 'default_category' => 'Artifacts dropped', 'to_ids' => 1],
'process-state' => ['desc' => __('State of a process'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0],
'target-user' => ['desc' => __('Attack Targets Username(s)'), 'default_category' => 'Targeting data', 'to_ids' => 0],
'target-email' => ['desc' => __('Attack Targets Email(s)'), 'default_category' => 'Targeting data', 'to_ids' => 0],
'target-machine' => ['desc' => __('Attack Targets Machine Name(s)'), 'default_category' => 'Targeting data', 'to_ids' => 0],
'target-org' => ['desc' => __('Attack Targets Department or Organization(s)'), 'default_category' => 'Targeting data', 'to_ids' => 0],
'target-location' => ['desc' => __('Attack Targets Physical Location(s)'), 'default_category' => 'Targeting data', 'to_ids' => 0],
'target-external' => ['desc' => __('External Target Organizations Affected by this Attack'), 'default_category' => 'Targeting data', 'to_ids' => 0],
'btc' => ['desc' => __('Bitcoin Address'), 'default_category' => 'Financial fraud', 'to_ids' => 1],
'dash' => ['desc' => __('Dash Address'), 'default_category' => 'Financial fraud', 'to_ids' => 1],
'xmr' => ['desc' => __('Monero Address'), 'default_category' => 'Financial fraud', 'to_ids' => 1],
'iban' => ['desc' => __('International Bank Account Number'), 'default_category' => 'Financial fraud', 'to_ids' => 1],
'bic' => ['desc' => __('Bank Identifier Code Number also known as SWIFT-BIC, SWIFT code or ISO 9362 code'), 'default_category' => 'Financial fraud', 'to_ids' => 1],
'bank-account-nr' => ['desc' => __('Bank account number without any routing number'), 'default_category' => 'Financial fraud', 'to_ids' => 1],
'aba-rtn' => ['desc' => __('ABA routing transit number'), 'default_category' => 'Financial fraud', 'to_ids' => 1],
'bin' => ['desc' => __('Bank Identification Number'), 'default_category' => 'Financial fraud', 'to_ids' => 1],
'cc-number' => ['desc' => __('Credit-Card Number'), 'default_category' => 'Financial fraud', 'to_ids' => 1],
'prtn' => ['desc' => __('Premium-Rate Telephone Number'), 'default_category' => 'Financial fraud', 'to_ids' => 1],
'phone-number' => ['desc' => __('Telephone Number'), 'default_category' => 'Person', 'to_ids' => 0],
'threat-actor' => ['desc' => __('A string identifying the threat actor'), 'default_category' => 'Attribution', 'to_ids' => 0],
'campaign-name' => ['desc' => __('Associated campaign name'), 'default_category' => 'Attribution', 'to_ids' => 0],
'campaign-id' => ['desc' => __('Associated campaign ID'), 'default_category' => 'Attribution', 'to_ids' => 0],
'malware-type' => ['desc' => '', 'default_category' => 'Payload delivery', 'to_ids' => 0],
'uri' => ['desc' => __('Uniform Resource Identifier'), 'default_category' => 'Network activity', 'to_ids' => 1],
'authentihash' => ['desc' => __('Authenticode executable signature hash'), 'formdesc' => __("You are encouraged to use filename|authentihash instead. Authenticode executable signature hash, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'vhash' => ['desc' => __('A VirusTotal checksum'), 'formdesc' => __("You are encouraged to use filename|vhash instead. A checksum from VirusTotal, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'ssdeep' => ['desc' => __('A checksum in ssdeep format'), 'formdesc' => __("You are encouraged to use filename|ssdeep instead. A checksum in the SSDeep format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'imphash' => ['desc' => __('Import hash - a hash created based on the imports in the sample.'), 'formdesc' => __("You are encouraged to use filename|imphash instead. A hash created based on the imports in the sample, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'telfhash' => ['desc' => __('telfhash is symbol hash for ELF files, just like imphash is imports hash for PE files.'), 'formdesc' => __("You are encouraged to use a file object with telfash"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'pehash' => ['desc' => __('peHash - a hash calculated based of certain pieces of a PE executable file'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'impfuzzy' => ['desc' => __('A fuzzy hash of import table of Portable Executable format'), 'formdesc' => __("You are encouraged to use filename|impfuzzy instead. A fuzzy hash created based on the imports in the sample, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'sha224' => ['desc' => __('A checksum in SHA-224 format'), 'formdesc' => __("You are encouraged to use filename|sha224 instead. A checksum in sha224 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'sha384' => ['desc' => __('A checksum in SHA-384 format'), 'formdesc' => __("You are encouraged to use filename|sha384 instead. A checksum in sha384 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'sha512' => ['desc' => __('A checksum in SHA-512 format'), 'formdesc' => __("You are encouraged to use filename|sha512 instead. A checksum in sha512 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'sha512/224' => ['desc' => __('A checksum in the SHA-512/224 format'), 'formdesc' => __("You are encouraged to use filename|sha512/224 instead. A checksum in sha512/224 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'sha512/256' => ['desc' => __('A checksum in the SHA-512/256 format'), 'formdesc' => __("You are encouraged to use filename|sha512/256 instead. A checksum in sha512/256 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'sha3-224' => ['desc' => __('A checksum in SHA3-224 format'), 'formdesc' => __("You are encouraged to use filename|sha3-224 instead. A checksum in sha3-224 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'sha3-256' => ['desc' => __('A checksum in SHA3-256 format'), 'formdesc' => __("You are encouraged to use filename|sha3-256 instead. A checksum in sha3-256 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'sha3-384' => ['desc' => __('A checksum in SHA3-384 format'), 'formdesc' => __("You are encouraged to use filename|sha3-384 instead. A checksum in sha3-384 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'sha3-512' => ['desc' => __('A checksum in SHA3-512 format'), 'formdesc' => __("You are encouraged to use filename|sha3-512 instead. A checksum in sha3-512 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'tlsh' => ['desc' => __('A checksum in the Trend Micro Locality Sensitive Hash format'), 'formdesc' => __("You are encouraged to use filename|tlsh instead. A checksum in the Trend Micro Locality Sensitive Hash format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'cdhash' => ['desc' => __('An Apple Code Directory Hash, identifying a code-signed Mach-O executable file'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'filename|authentihash' => ['desc' => __('A filename and Authenticode executable signature hash'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'filename|vhash' => ['desc' => __('A filename and a VirusTotal hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'filename|ssdeep' => ['desc' => __('A checksum in ssdeep format'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'filename|imphash' => ['desc' => __('Import hash - a hash created based on the imports in the sample.'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'filename|impfuzzy' => ['desc' => __('Import fuzzy hash - a fuzzy hash created based on the imports in the sample.'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'filename|pehash' => ['desc' => __('A filename and a peHash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'filename|sha224' => ['desc' => __('A filename and a SHA-224 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'filename|sha384' => ['desc' => __('A filename and a SHA-384 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'filename|sha512' => ['desc' => __('A filename and a SHA-512 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'filename|sha512/224' => ['desc' => __('A filename and a SHa-512/224 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'filename|sha512/256' => ['desc' => __('A filename and a SHA-512/256 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'filename|sha3-224' => ['desc' => __('A filename and an SHA3-224 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'filename|sha3-256' => ['desc' => __('A filename and an SHA3-256 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'filename|sha3-384' => ['desc' => __('A filename and an SHA3-384 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'filename|sha3-512' => ['desc' => __('A filename and an SHA3-512 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'filename|tlsh' => ['desc' => __('A filename and a Trend Micro Locality Sensitive Hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'windows-scheduled-task' => ['desc' => __('A scheduled task in windows'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0],
'windows-service-name' => ['desc' => __('A windows service name. This is the name used internally by windows. Not to be confused with the windows-service-displayname.'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0],
'windows-service-displayname' => ['desc' => __('A windows service\'s displayname, not to be confused with the windows-service-name. This is the name that applications will generally display as the service\'s name in applications.'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0],
'whois-registrant-email' => ['desc' => __('The e-mail of a domain\'s registrant, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0],
'whois-registrant-phone' => ['desc' => __('The phone number of a domain\'s registrant, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0],
'whois-registrant-name' => ['desc' => __('The name of a domain\'s registrant, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0],
'whois-registrant-org' => ['desc' => __('The org of a domain\'s registrant, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0],
'whois-registrar' => ['desc' => __('The registrar of the domain, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0],
'whois-creation-date' => ['desc' => __('The date of domain\'s creation, obtained from the WHOIS information.'), 'default_category' => 'Attribution', 'to_ids' => 0],
// 'targeted-threat-index' => array('desc' => ''), // currently not mapped!
// 'mailslot' => array('desc' => 'MailSlot interprocess communication'), // currently not mapped!
// 'pipe' => array('desc' => 'Pipeline (for named pipes use the attribute type "named pipe")'), // currently not mapped!
// 'ssl-cert-attributes' => array('desc' => 'SSL certificate attributes'), // currently not mapped!
'x509-fingerprint-sha1' => array('desc' => __('X509 fingerprint in SHA-1 format'), 'default_category' => 'Network activity', 'to_ids' => 1),
'x509-fingerprint-md5' => array('desc' => __('X509 fingerprint in MD5 format'), 'default_category' => 'Network activity', 'to_ids' => 1),
'x509-fingerprint-sha256' => array('desc' => __('X509 fingerprint in SHA-256 format'), 'default_category' => 'Network activity', 'to_ids' => 1),
'dns-soa-email' => array('desc' => __('RFC 1035 mandates that DNS zones should have a SOA (Statement Of Authority) record that contains an email address where a PoC for the domain could be contacted. This can sometimes be used for attribution/linkage between different domains even if protected by whois privacy'), 'default_category' => 'Attribution', 'to_ids' => 0),
'size-in-bytes' => array('desc' => __('Size expressed in bytes'), 'default_category' => 'Other', 'to_ids' => 0),
'counter' => array('desc' => __('An integer counter, generally to be used in objects'), 'default_category' => 'Other', 'to_ids' => 0),
'datetime' => array('desc' => __('Datetime in the ISO 8601 format'), 'default_category' => 'Other', 'to_ids' => 0),
'port' => array('desc' => __('Port number'), 'default_category' => 'Network activity', 'to_ids' => 0),
'ip-dst|port' => array('desc' => __('IP destination and port number separated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1),
'ip-src|port' => array('desc' => __('IP source and port number separated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1),
'hostname|port' => array('desc' => __('Hostname and port number separated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1),
'mac-address' => array('desc' => __('MAC address'), 'default_category' => 'Network activity', 'to_ids' => 0),
'mac-eui-64' => array('desc' => __('MAC EUI-64 address'), 'default_category' => 'Network activity', 'to_ids' => 0),
'x509-fingerprint-sha1' => ['desc' => __('X509 fingerprint in SHA-1 format'), 'default_category' => 'Network activity', 'to_ids' => 1],
'x509-fingerprint-md5' => ['desc' => __('X509 fingerprint in MD5 format'), 'default_category' => 'Network activity', 'to_ids' => 1],
'x509-fingerprint-sha256' => ['desc' => __('X509 fingerprint in SHA-256 format'), 'default_category' => 'Network activity', 'to_ids' => 1],
'dns-soa-email' => ['desc' => __('RFC 1035 mandates that DNS zones should have a SOA (Statement Of Authority) record that contains an email address where a PoC for the domain could be contacted. This can sometimes be used for attribution/linkage between different domains even if protected by whois privacy'), 'default_category' => 'Attribution', 'to_ids' => 0],
'size-in-bytes' => ['desc' => __('Size expressed in bytes'), 'default_category' => 'Other', 'to_ids' => 0],
'counter' => ['desc' => __('An integer counter, generally to be used in objects'), 'default_category' => 'Other', 'to_ids' => 0],
'datetime' => ['desc' => __('Datetime in the ISO 8601 format'), 'default_category' => 'Other', 'to_ids' => 0],
'port' => ['desc' => __('Port number'), 'default_category' => 'Network activity', 'to_ids' => 0],
'ip-dst|port' => ['desc' => __('IP destination and port number separated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1],
'ip-src|port' => ['desc' => __('IP source and port number separated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1],
'hostname|port' => ['desc' => __('Hostname and port number separated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1],
'mac-address' => ['desc' => __('MAC address'), 'default_category' => 'Network activity', 'to_ids' => 0],
'mac-eui-64' => ['desc' => __('MAC EUI-64 address'), 'default_category' => 'Network activity', 'to_ids' => 0],
// verify IDS flag defaults for these
'email-dst-display-name' => array('desc' => __('Email destination display name'), 'default_category' => 'Payload delivery', 'to_ids' => 0),
'email-src-display-name' => array('desc' => __('Email source display name'), 'default_category' => 'Payload delivery', 'to_ids' => 0),
'email-header' => array('desc' => __('Email header'), 'default_category' => 'Payload delivery', 'to_ids' => 0),
'email-reply-to' => array('desc' => __('Email reply to header'), 'default_category' => 'Payload delivery', 'to_ids' => 0),
'email-x-mailer' => array('desc' => __('Email x-mailer header'), 'default_category' => 'Payload delivery', 'to_ids' => 0),
'email-mime-boundary' => array('desc' => __('The email mime boundary separating parts in a multipart email'), 'default_category' => 'Payload delivery', 'to_ids' => 0),
'email-thread-index' => array('desc' => __('The email thread index header'), 'default_category' => 'Payload delivery', 'to_ids' => 0),
'email-message-id' => array('desc' => __('The email message ID'), 'default_category' => 'Payload delivery', 'to_ids' => 0),
'github-username' => array('desc' => __('A GitHub user name'), 'default_category' => 'Social network', 'to_ids' => 0),
'github-repository' => array('desc' => __('A Github repository'), 'default_category' => 'Social network', 'to_ids' => 0),
'github-organisation' => array('desc' => __('A GitHub organisation'), 'default_category' => 'Social network', 'to_ids' => 0),
'jabber-id' => array('desc' => __('Jabber ID'), 'default_category' => 'Social network', 'to_ids' => 0),
'twitter-id' => array('desc' => __('Twitter ID'), 'default_category' => 'Social network', 'to_ids' => 0),
'dkim' => array('desc' => __('DKIM public key'), 'default_category' => 'Network activity', 'to_ids' => 0),
'dkim-signature' => array('desc' => __('DKIM signature'), 'default_category' => 'Network activity', 'to_ids' => 0),
'first-name' => array('desc' => __('First name of a natural person'), 'default_category' => 'Person', 'to_ids' => 0),
'middle-name' => array('desc' => __('Middle name of a natural person'), 'default_category' => 'Person', 'to_ids' => 0),
'last-name' => array('desc' => __('Last name of a natural person'), 'default_category' => 'Person', 'to_ids' => 0),
'full-name' => array('desc' => __('Full name of a natural person'), 'default_category' => 'Person', 'to_ids' => 0),
'date-of-birth' => array('desc' => __('Date of birth of a natural person (in YYYY-MM-DD format)'), 'default_category' => 'Person', 'to_ids' => 0),
'place-of-birth' => array('desc' => __('Place of birth of a natural person'), 'default_category' => 'Person', 'to_ids' => 0),
'gender' => array('desc' => __('The gender of a natural person (Male, Female, Other, Prefer not to say)'), 'default_category' => 'Person', 'to_ids' => 0),
'passport-number' => array('desc' => __('The passport number of a natural person'), 'default_category' => 'Person', 'to_ids' => 0),
'passport-country' => array('desc' => __('The country in which the passport was issued'), 'default_category' => 'Person', 'to_ids' => 0),
'passport-expiration' => array('desc' => __('The expiration date of a passport'), 'default_category' => 'Person', 'to_ids' => 0),
'redress-number' => array('desc' => __('The Redress Control Number is the record identifier for people who apply for redress through the DHS Travel Redress Inquiry Program (DHS TRIP). DHS TRIP is for travelers who have been repeatedly identified for additional screening and who want to file an inquiry to have erroneous information corrected in DHS systems'), 'default_category' => 'Person', 'to_ids' => 0),
'nationality' => array('desc' => __('The nationality of a natural person'), 'default_category' => 'Person', 'to_ids' => 0),
'visa-number' => array('desc' => __('Visa number'), 'default_category' => 'Person', 'to_ids' => 0),
'issue-date-of-the-visa' => array('desc' => __('The date on which the visa was issued'), 'default_category' => 'Person', 'to_ids' => 0),
'primary-residence' => array('desc' => __('The primary residence of a natural person'), 'default_category' => 'Person', 'to_ids' => 0),
'country-of-residence' => array('desc' => __('The country of residence of a natural person'), 'default_category' => 'Person', 'to_ids' => 0),
'special-service-request' => array('desc' => __('A Special Service Request is a function to an airline to provide a particular facility for A Passenger or passengers. '), 'default_category' => 'Person', 'to_ids' => 0),
'frequent-flyer-number' => array('desc' => __('The frequent flyer number of a passenger'), 'default_category' => 'Person', 'to_ids' => 0),
'email-dst-display-name' => ['desc' => __('Email destination display name'), 'default_category' => 'Payload delivery', 'to_ids' => 0],
'email-src-display-name' => ['desc' => __('Email source display name'), 'default_category' => 'Payload delivery', 'to_ids' => 0],
'email-header' => ['desc' => __('Email header'), 'default_category' => 'Payload delivery', 'to_ids' => 0],
'email-reply-to' => ['desc' => __('Email reply to header'), 'default_category' => 'Payload delivery', 'to_ids' => 0],
'email-x-mailer' => ['desc' => __('Email x-mailer header'), 'default_category' => 'Payload delivery', 'to_ids' => 0],
'email-mime-boundary' => ['desc' => __('The email mime boundary separating parts in a multipart email'), 'default_category' => 'Payload delivery', 'to_ids' => 0],
'email-thread-index' => ['desc' => __('The email thread index header'), 'default_category' => 'Payload delivery', 'to_ids' => 0],
'email-message-id' => ['desc' => __('The email message ID'), 'default_category' => 'Payload delivery', 'to_ids' => 0],
'github-username' => ['desc' => __('A GitHub user name'), 'default_category' => 'Social network', 'to_ids' => 0],
'github-repository' => ['desc' => __('A Github repository'), 'default_category' => 'Social network', 'to_ids' => 0],
'github-organisation' => ['desc' => __('A GitHub organisation'), 'default_category' => 'Social network', 'to_ids' => 0],
'jabber-id' => ['desc' => __('Jabber ID'), 'default_category' => 'Social network', 'to_ids' => 0],
'twitter-id' => ['desc' => __('Twitter ID'), 'default_category' => 'Social network', 'to_ids' => 0],
'dkim' => ['desc' => __('DKIM public key'), 'default_category' => 'Network activity', 'to_ids' => 0],
'dkim-signature' => ['desc' => __('DKIM signature'), 'default_category' => 'Network activity', 'to_ids' => 0],
'first-name' => ['desc' => __('First name of a natural person'), 'default_category' => 'Person', 'to_ids' => 0],
'middle-name' => ['desc' => __('Middle name of a natural person'), 'default_category' => 'Person', 'to_ids' => 0],
'last-name' => ['desc' => __('Last name of a natural person'), 'default_category' => 'Person', 'to_ids' => 0],
'full-name' => ['desc' => __('Full name of a natural person'), 'default_category' => 'Person', 'to_ids' => 0],
'date-of-birth' => ['desc' => __('Date of birth of a natural person (in YYYY-MM-DD format)'), 'default_category' => 'Person', 'to_ids' => 0],
'place-of-birth' => ['desc' => __('Place of birth of a natural person'), 'default_category' => 'Person', 'to_ids' => 0],
'gender' => ['desc' => __('The gender of a natural person (Male, Female, Other, Prefer not to say)'), 'default_category' => 'Person', 'to_ids' => 0],
'passport-number' => ['desc' => __('The passport number of a natural person'), 'default_category' => 'Person', 'to_ids' => 0],
'passport-country' => ['desc' => __('The country in which the passport was issued'), 'default_category' => 'Person', 'to_ids' => 0],
'passport-expiration' => ['desc' => __('The expiration date of a passport'), 'default_category' => 'Person', 'to_ids' => 0],
'redress-number' => ['desc' => __('The Redress Control Number is the record identifier for people who apply for redress through the DHS Travel Redress Inquiry Program (DHS TRIP). DHS TRIP is for travelers who have been repeatedly identified for additional screening and who want to file an inquiry to have erroneous information corrected in DHS systems'), 'default_category' => 'Person', 'to_ids' => 0],
'nationality' => ['desc' => __('The nationality of a natural person'), 'default_category' => 'Person', 'to_ids' => 0],
'visa-number' => ['desc' => __('Visa number'), 'default_category' => 'Person', 'to_ids' => 0],
'issue-date-of-the-visa' => ['desc' => __('The date on which the visa was issued'), 'default_category' => 'Person', 'to_ids' => 0],
'primary-residence' => ['desc' => __('The primary residence of a natural person'), 'default_category' => 'Person', 'to_ids' => 0],
'country-of-residence' => ['desc' => __('The country of residence of a natural person'), 'default_category' => 'Person', 'to_ids' => 0],
'special-service-request' => ['desc' => __('A Special Service Request is a function to an airline to provide a particular facility for A Passenger or passengers. '), 'default_category' => 'Person', 'to_ids' => 0],
'frequent-flyer-number' => ['desc' => __('The frequent flyer number of a passenger'), 'default_category' => 'Person', 'to_ids' => 0],
// Do we really need remarks? Or just use comment/text for this?
//'remarks' => array('desc' => '', 'default_category' => 'Person', 'to_ids' => 0),
'travel-details' => array('desc' => __('Travel details'), 'default_category' => 'Person', 'to_ids' => 0),
'payment-details' => array('desc' => __('Payment details'), 'default_category' => 'Person', 'to_ids' => 0),
'place-port-of-original-embarkation' => array('desc' => __('The original port of embarkation'), 'default_category' => 'Person', 'to_ids' => 0),
'place-port-of-clearance' => array('desc' => __('The port of clearance'), 'default_category' => 'Person', 'to_ids' => 0),
'place-port-of-onward-foreign-destination' => array('desc' => __('A Port where the passenger is transiting to'), 'default_category' => 'Person', 'to_ids' => 0),
'passenger-name-record-locator-number' => array('desc' => __('The Passenger Name Record Locator is a key under which the reservation for a trip is stored in the system. The PNR contains, among other data, the name, flight segments and address of the passenger. It is defined by a combination of five or six letters and numbers.'), 'default_category' => 'Person', 'to_ids' => 0),
'mobile-application-id' => array('desc' => __('The application id of a mobile application'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'azure-application-id' => array('desc' => __('Azure Application ID.'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'chrome-extension-id' => array('desc' => __('Chrome extension id'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
'cortex' => array('desc' => __('Cortex analysis result'), 'default_category' => 'External analysis', 'to_ids' => 0),
'boolean' => array('desc' => __('Boolean value - to be used in objects'), 'default_category' => 'Other', 'to_ids' => 0),
'anonymised' => array('desc' => __('Anonymised value - described with the anonymisation object via a relationship'), 'formdesc' => __('Anonymised value - described with the anonymisation object via a relationship.'), 'default_category' => 'Other', 'to_ids' => 0)
'travel-details' => ['desc' => __('Travel details'), 'default_category' => 'Person', 'to_ids' => 0],
'payment-details' => ['desc' => __('Payment details'), 'default_category' => 'Person', 'to_ids' => 0],
'place-port-of-original-embarkation' => ['desc' => __('The original port of embarkation'), 'default_category' => 'Person', 'to_ids' => 0],
'place-port-of-clearance' => ['desc' => __('The port of clearance'), 'default_category' => 'Person', 'to_ids' => 0],
'place-port-of-onward-foreign-destination' => ['desc' => __('A Port where the passenger is transiting to'), 'default_category' => 'Person', 'to_ids' => 0],
'passenger-name-record-locator-number' => ['desc' => __('The Passenger Name Record Locator is a key under which the reservation for a trip is stored in the system. The PNR contains, among other data, the name, flight segments and address of the passenger. It is defined by a combination of five or six letters and numbers.'), 'default_category' => 'Person', 'to_ids' => 0],
'mobile-application-id' => ['desc' => __('The application id of a mobile application'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'azure-application-id' => ['desc' => __('Azure Application ID.'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'chrome-extension-id' => ['desc' => __('Chrome extension id'), 'default_category' => 'Payload delivery', 'to_ids' => 1],
'cortex' => ['desc' => __('Cortex analysis result'), 'default_category' => 'External analysis', 'to_ids' => 0],
'boolean' => ['desc' => __('Boolean value - to be used in objects'), 'default_category' => 'Other', 'to_ids' => 0],
'anonymised' => ['desc' => __('Anonymised value - described with the anonymisation object via a relationship'), 'formdesc' => __('Anonymised value - described with the anonymisation object via a relationship.'), 'default_category' => 'Other', 'to_ids' => 0]
// Not convinced about this.
//'url-regex' => array('desc' => '', 'default_category' => 'Person', 'to_ids' => 0),
);
];
}
}

View File

@ -3,7 +3,11 @@
namespace App\Model\Table;
use App\Model\Table\AppTable;
use ArrayObject;
use Cake\Core\Configure;
use Cake\Datasource\EntityInterface;
use Cake\Event\EventInterface;
use Cake\Utility\Text;
class EventsTable extends AppTable
{
@ -21,9 +25,25 @@ class EventsTable extends AppTable
'foreignKey' => 'sharing_group_id'
]
);
$this->hasMany(
'Attributes',
[
'dependent' => true,
'propertyName' => 'Attribute'
]
);
$this->setDisplayField('title');
}
public function beforeSave(EventInterface $event, EntityInterface $entity, ArrayObject $options)
{
if ($entity->uuid === null) {
$entity->uuid = Text::uuid();
}
return true;
}
public function createEventConditions($user)
{
$conditions = [];
@ -112,9 +132,8 @@ class EventsTable extends AppTable
}
if (!isset($data['Event']['orgc_id']) && !isset($data['Event']['orgc'])) {
$data['Event']['orgc_id'] = $data['Event']['org_id'];
} else {
$orgc_id = $data['Event']['orgc_id'] ?? null;
}
$event = $this->newEntity($data['Event']);
$this->saveOrFail($event);

View File

@ -13,6 +13,7 @@ use App\Lib\Tools\SyncTool;
use App\Lib\Tools\TmpFileTool;
use App\Model\Entity\Attribute;
use App\Model\Entity\Feed;
use App\Model\Entity\User;
use App\Model\Table\AppTable;
use Cake\Core\Configure;
use Cake\Http\Client as HttpClient;
@ -386,8 +387,10 @@ class FeedsTable extends AppTable
}
}
$content = $response->getBody()->getContents();
try {
FileAccessTool::writeCompressedFile($feedCache, $response->getBody());
FileAccessTool::writeCompressedFile($feedCache, $content);
if ($response->getHeader('etag')) {
FileAccessTool::writeToFile($feedCacheEtag, $response->getHeader('etag')[0]);
}
@ -396,7 +399,7 @@ class FeedsTable extends AppTable
$this->logException("Could not save file `$feedCache` to cache.", $e, LOG_NOTICE);
}
return $response->getBody();
return $content;
}
/**
@ -1209,7 +1212,7 @@ class FeedsTable extends AppTable
/**
* @param int $feedId
* @param array $user
* @param User $user
* @param int|false $jobId
* @return array|bool
* @throws Exception
@ -1227,10 +1230,6 @@ class FeedsTable extends AppTable
throw new Exception("Feed with ID $feedId not found.");
}
if (!empty($feed['settings'])) {
$feed['settings'] = json_decode($feed['settings'], true);
}
$HttpSocket = $this->isFeedLocal($feed) ? null : $this->__setupHttpSocket();
if ($feed['source_format'] === 'misp') {
$this->jobProgress($jobId, 'Fetching event manifest.');
@ -1300,31 +1299,34 @@ class FeedsTable extends AppTable
}
/**
* @param array $feed
* @param Feed $feed
* @param array $data
* @param array $user
* @param User $user
* @param int|bool $jobId
* @return bool
* @throws Exception
*/
public function saveFreetextFeedData(array $feed, array $data, array $user, $jobId = false)
public function saveFreetextFeedData(Feed $feed, array $data, User $user, $jobId = false)
{
$EventsTable = $this->fetchTable('Events');
if ($feed['fixed_event'] && $feed['event_id']) {
$event = $this->Event->find('first', ['conditions' => ['Event.id' => $feed['event_id']], 'recursive' => -1]);
$event = $EventsTable->find('all', ['conditions' => ['Event.id' => $feed['event_id']], 'recursive' => -1])->first();
if (empty($event)) {
throw new Exception("The target event is no longer valid. Make sure that the target event {$feed['event_id']} exists.");
}
} else {
$this->Event->create();
$orgc_id = $user['org_id'];
if (!empty($feed['orgc_id'])) {
$orgc_id = $feed['orgc_id'];
}
$disableCorrelation = false;
if (!empty($feed['settings'])) {
$disableCorrelation = (bool) $feed['settings']['disable_correlation'] ?? false;
if (!empty($feed['settings']['disable_correlation'])) {
$disableCorrelation = (bool) $feed['settings']['disable_correlation'];
} else {
$disableCorrelation = false;
}
}
$event = [
'info' => $feed['name'] . ' feed',
@ -1338,30 +1340,29 @@ class FeedsTable extends AppTable
'user_id' => $user['id'],
'disable_correlation' => $disableCorrelation,
];
$result = $this->Event->save($event);
$eventEntity = $EventsTable->newEntity($event);
$result = $EventsTable->save($eventEntity);
if (!$result) {
throw new Exception('Something went wrong while creating a new event.');
}
$event = $this->Event->find('first', ['conditions' => ['Event.id' => $this->Event->id], 'recursive' => -1]);
if (empty($event)) {
throw new Exception("The newly created event is no longer valid. Make sure that the target event {$this->Event->id} exists.");
if (empty($eventEntity)) {
throw new Exception("The newly created event is no longer valid. Make sure that the target event {$eventEntity->id} exists.");
}
if ($feed['fixed_event']) {
$feed['event_id'] = $event['Event']['id'];
$feed['event_id'] = $eventEntity['id'];
if (!empty($feed['settings'])) {
$feed['settings'] = json_encode($feed['settings']);
}
$feedEntity = $this->newEntity($feed);
$this->save($feedEntity);
$this->save($feed);
}
}
if ($feed['fixed_event']) {
$existsAttributesValueToId = $this->Event->Attribute->find(
$existsAttributesValueToId = $EventsTable->Attributes->find(
'list',
[
'conditions' => [
'Attribute.deleted' => 0,
'Attribute.event_id' => $event['Event']['id']
'Attribute.event_id' => $eventEntity['id']
],
'recursive' => -1,
'fields' => ['value', 'id']
@ -1386,7 +1387,7 @@ class FeedsTable extends AppTable
}
}
if ($feed['delta_merge'] && !empty($existsAttributesValueToId)) {
$attributesToDelete = $this->Event->Attribute->find(
$attributesToDelete = $EventsTable->Attributes->find(
'all',
[
'conditions' => [
@ -1399,9 +1400,9 @@ class FeedsTable extends AppTable
$attributesToDelete[$k]['Attribute']['deleted'] = 1;
unset($attributesToDelete[$k]['Attribute']['timestamp']);
}
$this->Event->Attribute->saveMany($attributesToDelete); // We need to trigger callback methods
$EventsTable->Attributes->saveMany($attributesToDelete); // We need to trigger callback methods
if (!empty($attributesToDelete)) {
$this->Event->unpublishEvent($feed['event_id']);
$EventsTable->unpublishEvent($feed['event_id']);
}
}
}
@ -1415,7 +1416,7 @@ class FeedsTable extends AppTable
unset($data[$key]);
continue;
}
$data[$key]['event_id'] = $event['Event']['id'];
$data[$key]['event_id'] = $eventEntity['id'];
$data[$key]['distribution'] = 5;
$data[$key]['sharing_group_id'] = 0;
$data[$key]['to_ids'] = $feed['override_ids'] ? 0 : $value['to_ids'];
@ -1423,19 +1424,19 @@ class FeedsTable extends AppTable
}
$chunks = array_chunk($data, 100);
foreach ($chunks as $k => $chunk) {
$this->Event->Attribute->saveMany($chunk, ['validate' => true, 'parentEvent' => $event]);
$EventsTable->Attributes->saveMany($EventsTable->Attributes->newEntities($chunk), ['validate' => true, 'parentEvent' => $event]);
$this->jobProgress($jobId, null, 50 + round(($k * 100 + 1) / count($data) * 50));
}
if (!empty($data) || !empty($attributesToDelete)) {
unset($event['Event']['timestamp']);
unset($event['Event']['attribute_count']);
$this->Event->save($event);
unset($eventEntity['timestamp']);
unset($eventEntity['attribute_count']);
$EventsTable->save($eventEntity);
}
if ($feed['publish']) {
$this->Event->publishRouter($event['Event']['id'], null, $user);
$EventsTable->publishRouter($eventEntity['id'], null, $user);
}
if ($feed['tag_id']) {
$this->Event->EventTag->attachTagToEvent($event['Event']['id'], ['id' => $feed['tag_id']]);
$EventsTable->EventTag->attachTagToEvent($eventEntity['id'], ['id' => $feed['tag_id']]);
}
return true;
}

View File

@ -124,7 +124,7 @@ class JobsTable extends AppTable
}
$jobData = [
$this->primaryKey => $jobId,
'id' => $jobId,
];
if ($message !== null) {
$jobData['message'] = $message;
@ -163,14 +163,18 @@ class JobsTable extends AppTable
$message = $success ? __('Job done.') : __('Job failed.');
}
$jobData = $this->newEntity(
[
$this->primaryKey => $jobId,
'status' => $success ? Job::STATUS_COMPLETED : Job::STATUS_FAILED,
'message' => $message,
'progress' => 100,
]
);
$jobData = $this->get($jobId);
if (!$jobData) {
$jobData = $this->newEntity(
[
'id' => $jobId,
'status' => $success ? Job::STATUS_COMPLETED : Job::STATUS_FAILED,
'message' => $message,
'progress' => 100,
]
);
}
try {
if ($this->save($jobData)) {

View File

@ -0,0 +1,18 @@
<?php
declare(strict_types=1);
namespace App\Test\Fixture;
use Cake\TestSuite\Fixture\TestFixture;
class AttributesFixture extends TestFixture
{
public $connection = 'test';
public function init(): void
{
$this->records = [];
parent::init();
}
}

View File

@ -18,6 +18,10 @@ class FeedsFixture extends TestFixture
public const FEED_2_NAME = 'test-feed-2';
public const FEED_2_URL = 'http://feed2.local/misp/test-feed-2';
public const FEED_3_ID = 3000;
public const FEED_3_NAME = 'test-feed-3';
public const FEED_3_URL = 'http://feed3.local/freetext/test-feed-3';
public function init(): void
{
$this->records = [
@ -27,6 +31,8 @@ class FeedsFixture extends TestFixture
'provider' => 'test-provider',
'url' => self::FEED_1_URL,
"source_format" => 'misp',
"input_source" => "network",
"settings" => "[]",
'enabled' => true,
],
[
@ -35,7 +41,19 @@ class FeedsFixture extends TestFixture
'provider' => 'test-provider',
'url' => self::FEED_2_URL,
"source_format" => 'misp',
"input_source" => "network",
"settings" => "[]",
'enabled' => false,
],
[
'id' => self::FEED_3_ID,
'name' => self::FEED_3_NAME,
'provider' => 'test-provider',
'url' => self::FEED_3_URL,
"source_format" => 'freetext',
"input_source" => "network",
"settings" => "{\"csv\":{\"value\":\"\",\"delimiter\":\",\"},\"common\":{\"excluderegex\":\"\"}}",
'enabled' => true,
]
];
parent::init();

View File

@ -7,6 +7,7 @@ namespace App\Test\TestCase\Api\Feeds;
use App\Test\Fixture\AuthKeysFixture;
use App\Test\Fixture\FeedsFixture;
use App\Test\Helper\ApiTestTrait;
use Cake\Core\Configure;
use Cake\Http\TestSuite\HttpClientTrait;
use Cake\TestSuite\TestCase;
@ -24,12 +25,15 @@ class FetchFromFeedApiTest extends TestCase
'app.AuthKeys',
'app.Feeds',
'app.Events',
'app.Attributes',
];
public function testFetchFromMispFeedById(): void
{
$this->skipOpenApiValidations();
Configure::write('BackgroundJobs.enabled', false);
$this->setAuthToken(AuthKeysFixture::ADMIN_API_KEY);
$url = sprintf('%s/%d', self::ENDPOINT, FeedsFixture::FEED_1_ID);
@ -139,8 +143,8 @@ class FetchFromFeedApiTest extends TestCase
// check that the event was added
$this->assertDbRecordExists('Events', ['uuid' => '56bf399d-c46c-4fdb-a9cf-d9bb02de0b81']);
$this->assertDbRecordExists('Attributes', ['uuid' => '56bf39c9-c078-4368-9555-6cf802de0b81']);
// TODO: check that the attributes were added
// TODO: check that the objects were added
// TODO: check that the event reports were added
// TODO: check that the sightings were added
@ -148,4 +152,46 @@ class FetchFromFeedApiTest extends TestCase
// TODO: check that the galaxies were added
// TODO: check that the cryptographic key were added
}
public function testFetchFromFreetextFeedById(): void
{
$this->skipOpenApiValidations();
Configure::write('BackgroundJobs.enabled', false);
$this->setAuthToken(AuthKeysFixture::ADMIN_API_KEY);
$url = sprintf('%s/%d', self::ENDPOINT, FeedsFixture::FEED_3_ID);
$headers = [
'Content-Type: text/plain',
'Connection: close',
];
$feedContent = implode(
" ",
[
'8.8.8.8',
'8.8.4.4',
]
);
$this->mockClientGet(
FeedsFixture::FEED_3_URL,
$this->newClientResponse(200, $headers, $feedContent)
);
$this->post($url);
$this->assertResponseOk();
$response = $this->getJsonResponseAsArray();
$this->assertEquals(
'Fetching the feed has successfully completed.',
$response['result']
);
// check that the event was added
$this->assertDbRecordExists('Events', ['info' => FeedsFixture::FEED_3_NAME . ' feed']);
$this->assertDbRecordExists('Attributes', ['value1' => '8.8.8.8']);
$this->assertDbRecordExists('Attributes', ['value1' => '8.8.4.4']);
}
}