mirror of https://github.com/MISP/MISP
new: [wip] migrate feeds controller main api endpoints
Luciano Righetti
parent
1b8eb67a00
commit
e107d6e8bb
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,757 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Lib\Tools;
|
||||
|
||||
use Exception;
|
||||
use InvalidArgumentException;
|
||||
use Cake\I18n\FrozenTime;
|
||||
|
||||
class AttributeValidationTool
|
||||
{
|
||||
// private
|
||||
const HASH_HEX_LENGTH = [
|
||||
'authentihash' => 64,
|
||||
'md5' => 32,
|
||||
'imphash' => 32,
|
||||
'telfhash' => 70,
|
||||
'sha1' => 40,
|
||||
'git-commit-id' => 40,
|
||||
'x509-fingerprint-md5' => 32,
|
||||
'x509-fingerprint-sha1' => 40,
|
||||
'x509-fingerprint-sha256' => 64,
|
||||
'ja3-fingerprint-md5' => 32,
|
||||
'jarm-fingerprint' => 62,
|
||||
'hassh-md5' => 32,
|
||||
'hasshserver-md5' => 32,
|
||||
'pehash' => 40,
|
||||
'sha224' => 56,
|
||||
'sha256' => 64,
|
||||
'sha384' => 96,
|
||||
'sha512' => 128,
|
||||
'sha512/224' => 56,
|
||||
'sha512/256' => 64,
|
||||
'sha3-224' => 56,
|
||||
'sha3-256' => 64,
|
||||
'sha3-384' => 96,
|
||||
'sha3-512' => 128,
|
||||
];
|
||||
|
||||
/**
|
||||
* Do some last second modifications before the validation
|
||||
* @param string $type
|
||||
* @param mixed $value
|
||||
* @return string
|
||||
*/
|
||||
public static function modifyBeforeValidation($type, $value)
|
||||
{
|
||||
$value = self::handle4ByteUnicode($value);
|
||||
switch ($type) {
|
||||
case 'ip-src':
|
||||
case 'ip-dst':
|
||||
return self::normalizeIp($value);
|
||||
case 'md5':
|
||||
case 'sha1':
|
||||
case 'sha224':
|
||||
case 'sha256':
|
||||
case 'sha384':
|
||||
case 'sha512':
|
||||
case 'sha512/224':
|
||||
case 'sha512/256':
|
||||
case 'sha3-224':
|
||||
case 'sha3-256':
|
||||
case 'sha3-384':
|
||||
case 'sha3-512':
|
||||
case 'ja3-fingerprint-md5':
|
||||
case 'jarm-fingerprint':
|
||||
case 'hassh-md5':
|
||||
case 'hasshserver-md5':
|
||||
case 'hostname':
|
||||
case 'pehash':
|
||||
case 'authentihash':
|
||||
case 'vhash':
|
||||
case 'imphash':
|
||||
case 'telfhash':
|
||||
case 'tlsh':
|
||||
case 'anonymised':
|
||||
case 'cdhash':
|
||||
case 'email':
|
||||
case 'email-src':
|
||||
case 'email-dst':
|
||||
case 'target-email':
|
||||
case 'whois-registrant-email':
|
||||
return strtolower($value);
|
||||
case 'domain':
|
||||
$value = strtolower($value);
|
||||
$value = trim($value, '.');
|
||||
// Domain is not valid, try to convert to punycode
|
||||
if (!self::isDomainValid($value) && function_exists('idn_to_ascii')) {
|
||||
$punyCode = idn_to_ascii($value);
|
||||
if ($punyCode !== false) {
|
||||
$value = $punyCode;
|
||||
}
|
||||
}
|
||||
return $value;
|
||||
case 'domain|ip':
|
||||
$value = strtolower($value);
|
||||
$parts = explode('|', $value);
|
||||
if (!isset($parts[1])) {
|
||||
return $value; // not a composite
|
||||
}
|
||||
$parts[0] = trim($parts[0], '.');
|
||||
// Domain is not valid, try to convert to punycode
|
||||
if (!self::isDomainValid($parts[0]) && function_exists('idn_to_ascii')) {
|
||||
$punyCode = idn_to_ascii($parts[0]);
|
||||
if ($punyCode !== false) {
|
||||
$parts[0] = $punyCode;
|
||||
}
|
||||
}
|
||||
$parts[1] = self::normalizeIp($parts[1]);
|
||||
return "$parts[0]|$parts[1]";
|
||||
case 'filename|md5':
|
||||
case 'filename|sha1':
|
||||
case 'filename|imphash':
|
||||
case 'filename|sha224':
|
||||
case 'filename|sha256':
|
||||
case 'filename|sha384':
|
||||
case 'filename|sha512':
|
||||
case 'filename|sha512/224':
|
||||
case 'filename|sha512/256':
|
||||
case 'filename|sha3-224':
|
||||
case 'filename|sha3-256':
|
||||
case 'filename|sha3-384':
|
||||
case 'filename|sha3-512':
|
||||
case 'filename|authentihash':
|
||||
case 'filename|vhash':
|
||||
case 'filename|pehash':
|
||||
case 'filename|tlsh':
|
||||
// Convert hash to lowercase
|
||||
$pos = strpos($value, '|');
|
||||
return substr($value, 0, $pos) . strtolower(substr($value, $pos));
|
||||
case 'http-method':
|
||||
case 'hex':
|
||||
return strtoupper($value);
|
||||
case 'vulnerability':
|
||||
case 'weakness':
|
||||
$value = str_replace('–', '-', $value);
|
||||
return strtoupper($value);
|
||||
case 'cc-number':
|
||||
case 'bin':
|
||||
return preg_replace('/[^0-9]+/', '', $value);
|
||||
case 'iban':
|
||||
case 'bic':
|
||||
$value = strtoupper($value);
|
||||
return preg_replace('/[^0-9A-Z]+/', '', $value);
|
||||
case 'prtn':
|
||||
case 'whois-registrant-phone':
|
||||
case 'phone-number':
|
||||
if (substr($value, 0, 2) == '00') {
|
||||
$value = '+' . substr($value, 2);
|
||||
}
|
||||
$value = preg_replace('/\(0\)/', '', $value);
|
||||
return preg_replace('/[^\+0-9]+/', '', $value);
|
||||
case 'x509-fingerprint-md5':
|
||||
case 'x509-fingerprint-sha256':
|
||||
case 'x509-fingerprint-sha1':
|
||||
$value = str_replace(':', '', $value);
|
||||
return strtolower($value);
|
||||
case 'ip-dst|port':
|
||||
case 'ip-src|port':
|
||||
if (substr_count($value, ':') >= 2) { // (ipv6|port) - tokenize ip and port
|
||||
if (strpos($value, '|')) { // 2001:db8::1|80
|
||||
$parts = explode('|', $value);
|
||||
} elseif (strpos($value, '[') === 0 && strpos($value, ']') !== false) { // [2001:db8::1]:80
|
||||
$ipv6 = substr($value, 1, strpos($value, ']') - 1);
|
||||
$port = explode(':', substr($value, strpos($value, ']')))[1];
|
||||
$parts = array($ipv6, $port);
|
||||
} elseif (strpos($value, '.')) { // 2001:db8::1.80
|
||||
$parts = explode('.', $value);
|
||||
} elseif (strpos($value, ' port ')) { // 2001:db8::1 port 80
|
||||
$parts = explode(' port ', $value);
|
||||
} elseif (strpos($value, 'p')) { // 2001:db8::1p80
|
||||
$parts = explode('p', $value);
|
||||
} elseif (strpos($value, '#')) { // 2001:db8::1#80
|
||||
$parts = explode('#', $value);
|
||||
} else { // 2001:db8::1:80 this one is ambiguous
|
||||
$temp = explode(':', $value);
|
||||
$parts = array(implode(':', array_slice($temp, 0, count($temp) - 1)), end($temp));
|
||||
}
|
||||
} elseif (strpos($value, ':')) { // (ipv4:port)
|
||||
$parts = explode(':', $value);
|
||||
} elseif (strpos($value, '|')) { // (ipv4|port)
|
||||
$parts = explode('|', $value);
|
||||
} else {
|
||||
return $value;
|
||||
}
|
||||
return self::normalizeIp($parts[0]) . '|' . $parts[1];
|
||||
case 'mac-address':
|
||||
case 'mac-eui-64':
|
||||
$value = str_replace(array('.', ':', '-', ' '), '', strtolower($value));
|
||||
return wordwrap($value, 2, ':', true);
|
||||
case 'hostname|port':
|
||||
$value = strtolower($value);
|
||||
return str_replace(':', '|', $value);
|
||||
case 'boolean':
|
||||
$value = trim(strtolower($value));
|
||||
if ('true' === $value) {
|
||||
$value = 1;
|
||||
} else if ('false' === $value) {
|
||||
$value = 0;
|
||||
}
|
||||
return $value ? '1' : '0';
|
||||
case 'datetime':
|
||||
try {
|
||||
return (new FrozenTime($value, 'GMT'))->format('Y-m-d\TH:i:s.uO'); // ISO8601 formatting with microseconds
|
||||
} catch (Exception $e) {
|
||||
return $value; // silently skip. Rejection will be done in validation()
|
||||
}
|
||||
case 'AS':
|
||||
if (strtoupper(substr($value, 0, 2)) === 'AS') {
|
||||
$value = substr($value, 2); // remove 'AS'
|
||||
}
|
||||
if (strpos($value, '.') !== false) { // maybe value is in asdot notation
|
||||
$parts = explode('.', $value, 2);
|
||||
if (self::isPositiveInteger($parts[0]) && self::isPositiveInteger($parts[1])) {
|
||||
return $parts[0] * 65536 + $parts[1];
|
||||
}
|
||||
}
|
||||
return $value;
|
||||
}
|
||||
return $value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate if value is valid for given attribute type.
|
||||
* At this point, we can be sure, that composite type is really composite.
|
||||
* @param string $type
|
||||
* @param string $value
|
||||
* @return bool|string
|
||||
*/
|
||||
public static function validate($type, $value)
|
||||
{
|
||||
switch ($type) {
|
||||
case 'md5':
|
||||
case 'imphash':
|
||||
case 'sha1':
|
||||
case 'sha224':
|
||||
case 'sha256':
|
||||
case 'sha384':
|
||||
case 'sha512':
|
||||
case 'sha512/224':
|
||||
case 'sha512/256':
|
||||
case 'sha3-224':
|
||||
case 'sha3-256':
|
||||
case 'sha3-384':
|
||||
case 'sha3-512':
|
||||
case 'authentihash':
|
||||
case 'ja3-fingerprint-md5':
|
||||
case 'jarm-fingerprint':
|
||||
case 'hassh-md5':
|
||||
case 'hasshserver-md5':
|
||||
case 'x509-fingerprint-md5':
|
||||
case 'x509-fingerprint-sha256':
|
||||
case 'x509-fingerprint-sha1':
|
||||
case 'git-commit-id':
|
||||
if (self::isHashValid($type, $value)) {
|
||||
return true;
|
||||
}
|
||||
$length = self::HASH_HEX_LENGTH[$type];
|
||||
return __('Checksum has an invalid length or format (expected: %s hexadecimal characters). Please double check the value or select type "other".', $length);
|
||||
case 'tlsh':
|
||||
if (self::isTlshValid($value)) {
|
||||
return true;
|
||||
}
|
||||
return __('Checksum has an invalid length or format (expected: at least 35 hexadecimal characters, optionally starting with t1 instead of hexadecimal characters). Please double check the value or select type "other".');
|
||||
case 'telfhash':
|
||||
if (self::isTelfhashValid($value)) {
|
||||
return true;
|
||||
}
|
||||
return __('Checksum has an invalid length or format (expected: %s or %s hexadecimal characters). Please double check the value or select type "other".', 70, 72);
|
||||
case 'pehash':
|
||||
if (self::isHashValid('pehash', $value)) {
|
||||
return true;
|
||||
}
|
||||
return __('The input doesn\'t match the expected sha1 format (expected: 40 hexadecimal characters). Keep in mind that MISP currently only supports SHA1 for PEhashes, if you would like to get the support extended to other hash types, make sure to create a github ticket about it at https://github.com/MISP/MISP!');
|
||||
case 'ssdeep':
|
||||
if (self::isSsdeep($value)) {
|
||||
return true;
|
||||
}
|
||||
return __('Invalid SSDeep hash. The format has to be blocksize:hash:hash');
|
||||
case 'impfuzzy':
|
||||
if (substr_count($value, ':') === 2) {
|
||||
$parts = explode(':', $value);
|
||||
if (self::isPositiveInteger($parts[0])) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return __('Invalid impfuzzy format. The format has to be imports:hash:hash');
|
||||
case 'cdhash':
|
||||
if (preg_match("#^[0-9a-f]{40,}$#", $value)) {
|
||||
return true;
|
||||
}
|
||||
return __('The input doesn\'t match the expected format (expected: 40 or more hexadecimal characters)');
|
||||
case 'http-method':
|
||||
if (preg_match("#(OPTIONS|GET|HEAD|POST|PUT|DELETE|TRACE|CONNECT|PROPFIND|PROPPATCH|MKCOL|COPY|MOVE|LOCK|UNLOCK|VERSION-CONTROL|REPORT|CHECKOUT|CHECKIN|UNCHECKOUT|MKWORKSPACE|UPDATE|LABEL|MERGE|BASELINE-CONTROL|MKACTIVITY|ORDERPATCH|ACL|PATCH|SEARCH)#", $value)) {
|
||||
return true;
|
||||
}
|
||||
return __('Unknown HTTP method.');
|
||||
case 'filename|pehash':
|
||||
// no newline
|
||||
if (preg_match("#^.+\|[0-9a-f]{40}$#", $value)) {
|
||||
return true;
|
||||
}
|
||||
return __('The input doesn\'t match the expected filename|sha1 format (expected: filename|40 hexadecimal characters). Keep in mind that MISP currently only supports SHA1 for PEhashes, if you would like to get the support extended to other hash types, make sure to create a github ticket about it at https://github.com/MISP/MISP!');
|
||||
case 'filename|md5':
|
||||
case 'filename|sha1':
|
||||
case 'filename|imphash':
|
||||
case 'filename|sha224':
|
||||
case 'filename|sha256':
|
||||
case 'filename|sha384':
|
||||
case 'filename|sha512':
|
||||
case 'filename|sha512/224':
|
||||
case 'filename|sha512/256':
|
||||
case 'filename|sha3-224':
|
||||
case 'filename|sha3-256':
|
||||
case 'filename|sha3-384':
|
||||
case 'filename|sha3-512':
|
||||
case 'filename|authentihash':
|
||||
$hashType = substr($type, 9); // strip `filename|`
|
||||
$length = self::HASH_HEX_LENGTH[$hashType];
|
||||
if (preg_match("#^.+\|[0-9a-f]{" . $length . "}$#", $value)) {
|
||||
return true;
|
||||
}
|
||||
return __('Checksum has an invalid length or format (expected: filename|%s hexadecimal characters). Please double check the value or select type "other".', $length);
|
||||
case 'filename|ssdeep':
|
||||
$composite = explode('|', $value);
|
||||
if (strpos($composite[0], "\n") !== false) {
|
||||
return __('Filename must not contain new line character.');
|
||||
}
|
||||
if (self::isSsdeep($composite[1])) {
|
||||
return true;
|
||||
}
|
||||
return __('Invalid ssdeep hash (expected: blocksize:hash:hash).');
|
||||
case 'filename|tlsh':
|
||||
$composite = explode('|', $value);
|
||||
if (strpos($composite[0], "\n") !== false) {
|
||||
return __('Filename must not contain new line character.');
|
||||
}
|
||||
if (self::isTlshValid($composite[1])) {
|
||||
return true;
|
||||
}
|
||||
return __('TLSH hash has an invalid length or format (expected: filename|at least 35 hexadecimal characters, optionally starting with t1 instead of hexadecimal characters). Please double check the value or select type "other".');
|
||||
case 'filename|vhash':
|
||||
if (preg_match('#^.+\|.+$#', $value)) {
|
||||
return true;
|
||||
}
|
||||
return __('Checksum has an invalid length or format (expected: filename|string characters). Please double check the value or select type "other".');
|
||||
case 'ip-src':
|
||||
case 'ip-dst':
|
||||
if (strpos($value, '/') !== false) {
|
||||
$parts = explode("/", $value);
|
||||
if (count($parts) !== 2 || !self::isPositiveInteger($parts[1])) {
|
||||
return __('Invalid CIDR notation value found.');
|
||||
}
|
||||
|
||||
if (filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) {
|
||||
if ($parts[1] > 32) {
|
||||
return __('Invalid CIDR notation value found, for IPv4 must be lower or equal 32.');
|
||||
}
|
||||
} else if (filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) {
|
||||
if ($parts[1] > 128) {
|
||||
return __('Invalid CIDR notation value found, for IPv6 must be lower or equal 128.');
|
||||
}
|
||||
} else {
|
||||
return __('IP address has an invalid format.');
|
||||
}
|
||||
} else if (!filter_var($value, FILTER_VALIDATE_IP)) {
|
||||
return __('IP address has an invalid format.');
|
||||
}
|
||||
return true;
|
||||
case 'port':
|
||||
if (!self::isPortValid($value)) {
|
||||
return __('Port numbers have to be integers between 1 and 65535.');
|
||||
}
|
||||
return true;
|
||||
case 'ip-dst|port':
|
||||
case 'ip-src|port':
|
||||
$parts = explode('|', $value);
|
||||
if (!filter_var($parts[0], FILTER_VALIDATE_IP)) {
|
||||
return __('IP address has an invalid format.');
|
||||
}
|
||||
if (!self::isPortValid($parts[1])) {
|
||||
return __('Port numbers have to be integers between 1 and 65535.');
|
||||
}
|
||||
return true;
|
||||
case 'mac-address':
|
||||
return preg_match('/^([a-fA-F0-9]{2}[:]?){6}$/', $value) === 1;
|
||||
case 'mac-eui-64':
|
||||
return preg_match('/^([a-fA-F0-9]{2}[:]?){8}$/', $value) === 1;
|
||||
case 'hostname':
|
||||
case 'domain':
|
||||
if (self::isDomainValid($value)) {
|
||||
return true;
|
||||
}
|
||||
return __('%s has an invalid format. Please double check the value or select type "other".', ucfirst($type));
|
||||
case 'hostname|port':
|
||||
$parts = explode('|', $value);
|
||||
if (!self::isDomainValid($parts[0])) {
|
||||
return __('Hostname has an invalid format.');
|
||||
}
|
||||
if (!self::isPortValid($parts[1])) {
|
||||
return __('Port numbers have to be integers between 1 and 65535.');
|
||||
}
|
||||
return true;
|
||||
case 'domain|ip':
|
||||
$parts = explode('|', $value);
|
||||
if (!self::isDomainValid($parts[0])) {
|
||||
return __('Domain has an invalid format.');
|
||||
}
|
||||
if (!filter_var($parts[1], FILTER_VALIDATE_IP)) {
|
||||
return __('IP address has an invalid format.');
|
||||
}
|
||||
return true;
|
||||
case 'email':
|
||||
case 'email-src':
|
||||
case 'eppn':
|
||||
case 'email-dst':
|
||||
case 'target-email':
|
||||
case 'whois-registrant-email':
|
||||
case 'dns-soa-email':
|
||||
case 'jabber-id':
|
||||
// we don't use the native function to prevent issues with partial email addresses
|
||||
if (preg_match("#^.[^\s]*\@.*\..*$#i", $value)) {
|
||||
return true;
|
||||
}
|
||||
return __('Email address has an invalid format. Please double check the value or select type "other".');
|
||||
case 'vulnerability':
|
||||
if (preg_match("#^CVE-[0-9]{4}-[0-9]{4,}$#", $value)) {
|
||||
return true;
|
||||
}
|
||||
return __('Invalid format. Expected: CVE-xxxx-xxxx...');
|
||||
case 'weakness':
|
||||
if (preg_match("#^CWE-[0-9]+$#", $value)) {
|
||||
return true;
|
||||
}
|
||||
return __('Invalid format. Expected: CWE-x...');
|
||||
case 'windows-service-name':
|
||||
case 'windows-service-displayname':
|
||||
if (strlen($value) > 256 || preg_match('#[\\\/]#', $value)) {
|
||||
return __('Invalid format. Only values shorter than 256 characters that don\'t include any forward or backward slashes are allowed.');
|
||||
}
|
||||
return true;
|
||||
case 'mutex':
|
||||
case 'process-state':
|
||||
case 'snort':
|
||||
case 'bro':
|
||||
case 'zeek':
|
||||
case 'community-id':
|
||||
case 'anonymised':
|
||||
case 'pattern-in-file':
|
||||
case 'pattern-in-traffic':
|
||||
case 'pattern-in-memory':
|
||||
case 'filename-pattern':
|
||||
case 'pgp-public-key':
|
||||
case 'pgp-private-key':
|
||||
case 'yara':
|
||||
case 'stix2-pattern':
|
||||
case 'sigma':
|
||||
case 'gene':
|
||||
case 'kusto-query':
|
||||
case 'mime-type':
|
||||
case 'identity-card-number':
|
||||
case 'cookie':
|
||||
case 'attachment':
|
||||
case 'malware-sample':
|
||||
case 'comment':
|
||||
case 'text':
|
||||
case 'other':
|
||||
case 'cpe':
|
||||
case 'email-attachment':
|
||||
case 'email-body':
|
||||
case 'email-header':
|
||||
case 'first-name':
|
||||
case 'middle-name':
|
||||
case 'last-name':
|
||||
case 'full-name':
|
||||
return true;
|
||||
case 'link':
|
||||
// Moved to a native function whilst still enforcing the scheme as a requirement
|
||||
return (bool)filter_var($value);
|
||||
case 'hex':
|
||||
return ctype_xdigit($value);
|
||||
case 'target-user':
|
||||
case 'campaign-name':
|
||||
case 'campaign-id':
|
||||
case 'threat-actor':
|
||||
case 'target-machine':
|
||||
case 'target-org':
|
||||
case 'target-location':
|
||||
case 'target-external':
|
||||
case 'email-subject':
|
||||
case 'malware-type':
|
||||
// TODO: review url/uri validation
|
||||
case 'url':
|
||||
case 'uri':
|
||||
case 'user-agent':
|
||||
case 'regkey':
|
||||
case 'regkey|value':
|
||||
case 'filename':
|
||||
case 'pdb':
|
||||
case 'windows-scheduled-task':
|
||||
case 'whois-registrant-name':
|
||||
case 'whois-registrant-org':
|
||||
case 'whois-registrar':
|
||||
case 'whois-creation-date':
|
||||
case 'date-of-birth':
|
||||
case 'place-of-birth':
|
||||
case 'gender':
|
||||
case 'passport-number':
|
||||
case 'passport-country':
|
||||
case 'passport-expiration':
|
||||
case 'redress-number':
|
||||
case 'nationality':
|
||||
case 'visa-number':
|
||||
case 'issue-date-of-the-visa':
|
||||
case 'primary-residence':
|
||||
case 'country-of-residence':
|
||||
case 'special-service-request':
|
||||
case 'frequent-flyer-number':
|
||||
case 'travel-details':
|
||||
case 'payment-details':
|
||||
case 'place-port-of-original-embarkation':
|
||||
case 'place-port-of-clearance':
|
||||
case 'place-port-of-onward-foreign-destination':
|
||||
case 'passenger-name-record-locator-number':
|
||||
case 'email-dst-display-name':
|
||||
case 'email-src-display-name':
|
||||
case 'email-reply-to':
|
||||
case 'email-x-mailer':
|
||||
case 'email-mime-boundary':
|
||||
case 'email-thread-index':
|
||||
case 'email-message-id':
|
||||
case 'github-username':
|
||||
case 'github-repository':
|
||||
case 'github-organisation':
|
||||
case 'twitter-id':
|
||||
case 'dkim':
|
||||
case 'dkim-signature':
|
||||
case 'favicon-mmh3':
|
||||
case 'chrome-extension-id':
|
||||
case 'mobile-application-id':
|
||||
case 'azure-application-id':
|
||||
case 'named pipe':
|
||||
if (strpos($value, "\n") !== false) {
|
||||
return __('Value must not contain new line character.');
|
||||
}
|
||||
return true;
|
||||
case 'ssh-fingerprint':
|
||||
if (self::isSshFingerprint($value)) {
|
||||
return true;
|
||||
}
|
||||
return __('SSH fingerprint must be in MD5 or SHA256 format.');
|
||||
case 'datetime':
|
||||
if (strtotime($value) !== false) {
|
||||
return true;
|
||||
}
|
||||
return __('Datetime has to be in the ISO 8601 format.');
|
||||
case 'size-in-bytes':
|
||||
case 'counter':
|
||||
if (self::isPositiveInteger($value)) {
|
||||
return true;
|
||||
}
|
||||
return __('The value has to be a whole number greater or equal 0.');
|
||||
/* case 'targeted-threat-index':
|
||||
if (!is_numeric($value) || $value < 0 || $value > 10) {
|
||||
return __('The value has to be a number between 0 and 10.');
|
||||
}
|
||||
return true;*/
|
||||
case 'iban':
|
||||
case 'bic':
|
||||
case 'btc':
|
||||
case 'dash':
|
||||
case 'xmr':
|
||||
return preg_match('/^[a-zA-Z0-9]+$/', $value) === 1;
|
||||
case 'vhash':
|
||||
return preg_match('/^.+$/', $value) === 1;
|
||||
case 'bin':
|
||||
case 'cc-number':
|
||||
case 'bank-account-nr':
|
||||
case 'aba-rtn':
|
||||
case 'prtn':
|
||||
case 'phone-number':
|
||||
case 'whois-registrant-phone':
|
||||
case 'float':
|
||||
return is_numeric($value);
|
||||
case 'cortex':
|
||||
return JsonTool::isValid($value);
|
||||
case 'boolean':
|
||||
return $value == 1 || $value == 0;
|
||||
case 'AS':
|
||||
if (self::isPositiveInteger($value) && $value <= 4294967295) {
|
||||
return true;
|
||||
}
|
||||
return __('AS number have to be integer between 1 and 4294967295');
|
||||
}
|
||||
throw new InvalidArgumentException("Unknown type $type.");
|
||||
}
|
||||
|
||||
/**
|
||||
* This method will generate all valid types for given value.
|
||||
* @param array $types Typos to check
|
||||
* @param array $compositeTypes Composite types
|
||||
* @param string $value Values to check
|
||||
* @return array
|
||||
*/
|
||||
public static function validTypesForValue(array $types, array $compositeTypes, $value)
|
||||
{
|
||||
$possibleTypes = [];
|
||||
foreach ($types as $type) {
|
||||
if (in_array($type, $compositeTypes, true) && substr_count($value, '|') !== 1) {
|
||||
continue; // value is not in composite format
|
||||
}
|
||||
$modifiedValue = AttributeValidationTool::modifyBeforeValidation($type, $value);
|
||||
if (AttributeValidationTool::validate($type, $modifiedValue) === true) {
|
||||
$possibleTypes[] = $type;
|
||||
}
|
||||
}
|
||||
return $possibleTypes;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $value
|
||||
* @return bool
|
||||
*/
|
||||
private static function isDomainValid($value)
|
||||
{
|
||||
return preg_match("#^[A-Z0-9.\-_]+\.[A-Z0-9\-]{2,}$#i", $value) === 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $value
|
||||
* @return bool
|
||||
*/
|
||||
private static function isPortValid($value)
|
||||
{
|
||||
return self::isPositiveInteger($value) && $value >= 1 && $value <= 65535;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $value
|
||||
* @return bool
|
||||
*/
|
||||
private static function isTlshValid($value)
|
||||
{
|
||||
if ($value[0] === 't') {
|
||||
$value = substr($value, 1);
|
||||
}
|
||||
return strlen($value) > 35 && ctype_xdigit($value);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $value
|
||||
* @return bool
|
||||
*/
|
||||
private static function isTelfhashValid($value)
|
||||
{
|
||||
return strlen($value) == 70 || strlen($value) == 72;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @param string $type
|
||||
* @param string $value
|
||||
* @return bool
|
||||
*/
|
||||
private static function isHashValid($type, $value)
|
||||
{
|
||||
return strlen($value) === self::HASH_HEX_LENGTH[$type] && ctype_xdigit($value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if input value is positive integer or zero.
|
||||
* @param int|string $value
|
||||
* @return bool
|
||||
*/
|
||||
private static function isPositiveInteger($value)
|
||||
{
|
||||
return (is_int($value) && $value >= 0) || ctype_digit($value);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $value
|
||||
* @return bool
|
||||
*/
|
||||
private static function isSsdeep($value)
|
||||
{
|
||||
return preg_match('#^([0-9]+):([0-9a-zA-Z/+]*):([0-9a-zA-Z/+]*)$#', $value);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $value
|
||||
* @return bool
|
||||
*/
|
||||
private static function isSshFingerprint($value)
|
||||
{
|
||||
if (substr($value, 0, 7) === 'SHA256:') {
|
||||
$value = substr($value, 7);
|
||||
$decoded = base64_decode($value, true);
|
||||
return $decoded && strlen($decoded) === 32;
|
||||
} else if (substr($value, 0, 4) === 'MD5:') {
|
||||
$value = substr($value, 4);
|
||||
}
|
||||
|
||||
$value = str_replace(':', '', $value);
|
||||
return self::isHashValid('md5', $value);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $value
|
||||
* @return string
|
||||
*/
|
||||
private static function normalizeIp($value)
|
||||
{
|
||||
// If IP is a CIDR
|
||||
if (strpos($value, '/')) {
|
||||
list($ip, $range) = explode('/', $value, 2);
|
||||
|
||||
// Compress IPv6
|
||||
if (strpos($ip, ':') && $converted = inet_pton($ip)) {
|
||||
$ip = inet_ntop($converted);
|
||||
}
|
||||
|
||||
// If IP is in CIDR format, but the network is 32 for IPv4 or 128 for IPv6, normalize to non CIDR type
|
||||
if (($range === '32' && strpos($value, '.')) || ($range === '128' && strpos($value, ':'))) {
|
||||
return $ip;
|
||||
}
|
||||
|
||||
return "$ip/$range";
|
||||
}
|
||||
|
||||
// Compress IPv6
|
||||
if (strpos($value, ':') && $converted = inet_pton($value)) {
|
||||
return inet_ntop($converted);
|
||||
}
|
||||
|
||||
return $value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Temporary solution for utf8 columns until we migrate to utf8mb4.
|
||||
* via https://stackoverflow.com/questions/16496554/can-php-detect-4-byte-encoded-utf8-chars
|
||||
* @param string $input
|
||||
* @return string
|
||||
*/
|
||||
private static function handle4ByteUnicode($input)
|
||||
{
|
||||
return preg_replace(
|
||||
'%(?:
|
||||
\xF0[\x90-\xBF][\x80-\xBF]{2}
|
||||
| [\xF1-\xF3][\x80-\xBF]{3}
|
||||
| \xF4[\x80-\x8F][\x80-\xBF]{2}
|
||||
)%xs',
|
||||
'?',
|
||||
$input
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,671 @@
|
|||
<?php
|
||||
|
||||
namespace App\Lib\Tools;
|
||||
|
||||
use Exception;
|
||||
|
||||
class ComplexTypeTool
|
||||
{
|
||||
const REFANG_REGEX_TABLE = array(
|
||||
array(
|
||||
'from' => '/^(hxxp|hxtp|htxp|meow|h\[tt\]p)/i',
|
||||
'to' => 'http',
|
||||
'types' => array('link', 'url')
|
||||
),
|
||||
array(
|
||||
'from' => '/(\[\.\]|\[dot\]|\(dot\))/',
|
||||
'to' => '.',
|
||||
'types' => array('link', 'url', 'ip-dst', 'ip-src', 'domain|ip', 'domain', 'hostname')
|
||||
),
|
||||
array(
|
||||
'from' => '/\[hxxp:\/\/\]/',
|
||||
'to' => 'http://',
|
||||
'types' => array('link', 'url')
|
||||
),
|
||||
array(
|
||||
'from' => '/[\@]|\[at\]/',
|
||||
'to' => '@',
|
||||
'types' => array('email', 'email-src', 'email-dst')
|
||||
),
|
||||
array(
|
||||
'from' => '/\[:\]/',
|
||||
'to' => ':',
|
||||
'types' => array('url', 'link')
|
||||
)
|
||||
);
|
||||
|
||||
const HEX_HASH_TYPES = [
|
||||
32 => ['single' => ['md5', 'imphash', 'x509-fingerprint-md5', 'ja3-fingerprint-md5'], 'composite' => ['filename|md5', 'filename|imphash']],
|
||||
40 => ['single' => ['sha1', 'pehash', 'x509-fingerprint-sha1', 'cdhash'], 'composite' => ['filename|sha1', 'filename|pehash']],
|
||||
56 => ['single' => ['sha224', 'sha512/224'], 'composite' => ['filename|sha224', 'filename|sha512/224']],
|
||||
64 => ['single' => ['sha256', 'authentihash', 'sha512/256', 'x509-fingerprint-sha256'], 'composite' => ['filename|sha256', 'filename|authentihash', 'filename|sha512/256']],
|
||||
96 => ['single' => ['sha384'], 'composite' => ['filename|sha384']],
|
||||
128 => ['single' => ['sha512'], 'composite' => ['filename|sha512']],
|
||||
];
|
||||
|
||||
private $__tlds;
|
||||
|
||||
/**
|
||||
* Hardcoded list if properly warninglist is not available
|
||||
* @var string[]
|
||||
*/
|
||||
private $securityVendorDomains = ['virustotal.com', 'hybrid-analysis.com'];
|
||||
|
||||
public static function refangValue($value, $type)
|
||||
{
|
||||
foreach (self::REFANG_REGEX_TABLE as $regex) {
|
||||
if (in_array($type, $regex['types'], true)) {
|
||||
$value = preg_replace($regex['from'], $regex['to'], $value);
|
||||
}
|
||||
}
|
||||
return $value;
|
||||
}
|
||||
|
||||
public function setTLDs($tlds = array())
|
||||
{
|
||||
$this->__tlds = [];
|
||||
foreach ($tlds as $tld) {
|
||||
$this->__tlds[$tld] = true;
|
||||
}
|
||||
}
|
||||
|
||||
public function setSecurityVendorDomains(array $securityVendorDomains)
|
||||
{
|
||||
if (empty($securityVendorDomains)) {
|
||||
return; // if provided warninglist is empty, keep hardcoded domains
|
||||
}
|
||||
$this->securityVendorDomains = $securityVendorDomains;
|
||||
}
|
||||
|
||||
public function checkComplexRouter($input, $type, $settings = array())
|
||||
{
|
||||
switch ($type) {
|
||||
case 'File':
|
||||
return $this->checkComplexFile($input);
|
||||
case 'CnC':
|
||||
return $this->checkComplexCnC($input);
|
||||
case 'freetext':
|
||||
case 'FreeText':
|
||||
return $this->checkFreeText($input, $settings);
|
||||
case 'csv':
|
||||
return $this->checkCSV($input, $settings);
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// checks if the passed input matches a valid file description attribute's pattern (filename, md5, sha1, sha256, filename|md5, filename|sha1, filename|sha256)
|
||||
public function checkComplexFile($input)
|
||||
{
|
||||
$original = $input;
|
||||
$type = '';
|
||||
$composite = false;
|
||||
if (strpos($input, '|')) {
|
||||
$composite = true;
|
||||
$result = explode('|', $input);
|
||||
if (count($result) != 2 || !preg_match("#^.+#", $result[0])) {
|
||||
$type = 'other';
|
||||
} else {
|
||||
$type = 'filename|';
|
||||
}
|
||||
$input = $result[1];
|
||||
}
|
||||
if (strlen($input) == 32 && preg_match("#[0-9a-f]{32}$#", $input)) {
|
||||
$type .= 'md5';
|
||||
}
|
||||
if (strlen($input) == 40 && preg_match("#[0-9a-f]{40}$#", $input)) {
|
||||
$type .= 'sha1';
|
||||
}
|
||||
if (strlen($input) == 64 && preg_match("#[0-9a-f]{64}$#", $input)) {
|
||||
$type .= 'sha256';
|
||||
}
|
||||
if ($type == '' && !$composite && preg_match("#^.+#", $input)) {
|
||||
$type = 'filename';
|
||||
}
|
||||
if ($type == '') {
|
||||
$type = 'other';
|
||||
}
|
||||
return array('type' => $type, 'value' => $original);
|
||||
}
|
||||
|
||||
public function checkComplexCnC($input)
|
||||
{
|
||||
$toReturn = array();
|
||||
// check if it's an IP address
|
||||
if (filter_var($input, FILTER_VALIDATE_IP)) {
|
||||
return array('type' => 'ip-dst', 'value' => $input);
|
||||
}
|
||||
if (preg_match("#^[A-Z0-9.-]+\.[A-Z]{2,4}$#i", $input)) {
|
||||
$result = explode('.', $input);
|
||||
if (count($result) > 2) {
|
||||
$toReturn['multi'][] = array('type' => 'hostname', 'value' => $input);
|
||||
$pos = strpos($input, '.');
|
||||
$toReturn['multi'][] = array('type' => 'domain', 'value' => substr($input, (1 + $pos)));
|
||||
return $toReturn;
|
||||
}
|
||||
return array('type' => 'domain', 'value' => $input);
|
||||
}
|
||||
|
||||
if (!preg_match("#\n#", $input)) {
|
||||
return array('type' => 'url', 'value' => $input);
|
||||
}
|
||||
return array('type' => 'other', 'value' => $input);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a CSV file with the given settings
|
||||
* All lines starting with # are stripped
|
||||
* The settings can contain the following:
|
||||
* delimiter: Expects a delimiter string (default is a simple comma).
|
||||
* For example, to split the following line: "value1##comma##value2" simply pass $settings['delimiter'] = "##comma##";
|
||||
* values: Expects an array (or a comma separated string) with numeric values denoting the columns containing indicators. If this is not set then every value will be checked. (column numbers start at 1)
|
||||
* @param string $input
|
||||
* @param array $settings
|
||||
* @return array
|
||||
* @throws Exception
|
||||
*/
|
||||
public function checkCSV($input, $settings = array())
|
||||
{
|
||||
if (empty($input)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$delimiter = !empty($settings['delimiter']) ? $settings['delimiter'] : ",";
|
||||
if ($delimiter === '\t') {
|
||||
$delimiter = "\t";
|
||||
}
|
||||
$values = !empty($settings['value']) ? $settings['value'] : array();
|
||||
if (!is_array($values)) {
|
||||
$values = explode(',', $values);
|
||||
}
|
||||
foreach ($values as $key => $value) {
|
||||
$values[$key] = intval($value);
|
||||
}
|
||||
|
||||
// Write to tmp file to save memory
|
||||
$tmpFile = new TmpFileTool();
|
||||
$tmpFile->write($input);
|
||||
unset($input);
|
||||
|
||||
$iocArray = [];
|
||||
foreach ($tmpFile->intoParsedCsv($delimiter) as $row) {
|
||||
if (!empty($row[0][0]) && $row[0][0] === '#') { // Comment
|
||||
continue;
|
||||
}
|
||||
foreach ($row as $elementPos => $element) {
|
||||
if (empty($element)) {
|
||||
continue;
|
||||
}
|
||||
if (empty($values) || in_array(($elementPos + 1), $values)) {
|
||||
$element = trim($element, " \t\n\r\0\x0B\"\'");
|
||||
if (empty($element)) {
|
||||
continue;
|
||||
}
|
||||
if (!empty($settings['excluderegex']) && preg_match($settings['excluderegex'], $element)) {
|
||||
continue;
|
||||
}
|
||||
$resolvedResult = $this->__resolveType($element);
|
||||
// Do not extract datetime from CSV
|
||||
if ($resolvedResult) {
|
||||
$iocArray[] = $resolvedResult;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $iocArray;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $input
|
||||
* @param array $settings
|
||||
* @return array
|
||||
*/
|
||||
public function checkFreeText($input, array $settings = [])
|
||||
{
|
||||
if (empty($input)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if ($input[0] === '{') {
|
||||
// If input looks like JSON, try to parse it as JSON
|
||||
try {
|
||||
return $this->parseJson($input, $settings);
|
||||
} catch (Exception $e) {
|
||||
}
|
||||
}
|
||||
|
||||
$iocArray = $this->parseFreetext($input);
|
||||
|
||||
$resultArray = [];
|
||||
foreach ($iocArray as $ioc) {
|
||||
$ioc = trim($ioc, '\'".,() ' . "\t\n\r\0\x0B"); // custom + default PHP trim
|
||||
if (empty($ioc)) {
|
||||
continue;
|
||||
}
|
||||
if (!empty($settings['excluderegex']) && preg_match($settings['excluderegex'], $ioc)) {
|
||||
continue;
|
||||
}
|
||||
$typeArray = $this->__resolveType($ioc);
|
||||
if ($typeArray === false) {
|
||||
continue;
|
||||
}
|
||||
// Remove duplicates
|
||||
if (isset($resultArray[$typeArray['value']])) {
|
||||
continue;
|
||||
}
|
||||
$typeArray['original_value'] = $ioc;
|
||||
$resultArray[$typeArray['value']] = $typeArray;
|
||||
}
|
||||
return array_values($resultArray);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $input
|
||||
* @throws JsonException
|
||||
*/
|
||||
private function parseJson($input, array $settings)
|
||||
{
|
||||
$parsed = JsonTool::decode($input);
|
||||
|
||||
$values = [];
|
||||
array_walk_recursive($parsed, function ($value) use (&$values) {
|
||||
if (is_bool($value) || is_int($value) || empty($value)) {
|
||||
return; // skip boolean, integer or empty values
|
||||
}
|
||||
|
||||
$values[] = $value;
|
||||
foreach ($this->parseFreetext($value) as $v) {
|
||||
if ($v !== $value) {
|
||||
$values[] = $v;
|
||||
}
|
||||
}
|
||||
});
|
||||
unset($parsed);
|
||||
|
||||
$resultArray = [];
|
||||
foreach ($values as $ioc) {
|
||||
$ioc = trim($ioc, '\'".,() ' . "\t\n\r\0\x0B"); // custom + default PHP trim
|
||||
if (empty($ioc)) {
|
||||
continue;
|
||||
}
|
||||
if (!empty($settings['excluderegex']) && preg_match($settings['excluderegex'], $ioc)) {
|
||||
continue;
|
||||
}
|
||||
$typeArray = $this->__resolveType($ioc);
|
||||
if ($typeArray === false) {
|
||||
continue;
|
||||
}
|
||||
// Remove duplicates
|
||||
if (isset($resultArray[$typeArray['value']])) {
|
||||
continue;
|
||||
}
|
||||
$typeArray['original_value'] = $ioc;
|
||||
$resultArray[$typeArray['value']] = $typeArray;
|
||||
}
|
||||
return array_values($resultArray);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $input
|
||||
* @return array|string[]
|
||||
*/
|
||||
private function parseFreetext($input)
|
||||
{
|
||||
$input = str_replace("\xc2\xa0", ' ', $input); // non breaking space to normal space
|
||||
$input = preg_replace('/\p{C}+/u', ' ', $input);
|
||||
$iocArray = preg_split("/\r\n|\n|\r|\s|\s+|,|\<|\>|;/", $input);
|
||||
|
||||
preg_match_all('/\"([^\"]+)\"/', $input, $matches);
|
||||
foreach ($matches[1] as $match) {
|
||||
$iocArray[] = $match;
|
||||
}
|
||||
return $iocArray;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $raw_input Trimmed value
|
||||
* @return array|false
|
||||
*/
|
||||
private function __resolveType($raw_input)
|
||||
{
|
||||
// Check if value is clean IP without doing expensive operations.
|
||||
if (filter_var($raw_input, FILTER_VALIDATE_IP)) {
|
||||
return [
|
||||
'types' => ['ip-dst', 'ip-src', 'ip-src/ip-dst'],
|
||||
'default_type' => 'ip-dst',
|
||||
'value' => $raw_input,
|
||||
];
|
||||
}
|
||||
|
||||
$input = ['raw' => $raw_input];
|
||||
|
||||
// Check hashes before refang and port extracting, it is not necessary for hashes. This speedups parsing
|
||||
// freetexts or CSVs with a lot of hashes.
|
||||
if ($result = $this->__checkForHashes($input)) {
|
||||
return $result;
|
||||
}
|
||||
|
||||
$input = $this->__refangInput($input);
|
||||
|
||||
// Check email before port extracting, it is not necessary for email. This speedups parsing
|
||||
// freetexts or CSVs with a lot of emails.
|
||||
if ($result = $this->__checkForEmail($input)) {
|
||||
return $result;
|
||||
}
|
||||
|
||||
$input = $this->__extractPort($input);
|
||||
if ($result = $this->__checkForIP($input)) {
|
||||
return $result;
|
||||
}
|
||||
if ($result = $this->__checkForDomainOrFilename($input)) {
|
||||
return $result;
|
||||
}
|
||||
if ($result = $this->__checkForSimpleRegex($input)) {
|
||||
return $result;
|
||||
}
|
||||
if ($result = $this->__checkForAS($input)) {
|
||||
return $result;
|
||||
}
|
||||
if ($result = $this->__checkForBTC($input)) {
|
||||
return $result;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private function __checkForBTC($input)
|
||||
{
|
||||
if (preg_match("#^([13][a-km-zA-HJ-NP-Z1-9]{25,34})|(bc|tb)1([023456789acdefghjklmnpqrstuvwxyz]{11,71})$#i", $input['raw'])) {
|
||||
return [
|
||||
'types' => ['btc'],
|
||||
'default_type' => 'btc',
|
||||
'value' => $input['raw'],
|
||||
];
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private function __checkForEmail($input)
|
||||
{
|
||||
// quick filter for an @ to see if we should validate a potential e-mail address
|
||||
if (strpos($input['refanged'], '@') !== false) {
|
||||
if (filter_var($input['refanged'], FILTER_VALIDATE_EMAIL)) {
|
||||
return [
|
||||
'types' => array('email', 'email-src', 'email-dst', 'target-email', 'whois-registrant-email'),
|
||||
'default_type' => 'email-src',
|
||||
'value' => $input['refanged'],
|
||||
];
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private function __checkForAS($input)
|
||||
{
|
||||
if (preg_match('#^as[0-9]+$#i', $input['raw'])) {
|
||||
$input['raw'] = strtoupper($input['raw']);
|
||||
return array('types' => array('AS'), 'default_type' => 'AS', 'value' => $input['raw']);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private function __checkForHashes($input)
|
||||
{
|
||||
// handle prepared composite values with the filename|hash format
|
||||
if (strpos($input['raw'], '|')) {
|
||||
$compositeParts = explode('|', $input['raw']);
|
||||
if (count($compositeParts) === 2) {
|
||||
if ($this->__resolveFilename($compositeParts[0])) {
|
||||
$hash = $this->__resolveHash($compositeParts[1]);
|
||||
if ($hash) {
|
||||
return array('types' => $hash['composite'], 'default_type' => $hash['composite'][0], 'value' => $input['raw']);
|
||||
}
|
||||
if ($this->__resolveSsdeep($compositeParts[1])) {
|
||||
return array('types' => array('filename|ssdeep'), 'default_type' => 'filename|ssdeep', 'value' => $input['raw']);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// check for hashes
|
||||
$hash = $this->__resolveHash($input['raw']);
|
||||
if ($hash) {
|
||||
$types = $hash['single'];
|
||||
if ($this->__checkForBTC($input)) {
|
||||
$types[] = 'btc';
|
||||
}
|
||||
return array('types' => $types, 'default_type' => $types[0], 'value' => $input['raw']);
|
||||
}
|
||||
// ssdeep has a different pattern
|
||||
if ($this->__resolveSsdeep($input['raw'])) {
|
||||
return array('types' => array('ssdeep'), 'default_type' => 'ssdeep', 'value' => $input['raw']);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private function __extractPort($input)
|
||||
{
|
||||
// note down and remove the port if it's a url / domain name / hostname / ip
|
||||
// input2 from here on is the variable containing the original input with the port removed. It is only used by url / domain name / hostname / ip
|
||||
if (preg_match('/(:[0-9]{2,5})$/', $input['refanged'], $port)) {
|
||||
$input['comment'] = 'On port ' . substr($port[0], 1);
|
||||
$input['refanged_no_port'] = str_replace($port[0], '', $input['refanged']);
|
||||
$input['port'] = substr($port[0], 1);
|
||||
} else {
|
||||
$input['comment'] = false;
|
||||
$input['refanged_no_port'] = $input['refanged'];
|
||||
}
|
||||
return $input;
|
||||
}
|
||||
|
||||
private function __refangInput($input)
|
||||
{
|
||||
$refanged = $input['raw'];
|
||||
foreach (self::REFANG_REGEX_TABLE as $regex) {
|
||||
$refanged = preg_replace($regex['from'], $regex['to'], $refanged);
|
||||
}
|
||||
$refanged = rtrim($refanged, ".");
|
||||
$input['refanged'] = preg_replace_callback(
|
||||
'/\[.\]/',
|
||||
function ($matches) {
|
||||
return trim($matches[0], '[]');
|
||||
},
|
||||
$refanged
|
||||
);
|
||||
return $input;
|
||||
}
|
||||
|
||||
private function __checkForSimpleRegex($input)
|
||||
{
|
||||
// CVE numbers
|
||||
if (preg_match("#^cve-[0-9]{4}-[0-9]{4,9}$#i", $input['raw'])) {
|
||||
return [
|
||||
'types' => ['vulnerability'],
|
||||
'default_type' => 'vulnerability',
|
||||
'value' => strtoupper($input['raw']), // 'CVE' must be uppercase
|
||||
];
|
||||
}
|
||||
|
||||
// Phone numbers - for automatic recognition, needs to start with + or include dashes
|
||||
if ($input['raw'][0] === '+' || strpos($input['raw'], '-')) {
|
||||
if (!preg_match('#^[0-9]{4}-[0-9]{2}-[0-9]{2}$#i', $input['raw']) && preg_match("#^(\+)?([0-9]{1,3}(\(0\))?)?[0-9\/\-]{5,}[0-9]$#i", $input['raw'])) {
|
||||
return array('types' => array('phone-number', 'prtn', 'whois-registrant-phone'), 'default_type' => 'phone-number', 'value' => $input['raw']);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private function __checkForIP(array $input)
|
||||
{
|
||||
if (filter_var($input['refanged_no_port'], FILTER_VALIDATE_IP)) {
|
||||
if (isset($input['port'])) {
|
||||
return array('types' => array('ip-dst|port', 'ip-src|port', 'ip-src|port/ip-dst|port'), 'default_type' => 'ip-dst|port', 'comment' => $input['comment'], 'value' => $input['refanged_no_port'] . '|' . $input['port']);
|
||||
} else {
|
||||
return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
|
||||
}
|
||||
}
|
||||
// IPv6 address that is considered as IP address with port
|
||||
if (filter_var($input['refanged'], FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) {
|
||||
return [
|
||||
'types' => ['ip-dst', 'ip-src', 'ip-src/ip-dst'],
|
||||
'default_type' => 'ip-dst',
|
||||
'comment' => '',
|
||||
'value' => $input['refanged'],
|
||||
];
|
||||
}
|
||||
// IPv6 with port in `[1fff:0:a88:85a3::ac1f]:8001` format
|
||||
if (
|
||||
isset($input['port']) &&
|
||||
!empty($input['refanged_no_port']) &&
|
||||
$input['refanged_no_port'][0] === '[' &&
|
||||
filter_var(substr($input['refanged_no_port'], 1, -1), FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)
|
||||
) {
|
||||
$value = substr($input['refanged_no_port'], 1, -1); // remove brackets
|
||||
return [
|
||||
'types' => ['ip-dst|port', 'ip-src|port', 'ip-src|port/ip-dst|port'],
|
||||
'default_type' => 'ip-dst|port',
|
||||
'comment' => $input['comment'],
|
||||
'value' => "$value|{$input['port']}",
|
||||
];
|
||||
}
|
||||
// it could still be a CIDR block
|
||||
if (strpos($input['refanged_no_port'], '/')) {
|
||||
$temp = explode('/', $input['refanged_no_port']);
|
||||
if (count($temp) === 2 && filter_var($temp[0], FILTER_VALIDATE_IP) && is_numeric($temp[1])) {
|
||||
return array('types' => array('ip-dst', 'ip-src', 'ip-src/ip-dst'), 'default_type' => 'ip-dst', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private function __checkForDomainOrFilename(array $input)
|
||||
{
|
||||
if (strpos($input['refanged_no_port'], '.') !== false) {
|
||||
$temp = explode('.', $input['refanged_no_port']);
|
||||
$domainDetection = true;
|
||||
if (preg_match('/^([-\pL\pN]+\.)+[a-z0-9-]+$/iu', $input['refanged_no_port'])) {
|
||||
if (!$this->isTld(end($temp))) {
|
||||
$domainDetection = false;
|
||||
}
|
||||
} else {
|
||||
$domainDetection = false;
|
||||
}
|
||||
if ($domainDetection) {
|
||||
if (count($temp) > 2) {
|
||||
return array('types' => array('hostname', 'domain', 'url', 'filename'), 'default_type' => 'hostname', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
|
||||
} else {
|
||||
return array('types' => array('domain', 'filename'), 'default_type' => 'domain', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
|
||||
}
|
||||
} else {
|
||||
// check if it is a URL
|
||||
// Adding http:// infront of the input in case it was left off. github.com/MISP/MISP should still be counted as a valid link
|
||||
if (count($temp) > 1 && (filter_var($input['refanged_no_port'], FILTER_VALIDATE_URL) || filter_var('http://' . $input['refanged_no_port'], FILTER_VALIDATE_URL))) {
|
||||
// Even though some domains are valid, we want to exclude them as they are known security vendors / etc
|
||||
if ($this->isLink($input['refanged_no_port'])) {
|
||||
return array('types' => array('link'), 'default_type' => 'link', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
|
||||
}
|
||||
if (strpos($input['refanged_no_port'], '/')) {
|
||||
return array('types' => array('url'), 'default_type' => 'url', 'comment' => $input['comment'], 'value' => $input['refanged_no_port']);
|
||||
}
|
||||
}
|
||||
if ($this->__resolveFilename($input['raw'])) {
|
||||
return array('types' => array('filename'), 'default_type' => 'filename', 'value' => $input['raw']);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (strpos($input['raw'], '\\') !== false) {
|
||||
$temp = explode('\\', $input['raw']);
|
||||
if (strpos(end($temp), '.') || preg_match('/^.:/i', $temp[0])) {
|
||||
if ($this->__resolveFilename(end($temp))) {
|
||||
return array('types' => array('filename'), 'default_type' => 'filename', 'value' => $input['raw']);
|
||||
}
|
||||
} else if (!empty($temp[0])) {
|
||||
return array('types' => array('regkey'), 'default_type' => 'regkey', 'value' => $input['raw']);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private function __resolveFilename($param)
|
||||
{
|
||||
if ((preg_match('/^.:/', $param) || strpos($param, '.') != 0)) {
|
||||
$parts = explode('.', $param);
|
||||
if (!is_numeric(end($parts)) && ctype_alnum(end($parts))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $value
|
||||
* @return bool
|
||||
*/
|
||||
private function __resolveSsdeep($value)
|
||||
{
|
||||
return preg_match('#^[0-9]+:[0-9a-zA-Z/+]+:[0-9a-zA-Z/+]+$#', $value) && !preg_match('#^[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}$#', $value);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $value
|
||||
* @return bool|string[][]
|
||||
*/
|
||||
private function __resolveHash($value)
|
||||
{
|
||||
$strlen = strlen($value);
|
||||
if (isset(self::HEX_HASH_TYPES[$strlen]) && ctype_xdigit($value)) {
|
||||
return self::HEX_HASH_TYPES[$strlen];
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $tld
|
||||
* @return bool
|
||||
*/
|
||||
private function isTld($tld)
|
||||
{
|
||||
if ($this->__tlds === null) {
|
||||
$this->setTLDs($this->__generateTLDList());
|
||||
}
|
||||
return isset($this->__tlds[strtolower($tld)]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if URL should be considered as link attribute type
|
||||
* @param string $value
|
||||
* @return bool
|
||||
*/
|
||||
private function isLink($value)
|
||||
{
|
||||
if (!preg_match('/^https:\/\/([^\/]*)/i', $value, $matches)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$domainToCheck = '';
|
||||
$domainParts = array_reverse(explode('.', strtolower($matches[1])));
|
||||
foreach ($domainParts as $domainPart) {
|
||||
$domainToCheck = $domainPart . $domainToCheck;
|
||||
if (in_array($domainToCheck, $this->securityVendorDomains, true)) {
|
||||
return true;
|
||||
}
|
||||
$domainToCheck = '.' . $domainToCheck;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private function __generateTLDList()
|
||||
{
|
||||
$tlds = array('biz', 'cat', 'com', 'edu', 'gov', 'int', 'mil', 'net', 'org', 'pro', 'tel', 'aero', 'arpa', 'asia', 'coop', 'info', 'jobs', 'mobi', 'name', 'museum', 'travel', 'onion');
|
||||
$char1 = $char2 = 'a';
|
||||
for ($i = 0; $i < 26; $i++) {
|
||||
for ($j = 0; $j < 26; $j++) {
|
||||
$tlds[] = $char1 . $char2;
|
||||
$char2++;
|
||||
}
|
||||
$char1++;
|
||||
$char2 = 'a';
|
||||
}
|
||||
return $tlds;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Lib\Tools;
|
||||
|
||||
use InvalidArgumentException;
|
||||
use LogicException;
|
||||
|
||||
class RandomTool
|
||||
{
|
||||
/**
|
||||
* Generate a random string
|
||||
*
|
||||
* Generate a random string, using a cryptographically secure
|
||||
* pseudorandom number generator (random_int)
|
||||
*
|
||||
* For PHP 7, random_int is a PHP core function
|
||||
* For PHP 5.x, depends on https://github.com/paragonie/random_compat
|
||||
*
|
||||
* @link https://paragonie.com/b/JvICXzh_jhLyt4y3
|
||||
*
|
||||
* @param bool $crypto_secure - If a cryptographically secure or a fast random number generator should be used
|
||||
* @param int $length - How long should our random string be?
|
||||
* @param string $charset - A string of all possible characters to choose from
|
||||
* @return string
|
||||
* @throws Exception
|
||||
*/
|
||||
public static function random_str($crypto_secure = true, $length = 32, $charset = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ')
|
||||
{
|
||||
// Type checks:
|
||||
if (!is_bool($crypto_secure)) {
|
||||
throw new InvalidArgumentException('random_str - Argument 1 - expected a boolean');
|
||||
}
|
||||
if (!is_numeric($length)) {
|
||||
throw new InvalidArgumentException('random_str - Argument 2 - expected an integer');
|
||||
}
|
||||
if (!is_string($charset)) {
|
||||
throw new InvalidArgumentException('random_str - Argument 3 - expected a string');
|
||||
}
|
||||
|
||||
if ($length < 1) {
|
||||
// Just return an empty string. Any value < 1 is meaningless.
|
||||
return '';
|
||||
}
|
||||
|
||||
// Remove duplicate characters from $charset
|
||||
$charset = count_chars($charset, 3);
|
||||
|
||||
// This is the maximum index for all of the characters in the string $charset
|
||||
$charset_max = strlen($charset) - 1;
|
||||
if ($charset_max < 1) {
|
||||
// Avoid letting users do: random_str($int, 'a'); -> 'aaaaa...'
|
||||
throw new LogicException('random_str - Argument 3 - expected a string that contains at least 2 distinct characters');
|
||||
}
|
||||
// Now that we have good data, this is the meat of our function:
|
||||
$random_str = '';
|
||||
for ($i = 0; $i < $length; ++$i) {
|
||||
$r = $crypto_secure ? random_int(0, $charset_max) : mt_rand(0, $charset_max);
|
||||
$random_str .= $charset[$r];
|
||||
}
|
||||
return $random_str;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
<?php
|
||||
|
||||
namespace App\Model\Entity;
|
||||
|
||||
use App\Model\Entity\AppModel;
|
||||
|
||||
class Attribute extends AppModel
|
||||
{
|
||||
public const EDITABLE_FIELDS = [
|
||||
'timestamp',
|
||||
'category',
|
||||
'type',
|
||||
'value',
|
||||
'value1',
|
||||
'value2',
|
||||
'to_ids',
|
||||
'comment',
|
||||
'distribution',
|
||||
'sharing_group_id',
|
||||
'deleted',
|
||||
'disable_correlation',
|
||||
'first_seen',
|
||||
'last_seen',
|
||||
];
|
||||
|
||||
// if these then a category may have upload to be zipped
|
||||
public const ZIPPED_DEFINITION = ['malware-sample'];
|
||||
|
||||
// if these then a category may have upload
|
||||
public const UPLOAD_DEFINITIONS = ['attachment'];
|
||||
|
||||
// skip Correlation for the following types
|
||||
public const NON_CORRELATING_TYPES = [
|
||||
'comment',
|
||||
'http-method',
|
||||
'aba-rtn',
|
||||
'gender',
|
||||
'counter',
|
||||
'float',
|
||||
'port',
|
||||
'nationality',
|
||||
'cortex',
|
||||
'boolean',
|
||||
'anonymised'
|
||||
];
|
||||
|
||||
public const PRIMARY_ONLY_CORRELATING_TYPES = array(
|
||||
'ip-src|port',
|
||||
'ip-dst|port',
|
||||
'hostname|port',
|
||||
);
|
||||
|
||||
public const CAPTURE_FIELDS = array(
|
||||
'event_id',
|
||||
'category',
|
||||
'type',
|
||||
'value',
|
||||
'value1',
|
||||
'value2',
|
||||
'to_ids',
|
||||
'uuid',
|
||||
'timestamp',
|
||||
'distribution',
|
||||
'comment',
|
||||
'sharing_group_id',
|
||||
'deleted',
|
||||
'disable_correlation',
|
||||
'object_id',
|
||||
'object_relation',
|
||||
'first_seen',
|
||||
'last_seen'
|
||||
);
|
||||
|
||||
// typeGroupings are a mapping to high level groups for attributes
|
||||
// for example, IP addresses, domain names, hostnames and e-mail addresses are network related attribute types
|
||||
// whilst filenames and hashes are file related attribute types
|
||||
// This helps generate quick filtering for the event view, but we may reuse this and enhance it in the future for other uses (such as the API?)
|
||||
public const TYPE_GROUPINGS = [
|
||||
'file' => ['attachment', 'pattern-in-file', 'filename-pattern', 'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|pehash', 'malware-sample', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'x509-fingerprint-md5'],
|
||||
'network' => ['ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port', 'mac-address', 'mac-eui-64', 'hostname', 'hostname|port', 'domain', 'domain|ip', 'email-dst', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'bro', 'zeek', 'pattern-in-traffic', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'jarm-fingerprint', 'favicon-mmh3', 'hassh-md5', 'hasshserver-md5', 'community-id'],
|
||||
'financial' => ['btc', 'xmr', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn', 'phone-number']
|
||||
];
|
||||
}
|
||||
|
|
@ -6,4 +6,120 @@ use App\Model\Entity\AppModel;
|
|||
|
||||
class Event extends AppModel
|
||||
{
|
||||
public const NO_PUSH_DISTRIBUTION = 'distribution',
|
||||
NO_PUSH_SERVER_RULES = 'push_rules';
|
||||
|
||||
public $displayField = 'id';
|
||||
|
||||
public $fieldDescriptions = [
|
||||
'threat_level_id' => ['desc' => 'Risk levels: *low* means mass-malware, *medium* means APT malware, *high* means sophisticated APT malware or 0-day attack', 'formdesc' => 'Risk levels: low: mass-malware medium: APT malware high: sophisticated APT malware or 0-day attack'],
|
||||
'classification' => ['desc' => 'Set the Traffic Light Protocol classification. <ol><li><em>TLP:AMBER</em>- Share only within the organization on a need-to-know basis</li><li><em>TLP:GREEN:NeedToKnow</em>- Share within your constituency on the need-to-know basis.</li><li><em>TLP:GREEN</em>- Share within your constituency.</li></ol>'],
|
||||
'submittedioc' => ['desc' => '', 'formdesc' => ''],
|
||||
'analysis' => ['desc' => 'Analysis Levels: *Initial* means the event has just been created, *Ongoing* means that the event is being populated, *Complete* means that the event\'s creation is complete', 'formdesc' => 'Analysis levels: Initial: event has been started Ongoing: event population is in progress Complete: event creation has finished'],
|
||||
'distribution' => ['desc' => 'Describes who will have access to the event.']
|
||||
];
|
||||
|
||||
public $analysisDescriptions = [
|
||||
0 => ['desc' => '*Initial* means the event has just been created', 'formdesc' => 'Event has just been created and is in an initial state'],
|
||||
1 => ['desc' => '*Ongoing* means that the event is being populated', 'formdesc' => 'The analysis is still ongoing'],
|
||||
2 => ['desc' => '*Complete* means that the event\'s creation is complete', 'formdesc' => 'The event creator considers the analysis complete']
|
||||
];
|
||||
|
||||
public $distributionDescriptions = [
|
||||
Distribution::ORGANISATION_ONLY => [
|
||||
'desc' => 'This field determines the current distribution of the event',
|
||||
'formdesc' => "This setting will only allow members of your organisation on this server to see it.",
|
||||
],
|
||||
Distribution::COMMUNITY_ONLY => [
|
||||
'desc' => 'This field determines the current distribution of the event',
|
||||
'formdesc' => "Organisations that are part of this MISP community will be able to see the event.",
|
||||
],
|
||||
Distribution::CONNECTED_COMMUNITIES => [
|
||||
'desc' => 'This field determines the current distribution of the event',
|
||||
'formdesc' => "Organisations that are either part of this MISP community or part of a directly connected MISP community will be able to see the event.",
|
||||
],
|
||||
Distribution::ALL => [
|
||||
'desc' => 'This field determines the current distribution of the event',
|
||||
'formdesc' => "This will share the event with all MISP communities, allowing the event to be freely propagated from one server to the next.",
|
||||
],
|
||||
Distribution::SHARING_GROUP => [
|
||||
'desc' => 'This field determines the current distribution of the event',
|
||||
'formdesc' => "This distribution of this event will be handled by the selected sharing group.",
|
||||
],
|
||||
];
|
||||
|
||||
public const ANALYSIS_LEVELS = [
|
||||
0 => 'Initial', 1 => 'Ongoing', 2 => 'Completed'
|
||||
];
|
||||
|
||||
public $shortDist = [0 => 'Organisation', 1 => 'Community', 2 => 'Connected', 3 => 'All', 4 => ' sharing Group'];
|
||||
|
||||
public $validFormats = [
|
||||
'attack' => ['html', 'AttackExport', 'html'],
|
||||
'attack-sightings' => ['json', 'AttackSightingsExport', 'json'],
|
||||
'cache' => ['txt', 'CacheExport', 'cache'],
|
||||
'context' => ['html', 'ContextExport', 'html'],
|
||||
'context-markdown' => ['txt', 'ContextMarkdownExport', 'md'],
|
||||
'count' => ['txt', 'CountExport', 'txt'],
|
||||
'csv' => ['csv', 'CsvExport', 'csv'],
|
||||
'hashes' => ['txt', 'HashesExport', 'txt'],
|
||||
'hosts' => ['txt', 'HostsExport', 'txt'],
|
||||
'json' => ['json', 'JsonExport', 'json'],
|
||||
'netfilter' => ['txt', 'NetfilterExport', 'sh'],
|
||||
'opendata' => ['txt', 'OpendataExport', 'txt'],
|
||||
'openioc' => ['xml', 'OpeniocExport', 'ioc'],
|
||||
'rpz' => ['txt', 'RPZExport', 'rpz'],
|
||||
'snort' => ['txt', 'NidsSnortExport', 'rules'],
|
||||
'stix' => ['xml', 'Stix1Export', 'xml'],
|
||||
'stix-json' => ['json', 'Stix1Export', 'json'],
|
||||
'stix2' => ['json', 'Stix2Export', 'json'],
|
||||
'suricata' => ['txt', 'NidsSuricataExport', 'rules'],
|
||||
'text' => ['text', 'TextExport', 'txt'],
|
||||
'xml' => ['xml', 'XmlExport', 'xml'],
|
||||
'yara' => ['txt', 'YaraExport', 'yara'],
|
||||
'yara-json' => ['json', 'YaraExport', 'json']
|
||||
];
|
||||
|
||||
public $possibleOptions = [
|
||||
'eventid',
|
||||
'idList',
|
||||
'tags',
|
||||
'from',
|
||||
'to',
|
||||
'last',
|
||||
'to_ids',
|
||||
'includeAllTags', // include also non exportable tags, default `false`
|
||||
'includeAttachments',
|
||||
'event_uuid',
|
||||
'distribution',
|
||||
'sharing_group_id',
|
||||
'disableSiteAdmin',
|
||||
'metadata',
|
||||
'enforceWarninglist', // return just attributes that contains no warnings
|
||||
'sgReferenceOnly', // do not fetch additional information about sharing groups
|
||||
'flatten',
|
||||
'blockedAttributeTags',
|
||||
'eventsExtendingUuid',
|
||||
'extended',
|
||||
'extensionList',
|
||||
'excludeGalaxy',
|
||||
// 'includeCustomGalaxyCluster', // not used
|
||||
'includeRelatedTags',
|
||||
'excludeLocalTags',
|
||||
'includeDecayScore',
|
||||
'includeScoresOnEvent',
|
||||
'includeSightingdb',
|
||||
'includeFeedCorrelations',
|
||||
'includeServerCorrelations',
|
||||
'includeWarninglistHits',
|
||||
'includeGranularCorrelations',
|
||||
'noEventReports', // do not include event report in event data
|
||||
'noShadowAttributes', // do not fetch proposals,
|
||||
'limit',
|
||||
'page',
|
||||
'order',
|
||||
'protected',
|
||||
'published',
|
||||
'orgc_id',
|
||||
];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,39 @@
|
|||
<?php
|
||||
|
||||
namespace App\Model\Entity;
|
||||
|
||||
use App\Model\Entity\AppModel;
|
||||
|
||||
class Feed extends AppModel
|
||||
{
|
||||
public const DEFAULT_FEED_PULL_RULES = [
|
||||
'tags' => [
|
||||
"OR" => [],
|
||||
"NOT" => [],
|
||||
],
|
||||
'orgs' => [
|
||||
"OR" => [],
|
||||
"NOT" => [],
|
||||
],
|
||||
'url_params' => ''
|
||||
];
|
||||
|
||||
public const SUPPORTED_URL_PARAM_FILTERS = [
|
||||
'timestamp',
|
||||
'publish_timestamp',
|
||||
];
|
||||
|
||||
public const CACHE_DIR = APP . 'tmp' . DS . 'cache' . DS . 'feeds' . DS;
|
||||
|
||||
public const FEED_TYPES = array(
|
||||
'misp' => array(
|
||||
'name' => 'MISP Feed'
|
||||
),
|
||||
'freetext' => array(
|
||||
'name' => 'Freetext Parsed Feed'
|
||||
),
|
||||
'csv' => array(
|
||||
'name' => 'Simple CSV Parsed Feed'
|
||||
)
|
||||
);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,39 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Test\Fixture;
|
||||
|
||||
use Cake\TestSuite\Fixture\TestFixture;
|
||||
|
||||
class FeedsFixture extends TestFixture
|
||||
{
|
||||
public $connection = 'test';
|
||||
|
||||
public const FEED_1_ID = 1000;
|
||||
public const FEED_1_NAME = 'test-feed-1';
|
||||
|
||||
public const FEED_2_ID = 2000;
|
||||
public const FEED_2_NAME = 'test-feed-2';
|
||||
|
||||
public function init(): void
|
||||
{
|
||||
$faker = \Faker\Factory::create();
|
||||
|
||||
$this->records = [
|
||||
[
|
||||
'id' => self::FEED_1_ID,
|
||||
'name' => self::FEED_1_NAME,
|
||||
'provider' => 'test-provider',
|
||||
'url' => 'http://localhost/test-feed-1'
|
||||
],
|
||||
[
|
||||
'id' => self::FEED_2_ID,
|
||||
'name' => self::FEED_2_NAME,
|
||||
'provider' => 'test-provider',
|
||||
'url' => 'http://localhost/test-feed-2'
|
||||
]
|
||||
];
|
||||
parent::init();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Test\TestCase\Api\Feeds;
|
||||
|
||||
use App\Test\Fixture\AuthKeysFixture;
|
||||
use App\Test\Helper\ApiTestTrait;
|
||||
use Cake\TestSuite\TestCase;
|
||||
|
||||
class AddFeedApiTest extends TestCase
|
||||
{
|
||||
use ApiTestTrait;
|
||||
|
||||
protected const ENDPOINT = '/feeds/add';
|
||||
|
||||
protected $fixtures = [
|
||||
'app.Organisations',
|
||||
'app.Roles',
|
||||
'app.Users',
|
||||
'app.AuthKeys',
|
||||
'app.Feeds'
|
||||
];
|
||||
|
||||
public function testAddFeed(): void
|
||||
{
|
||||
$this->skipOpenApiValidations();
|
||||
|
||||
$this->setAuthToken(AuthKeysFixture::ADMIN_API_KEY);
|
||||
|
||||
$faker = \Faker\Factory::create();
|
||||
|
||||
$this->post(
|
||||
self::ENDPOINT,
|
||||
[
|
||||
"name" => "feed-osint",
|
||||
"provider" => "CIRCL",
|
||||
"url" => "https://www.circl.lu/doc/misp/feed-osint",
|
||||
"rules" => "{\"tags\":{\"OR\":[],\"NOT\":[]},\"orgs\":{\"OR\":[],\"NOT\":[]},\"url_params\":\"\"}",
|
||||
"source_format" => "1"
|
||||
]
|
||||
);
|
||||
|
||||
$this->assertResponseOk();
|
||||
$this->assertDbRecordExists('Feeds', ['name' => 'feed-osint']);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Test\TestCase\Api\Feeds;
|
||||
|
||||
use App\Test\Fixture\AuthKeysFixture;
|
||||
use App\Test\Fixture\FeedsFixture;
|
||||
use App\Test\Helper\ApiTestTrait;
|
||||
use Cake\TestSuite\TestCase;
|
||||
|
||||
class EditFeedApiTest extends TestCase
|
||||
{
|
||||
use ApiTestTrait;
|
||||
|
||||
protected const ENDPOINT = '/feeds/edit';
|
||||
|
||||
protected $fixtures = [
|
||||
'app.Organisations',
|
||||
'app.Roles',
|
||||
'app.Users',
|
||||
'app.AuthKeys',
|
||||
'app.Feeds'
|
||||
];
|
||||
|
||||
public function testEditFeed(): void
|
||||
{
|
||||
$this->skipOpenApiValidations();
|
||||
|
||||
$this->setAuthToken(AuthKeysFixture::ADMIN_API_KEY);
|
||||
|
||||
$url = sprintf('%s/%s', self::ENDPOINT, FeedsFixture::FEED_1_ID);
|
||||
|
||||
$this->put(
|
||||
$url,
|
||||
[
|
||||
"name" => "feed-osint",
|
||||
"provider" => "CIRCL",
|
||||
"url" => "https://www.circl.lu/doc/misp/feed-osint",
|
||||
"rules" => [
|
||||
"tags" => [
|
||||
"OR" => [],
|
||||
"NOT" => []
|
||||
],
|
||||
"orgs" => [
|
||||
"OR" => [],
|
||||
"NOT" => []
|
||||
],
|
||||
"url_params" => ""
|
||||
],
|
||||
"settings" => [
|
||||
"csv" => [
|
||||
"value" => "",
|
||||
"delimiter" => ""
|
||||
],
|
||||
"common" => [
|
||||
"excluderegex" => ""
|
||||
]
|
||||
]
|
||||
]
|
||||
);
|
||||
|
||||
$this->assertResponseOk();
|
||||
$this->assertDbRecordExists(
|
||||
'Feeds',
|
||||
[
|
||||
'id' => FeedsFixture::FEED_1_ID,
|
||||
'name' => 'feed-osint',
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App\Test\TestCase\Api\Feeds;
|
||||
|
||||
use App\Test\Fixture\AuthKeysFixture;
|
||||
use App\Test\Fixture\FeedsFixture;
|
||||
use App\Test\Helper\ApiTestTrait;
|
||||
use Cake\TestSuite\TestCase;
|
||||
|
||||
class IndexFeedsApiTest extends TestCase
|
||||
{
|
||||
use ApiTestTrait;
|
||||
|
||||
protected const ENDPOINT = '/feeds/index';
|
||||
|
||||
protected $fixtures = [
|
||||
'app.Organisations',
|
||||
'app.Roles',
|
||||
'app.Users',
|
||||
'app.AuthKeys',
|
||||
'app.Feeds',
|
||||
];
|
||||
|
||||
public function testIndexFeeds(): void
|
||||
{
|
||||
$this->skipOpenApiValidations();
|
||||
|
||||
$this->setAuthToken(AuthKeysFixture::ADMIN_API_KEY);
|
||||
|
||||
$this->get(self::ENDPOINT);
|
||||
|
||||
$this->assertResponseOk();
|
||||
$this->assertResponseContains(sprintf('"name": "%s"', FeedsFixture::FEED_1_NAME));
|
||||
$this->assertResponseContains(sprintf('"name": "%s"', FeedsFixture::FEED_2_NAME));
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue