MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch '2.4' into develop

pull/7392/head
iglocska 2021-05-04 09:46:58 +02:00
parent e441808fa1 f18fc7cb51
commit 43836e2fb4
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
9 changed files with 198 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
app/Model/Event.php
View File

@ -3154,6 +3154,21 @@ class Event extends AppModel
));
return true;
}
$banStatus = $this->getEventRepublishBanStatus($id);
if ($banStatus['active']) {
$this->Log = ClassRegistry::init('Log');
$this->Log->create();
$this->Log->save(array(
'org' => 'SYSTEM',
'model' => 'Event',
'model_id' => $id,
'email' => $user['email'],
'action' => 'publish',
'title' => __('E-mail alerts not sent out during publishing'),
'change' => $banStatus['message'],
));
return !$banStatus['error'];
}
if (Configure::read('MISP.background_jobs')) {
$job = ClassRegistry::init('Job');
$job->create();
@ -7304,4 +7319,59 @@ class Event extends AppModel
}
return [];
}
public function getEventRepublishBanStatus($eventID)
{
$banStatus = [
'error' => false,
'active' => false,
'message' => __('Event publish is not banned')
];
if (Configure::read('MISP.event_alert_republish_ban')) {
$event = $this->find('first', array(
'conditions' => array('Event.id' => $eventID),
'recursive' => -1,
'fields' => array('Event.uuid')
));
if (empty($event)) {
$banStatus['error'] = true;
$banStatus['active'] = true;
$banStatus['message'] = __('Event not found');
return $banStatus;
}
$banThresholdMinutes = intval(Configure::read('MISP.event_alert_republish_ban_threshold'));
$banThresholdSeconds = 60 * $banThresholdMinutes;
$redis = $this->setupRedis();
if ($redis === false) {
$banStatus['error'] = true;
$banStatus['active'] = true;
$banStatus['message'] = __('Reason: Could not reach redis to chech republish emailing ban status.');
return $banStatus;
}
$redisKey = "misp:event_alert_republish_ban:{$event['Event']['uuid']}";
$banLiftTimestamp = $redis->get($redisKey);
if (!empty($banLiftTimestamp)) {
$remainingMinutes = (intval($banLiftTimestamp) - time()) / 60;
$banStatus['active'] = true;
if (Configure::read('MISP.event_alert_republish_ban_refresh_on_retry')) {
$redis->multi(Redis::PIPELINE)
->set($redisKey, time() + $banThresholdSeconds)
->expire($redisKey, $banThresholdSeconds)
->exec();
$banStatus['message'] = __('Reason: Event is banned from sending out emails. Ban has been refreshed and will be lifted in %smin', $banThresholdMinutes);
} else {
$banStatus['message'] = __('Reason: Event is banned from sending out emails. Ban will be lifted in %smin %ssec.', floor($remainingMinutes), $remainingMinutes % 60);
}
return $banStatus;
} else {
$redis->multi(Redis::PIPELINE)
->set($redisKey, time() + $banThresholdSeconds)
->expire($redisKey, $banThresholdSeconds)
->exec();
return $banStatus;
}
}
$banStatus['message'] = __('Emailing republishing ban setting is not enabled');
return $banStatus;
}
}

27
app/Model/Server.php
View File

@ -5325,6 +5325,33 @@ class Server extends AppModel
'type' => 'string',
'null' => false,
),
'event_alert_republish_ban' => array(
'level' => 1,
'description' => __('Enable this setting to start blocking alert e-mails for events that have already been published since a specified amount of time. This threshold is defined by MISP.event_alert_republish_ban_threshold'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean',
'null' => false,
),
'event_alert_republish_ban_threshold' => array(
'level' => 1,
'description' => __('If the MISP.event_alert_republish_ban setting is set, this setting will control how long no alerting by email will be done. Expected format: integer, in minutes'),
'value' => 5,
'errorMessage' => '',
'test' => 'testForNumeric',
'type' => 'numeric',
'null' => false,
),
'event_alert_republish_ban_refresh_on_retry' => array(
'level' => 1,
'description' => __('If the MISP.event_alert_republish_ban setting is set, this setting will control if a ban time should be reset if emails are tried to be sent during the ban.'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean',
'null' => false,
),
'org_alert_threshold' => array(
'level' => 1,
'description' => __('Set a value to limit the number of email alerts that events can generate per creator organisation (for example, if an organisation pushes out 2000 events in one shot, only alert on the first 20).'),

2
app/files/misp-galaxy

@ -1 +1 @@
Subproject commit ef9989dbe85d798b62ebfb8acfe23559292cde6f
Subproject commit 94ec98d544177c9b00c2bc0fe7913a1a5270db0c

94
app/files/scripts/stix2/misp2stix2.py
View File

@ -49,12 +49,12 @@ class StixBuilder():
pathname = os.path.dirname(args[0])
filename = os.path.join(pathname, args[1])
with open(filename, 'rt', encoding='utf-8') as f:
self.json_event = json.loads(f.read())
self.json_event = self._get_event(json.loads(f.read()))
self.filename = filename
def buildEvent(self):
try:
stix_packages = [sdo for event in self.json_event['response'] for sdo in self.handler(event['Event'])] if self.json_event.get('response') else self.handler(self.json_event['Event'])
stix_packages = self._get_packages()
outputfile = "{}.out".format(self.filename)
with open(outputfile, 'wt', encoding='utf-8') as f:
f.write(json.dumps(stix_packages, cls=STIXJSONEncoder))
@ -62,6 +62,17 @@ class StixBuilder():
except Exception as e:
print(json.dumps({'error': e.__str__()}))
@staticmethod
def _get_event(events):
if events.get('response'):
return {'response': [event['Event'] if event.get('Event') else event for event in events['response']]}
return events['Event'] if events.get('Event') else events
def _get_packages(self):
if self.json_event.get('response'):
return [sdo for event in self.json_event['response'] for sdo in self.handler(event)]
return self.handler(self.json_event)
def eventReport(self):
if not self.object_refs and self.links:
self.add_custom(self.links.pop(0))
@ -390,7 +401,8 @@ class StixBuilder():
coa_args = {'id': coa_id, 'type': 'course-of-action', 'created_by_ref': self.identity_id}
coa_args['labels'] = self.create_object_labels(misp_object['name'], misp_object['meta-category'], to_ids)
for attribute in misp_object['Attribute']:
self.parse_galaxies(attribute['Galaxy'], coa_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], coa_id)
relation = attribute['object_relation']
if relation in ('name', 'description'):
coa_args[relation] = attribute['value']
@ -460,7 +472,8 @@ class StixBuilder():
def add_indicator(self, attribute):
indicator_id = "indicator--{}".format(attribute['uuid'])
self.parse_galaxies(attribute['Galaxy'], indicator_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], indicator_id)
category = attribute['category']
killchain = self.create_killchain(category)
labels, markings = self.create_labels(attribute)
@ -491,7 +504,8 @@ class StixBuilder():
def add_observed_data(self, attribute):
observed_data_id = "observed-data--{}".format(attribute['uuid'])
self.parse_galaxies(attribute['Galaxy'], observed_data_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], observed_data_id)
timestamp = self.get_datetime_from_timestamp(attribute['timestamp'])
labels, markings = self.create_labels(attribute)
observable = self.define_observable(attribute)
@ -735,10 +749,8 @@ class StixBuilder():
def fetch_custom_values(self, attributes, object_id):
values = defaultdict(list)
for attribute in attributes:
try:
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
except KeyError:
pass
attribute_type = '{}_{}'.format(attribute['type'], attribute['object_relation'].replace('.', '_DOT_'))
values[attribute_type].append(attribute['value'])
return {attribute_type: value[0] if len(value) == 1 else value for attribute_type, value in values.items()}
@ -1100,7 +1112,8 @@ class StixBuilder():
observable = {}
object_num = 0
for attribute in attributes:
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
relation = attribute['object_relation']
try:
stix_type = misp2stix2_mapping.asnObjectMapping[relation]
@ -1121,7 +1134,8 @@ class StixBuilder():
mapping = misp2stix2_mapping.objectsMapping['asn']['pattern']
pattern = []
for attribute in attributes:
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
relation = attribute['object_relation']
try:
stix_type = misp2stix2_mapping.asnObjectMapping[relation]
@ -1137,7 +1151,8 @@ class StixBuilder():
def resolve_credential_observable(self, attributes, object_id):
user_account = misp2stix2_mapping.objectsMapping['credential']['observable']
for attribute in attributes:
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
relation = attribute['object_relation']
try:
stix_type = misp2stix2_mapping.credentialObjectMapping[relation]
@ -1150,7 +1165,8 @@ class StixBuilder():
mapping = misp2stix2_mapping.objectsMapping['credential']['pattern']
pattern = []
for attribute in attributes:
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
relation = attribute['object_relation']
try:
stix_type = misp2stix2_mapping.credentialObjectMapping[relation]
@ -1161,7 +1177,8 @@ class StixBuilder():
def resolve_domain_ip_observable(self, attributes, object_id):
for attribute in attributes:
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute['type'] == 'ip-dst':
ip_value = attribute['value']
elif attribute['type'] == 'domain':
@ -1173,7 +1190,8 @@ class StixBuilder():
mapping = misp2stix2_mapping.objectsMapping['domain-ip']['pattern']
pattern = []
for attribute in attributes:
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
try:
stix_type = misp2stix2_mapping.domainIpObjectMapping[attribute['type']]
except KeyError:
@ -1187,7 +1205,8 @@ class StixBuilder():
additional_header = {}
object_num = 0
for attribute in attributes:
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
relation = attribute['object_relation']
attribute_value = attribute['value']
try:
@ -1230,7 +1249,8 @@ class StixBuilder():
pattern = []
n = 0
for attribute in attributes:
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
relation = attribute['object_relation']
try:
mapping = misp2stix2_mapping.emailObjectMapping[relation]
@ -1332,7 +1352,8 @@ class StixBuilder():
ip_address = {}
domain = {}
for attribute in attributes:
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
relation = attribute['object_relation']
attribute_value = attribute['value']
if relation == 'ip':
@ -1384,7 +1405,8 @@ class StixBuilder():
def resolve_ip_port_pattern(self, attributes, object_id):
pattern = []
for attribute in attributes:
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
relation = attribute['object_relation']
attribute_value = attribute['value']
if relation == 'domain':
@ -1452,7 +1474,8 @@ class StixBuilder():
states = []
tmp_attributes = {}
for attribute in attributes:
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
relation = attribute['object_relation']
if relation == 'state':
states.append(attribute['value'])
@ -1466,7 +1489,8 @@ class StixBuilder():
current_process['type'] = 'process'
n = 0
for attribute in attributes:
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
relation = attribute['object_relation']
if relation == 'parent-pid':
str_n = str(n)
@ -1495,7 +1519,8 @@ class StixBuilder():
mapping = misp2stix2_mapping.objectsMapping['process']['pattern']
pattern = []
for attribute in attributes:
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
try:
pattern.append(mapping.format(misp2stix2_mapping.processMapping[attribute['object_relation']], attribute['value']))
except KeyError:
@ -1507,7 +1532,8 @@ class StixBuilder():
values = {}
registry_value_types = ('data', 'data-type', 'name')
for attribute in attributes:
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
relation = attribute['object_relation']
try:
stix_type = misp2stix2_mapping.regkeyMapping[relation]
@ -1529,7 +1555,8 @@ class StixBuilder():
fields = ('key', 'value')
registry_value_types = ('data', 'data-type', 'name')
for attribute in attributes:
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
relation = attribute['object_relation']
try:
stix_type = misp2stix2_mapping.regkeyMapping[relation]
@ -1596,7 +1623,8 @@ class StixBuilder():
def resolve_url_observable(self, attributes, object_id):
url_args = {}
for attribute in attributes:
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute['type'] == 'url':
# If we have the url (WE SHOULD), we return the observable supported atm with the url value
observable = {'0': {'type': 'url', 'value': attribute['value']}}
@ -1621,7 +1649,8 @@ class StixBuilder():
def resolve_url_pattern(self, attributes, object_id):
pattern = []
for attribute in attributes:
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
attribute_type = attribute['type']
try:
stix_type = misp2stix2_mapping.urlMapping[attribute_type]
@ -1676,7 +1705,8 @@ class StixBuilder():
def parse_user_account_attributes(self, attributes, object_id):
tmp_attributes = defaultdict(list)
for attribute in attributes:
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
relation = attribute['object_relation']
if relation == 'group':
tmp_attributes[relation].append(attribute['value'])
@ -1693,7 +1723,8 @@ class StixBuilder():
hashes = {}
attributes2parse = defaultdict(list)
for attribute in attributes:
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
relation = attribute['object_relation']
if relation in ("x509-fingerprint-md5", "x509-fingerprint-sha1", "x509-fingerprint-sha256"):
hashes[relation.split('-')[2]] = attribute['value']
@ -1713,7 +1744,8 @@ class StixBuilder():
mapping = misp2stix2_mapping.objectsMapping['x509']['pattern']
pattern = []
for attribute in attributes:
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
relation = attribute['object_relation']
if relation in ("x509-fingerprint-md5", "x509-fingerprint-sha1", "x509-fingerprint-sha256"):
stix_type = f"hashes.'{relation.split('-')[2]}'"
@ -1743,7 +1775,8 @@ class StixBuilder():
attributes_dict = defaultdict(list)
for attribute in attributes:
attributes_dict[attribute['object_relation']].append(self._parse_attribute(attribute))
self.parse_galaxies(attribute['Galaxy'], object_id)
if attribute.get('Galaxy'):
self.parse_galaxies(attribute['Galaxy'], object_id)
return {key: value[0] if key not in multiple_fields and len(value) == 1 else value for key, value in attributes_dict.items()}
@staticmethod
@ -1789,8 +1822,9 @@ class StixBuilder():
@staticmethod
def handle_time_fields(attribute, timestamp, stix_type):
to_return = {'created': timestamp, 'modified': timestamp}
iso_timestamp = f"{timestamp.isoformat(timespec='milliseconds')}Z"
for misp_field, stix_field in zip(('first_seen', 'last_seen'), _time_fields[stix_type]):
to_return[stix_field] = datetime.strptime(attribute[misp_field].split('+')[0], '%Y-%m-%dT%H:%M:%S.%f') if attribute.get(misp_field) else timestamp
to_return[stix_field] = datetime.strptime(attribute[misp_field].split('+')[0], '%Y-%m-%dT%H:%M:%S.%f') if attribute.get(misp_field) else iso_timestamp
return to_return
@staticmethod

13
app/files/scripts/stix2/stix2misp.py
View File

@ -1454,6 +1454,17 @@ class ExternalStixParser(StixParser):
print(f'File extension type(s) not supported at the moment: {", ".join(extension_types)}', file=sys.stderr)
self.handle_import_case(observable, attributes, 'file', _force_object=('file-encoding', 'path'))
def parse_ip_address_observable(self, observable):
attributes = []
for observable_object in observable.objects.values():
attribute = {
'value': observable_object.value,
'to_ids': False
}
attribute.update(stix2misp_mapping.ip_attribute_mapping)
attributes.append(attribute)
self.handle_import_case(observable, attributes, 'ip-port')
def parse_ip_network_traffic_observable(self, observable):
network_traffic, references = self.filter_main_object(observable.objects, 'NetworkTraffic')
extension = self._network_traffic_has_extension(network_traffic)
@ -2000,7 +2011,7 @@ class ExternalStixParser(StixParser):
misp_object.add_attribute(**attribute)
self.misp_event.add_object(**misp_object)
else:
attribute = {field: attributes[0][field] for field in stix2misp_mapping.single_attribute_fields if attributes[0].get(field)}
attribute = {field: attributes[0][field] for field in stix2misp_mapping.single_attribute_fields if attributes[0].get(field) is not None}
attribute['uuid'] = stix_object.id.split('--')[1]
attribute.update(self.parse_timeline(stix_object))
if isinstance(stix_object, stix2.v20.Indicator):

12
app/files/scripts/stix2misp.py
View File

@ -1521,11 +1521,19 @@ def generate_event(filename, tries=0):
print(3)
sys.exit(0)
def is_from_misp(event):
try:
title = event.header.title
except AttributeError:
return False
return ('Export from ' in title and 'MISP' in title)
def main(args):
filename = '{}/tmp/{}'.format(os.path.dirname(args[0]), args[1])
event = generate_event(filename)
title = event.stix_header.title
from_misp = (title is not None and "Export from " in title and "MISP" in title)
from_misp = is_from_misp(event)
stix_parser = StixFromMISPParser() if from_misp else ExternalStixParser()
stix_parser.load_event(args[2:], filename, from_misp, event.version)
stix_parser.build_misp_event(event)

2
app/files/warninglists

@ -1 +1 @@
Subproject commit c2400b392a85fae1ae0c92cbde4ca510b23b6189
Subproject commit a993b89aaab3551cfc8f5656cc1673e532f24378

13
db_schema.json
View File

@ -890,6 +890,17 @@
"column_default": null,
"extra": ""
},
{
"column_name": "comment",
"is_nullable": "NO",
"data_type": "text",
"character_maximum_length": "65535",
"numeric_precision": null,
"collation_name": "utf8mb4_unicode_ci",
"column_type": "text",
"column_default": null,
"extra": ""
},
{
"column_name": "from_json",
"is_nullable": "YES",
@ -7986,5 +7997,5 @@
"id": true
}
},
"db_version": "67"
"db_version": "68"
}

2
docs/generic/supportFunctions.md
View File

@ -904,7 +904,7 @@ runTests () {
key = \"${AUTH_KEY}\"" |sudo tee ${PATH_TO_MISP}/PyMISP/tests/keys.py
sudo chown -R $WWW_USER:$WWW_USER $PATH_TO_MISP/PyMISP/
${SUDO_WWW} sh -c "cd $PATH_TO_MISP/PyMISP && git submodule foreach git pull origin master"
${SUDO_WWW} sh -c "cd $PATH_TO_MISP/PyMISP && git submodule foreach git pull origin main"
${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -e $PATH_TO_MISP/PyMISP/.[fileobjects,neo,openioc,virustotal,pdfexport]
${SUDO_WWW} sh -c "cd $PATH_TO_MISP/PyMISP && ${PATH_TO_MISP}/venv/bin/python tests/testlive_comprehensive.py"
}