MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'misp-stix' of github.com:MISP/MISP into develop

pull/9044/head
Christian Studer 2023-04-26 14:01:03 +02:00
parent eb9442f34f 2ea3bde1a2
commit ed1ac9c8c8
6 changed files with 304 additions and 2160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
app/Controller/EventsController.php
View File

@ -2374,20 +2374,54 @@ class EventsController extends AppController
$this->set('title_for_layout', __('Import from MISP Export File'));
}
public function upload_stix($stix_version = '1', $publish = false)
public function upload_stix($stix_version = '1', $publish = false, $galaxies_as_tags = true, $debug = false)
{
$sgs = $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1);
$initialDistribution = 0;
if (Configure::read('MISP.default_event_distribution') != null) {
$initialDistribution = Configure::read('MISP.default_event_distribution');
}
$distributionLevels = $this->Event->distributionLevels;
if ($this->request->is('post')) {
if ($this->_isRest()) {
if (isset($this->params['named']['publish'])) {
$publish = $this->params['named']['publish'];
}
if (isset($this->params['named']['distribution'])) {
$distribution = intval($this->params['named']['distribution']);
if (array_key_exists($distribution, $distributionLevels)) {
$initialDistribution = $distribution;
} else {
throw new MethodNotAllowedException(__('Wrong distribution level'));
}
}
$sharingGroupId = null;
if ($initialDistribution == 4) {
if (!isset($this->params['named']['sharing_group_id'])) {
throw new MethodNotAllowedException(__('The sharing group id is needed when the distribution is set to 4 ("Sharing group").'));
}
$sharingGroupId = intval($this->params['named']['sharing_group_id']);
if (!array_key_exists($sharingGroupId, $sgs)) {
throw new MethodNotAllowedException(__('Please select a valid sharing group id.'));
}
}
if (isset($this->params['named']['galaxies_as_tags'])) {
$galaxies_as_tags = $this->params['named']['galaxies_as_tags'];
}
if (isset($this->params['named']['debugging'])) {
$debug = $this->params['named']['debugging'];
}
$filePath = FileAccessTool::writeToTempFile($this->request->input());
$result = $this->Event->upload_stix(
$this->Auth->user(),
$filePath,
$stix_version,
'uploaded_stix_file.' . ($stix_version == '1' ? 'xml' : 'json'),
$publish
$publish,
$initialDistribution,
$sharingGroupId,
$galaxies_as_tags,
$debug
);
if (is_numeric($result)) {
$event = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $result));
@ -2406,12 +2440,19 @@ class EventsController extends AppController
if (!move_uploaded_file($this->data['Event']['stix']['tmp_name'], $filePath)) {
throw new Exception("Could not move uploaded STIX file.");
}
if (isset($this->data['Event']['debug'])) {
$debug = $this->data['Event']['debug'];
}
$result = $this->Event->upload_stix(
$this->Auth->user(),
$filePath,
$stix_version,
$original_file,
$this->data['Event']['publish']
$this->data['Event']['publish'],
$this->data['Event']['distribution'],
$this->data['Event']['sharing_group_id'],
!boolval($this->data['Event']['galaxies_parsing']),
$debug
);
if (is_numeric($result)) {
$this->Flash->success(__('STIX document imported.'));
@ -2429,6 +2470,20 @@ class EventsController extends AppController
}
}
$this->set('stix_version', $stix_version == 2 ? '2.x JSON' : '1.x XML');
$this->set('initialDistribution', $initialDistribution);
$distributions = array_keys($this->Event->distributionDescriptions);
$distributions = $this->_arrayToValuesIndexArray($distributions);
$this->set('distributions', $distributions);
$fieldDesc = array();
if (empty($sgs)) {
unset($distributionLevels[4]);
}
$this->set('distributionLevels', $distributionLevels);
foreach ($distributionLevels as $key => $value) {
$fieldDesc['distribution'][$key] = $this->Event->distributionDescriptions[$key]['formdesc'];
}
$this->set('sharingGroups', $sgs);
$this->set('fieldDesc', $fieldDesc);
}
public function merge($target_id=null, $source_id=null)
@ -4438,12 +4493,12 @@ class EventsController extends AppController
),
'STIX' => array(
'url' => $this->baseurl . '/events/upload_stix',
'text' => __('STIX 1.1.1 format (lossy)'),
'text' => __('STIX 1.x format (lossy)'),
'ajax' => false,
),
'STIX2' => array(
'url' => $this->baseurl . '/events/upload_stix/2',
'text' => __('STIX 2.0 format (lossy)'),
'text' => __('STIX 2.x format (lossy)'),
'ajax' => false,
)
);

86
app/Model/Event.php
View File

@ -5838,41 +5838,79 @@ class Event extends AppModel
* @throws InvalidArgumentException
* @throws Exception
*/
public function upload_stix(array $user, $file, $stix_version, $original_file, $publish)
public function upload_stix(array $user, $file, $stix_version, $original_file, $publish, $distribution, $sharingGroupId, $galaxiesAsTags, $debug = false)
{
$scriptDir = APP . 'files' . DS . 'scripts';
if ($stix_version == '2') {
if ($stix_version == '2' || $stix_version == '2.0' || $stix_version == '2.1') {
$scriptFile = $scriptDir . DS . 'stix2' . DS . 'stix2misp.py';
$output_path = $file . '.stix2';
$stix_version = "STIX 2.0";
$output_path = $file . '.out';
$shell_command = [
ProcessTool::pythonBin(),
$scriptFile,
'-i', $file,
'--distribution', $distribution
];
if ($distribution == 4) {
array_push($shell_command, '--sharing_group_id', $sharingGroupId);
}
if ($galaxiesAsTags) {
$shell_command[] = '--galaxies_as_tags';
}
if ($debug) {
$shell_command[] = '--debug';
}
$stix_version = "STIX 2.1";
} elseif ($stix_version == '1' || $stix_version == '1.1' || $stix_version == '1.2') {
$scriptFile = $scriptDir . DS . 'stix2misp.py';
$output_path = $file . '.json';
$shell_command = [
ProcessTool::pythonBin(),
$scriptFile,
$file,
Configure::read('MISP.default_event_distribution'),
Configure::read('MISP.default_attribute_distribution'),
$this->__getTagNamesFromSynonyms($scriptDir)
];
$stix_version = "STIX 1.1";
} else {
throw new InvalidArgumentException('Invalid STIX version');
}
$shell_command = [
ProcessTool::pythonBin(),
$scriptFile,
$file,
Configure::read('MISP.default_event_distribution'),
Configure::read('MISP.default_attribute_distribution'),
$this->__getTagNamesFromSynonyms($scriptDir),
];
$result = ProcessTool::execute($shell_command, null, true);
$result = preg_split("/\r\n|\n|\r/", trim($result));
$result = trim(end($result));
$tempFile = file_get_contents($file);
unlink($file);
if ($result === '1') {
$decoded = JsonTool::decode($result);
if (!empty($decoded['success'])) {
$data = FileAccessTool::readAndDelete($output_path);
$data = $this->jsonDecode($data);
if (empty($data['Event'])) {
$data = array('Event' => $data);
}
if (!$galaxiesAsTags) {
if (!isset($this->GalaxyCluster)) {
$this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
}
$this->__handleGalaxiesAndClusters($user, $data['Event']);
if (!empty($data['Event']['Attribute'])) {
foreach ($data['Event']['Attribute'] as &$attribute) {
$this->__handleGalaxiesAndClusters($user, $attribute);
}
}
if (!empty($data['Event']['Object'])) {
foreach ($data['Event']['Object'] as &$misp_object) {
if (!empty($misp_object['Attribute'])) {
foreach ($misp_object['Attribute'] as &$attribute) {
$this->__handleGalaxiesAndClusters($user, $attribute);
}
}
}
}
}
if (!empty($decoded['stix_version'])) {
$stix_version = 'STIX ' . $decoded['stix_version'];
}
$created_id = false;
$validationIssues = false;
$result = $this->_add($data, true, $user, '', null, false, null, $created_id, $validationIssues);
@ -5890,13 +5928,8 @@ class Event extends AppModel
return $result;
}
return $validationIssues;
} else if ($result === '2') {
$response = __('Issues while loading the stix file.');
} elseif ($result === '3') {
$response = __('Issues with the maec library.');
} else {
$response = __('Issues executing the ingestion script or invalid input.');
}
$response = __($decoded['error']);
if (!$user['Role']['perm_site_admin']) {
$response .= ' ' . __('Please ask your administrator to');
} else {
@ -5906,6 +5939,19 @@ class Event extends AppModel
return $response;
}
private function __handleGalaxiesAndClusters($user, &$data)
{
if (!empty($data['Galaxy'])) {
$tag_names = $this->GalaxyCluster->convertGalaxyClustersToTags($user, $data['Galaxy']);
if (empty($data['Tag'])) {
$data['Tag'] = [];
}
foreach ($tag_names as $tag_name) {
$data['Tag'][] = array('name' => $tag_name);
}
}
}
/**
* @param string $scriptDir
* @return string

27
app/Model/GalaxyCluster.php
View File

@ -2071,4 +2071,31 @@ class GalaxyCluster extends AppModel
}
return $CyCatRelations;
}
/**
* convertGalaxyClustersToTags
*
* @param array $user
* @param array $galaxies
* @return array The tag names extracted from galaxy clusters
*/
public function convertGalaxyClustersToTags($user, $galaxies)
{
$galaxyClusters = [];
$tag_names = [];
foreach ($galaxies as $galaxy) {
if (empty($galaxy['GalaxyCluster'])) {
continue;
}
$clusters = $galaxy['GalaxyCluster'];
unset($galaxy['GalaxyCluster']);
foreach ($clusters as $cluster) {
$cluster['Galaxy'] = $galaxy;
$galaxyClusters[] = array('GalaxyCluster' => $cluster);
$tag_names[] = !empty($cluster['tag_name']) ? $cluster['tag_name'] : 'misp-galaxy:' . $cluster['type'] . '="' . $cluster['uuid'] . '"';
}
}
$this->Galaxy->importGalaxyAndClusters($user, $galaxyClusters);
return $tag_names;
}
}

56
app/View/Events/upload_stix.ctp
View File

@ -1,6 +1,6 @@
<div class="events form">
<?php
echo $this->Form->create('Event', array('type' => 'file'));
echo $this->Form->create('Event', array('type' => 'file'));
?>
<fieldset>
<legend><?= __('Import STIX %s file', $stix_version); ?></legend>
@ -11,6 +11,34 @@
));
?>
<div class="input clear"></div>
<?php
$distributionFormInfo = $this->element(
'genericElements/Form/formInfo',
[
'field' => [
'field' => 'distribution'
],
'modelForForm' => 'Event',
'fieldDesc' => $fieldDesc['distribution'],
]
);
echo $this->Form->input('distribution', array(
'options' => $distributionLevels,
'label' => __('Distribution ') . $distributionFormInfo,
'selected' => $initialDistribution,
));
?>
<div id="SGContainer" style="display:none;">
<?php
if (!empty($sharingGroups)) {
echo $this->Form->input('sharing_group_id', array(
'options' => array($sharingGroups),
'label' => __('Sharing Group'),
));
}
?>
</div>
<div class="input clear"></div>
<?php
echo $this->Form->input('publish', array(
'checked' => false,
@ -23,6 +51,20 @@
'checked' => true,
'label' => __('Include the original imported file as attachment')
));
if ($me['Role']['perm_site_admin'] || $me['Role']['perm_galaxy_editor']) {
echo '<div class="input clear"></div>';
echo $this->Form->input('galaxies_parsing', array(
'checked' => false,
'label' => __('Use Galaxies 2.0')
));
}
if ($me['Role']['perm_site_admin']) {
echo '<div class="input clear"></div>';
echo $this->Form->input('debug', array(
'checked' => true,
'label' => __('Advanced conversion debugging')
));
}
?>
</fieldset>
<?php
@ -33,3 +75,15 @@
<?php
echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event-collection', 'menuItem' => 'import_from'));
?>
<script>
$(function(){
$('#EventDistribution').change(function() {
if ($(this).val() == 4) {
$('#SGContainer').show();
} else {
$('#SGContainer').hide();
}
}).change();
});
</script>

2204
app/files/scripts/stix2/stix2misp.py
View File

File diff suppressed because it is too large Load Diff

26
app/files/scripts/stix2misp.py
View File

@ -22,6 +22,7 @@ import time
import uuid
import base64
import pymisp
import traceback
import stix2misp_mapping
from operator import attrgetter
from collections import defaultdict
@ -1544,19 +1545,18 @@ def generate_event(filename, tries=0):
return STIXPackage.from_xml(filename)
except NamespaceNotFoundError:
if tries == 1:
print(4)
print(json.dump({'error': 'Cannot handle STIX namespace'}))
sys.exit()
_update_namespaces()
return generate_event(filename, 1)
except NotImplementedError:
print('ERROR - Missing python library: stix_edh', file=sys.stderr)
except Exception:
print(json.dumps({'error': 'Missing python library: stix_edh'}))
except Exception as e:
try:
import maec
print(2)
print(json.dumps({'error': f'Error while loading the STIX file: {e.__str__()}'}))
except ImportError:
print('ERROR - Missing python library: maec', file=sys.stderr)
print(3)
print(json.dumps({'error': 'Missing python library: maec'}))
sys.exit(0)
@ -1572,11 +1572,15 @@ def main(args):
filename = args[1] if args[1][0] == '/' else '{}/tmp/{}'.format(os.path.dirname(args[0]), args[1])
event = generate_event(filename)
from_misp = is_from_misp(event)
stix_parser = StixFromMISPParser() if from_misp else ExternalStixParser()
stix_parser.load_event(args[2:], filename, from_misp, event.version)
stix_parser.build_misp_event(event)
stix_parser.saveFile()
print(1)
try:
stix_parser = StixFromMISPParser() if from_misp else ExternalStixParser()
stix_parser.load_event(args[2:], filename, from_misp, event.version)
stix_parser.build_misp_event(event)
stix_parser.saveFile()
print(json.dumps({'success': 1}))
except Exception as e:
print(json.dumps({'error': e.__str__()}))
traceback.print_tb(e.__traceback__)
if __name__ == "__main__":
main(sys.argv)