MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'develop' into event_view_collapse

pull/9764/head
iglocska 2024-06-05 13:02:22 +02:00
parent f6b4719413 ec705502fc
commit e4ecea5b5c
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
12 changed files with 247 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
PyMISP

@ -1 +1 @@
Subproject commit 8b4f98ac4c2e6c8cc1dba064f937dac816b67d0f
Subproject commit f7b28e7bc98a58cc623b111d1476c9079ca83c20

53
app/Console/Command/EventShell.php
View File

@ -501,6 +501,57 @@ class EventShell extends AppShell
$log->createLogEntry($user, 'publish', 'GalaxyCluster', $clusterId, 'GalaxyCluster (' . $clusterId . '): published.', 'published () => (1)');
}
public function attribute_enrichment()
{
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Run attribute enrichment'] . PHP_EOL);
}
$userId = $this->args[0];
$user = $this->getUser($userId);
$id = $this->args[1];
$modulesRaw = $this->args[2];
try {
$modules = json_decode($modulesRaw, true);
} catch (Exception $e) {
die('Invalid module JSON');
}
if (!empty($this->args[3])) {
$jobId = $this->args[3];
} else {
$this->Job->create();
$data = [
'worker' => 'default',
'job_type' => 'enrichment',
'job_input' => 'Attribute: ' . $id . ' modules: ' . $modulesRaw,
'status' => 0,
'retries' => 0,
'org' => $user['Organisation']['name'],
'message' => 'Enriching event.',
];
$this->Job->save($data);
$jobId = $this->Job->id;
}
$job = $this->Job->read(null, $jobId);
$options = array(
'user' => $user,
'id' => $id,
'modules' => $modules
);
$result = $this->Attribute->enrichment($options);
$job['Job']['progress'] = 100;
$job['Job']['date_modified'] = date("Y-m-d H:i:s");
if ($result) {
$job['Job']['message'] = 'Added ' . $result . ' attribute' . ($result > 1 ? 's.' : '.');
} else {
$job['Job']['message'] = 'Enrichment finished, but no attributes added.';
}
echo $job['Job']['message'] . PHP_EOL;
$this->Job->save($job);
$log = ClassRegistry::init('Log');
$log->createLogEntry($user, 'enrichment', 'Attribute', $id, 'Attribute (' . $id . '): enriched.', 'enriched () => (1)');
}
public function enrichment()
{
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
@ -546,7 +597,7 @@ class EventShell extends AppShell
} else {
$job['Job']['message'] = 'Enrichment finished, but no attributes added.';
}
echo $job['Job']['message'] . PHP_EOL;
echo $job['Job']['message'] . PHP_EOL;
$this->Job->save($job);
$log = ClassRegistry::init('Log');
$log->createLogEntry($user, 'enrichment', 'Event', $eventId, 'Event (' . $eventId . '): enriched.', 'enriched () => (1)');

25
app/Controller/AttributesController.php
View File

@ -3050,4 +3050,29 @@ class AttributesController extends AppController
$this->set('object', $attribute[0]['Attribute']);
$this->set('seed', $seed);
}
public function enrich($id)
{
$conditions = $this->__idToConditions($id);
$attributes = $this->Attribute->fetchAttributes($this->Auth->user(), ['conditions' => $conditions, 'flatten' => true]);
if (empty($attributes)) {
throw new MethodNotAllowedException(__('Invalid Attribute'));
}
$attribute = $attributes[0];
if (!$this->request->is('post') || !$this->_isRest()) {
throw new MethodNotAllowedException(__('This endpoint allows for API POST requests only.'));
}
$modules = [];
foreach ($this->request->data as $module => $enabled) {
if ($enabled) {
$modules[] = $module;
}
}
$result = $this->Attribute->enrichmentRouter([
'user' => $this->Auth->user(),
'id' => $attribute['Attribute']['id'],
'modules' => $modules
]);
return $this->RestResponse->successResponse(0, $result);
}
}

1
app/Controller/Component/ACLComponent.php
View File

@ -65,6 +65,7 @@ class ACLComponent extends Component
'edit' => array('perm_add'),
'editField' => array('perm_add'),
'editSelected' => array('perm_add'),
'enrich' => ['perm_add'],
'exportSearch' => array('*'),
'fetchEditForm' => array('perm_add'),
'fetchViewValue' => array('*'),

4
app/Controller/Component/RestResponseComponent.php
View File

@ -648,6 +648,9 @@ class RestResponseComponent extends Component
} else {
$prettyPrint = !$this->isAutomaticTool(); // Do not pretty print response for automatic tools
$response = JsonTool::encode($response, $prettyPrint);
if ($format !== 'json' && $format !== 'application/json') {
$response = h($response);
}
}
} else {
if ($dumpSql) {
@ -669,7 +672,6 @@ class RestResponseComponent extends Component
$tmpFile->writeWithSeparator($response, null);
$response = $tmpFile;
}
if ($response instanceof TmpFileTool) {
$requestEtag = $this->requestEtag();
if ($requestEtag !== null) {

4
app/Controller/Component/RestSearchComponent.php
View File

@ -214,8 +214,8 @@ class RestSearchComponent extends Component
'galaxy_uuid',
'version',
'distribution',
'org',
'orgc',
'org_id',
'orgc_id',
'tag_name',
'custom',
'sgReferenceOnly',

2
app/Controller/ModulesController.php
View File

@ -22,7 +22,7 @@ class ModulesController extends AppController
if (!Configure::read('Plugin.Enrichment_' . $modname . '_enabled')) {
throw new MethodNotAllowedException('Module not found or not available.');
}
if (!$this->Module->canUse($this->Auth->user(), 'Enrichment', $modname)) {
if (!$this->Module->canUse($this->Auth->user(), 'Enrichment', $module)) {
throw new MethodNotAllowedException('Module not found or not available.');
}
$options = array();

142
app/Model/Attribute.php
View File

@ -1101,6 +1101,12 @@ class Attribute extends AppModel
$params[$tag_key] = $this->dissectArgs($params[$tag_key]);
foreach (array(0, 1, 2) as $tag_operator) {
$tagArray[$tag_operator] = $tag->fetchTagIdsSimple($params[$tag_key][$tag_operator]);
// If at least one of the ANDed tags is not found, invalidate the entire query by setting the lookup equal -1
if ($tag_operator === 2) {
if (count($params[$tag_key][2]) !== count($tagArray[2])) {
$tagArray[2] = [-1];
}
}
}
$temp = array();
if (!empty($tagArray[0])) {
@ -1126,7 +1132,7 @@ class Attribute extends AppModel
'tag_id' => $tagArray[0]
),
'fields' => array(
$options['scope'] === 'Event' ? 'Event.id' : 'attribute_id'
$options['scope'] === 'Event' ? 'event_id' : 'attribute_id'
)
);
$lookup_field = $options['scope'] === 'Event' ? 'Event.id' : 'Attribute.id';
@ -1200,7 +1206,7 @@ class Attribute extends AppModel
'tag_id' => $anded_tag
),
'fields' => array(
$options['scope'] === 'Event' ? 'Event.id' : 'attribute_id'
$options['scope'] === 'Event' ? 'event_id' : 'attribute_id'
)
);
$lookup_field = $options['scope'] === 'Event' ? 'Event.id' : 'Attribute.id';
@ -3871,4 +3877,136 @@ class Attribute extends AppModel
'fields' => ['Attribute.id', 'Attribute.uuid']
]);
}
public function enrichmentRouter($options)
{
if (Configure::read('MISP.background_jobs')) {
/** @var Job $job */
$job = ClassRegistry::init('Job');
$jobId = $job->createJob(
$options['user'],
Job::WORKER_PRIO,
'enrichment',
'Attribute ID: ' . $options['id'] . ' modules: ' . json_encode($options['modules']),
'Enriching attribute.'
);
$this->getBackgroundJobsTool()->enqueue(
BackgroundJobsTool::PRIO_QUEUE,
BackgroundJobsTool::CMD_EVENT,
[
'attribute_enrichment',
$options['user']['id'],
$options['id'],
json_encode($options['modules']),
$jobId
],
true,
$jobId
);
return __('Job queued (job ID: %s).', $jobId);
} else {
$result = $this->enrichment($options);
return __('#' . $result . ' attributes have been created during the enrichment process.');
}
}
public function enrichment($params)
{
$option_fields = ['user', 'id', 'modules'];
foreach ($option_fields as $option_field) {
if (empty($params[$option_field])) {
throw new MethodNotAllowedException(__('%s not set', $option_field));
}
}
$attribute = $this->fetchAttributes($params['user'], [
'conditions' => [
'Attribute.id' => $params['id'],
],
'withAttachments' => 1,
]);
if (empty($attribute)) {
throw new MethodNotAllowedException('Invalid attribute.');
}
$attribute = $attribute[0]['Attribute'];
$this->Module = ClassRegistry::init('Module');
$enabledModules = $this->Module->getEnabledModules($params['user']);
if (empty($enabledModules) || is_string($enabledModules)) {
return true;
}
$options = array();
foreach ($enabledModules['modules'] as $k => $temp) {
if (isset($temp['meta']['config'])) {
$settings = array();
foreach ($temp['meta']['config'] as $conf) {
$settings[$conf] = Configure::read('Plugin.Enrichment_' . $temp['name'] . '_' . $conf);
}
$enabledModules['modules'][$k]['config'] = $settings;
}
}
$attributes_added = 0;
$initial_objects = array();
$event_id = $attribute['event_id'];
$event = $this->Event->find('first', ['conditions' => ['Event.id' => $event_id], 'recursive' => -1]);
if (empty($event)) {
throw new MethodNotAllowedException('Invalid event.');
}
$object_id = $attribute['object_id'];
if ($object_id != '0' && empty($initial_objects[$object_id])) {
$initial_objects[$object_id] = $this->Event->fetchInitialObject($event_id, $object_id);
}
foreach ($enabledModules['modules'] as $module) {
if (in_array($module['name'], $params['modules'])) {
if (in_array($attribute['type'], $module['mispattributes']['input'])) {
$data = array('module' => $module['name'], 'event_id' => $event_id, 'attribute_uuid' => $attribute['uuid']);
if (!empty($module['config'])) {
$data['config'] = $module['config'];
}
if (!empty($module['mispattributes']['format']) && $module['mispattributes']['format'] == 'misp_standard') {
$data['attribute'] = $attribute;
} else {
$data[$attribute['type']] = $attribute['value'];
}
if ($object_id != '0' && !empty($initial_objects[$object_id])) {
$attribute['Object'] = $initial_objects[$object_id]['Object'];
}
$triggerData = $event;
$triggerData['Attribute'] = [$attribute];
$result = $this->Module->queryModuleServer($data, false, 'Enrichment', false, $triggerData);
if ($result === false) {
throw new MethodNotAllowedException(h($module['name']) . ' service not reachable.');
} else if (!is_array($result)) {
continue;
}
if (!empty($module['mispattributes']['format']) && $module['mispattributes']['format'] == 'misp_standard') {
if ($object_id != '0' && !empty($initial_objects[$object_id])) {
$result['initialObject'] = $initial_objects[$object_id];
}
$default_comment = $attribute['value'] . ': enriched via the ' . $module['name'] . ' module.';
$attributes_added += $this->Event->processModuleResultsData($params['user'], $result['results'], $event_id, $default_comment, false, false, true);
} else {
$attributes = $this->Event->handleModuleResult($result, $event_id);
foreach ($attributes as $a) {
$this->create();
$a['distribution'] = $attribute['distribution'];
$a['sharing_group_id'] = $attribute['sharing_group_id'];
$comment = 'Attribute #' . $attribute['id'] . ' enriched by ' . $module['name'] . '.';
if (!empty($a['comment'])) {
$a['comment'] .= PHP_EOL . $comment;
} else {
$a['comment'] = $comment;
}
$a['type'] = empty($a['default_type']) ? $a['types'][0] : $a['default_type'];
$result = $this->save($a);
if ($result) {
$attributes_added++;
}
}
}
}
}
}
return $attributes_added;
}
}

2
app/Model/Event.php
View File

@ -1532,7 +1532,7 @@ class Event extends AppModel
'eventid' => array('function' => 'set_filter_eventid', 'pop' => true),
'eventinfo' => array('function' => 'set_filter_eventinfo'),
'ignore' => array('function' => 'set_filter_ignore'),
'tags' => array('function' => 'set_filter_tags'),
'tags' => array('function' => 'set_filter_tags', 'pop' => true),
'event_tags' => array('function' => 'set_filter_tags', 'pop' => true),
'from' => array('function' => 'set_filter_timestamp', 'pop' => true),
'to' => array('function' => 'set_filter_timestamp', 'pop' => true),

2
app/Model/Server.php
View File

@ -6254,6 +6254,7 @@ class Server extends AppModel
'value' => null,
'type' => 'string',
'null' => true,
'cli_only' => true,
],
'menu_custom_right_link_html' => [
'level' => self::SETTING_OPTIONAL,
@ -6261,6 +6262,7 @@ class Server extends AppModel
'value' => null,
'type' => 'string',
'null' => true,
'cli_only' => true,
],
'enable_synchronisation_filtering_on_type' => [
'level' => self::SETTING_OPTIONAL,

2
app/View/Elements/genericElements/SideMenu/side_menu.ctp
View File

@ -366,7 +366,7 @@ $divider = '<li class="divider"></li>';
'url' => $baseurl . '/collections/index',
'text' => __('List Collections')
));
if ($this->Acl->canAccess('collection', 'add')) {
if ($this->Acl->canAccess('collections', 'add')) {
echo $this->element('/genericElements/SideMenu/side_menu_link', array(
'element_id' => 'add',
'url' => $baseurl . '/collections/add',

19
app/files/community-metadata/defaults.json
View File

@ -229,5 +229,22 @@
"misp_project_vetted": false,
"scope_of_data_to_be_shared": "",
"self_registration": false
}
},
{
"name": "misp-lea.org",
"logo": "https://www.misp-lea.org/assets/img/misp-lea.png",
"uuid": "3848ab01-e06f-4f91-a516-05e550dbc659",
"org_uuid": "89ed13e7-d560-4d48-8ee7-a3aadad5e0ae",
"org_name": "misp-lea.org",
"description": "MISP-LEA project consists in an law enforcement agency information sharing community. It’s powered by MISP and AIL project, two leading open source projects led by CIRCL. The community is only accessible to law enforcement agencies.",
"url": "https://www.misp-lea.org/",
"sector": "Law Enforcement",
"nationality": "International",
"type": "Information Sharing Community",
"email": "info@misp-lea.org",
"pgp_key": null,
"misp_project_vetted": true,
"scope_of_data_to_be_shared": "",
"self_registration": false
}
]