From b01a3bf83e222a87e9997be217711ea62f121ed7 Mon Sep 17 00:00:00 2001 From: iglocska Date: Thu, 5 Oct 2023 11:05:20 +0200 Subject: [PATCH] new: [topology UI] added --- .gitignore | 2 + src/Controller/AuditLogsController.php | 1 - src/Controller/InstanceController.php | 7 + .../CommonConnectorTools.php | 70 ++++ .../local_tool_connectors/MispConnector.php | 301 +++++++++++++++++- src/Model/Table/InstanceTable.php | 124 ++++++++ .../IndexTable/Fields/status.php | 10 + .../ListTopBar/element_simple.php | 19 ++ templates/element/genericElements/mermaid.php | 5 + 9 files changed, 533 insertions(+), 6 deletions(-) create mode 100644 templates/element/genericElements/IndexTable/Fields/status.php create mode 100644 templates/element/genericElements/mermaid.php diff --git a/.gitignore b/.gitignore index 4c169ac..592284b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ tmp vendor webroot/theme/node_modules webroot/scss/*.css +webroot/js/node_modules/ +!webroot/js/node_modules/mermaid/dist/ .vscode docker/run/ .phpunit.result.cache diff --git a/src/Controller/AuditLogsController.php b/src/Controller/AuditLogsController.php index 8836d49..65f191e 100644 --- a/src/Controller/AuditLogsController.php +++ b/src/Controller/AuditLogsController.php @@ -40,5 +40,4 @@ class AuditLogsController extends AppController { $this->CRUD->filtering(); } - } diff --git a/src/Controller/InstanceController.php b/src/Controller/InstanceController.php index e641940..6c8a77f 100644 --- a/src/Controller/InstanceController.php +++ b/src/Controller/InstanceController.php @@ -181,4 +181,11 @@ class InstanceController extends AppController } } } + + public function topology() + { + $this->set('title', __('Topology')); + $this->set('description', __('A list of all instances and local tools connected .')); + $this->set('data', $this->Instance->getTopology()); + } } diff --git a/src/Lib/default/local_tool_connectors/CommonConnectorTools.php b/src/Lib/default/local_tool_connectors/CommonConnectorTools.php index d01e9d9..cacd34c 100644 --- a/src/Lib/default/local_tool_connectors/CommonConnectorTools.php +++ b/src/Lib/default/local_tool_connectors/CommonConnectorTools.php @@ -4,6 +4,8 @@ namespace CommonConnectorTools; use Cake\ORM\Locator\LocatorAwareTrait; use Cake\Log\Log; use Cake\Log\Engine\FileLog; +use Cake\Utility\Hash; + class CommonConnectorTools { @@ -88,6 +90,74 @@ class CommonConnectorTools return true; } + public function getOrganisation(string $uuid): ?array + { + $organisations = \Cake\ORM\TableRegistry::getTableLocator()->get('Organisations'); + $org = $organisations->find()->where(['Organisations.uuid' => $uuid])->disableHydration()->first(); + return $org; + } + + public function getOrganisations(): array + { + $organisations = \Cake\ORM\TableRegistry::getTableLocator()->get('Organisations'); + $orgs = $organisations->find()->disableHydration()->toArray(); + return $orgs; + } + + public function getFilteredOrganisations($filters, $returnObjects = false): array + { + $organisations = \Cake\ORM\TableRegistry::getTableLocator()->get('Organisations'); + $orgs = $organisations->find(); + $filterFields = ['type', 'nationality', 'sector']; + foreach ($filterFields as $fieldField) { + if (!empty($filters[$fieldField]) && $filters[$fieldField] !== 'ALL') { + $orgs = $orgs->where([$fieldField => $filters[$fieldField]]); + } + } + if (!empty($filters['local']) && $filters['local'] !== '0') { + $users = \Cake\ORM\TableRegistry::getTableLocator()->get('users'); + $org_ids = array_values(array_unique($users->find('list', [ + 'valueField' => 'organisation_id' + ])->toArray())); + $orgs = $orgs->where(['id IN' => $org_ids]); + } + if ($returnObjects) { + $orgs = $orgs->toArray(); + } else { + $orgs = $orgs->disableHydration()->all(); + } + return $orgs; + } + + public function getOrganisationSelectorValues(): array + { + $results = []; + $orgTable = \Cake\ORM\TableRegistry::getTableLocator()->get('Organisations'); + $fields = [ + 'nationality' => 'nat', + 'sector' => 'sect', + 'type' => 'typ' + ]; + foreach ($fields as $field => $temp_field) { + $temp = Hash::extract( + $orgTable->find() + ->select([$temp_field => 'DISTINCT (' . $field . ')']) + ->order([$temp_field => 'DESC']) + ->disableHydration()->toArray(), + '{n}.' . $temp_field + ); + foreach ($temp as $k => $v) { + if (empty($v)) { + unset($temp[$k]); + } + } + asort($temp, SORT_FLAG_CASE | SORT_NATURAL); + $temp = array_merge(['ALL' => 'ALL'], $temp); + $results[$field] = array_combine($temp, $temp); + } + return $results; + } + public function captureSharingGroup($input): bool { if (empty($input['uuid'])) { diff --git a/src/Lib/default/local_tool_connectors/MispConnector.php b/src/Lib/default/local_tool_connectors/MispConnector.php index a5adb59..94f89e9 100644 --- a/src/Lib/default/local_tool_connectors/MispConnector.php +++ b/src/Lib/default/local_tool_connectors/MispConnector.php @@ -56,6 +56,30 @@ class MispConnector extends CommonConnectorTools ], 'redirect' => 'organisationsAction' ], + 'fetchSelectedOrganisationsAction' => [ + 'type' => 'formAction', + 'scope' => 'childAction', + 'params' => [ + 'uuid' + ], + 'redirect' => 'organisationsAction' + ], + 'pushOrganisationAction' => [ + 'type' => 'formAction', + 'scope' => 'childAction', + 'params' => [ + 'uuid' + ], + 'redirect' => 'organisationsAction' + ], + 'pushOrganisationsAction' => [ + 'type' => 'formAction', + 'scope' => 'childAction', + 'params' => [ + 'uuid' + ], + 'redirect' => 'organisationsAction' + ], 'fetchSharingGroupAction' => [ 'type' => 'formAction', 'scope' => 'childAction', @@ -229,9 +253,6 @@ class MispConnector extends CommonConnectorTools $list = explode('.', $params['sort']); $params['sort'] = end($list); } - if (!isset($params['limit'])) { - $params['limit'] = 50; - } $url = $this->urlAppendParams($url, $params); $response = $this->HTTPClientGET($url, $params['connection']); if ($response->isOk()) { @@ -261,6 +282,9 @@ class MispConnector extends CommonConnectorTools if ($response->isOk()) { return $response; } else { + if (!empty($params['softError'])) { + return $response; + } $errorMsg = __('Could not post to the requested resource for `{0}`. Remote returned:', $url) . PHP_EOL . $response->getStringBody(); $this->logError($errorMsg); throw new NotFoundException($errorMsg); @@ -563,6 +587,48 @@ class MispConnector extends CommonConnectorTools $urlParams = h($params['connection']['id']) . '/organisationsAction'; $response = $this->getData('/organisations/index', $params); $data = $response->getJson(); + $temp = $this->getOrganisations(); + $existingOrgs = []; + foreach ($temp as $k => $v) { + $existingOrgs[$v['uuid']] = $v; + unset($temp[$k]); + } + $statusLevels = [ + 'same' => [ + 'colour' => 'success', + 'message' => __('Remote organisation is the same as local copy'), + 'icon' => 'check-circle' + ], + 'different' => [ + 'colour' => 'warning', + 'message' => __('Local and remote versions of the organisations are different.'), + 'icon' => 'exclamation-circle' + ], + 'not_found' => [ + 'colour' => 'danger', + 'message' => __('Local organisation not found'), + 'icon' => 'exclamation-triangle' + ] + ]; + foreach ($data as $k => $v) { + $data[$k]['Organisation']['local_copy'] = false; + if (!empty($existingOrgs[$v['Organisation']['uuid']])) { + $remoteOrg = $existingOrgs[$v['Organisation']['uuid']]; + $localOrg = $v['Organisation']; + $same = true; + $fieldsToCheck = [ + 'nationality', 'sector', 'type', 'name' + ]; + foreach (['nationality', 'sector', 'type', 'name'] as $fieldToCheck) { + if ($remoteOrg[$fieldToCheck] != $localOrg[$fieldToCheck]) { + $same = false; + } + } + $data[$k]['Organisation']['local_copy'] = $same ? 'same' : 'different'; + } else { + $data[$k]['Organisation']['local_copy'] = 'not_found'; + } + } if (!empty($data)) { return [ 'type' => 'index', @@ -571,6 +637,25 @@ class MispConnector extends CommonConnectorTools 'skip_pagination' => 1, 'top_bar' => [ 'children' => [ + [ + 'type' => 'simple', + 'children' => [ + [ + 'class' => 'hidden mass-select', + 'text' => __('Fetch selected organisations'), + 'html' => ' ', + 'reload_url' => '/localTools/action/' . h($params['connection']['id']) . '/organisationsAction', + 'popover_url' => '/localTools/action/' . h($params['connection']['id']) . '/fetchSelectedOrganisationsAction' + ], + [ + 'text' => __('Push organisations'), + 'html' => ' ', + 'class' => 'btn btn-primary', + 'reload_url' => '/localTools/action/' . h($params['connection']['id']) . '/organisationsAction', + 'popover_url' => '/localTools/action/' . h($params['connection']['id']) . '/pushOrganisationsAction' + ] + ] + ], [ 'type' => 'search', 'button' => __('Search'), @@ -582,11 +667,32 @@ class MispConnector extends CommonConnectorTools ] ], 'fields' => [ + [ + 'element' => 'selector', + 'class' => 'short', + 'data' => [ + 'id' => [ + 'value_path' => 'Organisation.uuid' + ] + ] + ], [ 'name' => 'Name', 'sort' => 'Organisation.name', 'data_path' => 'Organisation.name', ], + [ + 'name' => 'Status', + 'sort' => 'Organisation.local_copy', + 'data_path' => 'Organisation.local_copy', + 'element' => 'status', + 'status_levels' => $statusLevels + ], + [ + 'name' => 'uuid', + 'sort' => 'Organisation.uuid', + 'data_path' => 'Organisation.uuid' + ], [ 'name' => 'uuid', 'sort' => 'Organisation.uuid', @@ -597,6 +703,11 @@ class MispConnector extends CommonConnectorTools 'sort' => 'Organisation.nationality', 'data_path' => 'Organisation.nationality' ], + [ + 'name' => 'local', + 'sort' => 'Organisation.local', + 'data_path' => 'Organisation.local' + ], [ 'name' => 'sector', 'sort' => 'Organisation.sector', @@ -607,11 +718,33 @@ class MispConnector extends CommonConnectorTools 'description' => false, 'pull' => 'right', 'actions' => [ + [ + 'open_modal' => '/localTools/action/' . h($params['connection']['id']) . '/pushOrganisationAction?uuid={{0}}', + 'modal_params_data_path' => ['Organisation.uuid'], + 'icon' => 'upload', + 'reload_url' => '/localTools/action/' . h($params['connection']['id']) . '/organisationsAction', + 'complex_requirement' => [ + 'function' => function ($row, $options) { + if ($row['Organisation']['local_copy'] === 'different') { + return true; + } + return false; + } + ] + ], [ 'open_modal' => '/localTools/action/' . h($params['connection']['id']) . '/fetchOrganisationAction?uuid={{0}}', 'modal_params_data_path' => ['Organisation.uuid'], 'icon' => 'download', - 'reload_url' => '/localTools/action/' . h($params['connection']['id']) . '/organisationsAction' + 'reload_url' => '/localTools/action/' . h($params['connection']['id']) . '/organisationsAction', + 'complex_requirement' => [ + 'function' => function ($row, $options) { + if ($row['Organisation']['local_copy'] === 'different' || $row['Organisation']['local_copy'] === 'not_found') { + return true; + } + return false; + } + ] ] ] ] @@ -703,7 +836,7 @@ class MispConnector extends CommonConnectorTools return [ 'data' => [ 'title' => __('Fetch organisation'), - 'description' => __('Fetch and create/update organisation ({0}) from MISP.', $params['uuid']), + 'description' => __('Fetch and create/update organisation ({0}) from MISP?', $params['uuid']), 'submit' => [ 'action' => $params['request']->getParam('action') ], @@ -726,6 +859,164 @@ class MispConnector extends CommonConnectorTools throw new MethodNotAllowedException(__('Invalid http request type for the given action.')); } + public function fetchSelectedOrganisationsAction(array $params): array + { + $ids = $params['request']->getQuery('ids'); + if ($params['request']->is(['get'])) { + return [ + 'data' => [ + 'title' => __('Fetch organisations'), + 'description' => __('Fetch and create/update the selected {0} organisations from MISP?', count($ids)), + 'submit' => [ + 'action' => $params['request']->getParam('action') + ], + 'url' => ['controller' => 'localTools', 'action' => 'action', $params['connection']['id'], 'fetchSelectedOrganisationsAction'] + ] + ]; + } elseif ($params['request']->is(['post'])) { + $successes = 0; + $errors = 0; + foreach ($ids as $id) { + $response = $this->getData('/organisations/view/' . $id, $params); + $result = $this->captureOrganisation($response->getJson()['Organisation']); + if ($response->getStatusCode() == 200) { + $successes++; + } else { + $errors++; + } + } + if ($successes) { + return ['success' => 1, 'message' => __('The fetching of organisations has succeeded. {0} organisations created/modified and {1} organisations could not be created/modified.', $successes, $errors)]; + } else { + return ['success' => 0, 'message' => __('The fetching of organisations has failed. {0} organisations could not be created/modified.', $errors)]; + } + } + throw new MethodNotAllowedException(__('Invalid http request type for the given action.')); + } + + public function pushOrganisationAction(array $params): array + { + if ($params['request']->is(['get'])) { + return [ + 'data' => [ + 'title' => __('Push organisation'), + 'description' => __('Push or update organisation ({0}) on MISP.', $params['uuid']), + 'submit' => [ + 'action' => $params['request']->getParam('action') + ], + 'url' => ['controller' => 'localTools', 'action' => 'action', $params['connection']['id'], 'pushOrganisationAction', $params['uuid']] + ] + ]; + } elseif ($params['request']->is(['post'])) { + $org = $this->getOrganisation($params['uuid']); + if (empty($org)) { + return ['success' => 0, 'message' => __('Could not find the organisation.')]; + } + $params['body'] = json_encode($org); + $response = $this->getData('/organisations/view/' . $params['uuid'], $params); + if ($response->getStatusCode() == 200) { + $response = $this->postData('/admin/organisations/edit/' . $params['uuid'], $params); + $result = $this->captureOrganisation($response->getJson()['Organisation']); + if ($response->getStatusCode() == 200 && $result) { + return ['success' => 1, 'message' => __('Organisation modified.')]; + } else { + return ['success' => 0, 'message' => __('Could not save the changes to the organisation.')]; + } + } else { + $response = $this->postData('/admin/organisations/add/', $params); + $result = $this->captureOrganisation($response->getJson()['Organisation']); + if ($response->getStatusCode() == 200 && $result) { + return ['success' => 1, 'message' => __('Organisation created.')]; + } else { + return ['success' => 0, 'message' => __('Could not create the organisation.')]; + } + } + } + throw new MethodNotAllowedException(__('Invalid http request type for the given action.')); + } + + public function pushOrganisationsAction(array $params): array + { + $orgSelectorValues = $this->getOrganisationSelectorValues(); + if ($params['request']->is(['get'])) { + return [ + 'data' => [ + 'title' => __('Push organisation'), + 'description' => __('Push or update organisations on MISP.'), + 'fields' => [ + [ + 'field' => 'local', + 'label' => __('Only organisations with users'), + 'type' => 'checkbox' + ], + [ + 'field' => 'type', + 'label' => __('Type'), + 'type' => 'select', + 'options' => $orgSelectorValues['type'] + + ], + [ + 'field' => 'sector', + 'label' => __('Sector'), + 'type' => 'select', + 'options' => $orgSelectorValues['sector'] + + ], + [ + 'field' => 'nationality', + 'label' => __('Country'), + 'type' => 'select', + 'options' => $orgSelectorValues['nationality'] + + ] + ], + 'submit' => [ + 'action' => $params['request']->getParam('action') + ], + 'url' => ['controller' => 'localTools', 'action' => 'action', $params['connection']['id'], 'pushOrganisationAction'] + ] + ]; + } elseif ($params['request']->is(['post'])) { + $filters = $params['request']->getData(); + $orgs = $this->getFilteredOrganisations($filters, true); + $created = 0; + $modified = 0; + $errors = 0; + $params['softError'] = 1; + if (empty($orgs)) { + return ['success' => 0, 'message' => __('Could not find any organisations matching the criteria.')]; + } + foreach ($orgs as $org) { + $params['body'] = null; + $response = $this->getData('/organisations/view/' . $org->uuid, $params); + if ($response->getStatusCode() == 200) { + $params['body'] = json_encode($org); + $response = $this->postData('/admin/organisations/edit/' . $org->uuid, $params); + if ($response->getStatusCode() == 200) { + $modified++; + } else { + $errors++; + } + } else { + $params['body'] = json_encode($org); + $response = $this->postData('/admin/organisations/add', $params); + if ($response->getStatusCode() == 200) { + $created++; + } else { + $errors++; + } + } + } + if ($created || $modified) { + return ['success' => 1, 'message' => __('Organisations created: {0}, modified: {1}, errors: {2}', $created, $modified, $errors)]; + } else { + return ['success' => 0, 'message' => __('Organisations could not be pushed. Errors: {0}', $errors)]; + } + } + throw new MethodNotAllowedException(__('Invalid http request type for the given action.')); + } + public function fetchSharingGroupAction(array $params): array { if ($params['request']->is(['get'])) { diff --git a/src/Model/Table/InstanceTable.php b/src/Model/Table/InstanceTable.php index afd04e0..83b29bd 100644 --- a/src/Model/Table/InstanceTable.php +++ b/src/Model/Table/InstanceTable.php @@ -236,4 +236,128 @@ class InstanceTable extends AppTable } return $themes; } + + public function getTopology($mermaid = true): mixed + { + $BroodsModel = TableRegistry::getTableLocator()->get('Broods'); + $LocalToolsModel = TableRegistry::getTableLocator()->get('LocalTools'); + $connectors = $LocalToolsModel->getConnectors(); + $connections = $LocalToolsModel->extractMeta($connectors, true); + $broods = $BroodsModel->find()->select(['id', 'uuid', 'url', 'name', 'pull'])->disableHydration()->toArray(); + foreach ($broods as $k => $brood) { + $broods[$k]['status'] = $BroodsModel->queryStatus($brood['id']); + } + $data = [ + 'broods' => $broods, + 'tools' => $LocalToolsModel->extractMeta($connectors, true) + ]; + if ($mermaid) { + return $this->generateTopologyMermaid($data); + } + return $data; + } + + public function generateTopologyMermaid($data) + { + $version = json_decode(file_get_contents(APP . 'VERSION.json'), true)["version"]; + $newest = $version; + $broods = ''; + $edges = ''; + // pre-run the loop to get the latest version + foreach ($data['broods'] as $brood) { + if ($brood['status']['code'] === 200) { + if (version_compare($brood['status']['response']['version'], $newest) > 0) { + $newest = $brood['status']['response']['version']; + } + } + } + foreach ($data['broods'] as $brood) { + $status = ''; + if ($brood['status']['code'] === 200) { + $status = sprintf( + '
Ping: %sms
Version: v%s
Role: %s
', + h($brood['status']['ping']), + $brood['status']['response']['version'] === $newest ? 'text-success' : 'text-danger', + h($brood['status']['response']['version']) . ($brood['status']['response']['version'] !== $newest ? ' - outdated' : ''), + h($brood['status']['response']['role']['name']) + ); + } + $broods .= sprintf( + "%s%s end" . PHP_EOL, + sprintf( + ' subgraph brood_%s[fas:fa-network-wired Brood #%s]' . PHP_EOL, + h($brood['id']), + h($brood['id']) + ), + sprintf( + " cerebrate_%s[%s
%sfas:fa-eye]" . PHP_EOL, + h($brood['id']), + '' . h($brood['name']) . '', + sprintf( + "Connected: %s%s", + $brood['status']['code'] === 200 ? 'text-success' : 'text-danger', + h($brood['status']['code']), + $brood['status']['code'] === 200 ? 'fas:fa-check' : 'fas:fa-times', + $status + ), + h($brood['id']), + ) + + ); + $edges .= sprintf( + ' C1%s---cerebrate_%s' . PHP_EOL, + $brood['pull'] ? '<' : '', + h($brood['id']) + ); + } + $tools = ''; + foreach ($data['tools'] as $tool) { + $tools .= sprintf( + ' subgraph instance_local_tools_%s[%s %s connector]' . PHP_EOL . ' direction TB' . PHP_EOL, + h($tool['name']), + isset($tool['logo']) ? '' : 'fas:fa-wrench', + h($tool['name']) + ); + foreach ($tool['connections'] as $connection) { + $tools .= sprintf( + " %s[%s
%s
%s]" . PHP_EOL, + h($connection['name']), + h($connection['name']), + sprintf( + __('Health') . ': %s', + h($connection['message']), + $connection['health'] === 1 ? 'text-success' : 'text-danger', + $connection['health'] === 1 ? 'fas:fa-check' : 'fas:fa-times' + ), + sprintf( + "fas:fa-eye", + h($connection['url']) + ) + ); + } + $tools .= ' end' . PHP_EOL; + } + $this_cerebrate = sprintf( + 'C1[My Cerebrate
Version: v%s]', + $version === $newest ? 'text-success' : 'text-danger', + $version + ); + $md = sprintf( +'flowchart TB + subgraph instance[fas:fa-network-wired My Brood] + direction TB + %s + subgraph instance_local_tools[fa:fa-tools Local Tools] + direction LR +%s + end + end +%s%s', + $this_cerebrate, + $tools, + $broods, + $edges + ); + return $md; + } } diff --git a/templates/element/genericElements/IndexTable/Fields/status.php b/templates/element/genericElements/IndexTable/Fields/status.php new file mode 100644 index 0000000..95d8544 --- /dev/null +++ b/templates/element/genericElements/IndexTable/Fields/status.php @@ -0,0 +1,10 @@ +Hash->extract($row, $field['data_path'])[0]; + $status_levels = $field['status_levels']; + echo sprintf( + '', + h($field['status_levels'][$data]['colour']), + empty($field['status_levels'][$data]['icon']) ? 'circle' : h($field['status_levels'][$data]['icon']), + h($field['status_levels'][$data]['message']) + ); +?> diff --git a/templates/element/genericElements/ListTopBar/element_simple.php b/templates/element/genericElements/ListTopBar/element_simple.php index 5573254..63b4ea8 100644 --- a/templates/element/genericElements/ListTopBar/element_simple.php +++ b/templates/element/genericElements/ListTopBar/element_simple.php @@ -73,6 +73,25 @@ \ No newline at end of file