new: [topology UI] added

refacto/CRUDComponent
iglocska 2023-10-05 11:05:20 +02:00
parent a0fedb011c
commit b01a3bf83e
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
9 changed files with 533 additions and 6 deletions

2
.gitignore vendored
View File

@ -6,6 +6,8 @@ tmp
vendor vendor
webroot/theme/node_modules webroot/theme/node_modules
webroot/scss/*.css webroot/scss/*.css
webroot/js/node_modules/
!webroot/js/node_modules/mermaid/dist/
.vscode .vscode
docker/run/ docker/run/
.phpunit.result.cache .phpunit.result.cache

View File

@ -40,5 +40,4 @@ class AuditLogsController extends AppController
{ {
$this->CRUD->filtering(); $this->CRUD->filtering();
} }
} }

View File

@ -181,4 +181,11 @@ class InstanceController extends AppController
} }
} }
} }
public function topology()
{
$this->set('title', __('Topology'));
$this->set('description', __('A list of all instances and local tools connected .'));
$this->set('data', $this->Instance->getTopology());
}
} }

View File

@ -4,6 +4,8 @@ namespace CommonConnectorTools;
use Cake\ORM\Locator\LocatorAwareTrait; use Cake\ORM\Locator\LocatorAwareTrait;
use Cake\Log\Log; use Cake\Log\Log;
use Cake\Log\Engine\FileLog; use Cake\Log\Engine\FileLog;
use Cake\Utility\Hash;
class CommonConnectorTools class CommonConnectorTools
{ {
@ -88,6 +90,74 @@ class CommonConnectorTools
return true; return true;
} }
public function getOrganisation(string $uuid): ?array
{
$organisations = \Cake\ORM\TableRegistry::getTableLocator()->get('Organisations');
$org = $organisations->find()->where(['Organisations.uuid' => $uuid])->disableHydration()->first();
return $org;
}
public function getOrganisations(): array
{
$organisations = \Cake\ORM\TableRegistry::getTableLocator()->get('Organisations');
$orgs = $organisations->find()->disableHydration()->toArray();
return $orgs;
}
public function getFilteredOrganisations($filters, $returnObjects = false): array
{
$organisations = \Cake\ORM\TableRegistry::getTableLocator()->get('Organisations');
$orgs = $organisations->find();
$filterFields = ['type', 'nationality', 'sector'];
foreach ($filterFields as $fieldField) {
if (!empty($filters[$fieldField]) && $filters[$fieldField] !== 'ALL') {
$orgs = $orgs->where([$fieldField => $filters[$fieldField]]);
}
}
if (!empty($filters['local']) && $filters['local'] !== '0') {
$users = \Cake\ORM\TableRegistry::getTableLocator()->get('users');
$org_ids = array_values(array_unique($users->find('list', [
'valueField' => 'organisation_id'
])->toArray()));
$orgs = $orgs->where(['id IN' => $org_ids]);
}
if ($returnObjects) {
$orgs = $orgs->toArray();
} else {
$orgs = $orgs->disableHydration()->all();
}
return $orgs;
}
public function getOrganisationSelectorValues(): array
{
$results = [];
$orgTable = \Cake\ORM\TableRegistry::getTableLocator()->get('Organisations');
$fields = [
'nationality' => 'nat',
'sector' => 'sect',
'type' => 'typ'
];
foreach ($fields as $field => $temp_field) {
$temp = Hash::extract(
$orgTable->find()
->select([$temp_field => 'DISTINCT (' . $field . ')'])
->order([$temp_field => 'DESC'])
->disableHydration()->toArray(),
'{n}.' . $temp_field
);
foreach ($temp as $k => $v) {
if (empty($v)) {
unset($temp[$k]);
}
}
asort($temp, SORT_FLAG_CASE | SORT_NATURAL);
$temp = array_merge(['ALL' => 'ALL'], $temp);
$results[$field] = array_combine($temp, $temp);
}
return $results;
}
public function captureSharingGroup($input): bool public function captureSharingGroup($input): bool
{ {
if (empty($input['uuid'])) { if (empty($input['uuid'])) {

View File

@ -56,6 +56,30 @@ class MispConnector extends CommonConnectorTools
], ],
'redirect' => 'organisationsAction' 'redirect' => 'organisationsAction'
], ],
'fetchSelectedOrganisationsAction' => [
'type' => 'formAction',
'scope' => 'childAction',
'params' => [
'uuid'
],
'redirect' => 'organisationsAction'
],
'pushOrganisationAction' => [
'type' => 'formAction',
'scope' => 'childAction',
'params' => [
'uuid'
],
'redirect' => 'organisationsAction'
],
'pushOrganisationsAction' => [
'type' => 'formAction',
'scope' => 'childAction',
'params' => [
'uuid'
],
'redirect' => 'organisationsAction'
],
'fetchSharingGroupAction' => [ 'fetchSharingGroupAction' => [
'type' => 'formAction', 'type' => 'formAction',
'scope' => 'childAction', 'scope' => 'childAction',
@ -229,9 +253,6 @@ class MispConnector extends CommonConnectorTools
$list = explode('.', $params['sort']); $list = explode('.', $params['sort']);
$params['sort'] = end($list); $params['sort'] = end($list);
} }
if (!isset($params['limit'])) {
$params['limit'] = 50;
}
$url = $this->urlAppendParams($url, $params); $url = $this->urlAppendParams($url, $params);
$response = $this->HTTPClientGET($url, $params['connection']); $response = $this->HTTPClientGET($url, $params['connection']);
if ($response->isOk()) { if ($response->isOk()) {
@ -261,6 +282,9 @@ class MispConnector extends CommonConnectorTools
if ($response->isOk()) { if ($response->isOk()) {
return $response; return $response;
} else { } else {
if (!empty($params['softError'])) {
return $response;
}
$errorMsg = __('Could not post to the requested resource for `{0}`. Remote returned:', $url) . PHP_EOL . $response->getStringBody(); $errorMsg = __('Could not post to the requested resource for `{0}`. Remote returned:', $url) . PHP_EOL . $response->getStringBody();
$this->logError($errorMsg); $this->logError($errorMsg);
throw new NotFoundException($errorMsg); throw new NotFoundException($errorMsg);
@ -563,6 +587,48 @@ class MispConnector extends CommonConnectorTools
$urlParams = h($params['connection']['id']) . '/organisationsAction'; $urlParams = h($params['connection']['id']) . '/organisationsAction';
$response = $this->getData('/organisations/index', $params); $response = $this->getData('/organisations/index', $params);
$data = $response->getJson(); $data = $response->getJson();
$temp = $this->getOrganisations();
$existingOrgs = [];
foreach ($temp as $k => $v) {
$existingOrgs[$v['uuid']] = $v;
unset($temp[$k]);
}
$statusLevels = [
'same' => [
'colour' => 'success',
'message' => __('Remote organisation is the same as local copy'),
'icon' => 'check-circle'
],
'different' => [
'colour' => 'warning',
'message' => __('Local and remote versions of the organisations are different.'),
'icon' => 'exclamation-circle'
],
'not_found' => [
'colour' => 'danger',
'message' => __('Local organisation not found'),
'icon' => 'exclamation-triangle'
]
];
foreach ($data as $k => $v) {
$data[$k]['Organisation']['local_copy'] = false;
if (!empty($existingOrgs[$v['Organisation']['uuid']])) {
$remoteOrg = $existingOrgs[$v['Organisation']['uuid']];
$localOrg = $v['Organisation'];
$same = true;
$fieldsToCheck = [
'nationality', 'sector', 'type', 'name'
];
foreach (['nationality', 'sector', 'type', 'name'] as $fieldToCheck) {
if ($remoteOrg[$fieldToCheck] != $localOrg[$fieldToCheck]) {
$same = false;
}
}
$data[$k]['Organisation']['local_copy'] = $same ? 'same' : 'different';
} else {
$data[$k]['Organisation']['local_copy'] = 'not_found';
}
}
if (!empty($data)) { if (!empty($data)) {
return [ return [
'type' => 'index', 'type' => 'index',
@ -571,6 +637,25 @@ class MispConnector extends CommonConnectorTools
'skip_pagination' => 1, 'skip_pagination' => 1,
'top_bar' => [ 'top_bar' => [
'children' => [ 'children' => [
[
'type' => 'simple',
'children' => [
[
'class' => 'hidden mass-select',
'text' => __('Fetch selected organisations'),
'html' => '<i class="fas fa-download"></i> ',
'reload_url' => '/localTools/action/' . h($params['connection']['id']) . '/organisationsAction',
'popover_url' => '/localTools/action/' . h($params['connection']['id']) . '/fetchSelectedOrganisationsAction'
],
[
'text' => __('Push organisations'),
'html' => '<i class="fas fa-upload"></i> ',
'class' => 'btn btn-primary',
'reload_url' => '/localTools/action/' . h($params['connection']['id']) . '/organisationsAction',
'popover_url' => '/localTools/action/' . h($params['connection']['id']) . '/pushOrganisationsAction'
]
]
],
[ [
'type' => 'search', 'type' => 'search',
'button' => __('Search'), 'button' => __('Search'),
@ -582,11 +667,32 @@ class MispConnector extends CommonConnectorTools
] ]
], ],
'fields' => [ 'fields' => [
[
'element' => 'selector',
'class' => 'short',
'data' => [
'id' => [
'value_path' => 'Organisation.uuid'
]
]
],
[ [
'name' => 'Name', 'name' => 'Name',
'sort' => 'Organisation.name', 'sort' => 'Organisation.name',
'data_path' => 'Organisation.name', 'data_path' => 'Organisation.name',
], ],
[
'name' => 'Status',
'sort' => 'Organisation.local_copy',
'data_path' => 'Organisation.local_copy',
'element' => 'status',
'status_levels' => $statusLevels
],
[
'name' => 'uuid',
'sort' => 'Organisation.uuid',
'data_path' => 'Organisation.uuid'
],
[ [
'name' => 'uuid', 'name' => 'uuid',
'sort' => 'Organisation.uuid', 'sort' => 'Organisation.uuid',
@ -597,6 +703,11 @@ class MispConnector extends CommonConnectorTools
'sort' => 'Organisation.nationality', 'sort' => 'Organisation.nationality',
'data_path' => 'Organisation.nationality' 'data_path' => 'Organisation.nationality'
], ],
[
'name' => 'local',
'sort' => 'Organisation.local',
'data_path' => 'Organisation.local'
],
[ [
'name' => 'sector', 'name' => 'sector',
'sort' => 'Organisation.sector', 'sort' => 'Organisation.sector',
@ -607,11 +718,33 @@ class MispConnector extends CommonConnectorTools
'description' => false, 'description' => false,
'pull' => 'right', 'pull' => 'right',
'actions' => [ 'actions' => [
[
'open_modal' => '/localTools/action/' . h($params['connection']['id']) . '/pushOrganisationAction?uuid={{0}}',
'modal_params_data_path' => ['Organisation.uuid'],
'icon' => 'upload',
'reload_url' => '/localTools/action/' . h($params['connection']['id']) . '/organisationsAction',
'complex_requirement' => [
'function' => function ($row, $options) {
if ($row['Organisation']['local_copy'] === 'different') {
return true;
}
return false;
}
]
],
[ [
'open_modal' => '/localTools/action/' . h($params['connection']['id']) . '/fetchOrganisationAction?uuid={{0}}', 'open_modal' => '/localTools/action/' . h($params['connection']['id']) . '/fetchOrganisationAction?uuid={{0}}',
'modal_params_data_path' => ['Organisation.uuid'], 'modal_params_data_path' => ['Organisation.uuid'],
'icon' => 'download', 'icon' => 'download',
'reload_url' => '/localTools/action/' . h($params['connection']['id']) . '/organisationsAction' 'reload_url' => '/localTools/action/' . h($params['connection']['id']) . '/organisationsAction',
'complex_requirement' => [
'function' => function ($row, $options) {
if ($row['Organisation']['local_copy'] === 'different' || $row['Organisation']['local_copy'] === 'not_found') {
return true;
}
return false;
}
]
] ]
] ]
] ]
@ -703,7 +836,7 @@ class MispConnector extends CommonConnectorTools
return [ return [
'data' => [ 'data' => [
'title' => __('Fetch organisation'), 'title' => __('Fetch organisation'),
'description' => __('Fetch and create/update organisation ({0}) from MISP.', $params['uuid']), 'description' => __('Fetch and create/update organisation ({0}) from MISP?', $params['uuid']),
'submit' => [ 'submit' => [
'action' => $params['request']->getParam('action') 'action' => $params['request']->getParam('action')
], ],
@ -726,6 +859,164 @@ class MispConnector extends CommonConnectorTools
throw new MethodNotAllowedException(__('Invalid http request type for the given action.')); throw new MethodNotAllowedException(__('Invalid http request type for the given action.'));
} }
public function fetchSelectedOrganisationsAction(array $params): array
{
$ids = $params['request']->getQuery('ids');
if ($params['request']->is(['get'])) {
return [
'data' => [
'title' => __('Fetch organisations'),
'description' => __('Fetch and create/update the selected {0} organisations from MISP?', count($ids)),
'submit' => [
'action' => $params['request']->getParam('action')
],
'url' => ['controller' => 'localTools', 'action' => 'action', $params['connection']['id'], 'fetchSelectedOrganisationsAction']
]
];
} elseif ($params['request']->is(['post'])) {
$successes = 0;
$errors = 0;
foreach ($ids as $id) {
$response = $this->getData('/organisations/view/' . $id, $params);
$result = $this->captureOrganisation($response->getJson()['Organisation']);
if ($response->getStatusCode() == 200) {
$successes++;
} else {
$errors++;
}
}
if ($successes) {
return ['success' => 1, 'message' => __('The fetching of organisations has succeeded. {0} organisations created/modified and {1} organisations could not be created/modified.', $successes, $errors)];
} else {
return ['success' => 0, 'message' => __('The fetching of organisations has failed. {0} organisations could not be created/modified.', $errors)];
}
}
throw new MethodNotAllowedException(__('Invalid http request type for the given action.'));
}
public function pushOrganisationAction(array $params): array
{
if ($params['request']->is(['get'])) {
return [
'data' => [
'title' => __('Push organisation'),
'description' => __('Push or update organisation ({0}) on MISP.', $params['uuid']),
'submit' => [
'action' => $params['request']->getParam('action')
],
'url' => ['controller' => 'localTools', 'action' => 'action', $params['connection']['id'], 'pushOrganisationAction', $params['uuid']]
]
];
} elseif ($params['request']->is(['post'])) {
$org = $this->getOrganisation($params['uuid']);
if (empty($org)) {
return ['success' => 0, 'message' => __('Could not find the organisation.')];
}
$params['body'] = json_encode($org);
$response = $this->getData('/organisations/view/' . $params['uuid'], $params);
if ($response->getStatusCode() == 200) {
$response = $this->postData('/admin/organisations/edit/' . $params['uuid'], $params);
$result = $this->captureOrganisation($response->getJson()['Organisation']);
if ($response->getStatusCode() == 200 && $result) {
return ['success' => 1, 'message' => __('Organisation modified.')];
} else {
return ['success' => 0, 'message' => __('Could not save the changes to the organisation.')];
}
} else {
$response = $this->postData('/admin/organisations/add/', $params);
$result = $this->captureOrganisation($response->getJson()['Organisation']);
if ($response->getStatusCode() == 200 && $result) {
return ['success' => 1, 'message' => __('Organisation created.')];
} else {
return ['success' => 0, 'message' => __('Could not create the organisation.')];
}
}
}
throw new MethodNotAllowedException(__('Invalid http request type for the given action.'));
}
public function pushOrganisationsAction(array $params): array
{
$orgSelectorValues = $this->getOrganisationSelectorValues();
if ($params['request']->is(['get'])) {
return [
'data' => [
'title' => __('Push organisation'),
'description' => __('Push or update organisations on MISP.'),
'fields' => [
[
'field' => 'local',
'label' => __('Only organisations with users'),
'type' => 'checkbox'
],
[
'field' => 'type',
'label' => __('Type'),
'type' => 'select',
'options' => $orgSelectorValues['type']
],
[
'field' => 'sector',
'label' => __('Sector'),
'type' => 'select',
'options' => $orgSelectorValues['sector']
],
[
'field' => 'nationality',
'label' => __('Country'),
'type' => 'select',
'options' => $orgSelectorValues['nationality']
]
],
'submit' => [
'action' => $params['request']->getParam('action')
],
'url' => ['controller' => 'localTools', 'action' => 'action', $params['connection']['id'], 'pushOrganisationAction']
]
];
} elseif ($params['request']->is(['post'])) {
$filters = $params['request']->getData();
$orgs = $this->getFilteredOrganisations($filters, true);
$created = 0;
$modified = 0;
$errors = 0;
$params['softError'] = 1;
if (empty($orgs)) {
return ['success' => 0, 'message' => __('Could not find any organisations matching the criteria.')];
}
foreach ($orgs as $org) {
$params['body'] = null;
$response = $this->getData('/organisations/view/' . $org->uuid, $params);
if ($response->getStatusCode() == 200) {
$params['body'] = json_encode($org);
$response = $this->postData('/admin/organisations/edit/' . $org->uuid, $params);
if ($response->getStatusCode() == 200) {
$modified++;
} else {
$errors++;
}
} else {
$params['body'] = json_encode($org);
$response = $this->postData('/admin/organisations/add', $params);
if ($response->getStatusCode() == 200) {
$created++;
} else {
$errors++;
}
}
}
if ($created || $modified) {
return ['success' => 1, 'message' => __('Organisations created: {0}, modified: {1}, errors: {2}', $created, $modified, $errors)];
} else {
return ['success' => 0, 'message' => __('Organisations could not be pushed. Errors: {0}', $errors)];
}
}
throw new MethodNotAllowedException(__('Invalid http request type for the given action.'));
}
public function fetchSharingGroupAction(array $params): array public function fetchSharingGroupAction(array $params): array
{ {
if ($params['request']->is(['get'])) { if ($params['request']->is(['get'])) {

View File

@ -236,4 +236,128 @@ class InstanceTable extends AppTable
} }
return $themes; return $themes;
} }
public function getTopology($mermaid = true): mixed
{
$BroodsModel = TableRegistry::getTableLocator()->get('Broods');
$LocalToolsModel = TableRegistry::getTableLocator()->get('LocalTools');
$connectors = $LocalToolsModel->getConnectors();
$connections = $LocalToolsModel->extractMeta($connectors, true);
$broods = $BroodsModel->find()->select(['id', 'uuid', 'url', 'name', 'pull'])->disableHydration()->toArray();
foreach ($broods as $k => $brood) {
$broods[$k]['status'] = $BroodsModel->queryStatus($brood['id']);
}
$data = [
'broods' => $broods,
'tools' => $LocalToolsModel->extractMeta($connectors, true)
];
if ($mermaid) {
return $this->generateTopologyMermaid($data);
}
return $data;
}
public function generateTopologyMermaid($data)
{
$version = json_decode(file_get_contents(APP . 'VERSION.json'), true)["version"];
$newest = $version;
$broods = '';
$edges = '';
// pre-run the loop to get the latest version
foreach ($data['broods'] as $brood) {
if ($brood['status']['code'] === 200) {
if (version_compare($brood['status']['response']['version'], $newest) > 0) {
$newest = $brood['status']['response']['version'];
}
}
}
foreach ($data['broods'] as $brood) {
$status = '';
if ($brood['status']['code'] === 200) {
$status = sprintf(
'<br />Ping: %sms<br />Version: <span class="%s">v%s</span><br />Role: %s<br />',
h($brood['status']['ping']),
$brood['status']['response']['version'] === $newest ? 'text-success' : 'text-danger',
h($brood['status']['response']['version']) . ($brood['status']['response']['version'] !== $newest ? ' - outdated' : ''),
h($brood['status']['response']['role']['name'])
);
}
$broods .= sprintf(
"%s%s end" . PHP_EOL,
sprintf(
' subgraph brood_%s[fas:fa-network-wired Brood #%s]' . PHP_EOL,
h($brood['id']),
h($brood['id'])
),
sprintf(
" cerebrate_%s[%s<br />%s<a href='/broods/view/%s'>fas:fa-eye</a>]" . PHP_EOL,
h($brood['id']),
'<span class="font-weight-bold">' . h($brood['name']) . '</span>',
sprintf(
"Connected: <span class='%s' title='%s'>%s</span>%s",
$brood['status']['code'] === 200 ? 'text-success' : 'text-danger',
h($brood['status']['code']),
$brood['status']['code'] === 200 ? 'fas:fa-check' : 'fas:fa-times',
$status
),
h($brood['id']),
)
);
$edges .= sprintf(
' C1%s---cerebrate_%s' . PHP_EOL,
$brood['pull'] ? '<' : '',
h($brood['id'])
);
}
$tools = '';
foreach ($data['tools'] as $tool) {
$tools .= sprintf(
' subgraph instance_local_tools_%s[%s %s connector]' . PHP_EOL . ' direction TB' . PHP_EOL,
h($tool['name']),
isset($tool['logo']) ? '<img src="/img/local_tools/' . h($tool['logo']) . '" style="width: 50px; height:50px;" />' : 'fas:fa-wrench',
h($tool['name'])
);
foreach ($tool['connections'] as $connection) {
$tools .= sprintf(
" %s[%s<br />%s<br />%s]" . PHP_EOL,
h($connection['name']),
h($connection['name']),
sprintf(
__('Health') . ': <span title="%s" class="%s">%s</span>',
h($connection['message']),
$connection['health'] === 1 ? 'text-success' : 'text-danger',
$connection['health'] === 1 ? 'fas:fa-check' : 'fas:fa-times'
),
sprintf(
"<a href='%s'>fas:fa-eye</a>",
h($connection['url'])
)
);
}
$tools .= ' end' . PHP_EOL;
}
$this_cerebrate = sprintf(
'C1[My Cerebrate<br />Version: <span class="%s">v%s</span>]',
$version === $newest ? 'text-success' : 'text-danger',
$version
);
$md = sprintf(
'flowchart TB
subgraph instance[fas:fa-network-wired My Brood]
direction TB
%s
subgraph instance_local_tools[fa:fa-tools Local Tools]
direction LR
%s
end
end
%s%s',
$this_cerebrate,
$tools,
$broods,
$edges
);
return $md;
}
} }

View File

@ -0,0 +1,10 @@
<?php
$data = $this->Hash->extract($row, $field['data_path'])[0];
$status_levels = $field['status_levels'];
echo sprintf(
'<i class="text-%s fas fa-%s" title="%s"></i>',
h($field['status_levels'][$data]['colour']),
empty($field['status_levels'][$data]['icon']) ? 'circle' : h($field['status_levels'][$data]['icon']),
h($field['status_levels'][$data]['message'])
);
?>

View File

@ -73,6 +73,25 @@
<script> <script>
function openModalForButton<?= $seed ?>(clicked, url, reloadUrl='') { function openModalForButton<?= $seed ?>(clicked, url, reloadUrl='') {
var selected_ids = [];
$('.selectable_row:checkbox:checked').each(function () {
selected_ids.push($(this).data('id'));
});
if (selected_ids.length > 0) {
if (url.includes('?')) {
url += '&';
} else {
url += '?';
}
var first = true;
selected_ids.forEach(function (id) {
if (!first) {
url += '&';
}
url += 'ids[]=' + id;
first = false;
});
}
const fallbackReloadUrl = '<?= $this->Url->build(['action' => 'index']); ?>' const fallbackReloadUrl = '<?= $this->Url->build(['action' => 'index']); ?>'
reloadUrl = reloadUrl != '' ? reloadUrl : fallbackReloadUrl reloadUrl = reloadUrl != '' ? reloadUrl : fallbackReloadUrl
UI.overlayUntilResolve(clicked, UI.submissionModalForIndex(url, reloadUrl, '<?= $tableRandomValue ?>')) UI.overlayUntilResolve(clicked, UI.submissionModalForIndex(url, reloadUrl, '<?= $tableRandomValue ?>'))

View File

@ -0,0 +1,5 @@
<pre class="mermaid"><?= $data ?></pre>
<script type="module">
import mermaid from '/js/node_modules/mermaid/dist/mermaid.esm.min.mjs';
mermaid.initialize({ startOnLoad: true });
</script>