diff --git a/.gitignore b/.gitignore
index 4c169ac..592284b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,8 @@ tmp
vendor
webroot/theme/node_modules
webroot/scss/*.css
+webroot/js/node_modules/
+!webroot/js/node_modules/mermaid/dist/
.vscode
docker/run/
.phpunit.result.cache
diff --git a/src/Controller/AuditLogsController.php b/src/Controller/AuditLogsController.php
index 8836d49..65f191e 100644
--- a/src/Controller/AuditLogsController.php
+++ b/src/Controller/AuditLogsController.php
@@ -40,5 +40,4 @@ class AuditLogsController extends AppController
{
$this->CRUD->filtering();
}
-
}
diff --git a/src/Controller/InstanceController.php b/src/Controller/InstanceController.php
index e641940..6c8a77f 100644
--- a/src/Controller/InstanceController.php
+++ b/src/Controller/InstanceController.php
@@ -181,4 +181,11 @@ class InstanceController extends AppController
}
}
}
+
+ public function topology()
+ {
+ $this->set('title', __('Topology'));
+ $this->set('description', __('A list of all instances and local tools connected .'));
+ $this->set('data', $this->Instance->getTopology());
+ }
}
diff --git a/src/Lib/default/local_tool_connectors/CommonConnectorTools.php b/src/Lib/default/local_tool_connectors/CommonConnectorTools.php
index d01e9d9..cacd34c 100644
--- a/src/Lib/default/local_tool_connectors/CommonConnectorTools.php
+++ b/src/Lib/default/local_tool_connectors/CommonConnectorTools.php
@@ -4,6 +4,8 @@ namespace CommonConnectorTools;
use Cake\ORM\Locator\LocatorAwareTrait;
use Cake\Log\Log;
use Cake\Log\Engine\FileLog;
+use Cake\Utility\Hash;
+
class CommonConnectorTools
{
@@ -88,6 +90,74 @@ class CommonConnectorTools
return true;
}
+ public function getOrganisation(string $uuid): ?array
+ {
+ $organisations = \Cake\ORM\TableRegistry::getTableLocator()->get('Organisations');
+ $org = $organisations->find()->where(['Organisations.uuid' => $uuid])->disableHydration()->first();
+ return $org;
+ }
+
+ public function getOrganisations(): array
+ {
+ $organisations = \Cake\ORM\TableRegistry::getTableLocator()->get('Organisations');
+ $orgs = $organisations->find()->disableHydration()->toArray();
+ return $orgs;
+ }
+
+ public function getFilteredOrganisations($filters, $returnObjects = false): array
+ {
+ $organisations = \Cake\ORM\TableRegistry::getTableLocator()->get('Organisations');
+ $orgs = $organisations->find();
+ $filterFields = ['type', 'nationality', 'sector'];
+ foreach ($filterFields as $fieldField) {
+ if (!empty($filters[$fieldField]) && $filters[$fieldField] !== 'ALL') {
+ $orgs = $orgs->where([$fieldField => $filters[$fieldField]]);
+ }
+ }
+ if (!empty($filters['local']) && $filters['local'] !== '0') {
+ $users = \Cake\ORM\TableRegistry::getTableLocator()->get('users');
+ $org_ids = array_values(array_unique($users->find('list', [
+ 'valueField' => 'organisation_id'
+ ])->toArray()));
+ $orgs = $orgs->where(['id IN' => $org_ids]);
+ }
+ if ($returnObjects) {
+ $orgs = $orgs->toArray();
+ } else {
+ $orgs = $orgs->disableHydration()->all();
+ }
+ return $orgs;
+ }
+
+ public function getOrganisationSelectorValues(): array
+ {
+ $results = [];
+ $orgTable = \Cake\ORM\TableRegistry::getTableLocator()->get('Organisations');
+ $fields = [
+ 'nationality' => 'nat',
+ 'sector' => 'sect',
+ 'type' => 'typ'
+ ];
+ foreach ($fields as $field => $temp_field) {
+ $temp = Hash::extract(
+ $orgTable->find()
+ ->select([$temp_field => 'DISTINCT (' . $field . ')'])
+ ->order([$temp_field => 'DESC'])
+ ->disableHydration()->toArray(),
+ '{n}.' . $temp_field
+ );
+ foreach ($temp as $k => $v) {
+ if (empty($v)) {
+ unset($temp[$k]);
+ }
+ }
+ asort($temp, SORT_FLAG_CASE | SORT_NATURAL);
+ $temp = array_merge(['ALL' => 'ALL'], $temp);
+ $results[$field] = array_combine($temp, $temp);
+ }
+ return $results;
+ }
+
public function captureSharingGroup($input): bool
{
if (empty($input['uuid'])) {
diff --git a/src/Lib/default/local_tool_connectors/MispConnector.php b/src/Lib/default/local_tool_connectors/MispConnector.php
index a5adb59..94f89e9 100644
--- a/src/Lib/default/local_tool_connectors/MispConnector.php
+++ b/src/Lib/default/local_tool_connectors/MispConnector.php
@@ -56,6 +56,30 @@ class MispConnector extends CommonConnectorTools
],
'redirect' => 'organisationsAction'
],
+ 'fetchSelectedOrganisationsAction' => [
+ 'type' => 'formAction',
+ 'scope' => 'childAction',
+ 'params' => [
+ 'uuid'
+ ],
+ 'redirect' => 'organisationsAction'
+ ],
+ 'pushOrganisationAction' => [
+ 'type' => 'formAction',
+ 'scope' => 'childAction',
+ 'params' => [
+ 'uuid'
+ ],
+ 'redirect' => 'organisationsAction'
+ ],
+ 'pushOrganisationsAction' => [
+ 'type' => 'formAction',
+ 'scope' => 'childAction',
+ 'params' => [
+ 'uuid'
+ ],
+ 'redirect' => 'organisationsAction'
+ ],
'fetchSharingGroupAction' => [
'type' => 'formAction',
'scope' => 'childAction',
@@ -229,9 +253,6 @@ class MispConnector extends CommonConnectorTools
$list = explode('.', $params['sort']);
$params['sort'] = end($list);
}
- if (!isset($params['limit'])) {
- $params['limit'] = 50;
- }
$url = $this->urlAppendParams($url, $params);
$response = $this->HTTPClientGET($url, $params['connection']);
if ($response->isOk()) {
@@ -261,6 +282,9 @@ class MispConnector extends CommonConnectorTools
if ($response->isOk()) {
return $response;
} else {
+ if (!empty($params['softError'])) {
+ return $response;
+ }
$errorMsg = __('Could not post to the requested resource for `{0}`. Remote returned:', $url) . PHP_EOL . $response->getStringBody();
$this->logError($errorMsg);
throw new NotFoundException($errorMsg);
@@ -563,6 +587,48 @@ class MispConnector extends CommonConnectorTools
$urlParams = h($params['connection']['id']) . '/organisationsAction';
$response = $this->getData('/organisations/index', $params);
$data = $response->getJson();
+ $temp = $this->getOrganisations();
+ $existingOrgs = [];
+ foreach ($temp as $k => $v) {
+ $existingOrgs[$v['uuid']] = $v;
+ unset($temp[$k]);
+ }
+ $statusLevels = [
+ 'same' => [
+ 'colour' => 'success',
+ 'message' => __('Remote organisation is the same as local copy'),
+ 'icon' => 'check-circle'
+ ],
+ 'different' => [
+ 'colour' => 'warning',
+ 'message' => __('Local and remote versions of the organisations are different.'),
+ 'icon' => 'exclamation-circle'
+ ],
+ 'not_found' => [
+ 'colour' => 'danger',
+ 'message' => __('Local organisation not found'),
+ 'icon' => 'exclamation-triangle'
+ ]
+ ];
+ foreach ($data as $k => $v) {
+ $data[$k]['Organisation']['local_copy'] = false;
+ if (!empty($existingOrgs[$v['Organisation']['uuid']])) {
+ $remoteOrg = $existingOrgs[$v['Organisation']['uuid']];
+ $localOrg = $v['Organisation'];
+ $same = true;
+ $fieldsToCheck = [
+ 'nationality', 'sector', 'type', 'name'
+ ];
+ foreach (['nationality', 'sector', 'type', 'name'] as $fieldToCheck) {
+ if ($remoteOrg[$fieldToCheck] != $localOrg[$fieldToCheck]) {
+ $same = false;
+ }
+ }
+ $data[$k]['Organisation']['local_copy'] = $same ? 'same' : 'different';
+ } else {
+ $data[$k]['Organisation']['local_copy'] = 'not_found';
+ }
+ }
if (!empty($data)) {
return [
'type' => 'index',
@@ -571,6 +637,25 @@ class MispConnector extends CommonConnectorTools
'skip_pagination' => 1,
'top_bar' => [
'children' => [
+ [
+ 'type' => 'simple',
+ 'children' => [
+ [
+ 'class' => 'hidden mass-select',
+ 'text' => __('Fetch selected organisations'),
+ 'html' => ' ',
+ 'reload_url' => '/localTools/action/' . h($params['connection']['id']) . '/organisationsAction',
+ 'popover_url' => '/localTools/action/' . h($params['connection']['id']) . '/fetchSelectedOrganisationsAction'
+ ],
+ [
+ 'text' => __('Push organisations'),
+ 'html' => ' ',
+ 'class' => 'btn btn-primary',
+ 'reload_url' => '/localTools/action/' . h($params['connection']['id']) . '/organisationsAction',
+ 'popover_url' => '/localTools/action/' . h($params['connection']['id']) . '/pushOrganisationsAction'
+ ]
+ ]
+ ],
[
'type' => 'search',
'button' => __('Search'),
@@ -582,11 +667,32 @@ class MispConnector extends CommonConnectorTools
]
],
'fields' => [
+ [
+ 'element' => 'selector',
+ 'class' => 'short',
+ 'data' => [
+ 'id' => [
+ 'value_path' => 'Organisation.uuid'
+ ]
+ ]
+ ],
[
'name' => 'Name',
'sort' => 'Organisation.name',
'data_path' => 'Organisation.name',
],
+ [
+ 'name' => 'Status',
+ 'sort' => 'Organisation.local_copy',
+ 'data_path' => 'Organisation.local_copy',
+ 'element' => 'status',
+ 'status_levels' => $statusLevels
+ ],
+ [
+ 'name' => 'uuid',
+ 'sort' => 'Organisation.uuid',
+ 'data_path' => 'Organisation.uuid'
+ ],
[
'name' => 'uuid',
'sort' => 'Organisation.uuid',
@@ -597,6 +703,11 @@ class MispConnector extends CommonConnectorTools
'sort' => 'Organisation.nationality',
'data_path' => 'Organisation.nationality'
],
+ [
+ 'name' => 'local',
+ 'sort' => 'Organisation.local',
+ 'data_path' => 'Organisation.local'
+ ],
[
'name' => 'sector',
'sort' => 'Organisation.sector',
@@ -607,11 +718,33 @@ class MispConnector extends CommonConnectorTools
'description' => false,
'pull' => 'right',
'actions' => [
+ [
+ 'open_modal' => '/localTools/action/' . h($params['connection']['id']) . '/pushOrganisationAction?uuid={{0}}',
+ 'modal_params_data_path' => ['Organisation.uuid'],
+ 'icon' => 'upload',
+ 'reload_url' => '/localTools/action/' . h($params['connection']['id']) . '/organisationsAction',
+ 'complex_requirement' => [
+ 'function' => function ($row, $options) {
+ if ($row['Organisation']['local_copy'] === 'different') {
+ return true;
+ }
+ return false;
+ }
+ ]
+ ],
[
'open_modal' => '/localTools/action/' . h($params['connection']['id']) . '/fetchOrganisationAction?uuid={{0}}',
'modal_params_data_path' => ['Organisation.uuid'],
'icon' => 'download',
- 'reload_url' => '/localTools/action/' . h($params['connection']['id']) . '/organisationsAction'
+ 'reload_url' => '/localTools/action/' . h($params['connection']['id']) . '/organisationsAction',
+ 'complex_requirement' => [
+ 'function' => function ($row, $options) {
+ if ($row['Organisation']['local_copy'] === 'different' || $row['Organisation']['local_copy'] === 'not_found') {
+ return true;
+ }
+ return false;
+ }
+ ]
]
]
]
@@ -703,7 +836,7 @@ class MispConnector extends CommonConnectorTools
return [
'data' => [
'title' => __('Fetch organisation'),
- 'description' => __('Fetch and create/update organisation ({0}) from MISP.', $params['uuid']),
+ 'description' => __('Fetch and create/update organisation ({0}) from MISP?', $params['uuid']),
'submit' => [
'action' => $params['request']->getParam('action')
],
@@ -726,6 +859,164 @@ class MispConnector extends CommonConnectorTools
throw new MethodNotAllowedException(__('Invalid http request type for the given action.'));
}
+ public function fetchSelectedOrganisationsAction(array $params): array
+ {
+ $ids = $params['request']->getQuery('ids');
+ if ($params['request']->is(['get'])) {
+ return [
+ 'data' => [
+ 'title' => __('Fetch organisations'),
+ 'description' => __('Fetch and create/update the selected {0} organisations from MISP?', count($ids)),
+ 'submit' => [
+ 'action' => $params['request']->getParam('action')
+ ],
+ 'url' => ['controller' => 'localTools', 'action' => 'action', $params['connection']['id'], 'fetchSelectedOrganisationsAction']
+ ]
+ ];
+ } elseif ($params['request']->is(['post'])) {
+ $successes = 0;
+ $errors = 0;
+ foreach ($ids as $id) {
+ $response = $this->getData('/organisations/view/' . $id, $params);
+ $result = $this->captureOrganisation($response->getJson()['Organisation']);
+ if ($response->getStatusCode() == 200) {
+ $successes++;
+ } else {
+ $errors++;
+ }
+ }
+ if ($successes) {
+ return ['success' => 1, 'message' => __('The fetching of organisations has succeeded. {0} organisations created/modified and {1} organisations could not be created/modified.', $successes, $errors)];
+ } else {
+ return ['success' => 0, 'message' => __('The fetching of organisations has failed. {0} organisations could not be created/modified.', $errors)];
+ }
+ }
+ throw new MethodNotAllowedException(__('Invalid http request type for the given action.'));
+ }
+
+ public function pushOrganisationAction(array $params): array
+ {
+ if ($params['request']->is(['get'])) {
+ return [
+ 'data' => [
+ 'title' => __('Push organisation'),
+ 'description' => __('Push or update organisation ({0}) on MISP.', $params['uuid']),
+ 'submit' => [
+ 'action' => $params['request']->getParam('action')
+ ],
+ 'url' => ['controller' => 'localTools', 'action' => 'action', $params['connection']['id'], 'pushOrganisationAction', $params['uuid']]
+ ]
+ ];
+ } elseif ($params['request']->is(['post'])) {
+ $org = $this->getOrganisation($params['uuid']);
+ if (empty($org)) {
+ return ['success' => 0, 'message' => __('Could not find the organisation.')];
+ }
+ $params['body'] = json_encode($org);
+ $response = $this->getData('/organisations/view/' . $params['uuid'], $params);
+ if ($response->getStatusCode() == 200) {
+ $response = $this->postData('/admin/organisations/edit/' . $params['uuid'], $params);
+ $result = $this->captureOrganisation($response->getJson()['Organisation']);
+ if ($response->getStatusCode() == 200 && $result) {
+ return ['success' => 1, 'message' => __('Organisation modified.')];
+ } else {
+ return ['success' => 0, 'message' => __('Could not save the changes to the organisation.')];
+ }
+ } else {
+ $response = $this->postData('/admin/organisations/add/', $params);
+ $result = $this->captureOrganisation($response->getJson()['Organisation']);
+ if ($response->getStatusCode() == 200 && $result) {
+ return ['success' => 1, 'message' => __('Organisation created.')];
+ } else {
+ return ['success' => 0, 'message' => __('Could not create the organisation.')];
+ }
+ }
+ }
+ throw new MethodNotAllowedException(__('Invalid http request type for the given action.'));
+ }
+
+ public function pushOrganisationsAction(array $params): array
+ {
+ $orgSelectorValues = $this->getOrganisationSelectorValues();
+ if ($params['request']->is(['get'])) {
+ return [
+ 'data' => [
+ 'title' => __('Push organisation'),
+ 'description' => __('Push or update organisations on MISP.'),
+ 'fields' => [
+ [
+ 'field' => 'local',
+ 'label' => __('Only organisations with users'),
+ 'type' => 'checkbox'
+ ],
+ [
+ 'field' => 'type',
+ 'label' => __('Type'),
+ 'type' => 'select',
+ 'options' => $orgSelectorValues['type']
+
+ ],
+ [
+ 'field' => 'sector',
+ 'label' => __('Sector'),
+ 'type' => 'select',
+ 'options' => $orgSelectorValues['sector']
+
+ ],
+ [
+ 'field' => 'nationality',
+ 'label' => __('Country'),
+ 'type' => 'select',
+ 'options' => $orgSelectorValues['nationality']
+
+ ]
+ ],
+ 'submit' => [
+ 'action' => $params['request']->getParam('action')
+ ],
+ 'url' => ['controller' => 'localTools', 'action' => 'action', $params['connection']['id'], 'pushOrganisationAction']
+ ]
+ ];
+ } elseif ($params['request']->is(['post'])) {
+ $filters = $params['request']->getData();
+ $orgs = $this->getFilteredOrganisations($filters, true);
+ $created = 0;
+ $modified = 0;
+ $errors = 0;
+ $params['softError'] = 1;
+ if (empty($orgs)) {
+ return ['success' => 0, 'message' => __('Could not find any organisations matching the criteria.')];
+ }
+ foreach ($orgs as $org) {
+ $params['body'] = null;
+ $response = $this->getData('/organisations/view/' . $org->uuid, $params);
+ if ($response->getStatusCode() == 200) {
+ $params['body'] = json_encode($org);
+ $response = $this->postData('/admin/organisations/edit/' . $org->uuid, $params);
+ if ($response->getStatusCode() == 200) {
+ $modified++;
+ } else {
+ $errors++;
+ }
+ } else {
+ $params['body'] = json_encode($org);
+ $response = $this->postData('/admin/organisations/add', $params);
+ if ($response->getStatusCode() == 200) {
+ $created++;
+ } else {
+ $errors++;
+ }
+ }
+ }
+ if ($created || $modified) {
+ return ['success' => 1, 'message' => __('Organisations created: {0}, modified: {1}, errors: {2}', $created, $modified, $errors)];
+ } else {
+ return ['success' => 0, 'message' => __('Organisations could not be pushed. Errors: {0}', $errors)];
+ }
+ }
+ throw new MethodNotAllowedException(__('Invalid http request type for the given action.'));
+ }
+
public function fetchSharingGroupAction(array $params): array
{
if ($params['request']->is(['get'])) {
diff --git a/src/Model/Table/InstanceTable.php b/src/Model/Table/InstanceTable.php
index afd04e0..83b29bd 100644
--- a/src/Model/Table/InstanceTable.php
+++ b/src/Model/Table/InstanceTable.php
@@ -236,4 +236,128 @@ class InstanceTable extends AppTable
}
return $themes;
}
+
+ public function getTopology($mermaid = true): mixed
+ {
+ $BroodsModel = TableRegistry::getTableLocator()->get('Broods');
+ $LocalToolsModel = TableRegistry::getTableLocator()->get('LocalTools');
+ $connectors = $LocalToolsModel->getConnectors();
+ $connections = $LocalToolsModel->extractMeta($connectors, true);
+ $broods = $BroodsModel->find()->select(['id', 'uuid', 'url', 'name', 'pull'])->disableHydration()->toArray();
+ foreach ($broods as $k => $brood) {
+ $broods[$k]['status'] = $BroodsModel->queryStatus($brood['id']);
+ }
+ $data = [
+ 'broods' => $broods,
+ 'tools' => $LocalToolsModel->extractMeta($connectors, true)
+ ];
+ if ($mermaid) {
+ return $this->generateTopologyMermaid($data);
+ }
+ return $data;
+ }
+
+ public function generateTopologyMermaid($data)
+ {
+ $version = json_decode(file_get_contents(APP . 'VERSION.json'), true)["version"];
+ $newest = $version;
+ $broods = '';
+ $edges = '';
+ // pre-run the loop to get the latest version
+ foreach ($data['broods'] as $brood) {
+ if ($brood['status']['code'] === 200) {
+ if (version_compare($brood['status']['response']['version'], $newest) > 0) {
+ $newest = $brood['status']['response']['version'];
+ }
+ }
+ }
+ foreach ($data['broods'] as $brood) {
+ $status = '';
+ if ($brood['status']['code'] === 200) {
+ $status = sprintf(
+ '
Ping: %sms
Version: v%s
Role: %s
',
+ h($brood['status']['ping']),
+ $brood['status']['response']['version'] === $newest ? 'text-success' : 'text-danger',
+ h($brood['status']['response']['version']) . ($brood['status']['response']['version'] !== $newest ? ' - outdated' : ''),
+ h($brood['status']['response']['role']['name'])
+ );
+ }
+ $broods .= sprintf(
+ "%s%s end" . PHP_EOL,
+ sprintf(
+ ' subgraph brood_%s[fas:fa-network-wired Brood #%s]' . PHP_EOL,
+ h($brood['id']),
+ h($brood['id'])
+ ),
+ sprintf(
+ " cerebrate_%s[%s
%sfas:fa-eye]" . PHP_EOL,
+ h($brood['id']),
+ '' . h($brood['name']) . '',
+ sprintf(
+ "Connected: %s%s",
+ $brood['status']['code'] === 200 ? 'text-success' : 'text-danger',
+ h($brood['status']['code']),
+ $brood['status']['code'] === 200 ? 'fas:fa-check' : 'fas:fa-times',
+ $status
+ ),
+ h($brood['id']),
+ )
+
+ );
+ $edges .= sprintf(
+ ' C1%s---cerebrate_%s' . PHP_EOL,
+ $brood['pull'] ? '<' : '',
+ h($brood['id'])
+ );
+ }
+ $tools = '';
+ foreach ($data['tools'] as $tool) {
+ $tools .= sprintf(
+ ' subgraph instance_local_tools_%s[%s %s connector]' . PHP_EOL . ' direction TB' . PHP_EOL,
+ h($tool['name']),
+ isset($tool['logo']) ? '' : 'fas:fa-wrench',
+ h($tool['name'])
+ );
+ foreach ($tool['connections'] as $connection) {
+ $tools .= sprintf(
+ " %s[%s
%s
%s]" . PHP_EOL,
+ h($connection['name']),
+ h($connection['name']),
+ sprintf(
+ __('Health') . ': %s',
+ h($connection['message']),
+ $connection['health'] === 1 ? 'text-success' : 'text-danger',
+ $connection['health'] === 1 ? 'fas:fa-check' : 'fas:fa-times'
+ ),
+ sprintf(
+ "fas:fa-eye",
+ h($connection['url'])
+ )
+ );
+ }
+ $tools .= ' end' . PHP_EOL;
+ }
+ $this_cerebrate = sprintf(
+ 'C1[My Cerebrate
Version: v%s]',
+ $version === $newest ? 'text-success' : 'text-danger',
+ $version
+ );
+ $md = sprintf(
+'flowchart TB
+ subgraph instance[fas:fa-network-wired My Brood]
+ direction TB
+ %s
+ subgraph instance_local_tools[fa:fa-tools Local Tools]
+ direction LR
+%s
+ end
+ end
+%s%s',
+ $this_cerebrate,
+ $tools,
+ $broods,
+ $edges
+ );
+ return $md;
+ }
}
diff --git a/templates/element/genericElements/IndexTable/Fields/status.php b/templates/element/genericElements/IndexTable/Fields/status.php
new file mode 100644
index 0000000..95d8544
--- /dev/null
+++ b/templates/element/genericElements/IndexTable/Fields/status.php
@@ -0,0 +1,10 @@
+Hash->extract($row, $field['data_path'])[0];
+ $status_levels = $field['status_levels'];
+ echo sprintf(
+ '',
+ h($field['status_levels'][$data]['colour']),
+ empty($field['status_levels'][$data]['icon']) ? 'circle' : h($field['status_levels'][$data]['icon']),
+ h($field['status_levels'][$data]['message'])
+ );
+?>
diff --git a/templates/element/genericElements/ListTopBar/element_simple.php b/templates/element/genericElements/ListTopBar/element_simple.php
index 5573254..63b4ea8 100644
--- a/templates/element/genericElements/ListTopBar/element_simple.php
+++ b/templates/element/genericElements/ListTopBar/element_simple.php
@@ -73,6 +73,25 @@
\ No newline at end of file