cerebrate-project
/
cerebrate
mirror of https://github.com/cerebrate-project/cerebrate
1
0
Fork 0
Code Issues Releases Wiki Activity

chg: [wip] sharing group rework / MISP connector improvements

pull/116/merge
iglocska 2023-10-31 14:54:08 +01:00
parent d23e393a9a
commit 9305e7ceea
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
11 changed files with 481 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
config/Migrations/20231012000007_SGExtend.php Normal file
View File

@ -0,0 +1,34 @@
<?php
declare(strict_types=1);
use Migrations\AbstractMigration;
use Phinx\Db\Adapter\MysqlAdapter;
final class SGExtend extends AbstractMigration
{
public $autoId = false; // turn off automatic `id` column create. We want it to be `int(10) unsigned`
/**
* Change Method.
*
* Write your reversible migrations using this method.
*
* More information on writing migrations is available here:
* https://book.cakephp.org/phinx/0/en/migrations.html#the-change-method
*
* Remember to call "create()" or "update()" and NOT "save()" when working
* with the Table class.
*/
public function change(): void
{
$table = $this->table('sgo');
$exists = $table->hasColumn('extend');
if (!$exists) {
$table
->addColumn('extend', 'boolean', [
'default' => 0,
'null' => false,
])->update();
}
}
}

20
src/Controller/SharingGroupsController.php
View File

@ -8,6 +8,7 @@ use Cake\Utility\Text;
use \Cake\Database\Expression\QueryExpression;
use Cake\Error\Debugger;
use Cake\Http\Exception\NotFoundException;
use Cake\ORM\TableRegistry;
class SharingGroupsController extends AppController
{
@ -171,9 +172,13 @@ class SharingGroupsController extends AppController
$input['organisation_id'] = [$input['organisation_id']];
}
$result = true;
$this->SGO = TableRegistry::getTableLocator()->get('SGOs');
foreach ($input['organisation_id'] as $org_id) {
$org = $this->SharingGroups->SharingGroupOrgs->get($org_id);
$result &= (bool)$this->SharingGroups->SharingGroupOrgs->link($sharingGroup, [$org]);
$additional_data = [];
if (!empty($input['extend'])) {
$additional_data['extend'] = $input['extend'];
}
$result &= $this->SGO->attach($sharingGroup['id'], $org_id, $additional_data);
}
if ($result) {
$message = __('Organisation(s) added to the sharing group.');
@ -216,8 +221,8 @@ class SharingGroupsController extends AppController
throw new NotFoundException(__('Invalid SharingGroup.'));
}
if ($this->request->is('post')) {
$org = $this->SharingGroups->SharingGroupOrgs->get($org_id);
$result = (bool)$this->SharingGroups->SharingGroupOrgs->unlink($sharingGroup, [$org]);
$this->SGO = TableRegistry::getTableLocator()->get('SGOs');
$result = (bool)$this->SharingGroups->SharingGroupOrgs->unlink($sharingGroup['id'], $org_id);
if ($result) {
$message = __('Organisation(s) removed from the sharing group.');
} else {
@ -253,9 +258,10 @@ class SharingGroupsController extends AppController
public function listOrgs($id)
{
$sharingGroup = $this->SharingGroups->get($id, [
'contain' => 'SharingGroupOrgs'
]);
$sharingGroup = $this->SharingGroups->find()->where(['id' => $id])->contain(['SharingGroupOrgs'])->first();
foreach ($sharingGroup['sharing_group_orgs'] as $k => $org) {
$sharingGroup['sharing_group_orgs'][$k]['extend'] = $org['_joinData']['extend'];
}
$params = $this->ParamHandler->harvestParams(['quickFilter']);
if (!empty($params['quickFilter'])) {
foreach ($sharingGroup['sharing_group_orgs'] as $k => $org) {

15
src/Lib/default/local_tool_connectors/CommonConnectorTools.php
View File

@ -104,6 +104,16 @@ class CommonConnectorTools
return $orgs;
}
public function getSharingGroups(): array
{
$sgs = \Cake\ORM\TableRegistry::getTableLocator()->get('SharingGroups');
$sgs = $sgs->find()
->contain(['Organisations' => ['fields' => ['uuid']], 'SharingGroupOrgs' => ['fields' => ['uuid']]])
->disableHydration()
->toArray();
return $sgs;
}
public function getFilteredOrganisations($filters, $returnObjects = false): array
{
$organisations = \Cake\ORM\TableRegistry::getTableLocator()->get('Organisations');
@ -219,6 +229,11 @@ class CommonConnectorTools
$this->remoteToolConnectionStatus($params, self::STATE_CONNECTED);
return false;
}
public function diagnostics(array $params): array
{
return [];
}
}
?>

330
src/Lib/default/local_tool_connectors/MispConnector.php
View File

@ -15,6 +15,12 @@ class MispConnector extends CommonConnectorTools
public $name = 'MISP';
public $exposedFunctions = [
'diagnosticsAction' => [
'type' => 'index',
'scope' => 'child',
'params' => [
]
],
'serverSettingsAction' => [
'type' => 'index',
'scope' => 'child',
@ -48,6 +54,11 @@ class MispConnector extends CommonConnectorTools
'direction'
]
],
'restartWorkersAction' => [
'type' => 'formAction',
'scope' => 'childAction',
'redirect' => 'diagnosticsAction'
],
'fetchOrganisationAction' => [
'type' => 'formAction',
'scope' => 'childAction',
@ -132,7 +143,7 @@ class MispConnector extends CommonConnectorTools
'icon' => 'terminal',
'variant' => 'primary',
]
]
],
];
public $version = '0.1';
public $settings = [
@ -261,7 +272,7 @@ class MispConnector extends CommonConnectorTools
if (!empty($params['softError'])) {
return $response;
}
$errorMsg = __('Could not post to the requested resource for `{0}`. Remote returned:', $url) . PHP_EOL . $response->getStringBody();
$errorMsg = __('Could not GET from the requested resource for `{0}`. Remote returned:', $url) . PHP_EOL . $response->getStringBody();
$this->logError($errorMsg);
throw new NotFoundException($errorMsg);
}
@ -312,10 +323,183 @@ class MispConnector extends CommonConnectorTools
return $url;
}
public function diagnostics(array $params): array
{
$urlParams = h($params['connection']['id']) . '/serverSettingsAction';
$response = $this->getData('/servers/serverSettings', $params);
if (!$response->isOk()) {
return [];
}
$data = $response->getJson();
$issues = [];
if ($data['version']['upToDate'] !== 'same') {
$issues['version'] = [
'type' => 'danger',
'message' => __('Outdated ({0}).', $data['version']['current']),
'remediation' => [
'icon' => 'fa-sync',
'title' => __('Update MISP'),
'url' => '/localTools/action/' . h($params['connection']['id']) . '/updateMISP'
]
];
}
if ($data['phpSettings']['memory_limit']['value'] < $data['phpSettings']['memory_limit']['recommended']) {
$issues['php_memory'] = [
'type' => 'warning',
'message' => __('Low PHP memory ({0}M).', $data['phpSettings']['memory_limit']['value'])
];
}
$worker_issues = [];
foreach ($data['workers'] as $queue => $worker_data) {
if (in_array($queue, ['proc_accessible', 'scheduler', 'controls'])) {
continue;
}
if (empty($worker_data['ok'])) {
$worker_issues['down'][] = $queue;
}
if ($worker_data['jobCount'] > 100) {
$worker_issues['stalled'][] = $queue;
}
}
if (!empty($worker_issues['down'])) {
$issues['workers_down'] = [
'type' => 'danger',
'message' => __('Worker(s) down: {0}', implode(', ', $worker_issues['down'])),
'remediation' => [
'icon' => 'fa-sync',
'title' => __('Restart workers'),
'url' => '/localTools/action/' . h($params['connection']['id']) . '/restartWorkersAction'
]
];
}
if (!empty($worker_issues['stalled'])) {
$issues['workers_stalled'] = [
'type' => 'warning',
'message' => __('Worker(s) stalled: {0}', implode(', ', $worker_issues['stalled'])),
'remediation' => [
'icon' => 'fa-sync',
'title' => __('Restart workers'),
'url' => '/localTools/action/' . h($params['connection']['id']) . '/restartWorkersAction'
]
];
}
if (!empty($data['dbConfiguration'])) {
foreach ($data['dbConfiguration'] as $dbConfig) {
if ($dbConfig['name'] === 'innodb_buffer_pool_size' && $dbConfig['value'] < $dbConfig['recommended']) {
$issues['innodb_buffer_pool_size'] = [
'type' => 'warning',
'message' => __('InnoDB buffer pool size is low ({0}M).', (round($dbConfig['value']/1024/1024)))
];
}
}
}
if (!empty($data['dbSchemaDiagnostics'])) {
if ($data['dbSchemaDiagnostics']['expected_db_version'] > $data['dbSchemaDiagnostics']['actual_db_version'])
$issues['schema_version'] = [
'type' => 'danger',
'message' => __('DB schame outdated.'),
'remediation' => [
'icon' => 'fa-sync',
'title' => __('Update DB schema'),
'url' => '/localTools/action/' . h($params['connection']['id']) . '/updateSchemaAction'
]
];
}
return $issues;
}
public function restartWorkersAction(array $params): array
{
if ($params['request']->is(['get'])) {
return [
'data' => [
'title' => __('Restart workers'),
'description' => __('Would you like to trigger a restart of all attached workers?'),
'submit' => [
'action' => $params['request']->getParam('action')
],
'url' => ['controller' => 'localTools', 'action' => 'action', $params['connection']['id'], 'restartWorkersAction']
]
];
} elseif ($params['request']->is(['post'])) {
$response = $this->postData('/servers/restartWorkers', $params);
if ($response->isOk()) {
return ['success' => 1, 'message' => __('Workers restarted.')];
} else {
return ['success' => 0, 'message' => __('Could not restart workers.')];
}
}
$response = $this->postData('/servers/restartWorkers', $params);
if ($response->isOk()) {
return [
'type' => 'success',
'message' => __('Workers restarted.')
];
} else {
return [
'type' => 'danger',
'message' => __('Something went wrong.')
];
}
}
public function diagnosticsAction(array $params): array
{
$diagnostics = $this->diagnostics($params);
$data = [];
foreach ($diagnostics as $error => $error_data) {
$data[] = [
'error' => $error,
'type' => $error_data['type'],
'message' => $error_data['message'],
'remediation' => $error_data['remediation'] ?? false
];
}
return [
'type' => 'index',
'data' => [
'data' => $data,
'skip_pagination' => 1,
'top_bar' => [
'children' => []
],
'fields' => [
[
'name' => 'error',
'data_path' => 'error',
'name' => __('Error'),
],
[
'name' => 'message',
'data_path' => 'message',
'name' => __('Message'),
],
[
'name' => __('Remediation'),
'element' => 'function',
'function' => function($row, $context) {
$remediation = $context->Hash->extract($row, 'remediation');
if (!empty($remediation['title'])) {
echo sprintf(
'<a href="%s" class="btn btn-primary btn-sm" title="%s"><i class="fa %s"></i></a>',
h($remediation['url']),
h($remediation['title']),
h($remediation['icon'])
);
}
}
]
],
'title' => false,
'description' => false,
'pull' => 'right'
]
];
}
public function serverSettingsAction(array $params): array
@ -576,6 +760,56 @@ class MispConnector extends CommonConnectorTools
}
}
private function __compareOrgs(array $data, array $existingOrgs): array
{
foreach ($data as $k => $v) {
$data[$k]['Organisation']['local_copy'] = false;
if (!empty($existingOrgs[$v['Organisation']['uuid']])) {
$remoteOrg = $existingOrgs[$v['Organisation']['uuid']];
$localOrg = $v['Organisation'];
$same = true;
$fieldsToCheck = [
'nationality', 'sector', 'type', 'name'
];
foreach (['nationality', 'sector', 'type', 'name'] as $fieldToCheck) {
if ($remoteOrg[$fieldToCheck] != $localOrg[$fieldToCheck]) {
$same = false;
}
}
$data[$k]['Organisation']['local_copy'] = $same ? 'same' : 'different';
} else {
$data[$k]['Organisation']['local_copy'] = 'not_found';
}
}
return $data;
}
private function __compareSgs(array $data, array $existingSgs): array
{
debug($data);
debug($existingSgs);
foreach ($data as $k => $v) {
$data[$k]['SharingGroup']['local_copy'] = false;
if (!empty($existingOrgs[$v['SharingGroup']['uuid']])) {
$remoteOrg = $existingOrgs[$v['SharingGroup']['uuid']];
$localOrg = $v['SharingGroup'];
$same = true;
$fieldsToCheck = [
'nationality', 'sector', 'type', 'name'
];
foreach (['nationality', 'sector', 'type', 'name'] as $fieldToCheck) {
if ($remoteOrg[$fieldToCheck] != $localOrg[$fieldToCheck]) {
$same = false;
}
}
$data[$k]['SharingGroup']['local_copy'] = $same ? 'same' : 'different';
} else {
$data[$k]['SharingGroup']['local_copy'] = 'not_found';
}
}
return $data;
}
public function organisationsAction(array $params): array
{
@ -610,25 +844,7 @@ class MispConnector extends CommonConnectorTools
'icon' => 'exclamation-triangle'
]
];
foreach ($data as $k => $v) {
$data[$k]['Organisation']['local_copy'] = false;
if (!empty($existingOrgs[$v['Organisation']['uuid']])) {
$remoteOrg = $existingOrgs[$v['Organisation']['uuid']];
$localOrg = $v['Organisation'];
$same = true;
$fieldsToCheck = [
'nationality', 'sector', 'type', 'name'
];
foreach (['nationality', 'sector', 'type', 'name'] as $fieldToCheck) {
if ($remoteOrg[$fieldToCheck] != $localOrg[$fieldToCheck]) {
$same = false;
}
}
$data[$k]['Organisation']['local_copy'] = $same ? 'same' : 'different';
} else {
$data[$k]['Organisation']['local_copy'] = 'not_found';
}
}
$data = $this->__compareOrgs($data, $existingOrgs);
if (!empty($data)) {
return [
'type' => 'index',
@ -764,6 +980,35 @@ class MispConnector extends CommonConnectorTools
$urlParams = h($params['connection']['id']) . '/sharingGroupsAction';
$response = $this->getData('/sharing_groups/index', $params);
$data = $response->getJson();
$temp = $this->getSharingGroups();
$existingOrgs = [];
foreach ($temp as $k => $v) {
$existingSGs[$v['uuid']] = $v;
unset($temp[$k]);
}
$data = $this->__compareSgs($data, $existingSGs);
$existingSGs = [];
foreach ($temp as $k => $v) {
$existingSGs[$v['uuid']] = $v;
unset($temp[$k]);
}
$statusLevels = [
'same' => [
'colour' => 'success',
'message' => __('Remote sharing group is the same as local copy'),
'icon' => 'check-circle'
],
'different' => [
'colour' => 'warning',
'message' => __('Local and remote versions of the sharing groups are different.'),
'icon' => 'exclamation-circle'
],
'not_found' => [
'colour' => 'danger',
'message' => __('Local sharing group not found'),
'icon' => 'exclamation-triangle'
]
];
if (!empty($data)) {
return [
'type' => 'index',
@ -773,21 +1018,48 @@ class MispConnector extends CommonConnectorTools
'top_bar' => [
'children' => [
[
'type' => 'search',
'button' => __('Search'),
'placeholder' => __('Enter value to search'),
'data' => '',
'searchKey' => 'value',
'additionalUrlParams' => $urlParams
]
'type' => 'simple',
'children' => [
[
'class' => 'hidden mass-select',
'text' => __('Fetch selected sharing groups'),
'html' => '<i class="fas fa-download"></i> ',
'reload_url' => '/localTools/action/' . h($params['connection']['id']) . '/sharingGroupsAction',
'popover_url' => '/localTools/action/' . h($params['connection']['id']) . '/fetchSelectedSharingGroupsAction'
],
[
'text' => __('Push sharing groups'),
'html' => '<i class="fas fa-upload"></i> ',
'class' => 'btn btn-primary',
'reload_url' => '/localTools/action/' . h($params['connection']['id']) . '/SharingGroupsAction',
'popover_url' => '/localTools/action/' . h($params['connection']['id']) . '/pushSharingGroupsAction'
]
]
],
]
],
'fields' => [
[
'element' => 'selector',
'class' => 'short',
'data' => [
'id' => [
'value_path' => 'SharingGroup.uuid'
]
]
],
[
'name' => 'Name',
'sort' => 'SharingGroup.name',
'data_path' => 'SharingGroup.name',
],
[
'name' => 'Status',
'sort' => 'SharingGroup.local_copy',
'data_path' => 'SharingGroup.local_copy',
'element' => 'status',
'status_levels' => $statusLevels
],
[
'name' => 'uuid',
'sort' => 'SharingGroup.uuid',

11
src/Model/Entity/SGO.php Normal file
View File

@ -0,0 +1,11 @@
<?php
namespace App\Model\Entity;
use App\Model\Entity\AppModel;
use Cake\ORM\Entity;
class SGO extends AppModel
{
}

16
src/Model/Table/InstanceTable.php
View File

@ -249,7 +249,7 @@ class InstanceTable extends AppTable
}
$data = [
'broods' => $broods,
'tools' => $LocalToolsModel->extractMeta($connectors, true)
'tools' => $connections
];
if ($mermaid) {
return $this->generateTopologyMermaid($data);
@ -319,8 +319,19 @@ class InstanceTable extends AppTable
h($tool['name'])
);
foreach ($tool['connections'] as $k2 => $connection) {
$diagnostic_output = '';
if (!empty($connection['diagnostics'])) {
foreach ($connection['diagnostics'] as $diagnostic => $diagnostic_data) {
$diagnostic_output .= sprintf(
"%s: <span class='text-%s'>%s</span><br />",
h($diagnostic),
h($diagnostic_data['type']),
h($diagnostic_data['message'])
);
}
}
$tools .= sprintf(
" connection%s[%s<br />%s<br />%s]" . PHP_EOL,
" connection%s[\"%s<br />%s<br />%s%s\"]" . PHP_EOL,
h($k2),
h($connection['name']),
sprintf(
@ -329,6 +340,7 @@ class InstanceTable extends AppTable
$connection['health'] === 1 ? 'text-success' : 'text-danger',
$connection['health'] === 1 ? 'fas:fa-check' : 'fas:fa-times'
),
empty($diagnostic_data) ? '' : 'Diagnostics:<br />' . $diagnostic_output,
sprintf(
"<a href='%s'>fas:fa-eye</a>",
h($connection['url'])

23
src/Model/Table/LocalToolsTable.php
View File

@ -147,14 +147,14 @@ class LocalToolsTable extends AppTable
'connector_settings_placeholder' => $connector_class->settingsPlaceholder ?? [],
];
if ($includeConnections) {
$connector['connections'] = $this->healthCheck($connector_type, $connector_class);
$connector['connections'] = $this->healthCheck($connector_type, $connector_class, true);
}
$connectors[] = $connector;
}
return $connectors;
}
public function healthCheck(string $connector_type, Object $connector_class): array
public function healthCheck(string $connector_type, Object $connector_class, bool $includeDiagnostics = false): array
{
$query = $this->find();
$query->where([
@ -162,11 +162,28 @@ class LocalToolsTable extends AppTable
]);
$connections = $query->all()->toList();
foreach ($connections as &$connection) {
$connection = $this->healthCheckIndividual($connection);
$temp = $this->healthCheckIndividual($connection);
if ($includeDiagnostics && !empty($temp['health']) && $temp['health'] === 1) {
$temp['diagnostics'] = $this->diagnosticCheckIndividual($connection);
}
$connection = $temp;
}
return $connections;
}
public function diagnosticCheckIndividual(Object $connection): array
{
$connector_class = $this->getConnectors($connection->connector);
if (empty($connector_class[$connection->connector])) {
return [];
}
$connector_class = $connector_class[$connection->connector];
return $connector_class->diagnostics([
'connection' => $connection,
'softError' => 1
]);
}
public function healthCheckIndividual(Object $connection): array
{
$connector_class = $this->getConnectors($connection->connector);

53
src/Model/Table/SGOsTable.php Normal file
View File

@ -0,0 +1,53 @@
<?php
namespace App\Model\Table;
use App\Model\Table\AppTable;
use Cake\ORM\Table;
use Cake\Validation\Validator;
use Cake\ORM\RulesChecker;
use Cake\ORM\TableRegistry;
class SGOsTable extends AppTable
{
public function initialize(array $config): void
{
$this->setTable('sgo');
parent::initialize($config);
$this->belongsTo('SharingGroups');
$this->belongsTo('Organisations');
}
public function attach(int $sg_id, int $org_id, array $additional_data = []): bool
{
$sgo = $this->find()->where([
'sharing_group_id' => $sg_id,
'organisation_id' => $org_id
])->first();
if (empty($sgo)) {
$sgo = $this->newEmptyEntity();
$sgo->sharing_group_id = $sg_id;
$sgo->organisation_id = $org_id;
}
$sgo->extend = empty($additional_data['extend']) ? 0 : 1;
if ($this->save($sgo)) {
return true;
}
return false;
}
public function detach(): bool
{
$sgo = $this->find()->where([
'sharing_group_id' => $sg_id,
'organisation_id' => $org_id
])->first();
if (!empty($sgo)) {
if (!$this->delete($sgo)) {
return false;
}
}
return true;
}
}

13
src/Model/Table/SharingGroupsTable.php
View File

@ -25,11 +25,11 @@ class SharingGroupsTable extends AppTable
$this->belongsToMany(
'SharingGroupOrgs',
[
'through' => 'SGOs',
'className' => 'Organisations',
'foreignKey' => 'sharing_group_id',
'joinTable' => 'sgo',
'targetForeignKey' => 'organisation_id'
]
],
);
$this->setDisplayField('name');
}
@ -77,11 +77,14 @@ class SharingGroupsTable extends AppTable
public function postCaptureActions($savedEntity, $input): void
{
$orgs = [];
$additional_data = [];
if (!empty($input['extend'])) {
$additional_data['extend'] = $input['extend'];
}
$this->SGO = TableRegistry::getTableLocator()->get('SGOs');
foreach ($input['sharing_group_orgs'] as $sgo) {
$organisation_id = $this->Organisations->captureOrg($sgo);
$orgs[] = $this->SharingGroupOrgs->get($organisation_id);
$this->SGO->attach($savedEntity->id, $organisation_id, $additional_data);
}
$this->SharingGroupOrgs->link($savedEntity, $orgs);
}
}

5
templates/SharingGroups/add_org.php
View File

@ -9,6 +9,11 @@
'label' => __('Owner organisation'),
'options' => $dropdownData['organisation']
],
[
'field' => 'extend',
'type' => 'checkbox',
'label' => __('Can extend/administer')
],
],
'submit' => [
'action' => $this->request->getParam('action')

7
templates/SharingGroups/list_orgs.php
View File

@ -42,6 +42,13 @@ echo $this->element('genericElements/IndexTable/index_table', [
'sort' => 'uuid',
'class' => 'short',
'data_path' => 'uuid',
],
[
'name' => __('Can extend/administer'),
'sort' => 'extend',
'element' => 'boolean',
'class' => 'short',
'data_path' => 'extend',
]
],
'pull' => 'right',