new: [MISP connector] sync updated to properly support sharing group exchanges

src/Lib/default/local_tool_connectors/CommonConnectorTools.php

@@ -119,9 +119,9 @@ class CommonConnectorTools
         $organisations = \Cake\ORM\TableRegistry::getTableLocator()->get('Organisations');
         $orgs = $organisations->find();
         $filterFields = ['type', 'nationality', 'sector'];
-        foreach ($filterFields as $fieldField) {
+        foreach ($filterFields as $filterField) {
-            if (!empty($filters[$fieldField]) && $filters[$fieldField] !== 'ALL') {
+            if (!empty($filters[$filterField]) && $filters[$filterField] !== 'ALL') {
-                $orgs = $orgs->where([$fieldField => $filters[$fieldField]]);
+                $orgs = $orgs->where([$filterField => $filters[$filterField]]);
             }
         }
         if (!empty($filters['local']) && $filters['local'] !== '0') {
@@ -139,6 +139,30 @@ class CommonConnectorTools
         return $orgs;
     }
 
+    public function getFilteredSharingGroups($filters, $returnObjects = false): array
+    {
+        $SG = \Cake\ORM\TableRegistry::getTableLocator()->get('SharingGroups');
+        $sgs = $SG->find();
+        $filterFields = ['name', 'releasability'];
+        $sgs->contain(['SharingGroupOrgs', 'Organisations']);
+        foreach ($filterFields as $filterField) {
+            if (!empty($filters[$filterField]) && $filters[$filterField] !== 'ALL') {
+                if (is_string($filters[$filterField]) && strpos($filters[$filterField], '%') !== false) {
+                    $sgs = $sgs->where(['SharingGroups.' . $filterField . ' LIKE' => $filters[$filterField]]);
+                } else {
+                    
+                    $sgs = $sgs->where(['SharingGroups.' . $filterField => $filters[$filterField]]);
+                }
+            }
+        }
+        if ($returnObjects) {
+            $sgs = $sgs->toArray();
+        } else {
+            $sgs = $sgs->disableHydration()->all();
+        }
+        return $sgs;
+    }
+
     public function getOrganisationSelectorValues(): array
     {
         $results = [];
@@ -168,13 +192,13 @@ class CommonConnectorTools
         return $results;
     }
 
-    public function captureSharingGroup($input): bool
+    public function captureSharingGroup($input, $user_id): bool
     {
         if (empty($input['uuid'])) {
             return false;
         }
         $sharing_groups = \Cake\ORM\TableRegistry::getTableLocator()->get('SharingGroups');
-        $sharing_groups->captureSharingGroup($input);
+        $sharing_groups->captureSharingGroup($input, $user_id);
         return true;
     }
 

src/Lib/default/local_tool_connectors/MispConnector.php

@@ -75,6 +75,14 @@ class MispConnector extends CommonConnectorTools
             ],
             'redirect' => 'organisationsAction'
         ],
+        'fetchSelectedSharingGroupsAction' => [
+            'type' => 'formAction',
+            'scope' => 'childAction',
+            'params' => [
+                'uuid'
+            ],
+            'redirect' => 'sharingGroupsAction'
+        ],
         'pushOrganisationAction' => [
             'type' => 'formAction',
             'scope' => 'childAction',
@@ -91,6 +99,14 @@ class MispConnector extends CommonConnectorTools
             ],
             'redirect' => 'organisationsAction'
         ],
+        'pushSharingGroupsAction' => [
+            'type' => 'formAction',
+            'scope' => 'childAction',
+            'params' => [
+                'uuid'
+            ],
+            'redirect' => 'sharingGroupsAction'
+        ],
         'fetchSharingGroupAction' => [
             'type' => 'formAction',
             'scope' => 'childAction',
@@ -786,26 +802,54 @@ class MispConnector extends CommonConnectorTools
 
     private function __compareSgs(array $data, array $existingSgs): array
     {
-        debug($data);
+        foreach ($existingSgs as $k => $existingSg) {
-        debug($existingSgs);
+            $existingSgs[$k]['org_uuids'] = [];
+            foreach ($existingSg['sharing_group_orgs'] as $sgo) {
+                $existingSgs[$k]['org_uuids'][$sgo['uuid']] = $sgo['_joinData']['extend'];
+            }
+        }
         foreach ($data as $k => $v) {
             $data[$k]['SharingGroup']['local_copy'] = false;
-            if (!empty($existingOrgs[$v['SharingGroup']['uuid']])) {
+            $data[$k]['SharingGroup']['differences'] = [];
-                $remoteOrg = $existingOrgs[$v['SharingGroup']['uuid']];
+            if (!empty($existingSgs[$v['SharingGroup']['uuid']])) {
-                $localOrg = $v['SharingGroup'];
+                $data[$k]['SharingGroup']['roaming'] = !empty($v['SharingGroup']['roaming']);
+                $localSg = $existingSgs[$v['SharingGroup']['uuid']];
+                $remoteSg = $v['SharingGroup'];
                 $same = true;
-                $fieldsToCheck = [
+                foreach (['description', 'name', 'releasability', 'active'] as $fieldToCheck) {
-                    'nationality', 'sector', 'type', 'name'
+                    if ($remoteSg[$fieldToCheck] != $localSg[$fieldToCheck]) {
-                ];
-                foreach (['nationality', 'sector', 'type', 'name'] as $fieldToCheck) {
-                    if ($remoteOrg[$fieldToCheck] != $localOrg[$fieldToCheck]) {
                         $same = false;
+                        $data[$k]['differences']['metadata'] = true;
+                    }
+                }
+                if ($v['Organisation']['uuid'] != $localSg['organisation']['uuid']) {
+                    $same = false;
+                    $data[$k]['SharingGroup']['differences']['uuid'] = true;
+                }
+                $v['org_uuids'] = [];
+                foreach ($v['SharingGroupOrg'] as $sgo) {
+                    $v['org_uuids'][$sgo['Organisation']['uuid']] = $sgo['extend'];
+                }
+                if (count($localSg['org_uuids']) !== count($v['org_uuids'])) {
+                    $same = false;
+                    $data[$k]['SharingGroup']['differences']['orgs_count'] = true;
+                }
+                foreach ($localSg['org_uuids'] as $org_uuid => $extend) {
+                    if (!isset($v['org_uuids'][$org_uuid])) {
+                        $same = false;
+                        $data[$k]['SharingGroup']['differences']['remote_orgs_missing'] = true;
+                    } else {
+                        if ($extend != $v['org_uuids'][$org_uuid]) {
+                            $same = false;
+                            $data[$k]['SharingGroup']['differences']['remote_orgs_extend'] = true;
+                        }
                     }
                 }
                 $data[$k]['SharingGroup']['local_copy'] = $same ? 'same' : 'different';
             } else {
                 $data[$k]['SharingGroup']['local_copy'] = 'not_found';
             }
+            $data[$k]['SharingGroup']['differences'] = array_keys($data[$k]['SharingGroup']['differences']);
         }
         return $data;
     }
@@ -986,7 +1030,7 @@ class MispConnector extends CommonConnectorTools
             $existingSGs[$v['uuid']] = $v;
             unset($temp[$k]);
         }
-        $data = $this->__compareSgs($data, $existingSGs);
+        $data['response'] = $this->__compareSgs($data['response'], $existingSGs);
         $existingSGs = [];
         foreach ($temp as $k => $v) {
             $existingSGs[$v['uuid']] = $v;
@@ -1060,6 +1104,11 @@ class MispConnector extends CommonConnectorTools
                             'element' => 'status',
                             'status_levels' => $statusLevels
                         ],
+                        [
+                            'name' => 'Differences',
+                            'data_path' => 'SharingGroup.differences',
+                            'element' => 'list'
+                        ],
                         [
                             'name' => 'uuid',
                             'sort' => 'SharingGroup.uuid',
@@ -1067,21 +1116,16 @@ class MispConnector extends CommonConnectorTools
                         ],
                         [
                             'name' => 'Organisations',
-                            'sort' => 'Organisation',
+                            'sort' => 'SharingGroupOrg',
-                            'data_path' => 'Organisation',
+                            'data_path' => 'SharingGroupOrg',
-                            'element' => 'count_summary'
+                            'element' => 'count_summary',
+                            'title' => 'foo'
                         ],
                         [
                             'name' => 'Roaming',
                             'sort' => 'SharingGroup.roaming',
                             'data_path' => 'SharingGroup.roaming',
                             'element' => 'boolean'
-                        ],
-                        [
-                            'name' => 'External servers',
-                            'sort' => 'Server',
-                            'data_path' => 'Server',
-                            'element' => 'count_summary'
                         ]
                     ],
                     'title' => false,
@@ -1092,7 +1136,15 @@ class MispConnector extends CommonConnectorTools
                             'open_modal' => '/localTools/action/' . h($params['connection']['id']) . '/fetchSharingGroupAction?uuid={{0}}',
                             'modal_params_data_path' => ['SharingGroup.uuid'],
                             'icon' => 'download',
-                            'reload_url' => '/localTools/action/' . h($params['connection']['id']) . '/SharingGroupsAction'
+                            'reload_url' => '/localTools/action/' . h($params['connection']['id']) . '/SharingGroupsAction',
+                            'complex_requirement' => [
+                                'function' => function ($row, $options) {
+                                    if ($row['SharingGroup']['roaming']) {
+                                        return true;
+                                    }
+                                    return false;
+                                }
+                            ]
                         ]
                     ]
                 ]
@@ -1166,6 +1218,51 @@ class MispConnector extends CommonConnectorTools
         throw new MethodNotAllowedException(__('Invalid http request type for the given action.'));
     }
 
+    public function fetchSelectedSharingGroupsAction(array $params): array
+    {
+        $ids = $params['request']->getQuery('ids');
+        if ($params['request']->is(['get'])) {
+            return [
+                'data' => [
+                    'title' => __('Fetch sharing groups'),
+                    'description' => __('Fetch and create/update the selected {0} sharing groups from MISP?', count($ids)),
+                    'submit' => [
+                        'action' => $params['request']->getParam('action')
+                    ],
+                    'url' => ['controller' => 'localTools', 'action' => 'action', $params['connection']['id'], 'fetchSelectedSharingGroupsAction']
+                ]
+            ];
+        } elseif ($params['request']->is(['post'])) {
+            $successes = 0;
+            $errors = 0;
+            foreach ($ids as $id) {
+                $response = $this->getData('/sharingGroups/view/' . $id, $params);
+                $temp = $response->getJson();
+                $sg = $temp['SharingGroup'];
+                $sg['organisation'] = $temp['Organisation'];
+                foreach ($temp['SharingGroupOrg'] as $sgo) {
+                    $sg['sharing_group_orgs'][] = [
+                        'extend' => $sgo['extend'],
+                        'name' => $sgo['Organisation']['name'],
+                        'uuid' => $sgo['Organisation']['uuid']
+                    ];
+                }
+                $result = $this->captureSharingGroup($sg, $params['user_id']);
+                if ($response->getStatusCode() == 200) {
+                    $successes++;
+                } else {
+                    $errors++;
+                }
+            }
+            if ($successes) {
+                return ['success' => 1, 'message' => __('The fetching of organisations has succeeded. {0} organisations created/modified and {1} organisations could not be created/modified.', $successes, $errors)];
+            } else {
+                return ['success' => 0, 'message' => __('The fetching of organisations has failed. {0} organisations could not be created/modified.', $errors)];
+            }
+        }
+        throw new MethodNotAllowedException(__('Invalid http request type for the given action.'));
+    }
+
     public function pushOrganisationAction(array $params): array
     {
         if ($params['request']->is(['get'])) {
@@ -1246,7 +1343,7 @@ class MispConnector extends CommonConnectorTools
                     'submit' => [
                         'action' => $params['request']->getParam('action')
                     ],
-                    'url' => ['controller' => 'localTools', 'action' => 'action', $params['connection']['id'], 'pushOrganisationAction']
+                    'url' => ['controller' => 'localTools', 'action' => 'action', $params['connection']['id'], 'pushOrganisationsAction']
                 ]
             ];
         } elseif ($params['request']->is(['post'])) {
@@ -1289,6 +1386,91 @@ class MispConnector extends CommonConnectorTools
         throw new MethodNotAllowedException(__('Invalid http request type for the given action.'));
     }
 
+    public function pushSharingGroupsAction(array $params): array
+    {
+        if ($params['request']->is(['get'])) {
+            return [
+                'data' => [
+                    'title' => __('Push sharing groups'),
+                    'description' => __('Push or update sharinggroups to MISP.'),
+                    'fields' => [
+                        [
+                            'field' => 'name',
+                            'label' => __('Name'),
+                            'placeholder' => __('Search for sharing groups by name. You can use the % character as a wildcard.'),
+                        ],
+                        [
+                            'field' => 'releasability',
+                            'label' => __('Releasable to'),
+                            'placeholder' => __('Search for sharing groups by relesability. You can use the % character as a wildcard.'),
+                        ]
+                    ],
+                    'submit' => [
+                        'action' => $params['request']->getParam('action')
+                    ],
+                    'url' => ['controller' => 'localTools', 'action' => 'action', $params['connection']['id'], 'pushSharingGroupsAction']
+                ]
+            ];
+        } elseif ($params['request']->is(['post'])) {
+            $filters = $params['request']->getData();
+            $sgs = $this->getFilteredSharingGroups($filters, true);
+            $created = 0;
+            $modified = 0;
+            $errors = 0;
+            $params['softError'] = 1;
+            if (empty($sgs)) {
+                return ['success' => 0, 'message' => __('Could not find any sharing groups matching the criteria.')];
+            }
+            foreach ($sgs as $sg) {
+                $params['body'] = null;
+                $sgToPush = [
+                    'name' => $sg->name,
+                    'uuid' => $sg->uuid,
+                    'releasability' => $sg->releasability,
+                    'description' => $sg->description,
+                    'roaming' => true,
+                    'organisation_uuid' => $sg->organisation->uuid,
+                    'Organisation' => [
+                        'uuid' => $sg->organisation->uuid,
+                        'name' => $sg['organisation']['name']
+                    ],
+                    'SharingGroupOrg' => []
+                ];
+                foreach ($sg['sharing_group_orgs'] as $sgo) {
+                    $sgToPush['SharingGroupOrg'][] = [
+                        'extend' => $sgo['_joinData']['extend'],
+                        'uuid' => $sgo['uuid'],
+                        'name' => $sgo['name']
+                    ];
+                }
+                $response = $this->getData('/sharing_groups/view/' . $sg->uuid, $params);
+                if ($response->getStatusCode() == 200) {
+                    $params['body'] = json_encode($sgToPush);
+                    $response = $this->postData('/sharingGroups/edit/' . $sg->uuid, $params);
+                    if ($response->getStatusCode() == 200) {
+                        $modified++;
+                    } else {
+                        $errors++;
+                    }
+                } else {
+                    $params['body'] = json_encode($sgToPush);
+                    $response = $this->postData('/sharingGroups/add', $params);
+                    if ($response->getStatusCode() == 200) {
+                        $created++;
+                    } else {
+                        $errors++;
+                    }
+                }
+            }
+            if ($created || $modified) {
+                return ['success' => 1, 'message' => __('Sharing groups created: {0}, modified: {1}, errors: {2}', $created, $modified, $errors)];
+            } else {
+                return ['success' => 0, 'message' => __('Sharing groups could not be pushed. Errors: {0}', $errors)];
+            }
+        }
+        throw new MethodNotAllowedException(__('Invalid http request type for the given action.'));
+    }
+
     public function fetchSharingGroupAction(array $params): array
     {
         if ($params['request']->is(['get'])) {
@@ -1315,7 +1497,11 @@ class MispConnector extends CommonConnectorTools
                     'sharing_group_orgs' => []
                 ];
                 foreach ($mispSG['SharingGroupOrg'] as $sgo) {
-                    $sg['sharing_group_orgs'][] = $sgo['Organisation'];
+                    $sg['sharing_group_orgs'][] = [
+                        'name' => $sgo['Organisation']['name'],
+                        'uuid' => $sgo['Organisation']['uuid'],
+                        'extend' => $sgo['extend']
+                    ];
                 }
                 $result = $this->captureSharingGroup($sg, $params['user_id']);
                 if ($result) {

src/Model/Table/SharingGroupsTable.php

@@ -77,14 +77,17 @@ class SharingGroupsTable extends AppTable
 
     public function postCaptureActions($savedEntity, $input): void
     {
-        $additional_data = [];
+        $SGO = TableRegistry::getTableLocator()->get('SGOs');
-        if (!empty($input['extend'])) {
-            $additional_data['extend'] = $input['extend'];
-        }
-        $this->SGO = TableRegistry::getTableLocator()->get('SGOs');
         foreach ($input['sharing_group_orgs'] as $sgo) {
             $organisation_id = $this->Organisations->captureOrg($sgo);
-            $this->SGO->attach($savedEntity->id, $organisation_id, $additional_data);
+            $sgo_entity = $SGO->newEntity(
+                [
+                    'sharing_group_id' => $savedEntity->id,
+                    'organisation_id' => $organisation_id,
+                    'extend' => $sgo['extend']
+                ]
+            );
+            $SGO->save($sgo_entity);
         }
     }
 }