new: [sync+meta_fields] Initial work on meta_field synchronisation and meta_template_directory - WiP

The new directory allows to ingest meta_fields without knowing their associated meta_template. Improved the way data is re-arranged, how meta-templates are saved and a helper widget showing the difference local objects have with their remote counter-part

config/Migrations/20221130000000_MoreDataOnMetaFields.php

@@ -0,0 +1,160 @@
+<?php
+declare(strict_types=1);
+
+use Migrations\AbstractMigration;
+use Phinx\Db\Adapter\MysqlAdapter;
+
+final class MoreDataOnMetaFields extends AbstractMigration
+{
+    public $autoId = false; // turn off automatic `id` column create. We want it to be `int(10) unsigned`
+
+    /**
+     * Change Method.
+     *
+     * Write your reversible migrations using this method.
+     *
+     * More information on writing migrations is available here:
+     * https://book.cakephp.org/phinx/0/en/migrations.html#the-change-method
+     *
+     * Remember to call "create()" or "update()" and NOT "save()" when working
+     * with the Table class.
+     */
+    public function change(): void
+    {
+        $metaFieldTable = $this->table('meta_fields');
+        if (!$metaFieldTable->hasColumn('meta_template_directory_id')) {
+            $metaFieldTable
+                ->addColumn('meta_template_directory_id', 'integer', [
+                    'default' => null,
+                    'null' => false,
+                    'signed' => false,
+                    'length' => 10
+                ])
+                ->addIndex('meta_template_directory_id')
+                ->update();
+        }
+
+        $exists = $this->hasTable('meta_template_name_directory');
+        if (!$exists) {
+            $templateNameDirectoryTable = $this->table('meta_template_name_directory', [
+                'signed' => false,
+                'collation' => 'utf8mb4_unicode_ci'
+            ]);
+            $templateNameDirectoryTable
+                ->addColumn('id', 'integer', [
+                    'autoIncrement' => true,
+                    'limit' => 10,
+                    'signed' => false,
+                ])
+                ->addPrimaryKey('id')
+                ->addColumn('name', 'string', [
+                    'null' => false,
+                    'limit' => 191,
+                    'collation' => 'utf8mb4_unicode_ci',
+                    'encoding' => 'utf8mb4',
+                ])
+                ->addColumn('namespace', 'string', [
+                    'null' => false,
+                    'limit' => 191,
+                    'collation' => 'utf8mb4_unicode_ci',
+                    'encoding' => 'utf8mb4',
+                ])
+                ->addColumn('version', 'string', [
+                    'null' => false,
+                    'limit' => 191,
+                    'collation' => 'utf8mb4_unicode_ci',
+                    'encoding' => 'utf8mb4',
+                ])
+                ->addColumn('uuid', 'uuid', [
+                    'null' => false,
+                    'default' => null,
+                ]);
+
+            $templateNameDirectoryTable
+                ->addIndex(['uuid', 'version'], ['unique' => true])
+                ->addIndex('name')
+                ->addIndex('namespace');
+
+            $templateNameDirectoryTable->create();
+
+            $allTemplates = $this->getAllTemplates();
+            $this->populateTemplateDirectoryTable($allTemplates);
+
+            $metaTemplateTable = $this->table('meta_templates');
+            $metaTemplateTable
+                ->addColumn('meta_template_directory_id', 'integer', [
+                    'default' => null,
+                    'null' => false,
+                    'signed' => false,
+                    'length' => 10
+                ])
+                ->update();
+            $this->assignTemplateDirectory($allTemplates);
+            $metaTemplateTable
+                ->addForeignKey('meta_template_directory_id', 'meta_template_name_directory', 'id', ['delete' => 'CASCADE', 'update' => 'CASCADE'])
+                ->save();
+
+            $metaFieldTable
+                ->dropForeignKey('meta_template_id')
+                ->dropForeignKey('meta_template_field_id')
+                ->addForeignKey('meta_template_directory_id', 'meta_template_name_directory', 'id', ['delete' => 'CASCADE', 'update' => 'CASCADE'])
+                ->save();
+        }
+    }
+
+    private function populateTemplateDirectoryTable(array $allTemplates): void
+    {
+        $builder = $this->getQueryBuilder()
+            ->insert(['uuid', 'name', 'namespace', 'version'])
+            ->into('meta_template_name_directory');
+
+        if (!empty($allTemplates)) {
+            foreach ($allTemplates as $template) {
+                $builder->values([
+                    'uuid' => $template['uuid'],
+                    'name' => $template['name'],
+                    'namespace' => $template['namespace'],
+                    'version' => $template['version'],
+                ]);
+            }
+            $builder->execute();
+        }
+    }
+
+    private function assignTemplateDirectory(array $allTemplates): void
+    {
+        foreach ($allTemplates as $template) {
+            $directory_template = $this->getDirectoryTemplate($template['uuid'], $template['version'])[0];
+            $this->getQueryBuilder()
+                ->update('meta_templates')
+                ->set('meta_template_directory_id', $directory_template['id'])
+                ->where(['meta_template_id' => $template['id']])
+                ->execute();
+            $this->getQueryBuilder()
+                ->update('meta_fields')
+                ->set('meta_template_directory_id', $directory_template['id'])
+                ->where(['id' => $template['id']])
+                ->execute();
+        }
+    }
+
+    private function getAllTemplates(): array
+    {
+        return $this->getQueryBuilder()
+            ->select(['id', 'uuid', 'name', 'namespace', 'version'])
+            ->from('meta_templates')
+            ->execute()->fetchAll('assoc');
+    }
+
+    private function getDirectoryTemplate(string $uuid, string $version): array
+    {
+        return $this->getQueryBuilder()
+            ->select(['id', 'uuid', 'version'])
+            ->from('meta_template_name_directory')
+            ->where([
+                'uuid' => $uuid,
+                'version' => $version,
+            ])
+            ->execute()->fetchAll('assoc');
+    }
+}

src/Controller/BroodsController.php

@@ -14,18 +14,6 @@ class BroodsController extends AppController
     public $quickFilterFields = [['Broods.name' => true], 'Broods.uuid', ['Broods.description' => true]];
     public $containFields = ['Organisations'];
 
-    protected $previewScopes = [
-        'organisations' => [
-            'quickFilterFields' => ['uuid', ['name' => true], ],
-        ],
-        'individuals' => [
-            'quickFilterFields' => ['uuid', ['email' => true], ['first_name' => true], ['last_name' => true], ],
-        ],
-        'sharingGroups' => [
-            'quickFilterFields' => ['uuid', ['name' => true], ],
-        ],
-    ];
-
     public function index()
     {
         $this->CRUD->index([
@@ -108,23 +96,24 @@ class BroodsController extends AppController
 
     public function previewIndex($id, $scope)
     {
-        $validScopes = array_keys($this->previewScopes);
+        $validScopes = array_keys($this->Broods->previewScopes);
         if (!in_array($scope, $validScopes)) {
             throw new MethodNotAllowedException(__('Invalid scope. Valid options are: {0}', implode(', ', $validScopes)));
         }
         $filter = $this->request->getQuery('quickFilter');
-        $data = $this->Broods->queryIndex($id, $scope, $filter);
+        $data = $this->Broods->queryIndex($id, $scope, $filter, true);
         if (!is_array($data)) {
             $data = [];
         }
         if ($this->ParamHandler->isRest()) {
             return $this->RestResponse->viewData($data, 'json');
         } else {
+            $data = $this->Broods->attachAllSyncStatus($data, $scope);
             $data = $this->CustomPagination->paginate($data);
             $optionFilters = ['quickFilter'];
             $CRUDParams = $this->ParamHandler->harvestParams($optionFilters);
             $CRUDOptions = [
-                'quickFilters' => $this->previewScopes[$scope]['quickFilterFields'],
+                'quickFilters' => $this->Broods->previewScopes[$scope]['quickFilterFields'],
             ];
             $this->CRUD->setQuickFilterForView($CRUDParams, $CRUDOptions);
             $this->set('data', $data);
@@ -139,23 +128,43 @@ class BroodsController extends AppController
 
     public function downloadOrg($brood_id, $org_id)
     {
-        $result = $this->Broods->downloadOrg($brood_id, $org_id);
+        if ($this->request->is('post')) {
-        $success = __('Organisation fetched from remote.');
+            $result = $this->Broods->downloadOrg($brood_id, $org_id);
-        $fail = __('Could not save the remote organisation');
+            $success = __('Organisation fetched from remote.');
-        if ($this->ParamHandler->isRest()) {
+            $fail = __('Could not save the remote organisation');
-            if ($result) {
+            if ($this->ParamHandler->isRest()) {
-                return $this->RestResponse->saveSuccessResponse('Brood', 'downloadOrg', $brood_id, 'json', $success);
+                if ($result) {
+                    return $this->RestResponse->saveSuccessResponse('Brood', 'downloadOrg', $brood_id, 'json', $success);
+                } else {
+                    return $this->RestResponse->saveFailResponse('Brood', 'downloadOrg', $brood_id, $fail, 'json');
+                }
             } else {
-                return $this->RestResponse->saveFailResponse('Brood', 'downloadOrg', $brood_id, $fail, 'json');
+                if ($result) {
+                    $this->Flash->success($success);
+                } else {
+                    $this->Flash->error($fail);
+                }
+                $this->redirect($this->referer());
             }
-        } else {
-            if ($result) {
-                $this->Flash->success($success);
-            } else {
-                $this->Flash->error($fail);
-            }
-            $this->redirect($this->referer());
         }
+        if ($org_id === 'all') {
+            $question = __('All organisations from brood `{0}` will be downloaded. Continue?', h($brood_id));
+            $title = __('Download all organisations from this brood');
+            $actionName = __('Download all');
+        } else {
+            $question = __('The organisations `{0}` from brood `{1}` will be downloaded. Continue?', h($org_id), h($brood_id));
+            $title = __('Download organisation from this brood');
+            $actionName = __('Download organisation');
+        }
+        $this->set('title',  $title);
+        $this->set('question', $question);
+        $this->set('modalOptions', [
+            'confirmButton' => [
+                'variant' => $org_id === 'all' ? 'warning' : 'primary',
+                'text' => $actionName,
+            ],
+        ]);
+        $this->render('/genericTemplates/confirm');
     }
 
     public function downloadIndividual($brood_id, $individual_id)

src/Controller/Component/APIRearrangeComponent.php

@@ -16,17 +16,17 @@ use Cake\Collection\Collection;
 
 class APIRearrangeComponent extends Component
 {
-    public function rearrangeForAPI(object $data)
+    public static function rearrangeForAPI(object $data, array $options = [])
     {
         if (is_subclass_of($data, 'Iterator')) {
             $newData = [];
-            $data->each(function ($value, $key) use (&$newData) {
+            $data->each(function ($value, $key) use (&$newData, $options) {
-                $value->rearrangeForAPI();
+                $value->rearrangeForAPI($options);
                 $newData[] = $value;
             });
             return new Collection($newData);
         } else {
-            $data->rearrangeForAPI();
+            $data->rearrangeForAPI($options);
         }
         return $data;
     }

src/Controller/Component/CRUDComponent.php

@@ -432,6 +432,7 @@ class CRUDComponent extends Component
                                     'field' => $rawMetaTemplateField->field,
                                     'meta_template_id' => $rawMetaTemplateField->meta_template_id,
                                     'meta_template_field_id' => $rawMetaTemplateField->id,
+                                    'meta_template_directory_id' => $allMetaTemplates[$template_id]->meta_template_directory_id,
                                     'parent_id' => $entity->id,
                                     'uuid' => Text::uuid(),
                                 ]);
@@ -462,6 +463,7 @@ class CRUDComponent extends Component
                                     'field' => $rawMetaTemplateField->field,
                                     'meta_template_id' => $rawMetaTemplateField->meta_template_id,
                                     'meta_template_field_id' => $rawMetaTemplateField->id,
+                                    'meta_template_directory_id' => $template->meta_template_directory_id,
                                     'parent_id' => $entity->id,
                                     'uuid' => Text::uuid(),
                                 ]);

src/Controller/Component/RestResponseComponent.php

@@ -283,6 +283,7 @@ class RestResponseComponent extends Component
     private $__scopedFieldsConstraint = array();
 
     public function initialize(array $config): void {
+        parent::initialize($config);
         $this->__configureFieldConstraints();
         $this->Controller = $this->getController();
     }
@@ -559,7 +560,14 @@ class RestResponseComponent extends Component
             $data['errors'] = $errors;
         }
         if (!$raw && is_object($data)) {
-            $data = $this->APIRearrange->rearrangeForAPI($data);
+            $rearrangeOptions = [];
+            if (!empty($this->Controller->getRequest()->getQuery('includeMetatemplate', false))) {
+                $rearrangeOptions['includeMetatemplate'] = true;
+            }
+            if (!empty($this->Controller->getRequest()->getQuery('includeFullMetaFields', false))) {
+                $rearrangeOptions['includeFullMetaFields'] = true;
+            }
+            $data = $this->APIRearrange->rearrangeForAPI($data, $rearrangeOptions);
         }
         return $this->__sendResponse($data, 200, $format, $raw, $download, $headers);
     }

src/Model/Entity/AppModel.php

@@ -40,28 +40,47 @@ class AppModel extends Entity
         return TableRegistry::get($this->getSource());
     }
 
-    public function rearrangeForAPI(): void
+    public function rearrangeForAPI(array $options = []): void
     {
     }
 
-    public function rearrangeMetaFields(): void
+    public function rearrangeMetaFields(array $options = []): void
     {
-        $this->meta_fields = [];
+        if (!empty($options['includeFullMetaFields'])) {
-        foreach ($this->MetaTemplates as $template) {
+            $this->meta_fields = [];
-            foreach ($template['meta_template_fields'] as $field) {
+            foreach ($this->MetaTemplates as $template) {
-                if ($field['counter'] > 0) {
+                foreach ($template['meta_template_fields'] as $field) {
-                    foreach ($field['metaFields'] as $metaField) {
+                    if ($field['counter'] > 0) {
-                        if (!empty($this->meta_fields[$template['name']][$field['field']])) {
+                        foreach ($field['metaFields'] as $metaField) {
-                            if (!is_array($this->meta_fields[$template['name']][$field['field']])) {
+                            if (!empty($this->meta_fields[$template['name']][$field['field']])) {
-                                $this->meta_fields[$template['name']][$field['field']] = [$this->meta_fields[$template['name']][$field['field']]];
+                                if (!is_array($this->meta_fields[$template['name']][$field['field']])) {
+                                    $this->meta_fields[$template['name']][$field['field']] = [$this->meta_fields[$template['name']][$field['field']]];
+                                }
+                                $this->meta_fields[$template['name']][$field['field']][] = $metaField['value'];
+                            } else {
+                                $this->meta_fields[$template['name']][$field['field']] = $metaField['value'];
                             }
-                            $this->meta_fields[$template['name']][$field['field']][] = $metaField['value'];
-                        } else {
-                            $this->meta_fields[$template['name']][$field['field']] = $metaField['value'];
                         }
                     }
                 }
             }
+        } elseif (!empty($this->meta_fields)) {
+            $templateDirectoryTable = TableRegistry::get('MetaTemplateNameDirectory');
+            $templates = [];
+            foreach ($this->meta_fields as $i => $metafield) {
+                $templateDirectoryId = $metafield['meta_template_directory_id'];
+                if (empty($templates[$templateDirectoryId])) {
+                    $templates[$templateDirectoryId] = $templateDirectoryTable->find()->where(['id' => $templateDirectoryId])->first();
+                }
+                $this->meta_fields[$i]['template_uuid'] = $templates[$templateDirectoryId]['uuid'];
+                $this->meta_fields[$i]['template_version'] = $templates[$templateDirectoryId]['version'];
+                $this->meta_fields[$i]['template_name'] = $templates[$templateDirectoryId]['name'];
+                $this->meta_fields[$i]['template_namespace'] = $templates[$templateDirectoryId]['namespace'];
+            }
+        }
+        // if ((!isset($options['includeMetatemplate']) || empty($options['includeMetatemplate'])) && !empty($this->MetaTemplates)) {
+        if ((!isset($options['includeMetatemplate']) || empty($options['includeMetatemplate']))) {
+            unset($this->MetaTemplates);
         }
     }
 

src/Model/Entity/AuditLog.php

@@ -66,7 +66,7 @@ class AuditLog extends AppModel
         return $title;
     }
 
-    public function rearrangeForAPI(): void
+    public function rearrangeForAPI(array $options = []): void
     {
         if (!empty($this->user)) {
             $this->user = $this->user->toArray();

src/Model/Entity/EncryptionKey.php

@@ -8,7 +8,7 @@ use Cake\ORM\Entity;
 class EncryptionKey extends AppModel
 {
 
-    public function rearrangeForAPI(): void
+    public function rearrangeForAPI(array $options = []): void
     {
         $this->rearrangeSimplify(['organisation', 'individual']);
     }

src/Model/Entity/Individual.php

@@ -42,7 +42,7 @@ class Individual extends AppModel
         return $emails;
     }
 
-    public function rearrangeForAPI(): void
+    public function rearrangeForAPI(array $options = []): void
     {
         if (!empty($this->tags)) {
             $this->tags = $this->rearrangeTags($this->tags);
@@ -51,10 +51,7 @@ class Individual extends AppModel
             $this->alignments = $this->rearrangeAlignments($this->alignments);
         }
         if (!empty($this->meta_fields)) {
-            $this->rearrangeMetaFields();
+            $this->rearrangeMetaFields($options);
-        }
-        if (!empty($this->MetaTemplates)) {
-            unset($this->MetaTemplates);
         }
     }
 }

src/Model/Entity/MetaTemplateNameDirectory.php

@@ -0,0 +1,11 @@
+<?php
+
+namespace App\Model\Entity;
+
+use App\Model\Entity\AppModel;
+use Cake\ORM\Entity;
+
+class MetaTemplateNameDirectory extends AppModel
+{
+
+}

src/Model/Entity/Organisation.php

@@ -17,7 +17,7 @@ class Organisation extends AppModel
         'created' => true
     ];
 
-    public function rearrangeForAPI(): void
+    public function rearrangeForAPI(array $options = []): void
     {
         if (!empty($this->tags)) {
             $this->tags = $this->rearrangeTags($this->tags);
@@ -25,11 +25,9 @@ class Organisation extends AppModel
         if (!empty($this->alignments)) {
             $this->alignments = $this->rearrangeAlignments($this->alignments);
         }
-        if (!empty($this->meta_fields)) {
+        if (!empty($this->meta_fields) || !empty($this->MetaTemplates)) {
-            $this->rearrangeMetaFields();
+            $this->rearrangeMetaFields($options);
-        }
-        if (!empty($this->MetaTemplates)) {
-            unset($this->MetaTemplates);
         }
     }
+    // MetaTemplate object property is not unset!!
 }

src/Model/Entity/User.php

@@ -49,16 +49,13 @@ class User extends AppModel
         }
     }
 
-    public function rearrangeForAPI(): void
+    public function rearrangeForAPI(array $options = []): void
     {
         if (!empty($this->tags)) {
             $this->tags = $this->rearrangeTags($this->tags);
         }
         if (!empty($this->meta_fields)) {
-            $this->rearrangeMetaFields();
+            $this->rearrangeMetaFields($options);
-        }
-        if (!empty($this->MetaTemplates)) {
-            unset($this->MetaTemplates);
         }
         if (!empty($this->user_settings_by_name)) {
             $this->rearrangeUserSettings();

src/Model/Table/BroodsTable.php

@@ -2,10 +2,15 @@
 
 namespace App\Model\Table;
 
+require_once APP . DS . 'Utility/Utils.php';
 use App\Model\Table\AppTable;
+use function App\Utility\Utils\array_diff_recursive;
 use Cake\ORM\Table;
 use Cake\Validation\Validator;
 use Cake\Core\Configure;
+use Cake\Utility\Inflector;
+use Cake\Utility\Hash;
+use Cake\I18n\FrozenTime;
 use Cake\Http\Client;
 use Cake\Http\Client\Response;
 use Cake\Http\Exception\NotFoundException;
@@ -15,6 +20,27 @@ use Cake\Error\Debugger;
 
 class BroodsTable extends AppTable
 {
+
+    public $previewScopes = [
+        'organisations' => [
+            'quickFilterFields' => ['uuid', ['name' => true],],
+            'contain' => ['MetaFields' => ['MetaTemplateNameDirectory'], 'Tags'],
+            'compareFields' => ['name', 'url', 'nationality', 'sector', 'type', 'contacts', 'modified', 'tags', 'meta_fields',],
+        ],
+        'individuals' => [
+            'quickFilterFields' => ['uuid', ['email' => true], ['first_name' => true], ['last_name' => true],],
+            'contain' => ['MetaFields'],
+            'compareFields' => ['email', 'first_name', 'last_name', 'position', 'modified', 'meta_fields', 'tags',],
+        ],
+        'sharingGroups' => [
+            'quickFilterFields' => ['uuid', ['name' => true],],
+            'contain' => ['SharingGroupOrgs', 'Organisations'],
+            'compareFields' => ['name', 'releasability', 'description', 'organisation_id', 'user_id', 'active', 'local', 'modified', 'organisation', 'sharing_group_orgs',],
+        ],
+    ];
+
+    private $metaFieldCompareFields = ['modified', 'value'];
+
     public function initialize(array $config): void
     {
         parent::initialize($config);
@@ -119,13 +145,16 @@ class BroodsTable extends AppTable
         return $result;
     }
 
-    public function queryIndex($id, $scope, $filter)
+    public function queryIndex($id, $scope, $filter, $full = false)
     {
         $brood = $this->find()->where(['id' => $id])->first();
         if (empty($brood)) {
             throw new NotFoundException(__('Brood not found'));
         }
         $filterQuery = empty($filter) ? '' : '?quickFilter=' . urlencode($filter);
+        if (!empty($full)) {
+            $filterQuery .= (empty($filterQuery) ? '?' : '&') . 'full=1';
+        }
         $response = $this->HTTPClientGET(sprintf('/%s/index.json%s', $scope, $filterQuery), $brood);
         if ($response->isOk()) {
             return $response->getJson();
@@ -371,4 +400,124 @@ class BroodsTable extends AppTable
         $connector = $params['connector'][$params['remote_tool']['connector']];
         $connector->remoteToolConnectionStatus($params, constant(get_class($connector) . '::' . $status));
     }
+
+    public function attachAllSyncStatus(array $data, string $scope): array
+    {
+        $options = $this->previewScopes[$scope];
+        foreach ($data as $i => $entry) {
+            $data[$i] = $this->__attachSyncStatus($scope, $entry, $options);
+        }
+        return $data;
+    }
+
+    private function __attachSyncStatus(string $scope, array $entry, array $options = []): array
+    {
+        $table = TableRegistry::getTableLocator()->get(Inflector::camelize($scope));
+        $localEntry = $table
+            ->find()
+            ->where(['uuid' => $entry['uuid']])
+            ->first();
+        if (is_null($localEntry)) {
+            $entry['status'] = $this->__statusNotLocal();
+        } else {
+            if (!empty($options['contain'])) {
+                $localEntry = $table->loadInto($localEntry, $options['contain']);
+            }
+            $localEntry = json_decode(json_encode($localEntry), true);
+            $entry['status'] = $this->__statusLocal($entry, $localEntry, $options);
+        }
+
+        return $entry;
+    }
+
+    private function __statusNotLocal(): array
+    {
+        return self::__getStatus(false);
+    }
+
+    private function __statusLocal(array $remoteEntry, $localEntry, array $options = []): array
+    {
+        $isLocalNewer = (new FrozenTime($localEntry['modified']))->toUnixString() >= (new FrozenTime($remoteEntry['modified']))->toUnixString();
+        $compareFields = $options['compareFields'];
+        $fieldDifference = [];
+        $fieldDifference = array_diff_recursive($remoteEntry, $localEntry);
+        // if (in_array('meta_fields', $options['compareFields']) && !empty($fieldDifference['meta_fields'])) {
+        //     $fieldDifference['meta_fields'] = $this->_compareMetaFields($remoteEntry, $localEntry, $options);
+        // }
+        $fieldDifference = array_filter($fieldDifference, function($value, $field) use ($compareFields) {
+            return in_array($field, $compareFields);
+        }, ARRAY_FILTER_USE_BOTH);
+        foreach ($fieldDifference as $fieldName => $value) {
+            $fieldDifference[$fieldName] = [
+                'local' => $localEntry[$fieldName],
+                'remote' => $value,
+            ];
+        }
+        if (in_array('meta_fields', $options['compareFields']) && !empty($fieldDifference['meta_fields'])) {
+            $fieldDifference['meta_fields'] = $this->_compareMetaFields($remoteEntry, $localEntry, $options);
+        }
+
+        return self::__getStatus(true, $isLocalNewer, $fieldDifference);
+    }
+
+    private static function __getStatus($local=true, $updateToDate=false, array $data = []): array
+    {
+        $status = [
+            'local' => $local,
+            'up_to_date' => $updateToDate,
+            'data' => $data,
+        ];
+        if ($status['local'] && $status['up_to_date']) {
+            $status['title'] = __('This entity is up-to-date');
+        } else if ($status['local'] && !$status['up_to_date']) {
+            $status['title'] = __('This entity is known but differs with the remote');
+        } else {
+            $status['title'] = __('This entity is not known locally');
+        }
+        return $status;
+    }
+
+    private function _compareMetaFields($remoteEntry, $localEntry): array
+    {
+        $compareFields = $this->metaFieldCompareFields;
+        $indexedRemoteMF = [];
+        $indexedLocalMF = [];
+        foreach ($remoteEntry['meta_fields'] as $metafields) {
+            $indexedRemoteMF[$metafields['uuid']] = array_intersect_key($metafields, array_flip($compareFields));
+        }
+        foreach ($localEntry['meta_fields'] as $metafields) {
+            $indexedLocalMF[$metafields['uuid']] = array_intersect_key($metafields, array_flip($compareFields));
+        }
+        $fieldDifference = [];
+        foreach ($remoteEntry['meta_fields'] as $remoteMetafield) {
+            $uuid = $remoteMetafield['uuid'];
+            $metafieldName = $remoteMetafield['field'];
+            // $metafieldName = sprintf('%s(v%s) :: %s', $remoteMetafield['template_name'], $remoteMetafield['template_version'], $remoteMetafield['field']);
+            if (empty($fieldDifference[$metafieldName])) {
+                $fieldDifference[$metafieldName] = [
+                    'meta_template' => [
+                        'name' => $remoteMetafield['template_name'],
+                        'version' => $remoteMetafield['template_version'],
+                        'uuid' => $remoteMetafield['template_uuid']
+                    ],
+                    'delta' => [],
+                ];
+            }
+            if (empty($indexedLocalMF[$uuid])) {
+                $fieldDifference[$metafieldName]['delta'][] = [
+                    'local' => null,
+                    'remote' => $indexedRemoteMF[$uuid],
+                ];
+            } else {
+                $fieldDifferenceTmp = array_diff_recursive($indexedRemoteMF[$uuid], $indexedLocalMF[$uuid]);
+                if (!empty($fieldDifferenceTmp)) {
+                    $fieldDifference[$metafieldName]['delta'][] = [
+                        'local' => $indexedLocalMF[$uuid],
+                        'remote' => $indexedRemoteMF[$uuid],
+                    ];
+                }
+            }
+        }
+        return $fieldDifference;
+    }
 }

src/Model/Table/MetaFieldsTable.php

@@ -5,7 +5,9 @@ namespace App\Model\Table;
 use App\Model\Table\AppTable;
 use Cake\ORM\Table;
 use Cake\Validation\Validator;
+use Cake\Event\EventInterface;
 use Cake\ORM\RulesChecker;
+use ArrayObject;
 
 class MetaFieldsTable extends AppTable
 {
@@ -22,6 +24,8 @@ class MetaFieldsTable extends AppTable
         $this->addBehavior('Timestamp');
         $this->belongsTo('MetaTemplates');
         $this->belongsTo('MetaTemplateFields');
+        $this->belongsTo('MetaTemplateNameDirectory')
+            ->setForeignKey('meta_template_directory_id');
 
         $this->setDisplayField('field');
     }
@@ -35,7 +39,10 @@ class MetaFieldsTable extends AppTable
             ->notEmptyString('value')
             ->notEmptyString('meta_template_id')
             ->notEmptyString('meta_template_field_id')
-            ->requirePresence(['scope', 'field', 'value', 'uuid', 'meta_template_id', 'meta_template_field_id'], 'create');
+            // ->requirePresence(['scope', 'field', 'value', 'uuid', 'meta_template_id', 'meta_template_field_id'], 'create');
+            // ->requirePresence(['scope', 'field', 'value', 'uuid',], 'create');
+            ->notEmptyString('meta_template_directory_id')
+            ->requirePresence(['scope', 'field', 'value', 'uuid', 'meta_template_directory_id', ], 'create');
 
         $validator->add('value', 'validMetaField', [
             'rule' => 'isValidMetaField',
@@ -46,10 +53,28 @@ class MetaFieldsTable extends AppTable
         return $validator;
     }
 
+    public function beforeMarshal(EventInterface $event, ArrayObject $data, ArrayObject $options)
+    {
+        if (!isset($data['meta_template_directory_id'])) {
+            $data['meta_template_directory_id'] = $this->getTemplateDirectoryIdFromMetaTemplate($data['meta_template_id']);
+        }
+    }
+
+    public function getTemplateDirectoryIdFromMetaTemplate($metaTemplateId): int
+    {
+        return $this->MetaTemplates->find()
+            ->select('meta_template_directory_id')
+            ->where(['id' => $metaTemplateId])
+            ->first();
+    }
+
     public function isValidMetaField($value, array $context)
     {
         $metaFieldsTable = $context['providers']['table'];
         $entityData = $context['data'];
+        if (empty($entityData['meta_template_field_id'])) {
+            return true;
+        }
         $metaTemplateField = $metaFieldsTable->MetaTemplateFields->get($entityData['meta_template_field_id']);
         return $this->isValidMetaFieldForMetaTemplateField($value, $metaTemplateField);
     }

src/Model/Table/MetaTemplateNameDirectoryTable.php

@@ -0,0 +1,47 @@
+<?php
+
+namespace App\Model\Table;
+
+use App\Model\Entity\MetaTemplate;
+use App\Model\Entity\MetaTemplateNameDirectory;
+use App\Model\Table\AppTable;
+use Cake\Validation\Validator;
+
+class MetaTemplateNameDirectoryTable extends AppTable
+{
+
+    public function initialize(array $config): void
+    {
+        parent::initialize($config);
+        $this->hasMany(
+            'MetaFields',
+            [
+                'foreignKey' => 'meta_template_directory_id',
+            ]
+        );
+        $this->setDisplayField('name');
+    }
+
+    public function validationDefault(Validator $validator): Validator
+    {
+        $validator
+            ->notEmptyString('name')
+            ->notEmptyString('namespace')
+            ->notEmptyString('uuid')
+            ->notEmptyString('version')
+            ->requirePresence(['version', 'uuid', 'name', 'namespace'], 'create');
+        return $validator;
+    }
+
+    public function createFromMetaTemplate(MetaTemplate $metaTemplate): MetaTemplateNameDirectory
+    {
+        $metaTemplateDirectory = $this->newEntity([
+            'name' => $metaTemplate['name'],
+            'namespace' => $metaTemplate['namespace'],
+            'uuid' => $metaTemplate['uuid'],
+            'version' => $metaTemplate['version'],
+        ]);
+        $this->save($metaTemplateDirectory);
+        return $metaTemplateDirectory;
+    }
+}

src/Model/Table/MetaTemplatesTable.php

@@ -42,6 +42,9 @@ class MetaTemplatesTable extends AppTable
                 'cascadeCallbacks' => true,
             ]
         );
+        $this->hasOne('MetaTemplateNameDirectory')
+            ->setForeignKey('meta_template_directory_id');
+
         $this->setDisplayField('name');
     }
 
@@ -54,7 +57,7 @@ class MetaTemplatesTable extends AppTable
             ->notEmptyString('uuid')
             ->notEmptyString('version')
             ->notEmptyString('source')
-            ->requirePresence(['scope', 'source', 'version', 'uuid', 'name', 'namespace'], 'create');
+            ->requirePresence(['scope', 'source', 'version', 'uuid', 'name', 'namespace', 'meta_template_directory_id'], 'create');
         return $validator;
     }
 
@@ -731,6 +734,8 @@ class MetaTemplatesTable extends AppTable
         $metaTemplate = $this->newEntity($template, [
             'associated' => ['MetaTemplateFields']
         ]);
+        $metaTemplateDirectory = $this->MetaTemplateNameDirectory->createFromMetaTemplate($metaTemplate);
+        $metaTemplate->meta_template_directory_id = $metaTemplateDirectory->id;
         $tmp = $this->save($metaTemplate, [
             'associated' => ['MetaTemplateFields']
         ]);

templates/Broods/preview_individuals.php

@@ -5,6 +5,16 @@ echo $this->element('genericElements/IndexTable/index_table', [
         'top_bar' => [
             'pull' => 'right',
             'children' => [
+                [
+                    'type' => 'simple',
+                    'children' => [
+                        'data' => [
+                            'type' => 'simple',
+                            'text' => __('Download All'),
+                            'popover_url' => sprintf('/broods/downloadIndividual/%s/all', h($brood_id)),
+                        ]
+                    ]
+                ],
                 [
                     'type' => 'search',
                     'button' => __('Search'),
@@ -21,6 +31,13 @@ echo $this->element('genericElements/IndexTable/index_table', [
                 'sort' => 'id',
                 'data_path' => 'id',
             ],
+            [
+                'name' => __('Status'),
+                'class' => 'short',
+                'data_path' => 'status',
+                'sort' => 'status',
+                'element' => 'brood_sync_status',
+            ],
             [
                 'name' => __('Email'),
                 'sort' => 'email',
@@ -53,8 +70,8 @@ echo $this->element('genericElements/IndexTable/index_table', [
         'pull' => 'right',
         'actions' => [
             [
-                'url' => '/broods/downloadIndividual/' . $brood_id,
+                'open_modal' => '/broods/downloadIndividual/' . $brood_id . '/[onclick_params_data_path]',
-                'url_params_data_paths' => ['id'],
+                'modal_params_data_path' => 'id',
                 'title' => __('Download'),
                 'icon' => 'download'
             ]

templates/Broods/preview_organisations.php

@@ -3,8 +3,17 @@ echo $this->element('genericElements/IndexTable/index_table', [
     'data' => [
         'data' => $data,
         'top_bar' => [
-            'pull' => 'right',
             'children' => [
+                [
+                    'type' => 'simple',
+                    'children' => [
+                        'data' => [
+                            'type' => 'simple',
+                            'text' => __('Download All'),
+                            'popover_url' => sprintf('/broods/downloadOrg/%s/all', h($brood_id)),
+                        ]
+                    ]
+                ],
                 [
                     'type' => 'search',
                     'button' => __('Search'),
@@ -22,6 +31,14 @@ echo $this->element('genericElements/IndexTable/index_table', [
                 'class' => 'short',
                 'data_path' => 'id',
             ],
+            [
+                'name' => __('Status'),
+                'class' => 'short',
+                'data_path' => 'status',
+                'display_field_data_path' => 'name',
+                'sort' => 'status',
+                'element' => 'brood_sync_status',
+            ],
             [
                 'name' => __('Name'),
                 'class' => 'short',
@@ -58,8 +75,8 @@ echo $this->element('genericElements/IndexTable/index_table', [
         'pull' => 'right',
         'actions' => [
             [
-                'url' => '/broods/downloadOrg/' . $brood_id,
+                'open_modal' => '/broods/downloadOrg/' . $brood_id . '/[onclick_params_data_path]',
-                'url_params_data_paths' => ['id'],
+                'modal_params_data_path' => 'id',
                 'title' => __('Download'),
                 'icon' => 'download'
             ]

templates/Broods/preview_sharing_groups.php

@@ -5,6 +5,16 @@ echo $this->element('genericElements/IndexTable/index_table', [
         'top_bar' => [
             'pull' => 'right',
             'children' => [
+                [
+                    'type' => 'simple',
+                    'children' => [
+                        'data' => [
+                            'type' => 'simple',
+                            'text' => __('Download All'),
+                            'popover_url' => sprintf('/broods/downloadSharingGroup/%s/all', h($brood_id)),
+                        ]
+                    ]
+                ],
                 [
                     'type' => 'search',
                     'button' => __('Search'),
@@ -22,6 +32,13 @@ echo $this->element('genericElements/IndexTable/index_table', [
                 'class' => 'short',
                 'data_path' => 'id',
             ],
+            [
+                'name' => __('Status'),
+                'class' => 'short',
+                'data_path' => 'status',
+                'sort' => 'status',
+                'element' => 'brood_sync_status',
+            ],
             [
                 'name' => __('Name'),
                 'class' => 'short',
@@ -38,8 +55,8 @@ echo $this->element('genericElements/IndexTable/index_table', [
         'pull' => 'right',
         'actions' => [
             [
-                'url' => '/broods/downloadSharingGroup/' . $brood_id,
+                'open_modal' => '/broods/downloadSharingGroup/' . $brood_id . '/[onclick_params_data_path]',
-                'url_params_data_paths' => ['id'],
+                'modal_params_data_path' => 'id',
                 'title' => __('Download'),
                 'icon' => 'download'
             ]

templates/element/genericElements/IndexTable/Fields/brood_sync_status.php

@@ -0,0 +1,171 @@
+<?php
+$seed = 's-' . mt_rand();
+$status = $this->Hash->extract($row, $field['data_path']);
+$displayField = $this->Hash->get($row, $field['display_field_data_path']);
+
+if ($status['local'] && $status['up_to_date']) {
+    $variant = 'success';
+    $text = __('Ok');
+} else if ($status['local'] && !$status['up_to_date']) {
+    $variant = 'warning';
+    $text = __('Outdated');
+} else {
+    $variant = 'danger';
+    $text = __('N/A');
+}
+
+echo $this->Bootstrap->badge([
+    'id' => $seed,
+    'variant' => $variant,
+    'text' => $text,
+    'icon' => ($status['local'] && !$status['up_to_date']) ? 'question-circle' : false,
+    'title' => $status['title'],
+    'class' => [
+        (($status['local'] && !$status['up_to_date']) ? 'cursor-pointer' : ''),
+    ],
+]);
+?>
+
+<?php if ($status['local'] && !$status['up_to_date']) : ?>
+    <script>
+        $(document).ready(function() {
+
+            function genTable(status) {
+                status.forEach(function(row, i) {
+                    status[i][1] = buildTableEntry(status[i][1])
+                    status[i][2] = buildTableEntry(status[i][2])
+                });
+                const $table = HtmlHelper.table(
+                    ['<?= __('Field name') ?>', '<?= __('Local value') ?>', '<?= __('Remote value') ?>'],
+                    status, {
+                        small: true,
+                        caption: `${status.length} fields`,
+                    }
+                )
+
+                const $container = $('<div>')
+                const $header = $('<h4>').text('<?= __('Main fields') ?>')
+                $container.append($header, $table)
+                return $container[0].outerHTML
+            }
+
+            function genTableForMetafields(status) {
+                let rearrangedStatus = []
+                for (const [field, metafieldData] of Object.entries(status)) {
+                    rearrangedChanges = []
+                    const metaTemplate = metafieldData['meta_template']
+                    const metafields = metafieldData['delta']
+                    metafields.forEach(function(metaFields, i) {
+                        const localMetafield = metaFields.local
+                        const remoteMetafield = metaFields.remote
+                        rearrangedChanges.push([
+                            buildTableEntryForMetaField(localMetafield),
+                            buildTableEntryForMetaField(remoteMetafield),
+                        ])
+                    })
+                    const $changesTable = HtmlHelper.table(
+                        null,
+                        rearrangedChanges, {
+                            small: true,
+                            borderless: true,
+                            striped: true,
+                            fixed_layout: true,
+                            tableClass: 'mb-0',
+                        }
+                    )
+                    const $field = $('<td>')
+                        .css('min-width', '8em')
+                        .text(field)
+                    const $template = $('<td>')
+                        .css('min-width', '6em')
+                        .append(
+                            $('<span>').text(metaTemplate.name),
+                            $('<sup>').text(`v${metaTemplate.version}`),
+                        )
+                    rearrangedStatus.push([
+                        $template,
+                        $field,
+                        $('<td>').attr('colspan', 2).append($changesTable),
+                    ])
+                }
+                const $container = $('<div>')
+                const $header = $('<h4>').text('<?= __('Meta Fields') ?>')
+                const metafieldAmount = Object.values(status).reduce(function(carry, metaFields) {
+                    return carry + metaFields.length
+                }, 0)
+                const $table = HtmlHelper.table(
+                    ['<?= __('Template') ?>', '<?= __('Field name') ?>', '<?= __('Local value') ?>', '<?= __('Remote value') ?>'],
+                    // ['<?= __('Field name') ?>', '<?= __('Local value') ?>', '<?= __('Remote value') ?>'],
+                    rearrangedStatus, {
+                        small: true,
+                        caption: `${metafieldAmount} meta-fields`,
+                    }
+                )
+                $container.append($header, $table)
+                return $container[0].outerHTML
+            }
+
+            function buildTableEntry(value) {
+                let $elem
+                if (typeof value === 'object') {
+                    $elem = $('<span>')
+                        .html(syntaxHighlightJson(value, 2))
+                } else {
+                    $elem = $('<pre>')
+                        .text(value)
+                }
+                return $elem
+            }
+
+            function buildTableEntryForMetaField(metafieldDifferences) {
+                if (metafieldDifferences !== null) {
+                    const $container = $('<table>').addClass('table table-borderless table-xs mb-0')
+                    for (const [field, value] of Object.entries(metafieldDifferences)) {
+                        const $entry = $('<tr>')
+                            .append(
+                                $('<th>')
+                                .addClass('fw-normal')
+                                .text(field),
+                                $('<td>')
+                                .append(
+                                    $('<pre>')
+                                    .addClass('d-inline mb-0')
+                                    .text(value)
+                                )
+                            )
+                        $container.append($entry)
+                    }
+                    return $container
+                }
+                return $('<span>').html(syntaxHighlightJson(metafieldDifferences))
+            }
+
+            const status = <?= json_encode($status) ?>;
+            $('#<?= $seed ?>')
+                .data('sync-status', status)
+                .click(function() {
+                    const syncStatusData = $(this).data('sync-status')['data']
+                    console.log(syncStatusData);
+                    let rearrangedStatusData = []
+                    for (const [field, values] of Object.entries(syncStatusData)) {
+                        if (field !== 'meta_fields') {
+                            rearrangedStatusData.push([
+                                field,
+                                values.local,
+                                values.remote,
+                            ])
+                        }
+                    }
+                    const bodyHtml = genTable(rearrangedStatusData) +
+                        (syncStatusData['meta_fields'] ? genTableForMetafields(syncStatusData['meta_fields']) : '')
+                    const options = {
+                        title: '<?= __('Difference with the remote for `{0}`', $displayField) ?>',
+                        bodyHtml: bodyHtml,
+                        type: 'ok-only',
+                        size: 'xl',
+                    }
+                    UI.modal(options)
+                })
+        })
+    </script>
+<?php endif; ?>