new: [CRUD:index] Allow exporting data into csv

- Added CSVConverter tool and CSV server request detector

config/bootstrap.php

@@ -194,6 +194,11 @@ ServerRequest::addDetector('tablet', function ($request) {
 
     return $detector->isTablet();
 });
+ServerRequest::addDetector('csv', [
+    'accept' => ['text/csv',],
+    'param' => '_ext',
+    'value' => 'csv',
+]);
 
 /*
  * You can set whether the ORM uses immutable or mutable Time types.

config/routes.php

@@ -45,7 +45,7 @@ use Cake\Routing\RouteBuilder;
 /** @var \Cake\Routing\RouteBuilder $routes */
 $routes->setRouteClass(DashedRoute::class);
 $routes->scope('/', function (RouteBuilder $builder) {
-    $builder->setExtensions(['json']);
+    $builder->setExtensions(['json', 'csv']);
     // Register scoped middleware for in scopes.
     $builder->registerMiddleware('csrf', new CsrfProtectionMiddleware([
         'httponly' => true,

src/Controller/Component/CRUDComponent.php

@@ -19,7 +19,7 @@ use App\Utility\UI\IndexSetting;
 
 class CRUDComponent extends Component
 {
-    public $components = ['RestResponse'];
+    public $components = ['RestResponse', 'APIRearrange'];
 
     public function initialize(array $config): void
     {
@@ -107,7 +107,7 @@ class CRUDComponent extends Component
         }
         $data = $this->Controller->paginate($query, $this->Controller->paginate ?? []);
         $totalCount = $this->Controller->getRequest()->getAttribute('paging')[$this->TableAlias]['count'];
-        if ($this->Controller->ParamHandler->isRest()) {
+        if ($this->Controller->ParamHandler->isRest() || $this->request->is('csv')) {
             if (isset($options['hidden'])) {
                 $data->each(function($value, $key) use ($options) {
                     $hidden = is_array($options['hidden']) ? $options['hidden'] : [$options['hidden']];
@@ -138,9 +138,21 @@ class CRUDComponent extends Component
                     return $this->attachMetaTemplatesIfNeeded($value, $metaTemplates);
                 });
             }
-            $this->Controller->restResponsePayload = $this->RestResponse->viewData($data, 'json', false, false, false, [
+            if ($this->request->is('csv')) {
-                'X-Total-Count' => $totalCount,
+                require_once(ROOT . '/src/Lib/Tools/CsvConverter.php');
-            ]);
+                $rearranged = $this->APIRearrange->rearrangeForAPI($data, ['smartFlattenMetafields' => true]);
+                $rearranged = $rearranged->map(function($e) {
+                    return $e->toArray();
+                })->toList();
+                $data = \App\Lib\Tools\CsvConverter::flattenJSON($rearranged, []);
+                $this->Controller->restResponsePayload = $this->RestResponse->viewData($data, 'csv', false, false, false, [
+                    'X-Total-Count' => $totalCount,
+                ]);
+            } else {
+                $this->Controller->restResponsePayload = $this->RestResponse->viewData($data, 'json', false, false, false, [
+                    'X-Total-Count' => $totalCount,
+                ]);
+            }
         } else {
             $this->Controller->setResponse($this->Controller->getResponse()->withHeader('X-Total-Count', $totalCount));
             if (isset($options['afterFind'])) {
@@ -281,7 +293,7 @@ class CRUDComponent extends Component
      */
     public function getResponsePayload()
     {
-        if ($this->Controller->ParamHandler->isRest()) {
+        if ($this->Controller->ParamHandler->isRest() || $this->request->is('csv')) {
             return $this->Controller->restResponsePayload;
         } else if ($this->Controller->ParamHandler->isAjax() && $this->request->is(['post', 'put'])) {
             return $this->Controller->ajaxResponsePayload;
@@ -1305,6 +1317,7 @@ class CRUDComponent extends Component
             }
             $query = $this->setMetaFieldFilters($query, $filteringMetaFields);
         }
+        $activeFilters['_here'] = $this->request->getRequestTarget();
 
         $this->Controller->set('activeFilters', $activeFilters);
         return $query;

src/Controller/Component/RestResponseComponent.php

@@ -4,6 +4,7 @@ namespace App\Controller\Component;
 
 use Cake\Controller\Component;
 use Cake\Core\Configure;
+use Cake\Utility\Hash;
 use Cake\Utility\Inflector;
 
 class RestResponseComponent extends Component

src/Lib/Tools/CsvConverter.php

@@ -0,0 +1,84 @@
+<?php
+
+namespace App\Lib\Tools;
+
+use Cake\Utility\Hash;
+
+
+class CsvConverter
+{
+
+    /**
+     * @param array $data
+     * @param array $options
+     * @return string
+     */
+    public static function flattenJSON(array $data, $options=[]): string
+    {
+        $csv = '';
+        $toConvert = [];
+        if (!self::array_is_list($data)) {
+            $toConvert = [$data];
+        } else {
+            $toConvert = $data;
+        }
+
+        $headers = self::collectHeaders($toConvert);
+        $csv .= implode(',', self::quoteArray($headers)) . PHP_EOL;
+        foreach ($toConvert as $i => $item) {
+            $csv .= self::getRow($headers, $item);
+        }
+
+        return $csv;
+    }
+
+    private static function collectHeaders(array $items): array
+    {
+        $allHeaders = [];
+        foreach ($items as $item) {
+            $headers = Hash::flatten($item);
+            foreach ($headers as $head => $value) {
+                if (str_starts_with($head, '_')) {
+                    continue;
+                }
+                if (is_array($value) && empty($value)) {
+                    continue;
+                }
+                $allHeaders[$head] = 1;
+            }
+        }
+        return array_keys($allHeaders);
+    }
+
+    private static function getRow(array $headers, array $item): string
+    {
+        $tmp = [];
+        foreach ($headers as $header) {
+            $value = Hash::get($item, $header);
+            if (!isset($value)) {
+                $value = '';
+            }
+            if (is_bool($value)) {
+                $value = !empty($value) ? '1' : '0';
+            }
+            $tmp[] = '"' . $value . '"';
+        }
+        $row = implode(',', $tmp) . PHP_EOL;
+        return $row;
+    }
+
+    private static function quoteArray(array $arr): array
+    {
+        return array_map(function($item) {
+            return '"' . $item . '"';
+        }, $arr);
+    }
+
+    private static function array_is_list(array $arr): bool
+    {
+        if ($arr === []) {
+            return true;
+        }
+        return array_keys($arr) === range(0, count($arr) - 1);
+    }
+}

src/Model/Entity/AppModel.php

@@ -78,6 +78,15 @@ class AppModel extends Entity
                 $this->meta_fields[$i]['template_namespace'] = $templates[$templateDirectoryId]['namespace'];
             }
         }
+        if (!empty($options['smartFlattenMetafields'])) {
+            $smartFlatten = [];
+            foreach ($this->meta_fields as $metafield) {
+                $key = "{$metafield['template_name']}_v{$metafield['template_version']}:{$metafield['field']}";
+                $value = $metafield['value'];
+                $smartFlatten[$key] = $value;
+            }
+            $this->meta_fields = $smartFlatten;
+        }
         // if ((!isset($options['includeMetatemplate']) || empty($options['includeMetatemplate'])) && !empty($this->MetaTemplates)) {
         if ((!isset($options['includeMetatemplate']) || empty($options['includeMetatemplate']))) {
             unset($this->MetaTemplates);

src/View/Helper/BootstrapElements/BootstrapDropdownMenu.php

@@ -185,6 +185,9 @@ class BootstrapDropdownMenu extends BootstrapGeneric
             $classes = array_merge($classes, $entry['class']);
         }
         $params = $entry['attrs'] ?? [];
+        if (!empty($entry['onclick'])) {
+            $params['onclick'] = $entry['onclick'];
+        }
         $params['href'] = '#';
 
         if (!empty($entry['menu'])) {

templates/element/genericElements/ListTopBar/group_search.php

@@ -24,7 +24,10 @@
         $filteringButton = '';
         if (!empty($data['allowFilering'])) {
             $activeFilters = !empty($activeFilters) ? $activeFilters : [];
-            $numberActiveFilters = count($activeFilters);
+            $activeFiltersFiltered = array_filter($activeFilters, function ($k) {
+                return !str_starts_with($k, '_');
+            }, ARRAY_FILTER_USE_KEY);
+            $numberActiveFilters = count($activeFiltersFiltered);
             if (!empty($activeFilters['filteringMetaFields'])) {
                 $numberActiveFilters += count($activeFilters['filteringMetaFields']) - 1;
             }
@@ -34,7 +37,7 @@
                 'title' => __('Filter index'),
                 'id' => sprintf('toggleFilterButton-%s', h($tableRandomValue))
             ];
-            if (count($activeFilters) > 0) {
+            if (count($activeFiltersFiltered) > 0) {
                 $buttonConfig['badge'] = [
                     'variant' => 'light',
                     'text' => $numberActiveFilters,

templates/element/genericElements/ListTopBar/group_table_action.php

@@ -5,6 +5,10 @@ use App\Utility\UI\IndexSetting;
 if (empty($data['table_setting_id']) && empty($model)) {
     throw new Exception(__('`table_setting_id` must be set in order to use the `table_action` table topbar'));
 }
+
+$now = date("Y-m-d_H-i-s");
+$downloadFilename = sprintf('%s_%s', $data['table_setting_id'] ?? h($model), $now);
+
 $data['table_setting_id'] = !empty($data['table_setting_id']) ? $data['table_setting_id'] : IndexSetting::getIDFromTable($model);
 $tableSettings = IndexSetting::getTableSetting($loggedUser, $data['table_setting_id']);
 $compactDisplay = !empty($tableSettings['compact_display']);
@@ -52,6 +56,15 @@ $indexColumnMenu = array_merge(
     $metaTemplateColumnMenu
 );
 
+$indexDownloadMenu = [
+    ['header' => true, 'text' => 'JSON', 'icon' => 'file-code'],
+    ['text' => __('Download all'), 'onclick' => sprintf('downloadIndexTable(this, "%s")', $downloadFilename . '.json'), ],
+    ['text' => __('Download filtered table'), 'onclick' => sprintf('downloadIndexTable(this, "%s", true)', $downloadFilename . '.json'), ],
+    ['header' => true, 'text' => 'CSV', 'icon' => 'file-csv', ],
+    ['text' => __('Download all'), 'onclick' => sprintf('downloadIndexTable(this, "%s")', $downloadFilename . '.csv'), ],
+    ['text' => __('Download filtered table'), 'onclick' => sprintf('downloadIndexTable(this, "%s", true)', $downloadFilename . '.csv'), ],
+];
+
 $compactDisplayHtml = $this->element('/genericElements/ListTopBar/group_table_action/compactDisplay', [
     'table_data' => $table_data,
     'tableSettings' => $tableSettings,
@@ -68,8 +81,6 @@ $numberOfElementHtml = $this->element('/genericElements/ListTopBar/group_table_a
 ?>
 <?php if (!isset($data['requirement']) || $data['requirement']) : ?>
     <?php
-    $now = date("Y-m-d_H-i-s");
-    $downloadFilename = sprintf('%s_%s.json', $data['table_setting_id'] ?? h($model), $now);
     echo $this->Bootstrap->dropdownMenu([
         'dropdown-class' => 'ms-1',
         'alignment' => 'end',
@@ -95,9 +106,8 @@ $numberOfElementHtml = $this->element('/genericElements/ListTopBar/group_table_a
             [
                 'text' => __('Download'),
                 'icon' => 'download',
-                'attrs' => [
+                'keepOpen' => true,
-                    'onclick' => sprintf('downloadIndexTable(this, "%s")', $downloadFilename),
+                'menu' => $indexDownloadMenu,
-                ],
             ],
             [
                 'html' => $compactDisplayHtml,

webroot/js/api-helper.js

@@ -14,6 +14,9 @@ class AJAXApi {
     static genericRequestConfigGETJSON = {
         headers: new Headers(Object.assign({}, AJAXApi.genericRequestHeaders, {Accept: 'application/json'}))
     }
+    static genericRequestConfigGETCSV = {
+        headers: new Headers(Object.assign({}, AJAXApi.genericRequestHeaders, {Accept: 'text/csv'}))
+    }
 
     /**
      * @namespace
@@ -32,6 +35,7 @@ class AJAXApi {
         },
         successToastOptions: {
         },
+        fetchOptions: {},
     }
     options = {}
     loadingOverlay = false
@@ -45,6 +49,13 @@ class AJAXApi {
         this.mergeOptions(options)
     }
 
+    /**
+     * Merge the configuration object to be used by fetch based on the options passed in the constructor
+     */
+    mergeFetchConfig(base) {
+        return Object.assign({}, base, this.options.fetchOptions)
+    }
+
     /**
      * Based on the current configuration, provide feedback to the user via toast, console or do not
      * @param {Object} toastOptions - The options supported by Toaster#defaultOptions
@@ -149,11 +160,25 @@ class AJAXApi {
      * @return {Promise<Object>} Promise object resolving to the fetched HTML
      */
     static async quickFetchJSON(url, options={}) {
-        const constAlteredOptions = Object.assign({}, {provideFeedback: false}, options)
+        const constAlteredOptions = Object.assign({}, { provideFeedback: false, }, options)
         const tmpApi = new AJAXApi(constAlteredOptions)
         return tmpApi.fetchJSON(url, constAlteredOptions.skipRequestHooks)
     }
 
+    /**
+     * @param {string} url           - The URL to fetch
+     * @param {Object} [options={}]  - The options supported by AJAXApi#defaultOptions
+     * @return {Promise<Object>} Promise object resolving to the fetched CSV
+     */
+    static async quickFetchCSV(url, options={}) {
+        const constAlteredOptions = Object.assign({}, {
+            provideFeedback: false,
+            fetchOptions: AJAXApi.genericRequestConfigGETCSV
+        }, options)
+        const tmpApi = new AJAXApi(constAlteredOptions)
+        return tmpApi.fetchURL(url, constAlteredOptions.skipRequestHooks)
+    }
+
     /**
      * @param {string} url          - The URL to fetch
      * @param {Object} [options={}] - The options supported by AJAXApi#defaultOptions
@@ -213,7 +238,7 @@ class AJAXApi {
         }
         let toReturn
         try {
-            const response = await fetch(url, AJAXApi.genericRequestConfigGET);
+            const response = await fetch(url, this.mergeFetchConfig(AJAXApi.genericRequestConfigGET));
             if (!response.ok) {
                 throw new Error(`Network response was not ok. \`${response.statusText}\``)
             }

webroot/js/main.js

@@ -206,21 +206,34 @@ function deleteBookmark(bookmark, forSidebar=false) {
     }).catch((e) => { })
 }
 
-function downloadIndexTable(downloadButton, filename) {
+function downloadIndexTable(downloadButton, filename, filtered) {
     const $dropdownMenu = $(downloadButton).closest('.dropdown')
     const tableRandomValue = $dropdownMenu.attr('data-table-random-value')
     const $container = $dropdownMenu.closest('div[id^="table-container-"]')
     const $table = $container.find(`table[data-table-random-value="${tableRandomValue}"]`)
     const $filterButton = $(`#toggleFilterButton-${tableRandomValue}`)
     const activeFilters = $filterButton.data('activeFilters')
-    const additionalUrlParams = $filterButton.data('additionalUrlParams') ? $filterButton.data('additionalUrlParams') : ''
+    // const additionalUrlParams = $filterButton.data('additionalUrlParams') ? $filterButton.data('additionalUrlParams') : ''
-    const searchParam = jQuery.param(activeFilters);
+    // const searchParam = jQuery.param(activeFilters);
-    const url = $table.data('reload-url') + additionalUrlParams + '?' + searchParam
+    // const url = $table.data('reload-url') + additionalUrlParams + '?' + searchParam
+    let url = $table.data('reload-url')
+    if (filtered) {
+        url = activeFilters._here
+    }
     let options = {}
-    const downloadPromise = AJAXApi.quickFetchJSON(url, options)
+    let downloadPromise;
+    if (filename.endsWith('.csv')) {
+        downloadPromise = AJAXApi.quickFetchCSV(url, options)
+    } else {
+        downloadPromise = AJAXApi.quickFetchJSON(url, options)
+    }
     UI.overlayUntilResolve($dropdownMenu, downloadPromise)
     downloadPromise.then((data) => {
-        download(filename, JSON.stringify(data, undefined, 4))
+        if (filename.endsWith('.csv')) {
+            download(filename, data)
+        } else {
+            download(filename, JSON.stringify(data, undefined, 4))
+        }
     })
 }