new: [CRUD:index] Allow exporting data into csv

- Added CSVConverter tool and CSV server request detector
refacto/CRUDComponent
Sami Mokaddem 2023-11-02 08:08:06 +01:00
parent 1a7320e363
commit 63593cfd56
No known key found for this signature in database
GPG Key ID: 164C473F627A06FA
11 changed files with 188 additions and 22 deletions

View File

@ -194,6 +194,11 @@ ServerRequest::addDetector('tablet', function ($request) {
return $detector->isTablet(); return $detector->isTablet();
}); });
ServerRequest::addDetector('csv', [
'accept' => ['text/csv',],
'param' => '_ext',
'value' => 'csv',
]);
/* /*
* You can set whether the ORM uses immutable or mutable Time types. * You can set whether the ORM uses immutable or mutable Time types.

View File

@ -45,7 +45,7 @@ use Cake\Routing\RouteBuilder;
/** @var \Cake\Routing\RouteBuilder $routes */ /** @var \Cake\Routing\RouteBuilder $routes */
$routes->setRouteClass(DashedRoute::class); $routes->setRouteClass(DashedRoute::class);
$routes->scope('/', function (RouteBuilder $builder) { $routes->scope('/', function (RouteBuilder $builder) {
$builder->setExtensions(['json']); $builder->setExtensions(['json', 'csv']);
// Register scoped middleware for in scopes. // Register scoped middleware for in scopes.
$builder->registerMiddleware('csrf', new CsrfProtectionMiddleware([ $builder->registerMiddleware('csrf', new CsrfProtectionMiddleware([
'httponly' => true, 'httponly' => true,

View File

@ -19,7 +19,7 @@ use App\Utility\UI\IndexSetting;
class CRUDComponent extends Component class CRUDComponent extends Component
{ {
public $components = ['RestResponse']; public $components = ['RestResponse', 'APIRearrange'];
public function initialize(array $config): void public function initialize(array $config): void
{ {
@ -107,7 +107,7 @@ class CRUDComponent extends Component
} }
$data = $this->Controller->paginate($query, $this->Controller->paginate ?? []); $data = $this->Controller->paginate($query, $this->Controller->paginate ?? []);
$totalCount = $this->Controller->getRequest()->getAttribute('paging')[$this->TableAlias]['count']; $totalCount = $this->Controller->getRequest()->getAttribute('paging')[$this->TableAlias]['count'];
if ($this->Controller->ParamHandler->isRest()) { if ($this->Controller->ParamHandler->isRest() || $this->request->is('csv')) {
if (isset($options['hidden'])) { if (isset($options['hidden'])) {
$data->each(function($value, $key) use ($options) { $data->each(function($value, $key) use ($options) {
$hidden = is_array($options['hidden']) ? $options['hidden'] : [$options['hidden']]; $hidden = is_array($options['hidden']) ? $options['hidden'] : [$options['hidden']];
@ -138,9 +138,21 @@ class CRUDComponent extends Component
return $this->attachMetaTemplatesIfNeeded($value, $metaTemplates); return $this->attachMetaTemplatesIfNeeded($value, $metaTemplates);
}); });
} }
$this->Controller->restResponsePayload = $this->RestResponse->viewData($data, 'json', false, false, false, [ if ($this->request->is('csv')) {
'X-Total-Count' => $totalCount, require_once(ROOT . '/src/Lib/Tools/CsvConverter.php');
]); $rearranged = $this->APIRearrange->rearrangeForAPI($data, ['smartFlattenMetafields' => true]);
$rearranged = $rearranged->map(function($e) {
return $e->toArray();
})->toList();
$data = \App\Lib\Tools\CsvConverter::flattenJSON($rearranged, []);
$this->Controller->restResponsePayload = $this->RestResponse->viewData($data, 'csv', false, false, false, [
'X-Total-Count' => $totalCount,
]);
} else {
$this->Controller->restResponsePayload = $this->RestResponse->viewData($data, 'json', false, false, false, [
'X-Total-Count' => $totalCount,
]);
}
} else { } else {
$this->Controller->setResponse($this->Controller->getResponse()->withHeader('X-Total-Count', $totalCount)); $this->Controller->setResponse($this->Controller->getResponse()->withHeader('X-Total-Count', $totalCount));
if (isset($options['afterFind'])) { if (isset($options['afterFind'])) {
@ -281,7 +293,7 @@ class CRUDComponent extends Component
*/ */
public function getResponsePayload() public function getResponsePayload()
{ {
if ($this->Controller->ParamHandler->isRest()) { if ($this->Controller->ParamHandler->isRest() || $this->request->is('csv')) {
return $this->Controller->restResponsePayload; return $this->Controller->restResponsePayload;
} else if ($this->Controller->ParamHandler->isAjax() && $this->request->is(['post', 'put'])) { } else if ($this->Controller->ParamHandler->isAjax() && $this->request->is(['post', 'put'])) {
return $this->Controller->ajaxResponsePayload; return $this->Controller->ajaxResponsePayload;
@ -1305,6 +1317,7 @@ class CRUDComponent extends Component
} }
$query = $this->setMetaFieldFilters($query, $filteringMetaFields); $query = $this->setMetaFieldFilters($query, $filteringMetaFields);
} }
$activeFilters['_here'] = $this->request->getRequestTarget();
$this->Controller->set('activeFilters', $activeFilters); $this->Controller->set('activeFilters', $activeFilters);
return $query; return $query;

View File

@ -4,6 +4,7 @@ namespace App\Controller\Component;
use Cake\Controller\Component; use Cake\Controller\Component;
use Cake\Core\Configure; use Cake\Core\Configure;
use Cake\Utility\Hash;
use Cake\Utility\Inflector; use Cake\Utility\Inflector;
class RestResponseComponent extends Component class RestResponseComponent extends Component

View File

@ -0,0 +1,84 @@
<?php
namespace App\Lib\Tools;
use Cake\Utility\Hash;
class CsvConverter
{
/**
* @param array $data
* @param array $options
* @return string
*/
public static function flattenJSON(array $data, $options=[]): string
{
$csv = '';
$toConvert = [];
if (!self::array_is_list($data)) {
$toConvert = [$data];
} else {
$toConvert = $data;
}
$headers = self::collectHeaders($toConvert);
$csv .= implode(',', self::quoteArray($headers)) . PHP_EOL;
foreach ($toConvert as $i => $item) {
$csv .= self::getRow($headers, $item);
}
return $csv;
}
private static function collectHeaders(array $items): array
{
$allHeaders = [];
foreach ($items as $item) {
$headers = Hash::flatten($item);
foreach ($headers as $head => $value) {
if (str_starts_with($head, '_')) {
continue;
}
if (is_array($value) && empty($value)) {
continue;
}
$allHeaders[$head] = 1;
}
}
return array_keys($allHeaders);
}
private static function getRow(array $headers, array $item): string
{
$tmp = [];
foreach ($headers as $header) {
$value = Hash::get($item, $header);
if (!isset($value)) {
$value = '';
}
if (is_bool($value)) {
$value = !empty($value) ? '1' : '0';
}
$tmp[] = '"' . $value . '"';
}
$row = implode(',', $tmp) . PHP_EOL;
return $row;
}
private static function quoteArray(array $arr): array
{
return array_map(function($item) {
return '"' . $item . '"';
}, $arr);
}
private static function array_is_list(array $arr): bool
{
if ($arr === []) {
return true;
}
return array_keys($arr) === range(0, count($arr) - 1);
}
}

View File

@ -78,6 +78,15 @@ class AppModel extends Entity
$this->meta_fields[$i]['template_namespace'] = $templates[$templateDirectoryId]['namespace']; $this->meta_fields[$i]['template_namespace'] = $templates[$templateDirectoryId]['namespace'];
} }
} }
if (!empty($options['smartFlattenMetafields'])) {
$smartFlatten = [];
foreach ($this->meta_fields as $metafield) {
$key = "{$metafield['template_name']}_v{$metafield['template_version']}:{$metafield['field']}";
$value = $metafield['value'];
$smartFlatten[$key] = $value;
}
$this->meta_fields = $smartFlatten;
}
// if ((!isset($options['includeMetatemplate']) || empty($options['includeMetatemplate'])) && !empty($this->MetaTemplates)) { // if ((!isset($options['includeMetatemplate']) || empty($options['includeMetatemplate'])) && !empty($this->MetaTemplates)) {
if ((!isset($options['includeMetatemplate']) || empty($options['includeMetatemplate']))) { if ((!isset($options['includeMetatemplate']) || empty($options['includeMetatemplate']))) {
unset($this->MetaTemplates); unset($this->MetaTemplates);

View File

@ -185,6 +185,9 @@ class BootstrapDropdownMenu extends BootstrapGeneric
$classes = array_merge($classes, $entry['class']); $classes = array_merge($classes, $entry['class']);
} }
$params = $entry['attrs'] ?? []; $params = $entry['attrs'] ?? [];
if (!empty($entry['onclick'])) {
$params['onclick'] = $entry['onclick'];
}
$params['href'] = '#'; $params['href'] = '#';
if (!empty($entry['menu'])) { if (!empty($entry['menu'])) {

View File

@ -24,7 +24,10 @@
$filteringButton = ''; $filteringButton = '';
if (!empty($data['allowFilering'])) { if (!empty($data['allowFilering'])) {
$activeFilters = !empty($activeFilters) ? $activeFilters : []; $activeFilters = !empty($activeFilters) ? $activeFilters : [];
$numberActiveFilters = count($activeFilters); $activeFiltersFiltered = array_filter($activeFilters, function ($k) {
return !str_starts_with($k, '_');
}, ARRAY_FILTER_USE_KEY);
$numberActiveFilters = count($activeFiltersFiltered);
if (!empty($activeFilters['filteringMetaFields'])) { if (!empty($activeFilters['filteringMetaFields'])) {
$numberActiveFilters += count($activeFilters['filteringMetaFields']) - 1; $numberActiveFilters += count($activeFilters['filteringMetaFields']) - 1;
} }
@ -34,7 +37,7 @@
'title' => __('Filter index'), 'title' => __('Filter index'),
'id' => sprintf('toggleFilterButton-%s', h($tableRandomValue)) 'id' => sprintf('toggleFilterButton-%s', h($tableRandomValue))
]; ];
if (count($activeFilters) > 0) { if (count($activeFiltersFiltered) > 0) {
$buttonConfig['badge'] = [ $buttonConfig['badge'] = [
'variant' => 'light', 'variant' => 'light',
'text' => $numberActiveFilters, 'text' => $numberActiveFilters,

View File

@ -5,6 +5,10 @@ use App\Utility\UI\IndexSetting;
if (empty($data['table_setting_id']) && empty($model)) { if (empty($data['table_setting_id']) && empty($model)) {
throw new Exception(__('`table_setting_id` must be set in order to use the `table_action` table topbar')); throw new Exception(__('`table_setting_id` must be set in order to use the `table_action` table topbar'));
} }
$now = date("Y-m-d_H-i-s");
$downloadFilename = sprintf('%s_%s', $data['table_setting_id'] ?? h($model), $now);
$data['table_setting_id'] = !empty($data['table_setting_id']) ? $data['table_setting_id'] : IndexSetting::getIDFromTable($model); $data['table_setting_id'] = !empty($data['table_setting_id']) ? $data['table_setting_id'] : IndexSetting::getIDFromTable($model);
$tableSettings = IndexSetting::getTableSetting($loggedUser, $data['table_setting_id']); $tableSettings = IndexSetting::getTableSetting($loggedUser, $data['table_setting_id']);
$compactDisplay = !empty($tableSettings['compact_display']); $compactDisplay = !empty($tableSettings['compact_display']);
@ -52,6 +56,15 @@ $indexColumnMenu = array_merge(
$metaTemplateColumnMenu $metaTemplateColumnMenu
); );
$indexDownloadMenu = [
['header' => true, 'text' => 'JSON', 'icon' => 'file-code'],
['text' => __('Download all'), 'onclick' => sprintf('downloadIndexTable(this, "%s")', $downloadFilename . '.json'), ],
['text' => __('Download filtered table'), 'onclick' => sprintf('downloadIndexTable(this, "%s", true)', $downloadFilename . '.json'), ],
['header' => true, 'text' => 'CSV', 'icon' => 'file-csv', ],
['text' => __('Download all'), 'onclick' => sprintf('downloadIndexTable(this, "%s")', $downloadFilename . '.csv'), ],
['text' => __('Download filtered table'), 'onclick' => sprintf('downloadIndexTable(this, "%s", true)', $downloadFilename . '.csv'), ],
];
$compactDisplayHtml = $this->element('/genericElements/ListTopBar/group_table_action/compactDisplay', [ $compactDisplayHtml = $this->element('/genericElements/ListTopBar/group_table_action/compactDisplay', [
'table_data' => $table_data, 'table_data' => $table_data,
'tableSettings' => $tableSettings, 'tableSettings' => $tableSettings,
@ -68,8 +81,6 @@ $numberOfElementHtml = $this->element('/genericElements/ListTopBar/group_table_a
?> ?>
<?php if (!isset($data['requirement']) || $data['requirement']) : ?> <?php if (!isset($data['requirement']) || $data['requirement']) : ?>
<?php <?php
$now = date("Y-m-d_H-i-s");
$downloadFilename = sprintf('%s_%s.json', $data['table_setting_id'] ?? h($model), $now);
echo $this->Bootstrap->dropdownMenu([ echo $this->Bootstrap->dropdownMenu([
'dropdown-class' => 'ms-1', 'dropdown-class' => 'ms-1',
'alignment' => 'end', 'alignment' => 'end',
@ -95,9 +106,8 @@ $numberOfElementHtml = $this->element('/genericElements/ListTopBar/group_table_a
[ [
'text' => __('Download'), 'text' => __('Download'),
'icon' => 'download', 'icon' => 'download',
'attrs' => [ 'keepOpen' => true,
'onclick' => sprintf('downloadIndexTable(this, "%s")', $downloadFilename), 'menu' => $indexDownloadMenu,
],
], ],
[ [
'html' => $compactDisplayHtml, 'html' => $compactDisplayHtml,

View File

@ -14,6 +14,9 @@ class AJAXApi {
static genericRequestConfigGETJSON = { static genericRequestConfigGETJSON = {
headers: new Headers(Object.assign({}, AJAXApi.genericRequestHeaders, {Accept: 'application/json'})) headers: new Headers(Object.assign({}, AJAXApi.genericRequestHeaders, {Accept: 'application/json'}))
} }
static genericRequestConfigGETCSV = {
headers: new Headers(Object.assign({}, AJAXApi.genericRequestHeaders, {Accept: 'text/csv'}))
}
/** /**
* @namespace * @namespace
@ -32,6 +35,7 @@ class AJAXApi {
}, },
successToastOptions: { successToastOptions: {
}, },
fetchOptions: {},
} }
options = {} options = {}
loadingOverlay = false loadingOverlay = false
@ -45,6 +49,13 @@ class AJAXApi {
this.mergeOptions(options) this.mergeOptions(options)
} }
/**
* Merge the configuration object to be used by fetch based on the options passed in the constructor
*/
mergeFetchConfig(base) {
return Object.assign({}, base, this.options.fetchOptions)
}
/** /**
* Based on the current configuration, provide feedback to the user via toast, console or do not * Based on the current configuration, provide feedback to the user via toast, console or do not
* @param {Object} toastOptions - The options supported by Toaster#defaultOptions * @param {Object} toastOptions - The options supported by Toaster#defaultOptions
@ -149,11 +160,25 @@ class AJAXApi {
* @return {Promise<Object>} Promise object resolving to the fetched HTML * @return {Promise<Object>} Promise object resolving to the fetched HTML
*/ */
static async quickFetchJSON(url, options={}) { static async quickFetchJSON(url, options={}) {
const constAlteredOptions = Object.assign({}, {provideFeedback: false}, options) const constAlteredOptions = Object.assign({}, { provideFeedback: false, }, options)
const tmpApi = new AJAXApi(constAlteredOptions) const tmpApi = new AJAXApi(constAlteredOptions)
return tmpApi.fetchJSON(url, constAlteredOptions.skipRequestHooks) return tmpApi.fetchJSON(url, constAlteredOptions.skipRequestHooks)
} }
/**
* @param {string} url - The URL to fetch
* @param {Object} [options={}] - The options supported by AJAXApi#defaultOptions
* @return {Promise<Object>} Promise object resolving to the fetched CSV
*/
static async quickFetchCSV(url, options={}) {
const constAlteredOptions = Object.assign({}, {
provideFeedback: false,
fetchOptions: AJAXApi.genericRequestConfigGETCSV
}, options)
const tmpApi = new AJAXApi(constAlteredOptions)
return tmpApi.fetchURL(url, constAlteredOptions.skipRequestHooks)
}
/** /**
* @param {string} url - The URL to fetch * @param {string} url - The URL to fetch
* @param {Object} [options={}] - The options supported by AJAXApi#defaultOptions * @param {Object} [options={}] - The options supported by AJAXApi#defaultOptions
@ -213,7 +238,7 @@ class AJAXApi {
} }
let toReturn let toReturn
try { try {
const response = await fetch(url, AJAXApi.genericRequestConfigGET); const response = await fetch(url, this.mergeFetchConfig(AJAXApi.genericRequestConfigGET));
if (!response.ok) { if (!response.ok) {
throw new Error(`Network response was not ok. \`${response.statusText}\``) throw new Error(`Network response was not ok. \`${response.statusText}\``)
} }

View File

@ -206,21 +206,34 @@ function deleteBookmark(bookmark, forSidebar=false) {
}).catch((e) => { }) }).catch((e) => { })
} }
function downloadIndexTable(downloadButton, filename) { function downloadIndexTable(downloadButton, filename, filtered) {
const $dropdownMenu = $(downloadButton).closest('.dropdown') const $dropdownMenu = $(downloadButton).closest('.dropdown')
const tableRandomValue = $dropdownMenu.attr('data-table-random-value') const tableRandomValue = $dropdownMenu.attr('data-table-random-value')
const $container = $dropdownMenu.closest('div[id^="table-container-"]') const $container = $dropdownMenu.closest('div[id^="table-container-"]')
const $table = $container.find(`table[data-table-random-value="${tableRandomValue}"]`) const $table = $container.find(`table[data-table-random-value="${tableRandomValue}"]`)
const $filterButton = $(`#toggleFilterButton-${tableRandomValue}`) const $filterButton = $(`#toggleFilterButton-${tableRandomValue}`)
const activeFilters = $filterButton.data('activeFilters') const activeFilters = $filterButton.data('activeFilters')
const additionalUrlParams = $filterButton.data('additionalUrlParams') ? $filterButton.data('additionalUrlParams') : '' // const additionalUrlParams = $filterButton.data('additionalUrlParams') ? $filterButton.data('additionalUrlParams') : ''
const searchParam = jQuery.param(activeFilters); // const searchParam = jQuery.param(activeFilters);
const url = $table.data('reload-url') + additionalUrlParams + '?' + searchParam // const url = $table.data('reload-url') + additionalUrlParams + '?' + searchParam
let url = $table.data('reload-url')
if (filtered) {
url = activeFilters._here
}
let options = {} let options = {}
const downloadPromise = AJAXApi.quickFetchJSON(url, options) let downloadPromise;
if (filename.endsWith('.csv')) {
downloadPromise = AJAXApi.quickFetchCSV(url, options)
} else {
downloadPromise = AJAXApi.quickFetchJSON(url, options)
}
UI.overlayUntilResolve($dropdownMenu, downloadPromise) UI.overlayUntilResolve($dropdownMenu, downloadPromise)
downloadPromise.then((data) => { downloadPromise.then((data) => {
download(filename, JSON.stringify(data, undefined, 4)) if (filename.endsWith('.csv')) {
download(filename, data)
} else {
download(filename, JSON.stringify(data, undefined, 4))
}
}) })
} }