cerebrate-project
/
cerebrate
mirror of https://github.com/cerebrate-project/cerebrate
1
0
Fork 0
Code Issues Releases Wiki Activity

new: [CRUD:index] Allow exporting data into csv 

- Added CSVConverter tool and CSV server request detector
refacto/CRUDComponent
Sami Mokaddem 2023-11-02 08:08:06 +01:00
parent 1a7320e363
commit 63593cfd56
No known key found for this signature in database
GPG Key ID: 164C473F627A06FA
11 changed files with 188 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
config/bootstrap.php
View File

@ -194,6 +194,11 @@ ServerRequest::addDetector('tablet', function ($request) {
return $detector->isTablet();
});
ServerRequest::addDetector('csv', [
'accept' => ['text/csv',],
'param' => '_ext',
'value' => 'csv',
]);
/*
* You can set whether the ORM uses immutable or mutable Time types.

2
config/routes.php
View File

@ -45,7 +45,7 @@ use Cake\Routing\RouteBuilder;
/** @var \Cake\Routing\RouteBuilder $routes */
$routes->setRouteClass(DashedRoute::class);
$routes->scope('/', function (RouteBuilder $builder) {
$builder->setExtensions(['json']);
$builder->setExtensions(['json', 'csv']);
// Register scoped middleware for in scopes.
$builder->registerMiddleware('csrf', new CsrfProtectionMiddleware([
'httponly' => true,

25
src/Controller/Component/CRUDComponent.php
View File

@ -19,7 +19,7 @@ use App\Utility\UI\IndexSetting;
class CRUDComponent extends Component
{
public $components = ['RestResponse'];
public $components = ['RestResponse', 'APIRearrange'];
public function initialize(array $config): void
{
@ -107,7 +107,7 @@ class CRUDComponent extends Component
}
$data = $this->Controller->paginate($query, $this->Controller->paginate ?? []);
$totalCount = $this->Controller->getRequest()->getAttribute('paging')[$this->TableAlias]['count'];
if ($this->Controller->ParamHandler->isRest()) {
if ($this->Controller->ParamHandler->isRest() || $this->request->is('csv')) {
if (isset($options['hidden'])) {
$data->each(function($value, $key) use ($options) {
$hidden = is_array($options['hidden']) ? $options['hidden'] : [$options['hidden']];
@ -138,9 +138,21 @@ class CRUDComponent extends Component
return $this->attachMetaTemplatesIfNeeded($value, $metaTemplates);
});
}
$this->Controller->restResponsePayload = $this->RestResponse->viewData($data, 'json', false, false, false, [
'X-Total-Count' => $totalCount,
]);
if ($this->request->is('csv')) {
require_once(ROOT . '/src/Lib/Tools/CsvConverter.php');
$rearranged = $this->APIRearrange->rearrangeForAPI($data, ['smartFlattenMetafields' => true]);
$rearranged = $rearranged->map(function($e) {
return $e->toArray();
})->toList();
$data = \App\Lib\Tools\CsvConverter::flattenJSON($rearranged, []);
$this->Controller->restResponsePayload = $this->RestResponse->viewData($data, 'csv', false, false, false, [
'X-Total-Count' => $totalCount,
]);
} else {
$this->Controller->restResponsePayload = $this->RestResponse->viewData($data, 'json', false, false, false, [
'X-Total-Count' => $totalCount,
]);
}
} else {
$this->Controller->setResponse($this->Controller->getResponse()->withHeader('X-Total-Count', $totalCount));
if (isset($options['afterFind'])) {
@ -281,7 +293,7 @@ class CRUDComponent extends Component
*/
public function getResponsePayload()
{
if ($this->Controller->ParamHandler->isRest()) {
if ($this->Controller->ParamHandler->isRest() || $this->request->is('csv')) {
return $this->Controller->restResponsePayload;
} else if ($this->Controller->ParamHandler->isAjax() && $this->request->is(['post', 'put'])) {
return $this->Controller->ajaxResponsePayload;
@ -1305,6 +1317,7 @@ class CRUDComponent extends Component
}
$query = $this->setMetaFieldFilters($query, $filteringMetaFields);
}
$activeFilters['_here'] = $this->request->getRequestTarget();
$this->Controller->set('activeFilters', $activeFilters);
return $query;

1
src/Controller/Component/RestResponseComponent.php
View File

@ -4,6 +4,7 @@ namespace App\Controller\Component;
use Cake\Controller\Component;
use Cake\Core\Configure;
use Cake\Utility\Hash;
use Cake\Utility\Inflector;
class RestResponseComponent extends Component

84
src/Lib/Tools/CsvConverter.php Normal file
View File

@ -0,0 +1,84 @@
<?php
namespace App\Lib\Tools;
use Cake\Utility\Hash;
class CsvConverter
{
/**
* @param array $data
* @param array $options
* @return string
*/
public static function flattenJSON(array $data, $options=[]): string
{
$csv = '';
$toConvert = [];
if (!self::array_is_list($data)) {
$toConvert = [$data];
} else {
$toConvert = $data;
}
$headers = self::collectHeaders($toConvert);
$csv .= implode(',', self::quoteArray($headers)) . PHP_EOL;
foreach ($toConvert as $i => $item) {
$csv .= self::getRow($headers, $item);
}
return $csv;
}
private static function collectHeaders(array $items): array
{
$allHeaders = [];
foreach ($items as $item) {
$headers = Hash::flatten($item);
foreach ($headers as $head => $value) {
if (str_starts_with($head, '_')) {
continue;
}
if (is_array($value) && empty($value)) {
continue;
}
$allHeaders[$head] = 1;
}
}
return array_keys($allHeaders);
}
private static function getRow(array $headers, array $item): string
{
$tmp = [];
foreach ($headers as $header) {
$value = Hash::get($item, $header);
if (!isset($value)) {
$value = '';
}
if (is_bool($value)) {
$value = !empty($value) ? '1' : '0';
}
$tmp[] = '"' . $value . '"';
}
$row = implode(',', $tmp) . PHP_EOL;
return $row;
}
private static function quoteArray(array $arr): array
{
return array_map(function($item) {
return '"' . $item . '"';
}, $arr);
}
private static function array_is_list(array $arr): bool
{
if ($arr === []) {
return true;
}
return array_keys($arr) === range(0, count($arr) - 1);
}
}

9
src/Model/Entity/AppModel.php
View File

@ -78,6 +78,15 @@ class AppModel extends Entity
$this->meta_fields[$i]['template_namespace'] = $templates[$templateDirectoryId]['namespace'];
}
}
if (!empty($options['smartFlattenMetafields'])) {
$smartFlatten = [];
foreach ($this->meta_fields as $metafield) {
$key = "{$metafield['template_name']}_v{$metafield['template_version']}:{$metafield['field']}";
$value = $metafield['value'];
$smartFlatten[$key] = $value;
}
$this->meta_fields = $smartFlatten;
}
// if ((!isset($options['includeMetatemplate']) || empty($options['includeMetatemplate'])) && !empty($this->MetaTemplates)) {
if ((!isset($options['includeMetatemplate']) || empty($options['includeMetatemplate']))) {
unset($this->MetaTemplates);

3
src/View/Helper/BootstrapElements/BootstrapDropdownMenu.php
View File

@ -185,6 +185,9 @@ class BootstrapDropdownMenu extends BootstrapGeneric
$classes = array_merge($classes, $entry['class']);
}
$params = $entry['attrs'] ?? [];
if (!empty($entry['onclick'])) {
$params['onclick'] = $entry['onclick'];
}
$params['href'] = '#';
if (!empty($entry['menu'])) {

7
templates/element/genericElements/ListTopBar/group_search.php
View File

@ -24,7 +24,10 @@
$filteringButton = '';
if (!empty($data['allowFilering'])) {
$activeFilters = !empty($activeFilters) ? $activeFilters : [];
$numberActiveFilters = count($activeFilters);
$activeFiltersFiltered = array_filter($activeFilters, function ($k) {
return !str_starts_with($k, '_');
}, ARRAY_FILTER_USE_KEY);
$numberActiveFilters = count($activeFiltersFiltered);
if (!empty($activeFilters['filteringMetaFields'])) {
$numberActiveFilters += count($activeFilters['filteringMetaFields']) - 1;
}
@ -34,7 +37,7 @@
'title' => __('Filter index'),
'id' => sprintf('toggleFilterButton-%s', h($tableRandomValue))
];
if (count($activeFilters) > 0) {
if (count($activeFiltersFiltered) > 0) {
$buttonConfig['badge'] = [
'variant' => 'light',
'text' => $numberActiveFilters,

20
templates/element/genericElements/ListTopBar/group_table_action.php
View File

@ -5,6 +5,10 @@ use App\Utility\UI\IndexSetting;
if (empty($data['table_setting_id']) && empty($model)) {
throw new Exception(__('`table_setting_id` must be set in order to use the `table_action` table topbar'));
}
$now = date("Y-m-d_H-i-s");
$downloadFilename = sprintf('%s_%s', $data['table_setting_id'] ?? h($model), $now);
$data['table_setting_id'] = !empty($data['table_setting_id']) ? $data['table_setting_id'] : IndexSetting::getIDFromTable($model);
$tableSettings = IndexSetting::getTableSetting($loggedUser, $data['table_setting_id']);
$compactDisplay = !empty($tableSettings['compact_display']);
@ -52,6 +56,15 @@ $indexColumnMenu = array_merge(
$metaTemplateColumnMenu
);
$indexDownloadMenu = [
['header' => true, 'text' => 'JSON', 'icon' => 'file-code'],
['text' => __('Download all'), 'onclick' => sprintf('downloadIndexTable(this, "%s")', $downloadFilename . '.json'), ],
['text' => __('Download filtered table'), 'onclick' => sprintf('downloadIndexTable(this, "%s", true)', $downloadFilename . '.json'), ],
['header' => true, 'text' => 'CSV', 'icon' => 'file-csv', ],
['text' => __('Download all'), 'onclick' => sprintf('downloadIndexTable(this, "%s")', $downloadFilename . '.csv'), ],
['text' => __('Download filtered table'), 'onclick' => sprintf('downloadIndexTable(this, "%s", true)', $downloadFilename . '.csv'), ],
];
$compactDisplayHtml = $this->element('/genericElements/ListTopBar/group_table_action/compactDisplay', [
'table_data' => $table_data,
'tableSettings' => $tableSettings,
@ -68,8 +81,6 @@ $numberOfElementHtml = $this->element('/genericElements/ListTopBar/group_table_a
?>
<?php if (!isset($data['requirement']) || $data['requirement']) : ?>
<?php
$now = date("Y-m-d_H-i-s");
$downloadFilename = sprintf('%s_%s.json', $data['table_setting_id'] ?? h($model), $now);
echo $this->Bootstrap->dropdownMenu([
'dropdown-class' => 'ms-1',
'alignment' => 'end',
@ -95,9 +106,8 @@ $numberOfElementHtml = $this->element('/genericElements/ListTopBar/group_table_a
[
'text' => __('Download'),
'icon' => 'download',
'attrs' => [
'onclick' => sprintf('downloadIndexTable(this, "%s")', $downloadFilename),
],
'keepOpen' => true,
'menu' => $indexDownloadMenu,
],
[
'html' => $compactDisplayHtml,

29
webroot/js/api-helper.js
View File

@ -14,6 +14,9 @@ class AJAXApi {
static genericRequestConfigGETJSON = {
headers: new Headers(Object.assign({}, AJAXApi.genericRequestHeaders, {Accept: 'application/json'}))
}
static genericRequestConfigGETCSV = {
headers: new Headers(Object.assign({}, AJAXApi.genericRequestHeaders, {Accept: 'text/csv'}))
}
/**
* @namespace
@ -32,6 +35,7 @@ class AJAXApi {
},
successToastOptions: {
},
fetchOptions: {},
}
options = {}
loadingOverlay = false
@ -45,6 +49,13 @@ class AJAXApi {
this.mergeOptions(options)
}
/**
* Merge the configuration object to be used by fetch based on the options passed in the constructor
*/
mergeFetchConfig(base) {
return Object.assign({}, base, this.options.fetchOptions)
}
/**
* Based on the current configuration, provide feedback to the user via toast, console or do not
* @param {Object} toastOptions - The options supported by Toaster#defaultOptions
@ -149,11 +160,25 @@ class AJAXApi {
* @return {Promise<Object>} Promise object resolving to the fetched HTML
*/
static async quickFetchJSON(url, options={}) {
const constAlteredOptions = Object.assign({}, {provideFeedback: false}, options)
const constAlteredOptions = Object.assign({}, { provideFeedback: false, }, options)
const tmpApi = new AJAXApi(constAlteredOptions)
return tmpApi.fetchJSON(url, constAlteredOptions.skipRequestHooks)
}
/**
* @param {string} url - The URL to fetch
* @param {Object} [options={}] - The options supported by AJAXApi#defaultOptions
* @return {Promise<Object>} Promise object resolving to the fetched CSV
*/
static async quickFetchCSV(url, options={}) {
const constAlteredOptions = Object.assign({}, {
provideFeedback: false,
fetchOptions: AJAXApi.genericRequestConfigGETCSV
}, options)
const tmpApi = new AJAXApi(constAlteredOptions)
return tmpApi.fetchURL(url, constAlteredOptions.skipRequestHooks)
}
/**
* @param {string} url - The URL to fetch
* @param {Object} [options={}] - The options supported by AJAXApi#defaultOptions
@ -213,7 +238,7 @@ class AJAXApi {
}
let toReturn
try {
const response = await fetch(url, AJAXApi.genericRequestConfigGET);
const response = await fetch(url, this.mergeFetchConfig(AJAXApi.genericRequestConfigGET));
if (!response.ok) {
throw new Error(`Network response was not ok. \`${response.statusText}\``)
}

25
webroot/js/main.js
View File

@ -206,21 +206,34 @@ function deleteBookmark(bookmark, forSidebar=false) {
}).catch((e) => { })
}
function downloadIndexTable(downloadButton, filename) {
function downloadIndexTable(downloadButton, filename, filtered) {
const $dropdownMenu = $(downloadButton).closest('.dropdown')
const tableRandomValue = $dropdownMenu.attr('data-table-random-value')
const $container = $dropdownMenu.closest('div[id^="table-container-"]')
const $table = $container.find(`table[data-table-random-value="${tableRandomValue}"]`)
const $filterButton = $(`#toggleFilterButton-${tableRandomValue}`)
const activeFilters = $filterButton.data('activeFilters')
const additionalUrlParams = $filterButton.data('additionalUrlParams') ? $filterButton.data('additionalUrlParams') : ''
const searchParam = jQuery.param(activeFilters);
const url = $table.data('reload-url') + additionalUrlParams + '?' + searchParam
// const additionalUrlParams = $filterButton.data('additionalUrlParams') ? $filterButton.data('additionalUrlParams') : ''
// const searchParam = jQuery.param(activeFilters);
// const url = $table.data('reload-url') + additionalUrlParams + '?' + searchParam
let url = $table.data('reload-url')
if (filtered) {
url = activeFilters._here
}
let options = {}
const downloadPromise = AJAXApi.quickFetchJSON(url, options)
let downloadPromise;
if (filename.endsWith('.csv')) {
downloadPromise = AJAXApi.quickFetchCSV(url, options)
} else {
downloadPromise = AJAXApi.quickFetchJSON(url, options)
}
UI.overlayUntilResolve($dropdownMenu, downloadPromise)
downloadPromise.then((data) => {
download(filename, JSON.stringify(data, undefined, 4))
if (filename.endsWith('.csv')) {
download(filename, data)
} else {
download(filename, JSON.stringify(data, undefined, 4))
}
})
}