cerebrate-project
/
cerebrate
mirror of https://github.com/cerebrate-project/cerebrate
1
0
Fork 0
Code Issues Releases Wiki Activity

chg: [command] Generic importer improved and added support of value override

pull/35/head
mokaddem 2020-11-30 13:47:48 +01:00
parent 35ba595a82
commit fec6f817f6
1 changed files with 87 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
src/Command/ImporterCommand.php
View File

@ -14,6 +14,7 @@ use Cake\Http\Client;
class ImporterCommand extends Command class ImporterCommand extends Command
{ {
protected $modelClass = 'Organisations'; protected $modelClass = 'Organisations';
private $fieldsNoOverride = [];
protected function buildOptionParser(ConsoleOptionParser $parser): ConsoleOptionParser protected function buildOptionParser(ConsoleOptionParser $parser): ConsoleOptionParser
{ {
@ -27,7 +28,7 @@ class ImporterCommand extends Command
'required' => true 'required' => true
]); ]);
$parser->addArgument('primary_key', [ $parser->addArgument('primary_key', [
'help' => 'To avoid duplicates, entries having the value specified by the primary key will be updated instead of inserted. Empty if only insertion should be done', 'help' => 'To avoid duplicates, entries having the value specified by the primary key will be updated instead of inserted. Leave empty if only insertion should be done',
'required' => false, 'required' => false,
'default' => null, 'default' => null,
]); ]);
@ -43,23 +44,25 @@ class ImporterCommand extends Command
$table = $this->modelClass; $table = $this->modelClass;
$this->loadModel($table); $this->loadModel($table);
// $orgs = $this->{$table}->find()->enableHydration(false)->toList();
$config = $this->getConfigFromFile($configPath); $config = $this->getConfigFromFile($configPath);
$this->processConfig($config);
$sourceData = $this->getDataFromSource($source); $sourceData = $this->getDataFromSource($source);
// $sourceData = ['data' => [['name' => 'Test2', 'uuid' => '28de34ac-e284-495c-ae0e-9f46dd12da35', 'sector' => 'ss']]];
$data = $this->extractData($this->{$table}, $config, $sourceData); $data = $this->extractData($this->{$table}, $config, $sourceData);
$entities = $this->marshalData($this->{$table}, $data, $primary_key); $entities = $this->marshalData($this->{$table}, $data, $config, $primary_key);
$entitiesSample = array_slice($entities, 0, min(10, count($entities))); $entitiesSample = array_slice($entities, 0, min(10, count($entities)));
$ioTable = $this->transformEntitiesIntoTable($entitiesSample); $ioTable = $this->transformEntitiesIntoTable($entitiesSample);
$io->helper('Table')->output($ioTable); $io->helper('Table')->output($ioTable);
$selection = $io->askChoice('A sample of the data you are about to save is provided above. Would you like to proceed?', ['Y', 'N'], 'N'); $selection = $io->askChoice('A sample of the data you are about to save is provided above. Would you like to proceed?', ['Y', 'N'], 'N');
if ($selection == 'Y') { if ($selection == 'Y') {
$this->saveData($this->{$table}, $entities); $this->saveData($this->{$table}, $entities);
} }
} }
private function marshalData($table, $data, $primary_key=null) private function marshalData($table, $data, $config, $primary_key=null)
{ {
$this->loadModel('MetaFields');
$entities = []; $entities = [];
if (is_null($primary_key)) { if (is_null($primary_key)) {
$entities = $table->newEntities($data); $entities = $table->newEntities($data);
@ -68,10 +71,10 @@ class ImporterCommand extends Command
$query = $table->find('all') $query = $table->find('all')
->where(["${primary_key}" => $item[$primary_key]]); ->where(["${primary_key}" => $item[$primary_key]]);
$entity = $query->first(); $entity = $query->first();
if ($entity) { if (is_null($entity)) {
$entity->setAccess('uuid', false);
} else {
$entity = $table->newEmptyEntity(); $entity = $table->newEmptyEntity();
} else {
$this->lockAccess($config, $entity);
} }
$entity = $table->patchEntity($entity, $item); $entity = $table->patchEntity($entity, $item);
$entities[] = $entity; $entities[] = $entity;
@ -82,6 +85,29 @@ class ImporterCommand extends Command
if ($entity->hasErrors()) { if ($entity->hasErrors()) {
$hasErrors = true; $hasErrors = true;
$this->io->error(json_encode(['entity' => $entity, 'errors' => $entity->getErrors()], JSON_PRETTY_PRINT)); $this->io->error(json_encode(['entity' => $entity, 'errors' => $entity->getErrors()], JSON_PRETTY_PRINT));
} else {
$metaFields = [];
foreach ($entity['metaFields'] as $fieldName => $fieldValue) {
$metaEntity = null;
if (!$entity->isNew()) {
$query = $this->MetaFields->find('all')->where([
'parent_id' => $entity->id,
'field' => $fieldName
]);
$metaEntity = $query->first();
}
if (is_null($metaEntity)) {
$metaEntity = $this->MetaFields->newEmptyEntity();
$metaEntity->field = $fieldName;
$metaEntity->scope = $table->metaFields;
$metaEntity->parent_id = $entity->id;
}
if ($this->canBeOverriden($config, $metaEntity)) {
$metaEntity->value = $fieldValue;
}
$metaFields[] = $metaEntity;
}
$entities[$i]->metaFields = $metaFields;
} }
} }
if (!$hasErrors) { if (!$hasErrors) {
@ -99,31 +125,25 @@ class ImporterCommand extends Command
$this->io->error('Error while saving data'); $this->io->error('Error while saving data');
} }
$this->io->verbose('Saving meta fields'); $this->io->verbose('Saving meta fields');
$errorWhileSaving = 0;
foreach ($entities as $i => $entity) { foreach ($entities as $i => $entity) {
foreach ($entity['metaFields'] as $fieldName => $fieldValue) { $this->saveMetaFields($entity);
$query = $this->MetaFields->find('all')->where([ }
'parent_id' => $entity->id, }
'field' => $fieldName
]); private function saveMetaFields($entity)
$metaEntity = $query->first(); {
if (is_null($metaEntity)) { $errorWhileSaving = 0;
$metaEntity = $this->MetaFields->newEmptyEntity(); foreach ($entity->metaFields as $metaEntity) {
} $metaEntity->parent_id = $entity->id;
$metaEntity->field = $fieldName; $metaEntity = $this->MetaFields->save($metaEntity);
$metaEntity->value = $fieldValue; if ($metaEntity === false) {
$metaEntity->scope = $table->metaFields; $errorWhileSaving++;
$metaEntity->parent_id = $entity->id; $this->io->verbose('Error while saving metafield: ' . PHP_EOL . json_encode($metaEntity, JSON_PRETTY_PRINT));
$metaEntity = $this->MetaFields->save($metaEntity);
if ($metaEntity === false) {
$errorWhileSaving++;
$this->io->verbose('Error while saving metafield: ' . PHP_EOL . json_encode($metaEntity, JSON_PRETTY_PRINT));
}
}
if ($errorWhileSaving) {
$this->io->error('Error while saving meta data: ' . (string) $errorWhileSaving);
} }
} }
if ($errorWhileSaving) {
$this->io->error('Error while saving meta data: ' . (string) $errorWhileSaving);
}
} }
private function extractData($table, $config, $source) private function extractData($table, $config, $source)
@ -151,6 +171,18 @@ class ImporterCommand extends Command
return $this->invertArray($data); return $this->invertArray($data);
} }
private function lockAccess($config, &$entity)
{
foreach ($this->fieldsNoOverride as $fieldName) {
$entity->setAccess($fieldName, false);
}
}
private function canBeOverriden($config, $metaEntity)
{
return !in_array($metaEntity->field, $this->fieldsNoOverride);
}
private function getDataFromSource($source) private function getDataFromSource($source)
{ {
$data = $this->getDataFromFile($source); $data = $this->getDataFromFile($source);
@ -218,6 +250,18 @@ class ImporterCommand extends Command
} }
} }
private function processConfig($config)
{
$this->fieldsNoOverride = [];
foreach ($config as $fieldName => $fieldConfig) {
if (is_array($fieldConfig)) {
if (isset($fieldConfig['override']) && $fieldConfig['override'] === false) {
$this->fieldsNoOverride[] = $fieldName;
}
}
}
}
private function transformResultSetsIntoTable($result, $header=[]) private function transformResultSetsIntoTable($result, $header=[])
{ {
$table = [[]]; $table = [[]];
@ -248,8 +292,8 @@ class ImporterCommand extends Command
$tableHeader = array_filter($tableHeader, function($name) { $tableHeader = array_filter($tableHeader, function($name) {
return !in_array('metaFields', explode('.', $name)); return !in_array('metaFields', explode('.', $name));
}); });
foreach ($entities[0]['metaFields'] as $metaField => $metaValue) { foreach ($entities[0]->metaFields as $metaField) {
$tableHeader[] = "metaFields.$metaField"; $tableHeader[] = "metaFields.$metaField->field";
} }
$tableContent = []; $tableContent = [];
foreach ($entities as $entity) { foreach ($entities as $entity) {
@ -257,7 +301,17 @@ class ImporterCommand extends Command
foreach ($tableHeader as $key) { foreach ($tableHeader as $key) {
$subKeys = explode('.', $key); $subKeys = explode('.', $key);
if (in_array('metaFields', $subKeys)) { if (in_array('metaFields', $subKeys)) {
$row[] = (string) $entity['metaFields'][$subKeys[1]]; $found = false;
foreach ($entity->metaFields as $metaField) {
if ($metaField->field == $subKeys[1]) {
$row[] = (string) $metaField->value;
$found = true;
break;
}
}
if (!$found) {
$row[] = '';
}
} else { } else {
$row[] = (string) $entity[$key]; $row[] = (string) $entity[$key];
} }