cerebrate-project
/
cerebrate
mirror of https://github.com/cerebrate-project/cerebrate
1
0
Fork 0
Code Issues Releases Wiki Activity

chg: [command:importer] Improved tool and added support of more options

pull/40/head
mokaddem 2021-03-10 09:22:47 +01:00
parent 7f8e319c68
commit 008c674f4d
1 changed files with 65 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
src/Command/ImporterCommand.php
View File

@ -42,6 +42,9 @@ class ImporterCommand extends Command
protected $modelClass = 'Organisations'; protected $modelClass = 'Organisations';
private $fieldsNoOverride = []; private $fieldsNoOverride = [];
private $format = 'json'; private $format = 'json';
private $noMetaTemplate = false;
private $autoYes = false;
private $updateOnly = false;
protected function buildOptionParser(ConsoleOptionParser $parser): ConsoleOptionParser protected function buildOptionParser(ConsoleOptionParser $parser): ConsoleOptionParser
{ {
@ -64,6 +67,18 @@ class ImporterCommand extends Command
'help' => 'The target cerebrate model for the import', 'help' => 'The target cerebrate model for the import',
'default' => 'Organisations', 'default' => 'Organisations',
'choices' => ['Organisations', 'Individuals', 'AuthKeys'] 'choices' => ['Organisations', 'Individuals', 'AuthKeys']
]);
$parser->addOption('yes', [
'short' => 'y',
'help' => 'Automatically assume yes to any prompts',
'default' => false,
'boolean' => true
]);
$parser->addOption('update-only', [
'short' => 'u',
'help' => 'Only update existing record. No new record will be created. primary_key MUST be supplied',
'default' => false,
'boolean' => true
]); ]);
return $parser; return $parser;
} }
@ -78,12 +93,18 @@ class ImporterCommand extends Command
if (!is_null($model_class)) { if (!is_null($model_class)) {
$this->modelClass = $model_class; $this->modelClass = $model_class;
} }
$this->autoYes = $args->getOption('yes');
$this->updateOnly = $args->getOption('update-only');
if ($this->updateOnly && is_null($primary_key)) {
$io->error('A `primary_key` must be supplied when using `--update-only` mode.');
die(1);
}
$table = $this->modelClass; $table = $this->modelClass;
$this->loadModel($table); $this->loadModel($table);
$config = $this->getConfigFromFile($configPath); $config = $this->getConfigFromFile($configPath);
$this->processConfig($config); $this->processConfig($config);
$sourceData = $this->getDataFromSource($source); $sourceData = $this->getDataFromSource($source, $config);
$data = $this->extractData($this->{$table}, $config, $sourceData); $data = $this->extractData($this->{$table}, $config, $sourceData);
$entities = $this->marshalData($this->{$table}, $data, $config, $primary_key); $entities = $this->marshalData($this->{$table}, $data, $config, $primary_key);
@ -91,9 +112,13 @@ class ImporterCommand extends Command
$ioTable = $this->transformEntitiesIntoTable($entitiesSample); $ioTable = $this->transformEntitiesIntoTable($entitiesSample);
$io->helper('Table')->output($ioTable); $io->helper('Table')->output($ioTable);
$selection = $io->askChoice('A sample of the data you are about to save is provided above. Would you like to proceed?', ['Y', 'N'], 'N'); if ($this->autoYes) {
if ($selection == 'Y') {
$this->saveData($this->{$table}, $entities); $this->saveData($this->{$table}, $entities);
} else {
$selection = $io->askChoice('A sample of the data you about to be saved is provided above. Would you like to proceed?', ['Y', 'N'], 'N');
if ($selection == 'Y') {
$this->saveData($this->{$table}, $entities);
}
} }
} }
@ -113,32 +138,40 @@ class ImporterCommand extends Command
$entity = $query->first(); $entity = $query->first();
} }
if (is_null($entity)) { if (is_null($entity)) {
$entity = $table->newEmptyEntity(); if (!$this->updateOnly) {
$entity = $table->newEmptyEntity();
}
} else { } else {
$this->lockAccess($entity); $this->lockAccess($entity);
} }
$entity = $table->patchEntity($entity, $item); if (!is_null($entity)) {
$entities[] = $entity; $entity = $table->patchEntity($entity, $item);
$entities[] = $entity;
}
} }
} }
$hasErrors = false; $hasErrors = false;
$metaTemplate = $this->MetaTemplates->find() if (!$this->noMetaTemplate) {
->where(['uuid' => $config['metaTemplateUUID']]) $metaTemplate = $this->MetaTemplates->find()
->first(); ->where(['uuid' => $config['metaTemplateUUID']])
if (!is_null($metaTemplate)) { ->first();
$metaTemplateFieldsMapping = $this->MetaTemplates->MetaTemplateFields->find('list', [ if (!is_null($metaTemplate)) {
'keyField' => 'field', $metaTemplateFieldsMapping = $this->MetaTemplates->MetaTemplateFields->find('list', [
'valueField' => 'id' 'keyField' => 'field',
])->where(['meta_template_id' => $metaTemplate->id])->toArray(); 'valueField' => 'id'
} else { ])->where(['meta_template_id' => $metaTemplate->id])->toArray();
$this->io->error("Unkown template for UUID $metaTemplateUUID"); } else {
$this->io->error("Unkown template for UUID $metaTemplateUUID");
die(1);
}
} }
foreach ($entities as $i => $entity) { foreach ($entities as $i => $entity) {
if ($entity->hasErrors()) { if ($entity->hasErrors()) {
$hasErrors = true; $hasErrors = true;
$this->io->error(json_encode(['entity' => $entity, 'errors' => $entity->getErrors()], JSON_PRETTY_PRINT)); $this->io->error(json_encode(['entity' => $entity, 'errors' => $entity->getErrors()], JSON_PRETTY_PRINT));
} else { } else {
if (!is_null($metaTemplate)) { if (!$this->noMetaTemplate && !is_null($metaTemplate)) {
$metaFields = []; $metaFields = [];
foreach ($entity['metaFields'] as $fieldName => $fieldValue) { foreach ($entity['metaFields'] as $fieldName => $fieldValue) {
$metaEntity = null; $metaEntity = null;
@ -198,7 +231,9 @@ class ImporterCommand extends Command
'length' => 20 'length' => 20
]); ]);
foreach ($entities as $i => $entity) { foreach ($entities as $i => $entity) {
$this->saveMetaFields($entity); if (!$this->noMetaTemplate) {
$this->saveMetaFields($entity);
}
$progress->increment(1); $progress->increment(1);
$progress->draw(); $progress->draw();
} }
@ -293,16 +328,16 @@ class ImporterCommand extends Command
return !in_array($metaEntity->field, $this->fieldsNoOverride); return !in_array($metaEntity->field, $this->fieldsNoOverride);
} }
private function getDataFromSource($source) private function getDataFromSource($source, $config)
{ {
$data = $this->getDataFromFile($source); $data = $this->getDataFromFile($source);
if ($data === false) { if ($data === false) {
$data = $this->getDataFromURL($source); $data = $this->getDataFromURL($source, $config);
} }
return $data; return $data;
} }
private function getDataFromURL($url) private function getDataFromURL($url, $config)
{ {
$validator = new Validator(); $validator = new Validator();
$validator $validator
@ -316,7 +351,11 @@ class ImporterCommand extends Command
} }
$http = new Client(); $http = new Client();
$this->io->verbose('Downloading file'); $this->io->verbose('Downloading file');
$response = $http->get($url); $httpConfig = [
'headers' => !empty($config['sourceHeaders']) ? $config['sourceHeaders'] : []
];
$query = [];
$response = $http->get($url, $query, $httpConfig);
if ($this->format == 'json') { if ($this->format == 'json') {
return $response->getJson(); return $response->getJson();
} else if ($this->format == 'csv') { } else if ($this->format == 'csv') {
@ -378,6 +417,10 @@ class ImporterCommand extends Command
$this->io->error('Error while parsing the configuration file, mapping missing'); $this->io->error('Error while parsing the configuration file, mapping missing');
die(1); die(1);
} }
if (empty($config['metaTemplateUUID'])) {
$this->io->warning('No `metaTemplateUUID` provided. No meta fields will be created.');
$this->noMetaTemplate = true;
}
if (!empty($config['format'])) { if (!empty($config['format'])) {
$this->format = $config['format']; $this->format = $config['format'];
} }