From 355defee993f3a8bd138f1bed2f81087c09bf13f Mon Sep 17 00:00:00 2001 From: mokaddem Date: Fri, 27 Nov 2020 12:32:38 +0100 Subject: [PATCH 01/10] new: [command] First version of generic importer - WiP --- src/Command/ImporterCommand.php | 262 ++++++++++++++++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100644 src/Command/ImporterCommand.php diff --git a/src/Command/ImporterCommand.php b/src/Command/ImporterCommand.php new file mode 100644 index 0000000..b1e48aa --- /dev/null +++ b/src/Command/ImporterCommand.php @@ -0,0 +1,262 @@ +setDescription('Import data based on the provided configuration file.'); + $parser->addArgument('config', [ + 'help' => 'Configuration file path for the importer', + 'required' => true + ]); + $parser->addArgument('source', [ + 'help' => 'The source that should be imported. Can be either a file on the disk or an valid URL.', + 'required' => true + ]); + $parser->addArgument('primary_key', [ + 'help' => 'To avoid duplicates, entries having the value specified by the primary key will be updated instead of inserted. Empty if only insertion should be done', + 'required' => false, + 'default' => null, + ]); + return $parser; + } + + public function execute(Arguments $args, ConsoleIo $io) + { + $this->io = $io; + $configPath = $args->getArgument('config'); + $source = $args->getArgument('source'); + $primary_key = $args->getArgument('primary_key'); + + $table = $this->modelClass; + $this->loadModel($table); + // $orgs = $this->{$table}->find()->enableHydration(false)->toList(); + $config = $this->getConfigFromFile($configPath); + $sourceData = $this->getDataFromSource($source); + // $sourceData = ['data' => [['name' => 'Test2', 'uuid' => '28de34ac-e284-495c-ae0e-9f46dd12da35', 'sector' => 'ss']]]; + $data = $this->extractData($this->{$table}, $config, $sourceData); + $entities = $this->marshalData($this->{$table}, $data, $primary_key); + $entitiesSample = array_slice($entities, 0, min(10, count($entities))); + $ioTable = $this->transformEntitiesIntoTable($entitiesSample); + $io->helper('Table')->output($ioTable); + $selection = $io->askChoice('A sample of the data you are about to save is provided above. Would you like to proceed?', ['Y', 'N'], 'N'); + // debug(json_encode($entities, JSON_PRETTY_PRINT)); + if ($selection == 'Y') { + $this->saveData($this->{$table}, $entities); + } + } + + private function marshalData($table, $data, $primary_key=null) + { + $entities = []; + if (is_null($primary_key)) { + $entities = $table->newEntities($data); + } else { + foreach ($data as $i => $item) { + $query = $table->find('all') + ->where(["${primary_key}" => $item[$primary_key]]); + $entity = $query->first(); + if ($entity) { + $entity->setAccess('uuid', false); + } else { + $entity = $table->newEmptyEntity(); + } + $entity = $table->patchEntity($entity, $item); + $entities[] = $entity; + } + } + $hasErrors = false; + foreach ($entities as $entity) { + if ($entity->hasErrors()) { + $hasErrors = true; + $this->io->error(json_encode(['entity' => $entity, 'errors' => $entity->getErrors()], JSON_PRETTY_PRINT)); + } + } + if (!$hasErrors) { + $this->io->verbose('No validation errors'); + } + return $entities; + } + + private function saveData($table, $entities) + { + $this->io->verbose('Saving data'); + $result = $table->saveMany($entities); + if ($result == false) { + $this->io->error('Error while saving data'); + } + // take care of meta fields + } + + private function extractData($table, $config, $source) + { + $defaultFields = array_flip($table->getSchema()->columns()); + $this->io->verbose('Extracting data'); + $data = []; + foreach ($config as $key => $fieldConfig) { + $values = null; + if (!empty($fieldConfig['path'])) { + $values = Hash::extract($source, $fieldConfig['path']); + } + if (!empty($fieldConfig['massage'])) { + $values = array_map("self::{$fieldConfig['massage']}", $values); + } + if (isset($defaultFields[$key])) { + $data[$key] = $values; + } else { + $data['metaFields'][$key] = $values; + } + } + return $this->invertArray($data); + } + + private function getDataFromSource($source) + { + $data = $this->getDataFromFile($source); + if ($data === false) { + $data = $this->getDataFromURL($source); + } + return $data; + } + + private function getDataFromURL($url) + { + $validator = new Validator(); + $validator + ->requirePresence('url') + ->notEmptyString('url', 'Please provide a valid source') + ->url('url'); + $errors = $validator->validate(['url' => $url]); + if (!empty($errors)) { + $this->io->error(json_encode(Hash::extract($errors, '{s}'), JSON_PRETTY_PRINT)); + die(1); + } + $http = new Client(); + $this->io->verbose('Downloading file'); + $response = $http->get($url); + return $response->getJson(); + } + + private function getDataFromFile($path) + { + $file = new File($path); + if ($file->exists()) { + $this->io->verbose('Reading file'); + $data = $file->read(); + $file->close(); + if (!empty($data)) { + $data = json_decode($data, true); + if (is_null($data)) { + $this->io->error('Error while parsing the source file'); + die(1); + } + return $data; + } + } + return false; + } + + private function getConfigFromFile($configPath) + { + $file = new File($configPath); + if ($file->exists()) { + $config = $file->read(); + $file->close(); + if (!empty($config)) { + $config = json_decode($config, true); + if (is_null($config)) { + $this->io->error('Error while parsing the configuration file'); + die(1); + } + return $config; + } else { + $this->io->error('Configuration file cound not be read'); + } + } else { + $this->io->error('Configuration file not found'); + } + } + + private function transformResultSetsIntoTable($result, $header=[]) + { + $table = [[]]; + if (!empty($result)) { + $tableHeader = empty($header) ? array_keys($result[0]) : $header; + $tableContent = []; + foreach ($result as $item) { + if (empty($header)) { + $tableContent[] = array_map('strval', array_values($item)); + } else { + $row = []; + foreach ($tableHeader as $key) { + $row[] = (string) $item[$key]; + } + $tableContent[] = $row; + } + } + $table = array_merge([$tableHeader], $tableContent); + } + return $table; + } + + private function transformEntitiesIntoTable($entities, $header=[]) + { + $table = [[]]; + if (!empty($entities)) { + // $tableHeader = empty($header) ? array_keys($entities[0]->toArray()) : $header; + $tableHeader = empty($header) ? array_keys(Hash::flatten($entities[0]->toArray())) : $header; + $tableContent = []; + foreach ($entities as $entity) { + $row = []; + foreach ($tableHeader as $key) { + $subKeys = explode('.', $key); + if (in_array('metaFields', $subKeys)) { + $row[] = (string) Hash::get($entity, $key); + } else { + $row[] = (string) $entity[$key]; + } + } + $tableContent[] = $row; + } + $table = array_merge([$tableHeader], $tableContent); + } + return $table; + } + + private function invertArray($data) + { + $inverted = []; + foreach ($data as $key => $values) { + if ($key == 'metaFields') { + foreach ($values as $metaKey => $metaValues) { + foreach ($metaValues as $i => $metaValue) { + $inverted[$i]['metaFields'][$metaKey] = $metaValue; + } + } + } else { + foreach ($values as $i => $value) { + $inverted[$i][$key] = $value; + } + } + } + return $inverted; + } + + private function genUUID($value) + { + return Text::uuid(); + } +} \ No newline at end of file From 0c14d30ae72a10fffc7520729b21718846983c17 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Fri, 27 Nov 2020 16:22:54 +0100 Subject: [PATCH 02/10] chg: [command] Added support of meta-field in generic importer --- src/Command/ImporterCommand.php | 44 +++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/src/Command/ImporterCommand.php b/src/Command/ImporterCommand.php index b1e48aa..2e8d563 100644 --- a/src/Command/ImporterCommand.php +++ b/src/Command/ImporterCommand.php @@ -53,7 +53,6 @@ class ImporterCommand extends Command $ioTable = $this->transformEntitiesIntoTable($entitiesSample); $io->helper('Table')->output($ioTable); $selection = $io->askChoice('A sample of the data you are about to save is provided above. Would you like to proceed?', ['Y', 'N'], 'N'); - // debug(json_encode($entities, JSON_PRETTY_PRINT)); if ($selection == 'Y') { $this->saveData($this->{$table}, $entities); } @@ -79,7 +78,7 @@ class ImporterCommand extends Command } } $hasErrors = false; - foreach ($entities as $entity) { + foreach ($entities as $i => $entity) { if ($entity->hasErrors()) { $hasErrors = true; $this->io->error(json_encode(['entity' => $entity, 'errors' => $entity->getErrors()], JSON_PRETTY_PRINT)); @@ -93,12 +92,38 @@ class ImporterCommand extends Command private function saveData($table, $entities) { + $this->loadModel('MetaFields'); $this->io->verbose('Saving data'); - $result = $table->saveMany($entities); - if ($result == false) { + $entities = $table->saveMany($entities); + if ($entities === false) { $this->io->error('Error while saving data'); } - // take care of meta fields + $this->io->verbose('Saving meta fields'); + $errorWhileSaving = 0; + foreach ($entities as $i => $entity) { + foreach ($entity['metaFields'] as $fieldName => $fieldValue) { + $query = $this->MetaFields->find('all')->where([ + 'parent_id' => $entity->id, + 'field' => $fieldName + ]); + $metaEntity = $query->first(); + if (is_null($metaEntity)) { + $metaEntity = $this->MetaFields->newEmptyEntity(); + } + $metaEntity->field = $fieldName; + $metaEntity->value = $fieldValue; + $metaEntity->scope = $table->metaFields; + $metaEntity->parent_id = $entity->id; + $metaEntity = $this->MetaFields->save($metaEntity); + if ($metaEntity === false) { + $errorWhileSaving++; + $this->io->verbose('Error while saving metafield: ' . PHP_EOL . json_encode($metaEntity, JSON_PRETTY_PRINT)); + } + } + if ($errorWhileSaving) { + $this->io->error('Error while saving meta data: ' . (string) $errorWhileSaving); + } + } } private function extractData($table, $config, $source) @@ -216,15 +241,20 @@ class ImporterCommand extends Command { $table = [[]]; if (!empty($entities)) { - // $tableHeader = empty($header) ? array_keys($entities[0]->toArray()) : $header; $tableHeader = empty($header) ? array_keys(Hash::flatten($entities[0]->toArray())) : $header; + $tableHeader = array_filter($tableHeader, function($name) { + return !in_array('metaFields', explode('.', $name)); + }); + foreach ($entities[0]['metaFields'] as $metaField => $metaValue) { + $tableHeader[] = "metaFields.$metaField"; + } $tableContent = []; foreach ($entities as $entity) { $row = []; foreach ($tableHeader as $key) { $subKeys = explode('.', $key); if (in_array('metaFields', $subKeys)) { - $row[] = (string) Hash::get($entity, $key); + $row[] = (string) $entity['metaFields'][$subKeys[1]]; } else { $row[] = (string) $entity[$key]; } From 35ba595a82372fb1dc0ac7faec9e993fc299d5c4 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Fri, 27 Nov 2020 16:48:14 +0100 Subject: [PATCH 03/10] chg: [command] Generic import tool allow passing path directly --- src/Command/ImporterCommand.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Command/ImporterCommand.php b/src/Command/ImporterCommand.php index 2e8d563..b68f858 100644 --- a/src/Command/ImporterCommand.php +++ b/src/Command/ImporterCommand.php @@ -133,6 +133,9 @@ class ImporterCommand extends Command $data = []; foreach ($config as $key => $fieldConfig) { $values = null; + if (!is_array($fieldConfig)) { + $fieldConfig = ['path' => $fieldConfig]; + } if (!empty($fieldConfig['path'])) { $values = Hash::extract($source, $fieldConfig['path']); } From fec6f817f688b9377437fc274b262614b34aef53 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Mon, 30 Nov 2020 13:47:48 +0100 Subject: [PATCH 04/10] chg: [command] Generic importer improved and added support of value override --- src/Command/ImporterCommand.php | 120 +++++++++++++++++++++++--------- 1 file changed, 87 insertions(+), 33 deletions(-) diff --git a/src/Command/ImporterCommand.php b/src/Command/ImporterCommand.php index b68f858..6d6d75e 100644 --- a/src/Command/ImporterCommand.php +++ b/src/Command/ImporterCommand.php @@ -14,6 +14,7 @@ use Cake\Http\Client; class ImporterCommand extends Command { protected $modelClass = 'Organisations'; + private $fieldsNoOverride = []; protected function buildOptionParser(ConsoleOptionParser $parser): ConsoleOptionParser { @@ -27,7 +28,7 @@ class ImporterCommand extends Command 'required' => true ]); $parser->addArgument('primary_key', [ - 'help' => 'To avoid duplicates, entries having the value specified by the primary key will be updated instead of inserted. Empty if only insertion should be done', + 'help' => 'To avoid duplicates, entries having the value specified by the primary key will be updated instead of inserted. Leave empty if only insertion should be done', 'required' => false, 'default' => null, ]); @@ -43,23 +44,25 @@ class ImporterCommand extends Command $table = $this->modelClass; $this->loadModel($table); - // $orgs = $this->{$table}->find()->enableHydration(false)->toList(); $config = $this->getConfigFromFile($configPath); + $this->processConfig($config); $sourceData = $this->getDataFromSource($source); - // $sourceData = ['data' => [['name' => 'Test2', 'uuid' => '28de34ac-e284-495c-ae0e-9f46dd12da35', 'sector' => 'ss']]]; $data = $this->extractData($this->{$table}, $config, $sourceData); - $entities = $this->marshalData($this->{$table}, $data, $primary_key); + $entities = $this->marshalData($this->{$table}, $data, $config, $primary_key); + $entitiesSample = array_slice($entities, 0, min(10, count($entities))); $ioTable = $this->transformEntitiesIntoTable($entitiesSample); $io->helper('Table')->output($ioTable); + $selection = $io->askChoice('A sample of the data you are about to save is provided above. Would you like to proceed?', ['Y', 'N'], 'N'); if ($selection == 'Y') { $this->saveData($this->{$table}, $entities); } } - private function marshalData($table, $data, $primary_key=null) + private function marshalData($table, $data, $config, $primary_key=null) { + $this->loadModel('MetaFields'); $entities = []; if (is_null($primary_key)) { $entities = $table->newEntities($data); @@ -68,10 +71,10 @@ class ImporterCommand extends Command $query = $table->find('all') ->where(["${primary_key}" => $item[$primary_key]]); $entity = $query->first(); - if ($entity) { - $entity->setAccess('uuid', false); - } else { + if (is_null($entity)) { $entity = $table->newEmptyEntity(); + } else { + $this->lockAccess($config, $entity); } $entity = $table->patchEntity($entity, $item); $entities[] = $entity; @@ -82,6 +85,29 @@ class ImporterCommand extends Command if ($entity->hasErrors()) { $hasErrors = true; $this->io->error(json_encode(['entity' => $entity, 'errors' => $entity->getErrors()], JSON_PRETTY_PRINT)); + } else { + $metaFields = []; + foreach ($entity['metaFields'] as $fieldName => $fieldValue) { + $metaEntity = null; + if (!$entity->isNew()) { + $query = $this->MetaFields->find('all')->where([ + 'parent_id' => $entity->id, + 'field' => $fieldName + ]); + $metaEntity = $query->first(); + } + if (is_null($metaEntity)) { + $metaEntity = $this->MetaFields->newEmptyEntity(); + $metaEntity->field = $fieldName; + $metaEntity->scope = $table->metaFields; + $metaEntity->parent_id = $entity->id; + } + if ($this->canBeOverriden($config, $metaEntity)) { + $metaEntity->value = $fieldValue; + } + $metaFields[] = $metaEntity; + } + $entities[$i]->metaFields = $metaFields; } } if (!$hasErrors) { @@ -99,31 +125,25 @@ class ImporterCommand extends Command $this->io->error('Error while saving data'); } $this->io->verbose('Saving meta fields'); - $errorWhileSaving = 0; foreach ($entities as $i => $entity) { - foreach ($entity['metaFields'] as $fieldName => $fieldValue) { - $query = $this->MetaFields->find('all')->where([ - 'parent_id' => $entity->id, - 'field' => $fieldName - ]); - $metaEntity = $query->first(); - if (is_null($metaEntity)) { - $metaEntity = $this->MetaFields->newEmptyEntity(); - } - $metaEntity->field = $fieldName; - $metaEntity->value = $fieldValue; - $metaEntity->scope = $table->metaFields; - $metaEntity->parent_id = $entity->id; - $metaEntity = $this->MetaFields->save($metaEntity); - if ($metaEntity === false) { - $errorWhileSaving++; - $this->io->verbose('Error while saving metafield: ' . PHP_EOL . json_encode($metaEntity, JSON_PRETTY_PRINT)); - } - } - if ($errorWhileSaving) { - $this->io->error('Error while saving meta data: ' . (string) $errorWhileSaving); + $this->saveMetaFields($entity); + } + } + + private function saveMetaFields($entity) + { + $errorWhileSaving = 0; + foreach ($entity->metaFields as $metaEntity) { + $metaEntity->parent_id = $entity->id; + $metaEntity = $this->MetaFields->save($metaEntity); + if ($metaEntity === false) { + $errorWhileSaving++; + $this->io->verbose('Error while saving metafield: ' . PHP_EOL . json_encode($metaEntity, JSON_PRETTY_PRINT)); } } + if ($errorWhileSaving) { + $this->io->error('Error while saving meta data: ' . (string) $errorWhileSaving); + } } private function extractData($table, $config, $source) @@ -151,6 +171,18 @@ class ImporterCommand extends Command return $this->invertArray($data); } + private function lockAccess($config, &$entity) + { + foreach ($this->fieldsNoOverride as $fieldName) { + $entity->setAccess($fieldName, false); + } + } + + private function canBeOverriden($config, $metaEntity) + { + return !in_array($metaEntity->field, $this->fieldsNoOverride); + } + private function getDataFromSource($source) { $data = $this->getDataFromFile($source); @@ -218,6 +250,18 @@ class ImporterCommand extends Command } } + private function processConfig($config) + { + $this->fieldsNoOverride = []; + foreach ($config as $fieldName => $fieldConfig) { + if (is_array($fieldConfig)) { + if (isset($fieldConfig['override']) && $fieldConfig['override'] === false) { + $this->fieldsNoOverride[] = $fieldName; + } + } + } + } + private function transformResultSetsIntoTable($result, $header=[]) { $table = [[]]; @@ -248,8 +292,8 @@ class ImporterCommand extends Command $tableHeader = array_filter($tableHeader, function($name) { return !in_array('metaFields', explode('.', $name)); }); - foreach ($entities[0]['metaFields'] as $metaField => $metaValue) { - $tableHeader[] = "metaFields.$metaField"; + foreach ($entities[0]->metaFields as $metaField) { + $tableHeader[] = "metaFields.$metaField->field"; } $tableContent = []; foreach ($entities as $entity) { @@ -257,7 +301,17 @@ class ImporterCommand extends Command foreach ($tableHeader as $key) { $subKeys = explode('.', $key); if (in_array('metaFields', $subKeys)) { - $row[] = (string) $entity['metaFields'][$subKeys[1]]; + $found = false; + foreach ($entity->metaFields as $metaField) { + if ($metaField->field == $subKeys[1]) { + $row[] = (string) $metaField->value; + $found = true; + break; + } + } + if (!$found) { + $row[] = ''; + } } else { $row[] = (string) $entity[$key]; } From 7b393e58aacb8853d482143d75402d00377e87fd Mon Sep 17 00:00:00 2001 From: mokaddem Date: Mon, 30 Nov 2020 13:54:36 +0100 Subject: [PATCH 05/10] fix: [organisations] Make sure to cascade deletion calls --- src/Model/Table/OrganisationsTable.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Model/Table/OrganisationsTable.php b/src/Model/Table/OrganisationsTable.php index 146d715..52f08ac 100644 --- a/src/Model/Table/OrganisationsTable.php +++ b/src/Model/Table/OrganisationsTable.php @@ -23,6 +23,7 @@ class OrganisationsTable extends AppTable $this->hasMany( 'EncryptionKeys', [ + 'dependent' => true, 'foreignKey' => 'owner_id', 'conditions' => ['owner_type' => 'organisation'] ] @@ -30,6 +31,7 @@ class OrganisationsTable extends AppTable $this->hasMany( 'MetaFields', [ + 'dependent' => true, 'foreignKey' => 'parent_id', 'conditions' => ['scope' => 'organisation'] ] From 566d93b67c999663fc9d96c98a506fa3a193941e Mon Sep 17 00:00:00 2001 From: mokaddem Date: Mon, 30 Nov 2020 14:56:12 +0100 Subject: [PATCH 06/10] chg: [command] Added support of CSV and documentation --- src/Command/ImporterCommand.php | 116 +++++++++++++++++++++++++++----- 1 file changed, 100 insertions(+), 16 deletions(-) diff --git a/src/Command/ImporterCommand.php b/src/Command/ImporterCommand.php index 6d6d75e..11c2c67 100644 --- a/src/Command/ImporterCommand.php +++ b/src/Command/ImporterCommand.php @@ -1,4 +1,30 @@ setDescription('Import data based on the provided configuration file.'); $parser->addArgument('config', [ - 'help' => 'Configuration file path for the importer', + 'help' => 'JSON configuration file path for the importer.', 'required' => true ]); $parser->addArgument('source', [ @@ -74,7 +101,7 @@ class ImporterCommand extends Command if (is_null($entity)) { $entity = $table->newEmptyEntity(); } else { - $this->lockAccess($config, $entity); + $this->lockAccess($entity); } $entity = $table->patchEntity($entity, $item); $entities[] = $entity; @@ -102,7 +129,7 @@ class ImporterCommand extends Command $metaEntity->scope = $table->metaFields; $metaEntity->parent_id = $entity->id; } - if ($this->canBeOverriden($config, $metaEntity)) { + if ($this->canBeOverriden($metaEntity)) { $metaEntity->value = $fieldValue; } $metaFields[] = $metaEntity; @@ -148,10 +175,23 @@ class ImporterCommand extends Command private function extractData($table, $config, $source) { - $defaultFields = array_flip($table->getSchema()->columns()); $this->io->verbose('Extracting data'); + $defaultFields = array_flip($table->getSchema()->columns()); + if ($this->format == 'json') { + $data = $this->extractDataFromJSON($defaultFields, $config, $source); + } else if ($this->format == 'csv') { + $data = $this->extractDataFromCSV($defaultFields, $config, $source); + } else { + $this->io->error('Cannot extract data: Invalid file format'); + die(1); + } + return $data; + } + + private function extractDataFromJSON($defaultFields, $config, $source) + { $data = []; - foreach ($config as $key => $fieldConfig) { + foreach ($config['mapping'] as $key => $fieldConfig) { $values = null; if (!is_array($fieldConfig)) { $fieldConfig = ['path' => $fieldConfig]; @@ -171,14 +211,37 @@ class ImporterCommand extends Command return $this->invertArray($data); } - private function lockAccess($config, &$entity) + private function extractDataFromCSV($defaultFields, $config, $source) + { + $rows = array_map('str_getcsv', explode(PHP_EOL, $source)); + if (count($rows[0]) != count($rows[1])) { + $this->io->error('Error while parsing source data. CSV doesn\'t have the same number of columns'); + die(1); + } + $header = array_shift($rows); + $data = array(); + foreach($rows as $row) { + $dataRow = []; + foreach ($header as $i => $headerField) { + if (isset($defaultFields[$headerField])) { + $dataRow[$headerField] = $row[$i]; + } else { + $dataRow['metaFields'][$headerField] = $row[$i]; + } + } + $data[] = $dataRow; + } + return $data; + } + + private function lockAccess(&$entity) { foreach ($this->fieldsNoOverride as $fieldName) { $entity->setAccess($fieldName, false); } } - private function canBeOverriden($config, $metaEntity) + private function canBeOverriden($metaEntity) { return !in_array($metaEntity->field, $this->fieldsNoOverride); } @@ -207,7 +270,13 @@ class ImporterCommand extends Command $http = new Client(); $this->io->verbose('Downloading file'); $response = $http->get($url); - return $response->getJson(); + if ($this->format == 'json') { + return $response->getJson(); + } else if ($this->format == 'csv') { + return $response->getStringBody(); + } else { + $this->io->error('Cannot parse source data: Invalid file format'); + } } private function getDataFromFile($path) @@ -218,12 +287,18 @@ class ImporterCommand extends Command $data = $file->read(); $file->close(); if (!empty($data)) { - $data = json_decode($data, true); - if (is_null($data)) { - $this->io->error('Error while parsing the source file'); - die(1); + if ($this->format == 'json') { + $data = json_decode($data, true); + if (is_null($data)) { + $this->io->error('Error while parsing the source file'); + die(1); + } + return $data; + } else if ($this->format == 'csv') { + return $data; + } else { + $this->io->error('Cannot parse source data: Invalid file format'); } - return $data; } } return false; @@ -252,8 +327,15 @@ class ImporterCommand extends Command private function processConfig($config) { + if (empty($config['mapping'])) { + $this->io->error('Error while parsing the configuration file, mapping missing'); + die(1); + } + if (!empty($config['format'])) { + $this->format = $config['format']; + } $this->fieldsNoOverride = []; - foreach ($config as $fieldName => $fieldConfig) { + foreach ($config['mapping'] as $fieldName => $fieldConfig) { if (is_array($fieldConfig)) { if (isset($fieldConfig['override']) && $fieldConfig['override'] === false) { $this->fieldsNoOverride[] = $fieldName; @@ -292,8 +374,10 @@ class ImporterCommand extends Command $tableHeader = array_filter($tableHeader, function($name) { return !in_array('metaFields', explode('.', $name)); }); - foreach ($entities[0]->metaFields as $metaField) { - $tableHeader[] = "metaFields.$metaField->field"; + if (!empty($entities[0]->metaFields)) { + foreach ($entities[0]->metaFields as $metaField) { + $tableHeader[] = "metaFields.$metaField->field"; + } } $tableContent = []; foreach ($entities as $entity) { From a67a5d118d57ce573428a61c97c8ff0deeebb097 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Mon, 30 Nov 2020 15:36:13 +0100 Subject: [PATCH 07/10] chg: [command] Generic importer added more validation and progress bar --- src/Command/ImporterCommand.php | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/Command/ImporterCommand.php b/src/Command/ImporterCommand.php index 11c2c67..b6aa5b2 100644 --- a/src/Command/ImporterCommand.php +++ b/src/Command/ImporterCommand.php @@ -95,9 +95,12 @@ class ImporterCommand extends Command $entities = $table->newEntities($data); } else { foreach ($data as $i => $item) { - $query = $table->find('all') - ->where(["${primary_key}" => $item[$primary_key]]); - $entity = $query->first(); + $entity = null; + if (isset($item[$primary_key])) { + $query = $table->find('all') + ->where(["${primary_key}" => $item[$primary_key]]); + $entity = $query->first(); + } if (is_null($entity)) { $entity = $table->newEmptyEntity(); } else { @@ -139,6 +142,9 @@ class ImporterCommand extends Command } if (!$hasErrors) { $this->io->verbose('No validation errors'); + } else { + $this->io->error('Validation errors, please fix before importing'); + die(1); } return $entities; } @@ -147,13 +153,20 @@ class ImporterCommand extends Command { $this->loadModel('MetaFields'); $this->io->verbose('Saving data'); + $progress = $this->io->helper('Progress'); + $entities = $table->saveMany($entities); if ($entities === false) { $this->io->error('Error while saving data'); } $this->io->verbose('Saving meta fields'); + $progress->init([ + 'total' => count($entities) + ]); foreach ($entities as $i => $entity) { $this->saveMetaFields($entity); + $progress->increment(1); + $progress->draw(); } } @@ -162,6 +175,9 @@ class ImporterCommand extends Command $errorWhileSaving = 0; foreach ($entity->metaFields as $metaEntity) { $metaEntity->parent_id = $entity->id; + if ($metaEntity->hasErrors() || is_null($metaEntity->value)) { + continue; + } $metaEntity = $this->MetaFields->save($metaEntity); if ($metaEntity === false) { $errorWhileSaving++; From 3214787364aa5f9650d3d84c592ad1ecaef5c7ce Mon Sep 17 00:00:00 2001 From: mokaddem Date: Mon, 30 Nov 2020 15:41:36 +0100 Subject: [PATCH 08/10] chg: [command] Generic importer usage of saveMany and improved TUI --- src/Command/ImporterCommand.php | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/Command/ImporterCommand.php b/src/Command/ImporterCommand.php index b6aa5b2..5c902f8 100644 --- a/src/Command/ImporterCommand.php +++ b/src/Command/ImporterCommand.php @@ -160,32 +160,30 @@ class ImporterCommand extends Command $this->io->error('Error while saving data'); } $this->io->verbose('Saving meta fields'); + $this->io->out(''); $progress->init([ - 'total' => count($entities) + 'total' => count($entities), + 'length' => 20 ]); foreach ($entities as $i => $entity) { $this->saveMetaFields($entity); $progress->increment(1); $progress->draw(); } + $this->io->out(''); } private function saveMetaFields($entity) { - $errorWhileSaving = 0; - foreach ($entity->metaFields as $metaEntity) { + foreach ($entity->metaFields as $i => $metaEntity) { $metaEntity->parent_id = $entity->id; if ($metaEntity->hasErrors() || is_null($metaEntity->value)) { - continue; - } - $metaEntity = $this->MetaFields->save($metaEntity); - if ($metaEntity === false) { - $errorWhileSaving++; - $this->io->verbose('Error while saving metafield: ' . PHP_EOL . json_encode($metaEntity, JSON_PRETTY_PRINT)); + unset($entity->metaFields[$i]); } } - if ($errorWhileSaving) { - $this->io->error('Error while saving meta data: ' . (string) $errorWhileSaving); + $entity->metaFields = $this->MetaFields->saveMany($entity->metaFields); + if ($entity->metaFields === false) { + $this->io->error('Error while saving meta data'); } } From da1be4be0f2008cf17ae49165e90ae60c0169eb7 Mon Sep 17 00:00:00 2001 From: mokaddem Date: Mon, 30 Nov 2020 15:59:01 +0100 Subject: [PATCH 09/10] fix: [command] Generic importer correct usage of options --- src/Command/ImporterCommand.php | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/Command/ImporterCommand.php b/src/Command/ImporterCommand.php index 5c902f8..500e307 100644 --- a/src/Command/ImporterCommand.php +++ b/src/Command/ImporterCommand.php @@ -54,11 +54,17 @@ class ImporterCommand extends Command 'help' => 'The source that should be imported. Can be either a file on the disk or an valid URL.', 'required' => true ]); - $parser->addArgument('primary_key', [ + $parser->addOption('primary_key', [ + 'short' => 'p', 'help' => 'To avoid duplicates, entries having the value specified by the primary key will be updated instead of inserted. Leave empty if only insertion should be done', - 'required' => false, 'default' => null, ]); + $parser->addOption('model_class', [ + 'short' => 'm', + 'help' => 'The target cerebrate model for the import', + 'default' => 'Organisations', + 'choices' => ['Organisations', 'Individuals', 'AuthKeys'] + ]); return $parser; } @@ -67,7 +73,11 @@ class ImporterCommand extends Command $this->io = $io; $configPath = $args->getArgument('config'); $source = $args->getArgument('source'); - $primary_key = $args->getArgument('primary_key'); + $primary_key = $args->getOption('primary_key'); + $model_class = $args->getOption('model_class'); + if (!is_null($model_class)) { + $this->modelClass = $model_class; + } $table = $this->modelClass; $this->loadModel($table); From 1db86113fc86e4dbf844701ad8b4292d1d3d995a Mon Sep 17 00:00:00 2001 From: mokaddem Date: Mon, 30 Nov 2020 16:03:58 +0100 Subject: [PATCH 10/10] chg: [command] Added configuration file for enisa-csirts --- src/Command/config/config-enisa-csirts.json | 25 +++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 src/Command/config/config-enisa-csirts.json diff --git a/src/Command/config/config-enisa-csirts.json b/src/Command/config/config-enisa-csirts.json new file mode 100644 index 0000000..4e887e6 --- /dev/null +++ b/src/Command/config/config-enisa-csirts.json @@ -0,0 +1,25 @@ +{ + "format": "json", + "mapping": { + "name": "data.{n}.short-team-name", + "uuid": { + "path": "data.{n}.team-name", + "override": false, + "massage": "genUUID" + }, + "url": "data.{n}.website", + "nationality": "data.{n}.country-name", + "membership_type": "data.{n}.first-member-type", + "email": "data.{n}.email", + "country": "data.{n}.country-name", + "official_name": "data.{n}.team-name", + "established": "data.{n}.establishment", + "website": "data.{n}.website", + "constituency": "data.{n}.constituency", + "is_approved": "data.{n}.is_approved", + "enisa_geo_group": "data.{n}.enisa-geo-group", + "oes_coverage": "data.{n}.oes-coverage", + "enisa-tistatus": "data.{n}.enisa-tistatus", + "csirt_network_status": "data.{n}.csirt-network-status" + } +} \ No newline at end of file