From 00d6b0dda4b1964ab11127851c0fc7106cc0f912 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 26 Sep 2016 22:36:36 +0200 Subject: [PATCH] Add migration (for db, folders...) mechanism --- server.js | 46 ++++++++------- server/initializers/constants.js | 16 +++++- server/initializers/database.js | 1 + .../migrations/0005-create-application.js | 17 ++++++ .../migrations/0010-users-password.js | 23 ++++++++ server/initializers/migrator.js | 56 +++++++++++++++++++ server/models/application.js | 31 ++++++++++ server/models/user.js | 5 ++ 8 files changed, 174 insertions(+), 21 deletions(-) create mode 100644 server/initializers/migrations/0005-create-application.js create mode 100644 server/initializers/migrations/0010-users-password.js create mode 100644 server/initializers/migrator.js create mode 100644 server/models/application.js diff --git a/server.js b/server.js index 8010d3e06..b8564d05a 100644 --- a/server.js +++ b/server.js @@ -32,6 +32,7 @@ if (miss.length !== 0) { // ----------- PeerTube modules ----------- const customValidators = require('./server/helpers/custom-validators') const installer = require('./server/initializers/installer') +const migrator = require('./server/initializers/migrator') const mongoose = require('mongoose') const routes = require('./server/controllers') const utils = require('./server/helpers/utils') @@ -127,31 +128,36 @@ app.use(function (err, req, res, next) { installer.installApplication(function (err) { if (err) throw err - // Create/activate the webtorrent module - webtorrent.create(function () { - function cleanForExit () { - utils.cleanForExit(webtorrent.app) - } + // Run the migration scripts if needed + migrator.migrate(function (err) { + if (err) throw err - function exitGracefullyOnSignal () { - process.exit(-1) - } + // Create/activate the webtorrent module + webtorrent.create(function () { + function cleanForExit () { + utils.cleanForExit(webtorrent.app) + } - process.on('exit', cleanForExit) - process.on('SIGINT', exitGracefullyOnSignal) - process.on('SIGTERM', exitGracefullyOnSignal) + function exitGracefullyOnSignal () { + process.exit(-1) + } - // ----------- Make the server listening ----------- - server.listen(port, function () { - // Activate the pool requests - Request.activate() + process.on('exit', cleanForExit) + process.on('SIGINT', exitGracefullyOnSignal) + process.on('SIGTERM', exitGracefullyOnSignal) - Video.seedAllExisting(function (err) { - if (err) throw err + // ----------- Make the server listening ----------- + server.listen(port, function () { + // Activate the pool requests + Request.activate() - logger.info('Seeded all the videos') - logger.info('Server listening on port %d', port) - app.emit('ready') + Video.seedAllExisting(function (err) { + if (err) throw err + + logger.info('Seeded all the videos') + logger.info('Server listening on port %d', port) + app.emit('ready') + }) }) }) }) diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 76ebb8681..10ae48e95 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -54,6 +54,18 @@ const FRIEND_SCORE = { MAX: 1000 } +const MONGO_MIGRATION_SCRIPTS = [ + { + script: '0005-create-application', + version: 5 + }, + { + script: '0010-users-password', + version: 10 + } +] +const LAST_MONGO_SCHEMA_VERSION = 10 + // Time to wait between requests to the friends (10 min) let REQUESTS_INTERVAL = 600000 @@ -121,11 +133,13 @@ module.exports = { CONFIG: CONFIG, CONSTRAINTS_FIELDS: CONSTRAINTS_FIELDS, FRIEND_SCORE: FRIEND_SCORE, - REQUESTS_INTERVAL: REQUESTS_INTERVAL, + LAST_MONGO_SCHEMA_VERSION: LAST_MONGO_SCHEMA_VERSION, + MONGO_MIGRATION_SCRIPTS: MONGO_MIGRATION_SCRIPTS, OAUTH_LIFETIME: OAUTH_LIFETIME, PAGINATION_COUNT_DEFAULT: PAGINATION_COUNT_DEFAULT, PODS_SCORE: PODS_SCORE, REQUESTS_IN_PARALLEL: REQUESTS_IN_PARALLEL, + REQUESTS_INTERVAL: REQUESTS_INTERVAL, REQUESTS_LIMIT: REQUESTS_LIMIT, RETRY_REQUESTS: RETRY_REQUESTS, SEARCHABLE_COLUMNS: SEARCHABLE_COLUMNS, diff --git a/server/initializers/database.js b/server/initializers/database.js index 20dcc056e..1da574631 100644 --- a/server/initializers/database.js +++ b/server/initializers/database.js @@ -6,6 +6,7 @@ const constants = require('../initializers/constants') const logger = require('../helpers/logger') // Bootstrap models +require('../models/application') require('../models/user') require('../models/oauth-client') require('../models/oauth-token') diff --git a/server/initializers/migrations/0005-create-application.js b/server/initializers/migrations/0005-create-application.js new file mode 100644 index 000000000..e99dec019 --- /dev/null +++ b/server/initializers/migrations/0005-create-application.js @@ -0,0 +1,17 @@ +/* + Create the application collection in MongoDB. + Used to store the actual MongoDB scheme version. +*/ + +const mongoose = require('mongoose') + +const Application = mongoose.model('Application') + +exports.up = function (callback) { + const application = new Application() + application.save(callback) +} + +exports.down = function (callback) { + throw new Error('Not implemented.') +} diff --git a/server/initializers/migrations/0010-users-password.js b/server/initializers/migrations/0010-users-password.js new file mode 100644 index 000000000..e031fa142 --- /dev/null +++ b/server/initializers/migrations/0010-users-password.js @@ -0,0 +1,23 @@ +/* + Convert plain user password to encrypted user password. +*/ + +const mongoose = require('mongoose') + +const User = mongoose.model('User') + +exports.up = function (callback) { + User.list(function (err, users) { + if (err) return callback(err) + + users.forEach(function (user) { + user.save() + }) + + return callback(null) + }) +} + +exports.down = function (callback) { + throw new Error('Not implemented.') +} diff --git a/server/initializers/migrator.js b/server/initializers/migrator.js new file mode 100644 index 000000000..6b31d994f --- /dev/null +++ b/server/initializers/migrator.js @@ -0,0 +1,56 @@ +'use strict' + +const eachSeries = require('async/eachSeries') +const mongoose = require('mongoose') +const path = require('path') + +const constants = require('./constants') +const logger = require('../helpers/logger') + +const Application = mongoose.model('Application') + +const migrator = { + migrate: migrate +} + +function migrate (callback) { + Application.loadMongoSchemaVersion(function (err, actualVersion) { + if (err) return callback(err) + + // If there are a new mongo schemas + if (!actualVersion || actualVersion < constants.LAST_MONGO_SCHEMA_VERSION) { + logger.info('Begin migrations.') + + eachSeries(constants.MONGO_MIGRATION_SCRIPTS, function (entity, callbackEach) { + const versionScript = entity.version + + // Do not execute old migration scripts + if (versionScript <= actualVersion) return callbackEach(null) + + // Load the migration module and run it + const migrationScriptName = entity.script + logger.info('Executing %s migration script.', migrationScriptName) + + const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName)) + migrationScript.up(function (err) { + if (err) return callbackEach(err) + + // Update the new mongo version schema + Application.updateMongoSchemaVersion(versionScript, callbackEach) + }) + }, function (err) { + if (err) return callback(err) + + logger.info('Migrations finished. New mongo version schema: %s', constants.LAST_MONGO_SCHEMA_VERSION) + return callback(null) + }) + } else { + return callback(null) + } + }) +} + +// --------------------------------------------------------------------------- + +module.exports = migrator + diff --git a/server/models/application.js b/server/models/application.js new file mode 100644 index 000000000..8185f0915 --- /dev/null +++ b/server/models/application.js @@ -0,0 +1,31 @@ +const mongoose = require('mongoose') + +// --------------------------------------------------------------------------- + +const ApplicationSchema = mongoose.Schema({ + mongoSchemaVersion: { + type: Number, + default: 0 + } +}) + +ApplicationSchema.statics = { + loadMongoSchemaVersion: loadMongoSchemaVersion, + updateMongoSchemaVersion: updateMongoSchemaVersion +} + +mongoose.model('Application', ApplicationSchema) + +// --------------------------------------------------------------------------- + +function loadMongoSchemaVersion (callback) { + return this.findOne({}, { mongoSchemaVersion: 1 }, function (err, data) { + const version = data ? data.mongoSchemaVersion : 0 + + return callback(err, version) + }) +} + +function updateMongoSchemaVersion (newVersion, callback) { + return this.update({}, { mongoSchemaVersion: newVersion }, callback) +} diff --git a/server/models/user.js b/server/models/user.js index db6f1765b..c2c8807f0 100644 --- a/server/models/user.js +++ b/server/models/user.js @@ -28,6 +28,7 @@ UserSchema.methods = { UserSchema.statics = { countTotal: countTotal, getByUsername: getByUsername, + list: list, listForApi: listForApi, loadById: loadById, loadByUsername: loadByUsername @@ -71,6 +72,10 @@ function getByUsername (username) { return this.findOne({ username: username }) } +function list (callback) { + return this.find(callback) +} + function listForApi (start, count, sort, callback) { const query = {} return modelUtils.listForApiWithCount.call(this, query, start, count, sort, callback)