OpenCloud
/
PeerTube
mirror of https://github.com/Chocobozzz/PeerTube
1
0
Fork 0
Code Issues Releases Wiki Activity

Update migrations code

pull/30/head
Chocobozzz 2016-12-25 09:44:57 +01:00
parent dd6019932e
commit b769007f73
19 changed files with 161 additions and 348 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config/default.yaml
View File

@ -10,6 +10,8 @@ database:
hostname: 'localhost'
port: 5432
suffix: '_dev'
username: peertube
password: peertube
# From the project root directory
storage:

1
server.js
View File

@ -20,6 +20,7 @@ const constants = require('./server/initializers/constants')
const logger = require('./server/helpers/logger')
// Initialize database and models
const db = require('./server/initializers/database')
db.init()
// ----------- Checker -----------
const checker = require('./server/initializers/checker')

2
server/initializers/checker.js
View File

@ -27,7 +27,7 @@ function checkConfig () {
function checkMissedConfig () {
const required = [ 'listen.port',
'webserver.https', 'webserver.hostname', 'webserver.port',
'database.hostname', 'database.port', 'database.suffix',
'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password',
'storage.certs', 'storage.videos', 'storage.logs', 'storage.thumbnails', 'storage.previews'
]
const miss = []

43
server/initializers/constants.js
View File

@ -37,7 +37,9 @@ const CONFIG = {
DATABASE: {
DBNAME: 'peertube' + config.get('database.suffix'),
HOSTNAME: config.get('database.hostname'),
PORT: config.get('database.port')
PORT: config.get('database.port'),
USERNAME: config.get('database.username'),
PASSWORD: config.get('database.password')
},
STORAGE: {
CERT_DIR: path.join(__dirname, '..', '..', config.get('storage.certs')),
@ -87,41 +89,7 @@ const FRIEND_SCORE = {
// ---------------------------------------------------------------------------
const MIGRATION_SCRIPTS = [
{
script: '0005-create-application',
version: 5
},
{
script: '0010-users-password',
version: 10
},
{
script: '0015-admin-role',
version: 15
},
{
script: '0020-requests-endpoint',
version: 20
},
{
script: '0025-video-filenames',
version: 25
},
{
script: '0030-video-magnet',
version: 30
},
{
script: '0035-url-to-host',
version: 35
},
{
script: '0040-video-remote-id',
version: 40
}
]
const LAST_SQL_SCHEMA_VERSION = (maxBy(MIGRATION_SCRIPTS, 'version'))['version']
const LAST_MIGRATION_VERSION = 0
// ---------------------------------------------------------------------------
@ -197,8 +165,7 @@ module.exports = {
CONFIG,
CONSTRAINTS_FIELDS,
FRIEND_SCORE,
LAST_SQL_SCHEMA_VERSION,
MIGRATION_SCRIPTS,
LAST_MIGRATION_VERSION,
OAUTH_LIFETIME,
PAGINATION_COUNT_DEFAULT,
PODS_SCORE,

69
server/initializers/database.js
View File

@ -10,7 +10,11 @@ const utils = require('../helpers/utils')
const database = {}
const sequelize = new Sequelize(constants.CONFIG.DATABASE.DBNAME, 'peertube', 'peertube', {
const dbname = constants.CONFIG.DATABASE.DBNAME
const username = constants.CONFIG.DATABASE.USERNAME
const password = constants.CONFIG.DATABASE.PASSWORD
const sequelize = new Sequelize(dbname, username, password, {
dialect: 'postgres',
host: constants.CONFIG.DATABASE.HOSTNAME,
port: constants.CONFIG.DATABASE.PORT,
@ -26,33 +30,48 @@ const sequelize = new Sequelize(constants.CONFIG.DATABASE.DBNAME, 'peertube', 'p
}
})
const modelDirectory = path.join(__dirname, '..', 'models')
fs.readdir(modelDirectory, function (err, files) {
if (err) throw err
files.filter(function (file) {
if (file === 'utils.js') return false
return true
})
.forEach(function (file) {
const model = sequelize.import(path.join(modelDirectory, file))
database[model.name] = model
})
Object.keys(database).forEach(function (modelName) {
if ('associate' in database[modelName]) {
database[modelName].associate(database)
}
})
logger.info('Database is ready.')
})
database.sequelize = sequelize
database.Sequelize = Sequelize
database.init = init
// ---------------------------------------------------------------------------
module.exports = database
// ---------------------------------------------------------------------------
function init (silent, callback) {
if (!callback) {
callback = silent
silent = false
}
if (!callback) callback = function () {}
const modelDirectory = path.join(__dirname, '..', 'models')
fs.readdir(modelDirectory, function (err, files) {
if (err) throw err
files.filter(function (file) {
// For all models but not utils.js
if (file === 'utils.js') return false
return true
})
.forEach(function (file) {
const model = sequelize.import(path.join(modelDirectory, file))
database[model.name] = model
})
Object.keys(database).forEach(function (modelName) {
if ('associate' in database[modelName]) {
database[modelName].associate(database)
}
})
if (!silent) logger.info('Database is ready.')
return callback(null)
})
}

5
server/initializers/installer.js
View File

@ -121,9 +121,8 @@ function createOAuthAdminIfNotExist (callback) {
logger.info('Username: ' + username)
logger.info('User password: ' + password)
logger.info('Creating Application collection.')
const application = db.Application.build({ sqlSchemaVersion: constants.LAST_SQL_SCHEMA_VERSION })
application.save().asCallback(callback)
logger.info('Creating Application table.')
db.Application.create({ migrationVersion: constants.LAST_MIGRATION_VERSION }).asCallback(callback)
})
})
}

17
server/initializers/migrations/0005-create-application.js
View File

@ -1,17 +0,0 @@
/*
Create the application collection in MongoDB.
Used to store the actual MongoDB scheme version.
*/
const mongoose = require('mongoose')
const Application = mongoose.model('Application')
exports.up = function (callback) {
const application = new Application()
application.save(callback)
}
exports.down = function (callback) {
throw new Error('Not implemented.')
}

14
server/initializers/migrations/0005-example.js Normal file
View File

@ -0,0 +1,14 @@
/*
This is just an example.
*/
const db = require('../database')
// options contains the transaction
exports.up = function (options, callback) {
// db.Application.create({ migrationVersion: 42 }, { transaction: options.transaction }).asCallback(callback)
}
exports.down = function (options, callback) {
throw new Error('Not implemented.')
}

22
server/initializers/migrations/0010-users-password.js
View File

@ -1,22 +0,0 @@
/*
Convert plain user password to encrypted user password.
*/
const eachSeries = require('async/eachSeries')
const mongoose = require('mongoose')
const User = mongoose.model('User')
exports.up = function (callback) {
User.list(function (err, users) {
if (err) return callback(err)
eachSeries(users, function (user, callbackEach) {
user.save(callbackEach)
}, callback)
})
}
exports.down = function (callback) {
throw new Error('Not implemented.')
}

16
server/initializers/migrations/0015-admin-role.js
View File

@ -1,16 +0,0 @@
/*
Set the admin role to the root user.
*/
const constants = require('../constants')
const mongoose = require('mongoose')
const User = mongoose.model('User')
exports.up = function (callback) {
User.update({ username: 'root' }, { role: constants.USER_ROLES.ADMIN }, callback)
}
exports.down = function (callback) {
throw new Error('Not implemented.')
}

15
server/initializers/migrations/0020-requests-endpoint.js
View File

@ -1,15 +0,0 @@
/*
Set the endpoint videos for requests.
*/
const mongoose = require('mongoose')
const Request = mongoose.model('Request')
exports.up = function (callback) {
Request.update({ }, { endpoint: 'videos' }, callback)
}
exports.down = function (callback) {
throw new Error('Not implemented.')
}

57
server/initializers/migrations/0025-video-filenames.js
View File

@ -1,57 +0,0 @@
/*
Rename thumbnails and video filenames to _id.extension
*/
const each = require('async/each')
const fs = require('fs')
const path = require('path')
const mongoose = require('mongoose')
const constants = require('../constants')
const logger = require('../../helpers/logger')
const Video = mongoose.model('Video')
exports.up = function (callback) {
// Use of lean because the new Video scheme does not have filename field
Video.find({ filename: { $ne: null } }).lean().exec(function (err, videos) {
if (err) throw err
each(videos, function (video, callbackEach) {
const torrentName = video.filename + '.torrent'
const thumbnailName = video.thumbnail
const thumbnailExtension = path.extname(thumbnailName)
const videoName = video.filename
const videoExtension = path.extname(videoName)
const newTorrentName = video._id + '.torrent'
const newThumbnailName = video._id + thumbnailExtension
const newVideoName = video._id + videoExtension
const torrentsDir = constants.CONFIG.STORAGE.TORRENTS_DIR
const thumbnailsDir = constants.CONFIG.STORAGE.THUMBNAILS_DIR
const videosDir = constants.CONFIG.STORAGE.VIDEOS_DIR
logger.info('Renaming %s to %s.', torrentsDir + torrentName, torrentsDir + newTorrentName)
fs.renameSync(torrentsDir + torrentName, torrentsDir + newTorrentName)
logger.info('Renaming %s to %s.', thumbnailsDir + thumbnailName, thumbnailsDir + newThumbnailName)
fs.renameSync(thumbnailsDir + thumbnailName, thumbnailsDir + newThumbnailName)
logger.info('Renaming %s to %s.', videosDir + videoName, videosDir + newVideoName)
fs.renameSync(videosDir + videoName, videosDir + newVideoName)
Video.load(video._id, function (err, videoObj) {
if (err) return callbackEach(err)
videoObj.extname = videoExtension
videoObj.remoteId = null
videoObj.save(callbackEach)
})
}, callback)
})
}
exports.down = function (callback) {
throw new Error('Not implemented.')
}

32
server/initializers/migrations/0030-video-magnet.js
View File

@ -1,32 +0,0 @@
/*
Change video magnet structures
*/
const each = require('async/each')
const magnet = require('magnet-uri')
const mongoose = require('mongoose')
const Video = mongoose.model('Video')
exports.up = function (callback) {
// Use of lean because the new Video scheme does not have magnetUri field
Video.find({ }).lean().exec(function (err, videos) {
if (err) throw err
each(videos, function (video, callbackEach) {
const parsed = magnet.decode(video.magnetUri)
const infoHash = parsed.infoHash
Video.load(video._id, function (err, videoObj) {
if (err) return callbackEach(err)
videoObj.magnet.infoHash = infoHash
videoObj.save(callbackEach)
})
}, callback)
})
}
exports.down = function (callback) {
throw new Error('Not implemented.')
}

30
server/initializers/migrations/0035-url-to-host.js
View File

@ -1,30 +0,0 @@
/*
Change video magnet structures
*/
const each = require('async/each')
const mongoose = require('mongoose')
const Video = mongoose.model('Video')
exports.up = function (callback) {
// Use of lean because the new Video scheme does not have podUrl field
Video.find({ }).lean().exec(function (err, videos) {
if (err) throw err
each(videos, function (video, callbackEach) {
Video.load(video._id, function (err, videoObj) {
if (err) return callbackEach(err)
const host = video.podUrl.split('://')[1]
videoObj.podHost = host
videoObj.save(callbackEach)
})
}, callback)
})
}
exports.down = function (callback) {
throw new Error('Not implemented.')
}

59
server/initializers/migrations/0040-video-remote-id.js
View File

@ -1,59 +0,0 @@
/*
Use remote id as identifier
*/
const map = require('lodash/map')
const mongoose = require('mongoose')
const readline = require('readline')
const rl = readline.createInterface({
input: process.stdin,
output: process.stdout
})
const logger = require('../../helpers/logger')
const friends = require('../../lib/friends')
const Pod = mongoose.model('Pod')
const Video = mongoose.model('Video')
exports.up = function (callback) {
Pod.find({}).lean().exec(function (err, pods) {
if (err) return callback(err)
// We need to quit friends first
if (pods.length === 0) {
return setVideosRemoteId(callback)
}
const timeout = setTimeout(function () {
throw new Error('You need to enter a value!')
}, 10000)
rl.question('I am sorry but I need to quit friends for upgrading. Do you want to continue? (yes/*)', function (answer) {
if (answer !== 'yes') throw new Error('I cannot continue.')
clearTimeout(timeout)
rl.close()
const urls = map(pods, 'url')
logger.info('Saying goodbye to: ' + urls.join(', '))
setVideosRemoteId(function () {
friends.quitFriends(callback)
})
})
})
}
exports.down = function (callback) {
throw new Error('Not implemented.')
}
function setVideosRemoteId (callback) {
Video.update({ filename: { $ne: null } }, { remoteId: null }, function (err) {
if (err) throw err
Video.update({ filename: null }, { remoteId: mongoose.Types.ObjectId() }, callback)
})
}

90
server/initializers/migrator.js
View File

@ -1,6 +1,7 @@
'use strict'
const eachSeries = require('async/eachSeries')
const fs = require('fs')
const path = require('path')
const constants = require('./constants')
@ -12,35 +13,24 @@ const migrator = {
}
function migrate (callback) {
db.Application.loadSqlSchemaVersion(function (err, actualVersion) {
db.Application.loadMigrationVersion(function (err, actualVersion) {
if (err) return callback(err)
// If there are a new mongo schemas
if (!actualVersion || actualVersion < constants.LAST_SQL_SCHEMA_VERSION) {
// If there are a new migration scripts
if (actualVersion < constants.LAST_MIGRATION_VERSION) {
logger.info('Begin migrations.')
eachSeries(constants.MONGO_MIGRATION_SCRIPTS, function (entity, callbackEach) {
const versionScript = entity.version
// Do not execute old migration scripts
if (versionScript <= actualVersion) return callbackEach(null)
// Load the migration module and run it
const migrationScriptName = entity.script
logger.info('Executing %s migration script.', migrationScriptName)
const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName))
migrationScript.up(function (err) {
if (err) return callbackEach(err)
// Update the new mongo version schema
db.Application.updateSqlSchemaVersion(versionScript, callbackEach)
})
}, function (err) {
getMigrationScripts(function (err, migrationScripts) {
if (err) return callback(err)
logger.info('Migrations finished. New SQL version schema: %s', constants.LAST_SQL_SCHEMA_VERSION)
return callback(null)
eachSeries(migrationScripts, function (entity, callbackEach) {
executeMigration(actualVersion, entity, callbackEach)
}, function (err) {
if (err) return callback(err)
logger.info('Migrations finished. New migration version schema: %s', constants.LAST_MIGRATION_VERSION)
return callback(null)
})
})
} else {
return callback(null)
@ -52,3 +42,57 @@ function migrate (callback) {
module.exports = migrator
// ---------------------------------------------------------------------------
function getMigrationScripts (callback) {
fs.readdir(path.join(__dirname, 'migrations'), function (err, files) {
if (err) return callback(err)
const filesToMigrate = []
files.forEach(function (file) {
// Filename is something like 'version-blabla.js'
const version = file.split('-')[0]
filesToMigrate.push({
version,
script: file
})
})
return callback(err, filesToMigrate)
})
}
function executeMigration (actualVersion, entity, callback) {
const versionScript = entity.version
// Do not execute old migration scripts
if (versionScript <= actualVersion) return callback(null)
// Load the migration module and run it
const migrationScriptName = entity.script
logger.info('Executing %s migration script.', migrationScriptName)
const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName))
db.sequelize.transaction().asCallback(function (err, t) {
if (err) return callback(err)
migrationScript.up({ transaction: t }, function (err) {
if (err) {
t.rollback()
return callback(err)
}
// Update the new migration version
db.Application.updateMigrationVersion(versionScript, t, function (err) {
if (err) {
t.rollback()
return callback(err)
}
t.commit()
})
})
})
}

26
server/models/application.js
View File

@ -1,15 +1,15 @@
module.exports = function (sequelize, DataTypes) {
const Application = sequelize.define('Application',
{
sqlSchemaVersion: {
migrationVersion: {
type: DataTypes.INTEGER,
defaultValue: 0
}
},
{
classMethods: {
loadSqlSchemaVersion,
updateSqlSchemaVersion
loadMigrationVersion,
updateMigrationVersion
}
}
)
@ -19,18 +19,28 @@ module.exports = function (sequelize, DataTypes) {
// ---------------------------------------------------------------------------
function loadSqlSchemaVersion (callback) {
function loadMigrationVersion (callback) {
const query = {
attributes: [ 'sqlSchemaVersion' ]
attributes: [ 'migrationVersion' ]
}
return this.findOne(query).asCallback(function (err, data) {
const version = data ? data.sqlSchemaVersion : 0
const version = data ? data.migrationVersion : 0
return callback(err, version)
})
}
function updateSqlSchemaVersion (newVersion, callback) {
return this.update({ sqlSchemaVersion: newVersion }).asCallback(callback)
function updateMigrationVersion (newVersion, transaction, callback) {
const options = {
where: {}
}
if (!callback) {
transaction = callback
} else {
options.transaction = transaction
}
return this.update({ migrationVersion: newVersion }, options).asCallback(callback)
}

7
server/models/video.js
View File

@ -16,7 +16,7 @@ const modelUtils = require('./utils')
// ---------------------------------------------------------------------------
module.exports = function (sequelize, DataTypes) {
// TODO: add indexes on searchable columns
// TODO: add indexes on searchable columns
const Video = sequelize.define('Video',
{
id: {
@ -50,6 +50,7 @@ module.exports = function (sequelize, DataTypes) {
generateThumbnailFromBase64,
getDurationFromFile,
list,
listForApi,
listByHostAndRemoteId,
listOwnedAndPopulateAuthorAndTags,
@ -310,6 +311,10 @@ function getDurationFromFile (videoPath, callback) {
})
}
function list (callback) {
return this.find().asCallback()
}
function listForApi (start, count, sort, callback) {
const query = {
offset: start,

2
server/tests/utils/servers.js
View File

@ -103,7 +103,7 @@ function runServer (number, callback) {
if (serverRunString[key] === false) dontContinue = true
}
// If no, there is maybe one thing not already initialized (mongodb...)
// If no, there is maybe one thing not already initialized (client/user credentials generation...)
if (dontContinue === true) return
server.app.stdout.removeListener('data', onStdout)