From 052937db8a8d282eccdbdf38d487ed8d85d3c0a7 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 26 Jul 2016 22:30:46 +0200 Subject: [PATCH] First draft using only webseed for server --- .gitignore | 1 + client/package.json | 2 +- config/default.yaml | 1 + config/test-1.yaml | 1 + config/test-2.yaml | 1 + config/test-3.yaml | 1 + config/test-4.yaml | 1 + config/test-5.yaml | 1 + config/test-6.yaml | 1 + package.json | 6 +- server.js | 45 +++------ server/initializers/constants.js | 12 ++- server/lib/webtorrent-process.js | 92 ------------------ server/lib/webtorrent.js | 161 ------------------------------- server/models/video.js | 56 +++++------ 15 files changed, 57 insertions(+), 325 deletions(-) delete mode 100644 server/lib/webtorrent-process.js delete mode 100644 server/lib/webtorrent.js diff --git a/.gitignore b/.gitignore index debe118e1..ec5b4b2f0 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ uploads thumbnails config/production.yaml ffmpeg +torrents diff --git a/client/package.json b/client/package.json index b01e768b3..a5c5d092b 100644 --- a/client/package.json +++ b/client/package.json @@ -63,7 +63,7 @@ "webpack-md5-hash": "0.0.5", "webpack-merge": "^0.13.0", "webpack-notifier": "^1.3.0", - "webtorrent": "^0.93.2", + "webtorrent": "^0.95.2", "zone.js": "0.6.12" }, "devDependencies": { diff --git a/config/default.yaml b/config/default.yaml index b5fd7b06d..9a8a57879 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -17,6 +17,7 @@ storage: uploads: 'uploads/' logs: 'logs/' thumbnails: 'thumbnails/' + torrents: 'torrents/' network: friends: [] diff --git a/config/test-1.yaml b/config/test-1.yaml index 3afcb1b04..0998eaea1 100644 --- a/config/test-1.yaml +++ b/config/test-1.yaml @@ -14,6 +14,7 @@ storage: uploads: 'test1/uploads/' logs: 'test1/logs/' thumbnails: 'test1/thumbnails/' + torrents: 'test1/torrents/' network: friends: diff --git a/config/test-2.yaml b/config/test-2.yaml index 51cc186eb..ec2cff811 100644 --- a/config/test-2.yaml +++ b/config/test-2.yaml @@ -14,6 +14,7 @@ storage: uploads: 'test2/uploads/' logs: 'test2/logs/' thumbnails: 'test2/thumbnails/' + torrents: 'test2/torrents/' network: friends: diff --git a/config/test-3.yaml b/config/test-3.yaml index 7ef01ba4d..24f5533e0 100644 --- a/config/test-3.yaml +++ b/config/test-3.yaml @@ -14,6 +14,7 @@ storage: uploads: 'test3/uploads/' logs: 'test3/logs/' thumbnails: 'test3/thumbnails/' + torrents: 'test3/torrents/' network: friends: diff --git a/config/test-4.yaml b/config/test-4.yaml index a4d3bb164..1f884dbf2 100644 --- a/config/test-4.yaml +++ b/config/test-4.yaml @@ -14,6 +14,7 @@ storage: uploads: 'test4/uploads/' logs: 'test4/logs/' thumbnails: 'test4/thumbnails/' + torrents: 'test4/torrents/' network: friends: diff --git a/config/test-5.yaml b/config/test-5.yaml index 0435c17fe..08ed9f068 100644 --- a/config/test-5.yaml +++ b/config/test-5.yaml @@ -14,6 +14,7 @@ storage: uploads: 'test5/uploads/' logs: 'test5/logs/' thumbnails: 'test5/thumbnails/' + torrents: 'test5/torrents/' network: friends: diff --git a/config/test-6.yaml b/config/test-6.yaml index b7dbd4bdd..a57784cca 100644 --- a/config/test-6.yaml +++ b/config/test-6.yaml @@ -14,6 +14,7 @@ storage: uploads: 'test6/uploads/' logs: 'test6/logs/' thumbnails: 'test6/thumbnails/' + torrents: 'test6/torrents/' network: friends: diff --git a/package.json b/package.json index 9a0d4ecaa..63d014376 100644 --- a/package.json +++ b/package.json @@ -39,28 +39,28 @@ "body-parser": "^1.12.4", "concurrently": "^2.0.0", "config": "^1.14.0", + "create-torrent": "^3.24.5", "debug": "^2.2.0", "dezalgo": "^1.0.3", - "electron-spawn": "https://github.com/Chocobozzz/electron-spawn", "express": "^4.12.4", "express-oauth-server": "https://github.com/Chocobozzz/express-oauth-server", "express-validator": "^2.11.0", "fluent-ffmpeg": "^2.1.0", "js-yaml": "^3.5.4", "lodash": "^4.11.1", + "magnet-uri": "^5.1.4", "mkdirp": "^0.5.1", "mongoose": "^4.0.5", "morgan": "^1.5.3", "multer": "^1.1.0", - "node-ipc": "^8.0.0", "openssl-wrapper": "^0.3.4", + "parse-torrent": "^5.8.0", "password-generator": "^2.0.2", "request": "^2.57.0", "request-replay": "^1.0.2", "scripty": "^1.5.0", "segfault-handler": "^1.0.0", "ursa": "^0.9.1", - "webtorrent": "^0.93.2", "winston": "^2.1.1", "ws": "^1.1.1" }, diff --git a/server.js b/server.js index b2eeeff70..0033ed1db 100644 --- a/server.js +++ b/server.js @@ -34,10 +34,7 @@ const customValidators = require('./server/helpers/custom-validators') const installer = require('./server/initializers/installer') const mongoose = require('mongoose') const routes = require('./server/controllers') -const utils = require('./server/helpers/utils') -const webtorrent = require('./server/lib/webtorrent') const Request = mongoose.model('Request') -const Video = mongoose.model('Video') // Get configurations const port = config.get('listen.port') @@ -72,9 +69,16 @@ app.use('/client/*', function (req, res, next) { res.sendStatus(404) }) +const torrentsPhysicalPath = path.join(__dirname, config.get('storage.torrents')) +app.use(constants.STATIC_PATHS.TORRENTS, express.static(torrentsPhysicalPath, { maxAge: 0 })) + +// Uploads path for webseeding +const uploadsPhysicalPath = path.join(__dirname, config.get('storage.uploads')) +app.use(constants.STATIC_PATHS.WEBSEED, express.static(uploadsPhysicalPath, { maxAge: 0 })) + // Thumbnails path for express const thumbnailsPhysicalPath = path.join(__dirname, config.get('storage.thumbnails')) -app.use(constants.THUMBNAILS_STATIC_PATH, express.static(thumbnailsPhysicalPath, { maxAge: 0 })) +app.use(constants.STATIC_PATHS.THUMBNAILS, express.static(thumbnailsPhysicalPath, { maxAge: 0 })) // Client application app.use('/*', function (req, res, next) { @@ -121,33 +125,14 @@ app.use(function (err, req, res, next) { installer.installApplication(function (err) { if (err) throw err - // Create/activate the webtorrent module - webtorrent.create(function () { - function cleanForExit () { - utils.cleanForExit(webtorrent.app) - } + // ----------- Make the server listening ----------- + server.listen(port, function () { + // Activate the pool requests + Request.activate() - function exitGracefullyOnSignal () { - process.exit(-1) - } - - process.on('exit', cleanForExit) - process.on('SIGINT', exitGracefullyOnSignal) - process.on('SIGTERM', exitGracefullyOnSignal) - - // ----------- Make the server listening ----------- - server.listen(port, function () { - // Activate the pool requests - Request.activate() - - Video.seedAllExisting(function (err) { - if (err) throw err - - logger.info('Seeded all the videos') - logger.info('Server listening on port %d', port) - app.emit('ready') - }) - }) + logger.info('Seeded all the videos') + logger.info('Server listening on port %d', port) + app.emit('ready') }) }) diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 467816f2c..e0ea188af 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -49,12 +49,16 @@ const SORTABLE_COLUMNS = { VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdDate', '-createdDate' ] } +// Express static paths (router) +const STATIC_PATHS = { + THUMBNAILS: '/static/thumbnails', + TORRENTS: '/static/torrents/', + WEBSEED: '/static/webseed/' +} + // Videos thumbnail size const THUMBNAILS_SIZE = '200x110' -// Path for access to thumbnails with express router -const THUMBNAILS_STATIC_PATH = '/static/thumbnails' - const VIDEOS_CONSTRAINTS_FIELDS = { NAME: { min: 3, max: 50 }, // Length DESCRIPTION: { min: 3, max: 250 }, // Length @@ -89,8 +93,8 @@ module.exports = { SEARCHABLE_COLUMNS: SEARCHABLE_COLUMNS, SEEDS_IN_PARALLEL: SEEDS_IN_PARALLEL, SORTABLE_COLUMNS: SORTABLE_COLUMNS, + STATIC_PATHS: STATIC_PATHS, THUMBNAILS_SIZE: THUMBNAILS_SIZE, - THUMBNAILS_STATIC_PATH: THUMBNAILS_STATIC_PATH, VIDEOS_CONSTRAINTS_FIELDS: VIDEOS_CONSTRAINTS_FIELDS } diff --git a/server/lib/webtorrent-process.js b/server/lib/webtorrent-process.js deleted file mode 100644 index be7ac5bb4..000000000 --- a/server/lib/webtorrent-process.js +++ /dev/null @@ -1,92 +0,0 @@ -'use strict' - -const WebTorrent = require('webtorrent') -const ipc = require('node-ipc') - -function webtorrent (args) { - if (args.length !== 3) { - throw new Error('Wrong arguments number: ' + args.length + '/3') - } - - const host = args[1] - const port = args[2] - const nodeKey = 'webtorrentnode' + port - const processKey = 'webtorrentprocess' + port - - ipc.config.silent = true - ipc.config.id = processKey - - if (host === 'client' && port === '1') global.WEBTORRENT_ANNOUNCE = [] - else global.WEBTORRENT_ANNOUNCE = 'ws://' + host + ':' + port + '/tracker/socket' - const wt = new WebTorrent({ dht: false }) - - function seed (data) { - const args = data.args - const path = args.path - const _id = data._id - - wt.seed(path, { announceList: '' }, function (torrent) { - const toSend = { - magnetUri: torrent.magnetURI - } - - ipc.of[nodeKey].emit(nodeKey + '.seedDone.' + _id, toSend) - }) - } - - function add (data) { - const args = data.args - const magnetUri = args.magnetUri - const _id = data._id - - wt.add(magnetUri, function (torrent) { - const toSend = { - files: [] - } - - torrent.files.forEach(function (file) { - toSend.files.push({ path: file.path }) - }) - - ipc.of[nodeKey].emit(nodeKey + '.addDone.' + _id, toSend) - }) - } - - function remove (data) { - const args = data.args - const magnetUri = args.magnetUri - const _id = data._id - - try { - wt.remove(magnetUri, callback) - } catch (err) { - console.log('Cannot remove the torrent from WebTorrent.') - return callback(null) - } - - function callback () { - const toSend = {} - ipc.of[nodeKey].emit(nodeKey + '.removeDone.' + _id, toSend) - } - } - - console.log('Configuration: ' + host + ':' + port) - console.log('Connecting to IPC...') - - ipc.connectTo(nodeKey, function () { - ipc.of[nodeKey].on(processKey + '.seed', seed) - ipc.of[nodeKey].on(processKey + '.add', add) - ipc.of[nodeKey].on(processKey + '.remove', remove) - - ipc.of[nodeKey].emit(processKey + '.ready') - console.log('Ready.') - }) - - process.on('uncaughtException', function (e) { - ipc.of[nodeKey].emit(processKey + '.exception', { exception: e.toString() }) - }) -} - -// --------------------------------------------------------------------------- - -module.exports = webtorrent diff --git a/server/lib/webtorrent.js b/server/lib/webtorrent.js deleted file mode 100644 index bcd30139e..000000000 --- a/server/lib/webtorrent.js +++ /dev/null @@ -1,161 +0,0 @@ -'use strict' - -const config = require('config') -const ipc = require('node-ipc') -const pathUtils = require('path') -const spawn = require('electron-spawn') - -const logger = require('../helpers/logger') - -const electronDebug = config.get('electron.debug') -let host = config.get('webserver.host') -let port = config.get('webserver.port') -let nodeKey = 'webtorrentnode' + port -let processKey = 'webtorrentprocess' + port -ipc.config.silent = true -ipc.config.id = nodeKey - -const webtorrent = { - add: add, - app: null, // Pid of the app - create: create, - remove: remove, - seed: seed, - silent: false // Useful for beautiful tests -} - -function create (options, callback) { - if (typeof options === 'function') { - callback = options - options = {} - } - - // Override options - if (options.host) host = options.host - if (options.port) { - port = options.port - nodeKey = 'webtorrentnode' + port - processKey = 'webtorrentprocess' + port - ipc.config.id = nodeKey - } - - ipc.serve(function () { - if (!webtorrent.silent) logger.info('IPC server ready.') - - // Run a timeout of 30s after which we exit the process - const timeoutWebtorrentProcess = setTimeout(function () { - throw new Error('Timeout : cannot run the webtorrent process. Please ensure you have electron-prebuilt npm package installed with xvfb-run.') - }, 30000) - - ipc.server.on(processKey + '.ready', function () { - if (!webtorrent.silent) logger.info('Webtorrent process ready.') - clearTimeout(timeoutWebtorrentProcess) - callback() - }) - - ipc.server.on(processKey + '.exception', function (data) { - throw new Error('Received exception error from webtorrent process : ' + data.exception) - }) - - const webtorrentProcess = spawn(pathUtils.join(__dirname, 'webtorrent-process.js'), host, port, { detached: true }) - - if (electronDebug === true) { - webtorrentProcess.stderr.on('data', function (data) { - logger.debug('Webtorrent process stderr: ', data.toString()) - }) - - webtorrentProcess.stdout.on('data', function (data) { - logger.debug('Webtorrent process:', data.toString()) - }) - } - - webtorrent.app = webtorrentProcess - }) - - ipc.server.start() -} - -function seed (path, callback) { - const extension = pathUtils.extname(path) - const basename = pathUtils.basename(path, extension) - const data = { - _id: basename, - args: { - path: path - } - } - - if (!webtorrent.silent) logger.debug('Node wants to seed %s.', data._id) - - // Finish signal - const eventKey = nodeKey + '.seedDone.' + data._id - ipc.server.on(eventKey, function listener (received) { - if (!webtorrent.silent) logger.debug('Process seeded torrent %s.', received.magnetUri) - - // This is a fake object, we just use the magnetUri in this project - const torrent = { - magnetURI: received.magnetUri - } - - ipc.server.off(eventKey, '*') - callback(torrent) - }) - - ipc.server.broadcast(processKey + '.seed', data) -} - -function add (magnetUri, callback) { - const data = { - _id: magnetUri, - args: { - magnetUri: magnetUri - } - } - - if (!webtorrent.silent) logger.debug('Node wants to add ' + data._id) - - // Finish signal - const eventKey = nodeKey + '.addDone.' + data._id - ipc.server.on(eventKey, function (received) { - if (!webtorrent.silent) logger.debug('Process added torrent.') - - // This is a fake object, we just use the magnetUri in this project - const torrent = { - files: received.files - } - - ipc.server.off(eventKey, '*') - callback(torrent) - }) - - ipc.server.broadcast(processKey + '.add', data) -} - -function remove (magnetUri, callback) { - const data = { - _id: magnetUri, - args: { - magnetUri: magnetUri - } - } - - if (!webtorrent.silent) logger.debug('Node wants to stop seeding %s.', data._id) - - // Finish signal - const eventKey = nodeKey + '.removeDone.' + data._id - ipc.server.on(eventKey, function (received) { - if (!webtorrent.silent) logger.debug('Process removed torrent %s.', data._id) - - let err = null - if (received.err) err = received.err - - ipc.server.off(eventKey, '*') - callback(err) - }) - - ipc.server.broadcast(processKey + '.remove', data) -} - -// --------------------------------------------------------------------------- - -module.exports = webtorrent diff --git a/server/models/video.js b/server/models/video.js index 396aa505d..14e0df6f2 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -1,24 +1,27 @@ 'use strict' const config = require('config') -const eachLimit = require('async/eachLimit') +const createTorrent = require('create-torrent') const ffmpeg = require('fluent-ffmpeg') const fs = require('fs') const parallel = require('async/parallel') +const parseTorrent = require('parse-torrent') const pathUtils = require('path') +const magnet = require('magnet-uri') const mongoose = require('mongoose') const constants = require('../initializers/constants') const customValidators = require('../helpers/custom-validators') const logger = require('../helpers/logger') const utils = require('../helpers/utils') -const webtorrent = require('../lib/webtorrent') const http = config.get('webserver.https') === true ? 'https' : 'http' const host = config.get('webserver.host') const port = config.get('webserver.port') const uploadsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.uploads')) const thumbnailsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.thumbnails')) +const torrentsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.torrents')) +const webseedBaseUrl = http + '://' + host + ':' + port + constants.STATIC_PATHS.WEBSEED // --------------------------------------------------------------------------- @@ -66,8 +69,7 @@ VideoSchema.statics = { listOwned: listOwned, listRemotes: listRemotes, load: load, - search: search, - seedAllExisting: seedAllExisting + search: search } VideoSchema.pre('remove', function (next) { @@ -103,8 +105,21 @@ VideoSchema.pre('save', function (next) { this.podUrl = http + '://' + host + ':' + port tasks.push( + // TODO: refractoring function (callback) { - seed(videoPath, callback) + createTorrent(videoPath, { announceList: [ [ 'ws://' + host + ':' + port + '/tracker/socket' ] ], urlList: [ webseedBaseUrl + video.filename ] }, function (err, torrent) { + if (err) return callback(err) + + fs.writeFile(torrentsDir + video.filename + '.torrent', torrent, function (err) { + if (err) return callback(err) + + const parsedTorrent = parseTorrent(torrent) + parsedTorrent.xs = video.podUrl + constants.STATIC_PATHS.TORRENTS + video.filename + '.torrent' + video.magnetUri = magnet.encode(parsedTorrent) + + callback(null) + }) + }) }, function (callback) { createThumbnail(videoPath, callback) @@ -114,7 +129,6 @@ VideoSchema.pre('save', function (next) { parallel(tasks, function (err, results) { if (err) return next(err) - video.magnetUri = results[0].magnetURI video.thumbnail = results[1] return next() @@ -149,7 +163,7 @@ function toFormatedJSON () { author: this.author, duration: this.duration, tags: this.tags, - thumbnailPath: constants.THUMBNAILS_STATIC_PATH + '/' + this.thumbnail, + thumbnailPath: constants.STATIC_PATHS.THUMBNAILS + '/' + this.thumbnail, createdDate: this.createdDate } @@ -231,17 +245,6 @@ function search (value, field, start, count, sort, callback) { findWithCount.call(this, query, start, count, sort, callback) } -function seedAllExisting (callback) { - listOwned.call(this, function (err, videos) { - if (err) return callback(err) - - eachLimit(videos, constants.SEEDS_IN_PARALLEL, function (video, callbackEach) { - const videoPath = pathUtils.join(uploadsDir, video.filename) - seed(videoPath, callbackEach) - }, callback) - }) -} - // --------------------------------------------------------------------------- function findWithCount (query, start, count, sort, callback) { @@ -273,12 +276,7 @@ function removeFile (video, callback) { // Maybe the torrent is not seeded, but we catch the error to don't stop the removing process function removeTorrent (video, callback) { - try { - webtorrent.remove(video.magnetUri, callback) - } catch (err) { - logger.warn('Cannot remove the torrent from WebTorrent', { err: err }) - return callback(null) - } + fs.unlink(torrentsDir + video.filename + '.torrent') } function createThumbnail (videoPath, callback) { @@ -296,16 +294,6 @@ function createThumbnail (videoPath, callback) { }) } -function seed (path, callback) { - logger.info('Seeding %s...', path) - - webtorrent.seed(path, function (torrent) { - logger.info('%s seeded (%s).', path, torrent.magnetURI) - - return callback(null, torrent) - }) -} - function generateThumbnailFromBase64 (data, callback) { // Creating the thumbnail for this remote video utils.generateRandomString(16, function (err, randomString) {