From 61b3e146e16e997ea539cd4610af10d4b681e04a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 20 Feb 2018 18:01:38 +0100 Subject: [PATCH] Add ability to import videos from all supported youtube-dl sites --- CHANGELOG.md | 17 +++ README.md | 2 +- .../+video-edit/video-update.component.ts | 2 +- scripts/danger/clean/dev.sh | 2 +- scripts/danger/clean/prod.sh | 2 +- .../{import-youtube.ts => import-videos.ts} | 112 ++++++++++++------ .../{import-youtube.md => import-videos.md} | 18 +-- 7 files changed, 104 insertions(+), 51 deletions(-) rename server/tools/{import-youtube.ts => import-videos.ts} (62%) rename support/doc/{import-youtube.md => import-videos.md} (65%) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce7541109..d28674c6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,23 @@ # Changelog +## v0.0.26-alpha + +### BREAKING CHANGES + + * Renamed script `import-youtube.js` to `import-videos.js` + * Renamed `import-video.js` argument `youtube-url` to `target-url` + +### Features + + * Add "Support" attribute/button on videos + * Add ability to import from all [supported sites](https://rg3.github.io/youtube-dl/supportedsites.html) of youtube-dl + +### Bug fixes + + * Fix custom instance name overflow + + ## v0.0.25-alpha ### Features diff --git a/README.md b/README.md index 5ff05fcb8..95f274b7b 100644 --- a/README.md +++ b/README.md @@ -163,7 +163,7 @@ For now only on Github: ## Tools - * [YouTube import](/support/doc/import-youtube.md) + * [Import videos (YouTube, Dailymotion, Vimeo...)](/support/doc/import-videos.md) ## Architecture diff --git a/client/src/app/videos/+video-edit/video-update.component.ts b/client/src/app/videos/+video-edit/video-update.component.ts index 0ef3c0259..d97e00a3a 100644 --- a/client/src/app/videos/+video-edit/video-update.component.ts +++ b/client/src/app/videos/+video-edit/video-update.component.ts @@ -61,7 +61,7 @@ export class VideoUpdateComponent extends FormReactive implements OnInit { .switchMap(video => { return this.videoService .loadCompleteDescription(video.descriptionPath) - .map(description => Object.assign(video, { description })) + .map(description => Object.assign(video, { description })) }) .subscribe( video => { diff --git a/scripts/danger/clean/dev.sh b/scripts/danger/clean/dev.sh index 270ca0a2e..cd8456772 100755 --- a/scripts/danger/clean/dev.sh +++ b/scripts/danger/clean/dev.sh @@ -4,5 +4,5 @@ read -p "This will remove all directories and SQL tables. Are you sure? (y/*) " echo if [[ "$REPLY" =~ ^[Yy]$ ]]; then - NODE_ENV=test npm run ts-node "./scripts/danger/clean/cleaner" + NODE_ENV=test npm run ts-node -- --type-check "./scripts/danger/clean/cleaner" fi diff --git a/scripts/danger/clean/prod.sh b/scripts/danger/clean/prod.sh index 705987100..9103a7944 100755 --- a/scripts/danger/clean/prod.sh +++ b/scripts/danger/clean/prod.sh @@ -4,5 +4,5 @@ read -p "This will remove all directories and SQL tables. Are you sure? (y/*) " echo if [[ "$REPLY" =~ ^[Yy]$ ]]; then - NODE_ENV=production npm run ts-node "./scripts/danger/clean/cleaner" + NODE_ENV=production npm run ts-node -- --type-check "./scripts/danger/clean/cleaner" fi diff --git a/server/tools/import-youtube.ts b/server/tools/import-videos.ts similarity index 62% rename from server/tools/import-youtube.ts rename to server/tools/import-videos.ts index 20b4b0179..268101b41 100644 --- a/server/tools/import-youtube.ts +++ b/server/tools/import-videos.ts @@ -11,15 +11,16 @@ program .option('-u, --url ', 'Server url') .option('-U, --username ', 'Username') .option('-p, --password ', 'Password') - .option('-y, --youtube-url ', 'Youtube URL') + .option('-t, --target-url ', 'Video target URL') .option('-l, --language ', 'Language code') + .option('-v, --verbose', 'Verbose mode') .parse(process.argv) if ( !program['url'] || !program['username'] || !program['password'] || - !program['youtubeUrl'] + !program['targetUrl'] ) { console.error('All arguments are required.') process.exit(-1) @@ -28,6 +29,13 @@ if ( run().catch(err => console.error(err)) let accessToken: string +let client: { id: string, secret: string } + +const user = { + username: program['username'], + password: program['password'] +} + const processOptions = { cwd: __dirname, maxBuffer: Infinity @@ -35,74 +43,72 @@ const processOptions = { async function run () { const res = await getClient(program['url']) - const client = { + client = { id: res.body.client_id, secret: res.body.client_secret } - const user = { - username: program['username'], - password: program['password'] - } - const res2 = await login(program['url'], client, user) accessToken = res2.body.access_token const options = [ '-j', '--flat-playlist', '--playlist-reverse' ] - youtubeDL.getInfo(program['youtubeUrl'], options, processOptions, async (err, info) => { + youtubeDL.getInfo(program['targetUrl'], options, processOptions, async (err, info) => { if (err) throw err + let infoArray: any[] + // Normalize utf8 fields - info = info.map(i => normalizeObject(i)) + if (Array.isArray(info) === true) { + infoArray = info.map(i => normalizeObject(i)) + } else { + infoArray = [ normalizeObject(info) ] + } + console.log('Will download and upload %d videos.\n', infoArray.length) - const videos = info.map(i => { - return { url: 'https://www.youtube.com/watch?v=' + i.id, name: i.title } - }) - - console.log('Will download and upload %d videos.\n', videos.length) - - for (const video of videos) { - await processVideo(video, program['language'], client, user) + for (const info of infoArray) { + await processVideo(info, program['language']) } - console.log('I have finished!') + // https://www.youtube.com/watch?v=2Upx39TBc1s + console.log('I\'m finished!') process.exit(0) }) } -function processVideo (video: { name: string, url: string }, languageCode: number, client: { id: string, secret: string }, user: { username: string, password: string }) { +function processVideo (info: any, languageCode: number) { return new Promise(async res => { - const result = await searchVideo(program['url'], video.name) + if (program['verbose']) console.log('Fetching object.', info) + + const videoInfo = await fetchObject(info) + if (program['verbose']) console.log('Fetched object.', videoInfo) + + const result = await searchVideo(program['url'], videoInfo.title) console.log('############################################################\n') - if (result.body.data.find(v => v.name === video.name)) { - console.log('Video "%s" already exists, don\'t reupload it.\n', video.name) + if (result.body.data.find(v => v.name === videoInfo.title)) { + console.log('Video "%s" already exists, don\'t reupload it.\n', videoInfo.title) return res() } const path = join(__dirname, new Date().getTime() + '.mp4') - console.log('Downloading video "%s"...', video.name) + console.log('Downloading video "%s"...', videoInfo.title) - const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]', '-o', path ] - youtubeDL.exec(video.url, options, processOptions, async (err, output) => { + const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ] + youtubeDL.exec(videoInfo.url, options, processOptions, async (err, output) => { if (err) return console.error(err) console.log(output.join('\n')) - youtubeDL.getInfo(video.url, undefined, processOptions, async (err, videoInfo) => { - if (err) return console.error(err) + await uploadVideoOnPeerTube(normalizeObject(videoInfo), path, languageCode) - await uploadVideoOnPeerTube(normalizeObject(videoInfo), path, client, user, languageCode) - - return res() - }) + return res() }) }) } -async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, client: { id: string, secret: string }, user: { username: string, password: string }, language?: number) { +async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, language?: number) { const category = await getCategory(videoInfo.categories) const licence = getLicence(videoInfo.license) let tags = [] @@ -141,13 +147,16 @@ async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, client: console.log('\nUploading on PeerTube video "%s".', videoAttributes.name) try { await uploadVideo(program['url'], accessToken, videoAttributes) - } - catch (err) { - if ((err.message).search("401")) { - console.log("Get 401 Unauthorized, token may have expired, renewing token and retry.") - const res2 = await login(program['url'], client, user) - accessToken = res2.body.access_token + } catch (err) { + if (err.message.indexOf('401')) { + console.log('Got 401 Unauthorized, token may have expired, renewing token and retry.') + + const res = await login(program['url'], client, user) + accessToken = res.body.access_token + await uploadVideo(program['url'], accessToken, videoAttributes) + } else { + throw err } } @@ -160,6 +169,8 @@ async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, client: } async function getCategory (categories: string[]) { + if (!categories) return undefined + const categoryString = categories[0] if (categoryString === 'News & Politics') return 11 @@ -176,6 +187,8 @@ async function getCategory (categories: string[]) { } function getLicence (licence: string) { + if (!licence) return undefined + if (licence.indexOf('Creative Commons Attribution licence') !== -1) return 1 return undefined @@ -199,3 +212,24 @@ function normalizeObject (obj: any) { return newObj } + +function fetchObject (info: any) { + const url = buildUrl(info) + + return new Promise(async (res, rej) => { + youtubeDL.getInfo(url, undefined, processOptions, async (err, videoInfo) => { + if (err) return rej(err) + + const videoInfoWithUrl = Object.assign(videoInfo, { url }) + return res(normalizeObject(videoInfoWithUrl)) + }) + }) +} + +function buildUrl (info: any) { + const url = info.url as string + if (url && url.match(/^https?:\/\//)) return info.url + + // It seems youtube-dl does not return the video url + return 'https://www.youtube.com/watch?v=' + info.id +} diff --git a/support/doc/import-youtube.md b/support/doc/import-videos.md similarity index 65% rename from support/doc/import-youtube.md rename to support/doc/import-videos.md index 39f01b85b..166bb7c9f 100644 --- a/support/doc/import-youtube.md +++ b/support/doc/import-videos.md @@ -1,6 +1,6 @@ -# Import videos from Youtube guide +# Import videos guide -You can use this script to import videos from Youtube channel to Peertube. +You can use this script to import videos from all [supported sites of youtube-dl](https://rg3.github.io/youtube-dl/supportedsites.html) into PeerTube. Be sure you own the videos or have the author's authorization to do so. - [Installation](#installation) @@ -16,7 +16,6 @@ Importation can be launched directly from a PeerTube server (in this case you al ### Dependencies * [PeerTube dependencies](dependencies.md) - * git ### Installation @@ -46,16 +45,19 @@ You are now ready to run the script : ``` cd ${CLONE} -node dist/server/tools/import-youtube.js -u "PEERTUBE_URL" -U "PEERTUBE_USER" --password "PEERTUBE_PASSWORD" -y "YOUTUBE_URL" +node dist/server/tools/import-video.js -u "PEERTUBE_URL" -U "PEERTUBE_USER" --password "PEERTUBE_PASSWORD" -t "TARGET_URL" ``` * PEERTUBE_URL : the full URL of your PeerTube server where you want to import, eg: https://peertube.cpy.re/ * PEERTUBE_USER : your PeerTube account where videos will be uploaded * PEERTUBE_PASSWORD : password of your PeerTube account - * YOUTUBE_URL : the youtube video/user/channel/playlist you want to import. Examples: - * Channel: https://www.youtube.com/channel/ChannelId - * User https://www.youtube.com/c/UserName or https://www.youtube.com/user/UserName - * Video https://www.youtube.com/watch?v=blabla + * TARGET_URL : the target url you want to import. Examples: + * YouTube: + * Channel: https://www.youtube.com/channel/ChannelId + * User https://www.youtube.com/c/UserName or https://www.youtube.com/user/UserName + * Video https://www.youtube.com/watch?v=blabla + * Vimeo: https://vimeo.com/xxxxxx + * Dailymotion: https://www.dailymotion.com/xxxxx The script will get all public videos from Youtube, download them and upload to PeerTube. Already downloaded videos will not be uploaded twice, so you can run and re-run the script in case of crash, disconnection...