Import magnets with webtorrent

client/src/app/shared/video-import/video-import.service.ts

@@ -26,34 +26,23 @@ export class VideoImportService {
     private serverService: ServerService
   ) {}
 
-  importVideo (targetUrl: string, video: VideoUpdate): Observable<VideoImport> {
+  importVideoUrl (targetUrl: string, video: VideoUpdate): Observable<VideoImport> {
     const url = VideoImportService.BASE_VIDEO_IMPORT_URL
-    const language = video.language || null
-    const licence = video.licence || null
-    const category = video.category || null
-    const description = video.description || null
-    const support = video.support || null
-    const scheduleUpdate = video.scheduleUpdate || null
 
-    const body: VideoImportCreate = {
+    const body = this.buildImportVideoObject(video)
-      targetUrl,
+    body.targetUrl = targetUrl
 
-      name: video.name,
+    const data = objectToFormData(body)
-      category,
+    return this.authHttp.post<VideoImport>(url, data)
-      licence,
+               .pipe(catchError(res => this.restExtractor.handleError(res)))
-      language,
+  }
-      support,
+
-      description,
+  importVideoTorrent (target: string | Blob, video: VideoUpdate): Observable<VideoImport> {
-      channelId: video.channelId,
+    const url = VideoImportService.BASE_VIDEO_IMPORT_URL
-      privacy: video.privacy,
+    const body: VideoImportCreate = this.buildImportVideoObject(video)
-      tags: video.tags,
+
-      nsfw: video.nsfw,
+    if (typeof target === 'string') body.magnetUri = target
-      waitTranscoding: video.waitTranscoding,
+    else body.torrentfile = target
-      commentsEnabled: video.commentsEnabled,
-      thumbnailfile: video.thumbnailfile,
-      previewfile: video.previewfile,
-      scheduleUpdate
-    }
 
     const data = objectToFormData(body)
     return this.authHttp.post<VideoImport>(url, data)
@@ -73,6 +62,33 @@ export class VideoImportService {
                )
   }
 
+  private buildImportVideoObject (video: VideoUpdate): VideoImportCreate {
+    const language = video.language || null
+    const licence = video.licence || null
+    const category = video.category || null
+    const description = video.description || null
+    const support = video.support || null
+    const scheduleUpdate = video.scheduleUpdate || null
+
+    return {
+      name: video.name,
+      category,
+      licence,
+      language,
+      support,
+      description,
+      channelId: video.channelId,
+      privacy: video.privacy,
+      tags: video.tags,
+      nsfw: video.nsfw,
+      waitTranscoding: video.waitTranscoding,
+      commentsEnabled: video.commentsEnabled,
+      thumbnailfile: video.thumbnailfile,
+      previewfile: video.previewfile,
+      scheduleUpdate
+    }
+  }
+
   private extractVideoImports (result: ResultList<VideoImport>): Observable<ResultList<VideoImport>> {
     return this.serverService.localeObservable
                .pipe(

client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.html

@@ -0,0 +1,60 @@
+<div *ngIf="!hasImportedVideo" class="upload-video-container">
+  <div class="import-video-torrent">
+    <div class="icon icon-upload"></div>
+
+    <div class="form-group">
+      <label i18n for="magnetUri">Magnet URI</label>
+      <my-help
+        helpType="custom" i18n-customHtml
+        customHtml="You can import any torrent file that points to a mp4 file. You should make sure you have diffusion rights over the content it points to, otherwise it could cause legal trouble to yourself and your instance."
+      ></my-help>
+
+      <input type="text" id="magnetUri" [(ngModel)]="magnetUri" />
+    </div>
+
+    <div class="form-group">
+      <label i18n for="first-step-channel">Channel</label>
+      <div class="peertube-select-container">
+        <select id="first-step-channel" [(ngModel)]="firstStepChannelId">
+          <option *ngFor="let channel of userVideoChannels" [value]="channel.id">{{ channel.label }}</option>
+        </select>
+      </div>
+    </div>
+
+    <div class="form-group">
+      <label i18n for="first-step-privacy">Privacy</label>
+      <div class="peertube-select-container">
+        <select id="first-step-privacy" [(ngModel)]="firstStepPrivacyId">
+          <option *ngFor="let privacy of videoPrivacies" [value]="privacy.id">{{ privacy.label }}</option>
+        </select>
+      </div>
+    </div>
+
+    <input
+      type="button" i18n-value value="Import"
+      [disabled]="!isMagnetUrlValid() || isImportingVideo" (click)="importVideo()"
+    />
+  </div>
+</div>
+
+<div *ngIf="hasImportedVideo" class="alert alert-info" i18n>
+  Congratulations, the video will be imported with BitTorrent! You can already add information about this video.
+</div>
+
+<!-- Hidden because we want to load the component -->
+<form [hidden]="!hasImportedVideo" novalidate [formGroup]="form">
+  <my-video-edit
+    [form]="form" [formErrors]="formErrors" [videoCaptions]="videoCaptions" [schedulePublicationPossible]="false"
+    [validationMessages]="validationMessages" [videoPrivacies]="videoPrivacies" [userVideoChannels]="userVideoChannels"
+  ></my-video-edit>
+
+  <div class="submit-container">
+    <div class="submit-button"
+       (click)="updateSecondStep()"
+       [ngClass]="{ disabled: !form.valid || isUpdatingVideo === true }"
+    >
+      <span class="icon icon-validate"></span>
+      <input type="button" i18n-value value="Update" />
+    </div>
+  </div>
+</form>

client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.scss

@@ -0,0 +1,37 @@
+@import 'variables';
+@import 'mixins';
+
+$width-size: 190px;
+
+.peertube-select-container {
+  @include peertube-select-container($width-size);
+}
+
+.import-video-torrent {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+
+  .icon.icon-upload {
+    @include icon(90px);
+    margin-bottom: 25px;
+    cursor: default;
+
+    background-image: url('../../../../assets/images/video/upload.svg');
+  }
+
+  input[type=text] {
+    @include peertube-input-text($width-size);
+    display: block;
+  }
+
+  input[type=button] {
+    @include peertube-button;
+    @include orange-button;
+
+    width: $width-size;
+    margin-top: 30px;
+  }
+}
+
+

client/src/app/videos/+video-edit/video-add-components/video-import-torrent.component.ts

@@ -0,0 +1,132 @@
+import { Component, EventEmitter, OnInit, Output } from '@angular/core'
+import { Router } from '@angular/router'
+import { NotificationsService } from 'angular2-notifications'
+import { VideoPrivacy, VideoUpdate } from '../../../../../../shared/models/videos'
+import { AuthService, ServerService } from '../../../core'
+import { VideoService } from '../../../shared/video/video.service'
+import { I18n } from '@ngx-translate/i18n-polyfill'
+import { LoadingBarService } from '@ngx-loading-bar/core'
+import { VideoSend } from '@app/videos/+video-edit/video-add-components/video-send'
+import { CanComponentDeactivate } from '@app/shared/guards/can-deactivate-guard.service'
+import { VideoEdit } from '@app/shared/video/video-edit.model'
+import { FormValidatorService } from '@app/shared'
+import { VideoCaptionService } from '@app/shared/video-caption'
+import { VideoImportService } from '@app/shared/video-import'
+
+@Component({
+  selector: 'my-video-import-torrent',
+  templateUrl: './video-import-torrent.component.html',
+  styleUrls: [
+    '../shared/video-edit.component.scss',
+    './video-import-torrent.component.scss'
+  ]
+})
+export class VideoImportTorrentComponent extends VideoSend implements OnInit, CanComponentDeactivate {
+  @Output() firstStepDone = new EventEmitter<string>()
+
+  videoFileName: string
+  magnetUri = ''
+
+  isImportingVideo = false
+  hasImportedVideo = false
+  isUpdatingVideo = false
+
+  video: VideoEdit
+
+  protected readonly DEFAULT_VIDEO_PRIVACY = VideoPrivacy.PRIVATE
+
+  constructor (
+    protected formValidatorService: FormValidatorService,
+    protected loadingBar: LoadingBarService,
+    protected notificationsService: NotificationsService,
+    protected authService: AuthService,
+    protected serverService: ServerService,
+    protected videoService: VideoService,
+    protected videoCaptionService: VideoCaptionService,
+    private router: Router,
+    private videoImportService: VideoImportService,
+    private i18n: I18n
+  ) {
+    super()
+  }
+
+  ngOnInit () {
+    super.ngOnInit()
+  }
+
+  canDeactivate () {
+    return { canDeactivate: true }
+  }
+
+  isMagnetUrlValid () {
+    return !!this.magnetUri
+  }
+
+  importVideo () {
+    this.isImportingVideo = true
+
+    const videoUpdate: VideoUpdate = {
+      privacy: this.firstStepPrivacyId,
+      waitTranscoding: false,
+      commentsEnabled: true,
+      channelId: this.firstStepChannelId
+    }
+
+    this.loadingBar.start()
+
+    this.videoImportService.importVideoTorrent(this.magnetUri, videoUpdate).subscribe(
+      res => {
+        this.loadingBar.complete()
+        this.firstStepDone.emit(res.video.name)
+        this.isImportingVideo = false
+        this.hasImportedVideo = true
+
+        this.video = new VideoEdit(Object.assign(res.video, {
+          commentsEnabled: videoUpdate.commentsEnabled,
+          support: null,
+          thumbnailUrl: null,
+          previewUrl: null
+        }))
+        this.hydrateFormFromVideo()
+      },
+
+      err => {
+        this.loadingBar.complete()
+        this.isImportingVideo = false
+        this.notificationsService.error(this.i18n('Error'), err.message)
+      }
+    )
+  }
+
+  updateSecondStep () {
+    if (this.checkForm() === false) {
+      return
+    }
+
+    this.video.patch(this.form.value)
+
+    this.isUpdatingVideo = true
+
+    // Update the video
+    this.updateVideoAndCaptions(this.video)
+        .subscribe(
+          () => {
+            this.isUpdatingVideo = false
+            this.notificationsService.success(this.i18n('Success'), this.i18n('Video to import updated.'))
+
+            this.router.navigate([ '/my-account', 'video-imports' ])
+          },
+
+          err => {
+            this.isUpdatingVideo = false
+            this.notificationsService.error(this.i18n('Error'), err.message)
+            console.error(err)
+          }
+        )
+
+  }
+
+  private hydrateFormFromVideo () {
+    this.form.patchValue(this.video.toFormPatch())
+  }
+}

client/src/app/videos/+video-edit/video-add-components/video-import-url.component.html

@@ -1,5 +1,5 @@
 <div *ngIf="!hasImportedVideo" class="upload-video-container">
-  <div class="import-video">
+  <div class="import-video-url">
     <div class="icon icon-upload"></div>
 
     <div class="form-group">

client/src/app/videos/+video-edit/video-add-components/video-import-url.component.scss

@@ -7,7 +7,7 @@ $width-size: 190px;
   @include peertube-select-container($width-size);
 }
 
-.import-video {
+.import-video-url {
   display: flex;
   flex-direction: column;
   align-items: center;

client/src/app/videos/+video-edit/video-add-components/video-import-url.component.ts

@@ -74,7 +74,7 @@ export class VideoImportUrlComponent extends VideoSend implements OnInit, CanCom
 
     this.loadingBar.start()
 
-    this.videoImportService.importVideo(this.targetUrl, videoUpdate).subscribe(
+    this.videoImportService.importVideoUrl(this.targetUrl, videoUpdate).subscribe(
       res => {
         this.loadingBar.complete()
         this.firstStepDone.emit(res.video.name)

client/src/app/videos/+video-edit/video-add.component.html

@@ -10,8 +10,12 @@
       <my-video-upload #videoUpload (firstStepDone)="onFirstStepDone('upload', $event)"></my-video-upload>
     </tab>
 
-    <tab *ngIf="isVideoImportEnabled()" i18n-heading heading="Import with URL">
+    <tab *ngIf="isVideoImportHttpEnabled()" i18n-heading heading="Import with URL">
       <my-video-import-url #videoImportUrl (firstStepDone)="onFirstStepDone('import-url', $event)"></my-video-import-url>
     </tab>
+
+    <tab *ngIf="isVideoImportTorrentEnabled()" i18n-heading heading="Import with torrent">
+      <my-video-import-torrent #videoImportTorrent (firstStepDone)="onFirstStepDone('import-torrent', $event)"></my-video-import-torrent>
+    </tab>
   </tabset>
 </div>

client/src/app/videos/+video-edit/video-add.component.ts

@@ -3,6 +3,7 @@ import { CanComponentDeactivate } from '@app/shared/guards/can-deactivate-guard.
 import { VideoImportUrlComponent } from '@app/videos/+video-edit/video-add-components/video-import-url.component'
 import { VideoUploadComponent } from '@app/videos/+video-edit/video-add-components/video-upload.component'
 import { ServerService } from '@app/core'
+import { VideoImportTorrentComponent } from '@app/videos/+video-edit/video-add-components/video-import-torrent.component'
 
 @Component({
   selector: 'my-videos-add',
@@ -12,15 +13,16 @@ import { ServerService } from '@app/core'
 export class VideoAddComponent implements CanComponentDeactivate {
   @ViewChild('videoUpload') videoUpload: VideoUploadComponent
   @ViewChild('videoImportUrl') videoImportUrl: VideoImportUrlComponent
+  @ViewChild('videoImportTorrent') videoImportTorrent: VideoImportTorrentComponent
 
-  secondStepType: 'upload' | 'import-url'
+  secondStepType: 'upload' | 'import-url' | 'import-torrent'
   videoName: string
 
   constructor (
     private serverService: ServerService
   ) {}
 
-  onFirstStepDone (type: 'upload' | 'import-url', videoName: string) {
+  onFirstStepDone (type: 'upload' | 'import-url' | 'import-torrent', videoName: string) {
     this.secondStepType = type
     this.videoName = videoName
   }
@@ -28,11 +30,16 @@ export class VideoAddComponent implements CanComponentDeactivate {
   canDeactivate () {
     if (this.secondStepType === 'upload') return this.videoUpload.canDeactivate()
     if (this.secondStepType === 'import-url') return this.videoImportUrl.canDeactivate()
+    if (this.secondStepType === 'import-torrent') return this.videoImportTorrent.canDeactivate()
 
     return { canDeactivate: true }
   }
 
-  isVideoImportEnabled () {
+  isVideoImportHttpEnabled () {
+    return this.serverService.getConfig().import.videos.http.enabled
+  }
+
+  isVideoImportTorrentEnabled () {
     return this.serverService.getConfig().import.videos.http.enabled
   }
 }

client/src/app/videos/+video-edit/video-add.module.ts

@@ -7,6 +7,7 @@ import { VideoAddComponent } from './video-add.component'
 import { CanDeactivateGuard } from '../../shared/guards/can-deactivate-guard.service'
 import { VideoUploadComponent } from '@app/videos/+video-edit/video-add-components/video-upload.component'
 import { VideoImportUrlComponent } from '@app/videos/+video-edit/video-add-components/video-import-url.component'
+import { VideoImportTorrentComponent } from '@app/videos/+video-edit/video-add-components/video-import-torrent.component'
 
 @NgModule({
   imports: [
@@ -18,7 +19,8 @@ import { VideoImportUrlComponent } from '@app/videos/+video-edit/video-add-compo
   declarations: [
     VideoAddComponent,
     VideoUploadComponent,
-    VideoImportUrlComponent
+    VideoImportUrlComponent,
+    VideoImportTorrentComponent
   ],
   exports: [
     VideoAddComponent

package.json

@@ -134,6 +134,7 @@
     "uuid": "^3.1.0",
     "validator": "^10.2.0",
     "webfinger.js": "^2.6.6",
+    "webtorrent": "^0.100.0",
     "winston": "3.0.0",
     "ws": "^5.0.0",
     "youtube-dl": "^1.12.2"
@@ -187,7 +188,6 @@
     "tslint": "^5.7.0",
     "tslint-config-standard": "^7.0.0",
     "typescript": "^2.5.2",
-    "webtorrent": "^0.100.0",
     "xliff": "^3.0.1"
   },
   "scripty": {

server/controllers/api/videos/import.ts

@@ -1,3 +1,4 @@
+import * as magnetUtil from 'magnet-uri'
 import * as express from 'express'
 import { auditLoggerFactory, VideoImportAuditView } from '../../../helpers/audit-logger'
 import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
@@ -13,6 +14,10 @@ import { VideoImportModel } from '../../../models/video/video-import'
 import { JobQueue } from '../../../lib/job-queue/job-queue'
 import { processImage } from '../../../helpers/image-utils'
 import { join } from 'path'
+import { isArray } from '../../../helpers/custom-validators/misc'
+import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
+import { VideoChannelModel } from '../../../models/video/video-channel'
+import * as Bluebird from 'bluebird'
 
 const auditLogger = auditLoggerFactory('video-imports')
 const videoImportsRouter = express.Router()
@@ -41,7 +46,45 @@ export {
 
 // ---------------------------------------------------------------------------
 
-async function addVideoImport (req: express.Request, res: express.Response) {
+function addVideoImport (req: express.Request, res: express.Response) {
+  if (req.body.targetUrl) return addYoutubeDLImport(req, res)
+
+  if (req.body.magnetUri) return addTorrentImport(req, res)
+}
+
+async function addTorrentImport (req: express.Request, res: express.Response) {
+  const body: VideoImportCreate = req.body
+  const magnetUri = body.magnetUri
+
+  const parsed = magnetUtil.decode(magnetUri)
+  const magnetName = isArray(parsed.name) ? parsed.name[0] : parsed.name as string
+
+  const video = buildVideo(res.locals.videoChannel.id, body, { name: magnetName })
+
+  await processThumbnail(req, video)
+  await processPreview(req, video)
+
+  const tags = null
+  const videoImportAttributes = {
+    magnetUri,
+    state: VideoImportState.PENDING
+  }
+  const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes)
+
+  // Create job to import the video
+  const payload = {
+    type: 'magnet-uri' as 'magnet-uri',
+    videoImportId: videoImport.id,
+    magnetUri
+  }
+  await JobQueue.Instance.createJob({ type: 'video-import', payload })
+
+  auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON()))
+
+  return res.json(videoImport.toFormattedJSON()).end()
+}
+
+async function addYoutubeDLImport (req: express.Request, res: express.Response) {
   const body: VideoImportCreate = req.body
   const targetUrl = body.targetUrl
 
@@ -56,71 +99,17 @@ async function addVideoImport (req: express.Request, res: express.Response) {
     }).end()
   }
 
-  // Create video DB object
+  const video = buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
-  const videoData = {
+
-    name: body.name || youtubeDLInfo.name,
+  const downloadThumbnail = !await processThumbnail(req, video)
-    remote: false,
+  const downloadPreview = !await processPreview(req, video)
-    category: body.category || youtubeDLInfo.category,
+
-    licence: body.licence || youtubeDLInfo.licence,
+  const tags = body.tags || youtubeDLInfo.tags
-    language: body.language || undefined,
+  const videoImportAttributes = {
-    commentsEnabled: body.commentsEnabled || true,
+    targetUrl,
-    waitTranscoding: body.waitTranscoding || false,
+    state: VideoImportState.PENDING
-    state: VideoState.TO_IMPORT,
-    nsfw: body.nsfw || youtubeDLInfo.nsfw || false,
-    description: body.description || youtubeDLInfo.description,
-    support: body.support || null,
-    privacy: body.privacy || VideoPrivacy.PRIVATE,
-    duration: 0, // duration will be set by the import job
-    channelId: res.locals.videoChannel.id
   }
-  const video = new VideoModel(videoData)
+  const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes)
-  video.url = getVideoActivityPubUrl(video)
-
-  // Process thumbnail file?
-  const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
-  let downloadThumbnail = true
-  if (thumbnailField) {
-    const thumbnailPhysicalFile = thumbnailField[ 0 ]
-    await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE)
-    downloadThumbnail = false
-  }
-
-  // Process preview file?
-  const previewField = req.files ? req.files['previewfile'] : undefined
-  let downloadPreview = true
-  if (previewField) {
-    const previewPhysicalFile = previewField[0]
-    await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE)
-    downloadPreview = false
-  }
-
-  const videoImport: VideoImportModel = await sequelizeTypescript.transaction(async t => {
-    const sequelizeOptions = { transaction: t }
-
-    // Save video object in database
-    const videoCreated = await video.save(sequelizeOptions)
-    videoCreated.VideoChannel = res.locals.videoChannel
-
-    // Set tags to the video
-    const tags = body.tags ? body.tags : youtubeDLInfo.tags
-    if (tags !== undefined) {
-      const tagInstances = await TagModel.findOrCreateTags(tags, t)
-
-      await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
-      videoCreated.Tags = tagInstances
-    }
-
-    // Create video import object in database
-    const videoImport = await VideoImportModel.create({
-      targetUrl,
-      state: VideoImportState.PENDING,
-      videoId: videoCreated.id
-    }, sequelizeOptions)
-
-    videoImport.Video = videoCreated
-
-    return videoImport
-  })
 
   // Create job to import the video
   const payload = {
@@ -136,3 +125,82 @@ async function addVideoImport (req: express.Request, res: express.Response) {
 
   return res.json(videoImport.toFormattedJSON()).end()
 }
+
+function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo) {
+  const videoData = {
+    name: body.name || importData.name || 'Unknown name',
+    remote: false,
+    category: body.category || importData.category,
+    licence: body.licence || importData.licence,
+    language: body.language || undefined,
+    commentsEnabled: body.commentsEnabled || true,
+    waitTranscoding: body.waitTranscoding || false,
+    state: VideoState.TO_IMPORT,
+    nsfw: body.nsfw || importData.nsfw || false,
+    description: body.description || importData.description,
+    support: body.support || null,
+    privacy: body.privacy || VideoPrivacy.PRIVATE,
+    duration: 0, // duration will be set by the import job
+    channelId: channelId
+  }
+  const video = new VideoModel(videoData)
+  video.url = getVideoActivityPubUrl(video)
+
+  return video
+}
+
+async function processThumbnail (req: express.Request, video: VideoModel) {
+  const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
+  if (thumbnailField) {
+    const thumbnailPhysicalFile = thumbnailField[ 0 ]
+    await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE)
+
+    return true
+  }
+
+  return false
+}
+
+async function processPreview (req: express.Request, video: VideoModel) {
+  const previewField = req.files ? req.files['previewfile'] : undefined
+  if (previewField) {
+    const previewPhysicalFile = previewField[0]
+    await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE)
+
+    return true
+  }
+
+  return false
+}
+
+function insertIntoDB (
+  video: VideoModel,
+  videoChannel: VideoChannelModel,
+  tags: string[],
+  videoImportAttributes: FilteredModelAttributes<VideoImportModel>
+): Bluebird<VideoImportModel> {
+  return sequelizeTypescript.transaction(async t => {
+    const sequelizeOptions = { transaction: t }
+
+    // Save video object in database
+    const videoCreated = await video.save(sequelizeOptions)
+    videoCreated.VideoChannel = videoChannel
+
+    // Set tags to the video
+    if (tags !== undefined) {
+      const tagInstances = await TagModel.findOrCreateTags(tags, t)
+
+      await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
+      videoCreated.Tags = tagInstances
+    }
+
+    // Create video import object in database
+    const videoImport = await VideoImportModel.create(
+      Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
+      sequelizeOptions
+    )
+    videoImport.Video = videoCreated
+
+    return videoImport
+  })
+}

server/helpers/custom-validators/videos.ts

@@ -17,6 +17,7 @@ import { VideoModel } from '../../models/video/video'
 import { exists, isArray, isFileValid } from './misc'
 import { VideoChannelModel } from '../../models/video/video-channel'
 import { UserModel } from '../../models/account/user'
+import * as magnetUtil from 'magnet-uri'
 
 const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS
 const VIDEO_ABUSES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_ABUSES
@@ -126,6 +127,13 @@ function isVideoFileSizeValid (value: string) {
   return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.FILE_SIZE)
 }
 
+function isVideoMagnetUriValid (value: string) {
+  if (!exists(value)) return false
+
+  const parsed = magnetUtil.decode(value)
+  return parsed && isVideoFileInfoHashValid(parsed.infoHash)
+}
+
 function checkUserCanManageVideo (user: UserModel, video: VideoModel, right: UserRight, res: Response) {
   // Retrieve the user who did the request
   if (video.isOwned() === false) {
@@ -214,6 +222,7 @@ export {
   isScheduleVideoUpdatePrivacyValid,
   isVideoAbuseReasonValid,
   isVideoFile,
+  isVideoMagnetUriValid,
   isVideoStateValid,
   isVideoViewsValid,
   isVideoRatingTypeValid,

server/helpers/utils.ts

@@ -9,6 +9,8 @@ import { ApplicationModel } from '../models/application/application'
 import { pseudoRandomBytesPromise, unlinkPromise } from './core-utils'
 import { logger } from './logger'
 import { isArray } from './custom-validators/misc'
+import * as crypto from "crypto"
+import { join } from "path"
 
 const isCidr = require('is-cidr')
 
@@ -181,8 +183,14 @@ async function getServerActor () {
   return Promise.resolve(serverActor)
 }
 
+function generateVideoTmpPath (id: string) {
+  const hash = crypto.createHash('sha256').update(id).digest('hex')
+  return join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4')
+}
+
 type SortType = { sortModel: any, sortValue: string }
 
+
 // ---------------------------------------------------------------------------
 
 export {
@@ -195,5 +203,6 @@ export {
   computeResolutionsToTranscode,
   resetSequelizeInstance,
   getServerActor,
-  SortType
+  SortType,
+  generateVideoTmpPath
 }

server/helpers/webtorrent.ts

@@ -0,0 +1,31 @@
+import { logger } from './logger'
+import { generateVideoTmpPath } from './utils'
+import * as WebTorrent from 'webtorrent'
+import { createWriteStream } from 'fs'
+
+function downloadWebTorrentVideo (target: string) {
+  const path = generateVideoTmpPath(target)
+
+  logger.info('Importing torrent video %s', target)
+
+  return new Promise<string>((res, rej) => {
+    const webtorrent = new WebTorrent()
+
+    const torrent = webtorrent.add(target, torrent => {
+      if (torrent.files.length !== 1) throw new Error('The number of files is not equal to 1 for ' + target)
+
+      const file = torrent.files[ 0 ]
+      file.createReadStream().pipe(createWriteStream(path))
+    })
+
+    torrent.on('done', () => res(path))
+
+    torrent.on('error', err => rej(err))
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  downloadWebTorrentVideo
+}

server/helpers/youtube-dl.ts

@@ -1,18 +1,17 @@
 import * as youtubeDL from 'youtube-dl'
 import { truncate } from 'lodash'
-import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers'
+import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers'
-import { join } from 'path'
-import * as crypto from 'crypto'
 import { logger } from './logger'
+import { generateVideoTmpPath } from './utils'
 
 export type YoutubeDLInfo = {
-  name: string
+  name?: string
-  description: string
+  description?: string
-  category: number
+  category?: number
-  licence: number
+  licence?: number
-  nsfw: boolean
+  nsfw?: boolean
-  tags: string[]
+  tags?: string[]
-  thumbnailUrl: string
+  thumbnailUrl?: string
 }
 
 function getYoutubeDLInfo (url: string): Promise<YoutubeDLInfo> {
@@ -30,10 +29,9 @@ function getYoutubeDLInfo (url: string): Promise<YoutubeDLInfo> {
 }
 
 function downloadYoutubeDLVideo (url: string) {
-  const hash = crypto.createHash('sha256').update(url).digest('hex')
+  const path = generateVideoTmpPath(url)
-  const path = join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4')
 
-  logger.info('Importing video %s', url)
+  logger.info('Importing youtubeDL video %s', url)
 
   const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]
 

server/initializers/constants.ts

@@ -15,7 +15,7 @@ let config: IConfig = require('config')
 
 // ---------------------------------------------------------------------------
 
-const LAST_MIGRATION_VERSION = 240
+const LAST_MIGRATION_VERSION = 245
 
 // ---------------------------------------------------------------------------
 
@@ -271,7 +271,8 @@ const CONSTRAINTS_FIELDS = {
     }
   },
   VIDEO_IMPORTS: {
-    URL: { min: 3, max: 2000 } // Length
+    URL: { min: 3, max: 2000 }, // Length
+    TORRENT_NAME: { min: 3, max: 255 }, // Length
   },
   VIDEOS: {
     NAME: { min: 3, max: 120 }, // Length

server/initializers/migrations/0245-import-magnet.ts

@@ -0,0 +1,42 @@
+import * as Sequelize from 'sequelize'
+import { Migration } from '../../models/migrations'
+import { CONSTRAINTS_FIELDS } from '../index'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction
+  queryInterface: Sequelize.QueryInterface
+  sequelize: Sequelize.Sequelize
+}): Promise<any> {
+  {
+    const data = {
+      type: Sequelize.STRING,
+      allowNull: true,
+      defaultValue: null
+    } as Migration.String
+    await utils.queryInterface.changeColumn('videoImport', 'targetUrl', data)
+  }
+
+  {
+    const data = {
+      type: Sequelize.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max),
+      allowNull: true,
+      defaultValue: null
+    }
+    await utils.queryInterface.addColumn('videoImport', 'magnetUri', data)
+  }
+
+  {
+    const data = {
+      type: Sequelize.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_NAME.max),
+      allowNull: true,
+      defaultValue: null
+    }
+    await utils.queryInterface.addColumn('videoImport', 'torrentName', data)
+  }
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export { up, down }

server/lib/job-queue/handlers/video-import.ts

@@ -13,30 +13,99 @@ import { VideoState } from '../../../../shared'
 import { JobQueue } from '../index'
 import { federateVideoIfNeeded } from '../../activitypub'
 import { VideoModel } from '../../../models/video/video'
+import { downloadWebTorrentVideo } from '../../../helpers/webtorrent'
 
-export type VideoImportPayload = {
+type VideoImportYoutubeDLPayload = {
   type: 'youtube-dl'
   videoImportId: number
+
   thumbnailUrl: string
   downloadThumbnail: boolean
   downloadPreview: boolean
 }
 
+type VideoImportTorrentPayload = {
+  type: 'magnet-uri'
+  videoImportId: number
+}
+
+export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload
+
 async function processVideoImport (job: Bull.Job) {
   const payload = job.data as VideoImportPayload
-  logger.info('Processing video import in job %d.', job.id)
 
-  const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId)
+  if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, payload)
+  if (payload.type === 'magnet-uri') return processTorrentImport(job, payload)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  processVideoImport
+}
+
+// ---------------------------------------------------------------------------
+
+async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentPayload) {
+  logger.info('Processing torrent video import in job %d.', job.id)
+
+  const videoImport = await getVideoImportOrDie(payload.videoImportId)
+  const options = {
+    videoImportId: payload.videoImportId,
+
+    downloadThumbnail: false,
+    downloadPreview: false,
+
+    generateThumbnail: true,
+    generatePreview: true
+  }
+  return processFile(() => downloadWebTorrentVideo(videoImport.magnetUri), videoImport, options)
+}
+
+async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutubeDLPayload) {
+  logger.info('Processing youtubeDL video import in job %d.', job.id)
+
+  const videoImport = await getVideoImportOrDie(payload.videoImportId)
+  const options = {
+    videoImportId: videoImport.id,
+
+    downloadThumbnail: payload.downloadThumbnail,
+    downloadPreview: payload.downloadPreview,
+    thumbnailUrl: payload.thumbnailUrl,
+
+    generateThumbnail: false,
+    generatePreview: false
+  }
+
+  return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl), videoImport, options)
+}
+
+async function getVideoImportOrDie (videoImportId: number) {
+  const videoImport = await VideoImportModel.loadAndPopulateVideo(videoImportId)
   if (!videoImport || !videoImport.Video) {
     throw new Error('Cannot import video %s: the video import or video linked to this import does not exist anymore.')
   }
 
+  return videoImport
+}
+
+type ProcessFileOptions = {
+  videoImportId: number
+
+  downloadThumbnail: boolean
+  downloadPreview: boolean
+  thumbnailUrl?: string
+
+  generateThumbnail: boolean
+  generatePreview: boolean
+}
+async function processFile (downloader: () => Promise<string>, videoImport: VideoImportModel, options: ProcessFileOptions) {
   let tempVideoPath: string
   let videoDestFile: string
   let videoFile: VideoFileModel
   try {
     // Download video from youtubeDL
-    tempVideoPath = await downloadYoutubeDLVideo(videoImport.targetUrl)
+    tempVideoPath = await downloader()
 
     // Get information about this video
     const { videoFileResolution } = await getVideoFileResolution(tempVideoPath)
@@ -62,23 +131,27 @@ async function processVideoImport (job: Bull.Job) {
     tempVideoPath = null // This path is not used anymore
 
     // Process thumbnail
-    if (payload.downloadThumbnail) {
+    if (options.downloadThumbnail) {
-      if (payload.thumbnailUrl) {
+      if (options.thumbnailUrl) {
         const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName())
-        await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destThumbnailPath)
+        await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destThumbnailPath)
       } else {
         await videoImport.Video.createThumbnail(videoFile)
       }
+    } else if (options.generateThumbnail) {
+      await videoImport.Video.createThumbnail(videoFile)
     }
 
     // Process preview
-    if (payload.downloadPreview) {
+    if (options.downloadPreview) {
-      if (payload.thumbnailUrl) {
+      if (options.thumbnailUrl) {
         const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName())
-        await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destPreviewPath)
+        await doRequestAndSaveToFile({ method: 'GET', uri: options.thumbnailUrl }, destPreviewPath)
       } else {
         await videoImport.Video.createPreview(videoFile)
       }
+    } else if (options.generatePreview) {
+      await videoImport.Video.createPreview(videoFile)
     }
 
     // Create torrent
@@ -137,9 +210,3 @@ async function processVideoImport (job: Bull.Job) {
     throw err
   }
 }
-
-// ---------------------------------------------------------------------------
-
-export {
-  processVideoImport
-}

server/lib/job-queue/job-queue.ts

@@ -32,13 +32,6 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
   'video-import': processVideoImport
 }
 
-const jobsWithRequestTimeout: { [ id in JobType ]?: boolean } = {
-  'activitypub-http-broadcast': true,
-  'activitypub-http-unicast': true,
-  'activitypub-http-fetcher': true,
-  'activitypub-follow': true
-}
-
 const jobTypes: JobType[] = [
   'activitypub-follow',
   'activitypub-http-broadcast',

server/middlewares/validators/video-imports.ts

@@ -6,14 +6,19 @@ import { areValidationErrors } from './utils'
 import { getCommonVideoAttributes } from './videos'
 import { isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports'
 import { cleanUpReqFiles } from '../../helpers/utils'
-import { isVideoChannelOfAccountExist, isVideoNameValid } from '../../helpers/custom-validators/videos'
+import { isVideoChannelOfAccountExist, isVideoMagnetUriValid, isVideoNameValid } from '../../helpers/custom-validators/videos'
 import { CONFIG } from '../../initializers/constants'
 
 const videoImportAddValidator = getCommonVideoAttributes().concat([
-  body('targetUrl').custom(isVideoImportTargetUrlValid).withMessage('Should have a valid video import target URL'),
   body('channelId')
     .toInt()
     .custom(isIdValid).withMessage('Should have correct video channel id'),
+  body('targetUrl')
+    .optional()
+    .custom(isVideoImportTargetUrlValid).withMessage('Should have a valid video import target URL'),
+  body('magnetUri')
+    .optional()
+    .custom(isVideoMagnetUriValid).withMessage('Should have a valid video magnet URI'),
   body('name')
     .optional()
     .custom(isVideoNameValid).withMessage('Should have a valid name'),
@@ -34,6 +39,15 @@ const videoImportAddValidator = getCommonVideoAttributes().concat([
 
     if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req)
 
+    // Check we have at least 1 required param
+    if (!req.body.targetUrl && !req.body.magnetUri) {
+      cleanUpReqFiles(req)
+
+      return res.status(400)
+        .json({ error: 'Should have a magnetUri or a targetUrl.' })
+        .end()
+    }
+
     return next()
   }
 ])

server/models/video/video-import.ts

@@ -21,6 +21,7 @@ import { VideoImport, VideoImportState } from '../../../shared'
 import { VideoChannelModel } from './video-channel'
 import { AccountModel } from '../account/account'
 import { TagModel } from './tag'
+import { isVideoMagnetUriValid } from '../../helpers/custom-validators/videos'
 
 @DefaultScope({
   include: [
@@ -62,11 +63,23 @@ export class VideoImportModel extends Model<VideoImportModel> {
   @UpdatedAt
   updatedAt: Date
 
-  @AllowNull(false)
+  @AllowNull(true)
+  @Default(null)
   @Is('VideoImportTargetUrl', value => throwIfNotValid(value, isVideoImportTargetUrlValid, 'targetUrl'))
   @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max))
   targetUrl: string
 
+  @AllowNull(true)
+  @Default(null)
+  @Is('VideoImportMagnetUri', value => throwIfNotValid(value, isVideoMagnetUriValid, 'magnetUri'))
+  @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max)) // Use the same constraints than URLs
+  magnetUri: string
+
+  @AllowNull(true)
+  @Default(null)
+  @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_NAME.max))
+  torrentName: string
+
   @AllowNull(false)
   @Default(null)
   @Is('VideoImportState', value => throwIfNotValid(value, isVideoImportStateValid, 'state'))

shared/models/videos/video-import-create.model.ts

@@ -1,6 +1,9 @@
 import { VideoUpdate } from './video-update.model'
 
 export interface VideoImportCreate extends VideoUpdate {
-  targetUrl: string
+  targetUrl?: string
+  magnetUri?: string
+  torrentfile?: Blob
+
   channelId: number // Required
 }