OpenCloud
/
PeerTube
mirror of https://github.com/Chocobozzz/PeerTube
1
0
Fork 0
Code Issues Releases Wiki Activity

Channel sync (#5135) 

* Add external channel URL for channel update / creation (#754)

* Disallow synchronisation if user has no video quota (#754)

* More constraints serverside (#754)

* Disable sync if server configuration does not allow HTTP import (#754)

* Working version synchronizing videos with a job (#754)

TODO: refactoring, too much code duplication

* More logs and try/catch (#754)

* Fix eslint error (#754)

* WIP: support synchronization time change (#754)

* New frontend #754

* WIP: Create sync front (#754)

* Enhance UI, sync creation form (#754)

* Warning message when HTTP upload is disallowed

* More consistent names (#754)

* Binding Front with API (#754)

* Add a /me API (#754)

* Improve list UI (#754)

* Implement creation and deletion routes (#754)

* Lint (#754)

* Lint again (#754)

* WIP: UI for triggering import existing videos (#754)

* Implement jobs for syncing and importing channels

* Don't sync videos before sync creation + avoid concurrency issue (#754)

* Cleanup (#754)

* Cleanup: OpenAPI + API rework (#754)

* Remove dead code (#754)

* Eslint (#754)

* Revert the mess with whitespaces in constants.ts (#754)

* Some fixes after rebase (#754)

* Several fixes after PR remarks (#754)

* Front + API: Rename video-channels-sync to video-channel-syncs (#754)

* Allow enabling channel sync through UI (#754)

* getChannelInfo (#754)

* Minor fixes: openapi + model + sql (#754)

* Simplified API validators (#754)

* Rename MChannelSync to MChannelSyncChannel (#754)

* Add command for VideoChannelSync (#754)

* Use synchronization.enabled config (#754)

* Check parameters test + some fixes (#754)

* Fix conflict mistake (#754)

* Restrict access to video channel sync list API (#754)

* Start adding unit test for synchronization (#754)

* Continue testing (#754)

* Tests finished + convertion of job to scheduler (#754)

* Add lastSyncAt field (#754)

* Fix externalRemoteUrl sort + creation date not well formatted (#754)

* Small fix (#754)

* Factorize addYoutubeDLImport and buildVideo (#754)

* Check duplicates on channel not on users (#754)

* factorize thumbnail generation (#754)

* Fetch error should return status 400 (#754)

* Separate video-channel-import and video-channel-sync-latest (#754)

* Bump DB migration version after rebase (#754)

* Prettier states in UI table (#754)

* Add DefaultScope in VideoChannelSyncModel (#754)

* Fix audit logs (#754)

* Ensure user can upload when importing channel + minor fixes (#754)

* Mark synchronization as failed on exception + typos (#754)

* Change REST API for importing videos into channel (#754)

* Add option for fully synchronize a chnanel (#754)

* Return a whole sync object on creation to avoid tricks in Front (#754)

* Various remarks (#754)

* Single quotes by default (#754)

* Rename synchronization to video_channel_synchronization

* Add check.latest_videos_count and max_per_user options (#754)

* Better channel rendering in list #754

* Allow sorting with channel name and state (#754)

* Add missing tests for channel imports (#754)

* Prefer using a parent job for channel sync

* Styling

* Client styling

Co-authored-by: Chocobozzz <me@florianbigard.com>
pull/5190/head
Florent 2022-08-10 09:53:39 +02:00 committed by GitHub
parent 06ac128958
commit 2a491182e4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
90 changed files with 2944 additions and 359 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.html
View File

@ -267,10 +267,10 @@
inputName="importVideosHttpEnabled" formControlName="enabled"
i18n-labelText labelText="Allow import with HTTP URL (e.g. YouTube)"
>
<ng-container ngProjectAs="description">
<span i18n>⚠️ If enabled, we recommend to use <a class="link-orange" href="https://docs.joinpeertube.org/maintain-configuration?id=security">a HTTP proxy</a> to prevent private URL access from your PeerTube server</span>
</ng-container>
</my-peertube-checkbox>
<ng-container ngProjectAs="description">
<span i18n>⚠️ If enabled, we recommend to use <a class="link-orange" href="https://docs.joinpeertube.org/maintain-configuration?id=security">a HTTP proxy</a> to prevent private URL access from your PeerTube server</span>
</ng-container>
</my-peertube-checkbox>
</div>
<div class="form-group" formGroupName="torrent">
@ -285,6 +285,22 @@
</div>
</ng-container>
<ng-container formGroupName="videoChannelSynchronization">
<div class="form-group">
<my-peertube-checkbox
inputName="importSynchronizationEnabled" formControlName="enabled"
i18n-labelText labelText="Allow channel synchronization with channel of other platforms like YouTube (requires allowing import with HTTP URL)"
>
<ng-container ngProjectAs="description">
<span i18n [hidden]="isImportVideosHttpEnabled()">
⛔ You need to allow import with HTTP URL to be able to activate this feature.
</span>
</ng-container>
</my-peertube-checkbox>
</div>
</ng-container>
</ng-container>
<ng-container formGroupName="autoBlacklist">

26
client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.ts
View File

@ -25,11 +25,12 @@ export class EditBasicConfigurationComponent implements OnInit, OnChanges {
private configService: ConfigService,
private menuService: MenuService,
private themeService: ThemeService
) { }
) {}
ngOnInit () {
this.buildLandingPageOptions()
this.checkSignupField()
this.checkImportSyncField()
this.availableThemes = this.themeService.buildAvailableThemes()
}
@ -67,6 +68,14 @@ export class EditBasicConfigurationComponent implements OnInit, OnChanges {
return { 'disabled-checkbox-extra': !this.isSignupEnabled() }
}
isImportVideosHttpEnabled (): boolean {
return this.form.value['import']['videos']['http']['enabled'] === true
}
importSynchronizationChecked () {
return this.isImportVideosHttpEnabled() && this.form.value['import']['videoChannelSynchronization']['enabled']
}
hasUnlimitedSignup () {
return this.form.value['signup']['limit'] === -1
}
@ -97,6 +106,21 @@ export class EditBasicConfigurationComponent implements OnInit, OnChanges {
return this.themeService.getDefaultThemeLabel()
}
private checkImportSyncField () {
const importSyncControl = this.form.get('import.videoChannelSynchronization.enabled')
const importVideosHttpControl = this.form.get('import.videos.http.enabled')
importVideosHttpControl.valueChanges
.subscribe((httpImportEnabled) => {
importSyncControl.setValue(httpImportEnabled && importSyncControl.value)
if (httpImportEnabled) {
importSyncControl.enable()
} else {
importSyncControl.disable()
}
})
}
private checkSignupField () {
const signupControl = this.form.get('signup.enabled')

3
client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts
View File

@ -144,6 +144,9 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit {
torrent: {
enabled: null
}
},
videoChannelSynchronization: {
enabled: null
}
},
trending: {

3
client/src/app/+admin/system/jobs/jobs.component.ts
View File

@ -38,7 +38,8 @@ export class JobsComponent extends RestTable implements OnInit {
'video-redundancy',
'video-transcoding',
'videos-views-stats',
'move-to-object-storage'
'move-to-object-storage',
'video-channel-import'
]
jobs: Job[] = []

2
client/src/app/+manage/video-channel-edit/video-channel-edit.component.html
View File

@ -61,7 +61,7 @@
</div>
<div class="form-group">
<label for="support">Support</label>
<label i18n for="support">Support</label>
<my-help
helpType="markdownEnhanced" i18n-preHtml preHtml="Short text to tell people how they can support the channel (membership platform...).<br /><br />
When a video is uploaded in this channel, the video support field will be automatically filled by this text."

15
client/src/app/+my-library/+my-video-channels/my-video-channels.component.html
View File

@ -1,7 +1,16 @@
<h1>
<my-global-icon iconName="channel" aria-hidden="true"></my-global-icon>
<ng-container i18n>My channels</ng-container>
<span *ngIf="totalItems" class="pt-badge badge-secondary">{{ totalItems }}</span>
<span>
<my-global-icon iconName="channel" aria-hidden="true"></my-global-icon>
<ng-container i18n>My channels</ng-container>
<span *ngIf="totalItems" class="pt-badge badge-secondary">{{ totalItems }}</span>
</span>
<div>
<a routerLink="/my-library/video-channel-syncs" class="button-link">
<my-global-icon iconName="repeat" aria-hidden="true"></my-global-icon>
<ng-container i18n>My synchronizations</ng-container>
</a>
</div>
</h1>
<my-channels-setup-message [hideLink]="true"></my-channels-setup-message>

17
client/src/app/+my-library/+my-video-channels/my-video-channels.component.scss
View File

@ -1,9 +1,20 @@
@use '_variables' as *;
@use '_mixins' as *;
h1 my-global-icon {
position: relative;
top: -2px;
h1 {
display: flex;
justify-content: space-between;
my-global-icon {
position: relative;
top: -2px;
}
.button-link {
@include peertube-button-link;
@include grey-button;
@include button-with-icon(18px, 3px, -1px);
}
}
.create-button {

22
client/src/app/+my-library/my-library-routing.module.ts
View File

@ -6,6 +6,8 @@ import { MySubscriptionsComponent } from './my-follows/my-subscriptions.componen
import { MyHistoryComponent } from './my-history/my-history.component'
import { MyLibraryComponent } from './my-library.component'
import { MyOwnershipComponent } from './my-ownership/my-ownership.component'
import { MyVideoChannelSyncsComponent } from './my-video-channel-syncs/my-video-channel-syncs.component'
import { VideoChannelSyncEditComponent } from './my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component'
import { MyVideoImportsComponent } from './my-video-imports/my-video-imports.component'
import { MyVideoPlaylistCreateComponent } from './my-video-playlists/my-video-playlist-create.component'
import { MyVideoPlaylistElementsComponent } from './my-video-playlists/my-video-playlist-elements.component'
@ -131,6 +133,26 @@ const myLibraryRoutes: Routes = [
key: 'my-videos-history-list'
}
}
},
{
path: 'video-channel-syncs',
component: MyVideoChannelSyncsComponent,
data: {
meta: {
title: $localize`My synchronizations`
}
}
},
{
path: 'video-channel-syncs/create',
component: VideoChannelSyncEditComponent,
data: {
meta: {
title: $localize`Create new synchronization`
}
}
}
]
}

4
client/src/app/+my-library/my-library.module.ts
View File

@ -29,6 +29,8 @@ import { MyVideoPlaylistUpdateComponent } from './my-video-playlists/my-video-pl
import { MyVideoPlaylistsComponent } from './my-video-playlists/my-video-playlists.component'
import { VideoChangeOwnershipComponent } from './my-videos/modals/video-change-ownership.component'
import { MyVideosComponent } from './my-videos/my-videos.component'
import { MyVideoChannelSyncsComponent } from './my-video-channel-syncs/my-video-channel-syncs.component'
import { VideoChannelSyncEditComponent } from './my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component'
@NgModule({
imports: [
@ -63,6 +65,8 @@ import { MyVideosComponent } from './my-videos/my-videos.component'
MyOwnershipComponent,
MyAcceptOwnershipComponent,
MyVideoImportsComponent,
MyVideoChannelSyncsComponent,
VideoChannelSyncEditComponent,
MySubscriptionsComponent,
MyFollowersComponent,
MyHistoryComponent,

83
client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.html Normal file
View File

@ -0,0 +1,83 @@
<div *ngIf="error" class="alert alert-danger">{{ error }}</div>
<h1>
<my-global-icon iconName="refresh" aria-hidden="true"></my-global-icon>
<ng-container i18n>My synchronizations</ng-container>
</h1>
<div *ngIf="!syncEnabled()">
<p class="muted" i18n>⚠️ The instance doesn't allow channel synchronization</p>
</div>
<p-table
*ngIf="syncEnabled()" [value]="channelSyncs" [lazy]="true"
[paginator]="totalRecords > 0" [totalRecords]="totalRecords" [rows]="rowsPerPage" [rowsPerPageOptions]="rowsPerPageOptions"
[sortField]="sort.field" [sortOrder]="sort.order" (onLazyLoad)="loadLazy($event)" dataKey="id"
[showCurrentPageReport]="true" i18n-currentPageReportTemplate
currentPageReportTemplate="Showing {{'{first}'}} to {{'{last}'}} of {{'{totalRecords}'}} synchronizations"
[expandedRowKeys]="expandedRows"
>
<ng-template pTemplate="caption">
<div class="caption">
<div class="left-buttons">
<a class="add-sync" routerLink="{{ getSyncCreateLink() }}">
<my-global-icon iconName="add" aria-hidden="true"></my-global-icon>
<ng-container i18n>Add synchronization</ng-container>
</a>
</div>
</div>
</ng-template>
<ng-template pTemplate="header">
<tr>
<th style="width: 10%"><my-global-icon iconName="columns"></my-global-icon></th>
<th style="width: 25%" i18n pSortableColumn="externalChannelUrl">External Channel <p-sortIcon field="externalChannelUrl"></p-sortIcon></th>
<th style="width: 25%" i18n pSortableColumn="videoChannel">Channel <p-sortIcon field="videoChannel"></p-sortIcon></th>
<th style="width: 10%" i18n pSortableColumn="state">State <p-sortIcon field="state"></p-sortIcon></th>
<th style="width: 10%" i18n pSortableColumn="createdAt">Created <p-sortIcon field="createdAt"></p-sortIcon></th>
<th style="width: 10%" i18n pSortableColumn="lastSyncAt">Last synchronization at <p-sortIcon field="lastSyncAt"></p-sortIcon></th>
</tr>
</ng-template>
<ng-template pTemplate="body" let-expanded="expanded" let-videoChannelSync>
<tr>
<td class="action-cell">
<my-action-dropdown
container="body"
[actions]="videoChannelSyncActions" [entry]="videoChannelSync"
></my-action-dropdown>
</td>
<td>
<a [href]="videoChannelSync.externalChannelUrl" target="_blank" rel="noopener noreferrer">{{ videoChannelSync.externalChannelUrl }}</a>
</td>
<td>
<div class="actor">
<my-actor-avatar
class="channel"
[actor]="videoChannelSync.channel" actorType="channel"
[internalHref]="[ '/c', videoChannelSync.channel.name ]"
size="25"
></my-actor-avatar>
<div class="actor-info">
<a [routerLink]="[ '/c', videoChannelSync.channel.name ]" class="actor-names" i18n-title title="Channel page">
<div class="actor-display-name">{{ videoChannelSync.channel.displayName }}</div>
<div class="actor-name">{{ videoChannelSync.channel.name }}</div>
</a>
</div>
</div>
</td>
<td>
<span [ngClass]="getSyncStateClass(videoChannelSync.state.id)">
{{ videoChannelSync.state.label }}
</span>
</td>
<td>{{ videoChannelSync.createdAt | date: 'short' }}</td>
<td>{{ videoChannelSync.lastSyncAt | date: 'short' }}</td>
</tr>
</ng-template>
</p-table>

14
client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.scss Normal file
View File

@ -0,0 +1,14 @@
@use '_mixins' as *;
@use '_variables' as *;
@use '_actor' as *;
.add-sync {
@include create-button;
}
.actor {
@include actor-row($min-height: auto, $separator: true);
margin-bottom: 0;
padding-bottom: 0;
border: 0;
}

129
client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.ts Normal file
View File

@ -0,0 +1,129 @@
import { Component, OnInit } from '@angular/core'
import { AuthService, Notifier, RestPagination, RestTable, ServerService } from '@app/core'
import { DropdownAction, VideoChannelService, VideoChannelSyncService } from '@app/shared/shared-main'
import { HTMLServerConfig } from '@shared/models/server'
import { VideoChannelSync, VideoChannelSyncState } from '@shared/models/videos'
import { SortMeta } from 'primeng/api'
import { mergeMap } from 'rxjs'
@Component({
templateUrl: './my-video-channel-syncs.component.html',
styleUrls: [ './my-video-channel-syncs.component.scss' ]
})
export class MyVideoChannelSyncsComponent extends RestTable implements OnInit {
error: string
channelSyncs: VideoChannelSync[] = []
totalRecords = 0
videoChannelSyncActions: DropdownAction<VideoChannelSync>[][] = []
sort: SortMeta = { field: 'createdAt', order: 1 }
pagination: RestPagination = { count: this.rowsPerPage, start: 0 }
private static STATE_CLASS_BY_ID = {
[VideoChannelSyncState.FAILED]: 'badge-red',
[VideoChannelSyncState.PROCESSING]: 'badge-blue',
[VideoChannelSyncState.SYNCED]: 'badge-green',
[VideoChannelSyncState.WAITING_FIRST_RUN]: 'badge-yellow'
}
private serverConfig: HTMLServerConfig
constructor (
private videoChannelsSyncService: VideoChannelSyncService,
private serverService: ServerService,
private notifier: Notifier,
private authService: AuthService,
private videoChannelService: VideoChannelService
) {
super()
}
ngOnInit () {
this.serverConfig = this.serverService.getHTMLConfig()
this.initialize()
this.videoChannelSyncActions = [
[
{
label: $localize`Delete`,
iconName: 'delete',
handler: videoChannelSync => this.deleteSync(videoChannelSync)
},
{
label: $localize`Fully synchronize the channel`,
description: $localize`This fetches any missing videos on the local channel`,
iconName: 'refresh',
handler: videoChannelSync => this.fullySynchronize(videoChannelSync)
}
]
]
}
protected reloadData () {
this.error = undefined
this.authService.userInformationLoaded
.pipe(mergeMap(() => {
const user = this.authService.getUser()
return this.videoChannelsSyncService.listAccountVideoChannelsSyncs({
sort: this.sort,
account: user.account,
pagination: this.pagination
})
}))
.subscribe({
next: res => {
this.channelSyncs = res.data
},
error: err => {
this.error = err.message
}
})
}
syncEnabled () {
return this.serverConfig.import.videoChannelSynchronization.enabled
}
deleteSync (videoChannelSync: VideoChannelSync) {
this.videoChannelsSyncService.deleteSync(videoChannelSync.id)
.subscribe({
next: () => {
this.notifier.success($localize`Synchronization removed successfully for ${videoChannelSync.channel.displayName}.`)
this.reloadData()
},
error: err => {
this.error = err.message
}
})
}
fullySynchronize (videoChannelSync: VideoChannelSync) {
this.videoChannelService.importVideos(videoChannelSync.channel.name, videoChannelSync.externalChannelUrl)
.subscribe({
next: () => {
this.notifier.success($localize`Full synchronization requested successfully for ${videoChannelSync.channel.displayName}.`)
},
error: err => {
this.error = err.message
}
})
}
getSyncCreateLink () {
return '/my-library/video-channel-syncs/create'
}
getSyncStateClass (stateId: number) {
return [ 'pt-badge', MyVideoChannelSyncsComponent.STATE_CLASS_BY_ID[stateId] ]
}
getIdentifier () {
return 'MyVideoChannelsSyncComponent'
}
getChannelUrl (name: string) {
return '/c/' + name
}
}

64
client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.html Normal file
View File

@ -0,0 +1,64 @@
<div *ngIf="error" class="alert alert-danger">{{ error }}</div>
<div class="margin-content">
<form role="form" (ngSubmit)="formValidated()" [formGroup]="form">
<div class="row">
<div class="col-12 col-lg-4 col-xl-3">
<div class="video-channel-sync-title" i18n>NEW SYNCHRONIZATION</div>
</div>
<div class="col-12 col-lg-8 col-xl-9">
<div class="form-group">
<label i18n for="externalChannelUrl">Remote channel URL</label>
<div class="input-group">
<input
type="text"
id="externalChannelUrl"
i18n-placeholder
placeholder="Example: https://youtube.com/channel/UC_fancy_channel"
formControlName="externalChannelUrl"
[ngClass]="{ 'input-error': formErrors['externalChannelUrl'] }"
class="form-control"
>
</div>
<div *ngIf="formErrors['externalChannelUrl']" class="form-error">
{{ formErrors['externalChannelUrl'] }}
</div>
</div>
<div class="form-group">
<label i18n for="videoChannel">Video Channel</label>
<my-select-channel required [items]="userVideoChannels" formControlName="videoChannel"></my-select-channel>
<div *ngIf="formErrors['videoChannel']" class="form-error">
{{ formErrors['videoChannel'] }}
</div>
</div>
<div class="form-group">
<label for="existingVideoStrategy" i18n>Options for existing videos on remote channel:</label>
<div class="peertube-radio-container">
<input type="radio" name="existingVideoStrategy" id="import" value="import" formControlName="existingVideoStrategy" required />
<label for="import" i18n>Import all and watch for new publications</label>
</div>
<div class="peertube-radio-container">
<input type="radio" name="existingVideoStrategy" id="doNothing" value="nothing" formControlName="existingVideoStrategy" required />
<label for="doNothing" i18n>Only watch for new publications</label>
</div>
</div>
</div>
</div>
<div class="row"> <!-- submit placement block -->
<div class="col-md-7 col-xl-5"></div>
<div class="col-md-5 col-xl-5 d-inline-flex">
<input type="submit" class="peertube-button orange-button ms-auto" value="{{ getFormButtonTitle() }}" [disabled]="!form.valid">
</div>
</div>
</form>
</div>

17
client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.scss Normal file
View File

@ -0,0 +1,17 @@
@use '_variables' as *;
@use '_mixins' as *;
$form-base-input-width: 480px;
input[type=text] {
@include peertube-input-text($form-base-input-width);
}
.video-channel-sync-title {
@include settings-big-title;
}
my-select-channel {
display: block;
max-width: $form-base-input-width;
}

76
client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.ts Normal file
View File

@ -0,0 +1,76 @@
import { mergeMap } from 'rxjs'
import { SelectChannelItem } from 'src/types'
import { Component, OnInit } from '@angular/core'
import { Router } from '@angular/router'
import { AuthService, Notifier } from '@app/core'
import { listUserChannelsForSelect } from '@app/helpers'
import { VIDEO_CHANNEL_EXTERNAL_URL_VALIDATOR } from '@app/shared/form-validators/video-channel-validators'
import { FormReactive, FormValidatorService } from '@app/shared/shared-forms'
import { VideoChannelService, VideoChannelSyncService } from '@app/shared/shared-main'
import { VideoChannelSyncCreate } from '@shared/models/videos'
@Component({
selector: 'my-video-channel-sync-edit',
templateUrl: './video-channel-sync-edit.component.html',
styleUrls: [ './video-channel-sync-edit.component.scss' ]
})
export class VideoChannelSyncEditComponent extends FormReactive implements OnInit {
error: string
userVideoChannels: SelectChannelItem[] = []
existingVideosStrategy: string
constructor (
protected formValidatorService: FormValidatorService,
private authService: AuthService,
private router: Router,
private notifier: Notifier,
private videoChannelSyncService: VideoChannelSyncService,
private videoChannelService: VideoChannelService
) {
super()
}
ngOnInit () {
this.buildForm({
externalChannelUrl: VIDEO_CHANNEL_EXTERNAL_URL_VALIDATOR,
videoChannel: null,
existingVideoStrategy: null
})
listUserChannelsForSelect(this.authService)
.subscribe(channels => this.userVideoChannels = channels)
}
getFormButtonTitle () {
return $localize`Create`
}
formValidated () {
this.error = undefined
const body = this.form.value
const videoChannelSyncCreate: VideoChannelSyncCreate = {
externalChannelUrl: body.externalChannelUrl,
videoChannelId: body.videoChannel
}
const importExistingVideos = body['existingVideoStrategy'] === 'import'
this.videoChannelSyncService.createSync(videoChannelSyncCreate)
.pipe(mergeMap(({ videoChannelSync }) => {
return importExistingVideos
? this.videoChannelService.importVideos(videoChannelSync.channel.name, videoChannelSync.externalChannelUrl)
: Promise.resolve(null)
}))
.subscribe({
next: () => {
this.notifier.success($localize`Synchronization created successfully.`)
this.router.navigate([ '/my-library', 'video-channel-syncs' ])
},
error: err => {
this.error = err.message
}
})
}
}

13
client/src/app/shared/form-validators/video-channel-validators.ts
View File

@ -48,3 +48,16 @@ export const VIDEO_CHANNEL_SUPPORT_VALIDATOR: BuildFormValidator = {
maxlength: $localize`Support text cannot be more than 1000 characters long.`
}
}
export const VIDEO_CHANNEL_EXTERNAL_URL_VALIDATOR: BuildFormValidator = {
VALIDATORS: [
Validators.required,
Validators.pattern(/^https?:\/\//),
Validators.maxLength(1000)
],
MESSAGES: {
required: $localize`Remote channel url is required.`,
pattern: $localize`External channel URL must begin with "https://" or "http://"`,
maxlength: $localize`External channel URL cannot be more than 1000 characters long`
}
}

7
client/src/app/shared/shared-instance/instance-features-table.component.html
View File

@ -106,6 +106,13 @@
</td>
</tr>
<tr>
<th i18n class="sub-label" scope="row">Channel synchronization with other platforms (YouTube, Vimeo, ...)</th>
<td>
<my-feature-boolean [value]="serverConfig.import.videoChannelSynchronization.enabled"></my-feature-boolean>
</td>
</tr>
<tr>
<th i18n class="label" colspan="2">Search</th>
</tr>

1
client/src/app/shared/shared-main/index.ts
View File

@ -13,3 +13,4 @@ export * from './video'
export * from './video-caption'
export * from './video-channel'
export * from './shared-main.module'
export * from './video-channel-sync'

1
client/src/app/shared/shared-main/video-channel-sync/index.ts Normal file
View File

@ -0,0 +1 @@
export * from './video-channel-sync.service'

50
client/src/app/shared/shared-main/video-channel-sync/video-channel-sync.service.ts Normal file
View File

@ -0,0 +1,50 @@
import { SortMeta } from 'primeng/api'
import { catchError, Observable } from 'rxjs'
import { environment } from 'src/environments/environment'
import { HttpClient, HttpParams } from '@angular/common/http'
import { Injectable } from '@angular/core'
import { RestExtractor, RestPagination, RestService } from '@app/core'
import { ResultList } from '@shared/models/common'
import { VideoChannelSync, VideoChannelSyncCreate } from '@shared/models/videos'
import { Account, AccountService } from '../account'
@Injectable({
providedIn: 'root'
})
export class VideoChannelSyncService {
static BASE_VIDEO_CHANNEL_URL = environment.apiUrl + '/api/v1/video-channel-syncs'
constructor (
private authHttp: HttpClient,
private restExtractor: RestExtractor,
private restService: RestService
) { }
listAccountVideoChannelsSyncs (parameters: {
sort: SortMeta
pagination: RestPagination
account: Account
}): Observable<ResultList<VideoChannelSync>> {
const { pagination, sort, account } = parameters
let params = new HttpParams()
params = this.restService.addRestGetParams(params, pagination, sort)
const url = AccountService.BASE_ACCOUNT_URL + account.nameWithHost + '/video-channel-syncs'
return this.authHttp.get<ResultList<VideoChannelSync>>(url, { params })
.pipe(catchError(err => this.restExtractor.handleError(err)))
}
createSync (body: VideoChannelSyncCreate) {
return this.authHttp.post<{ videoChannelSync: VideoChannelSync }>(VideoChannelSyncService.BASE_VIDEO_CHANNEL_URL, body)
.pipe(catchError(err => this.restExtractor.handleError(err)))
}
deleteSync (videoChannelsSyncId: number) {
const url = `${VideoChannelSyncService.BASE_VIDEO_CHANNEL_URL}/${videoChannelsSyncId}`
return this.authHttp.delete(url)
.pipe(catchError(err => this.restExtractor.handleError(err)))
}
}

6
client/src/app/shared/shared-main/video-channel/video-channel.service.ts
View File

@ -95,4 +95,10 @@ export class VideoChannelService {
return this.authHttp.delete(VideoChannelService.BASE_VIDEO_CHANNEL_URL + videoChannel.nameWithHost)
.pipe(catchError(err => this.restExtractor.handleError(err)))
}
importVideos (videoChannelName: string, externalChannelUrl: string) {
const path = VideoChannelService.BASE_VIDEO_CHANNEL_URL + videoChannelName + '/import-videos'
return this.authHttp.post(path, { externalChannelUrl })
.pipe(catchError(err => this.restExtractor.handleError(err)))
}
}

11
config/default.yaml
View File

@ -546,6 +546,17 @@ import:
# See https://docs.joinpeertube.org/maintain-configuration?id=security for more information
enabled: false
# Add ability for your users to synchronize their channels with external channels, playlists, etc
video_channel_synchronization:
enabled: false
max_per_user: 10
check_interval: 1 hour
# Number of latest published videos to check and to potentially import when syncing a channel
videos_limit_per_synchronization: 10
auto_blacklist:
# New videos automatically blacklisted so moderators can review before publishing
videos:

5
config/dev.yaml
View File

@ -81,6 +81,11 @@ import:
enabled: true
torrent:
enabled: true
video_channel_synchronization:
enabled: true
max_per_user: 10
check_interval: 5 minutes
videos_limit_per_synchronization: 3
instance:
default_nsfw_policy: 'display'

11
config/production.yaml.example
View File

@ -556,6 +556,17 @@ import:
# See https://docs.joinpeertube.org/maintain-configuration?id=security for more information
enabled: false
# Add ability for your users to synchronize their channels with external channels, playlists, etc.
video_channel_synchronization:
enabled: false
max_per_user: 10
check_interval: 1 hour
# Number of latest published videos to check and to potentially import when syncing a channel
videos_limit_per_synchronization: 10
auto_blacklist:
# New videos automatically blacklisted so moderators can review before publishing
videos:

2
server.ts
View File

@ -139,6 +139,7 @@ import { VideoViewsManager } from '@server/lib/views/video-views-manager'
import { isTestOrDevInstance } from './server/helpers/core-utils'
import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics'
import { ApplicationModel } from '@server/models/application/application'
import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
// ----------- Command line -----------
@ -314,6 +315,7 @@ async function startApplication () {
PeerTubeVersionCheckScheduler.Instance.enable()
AutoFollowIndexInstances.Instance.enable()
RemoveDanglingResumableUploadsScheduler.Instance.enable()
VideoChannelSyncLatestScheduler.Instance.enable()
VideoViewsBufferScheduler.Instance.enable()
GeoIPUpdateScheduler.Instance.enable()
OpenTelemetryMetrics.Instance.registerMetrics()

28
server/controllers/api/accounts.ts
View File

@ -25,8 +25,10 @@ import {
accountsFollowersSortValidator,
accountsSortValidator,
ensureAuthUserOwnsAccountValidator,
ensureCanManageUser,
videoChannelsSortValidator,
videoChannelStatsValidator,
videoChannelSyncsSortValidator,
videosSortValidator
} from '../../middlewares/validators'
import { commonVideoPlaylistFiltersValidator, videoPlaylistsSearchValidator } from '../../middlewares/validators/videos/video-playlists'
@ -35,6 +37,7 @@ import { AccountVideoRateModel } from '../../models/account/account-video-rate'
import { VideoModel } from '../../models/video/video'
import { VideoChannelModel } from '../../models/video/video-channel'
import { VideoPlaylistModel } from '../../models/video/video-playlist'
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
const accountsRouter = express.Router()
@ -72,6 +75,17 @@ accountsRouter.get('/:accountName/video-channels',
asyncMiddleware(listAccountChannels)
)
accountsRouter.get('/:accountName/video-channel-syncs',
authenticate,
asyncMiddleware(accountNameWithHostGetValidator),
ensureCanManageUser,
paginationValidator,
videoChannelSyncsSortValidator,
setDefaultSort,
setDefaultPagination,
asyncMiddleware(listAccountChannelsSync)
)
accountsRouter.get('/:accountName/video-playlists',
optionalAuthenticate,
asyncMiddleware(accountNameWithHostGetValidator),
@ -146,6 +160,20 @@ async function listAccountChannels (req: express.Request, res: express.Response)
return res.json(getFormattedObjects(resultList.data, resultList.total))
}
async function listAccountChannelsSync (req: express.Request, res: express.Response) {
const options = {
accountId: res.locals.account.id,
start: req.query.start,
count: req.query.count,
sort: req.query.sort,
search: req.query.search
}
const resultList = await VideoChannelSyncModel.listByAccountForAPI(options)
return res.json(getFormattedObjects(resultList.data, resultList.total))
}
async function listAccountPlaylists (req: express.Request, res: express.Response) {
const serverActor = await getServerActor()

4
server/controllers/api/config.ts
View File

@ -273,6 +273,10 @@ function customConfig (): CustomConfig {
torrent: {
enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
}
},
videoChannelSynchronization: {
enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED,
maxPerUser: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER
}
},
trending: {

2
server/controllers/api/index.ts
View File

@ -20,6 +20,7 @@ import { usersRouter } from './users'
import { videoChannelRouter } from './video-channel'
import { videoPlaylistRouter } from './video-playlist'
import { videosRouter } from './videos'
import { videoChannelSyncRouter } from './video-channel-sync'
const apiRouter = express.Router()
@ -43,6 +44,7 @@ apiRouter.use('/config', configRouter)
apiRouter.use('/users', usersRouter)
apiRouter.use('/accounts', accountsRouter)
apiRouter.use('/video-channels', videoChannelRouter)
apiRouter.use('/video-channel-syncs', videoChannelSyncRouter)
apiRouter.use('/video-playlists', videoPlaylistRouter)
apiRouter.use('/videos', videosRouter)
apiRouter.use('/jobs', jobsRouter)

4
server/controllers/api/server/debug.ts
View File

@ -7,6 +7,7 @@ import { Debug, SendDebugCommand } from '@shared/models'
import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
import { UserRight } from '../../../../shared/models/users'
import { authenticate, ensureUserHasRight } from '../../../middlewares'
import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
const debugRouter = express.Router()
@ -43,7 +44,8 @@ async function runCommand (req: express.Request, res: express.Response) {
const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = {
'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(),
'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(),
'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats()
'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(),
'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute()
}
await processors[body.command]()

76
server/controllers/api/video-channel-sync.ts Normal file
View File

@ -0,0 +1,76 @@
import express from 'express'
import { auditLoggerFactory, getAuditIdFromRes, VideoChannelSyncAuditView } from '@server/helpers/audit-logger'
import { logger } from '@server/helpers/logger'
import {
asyncMiddleware,
asyncRetryTransactionMiddleware,
authenticate,
ensureCanManageChannel as ensureCanManageSyncedChannel,
ensureSyncExists,
ensureSyncIsEnabled,
videoChannelSyncValidator
} from '@server/middlewares'
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
import { MChannelSyncFormattable } from '@server/types/models'
import { HttpStatusCode, VideoChannelSyncState } from '@shared/models'
const videoChannelSyncRouter = express.Router()
const auditLogger = auditLoggerFactory('channel-syncs')
videoChannelSyncRouter.post('/',
authenticate,
ensureSyncIsEnabled,
asyncMiddleware(videoChannelSyncValidator),
ensureCanManageSyncedChannel,
asyncRetryTransactionMiddleware(createVideoChannelSync)
)
videoChannelSyncRouter.delete('/:id',
authenticate,
asyncMiddleware(ensureSyncExists),
ensureCanManageSyncedChannel,
asyncRetryTransactionMiddleware(removeVideoChannelSync)
)
export { videoChannelSyncRouter }
// ---------------------------------------------------------------------------
async function createVideoChannelSync (req: express.Request, res: express.Response) {
const syncCreated: MChannelSyncFormattable = new VideoChannelSyncModel({
externalChannelUrl: req.body.externalChannelUrl,
videoChannelId: req.body.videoChannelId,
state: VideoChannelSyncState.WAITING_FIRST_RUN
})
await syncCreated.save()
syncCreated.VideoChannel = res.locals.videoChannel
auditLogger.create(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncCreated.toFormattedJSON()))
logger.info(
'Video synchronization for channel "%s" with external channel "%s" created.',
syncCreated.VideoChannel.name,
syncCreated.externalChannelUrl
)
return res.json({
videoChannelSync: syncCreated.toFormattedJSON()
})
}
async function removeVideoChannelSync (req: express.Request, res: express.Response) {
const syncInstance = res.locals.videoChannelSync
await syncInstance.destroy()
auditLogger.delete(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncInstance.toFormattedJSON()))
logger.info(
'Video synchronization for channel "%s" with external channel "%s" deleted.',
syncInstance.VideoChannel.name,
syncInstance.externalChannelUrl
)
return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end()
}

28
server/controllers/api/video-channel.ts
View File

@ -36,7 +36,9 @@ import {
videoPlaylistsSortValidator
} from '../../middlewares'
import {
ensureChannelOwnerCanUpload,
ensureIsLocalChannel,
videoChannelImportVideosValidator,
videoChannelsFollowersSortValidator,
videoChannelsListValidator,
videoChannelsNameWithHostValidator,
@ -161,6 +163,16 @@ videoChannelRouter.get('/:nameWithHost/followers',
asyncMiddleware(listVideoChannelFollowers)
)
videoChannelRouter.post('/:nameWithHost/import-videos',
authenticate,
asyncMiddleware(videoChannelsNameWithHostValidator),
videoChannelImportVideosValidator,
ensureIsLocalChannel,
ensureCanManageChannel,
asyncMiddleware(ensureChannelOwnerCanUpload),
asyncMiddleware(importVideosInChannel)
)
// ---------------------------------------------------------------------------
export {
@ -404,3 +416,19 @@ async function listVideoChannelFollowers (req: express.Request, res: express.Res
return res.json(getFormattedObjects(resultList.data, resultList.total))
}
async function importVideosInChannel (req: express.Request, res: express.Response) {
const { externalChannelUrl } = req.body
await JobQueue.Instance.createJob({
type: 'video-channel-import',
payload: {
externalChannelUrl,
videoChannelId: res.locals.videoChannel.id
}
})
logger.info('Video import job for channel "%s" with url "%s" created.', res.locals.videoChannel.name, externalChannelUrl)
return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end()
}

318
server/controllers/api/videos/import.ts
View File

@ -1,49 +1,20 @@
import express from 'express'
import { move, readFile, remove } from 'fs-extra'
import { move, readFile } from 'fs-extra'
import { decode } from 'magnet-uri'
import parseTorrent, { Instance } from 'parse-torrent'
import { join } from 'path'
import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
import { isResolvingToUnicastOnly } from '@server/helpers/dns'
import { Hooks } from '@server/lib/plugins/hooks'
import { ServerConfigManager } from '@server/lib/server-config-manager'
import { setVideoTags } from '@server/lib/video'
import { FilteredModelAttributes } from '@server/types'
import {
MChannelAccountDefault,
MThumbnail,
MUser,
MVideoAccountDefault,
MVideoCaption,
MVideoTag,
MVideoThumbnail,
MVideoWithBlacklistLight
} from '@server/types/models'
import { MVideoImportFormattable } from '@server/types/models/video/video-import'
import {
HttpStatusCode,
ServerErrorCode,
ThumbnailType,
VideoImportCreate,
VideoImportState,
VideoPrivacy,
VideoState
} from '@shared/models'
import { buildYoutubeDLImport, buildVideoFromImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-import'
import { MThumbnail, MVideoThumbnail } from '@server/types/models'
import { HttpStatusCode, ServerErrorCode, ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState } from '@shared/models'
import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger'
import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
import { isArray } from '../../../helpers/custom-validators/misc'
import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils'
import { logger } from '../../../helpers/logger'
import { getSecureTorrentName } from '../../../helpers/utils'
import { YoutubeDLInfo, YoutubeDLWrapper } from '../../../helpers/youtube-dl'
import { CONFIG } from '../../../initializers/config'
import { MIMETYPES } from '../../../initializers/constants'
import { sequelizeTypescript } from '../../../initializers/database'
import { getLocalVideoActivityPubUrl } from '../../../lib/activitypub/url'
import { JobQueue } from '../../../lib/job-queue/job-queue'
import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from '../../../lib/thumbnail'
import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
import { updateVideoMiniatureFromExisting } from '../../../lib/thumbnail'
import {
asyncMiddleware,
asyncRetryTransactionMiddleware,
@ -52,9 +23,6 @@ import {
videoImportCancelValidator,
videoImportDeleteValidator
} from '../../../middlewares'
import { VideoModel } from '../../../models/video/video'
import { VideoCaptionModel } from '../../../models/video/video-caption'
import { VideoImportModel } from '../../../models/video/video-import'
const auditLogger = auditLoggerFactory('video-imports')
const videoImportsRouter = express.Router()
@ -68,7 +36,7 @@ videoImportsRouter.post('/imports',
authenticate,
reqVideoFileImport,
asyncMiddleware(videoImportAddValidator),
asyncRetryTransactionMiddleware(addVideoImport)
asyncRetryTransactionMiddleware(handleVideoImport)
)
videoImportsRouter.post('/imports/:id/cancel',
@ -108,14 +76,14 @@ async function cancelVideoImport (req: express.Request, res: express.Response) {
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
}
function addVideoImport (req: express.Request, res: express.Response) {
if (req.body.targetUrl) return addYoutubeDLImport(req, res)
function handleVideoImport (req: express.Request, res: express.Response) {
if (req.body.targetUrl) return handleYoutubeDlImport(req, res)
const file = req.files?.['torrentfile']?.[0]
if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
if (req.body.magnetUri || file) return handleTorrentImport(req, res, file)
}
async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
async function handleTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
const body: VideoImportCreate = req.body
const user = res.locals.oauth.token.User
@ -135,12 +103,17 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
videoName = result.name
}
const video = await buildVideo(res.locals.videoChannel.id, body, { name: videoName })
const video = await buildVideoFromImport({
channelId: res.locals.videoChannel.id,
importData: { name: videoName },
importDataOverride: body,
importType: 'torrent'
})
const thumbnailModel = await processThumbnail(req, video)
const previewModel = await processPreview(req, video)
const videoImport = await insertIntoDB({
const videoImport = await insertFromImportIntoDB({
video,
thumbnailModel,
previewModel,
@ -155,13 +128,12 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
}
})
// Create job to import the video
const payload = {
const payload: VideoImportPayload = {
type: torrentfile
? 'torrent-file' as 'torrent-file'
: 'magnet-uri' as 'magnet-uri',
? 'torrent-file'
: 'magnet-uri',
videoImportId: videoImport.id,
magnetUri
preventException: false
}
await JobQueue.Instance.createJob({ type: 'video-import', payload })
@ -170,131 +142,49 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
return res.json(videoImport.toFormattedJSON()).end()
}
async function addYoutubeDLImport (req: express.Request, res: express.Response) {
function statusFromYtDlImportError (err: YoutubeDlImportError): number {
switch (err.code) {
case YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL:
return HttpStatusCode.FORBIDDEN_403
case YoutubeDlImportError.CODE.FETCH_ERROR:
return HttpStatusCode.BAD_REQUEST_400
default:
return HttpStatusCode.INTERNAL_SERVER_ERROR_500
}
}
async function handleYoutubeDlImport (req: express.Request, res: express.Response) {
const body: VideoImportCreate = req.body
const targetUrl = body.targetUrl
const user = res.locals.oauth.token.User
const youtubeDL = new YoutubeDLWrapper(
targetUrl,
ServerConfigManager.Instance.getEnabledResolutions('vod'),
CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
)
// Get video infos
let youtubeDLInfo: YoutubeDLInfo
try {
youtubeDLInfo = await youtubeDL.getInfoForDownload()
const { job, videoImport } = await buildYoutubeDLImport({
targetUrl,
channel: res.locals.videoChannel,
importDataOverride: body,
thumbnailFilePath: req.files?.['thumbnailfile']?.[0].path,
previewFilePath: req.files?.['previewfile']?.[0].path,
user
})
await JobQueue.Instance.createJob(job)
auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
return res.json(videoImport.toFormattedJSON()).end()
} catch (err) {
logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
logger.error('An error occurred while importing the video %s. ', targetUrl, { err })
return res.fail({
message: 'Cannot fetch remote information of this URL.',
message: err.message,
status: statusFromYtDlImportError(err),
data: {
targetUrl
}
})
}
if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
return res.fail({
status: HttpStatusCode.FORBIDDEN_403,
message: 'Cannot use non unicast IP as targetUrl.'
})
}
const video = await buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
// Process video thumbnail from request.files
let thumbnailModel = await processThumbnail(req, video)
// Process video thumbnail from url if processing from request.files failed
if (!thumbnailModel && youtubeDLInfo.thumbnailUrl) {
try {
thumbnailModel = await processThumbnailFromUrl(youtubeDLInfo.thumbnailUrl, video)
} catch (err) {
logger.warn('Cannot process thumbnail %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
}
}
// Process video preview from request.files
let previewModel = await processPreview(req, video)
// Process video preview from url if processing from request.files failed
if (!previewModel && youtubeDLInfo.thumbnailUrl) {
try {
previewModel = await processPreviewFromUrl(youtubeDLInfo.thumbnailUrl, video)
} catch (err) {
logger.warn('Cannot process preview %s from youtubedl.', youtubeDLInfo.thumbnailUrl, { err })
}
}
const videoImport = await insertIntoDB({
video,
thumbnailModel,
previewModel,
videoChannel: res.locals.videoChannel,
tags: body.tags || youtubeDLInfo.tags,
user,
videoImportAttributes: {
targetUrl,
state: VideoImportState.PENDING,
userId: user.id
}
})
// Get video subtitles
await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
let fileExt = `.${youtubeDLInfo.ext}`
if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
// Create job to import the video
const payload = {
type: 'youtube-dl' as 'youtube-dl',
videoImportId: videoImport.id,
fileExt
}
await JobQueue.Instance.createJob({ type: 'video-import', payload })
auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON()))
return res.json(videoImport.toFormattedJSON()).end()
}
async function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo): Promise<MVideoThumbnail> {
let videoData = {
name: body.name || importData.name || 'Unknown name',
remote: false,
category: body.category || importData.category,
licence: body.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
language: body.language || importData.language,
commentsEnabled: body.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
downloadEnabled: body.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
waitTranscoding: body.waitTranscoding || false,
state: VideoState.TO_IMPORT,
nsfw: body.nsfw || importData.nsfw || false,
description: body.description || importData.description,
support: body.support || null,
privacy: body.privacy || VideoPrivacy.PRIVATE,
duration: 0, // duration will be set by the import job
channelId,
originallyPublishedAt: body.originallyPublishedAt
? new Date(body.originallyPublishedAt)
: importData.originallyPublishedAt
}
videoData = await Hooks.wrapObject(
videoData,
body.targetUrl
? 'filter:api.video.import-url.video-attribute.result'
: 'filter:api.video.import-torrent.video-attribute.result'
)
const video = new VideoModel(videoData)
video.url = getLocalVideoActivityPubUrl(video)
return video
}
async function processThumbnail (req: express.Request, video: MVideoThumbnail) {
@ -329,69 +219,6 @@ async function processPreview (req: express.Request, video: MVideoThumbnail): Pr
return undefined
}
async function processThumbnailFromUrl (url: string, video: MVideoThumbnail) {
try {
return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.MINIATURE })
} catch (err) {
logger.warn('Cannot generate video thumbnail %s for %s.', url, video.url, { err })
return undefined
}
}
async function processPreviewFromUrl (url: string, video: MVideoThumbnail) {
try {
return updateVideoMiniatureFromUrl({ downloadUrl: url, video, type: ThumbnailType.PREVIEW })
} catch (err) {
logger.warn('Cannot generate video preview %s for %s.', url, video.url, { err })
return undefined
}
}
async function insertIntoDB (parameters: {
video: MVideoThumbnail
thumbnailModel: MThumbnail
previewModel: MThumbnail
videoChannel: MChannelAccountDefault
tags: string[]
videoImportAttributes: FilteredModelAttributes<VideoImportModel>
user: MUser
}): Promise<MVideoImportFormattable> {
const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
const videoImport = await sequelizeTypescript.transaction(async t => {
const sequelizeOptions = { transaction: t }
// Save video object in database
const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
videoCreated.VideoChannel = videoChannel
if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
await autoBlacklistVideoIfNeeded({
video: videoCreated,
user,
notify: false,
isRemote: false,
isNew: true,
transaction: t
})
await setVideoTags({ video: videoCreated, tags, transaction: t })
// Create video import object in database
const videoImport = await VideoImportModel.create(
Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
sequelizeOptions
) as MVideoImportFormattable
videoImport.Video = videoCreated
return videoImport
})
return videoImport
}
async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
const torrentName = torrentfile.originalname
@ -432,46 +259,3 @@ function processMagnetURI (body: VideoImportCreate) {
function extractNameFromArray (name: string | string[]) {
return isArray(name) ? name[0] : name
}
async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
try {
const subtitles = await youtubeDL.getSubtitles()
logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
for (const subtitle of subtitles) {
if (!await isVTTFileValid(subtitle.path)) {
await remove(subtitle.path)
continue
}
const videoCaption = new VideoCaptionModel({
videoId,
language: subtitle.language,
filename: VideoCaptionModel.generateCaptionName(subtitle.language)
}) as MVideoCaption
// Move physical file
await moveAndProcessCaptionFile(subtitle, videoCaption)
await sequelizeTypescript.transaction(async t => {
await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
})
}
} catch (err) {
logger.warn('Cannot get video subtitles.', { err })
}
}
async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
const uniqHosts = new Set(hosts)
for (const h of uniqHosts) {
if (await isResolvingToUnicastOnly(h) !== true) {
return false
}
}
return true
}

17
server/helpers/audit-logger.ts
View File

@ -5,7 +5,7 @@ import { chain } from 'lodash'
import { join } from 'path'
import { addColors, config, createLogger, format, transports } from 'winston'
import { AUDIT_LOG_FILENAME } from '@server/initializers/constants'
import { AdminAbuse, CustomConfig, User, VideoChannel, VideoComment, VideoDetails, VideoImport } from '@shared/models'
import { AdminAbuse, CustomConfig, User, VideoChannel, VideoChannelSync, VideoComment, VideoDetails, VideoImport } from '@shared/models'
import { CONFIG } from '../initializers/config'
import { jsonLoggerFormat, labelFormatter } from './logger'
@ -260,6 +260,18 @@ class CustomConfigAuditView extends EntityAuditView {
}
}
const channelSyncKeysToKeep = [
'id',
'externalChannelUrl',
'channel-id',
'channel-name'
]
class VideoChannelSyncAuditView extends EntityAuditView {
constructor (channelSync: VideoChannelSync) {
super(channelSyncKeysToKeep, 'channelSync', channelSync)
}
}
export {
getAuditIdFromRes,
@ -270,5 +282,6 @@ export {
UserAuditView,
VideoAuditView,
AbuseAuditView,
CustomConfigAuditView
CustomConfigAuditView,
VideoChannelSyncAuditView
}

6
server/helpers/custom-validators/video-channel-syncs.ts Normal file
View File

@ -0,0 +1,6 @@
import { VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants'
import { exists } from './misc'
export function isVideoChannelSyncStateValid (value: any) {
return exists(value) && VIDEO_CHANNEL_SYNC_STATE[value] !== undefined
}

27
server/helpers/youtube-dl/youtube-dl-cli.ts
View File

@ -87,6 +87,7 @@ export class YoutubeDLCLI {
return result.concat([
'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio',
'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
'bestvideo[ext=mp4]+bestaudio[ext=m4a]',
'best' // Ultimate fallback
]).join('/')
}
@ -103,11 +104,14 @@ export class YoutubeDLCLI {
timeout?: number
additionalYoutubeDLArgs?: string[]
}) {
let args = options.additionalYoutubeDLArgs || []
args = args.concat([ '--merge-output-format', 'mp4', '-f', options.format, '-o', options.output ])
return this.run({
url: options.url,
processOptions: options.processOptions,
timeout: options.timeout,
args: (options.additionalYoutubeDLArgs || []).concat([ '-f', options.format, '-o', options.output ])
args
})
}
@ -129,6 +133,25 @@ export class YoutubeDLCLI {
: info
}
getListInfo (options: {
url: string
latestVideosCount?: number
processOptions: execa.NodeOptions
}): Promise<{ upload_date: string, webpage_url: string }[]> {
const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ]
if (options.latestVideosCount !== undefined) {
additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
}
return this.getInfo({
url: options.url,
format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false),
processOptions: options.processOptions,
additionalYoutubeDLArgs
})
}
async getSubs (options: {
url: string
format: 'vtt'
@ -175,7 +198,7 @@ export class YoutubeDLCLI {
const output = await subProcess
logger.debug('Runned youtube-dl command.', { command: output.command, ...lTags() })
logger.debug('Run youtube-dl command.', { command: output.command, ...lTags() })
return output.stdout
? output.stdout.trim().split(/\r?\n/)

4
server/helpers/youtube-dl/youtube-dl-info-builder.ts
View File

@ -13,6 +13,7 @@ type YoutubeDLInfo = {
thumbnailUrl?: string
ext?: string
originallyPublishedAt?: Date
webpageUrl?: string
urls?: string[]
}
@ -81,7 +82,8 @@ class YoutubeDLInfoBuilder {
thumbnailUrl: obj.thumbnail || undefined,
urls: this.buildAvailableUrl(obj),
originallyPublishedAt: this.buildOriginallyPublishedAt(obj),
ext: obj.ext
ext: obj.ext,
webpageUrl: obj.webpage_url
}
}

20
server/helpers/youtube-dl/youtube-dl-wrapper.ts
View File

@ -46,6 +46,24 @@ class YoutubeDLWrapper {
return infoBuilder.getInfo()
}
async getInfoForListImport (options: {
latestVideosCount?: number
}) {
const youtubeDL = await YoutubeDLCLI.safeGet()
const list = await youtubeDL.getListInfo({
url: this.url,
latestVideosCount: options.latestVideosCount,
processOptions
})
return list.map(info => {
const infoBuilder = new YoutubeDLInfoBuilder(info)
return infoBuilder.getInfo()
})
}
async getSubtitles (): Promise<YoutubeDLSubs> {
const cwd = CONFIG.STORAGE.TMP_DIR
@ -103,7 +121,7 @@ class YoutubeDLWrapper {
return remove(path)
})
.catch(innerErr => logger.error('Cannot remove file in youtubeDL timeout.', { innerErr, ...lTags() }))
.catch(innerErr => logger.error('Cannot remove file in youtubeDL error.', { innerErr, ...lTags() }))
throw err
}

7
server/initializers/checker-after-init.ts
View File

@ -48,6 +48,7 @@ function checkConfig () {
checkRemoteRedundancyConfig()
checkStorageConfig()
checkTranscodingConfig()
checkImportConfig()
checkBroadcastMessageConfig()
checkSearchConfig()
checkLiveConfig()
@ -200,6 +201,12 @@ function checkTranscodingConfig () {
}
}
function checkImportConfig () {
if (CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED && !CONFIG.IMPORT.VIDEOS.HTTP) {
throw new Error('You need to enable HTTP import to allow synchronization')
}
}
function checkBroadcastMessageConfig () {
if (CONFIG.BROADCAST_MESSAGE.ENABLED) {
const currentLevel = CONFIG.BROADCAST_MESSAGE.LEVEL

2
server/initializers/checker-before-init.ts
View File

@ -32,6 +32,8 @@ function checkMissedConfig () {
'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p',
'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'video_studio.enabled',
'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout',
'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user',
'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization',
'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days',
'client.videos.miniature.display_author_avatar',
'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth',

9
server/initializers/config.ts
View File

@ -398,6 +398,14 @@ const CONFIG = {
TORRENT: {
get ENABLED () { return config.get<boolean>('import.videos.torrent.enabled') }
}
},
VIDEO_CHANNEL_SYNCHRONIZATION: {
get ENABLED () { return config.get<boolean>('import.video_channel_synchronization.enabled') },
get MAX_PER_USER () { return config.get<number>('import.video_channel_synchronization.max_per_user') },
get CHECK_INTERVAL () { return parseDurationToMs(config.get<string>('import.video_channel_synchronization.check_interval')) },
get VIDEOS_LIMIT_PER_SYNCHRONIZATION () {
return config.get<number>('import.video_channel_synchronization.videos_limit_per_synchronization')
}
}
},
AUTO_BLACKLIST: {
@ -499,6 +507,7 @@ const CONFIG = {
get IS_DEFAULT_SEARCH () { return config.get<boolean>('search.search_index.is_default_search') }
}
}
}
function registerConfigChangedHandler (fun: Function) {

29
server/initializers/constants.ts
View File

@ -6,6 +6,7 @@ import { randomInt, root } from '@shared/core-utils'
import {
AbuseState,
JobType,
VideoChannelSyncState,
VideoImportState,
VideoPrivacy,
VideoRateType,
@ -24,7 +25,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
// ---------------------------------------------------------------------------
const LAST_MIGRATION_VERSION = 725
const LAST_MIGRATION_VERSION = 730
// ---------------------------------------------------------------------------
@ -64,6 +65,7 @@ const SORTABLE_COLUMNS = {
JOBS: [ 'createdAt' ],
VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ],
VIDEO_IMPORTS: [ 'createdAt' ],
VIDEO_CHANNEL_SYNCS: [ 'externalChannelUrl', 'videoChannel', 'createdAt', 'lastSyncAt', 'state' ],
VIDEO_COMMENT_THREADS: [ 'createdAt', 'totalReplies' ],
VIDEO_COMMENTS: [ 'createdAt' ],
@ -156,6 +158,8 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
'video-live-ending': 1,
'video-studio-edition': 1,
'manage-video-torrent': 1,
'video-channel-import': 1,
'after-video-channel-import': 1,
'move-to-object-storage': 3,
'notify': 1,
'federate-video': 1
@ -178,6 +182,8 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im
'video-studio-edition': 1,
'manage-video-torrent': 1,
'move-to-object-storage': 1,
'video-channel-import': 1,
'after-video-channel-import': 1,
'notify': 5,
'federate-video': 3
}
@ -199,9 +205,11 @@ const JOB_TTL: { [id in JobType]: number } = {
'video-redundancy': 1000 * 3600 * 3, // 3 hours
'video-live-ending': 1000 * 60 * 10, // 10 minutes
'manage-video-torrent': 1000 * 3600 * 3, // 3 hours
'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours
'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours
'after-video-channel-import': 60000 * 5, // 5 minutes
'notify': 60000 * 5, // 5 minutes
'federate-video': 60000 * 5, // 5 minutes
'move-to-object-storage': 1000 * 60 * 60 * 3 // 3 hours
'federate-video': 60000 * 5 // 5 minutes
}
const REPEAT_JOBS: { [ id in JobType ]?: RepeatOptions } = {
'videos-views-stats': {
@ -246,7 +254,8 @@ const SCHEDULER_INTERVALS_MS = {
REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day
REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day
UPDATE_INBOX_STATS: 1000 * 60, // 1 minute
REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60 // 1 hour
REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60, // 1 hour
CHANNEL_SYNC_CHECK_INTERVAL: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.CHECK_INTERVAL
}
// ---------------------------------------------------------------------------
@ -276,8 +285,12 @@ const CONSTRAINTS_FIELDS = {
NAME: { min: 1, max: 120 }, // Length
DESCRIPTION: { min: 3, max: 1000 }, // Length
SUPPORT: { min: 3, max: 1000 }, // Length
EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 }, // Length
URL: { min: 3, max: 2000 } // Length
},
VIDEO_CHANNEL_SYNCS: {
EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 } // Length
},
VIDEO_CAPTIONS: {
CAPTION_FILE: {
EXTNAME: [ '.vtt', '.srt' ],
@ -478,6 +491,13 @@ const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = {
[VideoImportState.PROCESSING]: 'Processing'
}
const VIDEO_CHANNEL_SYNC_STATE: { [ id in VideoChannelSyncState ]: string } = {
[VideoChannelSyncState.FAILED]: 'Failed',
[VideoChannelSyncState.SYNCED]: 'Synchronized',
[VideoChannelSyncState.PROCESSING]: 'Processing',
[VideoChannelSyncState.WAITING_FIRST_RUN]: 'Waiting first run'
}
const ABUSE_STATES: { [ id in AbuseState ]: string } = {
[AbuseState.PENDING]: 'Pending',
[AbuseState.REJECTED]: 'Rejected',
@ -1005,6 +1025,7 @@ export {
JOB_COMPLETED_LIFETIME,
HTTP_SIGNATURE,
VIDEO_IMPORT_STATES,
VIDEO_CHANNEL_SYNC_STATE,
VIEW_LIFETIME,
CONTACT_FORM_LIFETIME,
VIDEO_PLAYLIST_PRIVACIES,

4
server/initializers/database.ts
View File

@ -50,6 +50,7 @@ import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-pla
import { VideoTagModel } from '../models/video/video-tag'
import { VideoViewModel } from '../models/view/video-view'
import { CONFIG } from './config'
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
@ -153,7 +154,8 @@ async function initDatabaseModels (silent: boolean) {
VideoTrackerModel,
PluginModel,
ActorCustomPageModel,
VideoJobInfoModel
VideoJobInfoModel,
VideoChannelSyncModel
])
// Check extensions exist in the database

36
server/initializers/migrations/0730-video-channel-sync.ts Normal file
View File

@ -0,0 +1,36 @@
import * as Sequelize from 'sequelize'
async function up (utils: {
transaction: Sequelize.Transaction
queryInterface: Sequelize.QueryInterface
sequelize: Sequelize.Sequelize
db: any
}): Promise<void> {
const query = `
CREATE TABLE IF NOT EXISTS "videoChannelSync" (
"id" SERIAL,
"externalChannelUrl" VARCHAR(2000) NOT NULL DEFAULT NULL,
"videoChannelId" INTEGER NOT NULL REFERENCES "videoChannel" ("id")
ON DELETE CASCADE
ON UPDATE CASCADE,
"state" INTEGER NOT NULL DEFAULT 1,
"createdAt" TIMESTAMP WITH TIME ZONE NOT NULL,
"updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL,
"lastSyncAt" TIMESTAMP WITH TIME ZONE,
PRIMARY KEY ("id")
);
`
await utils.sequelize.query(query, { transaction: utils.transaction })
}
async function down (utils: {
queryInterface: Sequelize.QueryInterface
transaction: Sequelize.Transaction
}) {
await utils.queryInterface.dropTable('videoChannelSync', { transaction: utils.transaction })
}
export {
up,
down
}

37
server/lib/job-queue/handlers/after-video-channel-import.ts Normal file
View File

@ -0,0 +1,37 @@
import { Job } from 'bullmq'
import { logger } from '@server/helpers/logger'
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
import { AfterVideoChannelImportPayload, VideoChannelSyncState, VideoImportPreventExceptionResult } from '@shared/models'
export async function processAfterVideoChannelImport (job: Job) {
const payload = job.data as AfterVideoChannelImportPayload
if (!payload.channelSyncId) return
logger.info('Processing after video channel import in job %s.', job.id)
const sync = await VideoChannelSyncModel.loadWithChannel(payload.channelSyncId)
if (!sync) {
logger.error('Unknown sync id %d.', payload.channelSyncId)
return
}
const childrenValues = await job.getChildrenValues<VideoImportPreventExceptionResult>()
let errors = 0
let successes = 0
for (const value of Object.values(childrenValues)) {
if (value.resultType === 'success') successes++
else if (value.resultType === 'error') errors++
}
if (errors > 0) {
sync.state = VideoChannelSyncState.FAILED
logger.error(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" with failures.`, { errors, successes })
} else {
sync.state = VideoChannelSyncState.SYNCED
logger.info(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" successfully.`, { successes })
}
await sync.save()
}

36
server/lib/job-queue/handlers/video-channel-import.ts Normal file
View File

@ -0,0 +1,36 @@
import { Job } from 'bullmq'
import { logger } from '@server/helpers/logger'
import { CONFIG } from '@server/initializers/config'
import { synchronizeChannel } from '@server/lib/sync-channel'
import { VideoChannelModel } from '@server/models/video/video-channel'
import { VideoChannelImportPayload } from '@shared/models'
export async function processVideoChannelImport (job: Job) {
const payload = job.data as VideoChannelImportPayload
logger.info('Processing video channel import in job %s.', job.id)
// Channel import requires only http upload to be allowed
if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) {
logger.error('Cannot import channel as the HTTP upload is disabled')
return
}
if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
logger.error('Cannot import channel as the synchronization is disabled')
return
}
const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId)
try {
logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `)
await synchronizeChannel({
channel: videoChannel,
externalChannelUrl: payload.externalChannelUrl
})
} catch (err) {
logger.error(`Failed to import channel ${videoChannel.name}`, { err })
}
}

20
server/lib/job-queue/handlers/video-import.ts
View File

@ -8,7 +8,7 @@ import { generateWebTorrentVideoFilename } from '@server/lib/paths'
import { Hooks } from '@server/lib/plugins/hooks'
import { ServerConfigManager } from '@server/lib/server-config-manager'
import { isAbleToUploadVideo } from '@server/lib/user'
import { buildOptimizeOrMergeAudioJob, buildMoveToObjectStorageJob } from '@server/lib/video'
import { buildMoveToObjectStorageJob, buildOptimizeOrMergeAudioJob } from '@server/lib/video'
import { VideoPathManager } from '@server/lib/video-path-manager'
import { buildNextVideoState } from '@server/lib/video-state'
import { ThumbnailModel } from '@server/models/video/thumbnail'
@ -18,6 +18,7 @@ import { isAudioFile } from '@shared/extra-utils'
import {
ThumbnailType,
VideoImportPayload,
VideoImportPreventExceptionResult,
VideoImportState,
VideoImportTorrentPayload,
VideoImportTorrentPayloadType,
@ -41,20 +42,29 @@ import { Notifier } from '../../notifier'
import { generateVideoMiniature } from '../../thumbnail'
import { JobQueue } from '../job-queue'
async function processVideoImport (job: Job) {
async function processVideoImport (job: Job): Promise<VideoImportPreventExceptionResult> {
const payload = job.data as VideoImportPayload
const videoImport = await getVideoImportOrDie(payload)
if (videoImport.state === VideoImportState.CANCELLED) {
logger.info('Do not process import since it has been cancelled', { payload })
return
return { resultType: 'success' }
}
videoImport.state = VideoImportState.PROCESSING
await videoImport.save()
if (payload.type === 'youtube-dl') return processYoutubeDLImport(job, videoImport, payload)
if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') return processTorrentImport(job, videoImport, payload)
try {
if (payload.type === 'youtube-dl') await processYoutubeDLImport(job, videoImport, payload)
if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') await processTorrentImport(job, videoImport, payload)
return { resultType: 'success' }
} catch (err) {
if (!payload.preventException) throw err
logger.warn('Catch error in video import to send value to parent job.', { payload, err })
return { resultType: 'error' }
}
}
// ---------------------------------------------------------------------------

19
server/lib/job-queue/job-queue.ts
View File

@ -22,6 +22,7 @@ import {
ActivitypubHttpFetcherPayload,
ActivitypubHttpUnicastPayload,
ActorKeysPayload,
AfterVideoChannelImportPayload,
DeleteResumableUploadMetaFilePayload,
EmailPayload,
FederateVideoPayload,
@ -31,6 +32,7 @@ import {
MoveObjectStoragePayload,
NotifyPayload,
RefreshPayload,
VideoChannelImportPayload,
VideoFileImportPayload,
VideoImportPayload,
VideoLiveEndingPayload,
@ -53,12 +55,14 @@ import { processFederateVideo } from './handlers/federate-video'
import { processManageVideoTorrent } from './handlers/manage-video-torrent'
import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage'
import { processNotify } from './handlers/notify'
import { processVideoChannelImport } from './handlers/video-channel-import'
import { processVideoFileImport } from './handlers/video-file-import'
import { processVideoImport } from './handlers/video-import'
import { processVideoLiveEnding } from './handlers/video-live-ending'
import { processVideoStudioEdition } from './handlers/video-studio-edition'
import { processVideoTranscoding } from './handlers/video-transcoding'
import { processVideosViewsStats } from './handlers/video-views-stats'
import { processAfterVideoChannelImport } from './handlers/after-video-channel-import'
export type CreateJobArgument =
{ type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@ -79,6 +83,9 @@ export type CreateJobArgument =
{ type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } |
{ type: 'video-studio-edition', payload: VideoStudioEditionPayload } |
{ type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } |
{ type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
{ type: 'video-channel-import', payload: VideoChannelImportPayload } |
{ type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } |
{ type: 'notify', payload: NotifyPayload } |
{ type: 'move-to-object-storage', payload: MoveObjectStoragePayload } |
{ type: 'federate-video', payload: FederateVideoPayload }
@ -106,8 +113,10 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
'video-redundancy': processVideoRedundancy,
'move-to-object-storage': processMoveToObjectStorage,
'manage-video-torrent': processManageVideoTorrent,
'notify': processNotify,
'video-studio-edition': processVideoStudioEdition,
'video-channel-import': processVideoChannelImport,
'after-video-channel-import': processAfterVideoChannelImport,
'notify': processNotify,
'federate-video': processFederateVideo
}
@ -134,6 +143,8 @@ const jobTypes: JobType[] = [
'move-to-object-storage',
'manage-video-torrent',
'video-studio-edition',
'video-channel-import',
'after-video-channel-import',
'notify',
'federate-video'
]
@ -306,7 +317,7 @@ class JobQueue {
.catch(err => logger.error('Cannot create job.', { err, options }))
}
async createJob (options: CreateJobArgument & CreateJobOptions) {
createJob (options: CreateJobArgument & CreateJobOptions) {
const queue: Queue = this.queues[options.type]
if (queue === undefined) {
logger.error('Unknown queue %s: cannot create job.', options.type)
@ -318,7 +329,7 @@ class JobQueue {
return queue.add('job', options.payload, jobOptions)
}
async createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) {
createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) {
let lastJob: FlowJob
for (const job of jobs) {
@ -336,7 +347,7 @@ class JobQueue {
return this.flowProducer.add(lastJob)
}
async createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) {
createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) {
return this.flowProducer.add({
...this.buildJobFlowOption(parent),

61
server/lib/schedulers/video-channel-sync-latest-scheduler.ts Normal file
View File

@ -0,0 +1,61 @@
import { logger } from '@server/helpers/logger'
import { CONFIG } from '@server/initializers/config'
import { VideoChannelModel } from '@server/models/video/video-channel'
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
import { VideoChannelSyncState } from '@shared/models'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
import { synchronizeChannel } from '../sync-channel'
import { AbstractScheduler } from './abstract-scheduler'
export class VideoChannelSyncLatestScheduler extends AbstractScheduler {
private static instance: AbstractScheduler
protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHANNEL_SYNC_CHECK_INTERVAL
private constructor () {
super()
}
protected async internalExecute () {
logger.debug('Running %s.%s', this.constructor.name, this.internalExecute.name)
if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
logger.info('Discard channels synchronization as the feature is disabled')
return
}
const channelSyncs = await VideoChannelSyncModel.listSyncs()
for (const sync of channelSyncs) {
const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
try {
logger.info(
'Creating video import jobs for "%s" sync with external channel "%s"',
channel.Actor.preferredUsername, sync.externalChannelUrl
)
const onlyAfter = sync.lastSyncAt || sync.createdAt
sync.state = VideoChannelSyncState.PROCESSING
sync.lastSyncAt = new Date()
await sync.save()
await synchronizeChannel({
channel,
externalChannelUrl: sync.externalChannelUrl,
videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION,
channelSync: sync,
onlyAfter
})
} catch (err) {
logger.error(`Failed to synchronize channel ${channel.Actor.preferredUsername}`, { err })
sync.state = VideoChannelSyncState.FAILED
await sync.save()
}
}
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}

3
server/lib/server-config-manager.ts
View File

@ -170,6 +170,9 @@ class ServerConfigManager {
torrent: {
enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
}
},
videoChannelSynchronization: {
enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED
}
},
autoBlacklist: {

81
server/lib/sync-channel.ts Normal file
View File

@ -0,0 +1,81 @@
import { logger } from '@server/helpers/logger'
import { YoutubeDLWrapper } from '@server/helpers/youtube-dl'
import { CONFIG } from '@server/initializers/config'
import { buildYoutubeDLImport } from '@server/lib/video-import'
import { UserModel } from '@server/models/user/user'
import { VideoImportModel } from '@server/models/video/video-import'
import { MChannelAccountDefault, MChannelSync } from '@server/types/models'
import { VideoChannelSyncState, VideoPrivacy } from '@shared/models'
import { CreateJobArgument, JobQueue } from './job-queue'
import { ServerConfigManager } from './server-config-manager'
export async function synchronizeChannel (options: {
channel: MChannelAccountDefault
externalChannelUrl: string
channelSync?: MChannelSync
videosCountLimit?: number
onlyAfter?: Date
}) {
const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options
const user = await UserModel.loadByChannelActorId(channel.actorId)
const youtubeDL = new YoutubeDLWrapper(
externalChannelUrl,
ServerConfigManager.Instance.getEnabledResolutions('vod'),
CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
)
const infoList = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
const targetUrls = infoList
.filter(videoInfo => {
if (!onlyAfter) return true
return videoInfo.originallyPublishedAt.getTime() >= onlyAfter.getTime()
})
.map(videoInfo => videoInfo.webpageUrl)
logger.info(
'Fetched %d candidate URLs for sync channel %s.',
targetUrls.length, channel.Actor.preferredUsername, { targetUrls }
)
if (targetUrls.length === 0) {
if (channelSync) {
channelSync.state = VideoChannelSyncState.SYNCED
await channelSync.save()
}
return
}
const children: CreateJobArgument[] = []
for (const targetUrl of targetUrls) {
if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', channel.name, targetUrl)
continue
}
const { job } = await buildYoutubeDLImport({
user,
channel,
targetUrl,
channelSync,
importDataOverride: {
privacy: VideoPrivacy.PUBLIC
}
})
children.push(job)
}
const parent: CreateJobArgument = {
type: 'after-video-channel-import',
payload: {
channelSyncId: channelSync?.id
}
}
await JobQueue.Instance.createJobWithChildren(parent, children)
}

308
server/lib/video-import.ts Normal file
View File

@ -0,0 +1,308 @@
import { remove } from 'fs-extra'
import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils'
import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions'
import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos'
import { isResolvingToUnicastOnly } from '@server/helpers/dns'
import { logger } from '@server/helpers/logger'
import { YoutubeDLInfo, YoutubeDLWrapper } from '@server/helpers/youtube-dl'
import { CONFIG } from '@server/initializers/config'
import { sequelizeTypescript } from '@server/initializers/database'
import { Hooks } from '@server/lib/plugins/hooks'
import { ServerConfigManager } from '@server/lib/server-config-manager'
import { setVideoTags } from '@server/lib/video'
import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
import { VideoModel } from '@server/models/video/video'
import { VideoCaptionModel } from '@server/models/video/video-caption'
import { VideoImportModel } from '@server/models/video/video-import'
import { FilteredModelAttributes } from '@server/types'
import {
MChannelAccountDefault,
MChannelSync,
MThumbnail,
MUser,
MVideoAccountDefault,
MVideoCaption,
MVideoImportFormattable,
MVideoTag,
MVideoThumbnail,
MVideoWithBlacklistLight
} from '@server/types/models'
import { ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState, VideoPrivacy, VideoState } from '@shared/models'
import { getLocalVideoActivityPubUrl } from './activitypub/url'
import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from './thumbnail'
class YoutubeDlImportError extends Error {
code: YoutubeDlImportError.CODE
cause?: Error // Property to remove once ES2022 is used
constructor ({ message, code }) {
super(message)
this.code = code
}
static fromError (err: Error, code: YoutubeDlImportError.CODE, message?: string) {
const ytDlErr = new this({ message: message ?? err.message, code })
ytDlErr.cause = err
ytDlErr.stack = err.stack // Useless once ES2022 is used
return ytDlErr
}
}
namespace YoutubeDlImportError {
export enum CODE {
FETCH_ERROR,
NOT_ONLY_UNICAST_URL
}
}
// ---------------------------------------------------------------------------
async function insertFromImportIntoDB (parameters: {
video: MVideoThumbnail
thumbnailModel: MThumbnail
previewModel: MThumbnail
videoChannel: MChannelAccountDefault
tags: string[]
videoImportAttributes: FilteredModelAttributes<VideoImportModel>
user: MUser
}): Promise<MVideoImportFormattable> {
const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters
const videoImport = await sequelizeTypescript.transaction(async t => {
const sequelizeOptions = { transaction: t }
// Save video object in database
const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag)
videoCreated.VideoChannel = videoChannel
if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
await autoBlacklistVideoIfNeeded({
video: videoCreated,
user,
notify: false,
isRemote: false,
isNew: true,
transaction: t
})
await setVideoTags({ video: videoCreated, tags, transaction: t })
// Create video import object in database
const videoImport = await VideoImportModel.create(
Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
sequelizeOptions
) as MVideoImportFormattable
videoImport.Video = videoCreated
return videoImport
})
return videoImport
}
async function buildVideoFromImport ({ channelId, importData, importDataOverride, importType }: {
channelId: number
importData: YoutubeDLInfo
importDataOverride?: Partial<VideoImportCreate>
importType: 'url' | 'torrent'
}): Promise<MVideoThumbnail> {
let videoData = {
name: importDataOverride?.name || importData.name || 'Unknown name',
remote: false,
category: importDataOverride?.category || importData.category,
licence: importDataOverride?.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE,
language: importDataOverride?.language || importData.language,
commentsEnabled: importDataOverride?.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED,
downloadEnabled: importDataOverride?.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED,
waitTranscoding: importDataOverride?.waitTranscoding || false,
state: VideoState.TO_IMPORT,
nsfw: importDataOverride?.nsfw || importData.nsfw || false,
description: importDataOverride?.description || importData.description,
support: importDataOverride?.support || null,
privacy: importDataOverride?.privacy || VideoPrivacy.PRIVATE,
duration: 0, // duration will be set by the import job
channelId,
originallyPublishedAt: importDataOverride?.originallyPublishedAt
? new Date(importDataOverride?.originallyPublishedAt)
: importData.originallyPublishedAt
}
videoData = await Hooks.wrapObject(
videoData,
importType === 'url'
? 'filter:api.video.import-url.video-attribute.result'
: 'filter:api.video.import-torrent.video-attribute.result'
)
const video = new VideoModel(videoData)
video.url = getLocalVideoActivityPubUrl(video)
return video
}
async function buildYoutubeDLImport (options: {
targetUrl: string
channel: MChannelAccountDefault
user: MUser
channelSync?: MChannelSync
importDataOverride?: Partial<VideoImportCreate>
thumbnailFilePath?: string
previewFilePath?: string
}) {
const { targetUrl, channel, channelSync, importDataOverride, thumbnailFilePath, previewFilePath, user } = options
const youtubeDL = new YoutubeDLWrapper(
targetUrl,
ServerConfigManager.Instance.getEnabledResolutions('vod'),
CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
)
// Get video infos
let youtubeDLInfo: YoutubeDLInfo
try {
youtubeDLInfo = await youtubeDL.getInfoForDownload()
} catch (err) {
throw YoutubeDlImportError.fromError(
err, YoutubeDlImportError.CODE.FETCH_ERROR, `Cannot fetch information from import for URL ${targetUrl}`
)
}
if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
throw new YoutubeDlImportError({
message: 'Cannot use non unicast IP as targetUrl.',
code: YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL
})
}
const video = await buildVideoFromImport({
channelId: channel.id,
importData: youtubeDLInfo,
importDataOverride,
importType: 'url'
})
const thumbnailModel = await forgeThumbnail({
inputPath: thumbnailFilePath,
downloadUrl: youtubeDLInfo.thumbnailUrl,
video,
type: ThumbnailType.MINIATURE
})
const previewModel = await forgeThumbnail({
inputPath: previewFilePath,
downloadUrl: youtubeDLInfo.thumbnailUrl,
video,
type: ThumbnailType.PREVIEW
})
const videoImport = await insertFromImportIntoDB({
video,
thumbnailModel,
previewModel,
videoChannel: channel,
tags: importDataOverride?.tags || youtubeDLInfo.tags,
user,
videoImportAttributes: {
targetUrl,
state: VideoImportState.PENDING,
userId: user.id
}
})
// Get video subtitles
await processYoutubeSubtitles(youtubeDL, targetUrl, video.id)
let fileExt = `.${youtubeDLInfo.ext}`
if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'
const payload: VideoImportPayload = {
type: 'youtube-dl' as 'youtube-dl',
videoImportId: videoImport.id,
fileExt,
// If part of a sync process, there is a parent job that will aggregate children results
preventException: !!channelSync
}
return {
videoImport,
job: { type: 'video-import' as 'video-import', payload }
}
}
// ---------------------------------------------------------------------------
export {
buildYoutubeDLImport,
YoutubeDlImportError,
insertFromImportIntoDB,
buildVideoFromImport
}
// ---------------------------------------------------------------------------
async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: {
inputPath?: string
downloadUrl?: string
video: MVideoThumbnail
type: ThumbnailType
}): Promise<MThumbnail> {
if (inputPath) {
return updateVideoMiniatureFromExisting({
inputPath,
video,
type,
automaticallyGenerated: false
})
} else if (downloadUrl) {
try {
return await updateVideoMiniatureFromUrl({ downloadUrl, video, type })
} catch (err) {
logger.warn('Cannot process thumbnail %s from youtubedl.', downloadUrl, { err })
}
}
return null
}
async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) {
try {
const subtitles = await youtubeDL.getSubtitles()
logger.info('Will create %s subtitles from youtube import %s.', subtitles.length, targetUrl)
for (const subtitle of subtitles) {
if (!await isVTTFileValid(subtitle.path)) {
await remove(subtitle.path)
continue
}
const videoCaption = new VideoCaptionModel({
videoId,
language: subtitle.language,
filename: VideoCaptionModel.generateCaptionName(subtitle.language)
}) as MVideoCaption
// Move physical file
await moveAndProcessCaptionFile(subtitle, videoCaption)
await sequelizeTypescript.transaction(async t => {
await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t)
})
}
} catch (err) {
logger.warn('Cannot get video subtitles.', { err })
}
}
async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) {
const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname)
const uniqHosts = new Set(hosts)
for (const h of uniqHosts) {
if (await isResolvingToUnicastOnly(h) !== true) {
return false
}
}
return true
}

11
server/middlewares/validators/config.ts
View File

@ -66,6 +66,8 @@ const customConfigUpdateValidator = [
body('import.videos.http.enabled').isBoolean().withMessage('Should have a valid import video http enabled boolean'),
body('import.videos.torrent.enabled').isBoolean().withMessage('Should have a valid import video torrent enabled boolean'),
body('import.videoChannelSynchronization.enabled').isBoolean().withMessage('Should have a valid synchronization enabled boolean'),
body('trending.videos.algorithms.default').exists().withMessage('Should have a valid default trending algorithm'),
body('trending.videos.algorithms.enabled').exists().withMessage('Should have a valid array of enabled trending algorithms'),
@ -110,6 +112,7 @@ const customConfigUpdateValidator = [
if (areValidationErrors(req, res)) return
if (!checkInvalidConfigIfEmailDisabled(req.body, res)) return
if (!checkInvalidTranscodingConfig(req.body, res)) return
if (!checkInvalidSynchronizationConfig(req.body, res)) return
if (!checkInvalidLiveConfig(req.body, res)) return
if (!checkInvalidVideoStudioConfig(req.body, res)) return
@ -157,6 +160,14 @@ function checkInvalidTranscodingConfig (customConfig: CustomConfig, res: express
return true
}
function checkInvalidSynchronizationConfig (customConfig: CustomConfig, res: express.Response) {
if (customConfig.import.videoChannelSynchronization.enabled && !customConfig.import.videos.http.enabled) {
res.fail({ message: 'You need to enable HTTP video import in order to enable channel synchronization' })
return false
}
return true
}
function checkInvalidLiveConfig (customConfig: CustomConfig, res: express.Response) {
if (customConfig.live.enabled === false) return true

2
server/middlewares/validators/sort.ts
View File

@ -52,6 +52,7 @@ const videoPlaylistsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_PLAY
const pluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.PLUGINS)
const availablePluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.AVAILABLE_PLUGINS)
const videoRedundanciesSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_REDUNDANCIES)
const videoChannelSyncsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_CHANNEL_SYNCS)
const accountsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNT_FOLLOWERS)
const videoChannelsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.CHANNEL_FOLLOWERS)
@ -84,5 +85,6 @@ export {
videoPlaylistsSearchSortValidator,
accountsFollowersSortValidator,
videoChannelsFollowersSortValidator,
videoChannelSyncsSortValidator,
pluginsSortValidator
}

1
server/middlewares/validators/videos/index.ts
View File

@ -14,3 +14,4 @@ export * from './video-stats'
export * from './video-studio'
export * from './video-transcoding'
export * from './videos'
export * from './video-channel-sync'

66
server/middlewares/validators/videos/video-channel-sync.ts Normal file
View File

@ -0,0 +1,66 @@
import * as express from 'express'
import { body, param } from 'express-validator'
import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
import { logger } from '@server/helpers/logger'
import { CONFIG } from '@server/initializers/config'
import { VideoChannelModel } from '@server/models/video/video-channel'
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models'
import { areValidationErrors, doesVideoChannelIdExist } from '../shared'
export const ensureSyncIsEnabled = (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
return res.fail({
status: HttpStatusCode.FORBIDDEN_403,
message: 'Synchronization is impossible as video channel synchronization is not enabled on the server'
})
}
return next()
}
export const videoChannelSyncValidator = [
body('externalChannelUrl').custom(isUrlValid).withMessage('Should have a valid channel url'),
body('videoChannelId').isInt().withMessage('Should have a valid video channel id'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking videoChannelSync parameters', { parameters: req.body })
if (areValidationErrors(req, res)) return
const body: VideoChannelSyncCreate = req.body
if (!await doesVideoChannelIdExist(body.videoChannelId, res)) return
const count = await VideoChannelSyncModel.countByAccount(res.locals.videoChannel.accountId)
if (count >= CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER) {
return res.fail({
message: `You cannot create more than ${CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER} channel synchronizations`
})
}
return next()
}
]
export const ensureSyncExists = [
param('id').exists().isInt().withMessage('Should have an sync id'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (areValidationErrors(req, res)) return
const syncId = parseInt(req.params.id, 10)
const sync = await VideoChannelSyncModel.loadWithChannel(syncId)
if (!sync) {
return res.fail({
status: HttpStatusCode.NOT_FOUND_404,
message: 'Synchronization not found'
})
}
res.locals.videoChannelSync = sync
res.locals.videoChannel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
return next()
}
]

55
server/middlewares/validators/videos/video-channels.ts
View File

@ -1,5 +1,6 @@
import express from 'express'
import { body, param, query } from 'express-validator'
import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
import { CONFIG } from '@server/initializers/config'
import { MChannelAccountDefault } from '@server/types/models'
import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
@ -13,9 +14,9 @@ import {
import { logger } from '../../../helpers/logger'
import { ActorModel } from '../../../models/actor/actor'
import { VideoChannelModel } from '../../../models/video/video-channel'
import { areValidationErrors, doesVideoChannelNameWithHostExist } from '../shared'
import { areValidationErrors, checkUserQuota, doesVideoChannelNameWithHostExist } from '../shared'
const videoChannelsAddValidator = [
export const videoChannelsAddValidator = [
body('name').custom(isVideoChannelUsernameValid).withMessage('Should have a valid channel name'),
body('displayName').custom(isVideoChannelDisplayNameValid).withMessage('Should have a valid display name'),
body('description').optional().custom(isVideoChannelDescriptionValid).withMessage('Should have a valid description'),
@ -45,7 +46,7 @@ const videoChannelsAddValidator = [
}
]
const videoChannelsUpdateValidator = [
export const videoChannelsUpdateValidator = [
param('nameWithHost').exists().withMessage('Should have an video channel name with host'),
body('displayName')
.optional()
@ -69,7 +70,7 @@ const videoChannelsUpdateValidator = [
}
]
const videoChannelsRemoveValidator = [
export const videoChannelsRemoveValidator = [
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking videoChannelsRemove parameters', { parameters: req.params })
@ -79,7 +80,7 @@ const videoChannelsRemoveValidator = [
}
]
const videoChannelsNameWithHostValidator = [
export const videoChannelsNameWithHostValidator = [
param('nameWithHost').exists().withMessage('Should have an video channel name with host'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
@ -93,7 +94,7 @@ const videoChannelsNameWithHostValidator = [
}
]
const ensureIsLocalChannel = [
export const ensureIsLocalChannel = [
(req: express.Request, res: express.Response, next: express.NextFunction) => {
if (res.locals.videoChannel.Actor.isOwned() === false) {
return res.fail({
@ -106,7 +107,18 @@ const ensureIsLocalChannel = [
}
]
const videoChannelStatsValidator = [
export const ensureChannelOwnerCanUpload = [
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
const channel = res.locals.videoChannel
const user = { id: channel.Account.userId }
if (!await checkUserQuota(user, 1, res)) return
next()
}
]
export const videoChannelStatsValidator = [
query('withStats')
.optional()
.customSanitizer(toBooleanOrNull)
@ -118,7 +130,7 @@ const videoChannelStatsValidator = [
}
]
const videoChannelsListValidator = [
export const videoChannelsListValidator = [
query('search').optional().not().isEmpty().withMessage('Should have a valid search'),
(req: express.Request, res: express.Response, next: express.NextFunction) => {
@ -130,17 +142,24 @@ const videoChannelsListValidator = [
}
]
// ---------------------------------------------------------------------------
export const videoChannelImportVideosValidator = [
body('externalChannelUrl').custom(isUrlValid).withMessage('Should have a valid channel url'),
export {
videoChannelsAddValidator,
videoChannelsUpdateValidator,
videoChannelsRemoveValidator,
videoChannelsNameWithHostValidator,
ensureIsLocalChannel,
videoChannelsListValidator,
videoChannelStatsValidator
}
(req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking videoChannelImport parameters', { parameters: req.body })
if (areValidationErrors(req, res)) return
if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) {
return res.fail({
status: HttpStatusCode.FORBIDDEN_403,
message: 'Channel import is impossible as video upload via HTTP is not enabled on the server'
})
}
return next()
}
]
// ---------------------------------------------------------------------------

11
server/models/utils.ts
View File

@ -117,6 +117,16 @@ function getInstanceFollowsSort (value: string, lastSort: OrderItem = [ 'id', 'A
return getSort(value, lastSort)
}
function getChannelSyncSort (value: string): OrderItem[] {
const { direction, field } = buildDirectionAndField(value)
if (field.toLowerCase() === 'videochannel') {
return [
[ literal('"VideoChannel.name"'), direction ]
]
}
return [ [ field, direction ] ]
}
function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) {
if (!model.createdAt || !model.updatedAt) {
throw new Error('Miss createdAt & updatedAt attributes to model')
@ -280,6 +290,7 @@ export {
getAdminUsersSort,
getVideoSort,
getBlacklistSort,
getChannelSyncSort,
createSimilarityAttribute,
throwIfNotValid,
buildServerIdsFollowedBy,

176
server/models/video/video-channel-sync.ts Normal file
View File

@ -0,0 +1,176 @@
import { Op } from 'sequelize'
import {
AllowNull,
BelongsTo,
Column,
CreatedAt,
DataType,
Default,
DefaultScope,
ForeignKey,
Is,
Model,
Table,
UpdatedAt
} from 'sequelize-typescript'
import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
import { isVideoChannelSyncStateValid } from '@server/helpers/custom-validators/video-channel-syncs'
import { CONSTRAINTS_FIELDS, VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants'
import { MChannelSync, MChannelSyncChannel, MChannelSyncFormattable } from '@server/types/models'
import { VideoChannelSync, VideoChannelSyncState } from '@shared/models'
import { AttributesOnly } from '@shared/typescript-utils'
import { AccountModel } from '../account/account'
import { UserModel } from '../user/user'
import { getChannelSyncSort, throwIfNotValid } from '../utils'
import { VideoChannelModel } from './video-channel'
@DefaultScope(() => ({
include: [
{
model: VideoChannelModel, // Default scope includes avatar and server
required: true
}
]
}))
@Table({
tableName: 'videoChannelSync',
indexes: [
{
fields: [ 'videoChannelId' ]
}
]
})
export class VideoChannelSyncModel extends Model<Partial<AttributesOnly<VideoChannelSyncModel>>> {
@AllowNull(false)
@Default(null)
@Is('VideoChannelExternalChannelUrl', value => throwIfNotValid(value, isUrlValid, 'externalChannelUrl', true))
@Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_CHANNEL_SYNCS.EXTERNAL_CHANNEL_URL.max))
externalChannelUrl: string
@CreatedAt
createdAt: Date
@UpdatedAt
updatedAt: Date
@ForeignKey(() => VideoChannelModel)
@Column
videoChannelId: number
@BelongsTo(() => VideoChannelModel, {
foreignKey: {
allowNull: false
},
onDelete: 'cascade'
})
VideoChannel: VideoChannelModel
@AllowNull(false)
@Default(VideoChannelSyncState.WAITING_FIRST_RUN)
@Is('VideoChannelSyncState', value => throwIfNotValid(value, isVideoChannelSyncStateValid, 'state'))
@Column
state: VideoChannelSyncState
@AllowNull(true)
@Column(DataType.DATE)
lastSyncAt: Date
static listByAccountForAPI (options: {
accountId: number
start: number
count: number
sort: string
}) {
const getQuery = (forCount: boolean) => {
const videoChannelModel = forCount
? VideoChannelModel.unscoped()
: VideoChannelModel
return {
offset: options.start,
limit: options.count,
order: getChannelSyncSort(options.sort),
include: [
{
model: videoChannelModel,
required: true,
where: {
accountId: options.accountId
}
}
]
}
}
return Promise.all([
VideoChannelSyncModel.unscoped().count(getQuery(true)),
VideoChannelSyncModel.unscoped().findAll(getQuery(false))
]).then(([ total, data ]) => ({ total, data }))
}
static countByAccount (accountId: number) {
const query = {
include: [
{
model: VideoChannelModel.unscoped(),
required: true,
where: {
accountId
}
}
]
}
return VideoChannelSyncModel.unscoped().count(query)
}
static loadWithChannel (id: number): Promise<MChannelSyncChannel> {
return VideoChannelSyncModel.findByPk(id)
}
static async listSyncs (): Promise<MChannelSync[]> {
const query = {
include: [
{
model: VideoChannelModel.unscoped(),
required: true,
include: [
{
model: AccountModel.unscoped(),
required: true,
include: [ {
attributes: [],
model: UserModel.unscoped(),
required: true,
where: {
videoQuota: {
[Op.ne]: 0
},
videoQuotaDaily: {
[Op.ne]: 0
}
}
} ]
}
]
}
]
}
return VideoChannelSyncModel.unscoped().findAll(query)
}
toFormattedJSON (this: MChannelSyncFormattable): VideoChannelSync {
return {
id: this.id,
state: {
id: this.state,
label: VIDEO_CHANNEL_SYNC_STATE[this.state]
},
externalChannelUrl: this.externalChannelUrl,
createdAt: this.createdAt.toISOString(),
channel: this.VideoChannel.toFormattedSummaryJSON(),
lastSyncAt: this.lastSyncAt?.toISOString()
}
}
}

24
server/models/video/video-import.ts
View File

@ -1,4 +1,4 @@
import { WhereOptions } from 'sequelize'
import { Op, WhereOptions } from 'sequelize'
import {
AfterUpdate,
AllowNull,
@ -161,6 +161,28 @@ export class VideoImportModel extends Model<Partial<AttributesOnly<VideoImportMo
]).then(([ total, data ]) => ({ total, data }))
}
static async urlAlreadyImported (channelId: number, targetUrl: string): Promise<boolean> {
const element = await VideoImportModel.unscoped().findOne({
where: {
targetUrl,
state: {
[Op.in]: [ VideoImportState.PENDING, VideoImportState.PROCESSING, VideoImportState.SUCCESS ]
}
},
include: [
{
model: VideoModel,
required: true,
where: {
channelId
}
}
]
})
return !!element
}
getTargetIdentifier () {
return this.targetUrl || this.magnetUri || this.torrentName
}

29
server/tests/api/check-params/config.ts
View File

@ -1,7 +1,8 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import { omit } from 'lodash'
import { merge, omit } from 'lodash'
import { CustomConfig, HttpStatusCode } from '@shared/models'
import {
cleanupTests,
createSingleServer,
@ -11,7 +12,6 @@ import {
PeerTubeServer,
setAccessTokensToServers
} from '@shared/server-commands'
import { CustomConfig, HttpStatusCode } from '@shared/models'
describe('Test config API validators', function () {
const path = '/api/v1/config/custom'
@ -162,6 +162,10 @@ describe('Test config API validators', function () {
torrent: {
enabled: false
}
},
videoChannelSynchronization: {
enabled: false,
maxPerUser: 10
}
},
trending: {
@ -346,7 +350,26 @@ describe('Test config API validators', function () {
})
})
it('Should success with the correct parameters', async function () {
it('Should fail with a disabled http upload & enabled sync', async function () {
const newUpdateParams: CustomConfig = merge({}, updateParams, {
import: {
videos: {
http: { enabled: false }
},
videoChannelSynchronization: { enabled: true }
}
})
await makePutBodyRequest({
url: server.url,
path,
fields: newUpdateParams,
token: server.accessToken,
expectedStatus: HttpStatusCode.BAD_REQUEST_400
})
})
it('Should succeed with the correct parameters', async function () {
await makePutBodyRequest({
url: server.url,
path,

1
server/tests/api/check-params/index.ts
View File

@ -27,6 +27,7 @@ import './video-channels'
import './video-comments'
import './video-files'
import './video-imports'
import './video-channel-syncs'
import './video-playlists'
import './video-source'
import './video-studio'

2
server/tests/api/check-params/upload-quota.ts
View File

@ -70,7 +70,7 @@ describe('Test upload quota', function () {
})
it('Should fail to import with HTTP/Torrent/magnet', async function () {
this.timeout(120000)
this.timeout(120_000)
const baseAttributes = {
channelId: server.store.channel.id,

318
server/tests/api/check-params/video-channel-syncs.ts Normal file
View File

@ -0,0 +1,318 @@
import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, FIXTURE_URLS } from '@server/tests/shared'
import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models'
import {
ChannelSyncsCommand,
createSingleServer,
makePostBodyRequest,
PeerTubeServer,
setAccessTokensToServers,
setDefaultVideoChannel
} from '@shared/server-commands'
describe('Test video channel sync API validator', () => {
const path = '/api/v1/video-channel-syncs'
let server: PeerTubeServer
let command: ChannelSyncsCommand
let rootChannelId: number
let rootChannelSyncId: number
const userInfo = {
accessToken: '',
username: 'user1',
id: -1,
channelId: -1,
syncId: -1
}
async function withChannelSyncDisabled<T> (callback: () => Promise<T>): Promise<void> {
try {
await server.config.disableChannelSync()
await callback()
} finally {
await server.config.enableChannelSync()
}
}
async function withMaxSyncsPerUser<T> (maxSync: number, callback: () => Promise<T>): Promise<void> {
const origConfig = await server.config.getCustomConfig()
await server.config.updateExistingSubConfig({
newConfig: {
import: {
videoChannelSynchronization: {
maxPerUser: maxSync
}
}
}
})
try {
await callback()
} finally {
await server.config.updateCustomConfig({ newCustomConfig: origConfig })
}
}
before(async function () {
this.timeout(30_000)
server = await createSingleServer(1)
await setAccessTokensToServers([ server ])
await setDefaultVideoChannel([ server ])
command = server.channelSyncs
rootChannelId = server.store.channel.id
{
userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username)
const { videoChannels, id: userId } = await server.users.getMyInfo({ token: userInfo.accessToken })
userInfo.id = userId
userInfo.channelId = videoChannels[0].id
}
await server.config.enableChannelSync()
})
describe('When creating a sync', function () {
let baseCorrectParams: VideoChannelSyncCreate
before(function () {
baseCorrectParams = {
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
videoChannelId: rootChannelId
}
})
it('Should fail when sync is disabled', async function () {
await withChannelSyncDisabled(async () => {
await command.create({
token: server.accessToken,
attributes: baseCorrectParams,
expectedStatus: HttpStatusCode.FORBIDDEN_403
})
})
})
it('Should fail with nothing', async function () {
const fields = {}
await makePostBodyRequest({
url: server.url,
path,
token: server.accessToken,
fields,
expectedStatus: HttpStatusCode.BAD_REQUEST_400
})
})
it('Should fail with no authentication', async function () {
await command.create({
token: null,
attributes: baseCorrectParams,
expectedStatus: HttpStatusCode.UNAUTHORIZED_401
})
})
it('Should fail without a target url', async function () {
const attributes: VideoChannelSyncCreate = {
...baseCorrectParams,
externalChannelUrl: null
}
await command.create({
token: server.accessToken,
attributes,
expectedStatus: HttpStatusCode.BAD_REQUEST_400
})
})
it('Should fail without a channelId', async function () {
const attributes: VideoChannelSyncCreate = {
...baseCorrectParams,
videoChannelId: null
}
await command.create({
token: server.accessToken,
attributes,
expectedStatus: HttpStatusCode.BAD_REQUEST_400
})
})
it('Should fail with a channelId refering nothing', async function () {
const attributes: VideoChannelSyncCreate = {
...baseCorrectParams,
videoChannelId: 42
}
await command.create({
token: server.accessToken,
attributes,
expectedStatus: HttpStatusCode.NOT_FOUND_404
})
})
it('Should fail to create a sync when the user does not own the channel', async function () {
await command.create({
token: userInfo.accessToken,
attributes: baseCorrectParams,
expectedStatus: HttpStatusCode.FORBIDDEN_403
})
})
it('Should succeed to create a sync with root and for another user\'s channel', async function () {
const { videoChannelSync } = await command.create({
token: server.accessToken,
attributes: {
...baseCorrectParams,
videoChannelId: userInfo.channelId
},
expectedStatus: HttpStatusCode.OK_200
})
userInfo.syncId = videoChannelSync.id
})
it('Should succeed with the correct parameters', async function () {
const { videoChannelSync } = await command.create({
token: server.accessToken,
attributes: baseCorrectParams,
expectedStatus: HttpStatusCode.OK_200
})
rootChannelSyncId = videoChannelSync.id
})
it('Should fail when the user exceeds allowed number of synchronizations', async function () {
await withMaxSyncsPerUser(1, async () => {
await command.create({
token: server.accessToken,
attributes: {
...baseCorrectParams,
videoChannelId: userInfo.channelId
},
expectedStatus: HttpStatusCode.BAD_REQUEST_400
})
})
})
})
describe('When listing my channel syncs', function () {
const myPath = '/api/v1/accounts/root/video-channel-syncs'
it('Should fail with a bad start pagination', async function () {
await checkBadStartPagination(server.url, myPath, server.accessToken)
})
it('Should fail with a bad count pagination', async function () {
await checkBadCountPagination(server.url, myPath, server.accessToken)
})
it('Should fail with an incorrect sort', async function () {
await checkBadSortPagination(server.url, myPath, server.accessToken)
})
it('Should succeed with the correct parameters', async function () {
await command.listByAccount({
accountName: 'root',
token: server.accessToken,
expectedStatus: HttpStatusCode.OK_200
})
})
it('Should fail with no authentication', async function () {
await command.listByAccount({
accountName: 'root',
token: null,
expectedStatus: HttpStatusCode.UNAUTHORIZED_401
})
})
it('Should fail when a simple user lists another user\'s synchronizations', async function () {
await command.listByAccount({
accountName: 'root',
token: userInfo.accessToken,
expectedStatus: HttpStatusCode.FORBIDDEN_403
})
})
it('Should succeed when root lists another user\'s synchronizations', async function () {
await command.listByAccount({
accountName: userInfo.username,
token: server.accessToken,
expectedStatus: HttpStatusCode.OK_200
})
})
it('Should succeed even with synchronization disabled', async function () {
await withChannelSyncDisabled(async function () {
await command.listByAccount({
accountName: 'root',
token: server.accessToken,
expectedStatus: HttpStatusCode.OK_200
})
})
})
})
describe('When triggering deletion', function () {
it('should fail with no authentication', async function () {
await command.delete({
channelSyncId: userInfo.syncId,
token: null,
expectedStatus: HttpStatusCode.UNAUTHORIZED_401
})
})
it('Should fail when channelSyncId does not refer to any sync', async function () {
await command.delete({
channelSyncId: 42,
token: server.accessToken,
expectedStatus: HttpStatusCode.NOT_FOUND_404
})
})
it('Should fail when sync is not owned by the user', async function () {
await command.delete({
channelSyncId: rootChannelSyncId,
token: userInfo.accessToken,
expectedStatus: HttpStatusCode.FORBIDDEN_403
})
})
it('Should succeed when root delete a sync they do not own', async function () {
await command.delete({
channelSyncId: userInfo.syncId,
token: server.accessToken,
expectedStatus: HttpStatusCode.NO_CONTENT_204
})
})
it('should succeed when user delete a sync they own', async function () {
const { videoChannelSync } = await command.create({
attributes: {
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
videoChannelId: userInfo.channelId
},
token: server.accessToken,
expectedStatus: HttpStatusCode.OK_200
})
await command.delete({
channelSyncId: videoChannelSync.id,
token: server.accessToken,
expectedStatus: HttpStatusCode.NO_CONTENT_204
})
})
it('Should succeed even when synchronization is disabled', async function () {
await withChannelSyncDisabled(async function () {
await command.delete({
channelSyncId: rootChannelSyncId,
token: server.accessToken,
expectedStatus: HttpStatusCode.NO_CONTENT_204
})
})
})
})
after(async function () {
await server?.kill()
})
})

134
server/tests/api/check-params/video-channels.ts
View File

@ -3,8 +3,8 @@
import 'mocha'
import * as chai from 'chai'
import { omit } from 'lodash'
import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared'
import { buildAbsoluteFixturePath } from '@shared/core-utils'
import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, FIXTURE_URLS } from '@server/tests/shared'
import { areHttpImportTestsDisabled, buildAbsoluteFixturePath } from '@shared/core-utils'
import { HttpStatusCode, VideoChannelUpdate } from '@shared/models'
import {
ChannelsCommand,
@ -23,7 +23,13 @@ const expect = chai.expect
describe('Test video channels API validator', function () {
const videoChannelPath = '/api/v1/video-channels'
let server: PeerTubeServer
let accessTokenUser: string
const userInfo = {
accessToken: '',
channelName: 'fake_channel',
id: -1,
videoQuota: -1,
videoQuotaDaily: -1
}
let command: ChannelsCommand
// ---------------------------------------------------------------
@ -35,14 +41,15 @@ describe('Test video channels API validator', function () {
await setAccessTokensToServers([ server ])
const user = {
const userCreds = {
username: 'fake',
password: 'fake_password'
}
{
await server.users.create({ username: user.username, password: user.password })
accessTokenUser = await server.login.getAccessToken(user)
const user = await server.users.create({ username: userCreds.username, password: userCreds.password })
userInfo.id = user.id
userInfo.accessToken = await server.login.getAccessToken(userCreds)
}
command = server.channels
@ -191,7 +198,7 @@ describe('Test video channels API validator', function () {
await makePutBodyRequest({
url: server.url,
path,
token: accessTokenUser,
token: userInfo.accessToken,
fields: baseCorrectParams,
expectedStatus: HttpStatusCode.FORBIDDEN_403
})
@ -339,7 +346,7 @@ describe('Test video channels API validator', function () {
})
it('Should fail with a another user', async function () {
await makeGetRequest({ url: server.url, path, token: accessTokenUser, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeGetRequest({ url: server.url, path, token: userInfo.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
})
it('Should succeed with the correct params', async function () {
@ -347,13 +354,122 @@ describe('Test video channels API validator', function () {
})
})
describe('When triggering full synchronization', function () {
it('Should fail when HTTP upload is disabled', async function () {
await server.config.disableImports()
await command.importVideos({
channelName: 'super_channel',
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
token: server.accessToken,
expectedStatus: HttpStatusCode.FORBIDDEN_403
})
await server.config.enableImports()
})
it('Should fail when externalChannelUrl is not provided', async function () {
await command.importVideos({
channelName: 'super_channel',
externalChannelUrl: null,
token: server.accessToken,
expectedStatus: HttpStatusCode.BAD_REQUEST_400
})
})
it('Should fail when externalChannelUrl is malformed', async function () {
await command.importVideos({
channelName: 'super_channel',
externalChannelUrl: 'not-a-url',
token: server.accessToken,
expectedStatus: HttpStatusCode.BAD_REQUEST_400
})
})
it('Should fail with no authentication', async function () {
await command.importVideos({
channelName: 'super_channel',
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
token: null,
expectedStatus: HttpStatusCode.UNAUTHORIZED_401
})
})
it('Should fail when sync is not owned by the user', async function () {
await command.importVideos({
channelName: 'super_channel',
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
token: userInfo.accessToken,
expectedStatus: HttpStatusCode.FORBIDDEN_403
})
})
it('Should fail when the user has no quota', async function () {
await server.users.update({
userId: userInfo.id,
videoQuota: 0
})
await command.importVideos({
channelName: 'fake_channel',
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
token: userInfo.accessToken,
expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413
})
await server.users.update({
userId: userInfo.id,
videoQuota: userInfo.videoQuota
})
})
it('Should fail when the user has no daily quota', async function () {
await server.users.update({
userId: userInfo.id,
videoQuotaDaily: 0
})
await command.importVideos({
channelName: 'fake_channel',
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
token: userInfo.accessToken,
expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413
})
await server.users.update({
userId: userInfo.id,
videoQuotaDaily: userInfo.videoQuotaDaily
})
})
it('Should succeed when sync is run by its owner', async function () {
if (!areHttpImportTestsDisabled()) return
await command.importVideos({
channelName: 'fake_channel',
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
token: userInfo.accessToken
})
})
it('Should succeed when sync is run with root and for another user\'s channel', async function () {
if (!areHttpImportTestsDisabled()) return
await command.importVideos({
channelName: 'fake_channel',
externalChannelUrl: FIXTURE_URLS.youtubeChannel
})
})
})
describe('When deleting a video channel', function () {
it('Should fail with a non authenticated user', async function () {
await command.delete({ token: 'coucou', channelName: 'super_channel', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
})
it('Should fail with another authenticated user', async function () {
await command.delete({ token: accessTokenUser, channelName: 'super_channel', expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await command.delete({ token: userInfo.accessToken, channelName: 'super_channel', expectedStatus: HttpStatusCode.FORBIDDEN_403 })
})
it('Should fail with an unknown video channel id', async function () {

8
server/tests/api/check-params/video-imports.ts
View File

@ -88,7 +88,13 @@ describe('Test video imports API validator', function () {
it('Should fail with nothing', async function () {
const fields = {}
await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
await makePostBodyRequest({
url: server.url,
path,
token: server.accessToken,
fields,
expectedStatus: HttpStatusCode.BAD_REQUEST_400
})
})
it('Should fail without a target url', async function () {

4
server/tests/api/server/config.ts
View File

@ -368,6 +368,10 @@ const newCustomConfig: CustomConfig = {
torrent: {
enabled: false
}
},
videoChannelSynchronization: {
enabled: false,
maxPerUser: 10
}
},
trending: {

50
server/tests/api/videos/channel-import-videos.ts Normal file
View File

@ -0,0 +1,50 @@
import { expect } from 'chai'
import { FIXTURE_URLS } from '@server/tests/shared'
import { areHttpImportTestsDisabled } from '@shared/core-utils'
import {
createSingleServer,
getServerImportConfig,
PeerTubeServer,
setAccessTokensToServers,
setDefaultVideoChannel,
waitJobs
} from '@shared/server-commands'
describe('Test videos import in a channel', function () {
if (areHttpImportTestsDisabled()) return
function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
describe('Import using ' + mode, function () {
let server: PeerTubeServer
before(async function () {
this.timeout(120_000)
server = await createSingleServer(1, getServerImportConfig(mode))
await setAccessTokensToServers([ server ])
await setDefaultVideoChannel([ server ])
await server.config.enableChannelSync()
})
it('Should import a whole channel', async function () {
this.timeout(240_000)
await server.channels.importVideos({ channelName: server.store.channel.name, externalChannelUrl: FIXTURE_URLS.youtubeChannel })
await waitJobs(server)
const videos = await server.videos.listByChannel({ handle: server.store.channel.name })
expect(videos.total).to.equal(2)
})
after(async function () {
await server?.kill()
})
})
}
runSuite('yt-dlp')
runSuite('youtube-dl')
})

2
server/tests/api/videos/index.ts
View File

@ -4,6 +4,8 @@ import './single-server'
import './video-captions'
import './video-change-ownership'
import './video-channels'
import './channel-import-videos'
import './video-channel-syncs'
import './video-comments'
import './video-description'
import './video-files'

226
server/tests/api/videos/video-channel-syncs.ts Normal file
View File

@ -0,0 +1,226 @@
import 'mocha'
import { expect } from 'chai'
import { FIXTURE_URLS } from '@server/tests/shared'
import { areHttpImportTestsDisabled } from '@shared/core-utils'
import { HttpStatusCode, VideoChannelSyncState, VideoInclude, VideoPrivacy } from '@shared/models'
import {
ChannelSyncsCommand,
createSingleServer,
getServerImportConfig,
PeerTubeServer,
setAccessTokensToServers,
setDefaultAccountAvatar,
setDefaultChannelAvatar,
setDefaultVideoChannel,
waitJobs
} from '@shared/server-commands'
describe('Test channel synchronizations', function () {
if (areHttpImportTestsDisabled()) return
function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
describe('Sync using ' + mode, function () {
let server: PeerTubeServer
let command: ChannelSyncsCommand
let startTestDate: Date
const userInfo = {
accessToken: '',
username: 'user1',
channelName: 'user1_channel',
channelId: -1,
syncId: -1
}
async function changeDateForSync (channelSyncId: number, newDate: string) {
await server.sql.updateQuery(
`UPDATE "videoChannelSync" ` +
`SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` +
`WHERE id=${channelSyncId}`
)
}
before(async function () {
this.timeout(120_000)
startTestDate = new Date()
server = await createSingleServer(1, getServerImportConfig(mode))
await setAccessTokensToServers([ server ])
await setDefaultVideoChannel([ server ])
await setDefaultChannelAvatar([ server ])
await setDefaultAccountAvatar([ server ])
await server.config.enableChannelSync()
command = server.channelSyncs
{
userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username)
const { videoChannels } = await server.users.getMyInfo({ token: userInfo.accessToken })
userInfo.channelId = videoChannels[0].id
}
})
it('Should fetch the latest channel videos of a remote channel', async function () {
this.timeout(120_000)
{
const { video } = await server.imports.importVideo({
attributes: {
channelId: server.store.channel.id,
privacy: VideoPrivacy.PUBLIC,
targetUrl: FIXTURE_URLS.youtube
}
})
expect(video.name).to.equal('small video - youtube')
const { total } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
expect(total).to.equal(1)
}
const { videoChannelSync } = await command.create({
attributes: {
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
videoChannelId: server.store.channel.id
},
token: server.accessToken,
expectedStatus: HttpStatusCode.OK_200
})
// Ensure any missing video not already fetched will be considered as new
await changeDateForSync(videoChannelSync.id, '1970-01-01')
await server.debug.sendCommand({
body: {
command: 'process-video-channel-sync-latest'
}
})
{
await waitJobs(server)
const { total, data } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
expect(total).to.equal(2)
expect(data[0].name).to.equal('test')
}
})
it('Should add another synchronization', async function () {
const externalChannelUrl = FIXTURE_URLS.youtubeChannel + '?foo=bar'
const { videoChannelSync } = await command.create({
attributes: {
externalChannelUrl,
videoChannelId: server.store.channel.id
},
token: server.accessToken,
expectedStatus: HttpStatusCode.OK_200
})
expect(videoChannelSync.externalChannelUrl).to.equal(externalChannelUrl)
expect(videoChannelSync.channel).to.include({
id: server.store.channel.id,
name: 'root_channel'
})
expect(videoChannelSync.state.id).to.equal(VideoChannelSyncState.WAITING_FIRST_RUN)
expect(new Date(videoChannelSync.createdAt)).to.be.above(startTestDate).and.to.be.at.most(new Date())
})
it('Should add a synchronization for another user', async function () {
const { videoChannelSync } = await command.create({
attributes: {
externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
videoChannelId: userInfo.channelId
},
token: userInfo.accessToken
})
userInfo.syncId = videoChannelSync.id
})
it('Should not import a channel if not asked', async function () {
await waitJobs(server)
const { data } = await command.listByAccount({ accountName: userInfo.username })
expect(data[0].state).to.contain({
id: VideoChannelSyncState.WAITING_FIRST_RUN,
label: 'Waiting first run'
})
})
it('Should only fetch the videos newer than the creation date', async function () {
this.timeout(120_000)
await changeDateForSync(userInfo.syncId, '2019-03-01')
await server.debug.sendCommand({
body: {
command: 'process-video-channel-sync-latest'
}
})
await waitJobs(server)
const { data, total } = await server.videos.listByChannel({
handle: userInfo.channelName,
include: VideoInclude.NOT_PUBLISHED_STATE
})
expect(total).to.equal(1)
expect(data[0].name).to.equal('test')
})
it('Should list channel synchronizations', async function () {
// Root
{
const { total, data } = await command.listByAccount({ accountName: 'root' })
expect(total).to.equal(2)
expect(data[0]).to.deep.contain({
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
state: {
id: VideoChannelSyncState.SYNCED,
label: 'Synchronized'
}
})
expect(new Date(data[0].lastSyncAt)).to.be.greaterThan(startTestDate)
expect(data[0].channel).to.contain({ id: server.store.channel.id })
expect(data[1]).to.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?foo=bar' })
}
// User
{
const { total, data } = await command.listByAccount({ accountName: userInfo.username })
expect(total).to.equal(1)
expect(data[0]).to.deep.contain({
externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
state: {
id: VideoChannelSyncState.SYNCED,
label: 'Synchronized'
}
})
}
})
it('Should remove user\'s channel synchronizations', async function () {
await command.delete({ channelSyncId: userInfo.syncId })
const { total } = await command.listByAccount({ accountName: userInfo.username })
expect(total).to.equal(0)
})
after(async function () {
await server?.kill()
})
})
}
runSuite('youtube-dl')
runSuite('yt-dlp')
})

20
server/tests/api/videos/video-imports.ts
View File

@ -12,6 +12,7 @@ import {
createMultipleServers,
createSingleServer,
doubleFollow,
getServerImportConfig,
PeerTubeServer,
setAccessTokensToServers,
setDefaultVideoChannel,
@ -84,24 +85,9 @@ describe('Test video imports', function () {
let servers: PeerTubeServer[] = []
before(async function () {
this.timeout(30_000)
this.timeout(60_000)
// Run servers
servers = await createMultipleServers(2, {
import: {
videos: {
http: {
youtube_dl_release: {
url: mode === 'youtube-dl'
? 'https://yt-dl.org/downloads/latest/youtube-dl'
: 'https://api.github.com/repos/yt-dlp/yt-dlp/releases',
name: mode
}
}
}
}
})
servers = await createMultipleServers(2, getServerImportConfig(mode))
await setAccessTokensToServers(servers)
await setDefaultVideoChannel(servers)

2
server/tests/shared/tests.ts
View File

@ -16,6 +16,8 @@ const FIXTURE_URLS = {
*/
youtubeHDR: 'https://www.youtube.com/watch?v=RQgnBB9z_N4',
youtubeChannel: 'https://youtube.com/channel/UCtnlZdXv3-xQzxiqfn6cjIA',
// eslint-disable-next-line max-len
magnet: 'magnet:?xs=https%3A%2F%2Fpeertube2.cpy.re%2Flazy-static%2Ftorrents%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.torrent&xt=urn:btih:0f498834733e8057ed5c6f2ee2b4efd8d84a76ee&dn=super+peertube2+video&tr=https%3A%2F%2Fpeertube2.cpy.re%2Ftracker%2Fannounce&tr=wss%3A%2F%2Fpeertube2.cpy.re%3A443%2Ftracker%2Fsocket&ws=https%3A%2F%2Fpeertube2.cpy.re%2Fstatic%2Fwebseed%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.mp4',

3
server/types/express.d.ts vendored
View File

@ -8,6 +8,7 @@ import {
MActorFollowActorsDefault,
MActorUrl,
MChannelBannerAccountDefault,
MChannelSyncChannel,
MStreamingPlaylist,
MVideoChangeOwnershipFull,
MVideoFile,
@ -145,6 +146,7 @@ declare module 'express' {
videoStreamingPlaylist?: MStreamingPlaylist
videoChannel?: MChannelBannerAccountDefault
videoChannelSync?: MChannelSyncChannel
videoPlaylistFull?: MVideoPlaylistFull
videoPlaylistSummary?: MVideoPlaylistFullSummary
@ -194,6 +196,7 @@ declare module 'express' {
plugin?: MPlugin
localViewerFull?: MLocalVideoViewerWithWatchSections
}
}
}

1
server/types/models/video/index.ts
View File

@ -8,6 +8,7 @@ export * from './video'
export * from './video-blacklist'
export * from './video-caption'
export * from './video-change-ownership'
export * from './video-channel-sync'
export * from './video-channels'
export * from './video-comment'
export * from './video-file'

17
server/types/models/video/video-channel-sync.ts Normal file
View File

@ -0,0 +1,17 @@
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
import { FunctionProperties, PickWith } from '@shared/typescript-utils'
import { MChannelAccountDefault, MChannelFormattable } from './video-channels'
type Use<K extends keyof VideoChannelSyncModel, M> = PickWith<VideoChannelSyncModel, K, M>
export type MChannelSync = Omit<VideoChannelSyncModel, 'VideoChannel'>
export type MChannelSyncChannel =
MChannelSync &
Use<'VideoChannel', MChannelAccountDefault> &
FunctionProperties<VideoChannelSyncModel>
export type MChannelSyncFormattable =
FunctionProperties<MChannelSyncChannel> &
Use<'VideoChannel', MChannelFormattable> &
MChannelSync

4
shared/models/server/custom-config.model.ts
View File

@ -165,6 +165,10 @@ export interface CustomConfig {
enabled: boolean
}
}
videoChannelSynchronization: {
enabled: boolean
maxPerUser: number
}
}
trending: {

5
shared/models/server/debug.model.ts
View File

@ -4,5 +4,8 @@ export interface Debug {
}
export interface SendDebugCommand {
command: 'remove-dandling-resumable-uploads' | 'process-video-views-buffer' | 'process-video-viewers'
command: 'remove-dandling-resumable-uploads'
| 'process-video-views-buffer'
| 'process-video-viewers'
| 'process-video-channel-sync-latest'
}

31
shared/models/server/job.model.ts
View File

@ -25,6 +25,8 @@ export type JobType =
| 'manage-video-torrent'
| 'move-to-object-storage'
| 'video-studio-edition'
| 'video-channel-import'
| 'after-video-channel-import'
| 'notify'
| 'federate-video'
@ -82,20 +84,32 @@ export type VideoFileImportPayload = {
filePath: string
}
// ---------------------------------------------------------------------------
export type VideoImportTorrentPayloadType = 'magnet-uri' | 'torrent-file'
export type VideoImportYoutubeDLPayloadType = 'youtube-dl'
export type VideoImportYoutubeDLPayload = {
export interface VideoImportYoutubeDLPayload {
type: VideoImportYoutubeDLPayloadType
videoImportId: number
fileExt?: string
}
export type VideoImportTorrentPayload = {
export interface VideoImportTorrentPayload {
type: VideoImportTorrentPayloadType
videoImportId: number
}
export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload
export type VideoImportPayload = (VideoImportYoutubeDLPayload | VideoImportTorrentPayload) & {
preventException: boolean
}
export interface VideoImportPreventExceptionResult {
resultType: 'success' | 'error'
}
// ---------------------------------------------------------------------------
export type VideoRedundancyPayload = {
videoId: number
@ -219,6 +233,17 @@ export interface VideoStudioEditionPayload {
// ---------------------------------------------------------------------------
export interface VideoChannelImportPayload {
externalChannelUrl: string
videoChannelId: number
}
export interface AfterVideoChannelImportPayload {
channelSyncId: number
}
// ---------------------------------------------------------------------------
export type NotifyPayload =
{
action: 'new-video'

3
shared/models/server/server-config.model.ts
View File

@ -188,6 +188,9 @@ export interface ServerConfig {
enabled: boolean
}
}
videoChannelSynchronization: {
enabled: boolean
}
}
autoBlacklist: {

3
shared/models/videos/channel-sync/index.ts Normal file
View File

@ -0,0 +1,3 @@
export * from './video-channel-sync-state.enum'
export * from './video-channel-sync.model'
export * from './video-channel-sync-create.model'

4
shared/models/videos/channel-sync/video-channel-sync-create.model.ts Normal file
View File

@ -0,0 +1,4 @@
export interface VideoChannelSyncCreate {
externalChannelUrl: string
videoChannelId: number
}

6
shared/models/videos/channel-sync/video-channel-sync-state.enum.ts Normal file
View File

@ -0,0 +1,6 @@
export const enum VideoChannelSyncState {
WAITING_FIRST_RUN = 1,
PROCESSING = 2,
SYNCED = 3,
FAILED = 4
}

14
shared/models/videos/channel-sync/video-channel-sync.model.ts Normal file
View File

@ -0,0 +1,14 @@
import { VideoChannelSummary } from '../channel/video-channel.model'
import { VideoConstant } from '../video-constant.model'
import { VideoChannelSyncState } from './video-channel-sync-state.enum'
export interface VideoChannelSync {
id: number
externalChannelUrl: string
createdAt: string
channel: VideoChannelSummary
state: VideoConstant<VideoChannelSyncState>
lastSyncAt: string
}

1
shared/models/videos/index.ts
View File

@ -11,6 +11,7 @@ export * from './playlist'
export * from './rate'
export * from './stats'
export * from './transcoding'
export * from './channel-sync'
export * from './nsfw-policy.type'

36
shared/server-commands/server/config-command.ts
View File

@ -18,17 +18,25 @@ export class ConfigCommand extends AbstractCommand {
}
}
disableImports () {
return this.setImportsEnabled(false)
}
enableImports () {
return this.setImportsEnabled(true)
}
private setImportsEnabled (enabled: boolean) {
return this.updateExistingSubConfig({
newConfig: {
import: {
videos: {
http: {
enabled: true
enabled
},
torrent: {
enabled: true
enabled
}
}
}
@ -36,6 +44,26 @@ export class ConfigCommand extends AbstractCommand {
})
}
private setChannelSyncEnabled (enabled: boolean) {
return this.updateExistingSubConfig({
newConfig: {
import: {
videoChannelSynchronization: {
enabled
}
}
}
})
}
enableChannelSync () {
return this.setChannelSyncEnabled(true)
}
disableChannelSync () {
return this.setChannelSyncEnabled(false)
}
enableLive (options: {
allowReplay?: boolean
transcoding?: boolean
@ -356,6 +384,10 @@ export class ConfigCommand extends AbstractCommand {
torrent: {
enabled: false
}
},
videoChannelSynchronization: {
enabled: false,
maxPerUser: 10
}
},
trending: {

3
shared/server-commands/server/server.ts
View File

@ -19,6 +19,7 @@ import {
CaptionsCommand,
ChangeOwnershipCommand,
ChannelsCommand,
ChannelSyncsCommand,
HistoryCommand,
ImportsCommand,
LiveCommand,
@ -118,6 +119,7 @@ export class PeerTubeServer {
playlists?: PlaylistsCommand
history?: HistoryCommand
imports?: ImportsCommand
channelSyncs?: ChannelSyncsCommand
streamingPlaylists?: StreamingPlaylistsCommand
channels?: ChannelsCommand
comments?: CommentsCommand
@ -390,6 +392,7 @@ export class PeerTubeServer {
this.playlists = new PlaylistsCommand(this)
this.history = new HistoryCommand(this)
this.imports = new ImportsCommand(this)
this.channelSyncs = new ChannelSyncsCommand(this)
this.streamingPlaylists = new StreamingPlaylistsCommand(this)
this.channels = new ChannelsCommand(this)
this.comments = new CommentsCommand(this)

21
shared/server-commands/server/servers.ts
View File

@ -39,11 +39,30 @@ async function cleanupTests (servers: PeerTubeServer[]) {
return Promise.all(p)
}
function getServerImportConfig (mode: 'youtube-dl' | 'yt-dlp') {
return {
import: {
videos: {
http: {
youtube_dl_release: {
url: mode === 'youtube-dl'
? 'https://yt-dl.org/downloads/latest/youtube-dl'
: 'https://api.github.com/repos/yt-dlp/yt-dlp/releases',
name: mode
}
}
}
}
}
}
// ---------------------------------------------------------------------------
export {
createSingleServer,
createMultipleServers,
cleanupTests,
killallServers
killallServers,
getServerImportConfig
}

55
shared/server-commands/videos/channel-syncs-command.ts Normal file
View File

@ -0,0 +1,55 @@
import { HttpStatusCode, ResultList, VideoChannelSync, VideoChannelSyncCreate } from '@shared/models'
import { pick } from '@shared/core-utils'
import { unwrapBody } from '../requests'
import { AbstractCommand, OverrideCommandOptions } from '../shared'
export class ChannelSyncsCommand extends AbstractCommand {
private static readonly API_PATH = '/api/v1/video-channel-syncs'
listByAccount (options: OverrideCommandOptions & {
accountName: string
start?: number
count?: number
sort?: string
}) {
const { accountName, sort = 'createdAt' } = options
const path = `/api/v1/accounts/${accountName}/video-channel-syncs`
return this.getRequestBody<ResultList<VideoChannelSync>>({
...options,
path,
query: { sort, ...pick(options, [ 'start', 'count' ]) },
implicitToken: true,
defaultExpectedStatus: HttpStatusCode.OK_200
})
}
async create (options: OverrideCommandOptions & {
attributes: VideoChannelSyncCreate
}) {
return unwrapBody<{ videoChannelSync: VideoChannelSync }>(this.postBodyRequest({
...options,
path: ChannelSyncsCommand.API_PATH,
fields: options.attributes,
implicitToken: true,
defaultExpectedStatus: HttpStatusCode.OK_200
}))
}
delete (options: OverrideCommandOptions & {
channelSyncId: number
}) {
const path = `${ChannelSyncsCommand.API_PATH}/${options.channelSyncId}`
return this.deleteRequest({
...options,
path,
implicitToken: true,
defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
})
}
}

18
shared/server-commands/videos/channels-command.ts
View File

@ -181,4 +181,22 @@ export class ChannelsCommand extends AbstractCommand {
defaultExpectedStatus: HttpStatusCode.OK_200
})
}
importVideos (options: OverrideCommandOptions & {
channelName: string
externalChannelUrl: string
}) {
const { channelName, externalChannelUrl } = options
const path = `/api/v1/video-channels/${channelName}/import-videos`
return this.postBodyRequest({
...options,
path,
fields: { externalChannelUrl },
implicitToken: true,
defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
})
}
}

1
shared/server-commands/videos/index.ts
View File

@ -3,6 +3,7 @@ export * from './captions-command'
export * from './change-ownership-command'
export * from './channels'
export * from './channels-command'
export * from './channel-syncs-command'
export * from './comments-command'
export * from './history-command'
export * from './imports-command'

143
support/doc/api/openapi.yaml
View File

@ -254,6 +254,8 @@ tags:
download it for you, saving you as much bandwidth and avoiding any instability or limitation your network might have.
- name: Video Imports
description: Operations dealing with listing, adding and removing video imports.
- name: Channels Sync
description: Operations dealing with synchronizing PeerTube user's channel with channels of other platforms
- name: Video Captions
description: Operations dealing with listing, adding and removing closed captions of a video.
- name: Video Channels
@ -327,6 +329,7 @@ x-tagGroups:
- Video Transcoding
- Live Videos
- Feeds
- Channels Sync
- name: Search
tags:
- Search
@ -3050,7 +3053,7 @@ paths:
tags:
- Video Channels
responses:
'204':
'200':
description: successful operation
content:
application/json:
@ -3288,6 +3291,59 @@ paths:
'204':
description: successful operation
'/video-channel-syncs':
post:
summary: Create a synchronization for a video channel
operationId: addVideoChannelSync
security:
- OAuth2: []
tags:
- Channels Sync
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/VideoChannelSyncCreate'
responses:
'200':
description: successful operation
content:
application/json:
schema:
type: object
properties:
videoChannelSync:
$ref: "#/components/schemas/VideoChannelSync"
'/video-channel-syncs/{channelSyncId}':
delete:
summary: Delete a video channel synchronization
operationId: delVideoChannelSync
security:
- OAuth2: []
tags:
- Channels Sync
parameters:
- $ref: '#/components/parameters/channelSyncId'
responses:
'204':
description: successful operation
'/video-channel-syncs/{channelSyncId}/sync':
post:
summary: Triggers the channel synchronization job, fetching all the videos from the remote channel
operationId: triggerVideoChannelSync
security:
- OAuth2: []
tags:
- Channels Sync
parameters:
- $ref: '#/components/parameters/channelSyncId'
responses:
'204':
description: successful operation
/video-playlists/privacies:
get:
summary: List available playlist privacy policies
@ -3659,6 +3715,26 @@ paths:
schema:
$ref: '#/components/schemas/VideoChannelList'
'/accounts/{name}/video-channel-syncs':
get:
summary: List the synchronizations of video channels of an account
tags:
- Video Channels
- Channels Sync
- Accounts
parameters:
- $ref: '#/components/parameters/name'
- $ref: '#/components/parameters/start'
- $ref: '#/components/parameters/count'
- $ref: '#/components/parameters/sort'
responses:
'200':
description: successful operation
content:
application/json:
schema:
$ref: '#/components/schemas/VideoChannelSyncList'
'/accounts/{name}/ratings':
get:
summary: List ratings of an account
@ -5141,6 +5217,13 @@ components:
schema:
type: string
example: my_username | my_username@example.com
channelSyncId:
name: channelSyncId
in: path
required: true
description: Channel Sync id
schema:
$ref: '#/components/schemas/Abuse/properties/id'
subscriptionHandle:
name: subscriptionHandle
in: path
@ -5347,6 +5430,7 @@ components:
- activitypub-refresher
- video-redundancy
- video-live-ending
- video-channel-import
followState:
name: state
in: query
@ -6497,6 +6581,11 @@ components:
properties:
enabled:
type: boolean
videoChannelSynchronization:
type: object
properties:
enabled:
type: boolean
autoBlacklist:
type: object
properties:
@ -6861,6 +6950,11 @@ components:
properties:
enabled:
type: boolean
video_channel_synchronization:
type: object
properties:
enabled:
type: boolean
autoBlacklist:
type: object
properties:
@ -6953,6 +7047,7 @@ components:
- videos-views-stats
- activitypub-refresher
- video-redundancy
- video-channel-import
data:
type: object
additionalProperties: true
@ -7473,6 +7568,7 @@ components:
type: integer
uuid:
$ref: '#/components/schemas/UUIDv4'
VideoChannelCreate:
allOf:
- $ref: '#/components/schemas/VideoChannel'
@ -7503,6 +7599,51 @@ components:
- $ref: '#/components/schemas/VideoChannel'
- $ref: '#/components/schemas/Actor'
VideoChannelSync:
type: object
properties:
id:
$ref: '#/components/schemas/id'
state:
type: object
properties:
id:
type: integer
example: 2
label:
type: string
example: PROCESSING
externalChannelUrl:
type: string
example: 'https://youtube.com/c/UC_myfancychannel'
createdAt:
type: string
format: date-time
lastSyncAt:
type: string
format: date-time
nullable: true
channel:
$ref: '#/components/schemas/VideoChannel'
VideoChannelSyncList:
type: object
properties:
total:
type: integer
example: 1
data:
type: array
items:
allOf:
- $ref: '#/components/schemas/VideoChannelSync'
VideoChannelSyncCreate:
type: object
properties:
externalChannelUrl:
type: string
example: https://youtube.com/c/UC_myfancychannel
videoChannelId:
$ref: '#/components/schemas/id'
MRSSPeerLink:
type: object
xml: