PeerTube/server/initializers/constants.ts

1350 lines
38 KiB
TypeScript
Raw Normal View History

2022-08-08 10:42:08 +02:00
import { RepeatOptions } from 'bullmq'
2022-10-10 11:12:23 +02:00
import { Encoding, randomBytes } from 'crypto'
import { invert } from 'lodash'
2019-04-11 11:33:44 +02:00
import { join } from 'path'
import { randomInt, root } from '@shared/core-utils'
2020-07-01 16:05:30 +02:00
import {
AbuseState,
JobType,
RunnerJobState,
2023-01-19 14:00:37 +01:00
UserRegistrationState,
Channel sync (#5135) * Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz <me@florianbigard.com>
2022-08-10 09:53:39 +02:00
VideoChannelSyncState,
2020-07-01 16:05:30 +02:00
VideoImportState,
VideoPrivacy,
VideoRateType,
VideoResolution,
VideoState,
VideoTranscodingFPS
2020-07-01 16:05:30 +02:00
} from '../../shared/models'
import { ActivityPubActorType } from '../../shared/models/activitypub'
import { ActorImageType, FollowState } from '../../shared/models/actors'
import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
2019-02-26 10:55:40 +01:00
import { VideoPlaylistPrivacy } from '../../shared/models/videos/playlist/video-playlist-privacy.model'
2019-03-05 10:58:44 +01:00
import { VideoPlaylistType } from '../../shared/models/videos/playlist/video-playlist-type.model'
// Do not use barrels, remain constants as independent as possible
import { isTestInstance, isTestOrDevInstance, parseDurationToMs, sanitizeHost, sanitizeUrl } from '../helpers/core-utils'
2019-04-11 11:33:44 +02:00
import { CONFIG, registerConfigChangedHandler } from './config'
2017-06-16 10:36:18 +02:00
2016-10-02 11:14:08 +02:00
// ---------------------------------------------------------------------------
const LAST_MIGRATION_VERSION = 765
2017-02-18 11:56:28 +01:00
// ---------------------------------------------------------------------------
2016-03-16 22:29:27 +01:00
const API_VERSION = 'v1'
2021-03-08 14:24:11 +01:00
const PEERTUBE_VERSION: string = require(join(root(), 'package.json')).version
2018-07-24 16:27:45 +02:00
const PAGINATION = {
GLOBAL: {
COUNT: {
DEFAULT: 15,
MAX: 100
}
},
OUTBOX: {
COUNT: {
MAX: 50
}
2018-07-24 16:27:45 +02:00
}
}
2016-10-02 11:14:08 +02:00
2019-04-11 11:33:44 +02:00
const WEBSERVER = {
URL: '',
HOST: '',
SCHEME: '',
WS: '',
HOSTNAME: '',
PORT: 0,
2021-11-05 11:36:03 +01:00
RTMP_URL: '',
RTMPS_URL: ''
2019-04-11 11:33:44 +02:00
}
2016-10-02 11:14:08 +02:00
// Sortable columns per schema
const SORTABLE_COLUMNS = {
ADMIN_USERS: [ 'id', 'username', 'videoQuotaUsed', 'createdAt', 'lastLoginDate', 'role' ],
USER_SUBSCRIPTIONS: [ 'id', 'createdAt' ],
2018-01-03 16:38:50 +01:00
ACCOUNTS: [ 'createdAt' ],
JOBS: [ 'createdAt' ],
2017-10-24 19:41:09 +02:00
VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ],
2018-08-02 17:48:50 +02:00
VIDEO_IMPORTS: [ 'createdAt' ],
Channel sync (#5135) * Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz <me@florianbigard.com>
2022-08-10 09:53:39 +02:00
VIDEO_CHANNEL_SYNCS: [ 'externalChannelUrl', 'videoChannel', 'createdAt', 'lastSyncAt', 'state' ],
2020-11-13 16:38:23 +01:00
2019-12-27 17:02:34 +01:00
VIDEO_COMMENT_THREADS: [ 'createdAt', 'totalReplies' ],
2020-11-13 16:38:23 +01:00
VIDEO_COMMENTS: [ 'createdAt' ],
VIDEO_RATES: [ 'createdAt' ],
2017-11-13 17:39:41 +01:00
BLACKLISTS: [ 'id', 'name', 'duration', 'views', 'likes', 'dislikes', 'uuid', 'createdAt' ],
2021-10-19 09:44:43 +02:00
2021-10-19 09:10:01 +02:00
INSTANCE_FOLLOWERS: [ 'createdAt', 'state', 'score' ],
INSTANCE_FOLLOWING: [ 'createdAt', 'redundancyAllowed', 'state' ],
2021-10-19 09:44:43 +02:00
ACCOUNT_FOLLOWERS: [ 'createdAt' ],
CHANNEL_FOLLOWERS: [ 'createdAt' ],
2018-07-19 16:17:54 +02:00
2023-01-19 14:00:37 +01:00
USER_REGISTRATIONS: [ 'createdAt', 'state' ],
RUNNERS: [ 'createdAt' ],
RUNNER_REGISTRATION_TOKENS: [ 'createdAt' ],
RUNNER_JOBS: [ 'updatedAt', 'createdAt', 'priority', 'state', 'progress' ],
VIDEOS: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'trending', 'hot', 'best' ],
2018-08-31 17:18:13 +02:00
2020-05-29 16:16:24 +02:00
// Don't forget to update peertube-search-index with the same values
VIDEOS_SEARCH: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'match' ],
VIDEO_CHANNELS_SEARCH: [ 'match', 'displayName', 'createdAt' ],
2021-06-17 16:02:38 +02:00
VIDEO_PLAYLISTS_SEARCH: [ 'match', 'displayName', 'createdAt' ],
2020-07-01 16:05:30 +02:00
ABUSES: [ 'id', 'createdAt', 'state' ],
ACCOUNTS_BLOCKLIST: [ 'createdAt' ],
2018-12-26 10:36:24 +01:00
SERVERS_BLOCKLIST: [ 'createdAt' ],
2020-07-15 11:15:50 +02:00
USER_NOTIFICATIONS: [ 'createdAt', 'read' ],
2019-02-26 10:55:40 +01:00
VIDEO_PLAYLISTS: [ 'name', 'displayName', 'createdAt', 'updatedAt' ],
PLUGINS: [ 'name', 'createdAt', 'updatedAt' ],
2020-01-10 10:11:28 +01:00
AVAILABLE_PLUGINS: [ 'npmName', 'popularity' ],
VIDEO_REDUNDANCIES: [ 'name' ]
2016-10-02 11:14:08 +02:00
}
2018-05-11 09:44:04 +02:00
const ROUTE_CACHE_LIFETIME = {
2018-07-21 23:00:25 +02:00
FEEDS: '15 minutes',
ROBOTS: '2 hours',
2018-12-05 17:27:24 +01:00
SITEMAP: '1 day',
SECURITYTXT: '2 hours',
2018-07-21 23:00:25 +02:00
NODEINFO: '10 minutes',
DNT_POLICY: '1 week',
2018-05-11 09:44:04 +02:00
ACTIVITY_PUB: {
2018-07-21 23:00:25 +02:00
VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example
2018-09-14 14:57:59 +02:00
},
STATS: '4 hours',
WELL_KNOWN: '1 day'
2018-05-11 09:44:04 +02:00
}
2016-10-02 11:14:08 +02:00
// ---------------------------------------------------------------------------
2016-08-25 17:57:37 +02:00
// Number of points we add/remove after a successful/bad request
const ACTOR_FOLLOW_SCORE = {
PENALTY: -10,
BONUS: 10,
BASE: 1000,
MAX: 10000
}
const FOLLOW_STATES: { [ id: string ]: FollowState } = {
PENDING: 'pending',
ACCEPTED: 'accepted',
REJECTED: 'rejected'
}
const REMOTE_SCHEME = {
HTTP: 'https',
WS: 'wss'
}
// ---------------------------------------------------------------------------
2020-01-10 10:11:28 +01:00
const JOB_ATTEMPTS: { [id in JobType]: number } = {
'activitypub-http-broadcast': 1,
2022-06-17 14:08:13 +02:00
'activitypub-http-broadcast-parallel': 1,
'activitypub-http-unicast': 1,
'activitypub-http-fetcher': 2,
'activitypub-follow': 5,
'activitypub-cleaner': 1,
'video-file-import': 1,
'video-transcoding': 1,
'video-import': 1,
2018-08-29 16:26:25 +02:00
'email': 5,
'actor-keys': 3,
'videos-views-stats': 1,
2020-01-10 10:11:28 +01:00
'activitypub-refresher': 1,
'video-redundancy': 1,
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 08:26:20 +02:00
'video-live-ending': 1,
2022-03-22 16:58:49 +01:00
'video-studio-edition': 1,
'manage-video-torrent': 1,
Channel sync (#5135) * Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz <me@florianbigard.com>
2022-08-10 09:53:39 +02:00
'video-channel-import': 1,
'after-video-channel-import': 1,
2022-08-08 15:48:17 +02:00
'move-to-object-storage': 3,
'transcoding-job-builder': 1,
2022-08-08 15:48:17 +02:00
'notify': 1,
'federate-video': 1
}
// Excluded keys are jobs that can be configured by admins
const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-import'>]: number } = {
'activitypub-http-broadcast': 1,
2022-06-17 14:08:13 +02:00
'activitypub-http-broadcast-parallel': 30,
2022-11-24 10:20:07 +01:00
'activitypub-http-unicast': 30,
2021-06-15 15:31:08 +02:00
'activitypub-http-fetcher': 3,
'activitypub-cleaner': 1,
2020-01-10 11:15:07 +01:00
'activitypub-follow': 1,
'video-file-import': 1,
2018-08-29 16:26:25 +02:00
'email': 5,
'actor-keys': 1,
'videos-views-stats': 1,
2020-01-10 10:11:28 +01:00
'activitypub-refresher': 1,
'video-redundancy': 1,
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 08:26:20 +02:00
'video-live-ending': 10,
2022-03-22 16:58:49 +01:00
'video-studio-edition': 1,
'manage-video-torrent': 1,
2022-08-08 15:48:17 +02:00
'move-to-object-storage': 1,
Channel sync (#5135) * Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz <me@florianbigard.com>
2022-08-10 09:53:39 +02:00
'video-channel-import': 1,
'after-video-channel-import': 1,
'transcoding-job-builder': 1,
2022-08-08 15:48:17 +02:00
'notify': 5,
'federate-video': 3
}
2020-01-10 10:11:28 +01:00
const JOB_TTL: { [id in JobType]: number } = {
2018-08-03 10:19:51 +02:00
'activitypub-http-broadcast': 60000 * 10, // 10 minutes
2022-06-17 14:08:13 +02:00
'activitypub-http-broadcast-parallel': 60000 * 10, // 10 minutes
2018-08-03 10:19:51 +02:00
'activitypub-http-unicast': 60000 * 10, // 10 minutes
2020-04-11 09:06:15 +02:00
'activitypub-http-fetcher': 1000 * 3600 * 10, // 10 hours
2018-08-03 10:19:51 +02:00
'activitypub-follow': 60000 * 10, // 10 minutes
'activitypub-cleaner': 1000 * 3600, // 1 hour
2018-08-03 10:19:51 +02:00
'video-file-import': 1000 * 3600, // 1 hour
'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long
2022-03-22 16:58:49 +01:00
'video-studio-edition': 1000 * 3600 * 10, // 10 hours
'video-import': CONFIG.IMPORT.VIDEOS.TIMEOUT,
2018-08-29 16:26:25 +02:00
'email': 60000 * 10, // 10 minutes
'actor-keys': 60000 * 20, // 20 minutes
'videos-views-stats': undefined, // Unlimited
2020-01-10 10:11:28 +01:00
'activitypub-refresher': 60000 * 10, // 10 minutes
'video-redundancy': 1000 * 3600 * 3, // 3 hours
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 08:26:20 +02:00
'video-live-ending': 1000 * 60 * 10, // 10 minutes
'manage-video-torrent': 1000 * 3600 * 3, // 3 hours
Channel sync (#5135) * Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz <me@florianbigard.com>
2022-08-10 09:53:39 +02:00
'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours
'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours
'after-video-channel-import': 60000 * 5, // 5 minutes
'transcoding-job-builder': 60000, // 1 minute
2022-08-08 15:48:17 +02:00
'notify': 60000 * 5, // 5 minutes
Channel sync (#5135) * Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz <me@florianbigard.com>
2022-08-10 09:53:39 +02:00
'federate-video': 60000 * 5 // 5 minutes
2018-08-03 10:19:51 +02:00
}
2022-08-08 10:42:08 +02:00
const REPEAT_JOBS: { [ id in JobType ]?: RepeatOptions } = {
'videos-views-stats': {
2023-02-16 14:10:11 +01:00
pattern: randomInt(1, 20) + ' * * * *' // Between 1-20 minutes past the hour
},
'activitypub-cleaner': {
2023-02-16 14:10:11 +01:00
pattern: '30 5 * * ' + randomInt(0, 7) // 1 time per week (random day) at 5:30 AM
2018-08-29 16:26:25 +02:00
}
}
const JOB_PRIORITY = {
TRANSCODING: 100
}
2018-08-29 16:26:25 +02:00
const JOB_REMOVAL_OPTIONS = {
COUNT: 10000, // Max jobs to store
SUCCESS: { // Success jobs
'DEFAULT': parseDurationToMs('2 days'),
'activitypub-http-broadcast-parallel': parseDurationToMs('10 minutes'),
'activitypub-http-unicast': parseDurationToMs('1 hour'),
'videos-views-stats': parseDurationToMs('3 hours'),
'activitypub-refresher': parseDurationToMs('10 hours')
},
FAILURE: { // Failed job
DEFAULT: parseDurationToMs('7 days')
}
}
2022-02-09 11:40:47 +01:00
const VIDEO_IMPORT_TIMEOUT = Math.floor(JOB_TTL['video-import'] * 0.9)
const RUNNER_JOBS = {
MAX_FAILURES: 5
}
// ---------------------------------------------------------------------------
const BROADCAST_CONCURRENCY = 30 // How many requests in parallel we do in activitypub-http-broadcast job
const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...)
const AP_CLEANER = {
CONCURRENCY: 10, // How many requests in parallel we do in activitypub-cleaner job
UNAVAILABLE_TRESHOLD: 3, // How many attempts we do before removing an unavailable remote resource
PERIOD: parseDurationToMs('1 week') // /!\ Has to be sync with REPEAT_JOBS
}
const REQUEST_TIMEOUTS = {
DEFAULT: 7000, // 7 seconds
FILE: 30000, // 30 seconds
REDUNDANCY: JOB_TTL['video-redundancy']
}
const SCHEDULER_INTERVALS_MS = {
RUNNER_JOB_WATCH_DOG: Math.min(CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD, CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE),
2021-10-22 10:28:00 +02:00
ACTOR_FOLLOW_SCORES: 60000 * 60, // 1 hour
REMOVE_OLD_JOBS: 60000 * 60, // 1 hour
UPDATE_VIDEOS: 60000, // 1 minute
YOUTUBE_DL_UPDATE: 60000 * 60 * 24, // 1 day
GEO_IP_UPDATE: 60000 * 60 * 24, // 1 day
VIDEO_VIEWS_BUFFER_UPDATE: CONFIG.VIEWS.VIDEOS.LOCAL_BUFFER_UPDATE_INTERVAL,
2021-10-22 10:28:00 +02:00
CHECK_PLUGINS: CONFIG.PLUGINS.INDEX.CHECK_LATEST_VERSIONS_INTERVAL,
CHECK_PEERTUBE_VERSION: 60000 * 60 * 24, // 1 day
AUTO_FOLLOW_INDEX_INSTANCES: 60000 * 60 * 24, // 1 day
REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day
REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day
UPDATE_INBOX_STATS: 1000 * 60, // 1 minute
Channel sync (#5135) * Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz <me@florianbigard.com>
2022-08-10 09:53:39 +02:00
REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60, // 1 hour
CHANNEL_SYNC_CHECK_INTERVAL: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.CHECK_INTERVAL
}
// ---------------------------------------------------------------------------
const CONSTRAINTS_FIELDS = {
2016-07-31 20:58:43 +02:00
USERS: {
NAME: { min: 1, max: 120 }, // Length
DESCRIPTION: { min: 3, max: 1000 }, // Length
USERNAME: { min: 1, max: 50 }, // Length
2017-09-04 20:07:54 +02:00
PASSWORD: { min: 6, max: 255 }, // Length
2018-08-08 17:36:10 +02:00
VIDEO_QUOTA: { min: -1 },
VIDEO_QUOTA_DAILY: { min: -1 },
VIDEO_LANGUAGES: { max: 500 }, // Array length
2018-08-08 17:36:10 +02:00
BLOCKED_REASON: { min: 3, max: 250 } // Length
2016-07-31 20:58:43 +02:00
},
2020-07-01 16:05:30 +02:00
ABUSES: {
REASON: { min: 2, max: 3000 }, // Length
MODERATION_COMMENT: { min: 2, max: 3000 } // Length
2017-01-04 20:59:23 +01:00
},
2020-07-24 15:05:51 +02:00
ABUSE_MESSAGES: {
MESSAGE: { min: 2, max: 3000 } // Length
},
2023-01-19 14:00:37 +01:00
USER_REGISTRATIONS: {
REASON_MESSAGE: { min: 2, max: 3000 }, // Length
MODERATOR_MESSAGE: { min: 2, max: 3000 } // Length
},
2018-08-13 16:57:13 +02:00
VIDEO_BLACKLIST: {
REASON: { min: 2, max: 300 } // Length
},
2017-10-24 19:41:09 +02:00
VIDEO_CHANNELS: {
NAME: { min: 1, max: 120 }, // Length
DESCRIPTION: { min: 3, max: 1000 }, // Length
SUPPORT: { min: 3, max: 1000 }, // Length
Channel sync (#5135) * Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz <me@florianbigard.com>
2022-08-10 09:53:39 +02:00
EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 }, // Length
2017-11-14 10:57:56 +01:00
URL: { min: 3, max: 2000 } // Length
2017-10-24 19:41:09 +02:00
},
Channel sync (#5135) * Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz <me@florianbigard.com>
2022-08-10 09:53:39 +02:00
VIDEO_CHANNEL_SYNCS: {
EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 } // Length
},
2018-07-12 19:02:00 +02:00
VIDEO_CAPTIONS: {
CAPTION_FILE: {
2018-07-16 14:22:16 +02:00
EXTNAME: [ '.vtt', '.srt' ],
2018-07-12 19:02:00 +02:00
FILE_SIZE: {
2022-01-14 08:47:27 +01:00
max: 20 * 1024 * 1024 // 20MB
2018-07-12 19:02:00 +02:00
}
}
},
VIDEO_IMPORTS: {
2018-08-06 17:13:39 +02:00
URL: { min: 3, max: 2000 }, // Length
TORRENT_NAME: { min: 3, max: 255 }, // Length
2018-08-07 09:54:36 +02:00
TORRENT_FILE: {
EXTNAME: [ '.torrent' ],
FILE_SIZE: {
max: 1024 * 200 // 200 KB
}
}
},
2018-09-11 16:27:07 +02:00
VIDEOS_REDUNDANCY: {
URL: { min: 3, max: 2000 } // Length
},
2018-11-14 15:01:28 +01:00
VIDEO_RATES: {
URL: { min: 3, max: 2000 } // Length
},
2016-07-31 20:58:43 +02:00
VIDEOS: {
NAME: { min: 3, max: 120 }, // Length
2018-04-23 14:39:52 +02:00
LANGUAGE: { min: 1, max: 10 }, // Length
2017-10-30 10:16:27 +01:00
TRUNCATED_DESCRIPTION: { min: 3, max: 250 }, // Length
DESCRIPTION: { min: 3, max: 10000 }, // Length
SUPPORT: { min: 3, max: 1000 }, // Length
IMAGE: {
2021-01-26 10:23:21 +01:00
EXTNAME: [ '.png', '.jpg', '.jpeg', '.webp' ],
FILE_SIZE: {
max: 4 * 1024 * 1024 // 4MB
}
},
2019-05-16 16:55:34 +02:00
EXTNAME: [] as string[],
2017-09-04 21:21:47 +02:00
INFO_HASH: { min: 40, max: 40 }, // Length, info hash is 20 bytes length but we represent it in hexadecimal so 20 * 2
DURATION: { min: 0 }, // Number
TAGS: { min: 0, max: 5 }, // Number of total tags
TAG: { min: 2, max: 30 }, // Length
2017-02-26 18:57:33 +01:00
VIEWS: { min: 0 },
LIKES: { min: 0 },
DISLIKES: { min: 0 },
FILE_SIZE: { min: -1 },
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
PARTIAL_UPLOAD_SIZE: { max: 50 * 1024 * 1024 * 1024 }, // 50GB
2017-11-14 10:57:56 +01:00
URL: { min: 3, max: 2000 } // Length
},
2019-02-26 10:55:40 +01:00
VIDEO_PLAYLISTS: {
NAME: { min: 1, max: 120 }, // Length
DESCRIPTION: { min: 3, max: 1000 }, // Length
URL: { min: 3, max: 2000 }, // Length
IMAGE: {
EXTNAME: [ '.jpg', '.jpeg' ],
FILE_SIZE: {
max: 4 * 1024 * 1024 // 4MB
2019-02-26 10:55:40 +01:00
}
}
},
2018-01-03 11:10:40 +01:00
ACTORS: {
2017-11-14 10:57:56 +01:00
PUBLIC_KEY: { min: 10, max: 5000 }, // Length
PRIVATE_KEY: { min: 10, max: 5000 }, // Length
2017-12-29 19:10:13 +01:00
URL: { min: 3, max: 2000 }, // Length
2021-04-06 17:01:35 +02:00
IMAGE: {
2021-01-26 10:23:21 +01:00
EXTNAME: [ '.png', '.jpeg', '.jpg', '.gif', '.webp' ],
2018-01-03 11:10:40 +01:00
FILE_SIZE: {
max: 4 * 1024 * 1024 // 4MB
2018-01-03 11:10:40 +01:00
}
2017-12-29 19:10:13 +01:00
}
2017-02-26 18:57:33 +01:00
},
VIDEO_EVENTS: {
COUNT: { min: 0 }
},
2017-12-22 10:50:07 +01:00
VIDEO_COMMENTS: {
2020-05-05 16:48:30 +02:00
TEXT: { min: 1, max: 10000 }, // Length
URL: { min: 3, max: 2000 } // Length
},
VIDEO_SHARE: {
URL: { min: 3, max: 2000 } // Length
2019-01-09 15:14:29 +01:00
},
CONTACT_FORM: {
FROM_NAME: { min: 1, max: 120 }, // Length
BODY: { min: 3, max: 5000 } // Length
},
PLUGINS: {
NAME: { min: 1, max: 214 }, // Length
DESCRIPTION: { min: 1, max: 20000 } // Length
},
COMMONS: {
URL: { min: 5, max: 2000 } // Length
2022-02-11 10:51:33 +01:00
},
2022-03-22 16:58:49 +01:00
VIDEO_STUDIO: {
2022-02-11 10:51:33 +01:00
TASKS: { min: 1, max: 10 }, // Number of tasks
CUT_TIME: { min: 0 } // Value
},
LOGS: {
CLIENT_MESSAGE: { min: 1, max: 1000 }, // Length
2022-08-09 11:34:56 +02:00
CLIENT_STACK_TRACE: { min: 1, max: 15000 }, // Length
CLIENT_META: { min: 1, max: 5000 }, // Length
CLIENT_USER_AGENT: { min: 1, max: 200 } // Length
},
RUNNERS: {
TOKEN: { min: 1, max: 1000 }, // Length
NAME: { min: 1, max: 100 }, // Length
DESCRIPTION: { min: 1, max: 1000 } // Length
},
RUNNER_JOBS: {
TOKEN: { min: 1, max: 1000 }, // Length
REASON: { min: 1, max: 5000 }, // Length
ERROR_MESSAGE: { min: 1, max: 5000 }, // Length
PROGRESS: { min: 0, max: 100 } // Value
2016-07-31 20:58:43 +02:00
}
}
2020-11-06 16:43:43 +01:00
const VIEW_LIFETIME = {
VIEW: CONFIG.VIEWS.VIDEOS.IP_VIEW_EXPIRATION,
2022-06-20 10:04:52 +02:00
VIEWER_COUNTER: 60000 * 2, // 2 minutes
VIEWER_STATS: 60000 * 60 // 1 hour
2020-11-06 16:42:23 +01:00
}
2022-06-17 09:04:45 +02:00
const MAX_LOCAL_VIEWER_WATCH_SECTIONS = 100
2019-01-09 15:14:29 +01:00
let CONTACT_FORM_LIFETIME = 60000 * 60 // 1 hour
const VIDEO_TRANSCODING_FPS: VideoTranscodingFPS = {
MIN: 1,
2020-01-31 16:56:52 +01:00
STANDARD: [ 24, 25, 30 ],
HD_STANDARD: [ 50, 60 ],
2022-02-11 10:51:33 +01:00
AUDIO_MERGE: 25,
AVERAGE: 30,
MAX: 60,
KEEP_ORIGIN_FPS_RESOLUTION_MIN: 720 // We keep the original FPS on high resolutions (720 minimum)
}
2018-02-23 16:39:51 +01:00
2019-05-16 16:55:34 +02:00
const DEFAULT_AUDIO_RESOLUTION = VideoResolution.H_480P
2017-06-16 10:36:18 +02:00
const VIDEO_RATE_TYPES: { [ id: string ]: VideoRateType } = {
2017-03-08 21:35:43 +01:00
LIKE: 'like',
DISLIKE: 'dislike'
}
const FFMPEG_NICE: { [ id: string ]: number } = {
2021-01-10 21:02:55 +01:00
// parent process defaults to niceness = 0
// reminder: lower = higher priority, max value is 19, lowest is -20
LIVE: 5, // prioritize over VOD and THUMBNAIL
THUMBNAIL: 10,
2021-01-10 21:02:55 +01:00
VOD: 15
}
2017-03-22 21:15:55 +01:00
const VIDEO_CATEGORIES = {
1: 'Music',
2: 'Films',
3: 'Vehicles',
4: 'Art',
5: 'Sports',
6: 'Travels',
7: 'Gaming',
8: 'People',
9: 'Comedy',
10: 'Entertainment',
11: 'News & Politics',
12: 'How To',
2017-03-22 21:15:55 +01:00
13: 'Education',
14: 'Activism',
15: 'Science & Technology',
16: 'Animals',
17: 'Kids',
18: 'Food'
}
2017-03-27 20:53:11 +02:00
// See https://creativecommons.org/licenses/?lang=en
const VIDEO_LICENCES = {
1: 'Attribution',
2: 'Attribution - Share Alike',
3: 'Attribution - No Derivatives',
4: 'Attribution - Non Commercial',
5: 'Attribution - Non Commercial - Share Alike',
6: 'Attribution - Non Commercial - No Derivatives',
7: 'Public Domain Dedication'
}
2020-01-31 16:56:52 +01:00
const VIDEO_LANGUAGES: { [id: string]: string } = {}
2017-04-07 12:13:37 +02:00
const VIDEO_PRIVACIES: { [ id in VideoPrivacy ]: string } = {
2020-01-31 16:56:52 +01:00
[VideoPrivacy.PUBLIC]: 'Public',
[VideoPrivacy.UNLISTED]: 'Unlisted',
[VideoPrivacy.PRIVATE]: 'Private',
[VideoPrivacy.INTERNAL]: 'Internal'
2017-10-31 11:52:52 +01:00
}
const VIDEO_STATES: { [ id in VideoState ]: string } = {
2020-01-31 16:56:52 +01:00
[VideoState.PUBLISHED]: 'Published',
[VideoState.TO_TRANSCODE]: 'To transcode',
[VideoState.TO_IMPORT]: 'To import',
[VideoState.WAITING_FOR_LIVE]: 'Waiting for livestream',
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 08:26:20 +02:00
[VideoState.LIVE_ENDED]: 'Livestream ended',
[VideoState.TO_MOVE_TO_EXTERNAL_STORAGE]: 'To move to an external storage',
[VideoState.TRANSCODING_FAILED]: 'Transcoding failed',
2022-02-11 10:51:33 +01:00
[VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED]: 'External storage move failed',
[VideoState.TO_EDIT]: 'To edit*'
}
const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = {
2020-01-31 16:56:52 +01:00
[VideoImportState.FAILED]: 'Failed',
[VideoImportState.PENDING]: 'Pending',
[VideoImportState.SUCCESS]: 'Success',
[VideoImportState.REJECTED]: 'Rejected',
[VideoImportState.CANCELLED]: 'Cancelled',
[VideoImportState.PROCESSING]: 'Processing'
}
Channel sync (#5135) * Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz <me@florianbigard.com>
2022-08-10 09:53:39 +02:00
const VIDEO_CHANNEL_SYNC_STATE: { [ id in VideoChannelSyncState ]: string } = {
[VideoChannelSyncState.FAILED]: 'Failed',
[VideoChannelSyncState.SYNCED]: 'Synchronized',
[VideoChannelSyncState.PROCESSING]: 'Processing',
[VideoChannelSyncState.WAITING_FIRST_RUN]: 'Waiting first run'
}
const ABUSE_STATES: { [ id in AbuseState ]: string } = {
2020-07-01 16:05:30 +02:00
[AbuseState.PENDING]: 'Pending',
[AbuseState.REJECTED]: 'Rejected',
[AbuseState.ACCEPTED]: 'Accepted'
}
2023-01-19 14:00:37 +01:00
const USER_REGISTRATION_STATES: { [ id in UserRegistrationState ]: string } = {
[UserRegistrationState.PENDING]: 'Pending',
[UserRegistrationState.REJECTED]: 'Rejected',
[UserRegistrationState.ACCEPTED]: 'Accepted'
}
const VIDEO_PLAYLIST_PRIVACIES: { [ id in VideoPlaylistPrivacy ]: string } = {
2020-01-31 16:56:52 +01:00
[VideoPlaylistPrivacy.PUBLIC]: 'Public',
[VideoPlaylistPrivacy.UNLISTED]: 'Unlisted',
[VideoPlaylistPrivacy.PRIVATE]: 'Private'
2019-02-26 10:55:40 +01:00
}
const VIDEO_PLAYLIST_TYPES: { [ id in VideoPlaylistType ]: string } = {
2020-01-31 16:56:52 +01:00
[VideoPlaylistType.REGULAR]: 'Regular',
[VideoPlaylistType.WATCH_LATER]: 'Watch later'
2019-03-05 10:58:44 +01:00
}
const RUNNER_JOB_STATES: { [ id in RunnerJobState ]: string } = {
[RunnerJobState.PROCESSING]: 'Processing',
[RunnerJobState.COMPLETED]: 'Completed',
[RunnerJobState.PENDING]: 'Pending',
[RunnerJobState.ERRORED]: 'Errored',
[RunnerJobState.WAITING_FOR_PARENT_JOB]: 'Waiting for parent job to finish',
[RunnerJobState.CANCELLED]: 'Cancelled',
[RunnerJobState.PARENT_ERRORED]: 'Parent job failed',
[RunnerJobState.PARENT_CANCELLED]: 'Parent job cancelled'
}
2018-12-11 14:52:50 +01:00
const MIMETYPES = {
2019-05-16 16:55:34 +02:00
AUDIO: {
MIMETYPE_EXT: {
'audio/mpeg': '.mp3',
'audio/mp3': '.mp3',
2022-04-20 10:23:07 +02:00
2019-05-16 16:55:34 +02:00
'application/ogg': '.ogg',
'audio/ogg': '.ogg',
2022-04-20 10:23:07 +02:00
2020-02-07 08:51:28 +01:00
'audio/x-ms-wma': '.wma',
'audio/wav': '.wav',
'audio/x-wav': '.wav',
2022-04-20 10:23:07 +02:00
2020-02-07 08:51:28 +01:00
'audio/x-flac': '.flac',
'audio/flac': '.flac',
2022-04-20 10:23:07 +02:00
'audio/vnd.dlna.adts': '.aac',
'audio/aac': '.aac',
2022-04-20 10:23:07 +02:00
2021-02-25 09:09:41 +01:00
'audio/m4a': '.m4a',
2021-02-25 09:14:24 +01:00
'audio/mp4': '.m4a',
2021-02-25 09:09:41 +01:00
'audio/x-m4a': '.m4a',
2022-04-20 10:23:07 +02:00
'audio/vnd.dolby.dd-raw': '.ac3',
'audio/ac3': '.ac3'
2019-05-16 16:55:34 +02:00
},
EXT_MIMETYPE: null as { [ id: string ]: string }
},
2018-12-11 14:52:50 +01:00
VIDEO: {
MIMETYPE_EXT: null as { [ id: string ]: string | string[] },
MIMETYPES_REGEX: null as string,
2018-12-11 14:52:50 +01:00
EXT_MIMETYPE: null as { [ id: string ]: string }
},
IMAGE: {
MIMETYPE_EXT: {
'image/png': '.png',
2021-04-06 17:01:35 +02:00
'image/gif': '.gif',
'image/webp': '.webp',
2018-12-11 14:52:50 +01:00
'image/jpg': '.jpg',
'image/jpeg': '.jpg'
},
EXT_MIMETYPE: null as { [ id: string ]: string }
2018-12-11 14:52:50 +01:00
},
VIDEO_CAPTIONS: {
MIMETYPE_EXT: {
'text/vtt': '.vtt',
2019-10-18 10:36:32 +02:00
'application/x-subrip': '.srt',
'text/plain': '.srt'
2018-12-11 14:52:50 +01:00
}
},
TORRENT: {
MIMETYPE_EXT: {
'application/x-bittorrent': '.torrent'
}
},
M3U8: {
MIMETYPE_EXT: {
'application/vnd.apple.mpegurl': '.m3u8'
}
2018-12-11 14:52:50 +01:00
}
2018-08-07 09:54:36 +02:00
}
2019-05-16 16:55:34 +02:00
MIMETYPES.AUDIO.EXT_MIMETYPE = invert(MIMETYPES.AUDIO.MIMETYPE_EXT)
MIMETYPES.IMAGE.EXT_MIMETYPE = invert(MIMETYPES.IMAGE.MIMETYPE_EXT)
2018-08-07 09:54:36 +02:00
const BINARY_CONTENT_TYPES = new Set([
'binary/octet-stream',
'application/octet-stream',
'application/x-binary'
])
2016-10-02 11:14:08 +02:00
// ---------------------------------------------------------------------------
2018-08-30 14:58:00 +02:00
const OVERVIEWS = {
VIDEOS: {
2018-08-31 14:26:51 +02:00
SAMPLE_THRESHOLD: 6,
SAMPLES_COUNT: 20
2018-08-30 14:58:00 +02:00
}
}
// ---------------------------------------------------------------------------
2017-12-14 17:38:41 +01:00
const SERVER_ACTOR_NAME = 'peertube'
2017-11-14 17:31:26 +01:00
2017-11-09 17:51:58 +01:00
const ACTIVITY_PUB = {
2017-11-30 13:15:25 +01:00
POTENTIAL_ACCEPT_HEADERS: [
'application/activity+json',
2018-01-11 19:17:43 +01:00
'application/ld+json',
'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'
],
2017-11-30 13:15:25 +01:00
ACCEPT_HEADER: 'application/activity+json, application/ld+json',
2017-11-17 15:20:42 +01:00
PUBLIC: 'https://www.w3.org/ns/activitystreams#Public',
2017-11-10 14:34:45 +01:00
COLLECTION_ITEMS_PER_PAGE: 10,
2020-04-10 09:46:01 +02:00
FETCH_PAGE_LIMIT: 2000,
2017-11-16 15:22:39 +01:00
URL_MIME_TYPES: {
2019-05-16 16:55:34 +02:00
VIDEO: [] as string[],
2017-11-16 15:22:39 +01:00
TORRENT: [ 'application/x-bittorrent' ],
MAGNET: [ 'application/x-bittorrent;x-scheme-handler/magnet' ]
2018-01-04 14:04:02 +01:00
},
2018-01-10 17:18:12 +01:00
MAX_RECURSION_COMMENTS: 100,
2019-03-19 11:15:42 +01:00
ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days
2019-03-19 14:13:53 +01:00
VIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days
VIDEO_PLAYLIST_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2 // 2 days
2017-11-09 17:51:58 +01:00
}
2017-12-14 17:38:41 +01:00
const ACTIVITY_PUB_ACTOR_TYPES: { [ id: string ]: ActivityPubActorType } = {
GROUP: 'Group',
PERSON: 'Person',
APPLICATION: 'Application',
ORGANIZATION: 'Organization',
SERVICE: 'Service'
2017-12-14 17:38:41 +01:00
}
const HTTP_SIGNATURE = {
HEADER_NAME: 'signature',
ALGORITHM: 'rsa-sha256',
HEADERS_TO_SIGN: [ '(request-target)', 'host', 'date', 'digest' ],
CLOCK_SKEW_SECONDS: 1800
}
2016-10-02 11:14:08 +02:00
// ---------------------------------------------------------------------------
let PRIVATE_RSA_KEY_SIZE = 2048
2016-10-02 11:14:08 +02:00
// Password encryption
const BCRYPT_SALT_SIZE = 10
2016-05-17 21:03:00 +02:00
2022-10-10 11:12:23 +02:00
const ENCRYPTION = {
ALGORITHM: 'aes-256-cbc',
IV: 16,
SALT: 'peertube',
ENCODING: 'hex' as Encoding
}
const USER_PASSWORD_RESET_LIFETIME = 60000 * 60 // 60 minutes
const USER_PASSWORD_CREATE_LIFETIME = 60000 * 60 * 24 * 7 // 7 days
2018-01-30 13:27:07 +01:00
const TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME = 60000 * 10 // 10 minutes
2023-01-19 14:00:37 +01:00
const EMAIL_VERIFY_LIFETIME = 60000 * 60 // 60 minutes
2019-04-11 11:33:44 +02:00
const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = {
DO_NOT_LIST: 'do_not_list',
BLUR: 'blur',
DISPLAY: 'display'
}
// ---------------------------------------------------------------------------
// Express static paths (router)
const STATIC_PATHS = {
2016-11-11 15:20:03 +01:00
THUMBNAILS: '/static/thumbnails/',
2017-12-29 19:10:13 +01:00
WEBSEED: '/static/webseed/',
PRIVATE_WEBSEED: '/static/webseed/private/',
2018-12-04 17:08:55 +01:00
REDUNDANCY: '/static/redundancy/',
STREAMING_PLAYLISTS: {
HLS: '/static/streaming-playlists/hls',
PRIVATE_HLS: '/static/streaming-playlists/hls/private/'
}
}
2018-05-29 18:30:11 +02:00
const STATIC_DOWNLOAD_PATHS = {
TORRENTS: '/download/torrents/',
VIDEOS: '/download/videos/',
HLS_VIDEOS: '/download/streaming-playlists/hls/videos/'
2018-05-29 18:30:11 +02:00
}
2019-08-09 11:32:40 +02:00
const LAZY_STATIC_PATHS = {
2021-04-06 11:35:56 +02:00
BANNERS: '/lazy-static/banners/',
2019-08-09 11:32:40 +02:00
AVATARS: '/lazy-static/avatars/',
2020-08-26 15:03:00 +02:00
PREVIEWS: '/lazy-static/previews/',
VIDEO_CAPTIONS: '/lazy-static/video-captions/',
TORRENTS: '/lazy-static/torrents/'
2019-08-09 11:32:40 +02:00
}
const OBJECT_STORAGE_PROXY_PATHS = {
PRIVATE_WEBSEED: '/object-storage-proxy/webseed/private/',
STREAMING_PLAYLISTS: {
PRIVATE_HLS: '/object-storage-proxy/streaming-playlists/hls/private/'
}
}
// Cache control
2020-01-31 16:56:52 +01:00
const STATIC_MAX_AGE = {
2019-07-29 15:20:36 +02:00
SERVER: '2h',
LAZY_SERVER: '2d',
2019-07-29 15:20:36 +02:00
CLIENT: '30d'
}
// Videos thumbnail size
2017-10-16 10:05:49 +02:00
const THUMBNAILS_SIZE = {
2021-04-05 11:38:45 +02:00
width: 280,
height: 157,
minWidth: 150
2017-10-16 10:05:49 +02:00
}
const PREVIEWS_SIZE = {
2019-05-17 11:56:12 +02:00
width: 850,
height: 480,
minWidth: 400
}
2022-11-15 15:00:19 +01:00
const ACTOR_IMAGES_SIZE: { [key in ActorImageType]: { width: number, height: number }[] } = {
[ActorImageType.AVATAR]: [
{
width: 120,
height: 120
},
{
width: 48,
height: 48
}
],
[ActorImageType.BANNER]: [
{
width: 1920,
height: 317 // 6/1 ratio
}
]
2018-01-03 11:36:03 +01:00
}
const EMBED_SIZE = {
width: 560,
height: 315
2017-10-16 10:05:49 +02:00
}
2017-09-04 21:45:05 +02:00
// Sub folders of cache directory
2019-03-19 14:23:17 +01:00
const FILES_CACHE = {
2018-07-16 14:22:16 +02:00
PREVIEWS: {
DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'previews'),
MAX_AGE: 1000 * 3600 * 3 // 3 hours
},
VIDEO_CAPTIONS: {
DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'video-captions'),
MAX_AGE: 1000 * 3600 * 3 // 3 hours
},
TORRENTS: {
DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'torrents'),
MAX_AGE: 1000 * 3600 * 3 // 3 hours
2017-07-12 11:56:02 +02:00
}
}
2019-08-09 11:32:40 +02:00
const LRU_CACHE = {
2019-03-19 14:23:17 +01:00
USER_TOKENS: {
2019-08-09 11:32:40 +02:00
MAX_SIZE: 1000
},
2021-04-06 11:35:56 +02:00
ACTOR_IMAGE_STATIC: {
2019-08-09 11:32:40 +02:00
MAX_SIZE: 500
},
STATIC_VIDEO_FILES_RIGHTS_CHECK: {
MAX_SIZE: 5000,
TTL: parseDurationToMs('10 seconds')
},
VIDEO_TOKENS: {
MAX_SIZE: 100_000,
TTL: parseDurationToMs('8 hours')
},
TRACKER_IPS: {
MAX_SIZE: 100_000
2019-03-19 14:23:17 +01:00
}
}
const DIRECTORIES = {
RESUMABLE_UPLOAD: join(CONFIG.STORAGE.TMP_DIR, 'resumable-uploads'),
HLS_STREAMING_PLAYLIST: {
PUBLIC: join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls'),
PRIVATE: join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls', 'private')
},
VIDEOS: {
PUBLIC: CONFIG.STORAGE.VIDEOS_DIR,
PRIVATE: join(CONFIG.STORAGE.VIDEOS_DIR, 'private')
},
HLS_REDUNDANCY: join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls')
}
2019-01-29 08:37:25 +01:00
const RESUMABLE_UPLOAD_SESSION_LIFETIME = SCHEDULER_INTERVALS_MS.REMOVE_DANGLING_RESUMABLE_UPLOADS
const VIDEO_LIVE = {
EXTENSION: '.ts',
2020-09-25 16:19:35 +02:00
CLEANUP_DELAY: 1000 * 60 * 5, // 5 minutes
2022-03-04 13:40:02 +01:00
SEGMENT_TIME_SECONDS: {
DEFAULT_LATENCY: 4, // 4 seconds
SMALL_LATENCY: 2 // 2 seconds
},
2020-09-25 16:19:35 +02:00
SEGMENTS_LIST_SIZE: 15, // 15 maximum segments in live playlist
REPLAY_DIRECTORY: 'replay',
2020-11-18 11:24:28 +01:00
EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION: 4,
MAX_SOCKET_WAITING_DATA: 1024 * 1000 * 100, // 100MB
RTMP: {
CHUNK_SIZE: 60000,
GOP_CACHE: true,
PING: 60,
PING_TIMEOUT: 30,
BASE_PATH: 'live'
}
}
2018-09-14 11:52:23 +02:00
const MEMOIZE_TTL = {
2020-01-03 13:47:45 +01:00
OVERVIEWS_SAMPLE: 1000 * 3600 * 4, // 4 hours
INFO_HASH_EXISTS: 1000 * 60, // 1 minute
2022-06-17 14:34:37 +02:00
VIDEO_DURATION: 1000 * 10, // 10 seconds
LIVE_ABLE_TO_UPLOAD: 1000 * 60, // 1 minute
2022-07-05 15:43:21 +02:00
LIVE_CHECK_SOCKET_HEALTH: 1000 * 60, // 1 minute
GET_STATS_FOR_OPEN_TELEMETRY_METRICS: 1000 * 60 // 1 minute
2020-01-03 13:47:45 +01:00
}
const MEMOIZE_LENGTH = {
2022-06-17 14:34:37 +02:00
INFO_HASH_EXISTS: 200,
VIDEO_DURATION: 200
2018-09-14 11:52:23 +02:00
}
const WORKER_THREADS = {
DOWNLOAD_IMAGE: {
CONCURRENCY: 3,
MAX_THREADS: 1
2022-06-27 11:53:12 +02:00
},
PROCESS_IMAGE: {
CONCURRENCY: 1,
MAX_THREADS: 5
}
2019-08-09 11:32:40 +02:00
}
2018-09-11 16:27:07 +02:00
const REDUNDANCY = {
VIDEOS: {
RANDOMIZED_FACTOR: 5
}
}
2017-12-19 10:34:56 +01:00
const ACCEPT_HEADERS = [ 'html', 'application/json' ].concat(ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS)
const OTP = {
HEADER_NAME: 'x-peertube-otp',
HEADER_REQUIRED_VALUE: 'required; app'
}
2017-11-30 13:37:11 +01:00
2019-05-16 16:55:34 +02:00
const ASSETS_PATH = {
DEFAULT_AUDIO_BACKGROUND: join(root(), 'dist', 'server', 'assets', 'default-audio-background.jpg'),
DEFAULT_LIVE_BACKGROUND: join(root(), 'dist', 'server', 'assets', 'default-live-background.jpg')
2019-05-16 16:55:34 +02:00
}
// ---------------------------------------------------------------------------
const CUSTOM_HTML_TAG_COMMENTS = {
TITLE: '<!-- title tag -->',
DESCRIPTION: '<!-- description tag -->',
CUSTOM_CSS: '<!-- custom css tag -->',
2021-05-14 12:04:44 +02:00
META_TAGS: '<!-- meta tags -->',
SERVER_CONFIG: '<!-- server config -->'
}
2017-07-07 16:57:28 +02:00
2019-04-10 15:26:33 +02:00
const MAX_LOGS_OUTPUT_CHARACTERS = 10 * 1000 * 1000
2019-12-11 14:14:01 +01:00
const LOG_FILENAME = 'peertube.log'
const AUDIT_LOG_FILENAME = 'peertube-audit.log'
2019-04-10 15:26:33 +02:00
2018-04-17 14:01:06 +02:00
// ---------------------------------------------------------------------------
2018-06-26 16:53:24 +02:00
const TRACKER_RATE_LIMITS = {
INTERVAL: 60000 * 5, // 5 minutes
2018-06-29 09:33:36 +02:00
ANNOUNCES_PER_IP_PER_INFOHASH: 15, // maximum announces per torrent in the interval
2020-06-25 16:27:35 +02:00
ANNOUNCES_PER_IP: 30, // maximum announces for all our torrents in the interval
BLOCK_IP_LIFETIME: parseDurationToMs('3 minutes')
2018-06-26 16:53:24 +02:00
}
2019-04-08 11:13:49 +02:00
const P2P_MEDIA_LOADER_PEER_VERSION = 2
2018-06-26 16:53:24 +02:00
// ---------------------------------------------------------------------------
const PLUGIN_GLOBAL_CSS_FILE_NAME = 'plugins-global.css'
const PLUGIN_GLOBAL_CSS_PATH = join(CONFIG.STORAGE.TMP_DIR, PLUGIN_GLOBAL_CSS_FILE_NAME)
2020-04-29 09:04:42 +02:00
let PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME = 1000 * 60 * 5 // 5 minutes
2019-07-10 14:06:19 +02:00
const DEFAULT_THEME_NAME = 'default'
const DEFAULT_USER_THEME_NAME = 'instance-default'
2019-07-09 11:45:19 +02:00
// ---------------------------------------------------------------------------
2020-05-29 16:16:24 +02:00
const SEARCH_INDEX = {
ROUTES: {
VIDEOS: '/api/v1/search/videos',
VIDEO_CHANNELS: '/api/v1/search/video-channels'
}
}
// ---------------------------------------------------------------------------
const STATS_TIMESERIE = {
MAX_DAYS: 365 * 10 // Around 10 years
}
// ---------------------------------------------------------------------------
// Special constants for a test instance
if (process.env.PRODUCTION_CONSTANTS !== 'true') {
if (isTestOrDevInstance()) {
PRIVATE_RSA_KEY_SIZE = 1024
ACTOR_FOLLOW_SCORE.BASE = 20
REMOTE_SCHEME.HTTP = 'http'
REMOTE_SCHEME.WS = 'ws'
STATIC_MAX_AGE.SERVER = '0'
SCHEDULER_INTERVALS_MS.ACTOR_FOLLOW_SCORES = 1000
SCHEDULER_INTERVALS_MS.REMOVE_OLD_JOBS = 10000
SCHEDULER_INTERVALS_MS.REMOVE_OLD_HISTORY = 5000
SCHEDULER_INTERVALS_MS.REMOVE_OLD_VIEWS = 5000
SCHEDULER_INTERVALS_MS.UPDATE_VIDEOS = 5000
SCHEDULER_INTERVALS_MS.AUTO_FOLLOW_INDEX_INSTANCES = 5000
SCHEDULER_INTERVALS_MS.UPDATE_INBOX_STATS = 5000
SCHEDULER_INTERVALS_MS.CHECK_PEERTUBE_VERSION = 2000
REPEAT_JOBS['videos-views-stats'] = { every: 5000 }
REPEAT_JOBS['activitypub-cleaner'] = { every: 5000 }
AP_CLEANER.PERIOD = 5000
2021-12-28 11:36:51 +01:00
REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1
2021-12-28 11:36:51 +01:00
CONTACT_FORM_LIFETIME = 1000 // 1 second
JOB_ATTEMPTS['email'] = 1
2018-09-11 16:27:07 +02:00
FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000
MEMOIZE_TTL.OVERVIEWS_SAMPLE = 3000
MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD = 3000
OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD = 2
PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME = 5000
JOB_REMOVAL_OPTIONS.SUCCESS['videos-views-stats'] = 10000
}
if (isTestInstance()) {
ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2
ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
ACTIVITY_PUB.VIDEO_PLAYLIST_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
2018-07-16 14:22:16 +02:00
CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max = 100 * 1024 // 100KB
CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max = 400 * 1024 // 400KB
2020-04-29 09:04:42 +02:00
VIEW_LIFETIME.VIEWER_COUNTER = 1000 * 5 // 5 second
VIEW_LIFETIME.VIEWER_STATS = 1000 * 5 // 5 second
2020-10-27 16:06:24 +01:00
VIDEO_LIVE.CLEANUP_DELAY = getIntEnv('PEERTUBE_TEST_CONSTANTS_VIDEO_LIVE_CLEANUP_DELAY') ?? 5000
VIDEO_LIVE.SEGMENT_TIME_SECONDS.DEFAULT_LATENCY = 2
VIDEO_LIVE.SEGMENT_TIME_SECONDS.SMALL_LATENCY = 1
VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION = 1
}
}
2018-12-11 14:52:50 +01:00
updateWebserverUrls()
2019-05-16 16:55:34 +02:00
updateWebserverConfig()
2019-04-11 11:33:44 +02:00
registerConfigChangedHandler(() => {
updateWebserverUrls()
updateWebserverConfig()
})
// ---------------------------------------------------------------------------
const FILES_CONTENT_HASH = {
MANIFEST: generateContentHash(),
FAVICON: generateContentHash(),
LOGO: generateContentHash()
}
// ---------------------------------------------------------------------------
2022-02-11 10:51:33 +01:00
const VIDEO_FILTERS = {
WATERMARK: {
SIZE_RATIO: 1 / 10,
HORIZONTAL_MARGIN_RATIO: 1 / 20,
VERTICAL_MARGIN_RATIO: 1 / 20
}
}
// ---------------------------------------------------------------------------
2017-05-15 22:22:03 +02:00
export {
2019-04-11 11:33:44 +02:00
WEBSERVER,
2016-10-02 11:14:08 +02:00
API_VERSION,
2022-10-10 11:12:23 +02:00
ENCRYPTION,
VIDEO_LIVE,
2019-07-17 10:03:55 +02:00
PEERTUBE_VERSION,
2019-08-09 11:32:40 +02:00
LAZY_STATIC_PATHS,
OBJECT_STORAGE_PROXY_PATHS,
2020-05-29 16:16:24 +02:00
SEARCH_INDEX,
DIRECTORIES,
RESUMABLE_UPLOAD_SESSION_LIFETIME,
RUNNER_JOB_STATES,
2019-04-08 11:13:49 +02:00
P2P_MEDIA_LOADER_PEER_VERSION,
2021-04-06 17:01:35 +02:00
ACTOR_IMAGES_SIZE,
2017-11-30 13:37:11 +01:00
ACCEPT_HEADERS,
2016-10-02 11:14:08 +02:00
BCRYPT_SALT_SIZE,
2018-06-26 16:53:24 +02:00
TRACKER_RATE_LIMITS,
2019-03-19 14:23:17 +01:00
FILES_CACHE,
2019-12-11 14:14:01 +01:00
LOG_FILENAME,
2016-10-02 11:14:08 +02:00
CONSTRAINTS_FIELDS,
EMBED_SIZE,
2018-09-11 16:27:07 +02:00
REDUNDANCY,
JOB_CONCURRENCY,
JOB_ATTEMPTS,
2021-12-28 11:36:51 +01:00
AP_CLEANER,
2016-12-25 09:44:57 +01:00
LAST_MIGRATION_VERSION,
CUSTOM_HTML_TAG_COMMENTS,
STATS_TIMESERIE,
2018-04-18 16:04:49 +02:00
BROADCAST_CONCURRENCY,
2019-12-11 14:14:01 +01:00
AUDIT_LOG_FILENAME,
2018-07-24 16:27:45 +02:00
PAGINATION,
ACTOR_FOLLOW_SCORE,
2016-11-11 15:20:03 +01:00
PREVIEWS_SIZE,
REMOTE_SCHEME,
2017-11-13 17:39:41 +01:00
FOLLOW_STATES,
2019-07-10 14:06:19 +02:00
DEFAULT_USER_THEME_NAME,
2017-12-14 17:38:41 +01:00
SERVER_ACTOR_NAME,
TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME,
PLUGIN_GLOBAL_CSS_FILE_NAME,
PLUGIN_GLOBAL_CSS_PATH,
2017-11-09 17:51:58 +01:00
PRIVATE_RSA_KEY_SIZE,
2022-02-11 10:51:33 +01:00
VIDEO_FILTERS,
2018-05-11 09:44:04 +02:00
ROUTE_CACHE_LIFETIME,
2016-10-02 11:14:08 +02:00
SORTABLE_COLUMNS,
2018-08-03 10:19:51 +02:00
JOB_TTL,
2019-07-10 14:06:19 +02:00
DEFAULT_THEME_NAME,
NSFW_POLICY_TYPES,
STATIC_MAX_AGE,
STATIC_PATHS,
VIDEO_IMPORT_TIMEOUT,
2019-03-05 10:58:44 +01:00
VIDEO_PLAYLIST_TYPES,
2019-04-10 15:26:33 +02:00
MAX_LOGS_OUTPUT_CHARACTERS,
2017-11-09 17:51:58 +01:00
ACTIVITY_PUB,
2017-12-14 17:38:41 +01:00
ACTIVITY_PUB_ACTOR_TYPES,
2016-10-02 11:14:08 +02:00
THUMBNAILS_SIZE,
2017-03-22 21:15:55 +01:00
VIDEO_CATEGORIES,
2020-01-03 13:47:45 +01:00
MEMOIZE_LENGTH,
2017-04-07 12:13:37 +02:00
VIDEO_LANGUAGES,
2017-10-31 11:52:52 +01:00
VIDEO_PRIVACIES,
2017-03-27 20:53:11 +02:00
VIDEO_LICENCES,
VIDEO_STATES,
WORKER_THREADS,
2017-11-10 14:34:45 +01:00
VIDEO_RATE_TYPES,
JOB_PRIORITY,
VIDEO_TRANSCODING_FPS,
FFMPEG_NICE,
2020-07-01 16:05:30 +02:00
ABUSE_STATES,
2023-01-19 14:00:37 +01:00
USER_REGISTRATION_STATES,
2019-08-09 11:32:40 +02:00
LRU_CACHE,
2021-11-29 15:45:02 +01:00
REQUEST_TIMEOUTS,
RUNNER_JOBS,
MAX_LOCAL_VIEWER_WATCH_SECTIONS,
2018-01-30 13:27:07 +01:00
USER_PASSWORD_RESET_LIFETIME,
USER_PASSWORD_CREATE_LIFETIME,
2018-09-14 11:52:23 +02:00
MEMOIZE_TTL,
2023-01-19 14:00:37 +01:00
EMAIL_VERIFY_LIFETIME,
2018-08-30 14:58:00 +02:00
OVERVIEWS,
SCHEDULER_INTERVALS_MS,
2018-08-29 16:26:25 +02:00
REPEAT_JOBS,
2018-05-29 18:30:11 +02:00
STATIC_DOWNLOAD_PATHS,
2018-12-11 14:52:50 +01:00
MIMETYPES,
CRAWL_REQUEST_CONCURRENCY,
2019-05-16 16:55:34 +02:00
DEFAULT_AUDIO_RESOLUTION,
BINARY_CONTENT_TYPES,
JOB_REMOVAL_OPTIONS,
HTTP_SIGNATURE,
VIDEO_IMPORT_STATES,
Channel sync (#5135) * Add external channel URL for channel update / creation (#754) * Disallow synchronisation if user has no video quota (#754) * More constraints serverside (#754) * Disable sync if server configuration does not allow HTTP import (#754) * Working version synchronizing videos with a job (#754) TODO: refactoring, too much code duplication * More logs and try/catch (#754) * Fix eslint error (#754) * WIP: support synchronization time change (#754) * New frontend #754 * WIP: Create sync front (#754) * Enhance UI, sync creation form (#754) * Warning message when HTTP upload is disallowed * More consistent names (#754) * Binding Front with API (#754) * Add a /me API (#754) * Improve list UI (#754) * Implement creation and deletion routes (#754) * Lint (#754) * Lint again (#754) * WIP: UI for triggering import existing videos (#754) * Implement jobs for syncing and importing channels * Don't sync videos before sync creation + avoid concurrency issue (#754) * Cleanup (#754) * Cleanup: OpenAPI + API rework (#754) * Remove dead code (#754) * Eslint (#754) * Revert the mess with whitespaces in constants.ts (#754) * Some fixes after rebase (#754) * Several fixes after PR remarks (#754) * Front + API: Rename video-channels-sync to video-channel-syncs (#754) * Allow enabling channel sync through UI (#754) * getChannelInfo (#754) * Minor fixes: openapi + model + sql (#754) * Simplified API validators (#754) * Rename MChannelSync to MChannelSyncChannel (#754) * Add command for VideoChannelSync (#754) * Use synchronization.enabled config (#754) * Check parameters test + some fixes (#754) * Fix conflict mistake (#754) * Restrict access to video channel sync list API (#754) * Start adding unit test for synchronization (#754) * Continue testing (#754) * Tests finished + convertion of job to scheduler (#754) * Add lastSyncAt field (#754) * Fix externalRemoteUrl sort + creation date not well formatted (#754) * Small fix (#754) * Factorize addYoutubeDLImport and buildVideo (#754) * Check duplicates on channel not on users (#754) * factorize thumbnail generation (#754) * Fetch error should return status 400 (#754) * Separate video-channel-import and video-channel-sync-latest (#754) * Bump DB migration version after rebase (#754) * Prettier states in UI table (#754) * Add DefaultScope in VideoChannelSyncModel (#754) * Fix audit logs (#754) * Ensure user can upload when importing channel + minor fixes (#754) * Mark synchronization as failed on exception + typos (#754) * Change REST API for importing videos into channel (#754) * Add option for fully synchronize a chnanel (#754) * Return a whole sync object on creation to avoid tricks in Front (#754) * Various remarks (#754) * Single quotes by default (#754) * Rename synchronization to video_channel_synchronization * Add check.latest_videos_count and max_per_user options (#754) * Better channel rendering in list #754 * Allow sorting with channel name and state (#754) * Add missing tests for channel imports (#754) * Prefer using a parent job for channel sync * Styling * Client styling Co-authored-by: Chocobozzz <me@florianbigard.com>
2022-08-10 09:53:39 +02:00
VIDEO_CHANNEL_SYNC_STATE,
2020-11-06 16:42:23 +01:00
VIEW_LIFETIME,
2019-01-09 15:14:29 +01:00
CONTACT_FORM_LIFETIME,
2019-02-26 10:55:40 +01:00
VIDEO_PLAYLIST_PRIVACIES,
2020-04-29 09:04:42 +02:00
PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME,
2019-05-16 16:55:34 +02:00
ASSETS_PATH,
FILES_CONTENT_HASH,
OTP,
loadLanguages,
buildLanguages,
generateContentHash
}
// ---------------------------------------------------------------------------
2018-12-11 14:52:50 +01:00
function buildVideoMimetypeExt () {
const data = {
// streamable formats that warrant cross-browser compatibility
2018-12-11 14:52:50 +01:00
'video/webm': '.webm',
// We'll add .ogg if additional extensions are enabled
// We could add .ogg here but since it could be an audio file,
// it would be confusing for users because PeerTube will refuse their file (based on the mimetype)
'video/ogg': [ '.ogv' ],
2018-12-11 14:52:50 +01:00
'video/mp4': '.mp4'
}
2019-05-16 16:55:34 +02:00
if (CONFIG.TRANSCODING.ENABLED) {
if (CONFIG.TRANSCODING.ALLOW_ADDITIONAL_EXTENSIONS) {
data['video/ogg'].push('.ogg')
2019-05-16 16:55:34 +02:00
Object.assign(data, {
'video/x-matroska': '.mkv',
// Developed by Apple
'video/quicktime': [ '.mov', '.qt', '.mqv' ], // often used as output format by editing software
2019-12-05 11:07:57 +01:00
'video/x-m4v': '.m4v',
'video/m4v': '.m4v',
// Developed by the Adobe Flash Platform
'video/x-flv': '.flv',
'video/x-f4v': '.f4v', // replacement for flv
// Developed by Microsoft
'video/x-ms-wmv': '.wmv',
'video/x-msvideo': '.avi',
'video/avi': '.avi',
// Developed by 3GPP
// common video formats for cell phones
'video/3gpp': [ '.3gp', '.3gpp' ],
'video/3gpp2': [ '.3g2', '.3gpp2' ],
// Developed by FFmpeg/Mplayer
'application/x-nut': '.nut',
// The standard video format used by many Sony and Panasonic HD camcorders.
// It is also used for storing high definition video on Blu-ray discs.
'video/mp2t': '.mts',
2022-04-20 10:23:07 +02:00
'video/vnd.dlna.mpeg-tts': '.mts',
'video/m2ts': '.m2ts',
// Old formats reliant on MPEG-1/MPEG-2
'video/mpv': '.mpv',
'video/mpeg2': '.m2v',
'video/mpeg': [ '.m1v', '.mpg', '.mpe', '.mpeg', '.vob' ],
'video/dvd': '.vob',
// Could be anything
'application/octet-stream': null,
'application/mxf': '.mxf' // often used as exchange format by editing software
2019-05-16 16:55:34 +02:00
})
}
if (CONFIG.TRANSCODING.ALLOW_AUDIO_FILES) {
Object.assign(data, MIMETYPES.AUDIO.MIMETYPE_EXT)
}
2018-12-11 14:52:50 +01:00
}
return data
}
function updateWebserverUrls () {
2019-04-11 11:33:44 +02:00
WEBSERVER.URL = sanitizeUrl(CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT)
WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP)
WEBSERVER.WS = CONFIG.WEBSERVER.WS
WEBSERVER.SCHEME = CONFIG.WEBSERVER.SCHEME
2019-04-11 11:33:44 +02:00
WEBSERVER.HOSTNAME = CONFIG.WEBSERVER.HOSTNAME
WEBSERVER.PORT = CONFIG.WEBSERVER.PORT
const rtmpHostname = CONFIG.LIVE.RTMP.PUBLIC_HOSTNAME || CONFIG.WEBSERVER.HOSTNAME
const rtmpsHostname = CONFIG.LIVE.RTMPS.PUBLIC_HOSTNAME || CONFIG.WEBSERVER.HOSTNAME
WEBSERVER.RTMP_URL = 'rtmp://' + rtmpHostname + ':' + CONFIG.LIVE.RTMP.PORT + '/' + VIDEO_LIVE.RTMP.BASE_PATH
WEBSERVER.RTMPS_URL = 'rtmps://' + rtmpsHostname + ':' + CONFIG.LIVE.RTMPS.PORT + '/' + VIDEO_LIVE.RTMP.BASE_PATH
}
2018-12-11 14:52:50 +01:00
function updateWebserverConfig () {
MIMETYPES.VIDEO.MIMETYPE_EXT = buildVideoMimetypeExt()
MIMETYPES.VIDEO.MIMETYPES_REGEX = buildMimetypesRegex(MIMETYPES.VIDEO.MIMETYPE_EXT)
2019-05-16 16:55:34 +02:00
ACTIVITY_PUB.URL_MIME_TYPES.VIDEO = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT)
MIMETYPES.VIDEO.EXT_MIMETYPE = buildVideoExtMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT)
CONSTRAINTS_FIELDS.VIDEOS.EXTNAME = Object.keys(MIMETYPES.VIDEO.EXT_MIMETYPE)
}
function buildVideoExtMimetype (obj: { [ id: string ]: string | string[] }) {
const result: { [id: string]: string } = {}
for (const mimetype of Object.keys(obj)) {
const value = obj[mimetype]
if (!value) continue
const extensions = Array.isArray(value) ? value : [ value ]
for (const extension of extensions) {
result[extension] = mimetype
}
}
return result
2018-12-11 14:52:50 +01:00
}
function buildMimetypesRegex (obj: { [id: string]: string | string[] }) {
return Object.keys(obj)
.map(m => `(${m})`)
.join('|')
2018-12-11 14:52:50 +01:00
}
function loadLanguages () {
Object.assign(VIDEO_LANGUAGES, buildLanguages())
}
2018-04-23 14:39:52 +02:00
function buildLanguages () {
const iso639 = require('iso-639-3')
2020-01-31 16:56:52 +01:00
const languages: { [id: string]: string } = {}
2018-04-23 14:39:52 +02:00
const additionalLanguages = {
2020-01-31 16:56:52 +01:00
sgn: true, // Sign languages (macro language)
ase: true, // American sign language
2022-12-09 09:18:39 +01:00
asq: true, // Austrian sign language
2020-01-31 16:56:52 +01:00
sdl: true, // Arabian sign language
bfi: true, // British sign language
bzs: true, // Brazilian sign language
csl: true, // Chinese sign language
cse: true, // Czech sign language
dsl: true, // Danish sign language
fsl: true, // French sign language
gsg: true, // German sign language
pks: true, // Pakistan sign language
jsl: true, // Japanese sign language
sfs: true, // South African sign language
swl: true, // Swedish sign language
2020-08-11 09:41:57 +02:00
rsl: true, // Russian sign language
kab: true, // Kabyle
2020-01-31 16:56:52 +01:00
2021-09-07 09:05:57 +02:00
lat: true, // Latin
2020-01-31 16:56:52 +01:00
epo: true, // Esperanto
tlh: true, // Klingon
jbo: true, // Lojban
avk: true, // Kotava
zxx: true // No linguistic content (ISO-639-2)
}
2018-04-23 14:39:52 +02:00
// Only add ISO639-1 languages and some sign languages (ISO639-3)
iso639
.filter(l => {
2020-02-06 17:06:11 +01:00
return (l.iso6391 !== undefined && l.type === 'living') ||
2020-01-31 16:56:52 +01:00
additionalLanguages[l.iso6393] === true
2018-04-23 14:39:52 +02:00
})
2020-01-31 16:56:52 +01:00
.forEach(l => { languages[l.iso6391 || l.iso6393] = l.name })
2018-04-23 14:39:52 +02:00
2018-09-05 15:59:43 +02:00
// Override Occitan label
2020-01-31 16:56:52 +01:00
languages['oc'] = 'Occitan'
languages['el'] = 'Greek'
2022-06-24 11:59:12 +02:00
languages['tok'] = 'Toki Pona'
2018-09-05 15:59:43 +02:00
// Chinese languages
languages['zh-Hans'] = 'Simplified Chinese'
languages['zh-Hant'] = 'Traditional Chinese'
2018-04-23 14:39:52 +02:00
return languages
}
function generateContentHash () {
return randomBytes(20).toString('hex')
}
function getIntEnv (path: string) {
if (process.env[path]) return parseInt(process.env[path])
return undefined
}