PeerTube/server/initializers/constants.ts

1093 lines
30 KiB
TypeScript
Raw Normal View History

import { CronRepeatOptions, EveryRepeatOptions } from 'bull'
2020-11-10 17:04:09 +01:00
import { randomBytes } from 'crypto'
import { invert } from 'lodash'
2019-04-11 11:33:44 +02:00
import { join } from 'path'
2021-07-26 15:04:37 +02:00
import { randomInt } from '../../shared/core-utils/common/miscs'
2020-07-01 16:05:30 +02:00
import {
AbuseState,
JobType,
2020-07-01 16:05:30 +02:00
VideoImportState,
VideoPrivacy,
VideoRateType,
VideoResolution,
VideoState,
VideoTranscodingFPS
2020-07-01 16:05:30 +02:00
} from '../../shared/models'
import { ActivityPubActorType } from '../../shared/models/activitypub'
import { FollowState } from '../../shared/models/actors'
import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
2019-02-26 10:55:40 +01:00
import { VideoPlaylistPrivacy } from '../../shared/models/videos/playlist/video-playlist-privacy.model'
2019-03-05 10:58:44 +01:00
import { VideoPlaylistType } from '../../shared/models/videos/playlist/video-playlist-type.model'
// Do not use barrels, remain constants as independent as possible
import { isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils'
2019-04-11 11:33:44 +02:00
import { CONFIG, registerConfigChangedHandler } from './config'
2017-06-16 10:36:18 +02:00
2016-10-02 11:14:08 +02:00
// ---------------------------------------------------------------------------
2021-08-30 16:28:36 +02:00
const LAST_MIGRATION_VERSION = 670
2017-02-18 11:56:28 +01:00
// ---------------------------------------------------------------------------
2016-03-16 22:29:27 +01:00
const API_VERSION = 'v1'
2021-03-08 14:24:11 +01:00
const PEERTUBE_VERSION: string = require(join(root(), 'package.json')).version
2018-07-24 16:27:45 +02:00
const PAGINATION = {
GLOBAL: {
COUNT: {
DEFAULT: 15,
MAX: 100
}
},
OUTBOX: {
COUNT: {
MAX: 50
}
2018-07-24 16:27:45 +02:00
}
}
2016-10-02 11:14:08 +02:00
2019-04-11 11:33:44 +02:00
const WEBSERVER = {
URL: '',
HOST: '',
SCHEME: '',
WS: '',
HOSTNAME: '',
PORT: 0,
2021-11-05 11:36:03 +01:00
RTMP_URL: '',
RTMPS_URL: ''
2019-04-11 11:33:44 +02:00
}
2016-10-02 11:14:08 +02:00
// Sortable columns per schema
const SORTABLE_COLUMNS = {
2020-07-15 11:15:50 +02:00
USERS: [ 'id', 'username', 'videoQuotaUsed', 'createdAt', 'lastLoginDate', 'role' ],
USER_SUBSCRIPTIONS: [ 'id', 'createdAt' ],
2018-01-03 16:38:50 +01:00
ACCOUNTS: [ 'createdAt' ],
JOBS: [ 'createdAt' ],
2017-10-24 19:41:09 +02:00
VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ],
2018-08-02 17:48:50 +02:00
VIDEO_IMPORTS: [ 'createdAt' ],
2020-11-13 16:38:23 +01:00
2019-12-27 17:02:34 +01:00
VIDEO_COMMENT_THREADS: [ 'createdAt', 'totalReplies' ],
2020-11-13 16:38:23 +01:00
VIDEO_COMMENTS: [ 'createdAt' ],
VIDEO_RATES: [ 'createdAt' ],
2017-11-13 17:39:41 +01:00
BLACKLISTS: [ 'id', 'name', 'duration', 'views', 'likes', 'dislikes', 'uuid', 'createdAt' ],
2021-10-19 09:44:43 +02:00
2021-10-19 09:10:01 +02:00
INSTANCE_FOLLOWERS: [ 'createdAt', 'state', 'score' ],
INSTANCE_FOLLOWING: [ 'createdAt', 'redundancyAllowed', 'state' ],
2021-10-19 09:44:43 +02:00
ACCOUNT_FOLLOWERS: [ 'createdAt' ],
CHANNEL_FOLLOWERS: [ 'createdAt' ],
2018-07-19 16:17:54 +02:00
VIDEOS: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'trending', 'hot', 'best' ],
2018-08-31 17:18:13 +02:00
2020-05-29 16:16:24 +02:00
// Don't forget to update peertube-search-index with the same values
VIDEOS_SEARCH: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'match' ],
VIDEO_CHANNELS_SEARCH: [ 'match', 'displayName', 'createdAt' ],
2021-06-17 16:02:38 +02:00
VIDEO_PLAYLISTS_SEARCH: [ 'match', 'displayName', 'createdAt' ],
2020-07-01 16:05:30 +02:00
ABUSES: [ 'id', 'createdAt', 'state' ],
ACCOUNTS_BLOCKLIST: [ 'createdAt' ],
2018-12-26 10:36:24 +01:00
SERVERS_BLOCKLIST: [ 'createdAt' ],
2020-07-15 11:15:50 +02:00
USER_NOTIFICATIONS: [ 'createdAt', 'read' ],
2019-02-26 10:55:40 +01:00
VIDEO_PLAYLISTS: [ 'name', 'displayName', 'createdAt', 'updatedAt' ],
PLUGINS: [ 'name', 'createdAt', 'updatedAt' ],
2020-01-10 10:11:28 +01:00
AVAILABLE_PLUGINS: [ 'npmName', 'popularity' ],
VIDEO_REDUNDANCIES: [ 'name' ]
2016-10-02 11:14:08 +02:00
}
2016-07-20 16:23:58 +02:00
const OAUTH_LIFETIME = {
2018-06-08 08:17:41 +02:00
ACCESS_TOKEN: 3600 * 24, // 1 day, for upload
2016-07-20 16:23:58 +02:00
REFRESH_TOKEN: 1209600 // 2 weeks
}
2018-05-11 09:44:04 +02:00
const ROUTE_CACHE_LIFETIME = {
2018-07-21 23:00:25 +02:00
FEEDS: '15 minutes',
ROBOTS: '2 hours',
2018-12-05 17:27:24 +01:00
SITEMAP: '1 day',
SECURITYTXT: '2 hours',
2018-07-21 23:00:25 +02:00
NODEINFO: '10 minutes',
DNT_POLICY: '1 week',
2018-05-11 09:44:04 +02:00
ACTIVITY_PUB: {
2018-07-21 23:00:25 +02:00
VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example
2018-09-14 14:57:59 +02:00
},
STATS: '4 hours'
2018-05-11 09:44:04 +02:00
}
2016-10-02 11:14:08 +02:00
// ---------------------------------------------------------------------------
2016-08-25 17:57:37 +02:00
// Number of points we add/remove after a successful/bad request
const ACTOR_FOLLOW_SCORE = {
PENALTY: -10,
BONUS: 10,
BASE: 1000,
MAX: 10000
}
const FOLLOW_STATES: { [ id: string ]: FollowState } = {
PENDING: 'pending',
ACCEPTED: 'accepted'
}
const REMOTE_SCHEME = {
HTTP: 'https',
WS: 'wss'
}
2020-01-10 10:11:28 +01:00
const JOB_ATTEMPTS: { [id in JobType]: number } = {
'activitypub-http-broadcast': 1,
'activitypub-http-unicast': 1,
'activitypub-http-fetcher': 2,
'activitypub-follow': 5,
'activitypub-cleaner': 1,
'video-file-import': 1,
'video-transcoding': 1,
'video-import': 1,
2018-08-29 16:26:25 +02:00
'email': 5,
'actor-keys': 3,
'videos-views': 1,
2020-01-10 10:11:28 +01:00
'activitypub-refresher': 1,
'video-redundancy': 1,
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 08:26:20 +02:00
'video-live-ending': 1,
'move-to-object-storage': 3
}
// Excluded keys are jobs that can be configured by admins
const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-import'>]: number } = {
'activitypub-http-broadcast': 1,
'activitypub-http-unicast': 10,
2021-06-15 15:31:08 +02:00
'activitypub-http-fetcher': 3,
'activitypub-cleaner': 1,
2020-01-10 11:15:07 +01:00
'activitypub-follow': 1,
'video-file-import': 1,
2018-08-29 16:26:25 +02:00
'email': 5,
'actor-keys': 1,
'videos-views': 1,
2020-01-10 10:11:28 +01:00
'activitypub-refresher': 1,
'video-redundancy': 1,
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 08:26:20 +02:00
'video-live-ending': 10,
'move-to-object-storage': 1
}
2020-01-10 10:11:28 +01:00
const JOB_TTL: { [id in JobType]: number } = {
2018-08-03 10:19:51 +02:00
'activitypub-http-broadcast': 60000 * 10, // 10 minutes
'activitypub-http-unicast': 60000 * 10, // 10 minutes
2020-04-11 09:06:15 +02:00
'activitypub-http-fetcher': 1000 * 3600 * 10, // 10 hours
2018-08-03 10:19:51 +02:00
'activitypub-follow': 60000 * 10, // 10 minutes
'activitypub-cleaner': 1000 * 3600, // 1 hour
2018-08-03 10:19:51 +02:00
'video-file-import': 1000 * 3600, // 1 hour
'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long
2020-04-11 09:06:15 +02:00
'video-import': 1000 * 3600 * 2, // 2 hours
2018-08-29 16:26:25 +02:00
'email': 60000 * 10, // 10 minutes
'actor-keys': 60000 * 20, // 20 minutes
'videos-views': undefined, // Unlimited
2020-01-10 10:11:28 +01:00
'activitypub-refresher': 60000 * 10, // 10 minutes
'video-redundancy': 1000 * 3600 * 3, // 3 hours
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 08:26:20 +02:00
'video-live-ending': 1000 * 60 * 10, // 10 minutes
'move-to-object-storage': 1000 * 60 * 60 * 3 // 3 hours
2018-08-03 10:19:51 +02:00
}
2018-08-29 16:26:25 +02:00
const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = {
'videos-views': {
cron: randomInt(1, 20) + ' * * * *' // Between 1-20 minutes past the hour
},
'activitypub-cleaner': {
cron: '30 5 * * ' + randomInt(0, 7) // 1 time per week (random day) at 5:30 AM
2018-08-29 16:26:25 +02:00
}
}
const JOB_PRIORITY = {
TRANSCODING: 100
}
2018-08-29 16:26:25 +02:00
2021-03-03 10:09:34 +01:00
const BROADCAST_CONCURRENCY = 30 // How many requests in parallel we do in activitypub-http-broadcast job
const AP_CLEANER_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-cleaner job
2018-08-23 10:30:53 +02:00
const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...)
2021-03-03 10:10:55 +01:00
const REQUEST_TIMEOUT = 7000 // 7 seconds
2018-05-09 09:08:22 +02:00
const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days
const VIDEO_IMPORT_TIMEOUT = 1000 * 3600 // 1 hour
const SCHEDULER_INTERVALS_MS = {
2021-10-22 10:28:00 +02:00
ACTOR_FOLLOW_SCORES: 60000 * 60, // 1 hour
REMOVE_OLD_JOBS: 60000 * 60, // 1 hour
UPDATE_VIDEOS: 60000, // 1 minute
YOUTUBE_DL_UPDATE: 60000 * 60 * 24, // 1 day
CHECK_PLUGINS: CONFIG.PLUGINS.INDEX.CHECK_LATEST_VERSIONS_INTERVAL,
CHECK_PEERTUBE_VERSION: 60000 * 60 * 24, // 1 day
AUTO_FOLLOW_INDEX_INSTANCES: 60000 * 60 * 24, // 1 day
REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day
REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day
UPDATE_INBOX_STATS: 1000 * 60, // 1 minute
REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60 * 16 // 16 hours
}
// ---------------------------------------------------------------------------
const CONSTRAINTS_FIELDS = {
2016-07-31 20:58:43 +02:00
USERS: {
NAME: { min: 1, max: 120 }, // Length
DESCRIPTION: { min: 3, max: 1000 }, // Length
USERNAME: { min: 1, max: 50 }, // Length
2017-09-04 20:07:54 +02:00
PASSWORD: { min: 6, max: 255 }, // Length
2018-08-08 17:36:10 +02:00
VIDEO_QUOTA: { min: -1 },
VIDEO_QUOTA_DAILY: { min: -1 },
VIDEO_LANGUAGES: { max: 500 }, // Array length
2018-08-08 17:36:10 +02:00
BLOCKED_REASON: { min: 3, max: 250 } // Length
2016-07-31 20:58:43 +02:00
},
2020-07-01 16:05:30 +02:00
ABUSES: {
REASON: { min: 2, max: 3000 }, // Length
MODERATION_COMMENT: { min: 2, max: 3000 } // Length
2017-01-04 20:59:23 +01:00
},
2020-07-24 15:05:51 +02:00
ABUSE_MESSAGES: {
MESSAGE: { min: 2, max: 3000 } // Length
},
2018-08-13 16:57:13 +02:00
VIDEO_BLACKLIST: {
REASON: { min: 2, max: 300 } // Length
},
2017-10-24 19:41:09 +02:00
VIDEO_CHANNELS: {
NAME: { min: 1, max: 120 }, // Length
DESCRIPTION: { min: 3, max: 1000 }, // Length
SUPPORT: { min: 3, max: 1000 }, // Length
2017-11-14 10:57:56 +01:00
URL: { min: 3, max: 2000 } // Length
2017-10-24 19:41:09 +02:00
},
2018-07-12 19:02:00 +02:00
VIDEO_CAPTIONS: {
CAPTION_FILE: {
2018-07-16 14:22:16 +02:00
EXTNAME: [ '.vtt', '.srt' ],
2018-07-12 19:02:00 +02:00
FILE_SIZE: {
max: 4 * 1024 * 1024 // 4MB
2018-07-12 19:02:00 +02:00
}
}
},
VIDEO_IMPORTS: {
2018-08-06 17:13:39 +02:00
URL: { min: 3, max: 2000 }, // Length
TORRENT_NAME: { min: 3, max: 255 }, // Length
2018-08-07 09:54:36 +02:00
TORRENT_FILE: {
EXTNAME: [ '.torrent' ],
FILE_SIZE: {
max: 1024 * 200 // 200 KB
}
}
},
2018-09-11 16:27:07 +02:00
VIDEOS_REDUNDANCY: {
URL: { min: 3, max: 2000 } // Length
},
2018-11-14 15:01:28 +01:00
VIDEO_RATES: {
URL: { min: 3, max: 2000 } // Length
},
2016-07-31 20:58:43 +02:00
VIDEOS: {
NAME: { min: 3, max: 120 }, // Length
2018-04-23 14:39:52 +02:00
LANGUAGE: { min: 1, max: 10 }, // Length
2017-10-30 10:16:27 +01:00
TRUNCATED_DESCRIPTION: { min: 3, max: 250 }, // Length
DESCRIPTION: { min: 3, max: 10000 }, // Length
SUPPORT: { min: 3, max: 1000 }, // Length
IMAGE: {
2021-01-26 10:23:21 +01:00
EXTNAME: [ '.png', '.jpg', '.jpeg', '.webp' ],
FILE_SIZE: {
max: 4 * 1024 * 1024 // 4MB
}
},
2019-05-16 16:55:34 +02:00
EXTNAME: [] as string[],
2017-09-04 21:21:47 +02:00
INFO_HASH: { min: 40, max: 40 }, // Length, info hash is 20 bytes length but we represent it in hexadecimal so 20 * 2
DURATION: { min: 0 }, // Number
TAGS: { min: 0, max: 5 }, // Number of total tags
TAG: { min: 2, max: 30 }, // Length
2017-02-26 18:57:33 +01:00
VIEWS: { min: 0 },
LIKES: { min: 0 },
DISLIKES: { min: 0 },
FILE_SIZE: { min: -1 },
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
PARTIAL_UPLOAD_SIZE: { max: 50 * 1024 * 1024 * 1024 }, // 50GB
2017-11-14 10:57:56 +01:00
URL: { min: 3, max: 2000 } // Length
},
2019-02-26 10:55:40 +01:00
VIDEO_PLAYLISTS: {
NAME: { min: 1, max: 120 }, // Length
DESCRIPTION: { min: 3, max: 1000 }, // Length
URL: { min: 3, max: 2000 }, // Length
IMAGE: {
EXTNAME: [ '.jpg', '.jpeg' ],
FILE_SIZE: {
max: 4 * 1024 * 1024 // 4MB
2019-02-26 10:55:40 +01:00
}
}
},
2018-01-03 11:10:40 +01:00
ACTORS: {
2017-11-14 10:57:56 +01:00
PUBLIC_KEY: { min: 10, max: 5000 }, // Length
PRIVATE_KEY: { min: 10, max: 5000 }, // Length
2017-12-29 19:10:13 +01:00
URL: { min: 3, max: 2000 }, // Length
2021-04-06 17:01:35 +02:00
IMAGE: {
2021-01-26 10:23:21 +01:00
EXTNAME: [ '.png', '.jpeg', '.jpg', '.gif', '.webp' ],
2018-01-03 11:10:40 +01:00
FILE_SIZE: {
max: 4 * 1024 * 1024 // 4MB
2018-01-03 11:10:40 +01:00
}
2017-12-29 19:10:13 +01:00
}
2017-02-26 18:57:33 +01:00
},
VIDEO_EVENTS: {
COUNT: { min: 0 }
},
2017-12-22 10:50:07 +01:00
VIDEO_COMMENTS: {
2020-05-05 16:48:30 +02:00
TEXT: { min: 1, max: 10000 }, // Length
URL: { min: 3, max: 2000 } // Length
},
VIDEO_SHARE: {
URL: { min: 3, max: 2000 } // Length
2019-01-09 15:14:29 +01:00
},
CONTACT_FORM: {
FROM_NAME: { min: 1, max: 120 }, // Length
BODY: { min: 3, max: 5000 } // Length
},
PLUGINS: {
NAME: { min: 1, max: 214 }, // Length
DESCRIPTION: { min: 1, max: 20000 } // Length
},
COMMONS: {
URL: { min: 5, max: 2000 } // Length
2016-07-31 20:58:43 +02:00
}
}
2020-11-06 16:43:43 +01:00
const VIEW_LIFETIME = {
2020-11-06 16:42:23 +01:00
VIDEO: 60000 * 60, // 1 hour
LIVE: 60000 * 5 // 5 minutes
}
2019-01-09 15:14:29 +01:00
let CONTACT_FORM_LIFETIME = 60000 * 60 // 1 hour
const VIDEO_TRANSCODING_FPS: VideoTranscodingFPS = {
MIN: 1,
2020-01-31 16:56:52 +01:00
STANDARD: [ 24, 25, 30 ],
HD_STANDARD: [ 50, 60 ],
AVERAGE: 30,
MAX: 60,
KEEP_ORIGIN_FPS_RESOLUTION_MIN: 720 // We keep the original FPS on high resolutions (720 minimum)
}
2018-02-23 16:39:51 +01:00
2019-05-16 16:55:34 +02:00
const DEFAULT_AUDIO_RESOLUTION = VideoResolution.H_480P
2017-06-16 10:36:18 +02:00
const VIDEO_RATE_TYPES: { [ id: string ]: VideoRateType } = {
2017-03-08 21:35:43 +01:00
LIKE: 'like',
DISLIKE: 'dislike'
}
const FFMPEG_NICE: { [ id: string ]: number } = {
2021-01-10 21:02:55 +01:00
// parent process defaults to niceness = 0
// reminder: lower = higher priority, max value is 19, lowest is -20
LIVE: 5, // prioritize over VOD and THUMBNAIL
THUMBNAIL: 10,
2021-01-10 21:02:55 +01:00
VOD: 15
}
2017-03-22 21:15:55 +01:00
const VIDEO_CATEGORIES = {
1: 'Music',
2: 'Films',
3: 'Vehicles',
4: 'Art',
5: 'Sports',
6: 'Travels',
7: 'Gaming',
8: 'People',
9: 'Comedy',
10: 'Entertainment',
11: 'News & Politics',
12: 'How To',
2017-03-22 21:15:55 +01:00
13: 'Education',
14: 'Activism',
15: 'Science & Technology',
16: 'Animals',
17: 'Kids',
18: 'Food'
}
2017-03-27 20:53:11 +02:00
// See https://creativecommons.org/licenses/?lang=en
const VIDEO_LICENCES = {
1: 'Attribution',
2: 'Attribution - Share Alike',
3: 'Attribution - No Derivatives',
4: 'Attribution - Non Commercial',
5: 'Attribution - Non Commercial - Share Alike',
6: 'Attribution - Non Commercial - No Derivatives',
7: 'Public Domain Dedication'
}
2020-01-31 16:56:52 +01:00
const VIDEO_LANGUAGES: { [id: string]: string } = {}
2017-04-07 12:13:37 +02:00
const VIDEO_PRIVACIES: { [ id in VideoPrivacy ]: string } = {
2020-01-31 16:56:52 +01:00
[VideoPrivacy.PUBLIC]: 'Public',
[VideoPrivacy.UNLISTED]: 'Unlisted',
[VideoPrivacy.PRIVATE]: 'Private',
[VideoPrivacy.INTERNAL]: 'Internal'
2017-10-31 11:52:52 +01:00
}
const VIDEO_STATES: { [ id in VideoState ]: string } = {
2020-01-31 16:56:52 +01:00
[VideoState.PUBLISHED]: 'Published',
[VideoState.TO_TRANSCODE]: 'To transcode',
[VideoState.TO_IMPORT]: 'To import',
[VideoState.WAITING_FOR_LIVE]: 'Waiting for livestream',
Add support for saving video files to object storage (#4290) * Add support for saving video files to object storage * Add support for custom url generation on s3 stored files Uses two config keys to support url generation that doesn't directly go to (compatible s3). Can be used to generate urls to any cache server or CDN. * Upload files to s3 concurrently and delete originals afterwards * Only publish after move to object storage is complete * Use base url instead of url template * Fix mistyped config field * Add rudenmentary way to download before transcode * Implement Chocobozzz suggestions https://github.com/Chocobozzz/PeerTube/pull/4290#issuecomment-891670478 The remarks in question: Try to use objectStorage prefix instead of s3 prefix for your function/variables/config names Prefer to use a tree for the config: s3.streaming_playlists_bucket -> object_storage.streaming_playlists.bucket Use uppercase for config: S3.STREAMING_PLAYLISTS_BUCKETINFO.bucket -> OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET (maybe BUCKET_NAME instead of BUCKET) I suggest to rename moveJobsRunning to pendingMovingJobs (or better, create a dedicated videoJobInfo table with a pendingMove & videoId columns so we could also use this table to track pending transcoding jobs) https://github.com/Chocobozzz/PeerTube/pull/4290/files#diff-3e26d41ca4bda1de8e1747af70ca2af642abcc1e9e0bfb94239ff2165acfbde5R19 uses a string instead of an integer I think we should store the origin object storage URL in fileUrl, without base_url injection. Instead, inject the base_url at "runtime" so admins can easily change this configuration without running a script to update DB URLs * Import correct function * Support multipart upload * Remove import of node 15.0 module stream/promises * Extend maximum upload job length Using the same value as for redundancy downloading seems logical * Use dynamic part size for really large uploads Also adds very small part size for local testing * Fix decreasePendingMove query * Resolve various PR comments * Move to object storage after optimize * Make upload size configurable and increase default * Prune webtorrent files that are stored in object storage * Move files after transcoding jobs * Fix federation * Add video path manager * Support move to external storage job in client * Fix live object storage tests Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-08-17 08:26:20 +02:00
[VideoState.LIVE_ENDED]: 'Livestream ended',
[VideoState.TO_MOVE_TO_EXTERNAL_STORAGE]: 'To move to an external storage'
}
const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = {
2020-01-31 16:56:52 +01:00
[VideoImportState.FAILED]: 'Failed',
[VideoImportState.PENDING]: 'Pending',
[VideoImportState.SUCCESS]: 'Success',
[VideoImportState.REJECTED]: 'Rejected'
}
const ABUSE_STATES: { [ id in AbuseState ]: string } = {
2020-07-01 16:05:30 +02:00
[AbuseState.PENDING]: 'Pending',
[AbuseState.REJECTED]: 'Rejected',
[AbuseState.ACCEPTED]: 'Accepted'
}
const VIDEO_PLAYLIST_PRIVACIES: { [ id in VideoPlaylistPrivacy ]: string } = {
2020-01-31 16:56:52 +01:00
[VideoPlaylistPrivacy.PUBLIC]: 'Public',
[VideoPlaylistPrivacy.UNLISTED]: 'Unlisted',
[VideoPlaylistPrivacy.PRIVATE]: 'Private'
2019-02-26 10:55:40 +01:00
}
const VIDEO_PLAYLIST_TYPES: { [ id in VideoPlaylistType ]: string } = {
2020-01-31 16:56:52 +01:00
[VideoPlaylistType.REGULAR]: 'Regular',
[VideoPlaylistType.WATCH_LATER]: 'Watch later'
2019-03-05 10:58:44 +01:00
}
2018-12-11 14:52:50 +01:00
const MIMETYPES = {
2019-05-16 16:55:34 +02:00
AUDIO: {
MIMETYPE_EXT: {
'audio/mpeg': '.mp3',
'audio/mp3': '.mp3',
'application/ogg': '.ogg',
'audio/ogg': '.ogg',
2020-02-07 08:51:28 +01:00
'audio/x-ms-wma': '.wma',
'audio/wav': '.wav',
'audio/x-wav': '.wav',
2020-02-07 08:51:28 +01:00
'audio/x-flac': '.flac',
'audio/flac': '.flac',
'audio/aac': '.aac',
2021-02-25 09:09:41 +01:00
'audio/m4a': '.m4a',
2021-02-25 09:14:24 +01:00
'audio/mp4': '.m4a',
2021-02-25 09:09:41 +01:00
'audio/x-m4a': '.m4a',
'audio/ac3': '.ac3'
2019-05-16 16:55:34 +02:00
},
EXT_MIMETYPE: null as { [ id: string ]: string }
},
2018-12-11 14:52:50 +01:00
VIDEO: {
MIMETYPE_EXT: null as { [ id: string ]: string | string[] },
MIMETYPES_REGEX: null as string,
2018-12-11 14:52:50 +01:00
EXT_MIMETYPE: null as { [ id: string ]: string }
},
IMAGE: {
MIMETYPE_EXT: {
'image/png': '.png',
2021-04-06 17:01:35 +02:00
'image/gif': '.gif',
'image/webp': '.webp',
2018-12-11 14:52:50 +01:00
'image/jpg': '.jpg',
'image/jpeg': '.jpg'
},
EXT_MIMETYPE: null as { [ id: string ]: string }
2018-12-11 14:52:50 +01:00
},
VIDEO_CAPTIONS: {
MIMETYPE_EXT: {
'text/vtt': '.vtt',
2019-10-18 10:36:32 +02:00
'application/x-subrip': '.srt',
'text/plain': '.srt'
2018-12-11 14:52:50 +01:00
}
},
TORRENT: {
MIMETYPE_EXT: {
'application/x-bittorrent': '.torrent'
}
}
2018-08-07 09:54:36 +02:00
}
2019-05-16 16:55:34 +02:00
MIMETYPES.AUDIO.EXT_MIMETYPE = invert(MIMETYPES.AUDIO.MIMETYPE_EXT)
MIMETYPES.IMAGE.EXT_MIMETYPE = invert(MIMETYPES.IMAGE.MIMETYPE_EXT)
2018-08-07 09:54:36 +02:00
const BINARY_CONTENT_TYPES = new Set([
'binary/octet-stream',
'application/octet-stream',
'application/x-binary'
])
2016-10-02 11:14:08 +02:00
// ---------------------------------------------------------------------------
2018-08-30 14:58:00 +02:00
const OVERVIEWS = {
VIDEOS: {
2018-08-31 14:26:51 +02:00
SAMPLE_THRESHOLD: 6,
SAMPLES_COUNT: 20
2018-08-30 14:58:00 +02:00
}
}
// ---------------------------------------------------------------------------
2017-12-14 17:38:41 +01:00
const SERVER_ACTOR_NAME = 'peertube'
2017-11-14 17:31:26 +01:00
2017-11-09 17:51:58 +01:00
const ACTIVITY_PUB = {
2017-11-30 13:15:25 +01:00
POTENTIAL_ACCEPT_HEADERS: [
'application/activity+json',
2018-01-11 19:17:43 +01:00
'application/ld+json',
'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'
],
2017-11-30 13:15:25 +01:00
ACCEPT_HEADER: 'application/activity+json, application/ld+json',
2017-11-17 15:20:42 +01:00
PUBLIC: 'https://www.w3.org/ns/activitystreams#Public',
2017-11-10 14:34:45 +01:00
COLLECTION_ITEMS_PER_PAGE: 10,
2020-04-10 09:46:01 +02:00
FETCH_PAGE_LIMIT: 2000,
2017-11-16 15:22:39 +01:00
URL_MIME_TYPES: {
2019-05-16 16:55:34 +02:00
VIDEO: [] as string[],
2017-11-16 15:22:39 +01:00
TORRENT: [ 'application/x-bittorrent' ],
MAGNET: [ 'application/x-bittorrent;x-scheme-handler/magnet' ]
2018-01-04 14:04:02 +01:00
},
2018-01-10 17:18:12 +01:00
MAX_RECURSION_COMMENTS: 100,
2019-03-19 11:15:42 +01:00
ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days
2019-03-19 14:13:53 +01:00
VIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days
VIDEO_PLAYLIST_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2 // 2 days
2017-11-09 17:51:58 +01:00
}
2017-12-14 17:38:41 +01:00
const ACTIVITY_PUB_ACTOR_TYPES: { [ id: string ]: ActivityPubActorType } = {
GROUP: 'Group',
PERSON: 'Person',
APPLICATION: 'Application',
ORGANIZATION: 'Organization',
SERVICE: 'Service'
2017-12-14 17:38:41 +01:00
}
const HTTP_SIGNATURE = {
HEADER_NAME: 'signature',
ALGORITHM: 'rsa-sha256',
HEADERS_TO_SIGN: [ '(request-target)', 'host', 'date', 'digest' ],
REQUIRED_HEADERS: {
ALL: [ '(request-target)', 'host', 'date' ],
POST: [ '(request-target)', 'host', 'date', 'digest' ]
},
CLOCK_SKEW_SECONDS: 1800
}
2016-10-02 11:14:08 +02:00
// ---------------------------------------------------------------------------
let PRIVATE_RSA_KEY_SIZE = 2048
2016-10-02 11:14:08 +02:00
// Password encryption
const BCRYPT_SALT_SIZE = 10
2016-05-17 21:03:00 +02:00
const USER_PASSWORD_RESET_LIFETIME = 60000 * 60 // 60 minutes
const USER_PASSWORD_CREATE_LIFETIME = 60000 * 60 * 24 * 7 // 7 days
2018-01-30 13:27:07 +01:00
const USER_EMAIL_VERIFY_LIFETIME = 60000 * 60 // 60 minutes
2019-04-11 11:33:44 +02:00
const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = {
DO_NOT_LIST: 'do_not_list',
BLUR: 'blur',
DISPLAY: 'display'
}
// ---------------------------------------------------------------------------
// Express static paths (router)
const STATIC_PATHS = {
2016-11-11 15:20:03 +01:00
THUMBNAILS: '/static/thumbnails/',
2017-12-29 19:10:13 +01:00
WEBSEED: '/static/webseed/',
2018-12-04 17:08:55 +01:00
REDUNDANCY: '/static/redundancy/',
STREAMING_PLAYLISTS: {
HLS: '/static/streaming-playlists/hls'
}
}
2018-05-29 18:30:11 +02:00
const STATIC_DOWNLOAD_PATHS = {
TORRENTS: '/download/torrents/',
VIDEOS: '/download/videos/',
HLS_VIDEOS: '/download/streaming-playlists/hls/videos/'
2018-05-29 18:30:11 +02:00
}
2019-08-09 11:32:40 +02:00
const LAZY_STATIC_PATHS = {
2021-04-06 11:35:56 +02:00
BANNERS: '/lazy-static/banners/',
2019-08-09 11:32:40 +02:00
AVATARS: '/lazy-static/avatars/',
2020-08-26 15:03:00 +02:00
PREVIEWS: '/lazy-static/previews/',
VIDEO_CAPTIONS: '/lazy-static/video-captions/',
TORRENTS: '/lazy-static/torrents/'
2019-08-09 11:32:40 +02:00
}
// Cache control
2020-01-31 16:56:52 +01:00
const STATIC_MAX_AGE = {
2019-07-29 15:20:36 +02:00
SERVER: '2h',
LAZY_SERVER: '2d',
2019-07-29 15:20:36 +02:00
CLIENT: '30d'
}
// Videos thumbnail size
2017-10-16 10:05:49 +02:00
const THUMBNAILS_SIZE = {
2021-04-05 11:38:45 +02:00
width: 280,
height: 157,
minWidth: 150
2017-10-16 10:05:49 +02:00
}
const PREVIEWS_SIZE = {
2019-05-17 11:56:12 +02:00
width: 850,
height: 480,
minWidth: 400
}
2021-04-06 17:01:35 +02:00
const ACTOR_IMAGES_SIZE = {
AVATARS: {
width: 120,
height: 120
},
BANNERS: {
width: 1920,
2021-04-08 09:13:42 +02:00
height: 317 // 6/1 ratio
2021-04-06 17:01:35 +02:00
}
2018-01-03 11:36:03 +01:00
}
const EMBED_SIZE = {
width: 560,
height: 315
2017-10-16 10:05:49 +02:00
}
2017-09-04 21:45:05 +02:00
// Sub folders of cache directory
2019-03-19 14:23:17 +01:00
const FILES_CACHE = {
2018-07-16 14:22:16 +02:00
PREVIEWS: {
DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'previews'),
MAX_AGE: 1000 * 3600 * 3 // 3 hours
},
VIDEO_CAPTIONS: {
DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'video-captions'),
MAX_AGE: 1000 * 3600 * 3 // 3 hours
},
TORRENTS: {
DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'torrents'),
MAX_AGE: 1000 * 3600 * 3 // 3 hours
2017-07-12 11:56:02 +02:00
}
}
2019-08-09 11:32:40 +02:00
const LRU_CACHE = {
2019-03-19 14:23:17 +01:00
USER_TOKENS: {
2019-08-09 11:32:40 +02:00
MAX_SIZE: 1000
},
2021-04-06 11:35:56 +02:00
ACTOR_IMAGE_STATIC: {
2019-08-09 11:32:40 +02:00
MAX_SIZE: 500
2019-03-19 14:23:17 +01:00
}
}
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
const RESUMABLE_UPLOAD_DIRECTORY = join(CONFIG.STORAGE.TMP_DIR, 'resumable-uploads')
const HLS_STREAMING_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls')
2019-01-29 08:37:25 +01:00
const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls')
const RESUMABLE_UPLOAD_SESSION_LIFETIME = SCHEDULER_INTERVALS_MS.REMOVE_DANGLING_RESUMABLE_UPLOADS
const VIDEO_LIVE = {
EXTENSION: '.ts',
2020-09-25 16:19:35 +02:00
CLEANUP_DELAY: 1000 * 60 * 5, // 5 minutes
2020-11-04 14:16:57 +01:00
SEGMENT_TIME_SECONDS: 4, // 4 seconds
2020-09-25 16:19:35 +02:00
SEGMENTS_LIST_SIZE: 15, // 15 maximum segments in live playlist
REPLAY_DIRECTORY: 'replay',
2020-11-18 11:24:28 +01:00
EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION: 4,
MAX_SOCKET_WAITING_DATA: 1024 * 1000 * 100, // 100MB
RTMP: {
CHUNK_SIZE: 60000,
GOP_CACHE: true,
PING: 60,
PING_TIMEOUT: 30,
BASE_PATH: 'live'
}
}
2018-09-14 11:52:23 +02:00
const MEMOIZE_TTL = {
2020-01-03 13:47:45 +01:00
OVERVIEWS_SAMPLE: 1000 * 3600 * 4, // 4 hours
2020-09-25 16:19:35 +02:00
INFO_HASH_EXISTS: 1000 * 3600 * 12, // 12 hours
LIVE_ABLE_TO_UPLOAD: 1000 * 60, // 1 minute
LIVE_CHECK_SOCKET_HEALTH: 1000 * 60 // 1 minute
2020-01-03 13:47:45 +01:00
}
const MEMOIZE_LENGTH = {
INFO_HASH_EXISTS: 200
2018-09-14 11:52:23 +02:00
}
2019-08-09 11:32:40 +02:00
const QUEUE_CONCURRENCY = {
2021-04-06 11:35:56 +02:00
ACTOR_PROCESS_IMAGE: 3
2019-08-09 11:32:40 +02:00
}
2018-09-11 16:27:07 +02:00
const REDUNDANCY = {
VIDEOS: {
RANDOMIZED_FACTOR: 5
}
}
2017-12-19 10:34:56 +01:00
const ACCEPT_HEADERS = [ 'html', 'application/json' ].concat(ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS)
2017-11-30 13:37:11 +01:00
2019-05-16 16:55:34 +02:00
const ASSETS_PATH = {
DEFAULT_AUDIO_BACKGROUND: join(root(), 'dist', 'server', 'assets', 'default-audio-background.jpg'),
DEFAULT_LIVE_BACKGROUND: join(root(), 'dist', 'server', 'assets', 'default-live-background.jpg')
2019-05-16 16:55:34 +02:00
}
// ---------------------------------------------------------------------------
const CUSTOM_HTML_TAG_COMMENTS = {
TITLE: '<!-- title tag -->',
DESCRIPTION: '<!-- description tag -->',
CUSTOM_CSS: '<!-- custom css tag -->',
2021-05-14 12:04:44 +02:00
META_TAGS: '<!-- meta tags -->',
SERVER_CONFIG: '<!-- server config -->'
}
2017-07-07 16:57:28 +02:00
// ---------------------------------------------------------------------------
2018-04-17 14:01:06 +02:00
const FEEDS = {
2018-05-11 09:44:04 +02:00
COUNT: 20
2018-04-17 14:01:06 +02:00
}
2019-04-10 15:26:33 +02:00
const MAX_LOGS_OUTPUT_CHARACTERS = 10 * 1000 * 1000
2019-12-11 14:14:01 +01:00
const LOG_FILENAME = 'peertube.log'
const AUDIT_LOG_FILENAME = 'peertube-audit.log'
2019-04-10 15:26:33 +02:00
2018-04-17 14:01:06 +02:00
// ---------------------------------------------------------------------------
2018-06-26 16:53:24 +02:00
const TRACKER_RATE_LIMITS = {
INTERVAL: 60000 * 5, // 5 minutes
2018-06-29 09:33:36 +02:00
ANNOUNCES_PER_IP_PER_INFOHASH: 15, // maximum announces per torrent in the interval
2020-06-25 16:27:35 +02:00
ANNOUNCES_PER_IP: 30, // maximum announces for all our torrents in the interval
2020-11-05 10:28:05 +01:00
BLOCK_IP_LIFETIME: 60000 * 3 // 3 minutes
2018-06-26 16:53:24 +02:00
}
2019-04-08 11:13:49 +02:00
const P2P_MEDIA_LOADER_PEER_VERSION = 2
2018-06-26 16:53:24 +02:00
// ---------------------------------------------------------------------------
const PLUGIN_GLOBAL_CSS_FILE_NAME = 'plugins-global.css'
const PLUGIN_GLOBAL_CSS_PATH = join(CONFIG.STORAGE.TMP_DIR, PLUGIN_GLOBAL_CSS_FILE_NAME)
2020-04-29 09:04:42 +02:00
let PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME = 1000 * 60 * 5 // 5 minutes
2019-07-10 14:06:19 +02:00
const DEFAULT_THEME_NAME = 'default'
const DEFAULT_USER_THEME_NAME = 'instance-default'
2019-07-09 11:45:19 +02:00
// ---------------------------------------------------------------------------
2020-05-29 16:16:24 +02:00
const SEARCH_INDEX = {
ROUTES: {
VIDEOS: '/api/v1/search/videos',
VIDEO_CHANNELS: '/api/v1/search/video-channels'
}
}
// ---------------------------------------------------------------------------
// Special constants for a test instance
if (isTestInstance() === true) {
PRIVATE_RSA_KEY_SIZE = 1024
ACTOR_FOLLOW_SCORE.BASE = 20
2016-11-11 15:20:03 +01:00
REMOTE_SCHEME.HTTP = 'http'
REMOTE_SCHEME.WS = 'ws'
2019-07-29 15:20:36 +02:00
STATIC_MAX_AGE.SERVER = '0'
ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2
2020-11-06 15:14:48 +01:00
ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
ACTIVITY_PUB.VIDEO_PLAYLIST_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
2021-04-06 17:01:35 +02:00
CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max = 100 * 1024 // 100KB
2021-01-26 11:37:31 +01:00
CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max = 400 * 1024 // 400KB
2021-10-22 10:28:00 +02:00
SCHEDULER_INTERVALS_MS.ACTOR_FOLLOW_SCORES = 1000
SCHEDULER_INTERVALS_MS.REMOVE_OLD_JOBS = 10000
SCHEDULER_INTERVALS_MS.REMOVE_OLD_HISTORY = 5000
SCHEDULER_INTERVALS_MS.REMOVE_OLD_VIEWS = 5000
SCHEDULER_INTERVALS_MS.UPDATE_VIDEOS = 5000
SCHEDULER_INTERVALS_MS.AUTO_FOLLOW_INDEX_INSTANCES = 5000
SCHEDULER_INTERVALS_MS.UPDATE_INBOX_STATS = 5000
SCHEDULER_INTERVALS_MS.CHECK_PEERTUBE_VERSION = 2000
2020-01-31 16:56:52 +01:00
REPEAT_JOBS['videos-views'] = { every: 5000 }
REPEAT_JOBS['activitypub-cleaner'] = { every: 5000 }
2018-09-11 16:27:07 +02:00
REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1
2020-11-06 16:42:23 +01:00
VIEW_LIFETIME.VIDEO = 1000 // 1 second
VIEW_LIFETIME.LIVE = 1000 * 5 // 5 second
2019-01-09 15:14:29 +01:00
CONTACT_FORM_LIFETIME = 1000 // 1 second
2020-01-31 16:56:52 +01:00
JOB_ATTEMPTS['email'] = 1
2018-07-16 14:22:16 +02:00
2019-03-19 14:23:17 +01:00
FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000
MEMOIZE_TTL.OVERVIEWS_SAMPLE = 3000
2020-11-03 15:33:30 +01:00
MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD = 3000
OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD = 2
2020-04-29 09:04:42 +02:00
PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME = 5000
2020-10-27 16:06:24 +01:00
2020-11-04 14:16:57 +01:00
VIDEO_LIVE.CLEANUP_DELAY = 5000
VIDEO_LIVE.SEGMENT_TIME_SECONDS = 2
VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION = 1
}
2018-12-11 14:52:50 +01:00
updateWebserverUrls()
2019-05-16 16:55:34 +02:00
updateWebserverConfig()
2019-04-11 11:33:44 +02:00
registerConfigChangedHandler(() => {
updateWebserverUrls()
updateWebserverConfig()
})
// ---------------------------------------------------------------------------
const FILES_CONTENT_HASH = {
MANIFEST: generateContentHash(),
FAVICON: generateContentHash(),
LOGO: generateContentHash()
}
// ---------------------------------------------------------------------------
2017-05-15 22:22:03 +02:00
export {
2019-04-11 11:33:44 +02:00
WEBSERVER,
2016-10-02 11:14:08 +02:00
API_VERSION,
VIDEO_LIVE,
2019-07-17 10:03:55 +02:00
PEERTUBE_VERSION,
2019-08-09 11:32:40 +02:00
LAZY_STATIC_PATHS,
2020-05-29 16:16:24 +02:00
SEARCH_INDEX,
Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
2021-05-10 11:13:41 +02:00
RESUMABLE_UPLOAD_DIRECTORY,
RESUMABLE_UPLOAD_SESSION_LIFETIME,
2019-01-29 08:37:25 +01:00
HLS_REDUNDANCY_DIRECTORY,
2019-04-08 11:13:49 +02:00
P2P_MEDIA_LOADER_PEER_VERSION,
2021-04-06 17:01:35 +02:00
ACTOR_IMAGES_SIZE,
2017-11-30 13:37:11 +01:00
ACCEPT_HEADERS,
2016-10-02 11:14:08 +02:00
BCRYPT_SALT_SIZE,
2018-06-26 16:53:24 +02:00
TRACKER_RATE_LIMITS,
2019-03-19 14:23:17 +01:00
FILES_CACHE,
2019-12-11 14:14:01 +01:00
LOG_FILENAME,
2016-10-02 11:14:08 +02:00
CONSTRAINTS_FIELDS,
EMBED_SIZE,
2018-09-11 16:27:07 +02:00
REDUNDANCY,
JOB_CONCURRENCY,
JOB_ATTEMPTS,
AP_CLEANER_CONCURRENCY,
2016-12-25 09:44:57 +01:00
LAST_MIGRATION_VERSION,
2016-10-02 11:14:08 +02:00
OAUTH_LIFETIME,
CUSTOM_HTML_TAG_COMMENTS,
2018-04-18 16:04:49 +02:00
BROADCAST_CONCURRENCY,
2019-12-11 14:14:01 +01:00
AUDIT_LOG_FILENAME,
2018-07-24 16:27:45 +02:00
PAGINATION,
ACTOR_FOLLOW_SCORE,
2016-11-11 15:20:03 +01:00
PREVIEWS_SIZE,
REMOTE_SCHEME,
2017-11-13 17:39:41 +01:00
FOLLOW_STATES,
2019-07-10 14:06:19 +02:00
DEFAULT_USER_THEME_NAME,
2017-12-14 17:38:41 +01:00
SERVER_ACTOR_NAME,
PLUGIN_GLOBAL_CSS_FILE_NAME,
PLUGIN_GLOBAL_CSS_PATH,
2017-11-09 17:51:58 +01:00
PRIVATE_RSA_KEY_SIZE,
2018-05-11 09:44:04 +02:00
ROUTE_CACHE_LIFETIME,
2016-10-02 11:14:08 +02:00
SORTABLE_COLUMNS,
HLS_STREAMING_PLAYLIST_DIRECTORY,
2018-04-17 14:01:06 +02:00
FEEDS,
2018-08-03 10:19:51 +02:00
JOB_TTL,
2019-07-10 14:06:19 +02:00
DEFAULT_THEME_NAME,
NSFW_POLICY_TYPES,
STATIC_MAX_AGE,
STATIC_PATHS,
VIDEO_IMPORT_TIMEOUT,
2019-03-05 10:58:44 +01:00
VIDEO_PLAYLIST_TYPES,
2019-04-10 15:26:33 +02:00
MAX_LOGS_OUTPUT_CHARACTERS,
2017-11-09 17:51:58 +01:00
ACTIVITY_PUB,
2017-12-14 17:38:41 +01:00
ACTIVITY_PUB_ACTOR_TYPES,
2016-10-02 11:14:08 +02:00
THUMBNAILS_SIZE,
2017-03-22 21:15:55 +01:00
VIDEO_CATEGORIES,
2020-01-03 13:47:45 +01:00
MEMOIZE_LENGTH,
2017-04-07 12:13:37 +02:00
VIDEO_LANGUAGES,
2017-10-31 11:52:52 +01:00
VIDEO_PRIVACIES,
2017-03-27 20:53:11 +02:00
VIDEO_LICENCES,
VIDEO_STATES,
2019-08-09 11:32:40 +02:00
QUEUE_CONCURRENCY,
2017-11-10 14:34:45 +01:00
VIDEO_RATE_TYPES,
JOB_PRIORITY,
VIDEO_TRANSCODING_FPS,
FFMPEG_NICE,
2020-07-01 16:05:30 +02:00
ABUSE_STATES,
2019-08-09 11:32:40 +02:00
LRU_CACHE,
2021-03-03 10:10:55 +01:00
REQUEST_TIMEOUT,
2018-01-30 13:27:07 +01:00
USER_PASSWORD_RESET_LIFETIME,
USER_PASSWORD_CREATE_LIFETIME,
2018-09-14 11:52:23 +02:00
MEMOIZE_TTL,
USER_EMAIL_VERIFY_LIFETIME,
2018-08-30 14:58:00 +02:00
OVERVIEWS,
SCHEDULER_INTERVALS_MS,
2018-08-29 16:26:25 +02:00
REPEAT_JOBS,
2018-05-29 18:30:11 +02:00
STATIC_DOWNLOAD_PATHS,
2018-12-11 14:52:50 +01:00
MIMETYPES,
CRAWL_REQUEST_CONCURRENCY,
2019-05-16 16:55:34 +02:00
DEFAULT_AUDIO_RESOLUTION,
BINARY_CONTENT_TYPES,
2018-02-23 16:39:51 +01:00
JOB_COMPLETED_LIFETIME,
HTTP_SIGNATURE,
VIDEO_IMPORT_STATES,
2020-11-06 16:42:23 +01:00
VIEW_LIFETIME,
2019-01-09 15:14:29 +01:00
CONTACT_FORM_LIFETIME,
2019-02-26 10:55:40 +01:00
VIDEO_PLAYLIST_PRIVACIES,
2020-04-29 09:04:42 +02:00
PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME,
2019-05-16 16:55:34 +02:00
ASSETS_PATH,
FILES_CONTENT_HASH,
loadLanguages,
buildLanguages,
generateContentHash
}
// ---------------------------------------------------------------------------
2018-12-11 14:52:50 +01:00
function buildVideoMimetypeExt () {
const data = {
// streamable formats that warrant cross-browser compatibility
2018-12-11 14:52:50 +01:00
'video/webm': '.webm',
// We'll add .ogg if additional extensions are enabled
// We could add .ogg here but since it could be an audio file,
// it would be confusing for users because PeerTube will refuse their file (based on the mimetype)
'video/ogg': [ '.ogv' ],
2018-12-11 14:52:50 +01:00
'video/mp4': '.mp4'
}
2019-05-16 16:55:34 +02:00
if (CONFIG.TRANSCODING.ENABLED) {
if (CONFIG.TRANSCODING.ALLOW_ADDITIONAL_EXTENSIONS) {
data['video/ogg'].push('.ogg')
2019-05-16 16:55:34 +02:00
Object.assign(data, {
'video/x-matroska': '.mkv',
// Developed by Apple
'video/quicktime': [ '.mov', '.qt', '.mqv' ], // often used as output format by editing software
2019-12-05 11:07:57 +01:00
'video/x-m4v': '.m4v',
'video/m4v': '.m4v',
// Developed by the Adobe Flash Platform
'video/x-flv': '.flv',
'video/x-f4v': '.f4v', // replacement for flv
// Developed by Microsoft
'video/x-ms-wmv': '.wmv',
'video/x-msvideo': '.avi',
'video/avi': '.avi',
// Developed by 3GPP
// common video formats for cell phones
'video/3gpp': [ '.3gp', '.3gpp' ],
'video/3gpp2': [ '.3g2', '.3gpp2' ],
// Developed by FFmpeg/Mplayer
'application/x-nut': '.nut',
// The standard video format used by many Sony and Panasonic HD camcorders.
// It is also used for storing high definition video on Blu-ray discs.
'video/mp2t': '.mts',
'video/m2ts': '.m2ts',
// Old formats reliant on MPEG-1/MPEG-2
'video/mpv': '.mpv',
'video/mpeg2': '.m2v',
'video/mpeg': [ '.m1v', '.mpg', '.mpe', '.mpeg', '.vob' ],
'video/dvd': '.vob',
// Could be anything
'application/octet-stream': null,
'application/mxf': '.mxf' // often used as exchange format by editing software
2019-05-16 16:55:34 +02:00
})
}
if (CONFIG.TRANSCODING.ALLOW_AUDIO_FILES) {
Object.assign(data, MIMETYPES.AUDIO.MIMETYPE_EXT)
}
2018-12-11 14:52:50 +01:00
}
return data
}
function updateWebserverUrls () {
2019-04-11 11:33:44 +02:00
WEBSERVER.URL = sanitizeUrl(CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT)
WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP)
WEBSERVER.WS = CONFIG.WEBSERVER.WS
WEBSERVER.SCHEME = CONFIG.WEBSERVER.SCHEME
2019-04-11 11:33:44 +02:00
WEBSERVER.HOSTNAME = CONFIG.WEBSERVER.HOSTNAME
WEBSERVER.PORT = CONFIG.WEBSERVER.PORT
WEBSERVER.RTMP_URL = 'rtmp://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.LIVE.RTMP.PORT + '/' + VIDEO_LIVE.RTMP.BASE_PATH
2021-11-05 11:36:03 +01:00
WEBSERVER.RTMPS_URL = 'rtmps://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.LIVE.RTMPS.PORT + '/' + VIDEO_LIVE.RTMP.BASE_PATH
}
2018-12-11 14:52:50 +01:00
function updateWebserverConfig () {
MIMETYPES.VIDEO.MIMETYPE_EXT = buildVideoMimetypeExt()
MIMETYPES.VIDEO.MIMETYPES_REGEX = buildMimetypesRegex(MIMETYPES.VIDEO.MIMETYPE_EXT)
2019-05-16 16:55:34 +02:00
ACTIVITY_PUB.URL_MIME_TYPES.VIDEO = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT)
MIMETYPES.VIDEO.EXT_MIMETYPE = buildVideoExtMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT)
CONSTRAINTS_FIELDS.VIDEOS.EXTNAME = Object.keys(MIMETYPES.VIDEO.EXT_MIMETYPE)
}
function buildVideoExtMimetype (obj: { [ id: string ]: string | string[] }) {
const result: { [id: string]: string } = {}
for (const mimetype of Object.keys(obj)) {
const value = obj[mimetype]
if (!value) continue
const extensions = Array.isArray(value) ? value : [ value ]
for (const extension of extensions) {
result[extension] = mimetype
}
}
return result
2018-12-11 14:52:50 +01:00
}
function buildMimetypesRegex (obj: { [id: string]: string | string[] }) {
return Object.keys(obj)
.map(m => `(${m})`)
.join('|')
2018-12-11 14:52:50 +01:00
}
function loadLanguages () {
Object.assign(VIDEO_LANGUAGES, buildLanguages())
}
2018-04-23 14:39:52 +02:00
function buildLanguages () {
const iso639 = require('iso-639-3')
2020-01-31 16:56:52 +01:00
const languages: { [id: string]: string } = {}
2018-04-23 14:39:52 +02:00
const additionalLanguages = {
2020-01-31 16:56:52 +01:00
sgn: true, // Sign languages (macro language)
ase: true, // American sign language
sdl: true, // Arabian sign language
bfi: true, // British sign language
bzs: true, // Brazilian sign language
csl: true, // Chinese sign language
cse: true, // Czech sign language
dsl: true, // Danish sign language
fsl: true, // French sign language
gsg: true, // German sign language
pks: true, // Pakistan sign language
jsl: true, // Japanese sign language
sfs: true, // South African sign language
swl: true, // Swedish sign language
2020-08-11 09:41:57 +02:00
rsl: true, // Russian sign language
kab: true, // Kabyle
2020-01-31 16:56:52 +01:00
2021-09-07 09:05:57 +02:00
lat: true, // Latin
2020-01-31 16:56:52 +01:00
epo: true, // Esperanto
tlh: true, // Klingon
jbo: true, // Lojban
avk: true // Kotava
}
2018-04-23 14:39:52 +02:00
// Only add ISO639-1 languages and some sign languages (ISO639-3)
iso639
.filter(l => {
2020-02-06 17:06:11 +01:00
return (l.iso6391 !== undefined && l.type === 'living') ||
2020-01-31 16:56:52 +01:00
additionalLanguages[l.iso6393] === true
2018-04-23 14:39:52 +02:00
})
2020-01-31 16:56:52 +01:00
.forEach(l => { languages[l.iso6391 || l.iso6393] = l.name })
2018-04-23 14:39:52 +02:00
2018-09-05 15:59:43 +02:00
// Override Occitan label
2020-01-31 16:56:52 +01:00
languages['oc'] = 'Occitan'
languages['el'] = 'Greek'
2018-09-05 15:59:43 +02:00
2018-04-23 14:39:52 +02:00
return languages
}
function generateContentHash () {
return randomBytes(20).toString('hex')
}