Implement user import/export in server

pull/6266/head
Chocobozzz 2024-02-12 10:47:52 +01:00 committed by Chocobozzz
parent 4d63e6f577
commit 8573e5a80a
196 changed files with 5661 additions and 722 deletions

View File

@ -1,4 +1,4 @@
# /!\ YOU SHOULD NOT UPDATE THIS FILE, USE production.yaml instead /!\ #
# /!\ DO NOT UPDATE THIS FILE, USE production.yaml instead /!\ #
listen:
hostname: '127.0.0.1'
@ -222,12 +222,16 @@ object_storage:
# Useful when you want to use a CDN/external proxy
base_url: '' # Example: 'https://mirror.example.com'
# Same settings but for web videos
web_videos:
bucket_name: 'web-videos'
prefix: ''
base_url: ''
user_exports:
bucket_name: 'user-exports'
prefix: ''
base_url: ''
log:
level: 'info' # 'debug' | 'info' | 'warn' | 'error'
@ -482,11 +486,14 @@ user:
videos:
# Enable or disable video history by default for new users.
enabled: true
# Default value of maximum video bytes the user can upload (does not take into account transcoded files)
# Default value of maximum video bytes the user can upload
# Does not take into account transcoded files or account export archives (that can include user uploaded files)
# Byte format is supported ("1GB" etc)
# -1 == unlimited
video_quota: -1
video_quota_daily: -1
default_channel_name: 'Main $1 channel' # The placeholder $1 is used to represent the user's username
video_channels:
@ -707,6 +714,24 @@ import:
# Max number of videos to import when the user asks for full sync
full_sync_videos_limit: 1000
users:
# Video quota is checked on import so the user doesn't upload a too big archive file
# Video quota (daily quota is not taken into account) is also checked for each video when PeerTube is processing the import
enabled: true
export:
users:
# Allow users to export their PeerTube data in a .zip for backup or re-import
# Only one export at a time is allowed per user
enabled: true
# Max size of the current user quota to accept or not the export
# Goal of this setting is to not store too big archive file on your server disk
max_user_video_quota: 10GB
# How long PeerTube should keep the user export
export_expiration: '2 days'
auto_blacklist:
# New videos automatically blacklisted so moderators can review before publishing
videos:
@ -867,6 +892,7 @@ client:
# By default PeerTube client displays author username
prefer_author_display_name: false
display_author_avatar: false
resumable_upload:
# Max size of upload chunks, e.g. '90MB'
# If null, it will be calculated based on network speed

View File

@ -220,12 +220,16 @@ object_storage:
# Useful when you want to use a CDN/external proxy
base_url: '' # Example: 'https://mirror.example.com'
# Same settings but for web videos
web_videos:
bucket_name: 'web-videos'
prefix: ''
base_url: ''
user_exports:
bucket_name: 'user-exports'
prefix: ''
base_url: ''
log:
level: 'info' # 'debug' | 'info' | 'warn' | 'error'
@ -492,11 +496,14 @@ user:
videos:
# Enable or disable video history by default for new users.
enabled: true
# Default value of maximum video bytes the user can upload (does not take into account transcoded files)
# Default value of maximum video bytes the user can upload
# Does not take into account transcoded files or account export archives (that can include user uploaded files)
# Byte format is supported ("1GB" etc)
# -1 == unlimited
video_quota: -1
video_quota_daily: -1
default_channel_name: 'Main $1 channel' # The placeholder $1 is used to represent the user's username
video_channels:
@ -717,6 +724,24 @@ import:
# Max number of videos to import when the user asks for full sync
full_sync_videos_limit: 1000
users:
# Video quota is checked on import so the user doesn't upload a too big archive file
# Video quota (daily quota is not taken into account) is also checked for each video when PeerTube is processing the import
enabled: true
export:
users:
# Allow users to export their PeerTube data in a .zip for backup or re-import
# Only one export at a time is allowed per user
enabled: true
# Max size of the current user quota to accept or not the export
# Goal of this setting is to not store too big archive file on your server disk
max_user_video_quota: 10GB
# How long PeerTube should keep the user export
export_expiration: '2 days'
auto_blacklist:
# New videos automatically blacklisted so moderators can review before publishing
videos:
@ -877,6 +902,7 @@ client:
# By default PeerTube client displays author username
prefer_author_display_name: false
display_author_avatar: false
resumable_upload:
# Max size of upload chunks, e.g. '90MB'
# If null, it will be calculated based on network speed

View File

@ -109,6 +109,7 @@
"@peertube/http-signature": "^1.7.0",
"@smithy/node-http-handler": "^2.1.7",
"@uploadx/core": "^6.0.0",
"archiver": "^6.0.1",
"async-mutex": "^0.4.0",
"bcrypt": "5.1.1",
"bencode": "^4.0.0",
@ -142,6 +143,7 @@
"jimp": "^0.22.4",
"js-yaml": "^4.0.0",
"jsonld": "~8.3.1",
"jsonwebtoken": "^9.0.2",
"lodash-es": "^4.17.21",
"lru-cache": "^10.0.1",
"magnet-uri": "^7.0.5",
@ -178,11 +180,13 @@
"webfinger.js": "^2.6.6",
"webtorrent": "^2.1.27",
"winston": "3.11.0",
"ws": "^8.0.0"
"ws": "^8.0.0",
"yauzl": "^2.10.0"
},
"devDependencies": {
"@peertube/maildev": "^1.2.0",
"@peertube/resolve-tspaths": "^0.8.14",
"@types/archiver": "^6.0.2",
"@types/bcrypt": "^5.0.0",
"@types/bencode": "^2.0.0",
"@types/bluebird": "^3.5.33",
@ -197,6 +201,7 @@
"@types/fluent-ffmpeg": "^2.1.16",
"@types/fs-extra": "^11.0.1",
"@types/jsonld": "^1.5.9",
"@types/jsonwebtoken": "^9.0.5",
"@types/lodash-es": "^4.17.8",
"@types/magnet-uri": "^5.1.1",
"@types/maildev": "^0.0.4",
@ -212,6 +217,7 @@
"@types/validator": "^13.9.0",
"@types/webtorrent": "^0.109.0",
"@types/ws": "^8.2.0",
"@types/yauzl": "^2.10.3",
"@typescript-eslint/eslint-plugin": "^6.7.5",
"autocannon": "^7.0.4",
"chai": "^4.1.1",
@ -228,6 +234,7 @@
"eslint-plugin-promise": "^6.0.0",
"fast-xml-parser": "^4.0.0-beta.8",
"jpeg-js": "^0.4.4",
"jszip": "^3.10.1",
"mocha": "^10.0.0",
"pixelmatch": "^5.3.0",
"pngjs": "^7.0.0",

View File

@ -19,6 +19,18 @@ function removeQueryParams (url: string) {
return objUrl.toString()
}
function queryParamsToObject (entries: any) {
const result: { [ id: string ]: string | number | boolean } = {}
for (const [ key, value ] of entries) {
result[key] = value
}
return result
}
// ---------------------------------------------------------------------------
function buildPlaylistLink (playlist: Pick<VideoPlaylist, 'shortUUID'>, base?: string) {
return (base ?? window.location.origin) + buildPlaylistWatchPath(playlist)
}
@ -123,6 +135,7 @@ function decoratePlaylistLink (options: {
export {
addQueryParams,
removeQueryParams,
queryParamsToObject,
buildPlaylistLink,
buildVideoLink,

View File

@ -18,7 +18,7 @@ export interface ActivityPubActor {
sharedInbox: string
}
summary: string
attributedTo: ActivityPubAttributedTo[]
attributedTo?: ActivityPubAttributedTo[]
support?: string
publicKey: {
@ -31,4 +31,8 @@ export interface ActivityPubActor {
icon?: ActivityIconObject | ActivityIconObject[]
published?: string
// For export
likes?: string
dislikes?: string
}

View File

@ -1,7 +1,7 @@
import { Activity } from './activity.js'
export interface ActivityPubCollection {
'@context': string[]
'@context': any[]
type: 'Collection' | 'CollectionPage'
totalItems: number
partOf?: string

View File

@ -1,5 +1,7 @@
export interface ActivityPubOrderedCollection<T> {
'@context': string[]
id: string
'@context': any[]
type: 'OrderedCollection' | 'OrderedCollectionPage'
totalItems: number
orderedItems: T[]

View File

@ -59,6 +59,16 @@ export interface VideoObject {
to?: string[]
cc?: string[]
// For export
attachment?: {
type: 'Video'
url: string
mediaType: string
height: number
size: number
fps: number
}[]
}
export interface ActivityPubStoryboard {

View File

@ -0,0 +1,6 @@
export const FileStorage = {
FILE_SYSTEM: 0,
OBJECT_STORAGE: 1
} as const
export type FileStorageType = typeof FileStorage[keyof typeof FileStorage]

View File

@ -1 +1,2 @@
export * from './file-storage.enum.js'
export * from './result-list.model.js'

View File

@ -0,0 +1,9 @@
export * from './peertube-export-format/index.js'
export * from './user-export-request-result.model.js'
export * from './user-export-request.model.js'
export * from './user-export-state.enum.js'
export * from './user-export.model.js'
export * from './user-import.model.js'
export * from './user-import-state.enum.js'
export * from './user-import-result.model.js'
export * from './user-import-upload-result.model.js'

View File

@ -0,0 +1,18 @@
import { UserActorImageJSON } from './actor-export.model.js'
export interface AccountExportJSON {
url: string
name: string
displayName: string
description: string
updatedAt: string
createdAt: string
avatars: UserActorImageJSON[]
archiveFiles: {
avatar: string | null
}
}

View File

@ -0,0 +1,6 @@
export interface UserActorImageJSON {
width: number
url: string
createdAt: string
updatedAt: string
}

View File

@ -0,0 +1,9 @@
export interface BlocklistExportJSON {
instances: {
host: string
}[]
actors: {
handle: string
}[]
}

View File

@ -0,0 +1,23 @@
import { UserActorImageJSON } from './actor-export.model.js'
export interface ChannelExportJSON {
channels: {
url: string
name: string
displayName: string
description: string
support: string
updatedAt: string
createdAt: string
avatars: UserActorImageJSON[]
banners: UserActorImageJSON[]
archiveFiles: {
avatar: string | null
banner: string | null
}
}[]
}

View File

@ -0,0 +1,12 @@
export interface CommentsExportJSON {
comments: {
url: string
text: string
createdAt: string
videoUrl: string
inReplyToCommentUrl?: string
archiveFiles?: never
}[]
}

View File

@ -0,0 +1,8 @@
export interface DislikesExportJSON {
dislikes: {
videoUrl: string
createdAt: string
archiveFiles?: never
}[]
}

View File

@ -0,0 +1,9 @@
export interface FollowersExportJSON {
followers: {
handle: string
createdAt: string
targetHandle: string
archiveFiles?: never
}[]
}

View File

@ -0,0 +1,9 @@
export interface FollowingExportJSON {
following: {
handle: string
targetHandle: string
createdAt: string
archiveFiles?: never
}[]
}

View File

@ -0,0 +1,12 @@
export * from './account-export.model.js'
export * from './actor-export.model.js'
export * from './blocklist-export.model.js'
export * from './channel-export.model.js'
export * from './comments-export.model.js'
export * from './dislikes-export.model.js'
export * from './followers-export.model.js'
export * from './following-export.model.js'
export * from './likes-export.model.js'
export * from './user-settings-export.model.js'
export * from './video-export.model.js'
export * from './video-playlists-export.model.js'

View File

@ -0,0 +1,8 @@
export interface LikesExportJSON {
likes: {
videoUrl: string
createdAt: string
archiveFiles?: never
}[]
}

View File

@ -0,0 +1,26 @@
import { UserNotificationSetting } from '../../users/user-notification-setting.model.js'
import { NSFWPolicyType } from '../../videos/nsfw-policy.type.js'
export interface UserSettingsExportJSON {
email: string
emailPublic: boolean
nsfwPolicy: NSFWPolicyType
autoPlayVideo: boolean
autoPlayNextVideo: boolean
autoPlayNextVideoPlaylist: boolean
p2pEnabled: boolean
videosHistoryEnabled: boolean
videoLanguages: string[]
theme: string
createdAt: Date
notificationSettings: UserNotificationSetting
archiveFiles?: never
}

View File

@ -0,0 +1,103 @@
import {
LiveVideoLatencyModeType,
VideoPrivacyType,
VideoStateType,
VideoStreamingPlaylistType_Type
} from '../../videos/index.js'
export interface VideoExportJSON {
videos: {
uuid: string
createdAt: string
updatedAt: string
publishedAt: string
originallyPublishedAt: string
name: string
category: number
licence: number
language: string
tags: string[]
privacy: VideoPrivacyType
passwords: string[]
duration: number
description: string
support: string
isLive: boolean
live?: {
saveReplay: boolean
permanentLive: boolean
latencyMode: LiveVideoLatencyModeType
streamKey: string
replaySettings?: {
privacy: VideoPrivacyType
}
}
url: string
thumbnailUrl: string
previewUrl: string
views: number
likes: number
dislikes: number
nsfw: boolean
commentsEnabled: boolean
downloadEnabled: boolean
channel: {
name: string
}
waitTranscoding: boolean
state: VideoStateType
captions: {
createdAt: string
updatedAt: string
language: string
filename: string
fileUrl: string
}[]
files: VideoFileExportJSON[]
streamingPlaylists: {
type: VideoStreamingPlaylistType_Type
playlistUrl: string
segmentsSha256Url: string
files: VideoFileExportJSON[]
}[]
source?: {
filename: string
}
archiveFiles: {
videoFile: string | null
thumbnail: string | null
captions: Record<string, string> // The key is the language code
}
}[]
}
// ---------------------------------------------------------------------------
export interface VideoFileExportJSON {
resolution: number
size: number // Bytes
fps: number
torrentUrl: string
fileUrl: string
}

View File

@ -0,0 +1,34 @@
import { VideoPlaylistPrivacyType } from '../../videos/playlist/video-playlist-privacy.model.js'
import { VideoPlaylistType_Type } from '../../videos/playlist/video-playlist-type.model.js'
export interface VideoPlaylistsExportJSON {
videoPlaylists: {
displayName: string
description: string
privacy: VideoPlaylistPrivacyType
url: string
uuid: string
type: VideoPlaylistType_Type
channel: {
name: string
}
createdAt: string
updatedAt: string
thumbnailUrl: string
elements: {
videoUrl: string
startTimestamp?: number
stopTimestamp?: number
}[]
archiveFiles: {
thumbnail: string | null
}
}[]
}

View File

@ -0,0 +1,5 @@
export interface UserExportRequestResult {
export: {
id: number
}
}

View File

@ -0,0 +1,3 @@
export interface UserExportRequest {
withVideoFiles: boolean
}

View File

@ -0,0 +1,8 @@
export const UserExportState = {
PENDING: 1,
PROCESSING: 2,
COMPLETED: 3,
ERRORED: 4
} as const
export type UserExportStateType = typeof UserExportState[keyof typeof UserExportState]

View File

@ -0,0 +1,18 @@
import { UserExportStateType } from './user-export-state.enum.js'
export interface UserExport {
id: number
state: {
id: UserExportStateType
label: string
}
// In bytes
size: number
privateDownloadUrl: string
createdAt: string | Date
expiresOn: string | Date
}

View File

@ -0,0 +1,20 @@
type Summary = {
success: number
duplicates: number
errors: number
}
export interface UserImportResultSummary {
stats: {
blocklist: Summary
channels: Summary
likes: Summary
dislikes: Summary
following: Summary
videoPlaylists: Summary
videos: Summary
account: Summary
userSettings: Summary
}
}

View File

@ -0,0 +1,8 @@
export const UserImportState = {
PENDING: 1,
PROCESSING: 2,
COMPLETED: 3,
ERRORED: 4
} as const
export type UserImportStateType = typeof UserImportState[keyof typeof UserImportState]

View File

@ -0,0 +1,5 @@
export interface UserImportUploadResult {
userImport: {
id: number
}
}

View File

@ -0,0 +1,10 @@
import { UserImportStateType } from './user-import-state.enum.js'
export interface UserImport {
id: number
state: {
id: UserImportStateType
label: string
}
createdAt: string
}

View File

@ -3,6 +3,7 @@ export * from './actors/index.js'
export * from './bulk/index.js'
export * from './common/index.js'
export * from './custom-markup/index.js'
export * from './import-export/index.js'
export * from './feeds/index.js'
export * from './http/index.js'
export * from './joinpeertube/index.js'

View File

@ -65,6 +65,8 @@ export const serverFilterHookObject = {
'filter:api.video.post-import-url.accept.result': true,
'filter:api.video.post-import-torrent.accept.result': true,
'filter:api.video.update-file.accept.result': true,
// PeerTube >= 6.1
'filter:api.video.user-import.accept.result': true,
// Filter the result of the accept comment (thread or reply) functions
// If the functions return false then the user cannot post its comment
'filter:api.video-thread.create.accept.result': true,
@ -75,6 +77,8 @@ export const serverFilterHookObject = {
'filter:api.video.import-url.video-attribute.result': true,
'filter:api.video.import-torrent.video-attribute.result': true,
'filter:api.video.live.video-attribute.result': true,
// PeerTube >= 6.1
'filter:api.video.user-import.video-attribute.result': true,
// Filter params/result used to list threads of a specific video
// (used by the video watch page)

View File

@ -193,10 +193,23 @@ export interface CustomConfig {
enabled: boolean
}
}
videoChannelSynchronization: {
enabled: boolean
maxPerUser: number
}
users: {
enabled: boolean
}
}
export: {
users: {
enabled: boolean
maxUserVideoQuota: number
exportExpiration: number
}
}
trending: {
@ -260,5 +273,4 @@ export interface CustomConfig {
storyboards: {
enabled: boolean
}
}

View File

@ -9,4 +9,5 @@ export interface SendDebugCommand {
| 'process-video-viewers'
| 'process-video-channel-sync-latest'
| 'process-update-videos-scheduler'
| 'remove-expired-user-exports'
}

View File

@ -31,6 +31,8 @@ export type JobType =
| 'video-transcoding'
| 'videos-views-stats'
| 'generate-video-storyboard'
| 'create-user-export'
| 'import-user-archive'
export interface Job {
id: number | string
@ -302,3 +304,15 @@ export interface GenerateStoryboardPayload {
videoUUID: string
federate: boolean
}
// ---------------------------------------------------------------------------
export interface CreateUserExportPayload {
userExportId: number
}
// ---------------------------------------------------------------------------
export interface ImportUserArchivePayload {
userImportId: number
}

View File

@ -207,9 +207,22 @@ export interface ServerConfig {
enabled: boolean
}
}
videoChannelSynchronization: {
enabled: boolean
}
users: {
enabled:boolean
}
}
export: {
users: {
enabled: boolean
exportExpiration: number
maxUserVideoQuota: number
}
}
autoBlacklist: {

View File

@ -54,7 +54,9 @@ export const ServerErrorCode = {
VIDEO_REQUIRES_PASSWORD:'video_requires_password',
INCORRECT_VIDEO_PASSWORD:'incorrect_video_password',
VIDEO_ALREADY_BEING_TRANSCODED:'video_already_being_transcoded'
VIDEO_ALREADY_BEING_TRANSCODED:'video_already_being_transcoded',
MAX_USER_VIDEO_QUOTA_EXCEEDED_FOR_USER_EXPORT: 'max_user_video_quota_exceeded_for_user_export'
} as const
/**

View File

@ -47,7 +47,10 @@ export const UserRight = {
MANAGE_REGISTRATIONS: 28,
MANAGE_RUNNERS: 29
MANAGE_RUNNERS: 29,
MANAGE_USER_EXPORTS: 30,
MANAGE_USER_IMPORTS: 31
} as const
export type UserRightType = typeof UserRight[keyof typeof UserRight]

View File

@ -29,7 +29,6 @@ export * from './video-rate.type.js'
export * from './video-schedule-update.model.js'
export * from './video-sort-field.type.js'
export * from './video-state.enum.js'
export * from './video-storage.enum.js'
export * from './video-source.model.js'
export * from './video-streaming-playlist.model.js'

View File

@ -1,6 +0,0 @@
export const VideoStorage = {
FILE_SYSTEM: 0,
OBJECT_STORAGE: 1
} as const
export type VideoStorageType = typeof VideoStorage[keyof typeof VideoStorage]

View File

@ -1,4 +1,4 @@
import { basename, extname, isAbsolute, join, resolve } from 'path'
import { basename, extname, isAbsolute, join, parse, resolve } from 'path'
import { fileURLToPath } from 'url'
let rootPath: string
@ -48,3 +48,15 @@ export function buildAbsoluteFixturePath (path: string, customCIPath = false) {
return join(root(), 'packages', 'tests', 'fixtures', path)
}
export function getFilenameFromUrl (url: string) {
return getFilename(new URL(url).pathname)
}
export function getFilename (path: string) {
return parse(path).base
}
export function getFilenameWithoutExt (path: string) {
return parse(path).name
}

View File

@ -45,3 +45,5 @@ export type DeepOmitArray<T extends any[], K> = {
}
export type Unpacked<T> = T extends (infer U)[] ? U : T
export type Awaitable<T> = T | PromiseLike<T>

View File

@ -6,6 +6,7 @@ import {
ABUSE_STATES,
buildLanguages,
RUNNER_JOB_STATES,
USER_EXPORT_STATES,
USER_REGISTRATION_STATES,
VIDEO_CATEGORIES,
VIDEO_CHANNEL_SYNC_STATE,
@ -14,6 +15,7 @@ import {
VIDEO_PLAYLIST_PRIVACIES,
VIDEO_PLAYLIST_TYPES,
VIDEO_PRIVACIES,
USER_IMPORT_STATES,
VIDEO_STATES
} from '@peertube/peertube-server/core/initializers/constants.js'
@ -96,6 +98,8 @@ Object.values(VIDEO_CATEGORIES)
.concat(Object.values(ABUSE_STATES))
.concat(Object.values(USER_REGISTRATION_STATES))
.concat(Object.values(RUNNER_JOB_STATES))
.concat(Object.values(USER_EXPORT_STATES))
.concat(Object.values(USER_IMPORT_STATES))
.concat([
'This video does not exist.',
'We cannot fetch the video. Please try again later.',

View File

@ -355,6 +355,16 @@ function customConfig (): CustomConfig {
videoChannelSynchronization: {
enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED,
maxPerUser: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER
},
users: {
enabled: CONFIG.IMPORT.USERS.ENABLED
}
},
export: {
users: {
enabled: CONFIG.EXPORT.USERS.ENABLED,
exportExpiration: CONFIG.EXPORT.USERS.EXPORT_EXPIRATION,
maxUserVideoQuota: CONFIG.EXPORT.USERS.MAX_USER_VIDEO_QUOTA
}
},
trending: {

View File

@ -50,7 +50,6 @@ apiRouter.use('/custom-pages', customPageRouter)
apiRouter.use('/blocklist', blocklistRouter)
apiRouter.use('/runners', runnersRouter)
// apiRouter.use(apiRateLimiter)
apiRouter.use('/ping', pong)
apiRouter.use('/*', badRequest)

View File

@ -9,7 +9,7 @@ import {
runnerJobGetVideoStudioTaskFileValidator,
runnerJobGetVideoTranscodingFileValidator
} from '@server/middlewares/validators/runners/job-files.js'
import { RunnerJobState, VideoStorage } from '@peertube/peertube-models'
import { RunnerJobState, FileStorage } from '@peertube/peertube-models'
const lTags = loggerTagsFactory('api', 'runner')
@ -57,7 +57,7 @@ async function getMaxQualityVideoFile (req: express.Request, res: express.Respon
const file = video.getMaxQualityFile()
if (file.storage === VideoStorage.OBJECT_STORAGE) {
if (file.storage === FileStorage.OBJECT_STORAGE) {
if (file.isHLS()) {
return proxifyHLS({
req,

View File

@ -151,7 +151,7 @@ async function searchVideoURI (url: string, res: express.Response) {
logger.info('Cannot search remote video %s.', url, { err })
}
} else {
video = await searchLocalUrl(sanitizeLocalUrl(url), url => VideoModel.loadByUrlAndPopulateAccount(url))
video = await searchLocalUrl(sanitizeLocalUrl(url), url => VideoModel.loadByUrlAndPopulateAccountAndFiles(url))
}
return res.json({

View File

@ -7,6 +7,7 @@ import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-ch
import { VideoViewsBufferScheduler } from '@server/lib/schedulers/video-views-buffer-scheduler.js'
import { VideoViewsManager } from '@server/lib/views/video-views-manager.js'
import { authenticate, ensureUserHasRight } from '../../../middlewares/index.js'
import { RemoveExpiredUserExportsScheduler } from '@server/lib/schedulers/remove-expired-user-exports-scheduler.js'
const debugRouter = express.Router()
@ -42,6 +43,7 @@ async function runCommand (req: express.Request, res: express.Response) {
const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = {
'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(),
'remove-expired-user-exports': () => RemoveExpiredUserExportsScheduler.Instance.execute(),
'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(),
'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(),
'process-update-videos-scheduler': () => UpdateVideosScheduler.Instance.execute(),

View File

@ -1,9 +1,7 @@
import 'multer'
import express from 'express'
import { HttpStatusCode, UserRight } from '@peertube/peertube-models'
import { logger } from '@server/helpers/logger.js'
import { getServerActor } from '@server/models/application/application.js'
import { UserNotificationModel } from '@server/models/user/user-notification.js'
import { getFormattedObjects } from '../../../helpers/utils.js'
import {
addAccountInBlocklist,
@ -105,15 +103,9 @@ async function blockAccount (req: express.Request, res: express.Response) {
const serverActor = await getServerActor()
const accountToBlock = res.locals.account
await addAccountInBlocklist(serverActor.Account.id, accountToBlock.id)
await addAccountInBlocklist({ byAccountId: serverActor.Account.id, targetAccountId: accountToBlock.id, removeNotificationOfUserId: null })
UserNotificationModel.removeNotificationsOf({
id: accountToBlock.id,
type: 'account',
forUserId: null // For all users
}).catch(err => logger.error('Cannot remove notifications after an account mute.', { err }))
return res.status(HttpStatusCode.NO_CONTENT_204).end()
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
}
async function unblockAccount (req: express.Request, res: express.Response) {
@ -121,7 +113,7 @@ async function unblockAccount (req: express.Request, res: express.Response) {
await removeAccountFromBlocklist(accountBlock)
return res.status(HttpStatusCode.NO_CONTENT_204).end()
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
}
async function listBlockedServers (req: express.Request, res: express.Response) {
@ -142,15 +134,13 @@ async function blockServer (req: express.Request, res: express.Response) {
const serverActor = await getServerActor()
const serverToBlock = res.locals.server
await addServerInBlocklist(serverActor.Account.id, serverToBlock.id)
await addServerInBlocklist({
byAccountId: serverActor.Account.id,
targetServerId: serverToBlock.id,
removeNotificationOfUserId: null
})
UserNotificationModel.removeNotificationsOf({
id: serverToBlock.id,
type: 'server',
forUserId: null // For all users
}).catch(err => logger.error('Cannot remove notifications after a server mute.', { err }))
return res.status(HttpStatusCode.NO_CONTENT_204).end()
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
}
async function unblockServer (req: express.Request, res: express.Response) {
@ -158,5 +148,5 @@ async function unblockServer (req: express.Request, res: express.Response) {
await removeServerFromBlocklist(serverBlock)
return res.status(HttpStatusCode.NO_CONTENT_204).end()
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
}

View File

@ -47,6 +47,8 @@ import { mySubscriptionsRouter } from './my-subscriptions.js'
import { myVideoPlaylistsRouter } from './my-video-playlists.js'
import { registrationsRouter } from './registrations.js'
import { twoFactorRouter } from './two-factor.js'
import { userExportsRouter } from './user-exports.js'
import { userImportRouter } from './user-imports.js'
const auditLogger = auditLoggerFactory('users')
@ -55,6 +57,8 @@ const usersRouter = express.Router()
usersRouter.use(apiRateLimiter)
usersRouter.use('/', emailVerificationRouter)
usersRouter.use('/', userExportsRouter)
usersRouter.use('/', userImportRouter)
usersRouter.use('/', registrationsRouter)
usersRouter.use('/', twoFactorRouter)
usersRouter.use('/', tokensRouter)

View File

@ -262,11 +262,12 @@ async function updateMyAvatar (req: express.Request, res: express.Response) {
const userAccount = await AccountModel.load(user.Account.id)
const avatars = await updateLocalActorImageFiles(
userAccount,
avatarPhysicalFile,
ActorImageType.AVATAR
)
const avatars = await updateLocalActorImageFiles({
accountOrChannel: userAccount,
imagePhysicalFile: avatarPhysicalFile,
type: ActorImageType.AVATAR,
sendActorUpdate: true
})
return res.json({
avatars: avatars.map(avatar => avatar.toFormattedJSON())

View File

@ -1,8 +1,6 @@
import 'multer'
import express from 'express'
import { HttpStatusCode } from '@peertube/peertube-models'
import { logger } from '@server/helpers/logger.js'
import { UserNotificationModel } from '@server/models/user/user-notification.js'
import { getFormattedObjects } from '../../../helpers/utils.js'
import {
addAccountInBlocklist,
@ -97,15 +95,9 @@ async function blockAccount (req: express.Request, res: express.Response) {
const user = res.locals.oauth.token.User
const accountToBlock = res.locals.account
await addAccountInBlocklist(user.Account.id, accountToBlock.id)
await addAccountInBlocklist({ byAccountId: user.Account.id, targetAccountId: accountToBlock.id, removeNotificationOfUserId: user.id })
UserNotificationModel.removeNotificationsOf({
id: accountToBlock.id,
type: 'account',
forUserId: user.id
}).catch(err => logger.error('Cannot remove notifications after an account mute.', { err }))
return res.status(HttpStatusCode.NO_CONTENT_204).end()
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
}
async function unblockAccount (req: express.Request, res: express.Response) {
@ -134,15 +126,13 @@ async function blockServer (req: express.Request, res: express.Response) {
const user = res.locals.oauth.token.User
const serverToBlock = res.locals.server
await addServerInBlocklist(user.Account.id, serverToBlock.id)
await addServerInBlocklist({
byAccountId: user.Account.id,
targetServerId: serverToBlock.id,
removeNotificationOfUserId: user.id
})
UserNotificationModel.removeNotificationsOf({
id: serverToBlock.id,
type: 'server',
forUserId: user.id
}).catch(err => logger.error('Cannot remove notifications after a server mute.', { err }))
return res.status(HttpStatusCode.NO_CONTENT_204).end()
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
}
async function unblockServer (req: express.Request, res: express.Response) {
@ -150,5 +140,5 @@ async function unblockServer (req: express.Request, res: express.Response) {
await removeServerFromBlocklist(serverBlock)
return res.status(HttpStatusCode.NO_CONTENT_204).end()
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
}

View File

@ -16,7 +16,7 @@ import {
listUserNotificationsValidator,
markAsReadUserNotificationsValidator,
updateNotificationSettingsValidator
} from '../../../middlewares/validators/user-notifications.js'
} from '../../../middlewares/validators/users/user-notifications.js'
import { UserNotificationSettingModel } from '../../../models/user/user-notification-setting.js'
import { meRouter } from './me.js'
@ -59,12 +59,6 @@ async function updateNotificationSettings (req: express.Request, res: express.Re
const user = res.locals.oauth.token.User
const body = req.body as UserNotificationSetting
const query = {
where: {
userId: user.id
}
}
const values: UserNotificationSetting = {
newVideoFromSubscription: body.newVideoFromSubscription,
newCommentOnMyVideo: body.newCommentOnMyVideo,
@ -85,9 +79,9 @@ async function updateNotificationSettings (req: express.Request, res: express.Re
myVideoStudioEditionFinished: body.myVideoStudioEditionFinished
}
await UserNotificationSettingModel.update(values, query)
await UserNotificationSettingModel.updateUserSettings(values, user.id)
return res.status(HttpStatusCode.NO_CONTENT_204).end()
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
}
async function listUserNotifications (req: express.Request, res: express.Response) {
@ -103,7 +97,7 @@ async function markAsReadUserNotifications (req: express.Request, res: express.R
await UserNotificationModel.markAsRead(user.id, req.body.ids)
return res.status(HttpStatusCode.NO_CONTENT_204).end()
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
}
async function markAsReadAllUserNotifications (req: express.Request, res: express.Response) {
@ -111,5 +105,5 @@ async function markAsReadAllUserNotifications (req: express.Request, res: expres
await UserNotificationModel.markAllAsRead(user.id)
return res.status(HttpStatusCode.NO_CONTENT_204).end()
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
}

View File

@ -0,0 +1,100 @@
import express from 'express'
import { FileStorage, HttpStatusCode, UserExportRequest, UserExportRequestResult, UserExportState } from '@peertube/peertube-models'
import {
apiRateLimiter,
asyncMiddleware,
authenticate,
userExportDeleteValidator,
userExportRequestValidator,
userExportsListValidator
} from '../../../middlewares/index.js'
import { UserExportModel } from '@server/models/user/user-export.js'
import { getFormattedObjects } from '@server/helpers/utils.js'
import { sequelizeTypescript } from '@server/initializers/database.js'
import { JobQueue } from '@server/lib/job-queue/job-queue.js'
import { CONFIG } from '@server/initializers/config.js'
const userExportsRouter = express.Router()
userExportsRouter.use(apiRateLimiter)
userExportsRouter.post('/:userId/exports/request',
authenticate,
asyncMiddleware(userExportRequestValidator),
asyncMiddleware(requestExport)
)
userExportsRouter.get('/:userId/exports',
authenticate,
asyncMiddleware(userExportsListValidator),
asyncMiddleware(listUserExports)
)
userExportsRouter.delete('/:userId/exports/:id',
authenticate,
asyncMiddleware(userExportDeleteValidator),
asyncMiddleware(deleteUserExport)
)
// ---------------------------------------------------------------------------
export {
userExportsRouter
}
// ---------------------------------------------------------------------------
async function requestExport (req: express.Request, res: express.Response) {
const body = req.body as UserExportRequest
const exportModel = new UserExportModel({
state: UserExportState.PENDING,
withVideoFiles: body.withVideoFiles,
storage: CONFIG.OBJECT_STORAGE.ENABLED
? FileStorage.OBJECT_STORAGE
: FileStorage.FILE_SYSTEM,
userId: res.locals.user.id,
createdAt: new Date()
})
exportModel.generateAndSetFilename()
await sequelizeTypescript.transaction(async transaction => {
await exportModel.save({ transaction })
})
await JobQueue.Instance.createJob({ type: 'create-user-export', payload: { userExportId: exportModel.id } })
return res.json({
export: {
id: exportModel.id
}
} as UserExportRequestResult)
}
async function listUserExports (req: express.Request, res: express.Response) {
const resultList = await UserExportModel.listForApi({
start: req.query.start,
count: req.query.count,
user: res.locals.user
})
return res.json(getFormattedObjects(resultList.data, resultList.total))
}
async function deleteUserExport (req: express.Request, res: express.Response) {
const userExport = res.locals.userExport
await sequelizeTypescript.transaction(async transaction => {
await userExport.reload({ transaction })
if (!userExport.canBeSafelyRemoved()) {
return res.sendStatus(HttpStatusCode.CONFLICT_409)
}
await userExport.destroy({ transaction })
})
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
}

View File

@ -0,0 +1,90 @@
import express from 'express'
import {
apiRateLimiter,
asyncMiddleware,
authenticate
} from '../../../middlewares/index.js'
import { uploadx } from '@server/lib/uploadx.js'
import {
getLatestImportStatusValidator,
userImportRequestResumableInitValidator,
userImportRequestResumableValidator
} from '@server/middlewares/validators/users/user-import.js'
import { HttpStatusCode, UserImportState, UserImportUploadResult } from '@peertube/peertube-models'
import { logger } from '@server/helpers/logger.js'
import { UserImportModel } from '@server/models/user/user-import.js'
import { getFSUserImportFilePath } from '@server/lib/paths.js'
import { move } from 'fs-extra/esm'
import { JobQueue } from '@server/lib/job-queue/job-queue.js'
import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'
const userImportRouter = express.Router()
userImportRouter.use(apiRateLimiter)
userImportRouter.post('/:userId/imports/import-resumable',
authenticate,
asyncMiddleware(userImportRequestResumableInitValidator),
(req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
)
userImportRouter.delete('/:userId/imports/import-resumable',
authenticate,
(req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
)
userImportRouter.put('/:userId/imports/import-resumable',
authenticate,
uploadx.upload, // uploadx doesn't next() before the file upload completes
asyncMiddleware(userImportRequestResumableValidator),
asyncMiddleware(addUserImportResumable)
)
userImportRouter.get('/:userId/imports/latest',
authenticate,
asyncMiddleware(getLatestImportStatusValidator),
asyncMiddleware(getLatestImport)
)
// ---------------------------------------------------------------------------
export {
userImportRouter
}
// ---------------------------------------------------------------------------
async function addUserImportResumable (req: express.Request, res: express.Response) {
const file = res.locals.importUserFileResumable
const user = res.locals.user
// Move import
const userImport = new UserImportModel({
state: UserImportState.PENDING,
userId: user.id,
createdAt: new Date()
})
userImport.generateAndSetFilename()
await move(file.path, getFSUserImportFilePath(userImport))
await saveInTransactionWithRetries(userImport)
// Create job
await JobQueue.Instance.createJob({ type: 'import-user-archive', payload: { userImportId: userImport.id } })
logger.info('User import request job created for user ' + user.username)
return res.json({
userImport: {
id: userImport.id
}
} as UserImportUploadResult)
}
async function getLatestImport (req: express.Request, res: express.Response) {
const userImport = await UserImportModel.loadLatestByUserId(res.locals.user.id)
if (!userImport) return res.sendStatus(HttpStatusCode.NOT_FOUND_404)
return res.json(userImport.toFormattedJSON())
}

View File

@ -213,7 +213,12 @@ async function updateVideoChannelBanner (req: express.Request, res: express.Resp
const videoChannel = res.locals.videoChannel
const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannel.toFormattedJSON())
const banners = await updateLocalActorImageFiles(videoChannel, bannerPhysicalFile, ActorImageType.BANNER)
const banners = await updateLocalActorImageFiles({
accountOrChannel: videoChannel,
imagePhysicalFile: bannerPhysicalFile,
type: ActorImageType.BANNER,
sendActorUpdate: true
})
auditLogger.update(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannel.toFormattedJSON()), oldVideoChannelAuditKeys)
@ -227,7 +232,13 @@ async function updateVideoChannelAvatar (req: express.Request, res: express.Resp
const videoChannel = res.locals.videoChannel
const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannel.toFormattedJSON())
const avatars = await updateLocalActorImageFiles(videoChannel, avatarPhysicalFile, ActorImageType.AVATAR)
const avatars = await updateLocalActorImageFiles({
accountOrChannel: videoChannel,
imagePhysicalFile: avatarPhysicalFile,
type: ActorImageType.AVATAR,
sendActorUpdate: true
})
auditLogger.update(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannel.toFormattedJSON()), oldVideoChannelAuditKeys)
return res.json({

View File

@ -192,7 +192,6 @@ async function addVideoPlaylist (req: express.Request, res: express.Response) {
const videoPlaylistCreated = await videoPlaylist.save({ transaction: t }) as MVideoPlaylistFull
if (thumbnailModel) {
thumbnailModel.automaticallyGenerated = false
await videoPlaylistCreated.setAndSaveThumbnail(thumbnailModel, t)
}

View File

@ -1,12 +1,8 @@
import express from 'express'
import { HttpStatusCode, UserVideoRateUpdate } from '@peertube/peertube-models'
import { logger } from '../../../helpers/logger.js'
import { VIDEO_RATE_TYPES } from '../../../initializers/constants.js'
import { sequelizeTypescript } from '../../../initializers/database.js'
import { getLocalRateUrl, sendVideoRateChange } from '../../../lib/activitypub/video-rates.js'
import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoUpdateRateValidator } from '../../../middlewares/index.js'
import { AccountModel } from '../../../models/account/account.js'
import { AccountVideoRateModel } from '../../../models/account/account-video-rate.js'
import { userRateVideo } from '@server/lib/rate.js'
const rateVideoRouter = express.Router()
@ -25,63 +21,16 @@ export {
// ---------------------------------------------------------------------------
async function rateVideo (req: express.Request, res: express.Response) {
const body: UserVideoRateUpdate = req.body
const rateType = body.rating
const videoInstance = res.locals.videoAll
const userAccount = res.locals.oauth.token.User.Account
const user = res.locals.oauth.token.User
const video = res.locals.videoAll
await sequelizeTypescript.transaction(async t => {
const sequelizeOptions = { transaction: t }
const accountInstance = await AccountModel.load(userAccount.id, t)
const previousRate = await AccountVideoRateModel.load(accountInstance.id, videoInstance.id, t)
// Same rate, nothing do to
if (rateType === 'none' && !previousRate || previousRate?.type === rateType) return
let likesToIncrement = 0
let dislikesToIncrement = 0
if (rateType === VIDEO_RATE_TYPES.LIKE) likesToIncrement++
else if (rateType === VIDEO_RATE_TYPES.DISLIKE) dislikesToIncrement++
// There was a previous rate, update it
if (previousRate) {
// We will remove the previous rate, so we will need to update the video count attribute
if (previousRate.type === 'like') likesToIncrement--
else if (previousRate.type === 'dislike') dislikesToIncrement--
if (rateType === 'none') { // Destroy previous rate
await previousRate.destroy(sequelizeOptions)
} else { // Update previous rate
previousRate.type = rateType
previousRate.url = getLocalRateUrl(rateType, userAccount.Actor, videoInstance)
await previousRate.save(sequelizeOptions)
}
} else if (rateType !== 'none') { // There was not a previous rate, insert a new one if there is a rate
const query = {
accountId: accountInstance.id,
videoId: videoInstance.id,
type: rateType,
url: getLocalRateUrl(rateType, userAccount.Actor, videoInstance)
}
await AccountVideoRateModel.create(query, sequelizeOptions)
}
const incrementQuery = {
likes: likesToIncrement,
dislikes: dislikesToIncrement
}
await videoInstance.increment(incrementQuery, sequelizeOptions)
await sendVideoRateChange(accountInstance, videoInstance, likesToIncrement, dislikesToIncrement, t)
logger.info('Account video rate for video %s of account %s updated.', videoInstance.name, accountInstance.name)
await userRateVideo({
account: user.Account,
rateType: (req.body as UserVideoRateUpdate).rating,
video
})
return res.type('json')
.status(HttpStatusCode.NO_CONTENT_204)
.end()
logger.info('Account video rate for video %s of account %s updated.', video.name, user.username)
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
}

View File

@ -5,7 +5,7 @@ import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-q
import { Hooks } from '@server/lib/plugins/hooks.js'
import { regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail.js'
import { uploadx } from '@server/lib/uploadx.js'
import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video.js'
import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js'
import { buildNewFile } from '@server/lib/video-file.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'

View File

@ -6,7 +6,7 @@ import { exists } from '@server/helpers/custom-validators/misc.js'
import { changeVideoChannelShare } from '@server/lib/activitypub/share.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { setVideoPrivacy } from '@server/lib/video-privacy.js'
import { addVideoJobsAfterUpdate, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video.js'
import { buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video.js'
import { openapiOperationDoc } from '@server/middlewares/doc.js'
import { VideoPasswordModel } from '@server/models/video/video-password.js'
import { FilteredModelAttributes } from '@server/types/index.js'
@ -23,6 +23,7 @@ import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videosU
import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update.js'
import { VideoModel } from '../../../models/video/video.js'
import { replaceChaptersFromDescriptionIfNeeded } from '@server/lib/video-chapters.js'
import { addVideoJobsAfterUpdate } from '@server/lib/video-jobs.js'
const lTags = loggerTagsFactory('api', 'video')
const auditLogger = auditLoggerFactory('videos')

View File

@ -3,14 +3,10 @@ import { move } from 'fs-extra/esm'
import { basename } from 'path'
import { getResumableUploadPath } from '@server/helpers/upload.js'
import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url.js'
import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue/index.js'
import { Redis } from '@server/lib/redis.js'
import { uploadx } from '@server/lib/uploadx.js'
import {
buildLocalVideoFromReq,
buildMoveJob,
buildStoryboardJobIfNeeded,
buildVideoThumbnailsFromReq,
buildLocalVideoFromReq, buildVideoThumbnailsFromReq,
setVideoTags
} from '@server/lib/video.js'
import { buildNewFile } from '@server/lib/video-file.js'
@ -21,7 +17,7 @@ import { VideoPasswordModel } from '@server/models/video/video-password.js'
import { VideoSourceModel } from '@server/models/video/video-source.js'
import { MVideoFile, MVideoFullLight, MVideoThumbnail } from '@server/types/models/index.js'
import { uuidToShort } from '@peertube/peertube-node-utils'
import { HttpStatusCode, ThumbnailType, VideoCreate, VideoPrivacy, VideoState } from '@peertube/peertube-models'
import { HttpStatusCode, ThumbnailType, VideoCreate, VideoPrivacy } from '@peertube/peertube-models'
import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger.js'
import { createReqFiles } from '../../../helpers/express-utils.js'
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
@ -43,6 +39,7 @@ import { VideoModel } from '../../../models/video/video.js'
import { ffprobePromise, getChaptersFromContainer } from '@peertube/peertube-ffmpeg'
import { replaceChapters, replaceChaptersFromDescriptionIfNeeded } from '@server/lib/video-chapters.js'
import { FfprobeData } from 'fluent-ffmpeg'
import { addVideoJobsAfterCreation } from '@server/lib/video-jobs.js'
const lTags = loggerTagsFactory('api', 'video')
const auditLogger = auditLoggerFactory('videos')
@ -230,7 +227,7 @@ async function addVideo (options: {
// Channel has a new content, set as updated
await videoCreated.VideoChannel.setAsUpdated()
addVideoJobsAfterUpload(videoCreated, videoFile)
addVideoJobsAfterCreation({ video: videoCreated, videoFile })
.catch(err => logger.error('Cannot build new video jobs of %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) }))
Hooks.runAction('action:api.video.uploaded', { video: videoCreated, req, res })
@ -244,55 +241,6 @@ async function addVideo (options: {
}
}
async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile) {
const jobs: (CreateJobArgument & CreateJobOptions)[] = [
{
type: 'manage-video-torrent' as 'manage-video-torrent',
payload: {
videoId: video.id,
videoFileId: videoFile.id,
action: 'create'
}
},
buildStoryboardJobIfNeeded({ video, federate: false }),
{
type: 'notify',
payload: {
action: 'new-video',
videoUUID: video.uuid
}
},
{
type: 'federate-video' as 'federate-video',
payload: {
videoUUID: video.uuid,
isNewVideoForFederation: true
}
}
]
if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
jobs.push(await buildMoveJob({ video, previousVideoState: undefined, type: 'move-to-object-storage' }))
}
if (video.state === VideoState.TO_TRANSCODE) {
jobs.push({
type: 'transcoding-job-builder' as 'transcoding-job-builder',
payload: {
videoUUID: video.uuid,
optimizeJob: {
isNewVideo: true
}
}
})
}
return JobQueue.Instance.createSequentialJobFlow(...jobs)
}
async function deleteUploadResumableCache (req: express.Request, res: express.Response, next: express.NextFunction) {
await Redis.Instance.deleteUploadSession(req.query.upload_id)

View File

@ -2,14 +2,30 @@ import cors from 'cors'
import express from 'express'
import { logger } from '@server/helpers/logger.js'
import { VideoTorrentsSimpleFileCache } from '@server/lib/files-cache/index.js'
import { generateHLSFilePresignedUrl, generateWebVideoPresignedUrl } from '@server/lib/object-storage/index.js'
import {
generateHLSFilePresignedUrl,
generateUserExportPresignedUrl,
generateWebVideoPresignedUrl
} from '@server/lib/object-storage/index.js'
import { Hooks } from '@server/lib/plugins/hooks.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models/index.js'
import {
MStreamingPlaylist,
MStreamingPlaylistVideo,
MUserExport,
MVideo,
MVideoFile,
MVideoFullLight
} from '@server/types/models/index.js'
import { forceNumber } from '@peertube/peertube-core-utils'
import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@peertube/peertube-models'
import { HttpStatusCode, FileStorage, VideoStreamingPlaylistType } from '@peertube/peertube-models'
import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants.js'
import { asyncMiddleware, optionalAuthenticate, videosDownloadValidator } from '../middlewares/index.js'
import {
asyncMiddleware, optionalAuthenticate,
userExportDownloadValidator,
videosDownloadValidator
} from '../middlewares/index.js'
import { getFSUserExportFilePath } from '@server/lib/paths.js'
const downloadRouter = express.Router()
@ -34,6 +50,12 @@ downloadRouter.use(
asyncMiddleware(downloadHLSVideoFile)
)
downloadRouter.use(
STATIC_DOWNLOAD_PATHS.USER_EXPORT + ':filename',
asyncMiddleware(userExportDownloadValidator), // Include JWT token authentication
asyncMiddleware(downloadUserExport)
)
// ---------------------------------------------------------------------------
export {
@ -99,8 +121,8 @@ async function downloadVideoFile (req: express.Request, res: express.Response) {
const videoName = video.name.replace(/[/\\]/g, '_')
const downloadFilename = `${videoName}-${videoFile.resolution}p${videoFile.extname}`
if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
return redirectToObjectStorage({ req, res, video, file: videoFile, downloadFilename })
if (videoFile.storage === FileStorage.OBJECT_STORAGE) {
return redirectVideoDownloadToObjectStorage({ res, video, file: videoFile, downloadFilename })
}
await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), path => {
@ -140,8 +162,8 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response
const videoName = video.name.replace(/\//g, '_')
const downloadFilename = `${videoName}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}`
if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
return redirectToObjectStorage({ req, res, video, streamingPlaylist, file: videoFile, downloadFilename })
if (videoFile.storage === FileStorage.OBJECT_STORAGE) {
return redirectVideoDownloadToObjectStorage({ res, video, streamingPlaylist, file: videoFile, downloadFilename })
}
await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(streamingPlaylist), path => {
@ -149,6 +171,21 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response
})
}
function downloadUserExport (req: express.Request, res: express.Response) {
const userExport = res.locals.userExport
const downloadFilename = userExport.filename
if (userExport.storage === FileStorage.OBJECT_STORAGE) {
return redirectUserExportToObjectStorage({ res, userExport, downloadFilename })
}
res.download(getFSUserExportFilePath(userExport), downloadFilename)
return Promise.resolve()
}
// ---------------------------------------------------------------------------
function getVideoFile (req: express.Request, files: MVideoFile[]) {
const resolution = forceNumber(req.params.resolution)
return files.find(f => f.resolution === resolution)
@ -194,8 +231,7 @@ function checkAllowResult (res: express.Response, allowParameters: any, result?:
return true
}
async function redirectToObjectStorage (options: {
req: express.Request
async function redirectVideoDownloadToObjectStorage (options: {
res: express.Response
video: MVideo
file: MVideoFile
@ -212,3 +248,17 @@ async function redirectToObjectStorage (options: {
return res.redirect(url)
}
async function redirectUserExportToObjectStorage (options: {
res: express.Response
downloadFilename: string
userExport: MUserExport
}) {
const { res, downloadFilename, userExport } = options
const url = await generateUserExportPresignedUrl({ userExport, downloadFilename })
logger.debug('Generating pre-signed URL %s for user export %s', url, userExport.filename)
return res.redirect(url)
}

View File

@ -1,14 +1,11 @@
import { createReadStream, createWriteStream } from 'fs'
import { move, remove } from 'fs-extra/esm'
import { join } from 'path'
import { Transform } from 'stream'
import { MVideoCaption } from '@server/types/models/index.js'
import { CONFIG } from '../initializers/config.js'
import { pipelinePromise } from './core-utils.js'
async function moveAndProcessCaptionFile (physicalFile: { filename: string, path: string }, videoCaption: MVideoCaption) {
const videoCaptionsDir = CONFIG.STORAGE.CAPTIONS_DIR
const destination = join(videoCaptionsDir, videoCaption.filename)
async function moveAndProcessCaptionFile (physicalFile: { filename?: string, path: string }, videoCaption: MVideoCaption) {
const destination = videoCaption.getFSPath()
// Convert this srt file to vtt
if (physicalFile.path.endsWith('.srt')) {
@ -19,7 +16,7 @@ async function moveAndProcessCaptionFile (physicalFile: { filename: string, path
}
// This is important in case if there is another attempt in the retry process
physicalFile.filename = videoCaption.filename
if (physicalFile.filename) physicalFile.filename = videoCaption.filename
physicalFile.path = destination
}

View File

@ -0,0 +1,55 @@
import { createWriteStream } from 'fs'
import { ensureDir } from 'fs-extra/esm'
import { dirname, join } from 'path'
import { pipeline } from 'stream'
import * as yauzl from 'yauzl'
import { logger, loggerTagsFactory } from './logger.js'
const lTags = loggerTagsFactory('unzip')
export async function unzip (source: string, destination: string) {
await ensureDir(destination)
logger.info(`Unzip ${source} to ${destination}`, lTags())
return new Promise<void>((res, rej) => {
yauzl.open(source, { lazyEntries: true }, (err, zipFile) => {
if (err) return rej(err)
zipFile.readEntry()
zipFile.on('entry', async entry => {
const entryPath = join(destination, entry.fileName)
try {
if (/\/$/.test(entry.fileName)) {
await ensureDir(entryPath)
logger.debug(`Creating directory from zip ${entryPath}`, lTags())
zipFile.readEntry()
return
}
await ensureDir(dirname(entryPath))
} catch (err) {
return rej(err)
}
zipFile.openReadStream(entry, (readErr, readStream) => {
if (readErr) return rej(readErr)
logger.debug(`Creating file from zip ${entryPath}`, lTags())
const writeStream = createWriteStream(entryPath)
writeStream.on('close', () => zipFile.readEntry())
pipeline(readStream, writeStream, pipelineErr => {
if (pipelineErr) return rej(pipelineErr)
})
})
})
zipFile.on('end', () => res())
})
})
}

View File

@ -153,6 +153,11 @@ const CONFIG = {
BUCKET_NAME: config.get<string>('object_storage.streaming_playlists.bucket_name'),
PREFIX: config.get<string>('object_storage.streaming_playlists.prefix'),
BASE_URL: config.get<string>('object_storage.streaming_playlists.base_url')
},
USER_EXPORTS: {
BUCKET_NAME: config.get<string>('object_storage.user_exports.bucket_name'),
PREFIX: config.get<string>('object_storage.user_exports.prefix'),
BASE_URL: config.get<string>('object_storage.user_exports.base_url')
}
},
WEBSERVER: {
@ -511,6 +516,16 @@ const CONFIG = {
get FULL_SYNC_VIDEOS_LIMIT () {
return config.get<number>('import.video_channel_synchronization.full_sync_videos_limit')
}
},
USERS: {
get ENABLED () { return config.get<boolean>('import.users.enabled') }
}
},
EXPORT: {
USERS: {
get ENABLED () { return config.get<boolean>('export.users.enabled') },
get MAX_USER_VIDEO_QUOTA () { return parseBytes(config.get<string>('export.users.max_user_video_quota')) },
get EXPORT_EXPIRATION () { return parseDurationToMs(config.get<string>('export.users.export_expiration')) }
}
},
AUTO_BLACKLIST: {

View File

@ -10,6 +10,10 @@ import {
NSFWPolicyType,
RunnerJobState,
RunnerJobStateType,
UserExportState,
UserExportStateType,
UserImportState,
UserImportStateType,
UserRegistrationState,
UserRegistrationStateType,
VideoChannelSyncState,
@ -41,7 +45,7 @@ import { cpus } from 'os'
// ---------------------------------------------------------------------------
const LAST_MIGRATION_VERSION = 805
const LAST_MIGRATION_VERSION = 815
// ---------------------------------------------------------------------------
@ -191,7 +195,9 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
'transcoding-job-builder': 1,
'generate-video-storyboard': 1,
'notify': 1,
'federate-video': 1
'federate-video': 1,
'create-user-export': 1,
'import-user-archive': 1
}
// Excluded keys are jobs that can be configured by admins
const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-import'>]: number } = {
@ -217,7 +223,9 @@ const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-im
'transcoding-job-builder': 1,
'generate-video-storyboard': 1,
'notify': 5,
'federate-video': 3
'federate-video': 3,
'create-user-export': 1,
'import-user-archive': 1
}
const JOB_TTL: { [id in JobType]: number } = {
'activitypub-http-broadcast': 60000 * 10, // 10 minutes
@ -244,7 +252,9 @@ const JOB_TTL: { [id in JobType]: number } = {
'after-video-channel-import': 60000 * 5, // 5 minutes
'transcoding-job-builder': 60000, // 1 minute
'notify': 60000 * 5, // 5 minutes
'federate-video': 60000 * 5 // 5 minutes
'federate-video': 60000 * 5, // 5 minutes,
'create-user-export': 60000 * 60 * 24, // 24 hours
'import-user-archive': 60000 * 60 * 24 // 24 hours
}
const REPEAT_JOBS: { [ id in JobType ]?: RepeatOptions } = {
'videos-views-stats': {
@ -313,6 +323,7 @@ const SCHEDULER_INTERVALS_MS = {
AUTO_FOLLOW_INDEX_INSTANCES: 60000 * 60 * 24, // 1 day
REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day
REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day
REMOVE_EXPIRED_USER_EXPORTS: 1000 * 3600, // 1 hour
UPDATE_INBOX_STATS: 1000 * 60, // 1 minute
REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60, // 1 hour
CHANNEL_SYNC_CHECK_INTERVAL: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.CHECK_INTERVAL
@ -503,6 +514,10 @@ const VIDEO_RATE_TYPES: { [ id: string ]: VideoRateType } = {
DISLIKE: 'dislike'
}
const USER_IMPORT = {
MAX_PLAYLIST_ELEMENTS: 1000
}
const FFMPEG_NICE = {
// parent process defaults to niceness = 0
// reminder: lower = higher priority, max value is 19, lowest is -20
@ -618,6 +633,20 @@ const RUNNER_JOB_STATES: { [ id in RunnerJobStateType ]: string } = {
[RunnerJobState.PARENT_CANCELLED]: 'Parent job cancelled'
}
const USER_EXPORT_STATES: { [ id in UserExportStateType ]: string } = {
[UserExportState.PENDING]: 'Pending',
[UserExportState.PROCESSING]: 'Processing',
[UserExportState.COMPLETED]: 'Completed',
[UserExportState.ERRORED]: 'Failed'
}
const USER_IMPORT_STATES: { [ id in UserImportStateType ]: string } = {
[UserImportState.PENDING]: 'Pending',
[UserImportState.PROCESSING]: 'Processing',
[UserImportState.COMPLETED]: 'Completed',
[UserImportState.ERRORED]: 'Failed'
}
const MIMETYPES = {
AUDIO: {
MIMETYPE_EXT: {
@ -773,6 +802,7 @@ const USER_PASSWORD_RESET_LIFETIME = 60000 * 60 // 60 minutes
const USER_PASSWORD_CREATE_LIFETIME = 60000 * 60 * 24 * 7 // 7 days
const TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME = 60000 * 10 // 10 minutes
let JWT_TOKEN_USER_EXPORT_FILE_LIFETIME = '15 minutes'
const EMAIL_VERIFY_LIFETIME = 60000 * 60 // 60 minutes
@ -807,7 +837,8 @@ const STATIC_PATHS = {
const STATIC_DOWNLOAD_PATHS = {
TORRENTS: '/download/torrents/',
VIDEOS: '/download/videos/',
HLS_VIDEOS: '/download/streaming-playlists/hls/videos/'
HLS_VIDEOS: '/download/streaming-playlists/hls/videos/',
USER_EXPORT: '/download/user-export/'
}
const LAZY_STATIC_PATHS = {
THUMBNAILS: '/lazy-static/thumbnails/',
@ -1125,6 +1156,8 @@ if (process.env.PRODUCTION_CONSTANTS !== 'true') {
VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION = 1
RUNNER_JOBS.LAST_CONTACT_UPDATE_INTERVAL = 2000
JWT_TOKEN_USER_EXPORT_FILE_LIFETIME = '2 seconds'
}
}
@ -1168,6 +1201,8 @@ export {
DIRECTORIES,
RESUMABLE_UPLOAD_SESSION_LIFETIME,
RUNNER_JOB_STATES,
USER_EXPORT_STATES,
USER_IMPORT_STATES,
P2P_MEDIA_LOADER_PEER_VERSION,
STORYBOARD,
ACTOR_IMAGES_SIZE,
@ -1187,6 +1222,7 @@ export {
STATS_TIMESERIE,
BROADCAST_CONCURRENCY,
AUDIT_LOG_FILENAME,
USER_IMPORT,
PAGINATION,
ACTOR_FOLLOW_SCORE,
PREVIEWS_SIZE,
@ -1195,6 +1231,7 @@ export {
DEFAULT_USER_THEME_NAME,
SERVER_ACTOR_NAME,
TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME,
JWT_TOKEN_USER_EXPORT_FILE_LIFETIME,
PLUGIN_GLOBAL_CSS_FILE_NAME,
PLUGIN_GLOBAL_CSS_PATH,
PRIVATE_RSA_KEY_SIZE,

View File

@ -60,6 +60,8 @@ import { VideoModel } from '../models/video/video.js'
import { VideoViewModel } from '../models/view/video-view.js'
import { CONFIG } from './config.js'
import { VideoChapterModel } from '@server/models/video/video-chapter.js'
import { UserExportModel } from '@server/models/user/user-export.js'
import { UserImportModel } from '@server/models/user/user-import.js'
pg.defaults.parseInt8 = true // Avoid BIGINT to be converted to string
@ -165,6 +167,7 @@ async function initDatabaseModels (silent: boolean) {
VideoTrackerModel,
PluginModel,
ActorCustomPageModel,
UserImportModel,
VideoJobInfoModel,
VideoChannelSyncModel,
UserRegistrationModel,
@ -172,7 +175,8 @@ async function initDatabaseModels (silent: boolean) {
RunnerRegistrationTokenModel,
RunnerModel,
RunnerJobModel,
StoryboardModel
StoryboardModel,
UserExportModel
])
// Check extensions exist in the database

View File

@ -1,5 +1,5 @@
import * as Sequelize from 'sequelize'
import { VideoStorage } from '@peertube/peertube-models'
import { FileStorage } from '@peertube/peertube-models'
async function up (utils: {
transaction: Sequelize.Transaction
@ -27,7 +27,7 @@ async function up (utils: {
await utils.queryInterface.addColumn('videoFile', 'storage', {
type: Sequelize.INTEGER,
allowNull: true,
defaultValue: VideoStorage.FILE_SYSTEM
defaultValue: FileStorage.FILE_SYSTEM
})
await utils.queryInterface.changeColumn('videoFile', 'storage', { type: Sequelize.INTEGER, allowNull: false, defaultValue: null })
}
@ -36,7 +36,7 @@ async function up (utils: {
await utils.queryInterface.addColumn('videoStreamingPlaylist', 'storage', {
type: Sequelize.INTEGER,
allowNull: true,
defaultValue: VideoStorage.FILE_SYSTEM
defaultValue: FileStorage.FILE_SYSTEM
})
await utils.queryInterface.changeColumn('videoStreamingPlaylist', 'storage', {
type: Sequelize.INTEGER,

View File

@ -0,0 +1,33 @@
import * as Sequelize from 'sequelize'
async function up (utils: {
transaction: Sequelize.Transaction
queryInterface: Sequelize.QueryInterface
sequelize: Sequelize.Sequelize
}): Promise<void> {
const query = `
CREATE TABLE IF NOT EXISTS "userExport" (
"id" SERIAL,
"filename" VARCHAR(255),
"withVideoFiles" BOOLEAN NOT NULL,
"state" INTEGER NOT NULL,
"error" TEXT,
"size" INTEGER,
"storage" INTEGER NOT NULL,
"userId" INTEGER NOT NULL REFERENCES "user" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
"createdAt" TIMESTAMP WITH TIME ZONE NOT NULL,
"updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL,
PRIMARY KEY ("id")
);`
await utils.sequelize.query(query, { transaction: utils.transaction })
}
function down (options) {
throw new Error('Not implemented.')
}
export {
up,
down
}

View File

@ -0,0 +1,31 @@
import * as Sequelize from 'sequelize'
async function up (utils: {
transaction: Sequelize.Transaction
queryInterface: Sequelize.QueryInterface
sequelize: Sequelize.Sequelize
}): Promise<void> {
const query = `
CREATE TABLE IF NOT EXISTS "userImport" (
"id" SERIAL,
"filename" VARCHAR(255),
"state" INTEGER NOT NULL,
"error" TEXT,
"resultSummary" JSONB,
"userId" INTEGER NOT NULL REFERENCES "user" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
"createdAt" TIMESTAMP WITH TIME ZONE NOT NULL,
"updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL,
PRIMARY KEY ("id")
);;`
await utils.sequelize.query(query, { transaction: utils.transaction })
}
function down (options) {
throw new Error('Not implemented.')
}
export {
up,
down
}

View File

@ -7,7 +7,7 @@ import { forceNumber } from '@peertube/peertube-core-utils'
type ActivityPubCollectionPaginationHandler = (start: number, count: number) => Bluebird<ResultList<any>> | Promise<ResultList<any>>
async function activityPubCollectionPagination (
export async function activityPubCollectionPagination (
baseUrl: string,
handler: ActivityPubCollectionPaginationHandler,
page?: any,
@ -56,8 +56,11 @@ async function activityPubCollectionPagination (
}
}
// ---------------------------------------------------------------------------
export {
activityPubCollectionPagination
export function activityPubCollection <T> (baseUrl: string, items: T[]) {
return {
id: baseUrl,
type: 'OrderedCollection' as 'OrderedCollection',
totalItems: items.length,
orderedItems: items
}
}

View File

@ -51,7 +51,7 @@ async function processDeleteActivity (options: APProcessorOptions<ActivityDelete
}
{
const videoInstance = await VideoModel.loadByUrlAndPopulateAccount(objectUrl)
const videoInstance = await VideoModel.loadByUrlAndPopulateAccountAndFiles(objectUrl)
if (videoInstance) {
if (videoInstance.isOwned()) throw new Error(`Remote instance cannot delete owned video ${videoInstance.url}.`)

View File

@ -47,7 +47,7 @@ async function processCreateAbuse (flag: ActivityFlag, byActor: MActorSignature)
logger.debug('Reporting remote abuse for object %s.', uri)
await sequelizeTypescript.transaction(async t => {
const video = await VideoModel.loadByUrlAndPopulateAccount(uri, t)
const video = await VideoModel.loadByUrlAndPopulateAccountAndFiles(uri, t)
let videoComment: MCommentOwnerVideo
let flaggedAccount: MAccountDefault

View File

@ -18,7 +18,7 @@ async function refreshVideoIfNeeded (options: {
// We need more attributes if the argument video was fetched with not enough joints
const video = options.fetchedType === 'all'
? options.video as MVideoAccountLightBlacklistAllFiles
: await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
: await VideoModel.loadByUrlAndPopulateAccountAndFiles(options.video.url)
const lTags = loggerTagsFactory('ap', 'video', 'refresh', video.uuid, video.url)

View File

@ -3,23 +3,51 @@ import { getServerActor } from '@server/models/application/application.js'
import { MAccountBlocklist, MAccountId, MAccountHost, MServerBlocklist } from '@server/types/models/index.js'
import { AccountBlocklistModel } from '../models/account/account-blocklist.js'
import { ServerBlocklistModel } from '../models/server/server-blocklist.js'
import { UserNotificationModel } from '@server/models/user/user-notification.js'
import { logger } from '@server/helpers/logger.js'
function addAccountInBlocklist (byAccountId: number, targetAccountId: number) {
return sequelizeTypescript.transaction(async t => {
async function addAccountInBlocklist (options: {
byAccountId: number
targetAccountId: number
removeNotificationOfUserId: number | null // If blocked by a user
}) {
const { byAccountId, targetAccountId, removeNotificationOfUserId } = options
await sequelizeTypescript.transaction(async t => {
return AccountBlocklistModel.upsert({
accountId: byAccountId,
targetAccountId
}, { transaction: t })
})
UserNotificationModel.removeNotificationsOf({
id: targetAccountId,
type: 'account',
forUserId: removeNotificationOfUserId
}).catch(err => logger.error('Cannot remove notifications after an account mute.', { err }))
}
function addServerInBlocklist (byAccountId: number, targetServerId: number) {
return sequelizeTypescript.transaction(async t => {
async function addServerInBlocklist (options: {
byAccountId: number
targetServerId: number
removeNotificationOfUserId: number | null
}) {
const { byAccountId, targetServerId, removeNotificationOfUserId } = options
await sequelizeTypescript.transaction(async t => {
return ServerBlocklistModel.upsert({
accountId: byAccountId,
targetServerId
}, { transaction: t })
})
UserNotificationModel.removeNotificationsOf({
id: targetServerId,
type: 'server',
forUserId: removeNotificationOfUserId
}).catch(err => logger.error('Cannot remove notifications after a server mute.', { err }))
}
function removeAccountFromBlocklist (accountBlock: MAccountBlocklist) {

View File

@ -1,5 +1,5 @@
import { arrayify } from '@peertube/peertube-core-utils'
import { EmailPayload, SendEmailDefaultOptions, UserRegistrationState } from '@peertube/peertube-models'
import { EmailPayload, SendEmailDefaultOptions, UserExportState, UserRegistrationState } from '@peertube/peertube-models'
import { isTestOrDevInstance, root } from '@peertube/peertube-node-utils'
import { readFileSync } from 'fs'
import merge from 'lodash-es/merge.js'
@ -8,8 +8,9 @@ import { join } from 'path'
import { bunyanLogger, logger } from '../helpers/logger.js'
import { CONFIG, isEmailEnabled } from '../initializers/config.js'
import { WEBSERVER } from '../initializers/constants.js'
import { MRegistration, MUser } from '../types/models/index.js'
import { MRegistration, MUser, MUserExport, MUserImport } from '../types/models/index.js'
import { JobQueue } from './job-queue/index.js'
import { UserModel } from '@server/models/user/user.js'
class Emailer {
@ -52,6 +53,8 @@ class Emailer {
}
}
// ---------------------------------------------------------------------------
addPasswordResetEmailJob (username: string, to: string, resetPasswordUrl: string) {
const emailPayload: EmailPayload = {
template: 'password-reset',
@ -160,13 +163,82 @@ class Emailer {
locals: {
username: registration.username,
moderationResponse: registration.moderationResponse,
loginLink: WEBSERVER.URL + '/login'
loginLink: WEBSERVER.URL + '/login',
hideNotificationPreferencesLink: true
}
}
return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload })
}
// ---------------------------------------------------------------------------
async addUserExportCompletedOrErroredJob (userExport: MUserExport) {
let template: string
let subject: string
if (userExport.state === UserExportState.COMPLETED) {
template = 'user-export-completed'
subject = `Your export archive has been created`
} else {
template = 'user-export-errored'
subject = `Failed to create your export archive`
}
const user = await UserModel.loadById(userExport.userId)
const emailPayload: EmailPayload = {
to: [ user.email ],
template,
subject,
locals: {
exportsUrl: WEBSERVER.URL + '/my-account/import-export',
errorMessage: userExport.error,
hideNotificationPreferencesLink: true
}
}
return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload })
}
async addUserImportErroredJob (userImport: MUserImport) {
const user = await UserModel.loadById(userImport.userId)
const emailPayload: EmailPayload = {
to: [ user.email ],
template: 'user-import-errored',
subject: 'Failed to import your archive',
locals: {
errorMessage: userImport.error,
hideNotificationPreferencesLink: true
}
}
return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload })
}
async addUserImportSuccessJob (userImport: MUserImport) {
const user = await UserModel.loadById(userImport.userId)
const emailPayload: EmailPayload = {
to: [ user.email ],
template: 'user-import-completed',
subject: 'Your archive import has finished',
locals: {
resultStats: userImport.resultSummary.stats,
hideNotificationPreferencesLink: true
}
}
return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload })
}
// ---------------------------------------------------------------------------
async sendMail (options: EmailPayload) {
if (!isEmailEnabled()) {
logger.info('Cannot send mail because SMTP is not configured.')
@ -233,14 +305,14 @@ class Emailer {
private initSMTPTransport () {
logger.info('Using %s:%s as SMTP server.', CONFIG.SMTP.HOSTNAME, CONFIG.SMTP.PORT)
let tls
let tls: { ca: [ Buffer ] }
if (CONFIG.SMTP.CA_FILE) {
tls = {
ca: [ readFileSync(CONFIG.SMTP.CA_FILE) ]
}
}
let auth
let auth: { user: string, pass: string }
if (CONFIG.SMTP.USERNAME && CONFIG.SMTP.PASSWORD) {
auth = {
user: CONFIG.SMTP.USERNAME,

View File

@ -0,0 +1,9 @@
extends ../common/greetings
include ../common/mixins.pug
block title
| Your export archive has been created
block content
p
| Your export archive has been created. You can download it in #[a(href=exportsUrl) your account export page].

View File

@ -0,0 +1,12 @@
extends ../common/greetings
include ../common/mixins.pug
block title
| Failed to create your export archive
block content
p
| We are sorry but the generation of your export archive has failed:
blockquote !{errorMessage}
p
| Please contact your administrator if the problem occurs again.

View File

@ -0,0 +1,46 @@
extends ../common/greetings
include ../common/mixins.pug
mixin displaySummary(stats)
ul
if stats.success
li Imported: #{stats.success}
if stats.duplicates
li Not imported as considered duplicate: #{stats.duplicates}
if stats.errors
li Not imported due to error: #{stats.errors}
block title
| Your archive import has finished
block content
p Your archive import has finished. Here is the summary of imported objects:
ul
li
strong User settings:
+displaySummary(resultStats.userSettings)
li
strong Account (name, description, avatar...):
+displaySummary(resultStats.account)
li
strong Blocklist:
+displaySummary(resultStats.blocklist)
li
strong Channels:
+displaySummary(resultStats.channels)
li
strong Likes:
+displaySummary(resultStats.likes)
li
strong Dislikes:
+displaySummary(resultStats.dislikes)
li
strong Subscriptions:
+displaySummary(resultStats.following)
li
strong Video Playlists:
+displaySummary(resultStats.videoPlaylists)
li
strong Videos:
+displaySummary(resultStats.videos)

View File

@ -0,0 +1,12 @@
extends ../common/greetings
include ../common/mixins.pug
block title
| Failed to import your archive
block content
p
| We are sorry but the import of your archive has failed:
blockquote !{errorMessage}
p
| Please contact your administrator if the problem occurs again.

View File

@ -1,7 +1,6 @@
import { join } from 'path'
import { logger } from '@server/helpers/logger.js'
import { doRequestAndSaveToFile } from '@server/helpers/requests.js'
import { CONFIG } from '../../initializers/config.js'
import { FILES_CACHE } from '../../initializers/constants.js'
import { VideoModel } from '../../models/video/video.js'
import { VideoCaptionModel } from '../../models/video/video-caption.js'
@ -24,7 +23,7 @@ class VideoCaptionsSimpleFileCache extends AbstractSimpleFileCache <string> {
if (!videoCaption) return undefined
if (videoCaption.isOwned()) {
return { isOwned: true, path: join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.filename) }
return { isOwned: true, path: videoCaption.getFSPath() }
}
return this.loadRemoteFile(filename)

View File

@ -1,6 +1,6 @@
import { uniqify, uuidRegex } from '@peertube/peertube-core-utils'
import { getVideoStreamDimensionsInfo } from '@peertube/peertube-ffmpeg'
import { VideoStorage } from '@peertube/peertube-models'
import { FileStorage } from '@peertube/peertube-models'
import { sha256 } from '@peertube/peertube-node-utils'
import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models/index.js'
import { ensureDir, move, outputJSON, remove } from 'fs-extra/esm'
@ -100,7 +100,7 @@ function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist
logger.info('Updating %s master playlist file of video %s', masterPlaylistPath, video.uuid, lTags(video.uuid))
if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
if (playlist.storage === FileStorage.OBJECT_STORAGE) {
playlist.playlistUrl = await storeHLSFileFromFilename(playlist, playlist.playlistFilename)
await remove(masterPlaylistPath)
}
@ -151,7 +151,7 @@ function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist
const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
await outputJSON(outputPath, json)
if (playlist.storage === VideoStorage.OBJECT_STORAGE) {
if (playlist.storage === FileStorage.OBJECT_STORAGE) {
playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlist, playlist.segmentsSha256Filename)
await remove(outputPath)
}

View File

@ -18,6 +18,10 @@ async function processActivityPubFollow (job: Job) {
const payload = job.data as ActivitypubFollowPayload
const host = payload.host
const handle = host
? `${payload.name}@${host}`
: payload.name
logger.info('Processing ActivityPub follow in job %s.', job.id)
let targetActor: MActorFull
@ -30,14 +34,24 @@ async function processActivityPubFollow (job: Job) {
let actorUrl: string
if (!payload.name) actorUrl = await getApplicationActorOfHost(sanitizedHost)
if (!actorUrl) actorUrl = await loadActorUrlOrGetFromWebfinger((payload.name || SERVER_ACTOR_NAME) + '@' + sanitizedHost)
try {
if (!payload.name) actorUrl = await getApplicationActorOfHost(sanitizedHost)
if (!actorUrl) actorUrl = await loadActorUrlOrGetFromWebfinger((payload.name || SERVER_ACTOR_NAME) + '@' + sanitizedHost)
targetActor = await getOrCreateAPActor(actorUrl, 'all')
targetActor = await getOrCreateAPActor(actorUrl, 'all')
} catch (err) {
logger.warn(`Do not follow ${handle} because we could not find the actor URL (in database or using webfinger)`)
return
}
}
if (!targetActor) {
logger.warn(`Do not follow ${handle} because we could not fetch/load the actor`)
return
}
if (payload.assertIsChannel && !targetActor.VideoChannel) {
logger.warn('Do not follow %s@%s because it is not a channel.', payload.name, host)
logger.warn(`Do not follow ${handle} because it is not a channel.`)
return
}

View File

@ -0,0 +1,34 @@
import { Job } from 'bullmq'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { CreateUserExportPayload } from '@peertube/peertube-models'
import { UserExportModel } from '@server/models/user/user-export.js'
import { UserExporter } from '@server/lib/user-import-export/user-exporter.js'
import { Emailer } from '@server/lib/emailer.js'
const lTags = loggerTagsFactory('user-export')
export async function processCreateUserExport (job: Job): Promise<void> {
const payload = job.data as CreateUserExportPayload
const exportModel = await UserExportModel.load(payload.userExportId)
logger.info('Processing create user export %s in job %s.', payload.userExportId, job.id, lTags())
if (!exportModel) {
logger.info(`User export ${payload.userExportId} does not exist anymore, do not create user export.`, lTags())
return
}
const exporter = new UserExporter()
try {
await exporter.export(exportModel)
await Emailer.Instance.addUserExportCompletedOrErroredJob(exportModel)
logger.info(`User export ${payload.userExportId} has been created`, lTags())
} catch (err) {
await Emailer.Instance.addUserExportCompletedOrErroredJob(exportModel)
throw err
}
}

View File

@ -0,0 +1,33 @@
import { Job } from 'bullmq'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { ImportUserArchivePayload } from '@peertube/peertube-models'
import { UserImportModel } from '@server/models/user/user-import.js'
import { UserImporter } from '@server/lib/user-import-export/user-importer.js'
import { Emailer } from '@server/lib/emailer.js'
const lTags = loggerTagsFactory('user-import')
export async function processImportUserArchive (job: Job): Promise<void> {
const payload = job.data as ImportUserArchivePayload
const importModel = await UserImportModel.load(payload.userImportId)
logger.info(`Processing importing user archive ${payload.userImportId} in job ${job.id}`, lTags())
if (!importModel) {
logger.info(`User import ${payload.userImportId} does not exist anymore, do not create import data.`, lTags())
return
}
const exporter = new UserImporter()
await exporter.import(importModel)
try {
await Emailer.Instance.addUserImportSuccessJob(importModel)
logger.info(`User import ${payload.userImportId} ended`, lTags())
} catch (err) {
await Emailer.Instance.addUserImportErroredJob(importModel)
throw err
}
}

View File

@ -1,6 +1,6 @@
import { Job } from 'bullmq'
import { join } from 'path'
import { MoveStoragePayload, VideoStateType, VideoStorage } from '@peertube/peertube-models'
import { MoveStoragePayload, VideoStateType, FileStorage } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { updateTorrentMetadata } from '@server/helpers/webtorrent.js'
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
@ -52,7 +52,7 @@ export async function onMoveToFileSystemFailure (job: Job, err: any) {
async function moveWebVideoFiles (video: MVideoWithAllFiles) {
for (const file of video.VideoFiles) {
if (file.storage === VideoStorage.FILE_SYSTEM) continue
if (file.storage === FileStorage.FILE_SYSTEM) continue
await makeWebVideoFileAvailable(file.filename, VideoPathManager.Instance.getFSVideoFileOutputPath(video, file))
await onFileMoved({
@ -68,7 +68,7 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
const playlistWithVideo = playlist.withVideo(video)
for (const file of playlist.VideoFiles) {
if (file.storage === VideoStorage.FILE_SYSTEM) continue
if (file.storage === FileStorage.FILE_SYSTEM) continue
// Resolution playlist
const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
@ -97,7 +97,7 @@ async function onFileMoved (options: {
const oldFileUrl = file.fileUrl
file.fileUrl = null
file.storage = VideoStorage.FILE_SYSTEM
file.storage = FileStorage.FILE_SYSTEM
await updateTorrentMetadata(videoOrPlaylist, file)
await file.save()
@ -114,7 +114,7 @@ async function doAfterLastMove (options: {
const { video, previousVideoState, isNewVideo } = options
for (const playlist of video.VideoStreamingPlaylists) {
if (playlist.storage === VideoStorage.FILE_SYSTEM) continue
if (playlist.storage === FileStorage.FILE_SYSTEM) continue
const playlistWithVideo = playlist.withVideo(video)
@ -124,7 +124,7 @@ async function doAfterLastMove (options: {
playlist.playlistUrl = null
playlist.segmentsSha256Url = null
playlist.storage = VideoStorage.FILE_SYSTEM
playlist.storage = FileStorage.FILE_SYSTEM
playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION

View File

@ -1,7 +1,7 @@
import { Job } from 'bullmq'
import { remove } from 'fs-extra/esm'
import { join } from 'path'
import { MoveStoragePayload, VideoStateType, VideoStorage } from '@peertube/peertube-models'
import { MoveStoragePayload, VideoStateType, FileStorage } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { updateTorrentMetadata } from '@server/helpers/webtorrent.js'
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
@ -45,7 +45,7 @@ export async function onMoveToObjectStorageFailure (job: Job, err: any) {
async function moveWebVideoFiles (video: MVideoWithAllFiles) {
for (const file of video.VideoFiles) {
if (file.storage !== VideoStorage.FILE_SYSTEM) continue
if (file.storage !== FileStorage.FILE_SYSTEM) continue
const fileUrl = await storeWebVideoFile(video, file)
@ -59,7 +59,7 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
const playlistWithVideo = playlist.withVideo(video)
for (const file of playlist.VideoFiles) {
if (file.storage !== VideoStorage.FILE_SYSTEM) continue
if (file.storage !== FileStorage.FILE_SYSTEM) continue
// Resolution playlist
const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
@ -84,7 +84,7 @@ async function onFileMoved (options: {
const { videoOrPlaylist, file, fileUrl, oldPath } = options
file.fileUrl = fileUrl
file.storage = VideoStorage.OBJECT_STORAGE
file.storage = FileStorage.OBJECT_STORAGE
await updateTorrentMetadata(videoOrPlaylist, file)
await file.save()
@ -101,13 +101,13 @@ async function doAfterLastMove (options: {
const { video, previousVideoState, isNewVideo } = options
for (const playlist of video.VideoStreamingPlaylists) {
if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue
if (playlist.storage === FileStorage.OBJECT_STORAGE) continue
const playlistWithVideo = playlist.withVideo(video)
playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename)
playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename)
playlist.storage = VideoStorage.OBJECT_STORAGE
playlist.storage = FileStorage.OBJECT_STORAGE
playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION

View File

@ -1,13 +1,12 @@
import { Job } from 'bullmq'
import { copy } from 'fs-extra/esm'
import { stat } from 'fs/promises'
import { VideoFileImportPayload, VideoStorage } from '@peertube/peertube-models'
import { VideoFileImportPayload, FileStorage } from '@peertube/peertube-models'
import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent.js'
import { CONFIG } from '@server/initializers/config.js'
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
import { generateWebVideoFilename } from '@server/lib/paths.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { buildMoveJob } from '@server/lib/video.js'
import { VideoFileModel } from '@server/models/video/video-file.js'
import { VideoModel } from '@server/models/video/video.js'
import { MVideoFullLight } from '@server/types/models/index.js'
@ -15,6 +14,7 @@ import { getLowercaseExtension } from '@peertube/peertube-node-utils'
import { getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@peertube/peertube-ffmpeg'
import { logger } from '../../../helpers/logger.js'
import { JobQueue } from '../job-queue.js'
import { buildMoveJob } from '@server/lib/video-jobs.js'
async function processVideoFileImport (job: Job) {
const payload = job.data as VideoFileImportPayload
@ -68,7 +68,7 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) {
resolution,
extname: fileExt,
filename: generateWebVideoFilename(resolution, fileExt),
storage: VideoStorage.FILE_SYSTEM,
storage: FileStorage.FILE_SYSTEM,
size,
fps,
videoId: video.id

View File

@ -22,10 +22,10 @@ import { generateWebVideoFilename } from '@server/lib/paths.js'
import { Hooks } from '@server/lib/plugins/hooks.js'
import { ServerConfigManager } from '@server/lib/server-config-manager.js'
import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job.js'
import { isAbleToUploadVideo } from '@server/lib/user.js'
import { isUserQuotaValid } from '@server/lib/user.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { buildNextVideoState } from '@server/lib/video-state.js'
import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video.js'
import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models/index.js'
import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import.js'
import { getLowercaseExtension } from '@peertube/peertube-node-utils'
@ -138,7 +138,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
// Get information about this video
const stats = await stat(tempVideoPath)
const isAble = await isAbleToUploadVideo(videoImport.User.id, stats.size)
const isAble = await isUserQuotaValid({ userId: videoImport.User.id, uploadSize: stats.size })
if (isAble === false) {
throw new Error('The user video quota is exceeded with this video to import.')
}

View File

@ -30,7 +30,7 @@ import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoS
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
import { JobQueue } from '../job-queue.js'
import { isVideoInPublicDirectory } from '@server/lib/video-privacy.js'
import { buildStoryboardJobIfNeeded } from '@server/lib/video.js'
import { buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
const lTags = loggerTagsFactory('live', 'job')

View File

@ -4,7 +4,7 @@ import { join } from 'path'
import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js'
import { CONFIG } from '@server/initializers/config.js'
import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles.js'
import { isAbleToUploadVideo } from '@server/lib/user.js'
import { isUserQuotaValid } from '@server/lib/user.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { approximateIntroOutroAdditionalSize, onVideoStudioEnded, safeCleanupStudioTMPFiles } from '@server/lib/video-studio.js'
import { UserModel } from '@server/models/user/user.js'
@ -170,7 +170,7 @@ async function checkUserQuotaOrThrow (video: MVideoFullLight, payload: VideoStud
const filePathFinder = (i: number) => (payload.tasks[i] as VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload).options.file
const additionalBytes = await approximateIntroOutroAdditionalSize(video, payload.tasks, filePathFinder)
if (await isAbleToUploadVideo(user.id, additionalBytes) === false) {
if (await isUserQuotaValid({ userId: user.id, uploadSize: additionalBytes }) === false) {
throw new Error('Quota exceeded for this user to edit the video')
}
}

View File

@ -18,10 +18,12 @@ import {
ActivitypubHttpUnicastPayload,
ActorKeysPayload,
AfterVideoChannelImportPayload,
CreateUserExportPayload,
DeleteResumableUploadMetaFilePayload,
EmailPayload,
FederateVideoPayload,
GenerateStoryboardPayload,
ImportUserArchivePayload,
JobState,
JobType,
ManageVideoTorrentPayload,
@ -71,6 +73,8 @@ import { processVideoStudioEdition } from './handlers/video-studio-edition.js'
import { processVideoTranscoding } from './handlers/video-transcoding.js'
import { processVideosViewsStats } from './handlers/video-views-stats.js'
import { onMoveToFileSystemFailure, processMoveToFileSystem } from './handlers/move-to-file-system.js'
import { processCreateUserExport } from './handlers/create-user-export.js'
import { processImportUserArchive } from './handlers/import-user-archive.js'
export type CreateJobArgument =
{ type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@ -98,7 +102,9 @@ export type CreateJobArgument =
{ type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } |
{ type: 'notify', payload: NotifyPayload } |
{ type: 'federate-video', payload: FederateVideoPayload } |
{ type: 'generate-video-storyboard', payload: GenerateStoryboardPayload }
{ type: 'create-user-export', payload: CreateUserExportPayload } |
{ type: 'generate-video-storyboard', payload: GenerateStoryboardPayload } |
{ type: 'import-user-archive', payload: ImportUserArchivePayload }
export type CreateJobOptions = {
delay?: number
@ -131,7 +137,9 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
'video-studio-edition': processVideoStudioEdition,
'video-transcoding': processVideoTranscoding,
'videos-views-stats': processVideosViewsStats,
'generate-video-storyboard': processGenerateStoryboard
'generate-video-storyboard': processGenerateStoryboard,
'create-user-export': processCreateUserExport,
'import-user-archive': processImportUserArchive
}
const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = {
@ -164,7 +172,9 @@ const jobTypes: JobType[] = [
'video-redundancy',
'video-studio-edition',
'video-transcoding',
'videos-views-stats'
'videos-views-stats',
'create-user-export',
'import-user-archive'
]
const silentFailure = new Set<JobType>([ 'activitypub-http-unicast' ])

View File

@ -1,7 +1,7 @@
import { pathExists, remove } from 'fs-extra/esm'
import { readdir } from 'fs/promises'
import { basename, join } from 'path'
import { LiveVideoLatencyMode, LiveVideoLatencyModeType, VideoStorage } from '@peertube/peertube-models'
import { LiveVideoLatencyMode, LiveVideoLatencyModeType, FileStorage } from '@peertube/peertube-models'
import { logger } from '@server/helpers/logger.js'
import { VIDEO_LIVE } from '@server/initializers/constants.js'
import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models/index.js'
@ -24,7 +24,7 @@ async function cleanupUnsavedNormalLive (video: MVideo, streamingPlaylist: MStre
const hlsDirectory = getLiveDirectory(video)
// We uploaded files to object storage too, remove them
if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
if (streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) {
await removeHLSObjectStorage(streamingPlaylist.withVideo(video))
}
@ -86,7 +86,7 @@ async function cleanupTMPLiveFilesFromFilesystem (video: MVideo) {
}
async function cleanupTMPLiveFilesFromObjectStorage (streamingPlaylist: MStreamingPlaylistVideo) {
if (streamingPlaylist.storage !== VideoStorage.OBJECT_STORAGE) return
if (streamingPlaylist.storage !== FileStorage.OBJECT_STORAGE) return
logger.info('Cleanup TMP live files from object storage for %s.', streamingPlaylist.Video.uuid)

View File

@ -14,14 +14,14 @@ import { removeHLSFileObjectStorageByPath, storeHLSFileFromContent, storeHLSFile
import { VideoFileModel } from '@server/models/video/video-file.js'
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models/index.js'
import { LiveVideoError, VideoStorage, VideoStreamingPlaylistType } from '@peertube/peertube-models'
import { LiveVideoError, FileStorage, VideoStreamingPlaylistType } from '@peertube/peertube-models'
import {
generateHLSMasterPlaylistFilename,
generateHlsSha256SegmentsFilename,
getLiveDirectory,
getLiveReplayBaseDirectory
} from '../../paths.js'
import { isAbleToUploadVideo } from '../../user.js'
import { isUserQuotaValid } from '../../user.js'
import { LiveQuotaStore } from '../live-quota-store.js'
import { LiveSegmentShaStore } from '../live-segment-sha-store.js'
import { buildConcatenatedName, getLiveSegmentTime } from '../live-utils.js'
@ -95,7 +95,7 @@ class MuxingSession extends EventEmitter {
private aborted = false
private readonly isAbleToUploadVideoWithCache = memoizee((userId: number) => {
return isAbleToUploadVideo(userId, 1000)
return isUserQuotaValid({ userId, uploadSize: 1000 })
}, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD })
private readonly hasClientSocketInBadHealthWithCache = memoizee((sessionId: string) => {
@ -186,7 +186,7 @@ class MuxingSession extends EventEmitter {
if (this.masterPlaylistCreated === true) return
try {
if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
if (this.streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) {
let masterContent = await readFile(path, 'utf-8')
// If the disk sync is slow, don't upload an empty master playlist on object storage
@ -260,7 +260,7 @@ class MuxingSession extends EventEmitter {
logger.warn('Cannot remove segment sha %s from sha store', segmentPath, { err, ...this.lTags() })
}
if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
if (this.streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) {
try {
await removeHLSFileObjectStorageByPath(this.streamingPlaylist, segmentPath)
} catch (err) {
@ -345,7 +345,7 @@ class MuxingSession extends EventEmitter {
await this.addSegmentToReplay(segmentPath)
}
if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) {
if (this.streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) {
try {
await storeHLSFileFromPath(this.streamingPlaylist, segmentPath)
@ -464,8 +464,8 @@ class MuxingSession extends EventEmitter {
playlist.type = VideoStreamingPlaylistType.HLS
playlist.storage = CONFIG.OBJECT_STORAGE.ENABLED
? VideoStorage.OBJECT_STORAGE
: VideoStorage.FILE_SYSTEM
? FileStorage.OBJECT_STORAGE
: FileStorage.FILE_SYSTEM
return playlist.save()
}

View File

@ -30,13 +30,16 @@ export function buildActorInstance (type: ActivityPubActorType, url: string, pre
}) as MActor
}
export async function updateLocalActorImageFiles (
accountOrChannel: MAccountDefault | MChannelDefault,
imagePhysicalFile: Express.Multer.File,
export async function updateLocalActorImageFiles (options: {
accountOrChannel: MAccountDefault | MChannelDefault
imagePhysicalFile: { path: string }
type: ActorImageType_Type
) {
sendActorUpdate: boolean
}) {
const { accountOrChannel, imagePhysicalFile, type, sendActorUpdate } = options
const processImageSize = async (imageSize: { width: number, height: number }) => {
const extension = getLowercaseExtension(imagePhysicalFile.filename)
const extension = getLowercaseExtension(imagePhysicalFile.path)
const imageName = buildUUID() + extension
const destination = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, imageName)
@ -63,7 +66,9 @@ export async function updateLocalActorImageFiles (
const updatedActor = await updateActorImages(accountOrChannel.Actor, type, actorImagesInfo, t)
await updatedActor.save({ transaction: t })
await sendUpdateActor(accountOrChannel, t)
if (sendActorUpdate) {
await sendUpdateActor(accountOrChannel, t)
}
return type === ActorImageType.AVATAR
? updatedActor.Avatars

View File

@ -1,3 +1,4 @@
import { CONFIG } from '@server/initializers/config.js'
import { VideoModel } from '@server/models/video/video.js'
import {
MVideoAccountLightBlacklistAllFiles,
@ -7,6 +8,7 @@ import {
MVideoImmutable,
MVideoThumbnail
} from '@server/types/models/index.js'
import { getOrCreateAPVideo } from '../activitypub/videos/get.js'
type VideoLoadType = 'for-api' | 'all' | 'only-video' | 'id' | 'none' | 'only-immutable-attributes'
@ -50,17 +52,36 @@ function loadVideoByUrl (
url: string,
fetchType: VideoLoadByUrlType
): Promise<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url)
if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccountAndFiles(url)
if (fetchType === 'only-immutable-attributes') return VideoModel.loadByUrlImmutableAttributes(url)
if (fetchType === 'only-video') return VideoModel.loadByUrl(url)
}
async function loadOrCreateVideoIfAllowedForUser (videoUrl: string) {
if (CONFIG.SEARCH.REMOTE_URI.USERS) {
try {
const res = await getOrCreateAPVideo({
videoObject: videoUrl,
fetchType: 'only-immutable-attributes',
allowRefresh: false
})
return res?.video
} catch {
return undefined
}
}
return VideoModel.loadByUrlImmutableAttributes(videoUrl)
}
export {
type VideoLoadType,
type VideoLoadByUrlType,
loadVideo,
loadVideoByUrl
loadVideoByUrl,
loadOrCreateVideoIfAllowedForUser
}

View File

@ -17,6 +17,7 @@ import {
MCommentAbuseAccountVideo,
MCommentOwnerVideo,
MUser,
MUserDefault,
MVideoAbuseVideoFull,
MVideoAccountLightBlacklistAllFiles
} from '@server/types/models/index.js'
@ -38,7 +39,7 @@ export type AcceptResult = {
function isLocalVideoFileAccepted (object: {
videoBody: VideoCreate
videoFile: VideoUploadFile
user: UserModel
user: MUserDefault
}): AcceptResult {
return { accepted: true }
}

View File

@ -13,8 +13,13 @@ function generateWebVideoObjectStorageKey (filename: string) {
return filename
}
function generateUserExportObjectStorageKey (filename: string) {
return filename
}
export {
generateHLSObjectStorageKey,
generateHLSObjectBaseStorageKey,
generateWebVideoObjectStorageKey
generateWebVideoObjectStorageKey,
generateUserExportObjectStorageKey
}

Some files were not shown because too many files have changed in this diff Show More