OpenCloud
/
PeerTube
mirror of https://github.com/Chocobozzz/PeerTube
1
0
Fork 0
Code Issues Releases Wiki Activity

Use random names for VOD HLS playlists

pull/4283/head
Chocobozzz 2021-07-23 11:20:00 +02:00 committed by Chocobozzz
parent 83903cb65d
commit 764b1a14fc
44 changed files with 508 additions and 281 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
scripts/optimize-old-videos.ts
View File

@ -19,13 +19,13 @@ run()
process.exit(-1)
})
let currentVideoId = null
let currentFile = null
let currentVideoId: string
let currentFilePath: string
process.on('SIGINT', async function () {
console.log('Cleaning up temp files')
await remove(`${currentFile}_backup`)
await remove(`${dirname(currentFile)}/${currentVideoId}-transcoded.mp4`)
await remove(`${currentFilePath}_backup`)
await remove(`${dirname(currentFilePath)}/${currentVideoId}-transcoded.mp4`)
process.exit(0)
})
@ -40,12 +40,12 @@ async function run () {
currentVideoId = video.id
for (const file of video.VideoFiles) {
currentFile = getVideoFilePath(video, file)
currentFilePath = getVideoFilePath(video, file)
const [ videoBitrate, fps, resolution ] = await Promise.all([
getVideoFileBitrate(currentFile),
getVideoFileFPS(currentFile),
getVideoFileResolution(currentFile)
getVideoFileBitrate(currentFilePath),
getVideoFileFPS(currentFilePath),
getVideoFileResolution(currentFilePath)
])
const maxBitrate = getMaxBitrate(resolution.videoFileResolution, fps, VIDEO_TRANSCODING_FPS)
@ -53,25 +53,27 @@ async function run () {
if (isMaxBitrateExceeded) {
console.log(
'Optimizing video file %s with bitrate %s kbps (max: %s kbps)',
basename(currentFile), videoBitrate / 1000, maxBitrate / 1000
basename(currentFilePath), videoBitrate / 1000, maxBitrate / 1000
)
const backupFile = `${currentFile}_backup`
await copy(currentFile, backupFile)
const backupFile = `${currentFilePath}_backup`
await copy(currentFilePath, backupFile)
await optimizeOriginalVideofile(video, file)
// Update file path, the video filename changed
currentFilePath = getVideoFilePath(video, file)
const originalDuration = await getDurationFromVideoFile(backupFile)
const newDuration = await getDurationFromVideoFile(currentFile)
const newDuration = await getDurationFromVideoFile(currentFilePath)
if (originalDuration === newDuration) {
console.log('Finished optimizing %s', basename(currentFile))
console.log('Finished optimizing %s', basename(currentFilePath))
await remove(backupFile)
continue
}
console.log('Failed to optimize %s, restoring original', basename(currentFile))
await move(backupFile, currentFile, { overwrite: true })
console.log('Failed to optimize %s, restoring original', basename(currentFilePath))
await move(backupFile, currentFilePath, { overwrite: true })
await createTorrentAndSetInfoHash(video, file)
await file.save()
}

62
scripts/prune-storage.ts
View File

@ -2,11 +2,11 @@ import { registerTSPaths } from '../server/helpers/register-ts-paths'
registerTSPaths()
import * as prompt from 'prompt'
import { join } from 'path'
import { join, basename } from 'path'
import { CONFIG } from '../server/initializers/config'
import { VideoModel } from '../server/models/video/video'
import { initDatabaseModels } from '../server/initializers/database'
import { readdir, remove } from 'fs-extra'
import { readdir, remove, stat } from 'fs-extra'
import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy'
import * as Bluebird from 'bluebird'
import { getUUIDFromFilename } from '../server/helpers/utils'
@ -14,6 +14,7 @@ import { ThumbnailModel } from '../server/models/video/thumbnail'
import { ActorImageModel } from '../server/models/actor/actor-image'
import { uniq, values } from 'lodash'
import { ThumbnailType } from '@shared/models'
import { VideoFileModel } from '@server/models/video/video-file'
run()
.then(() => process.exit(0))
@ -37,8 +38,8 @@ async function run () {
console.log('Detecting files to remove, it could take a while...')
toDelete = toDelete.concat(
await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesVideoExist(true)),
await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesVideoExist(true)),
await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesWebTorrentFileExist()),
await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesTorrentFileExist()),
await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist),
@ -78,26 +79,27 @@ async function pruneDirectory (directory: string, existFun: ExistFun) {
const toDelete: string[] = []
await Bluebird.map(files, async file => {
if (await existFun(file) !== true) {
toDelete.push(join(directory, file))
const filePath = join(directory, file)
if (await existFun(filePath) !== true) {
toDelete.push(filePath)
}
}, { concurrency: 20 })
return toDelete
}
function doesVideoExist (keepOnlyOwned: boolean) {
return async (file: string) => {
const uuid = getUUIDFromFilename(file)
const video = await VideoModel.load(uuid)
function doesWebTorrentFileExist () {
return (filePath: string) => VideoFileModel.doesOwnedWebTorrentVideoFileExist(basename(filePath))
}
return video && (keepOnlyOwned === false || video.isOwned())
}
function doesTorrentFileExist () {
return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath))
}
function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType) {
return async (file: string) => {
const thumbnail = await ThumbnailModel.loadByFilename(file, type)
return async (filePath: string) => {
const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type)
if (!thumbnail) return false
if (keepOnlyOwned) {
@ -109,21 +111,20 @@ function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType) {
}
}
async function doesActorImageExist (file: string) {
const image = await ActorImageModel.loadByName(file)
async function doesActorImageExist (filePath: string) {
const image = await ActorImageModel.loadByName(basename(filePath))
return !!image
}
async function doesRedundancyExist (file: string) {
const uuid = getUUIDFromFilename(file)
const video = await VideoModel.loadWithFiles(uuid)
if (!video) return false
const isPlaylist = file.includes('.') === false
async function doesRedundancyExist (filePath: string) {
const isPlaylist = (await stat(filePath)).isDirectory()
if (isPlaylist) {
const uuid = getUUIDFromFilename(filePath)
const video = await VideoModel.loadWithFiles(uuid)
if (!video) return false
const p = video.getHLSPlaylist()
if (!p) return false
@ -131,19 +132,10 @@ async function doesRedundancyExist (file: string) {
return !!redundancy
}
const resolution = parseInt(file.split('-')[5], 10)
if (isNaN(resolution)) {
console.error('Cannot prune %s because we cannot guess guess the resolution.', file)
return true
}
const file = await VideoFileModel.loadByFilename(basename(filePath))
if (!file) return false
const videoFile = video.getWebTorrentFile(resolution)
if (!videoFile) {
console.error('Cannot find webtorrent file of video %s - %d', video.url, resolution)
return true
}
const redundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id)
const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
return !!redundancy
}

13
scripts/update-host.ts
View File

@ -16,7 +16,6 @@ import { VideoShareModel } from '../server/models/video/video-share'
import { VideoCommentModel } from '../server/models/video/video-comment'
import { AccountModel } from '../server/models/account/account'
import { VideoChannelModel } from '../server/models/video/video-channel'
import { VideoStreamingPlaylistModel } from '../server/models/video/video-streaming-playlist'
import { initDatabaseModels } from '../server/initializers/database'
import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
import { getServerActor } from '@server/models/application/application'
@ -128,13 +127,17 @@ async function run () {
for (const file of video.VideoFiles) {
console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid)
await createTorrentAndSetInfoHash(video, file)
await file.save()
}
for (const playlist of video.VideoStreamingPlaylists) {
playlist.playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
playlist.segmentsSha256Url = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive)
const playlist = video.getHLSPlaylist()
for (const file of (playlist?.VideoFiles || [])) {
console.log('Updating fragmented torrent file %s of video %s.', file.resolution, video.uuid)
await playlist.save()
await createTorrentAndSetInfoHash(video, file)
await file.save()
}
}
}

12
server/controllers/api/videos/upload.ts
View File

@ -209,10 +209,12 @@ async function addVideo (options: {
})
createTorrentFederate(video, videoFile)
.then(() => {
if (video.state !== VideoState.TO_TRANSCODE) return
if (video.state === VideoState.TO_TRANSCODE) {
await addOptimizeOrMergeAudioJob(videoCreated, videoFile, user)
}
return addOptimizeOrMergeAudioJob(videoCreated, videoFile, user)
})
.catch(err => logger.error('Cannot add optimize/merge audio job for %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) }))
Hooks.runAction('action:api.video.uploaded', { video: videoCreated })
@ -259,9 +261,9 @@ async function createTorrentAndSetInfoHashAsync (video: MVideo, fileArg: MVideoF
return refreshedFile.save()
}
function createTorrentFederate (video: MVideoFullLight, videoFile: MVideoFile): void {
function createTorrentFederate (video: MVideoFullLight, videoFile: MVideoFile) {
// Create the torrent file in async way because it could be long
createTorrentAndSetInfoHashAsync(video, videoFile)
return createTorrentAndSetInfoHashAsync(video, videoFile)
.catch(err => logger.error('Cannot create torrent file for video %s', video.url, { err, ...lTags(video.uuid) }))
.then(() => VideoModel.loadAndPopulateAccountAndServerAndTags(video.id))
.then(refreshedVideo => {

31
server/helpers/database-utils.ts
View File

@ -1,6 +1,6 @@
import * as retry from 'async/retry'
import * as Bluebird from 'bluebird'
import { QueryTypes, Transaction } from 'sequelize'
import { BindOrReplacements, QueryTypes, Transaction } from 'sequelize'
import { Model } from 'sequelize-typescript'
import { sequelizeTypescript } from '@server/initializers/database'
import { logger } from './logger'
@ -84,13 +84,15 @@ function resetSequelizeInstance (instance: Model<any>, savedFields: object) {
})
}
function deleteNonExistingModels <T extends { hasSameUniqueKeysThan (other: T): boolean } & Pick<Model, 'destroy'>> (
function filterNonExistingModels <T extends { hasSameUniqueKeysThan (other: T): boolean }> (
fromDatabase: T[],
newModels: T[],
t: Transaction
newModels: T[]
) {
return fromDatabase.filter(f => !newModels.find(newModel => newModel.hasSameUniqueKeysThan(f)))
.map(f => f.destroy({ transaction: t }))
}
function deleteAllModels <T extends Pick<Model, 'destroy'>> (models: T[], transaction: Transaction) {
return Promise.all(models.map(f => f.destroy({ transaction })))
}
// Sequelize always skip the update if we only update updatedAt field
@ -121,13 +123,28 @@ function afterCommitIfTransaction (t: Transaction, fn: Function) {
// ---------------------------------------------------------------------------
function doesExist (query: string, bind?: BindOrReplacements) {
const options = {
type: QueryTypes.SELECT as QueryTypes.SELECT,
bind,
raw: true
}
return sequelizeTypescript.query(query, options)
.then(results => results.length === 1)
}
// ---------------------------------------------------------------------------
export {
resetSequelizeInstance,
retryTransactionWrapper,
transactionRetryer,
updateInstanceWithAnother,
afterCommitIfTransaction,
deleteNonExistingModels,
filterNonExistingModels,
deleteAllModels,
setAsUpdated,
runInReadCommittedTransaction
runInReadCommittedTransaction,
doesExist
}

15
server/helpers/ffmpeg-utils.ts
View File

@ -212,14 +212,17 @@ async function transcode (options: TranscodeOptions) {
async function getLiveTranscodingCommand (options: {
rtmpUrl: string
outPath: string
masterPlaylistName: string
resolutions: number[]
fps: number
availableEncoders: AvailableEncoders
profile: string
}) {
const { rtmpUrl, outPath, resolutions, fps, availableEncoders, profile } = options
const { rtmpUrl, outPath, resolutions, fps, availableEncoders, profile, masterPlaylistName } = options
const input = rtmpUrl
const command = getFFmpeg(input, 'live')
@ -301,14 +304,14 @@ async function getLiveTranscodingCommand (options: {
command.complexFilter(complexFilter)
addDefaultLiveHLSParams(command, outPath)
addDefaultLiveHLSParams(command, outPath, masterPlaylistName)
command.outputOption('-var_stream_map', varStreamMap.join(' '))
return command
}
function getLiveMuxingCommand (rtmpUrl: string, outPath: string) {
function getLiveMuxingCommand (rtmpUrl: string, outPath: string, masterPlaylistName: string) {
const command = getFFmpeg(rtmpUrl, 'live')
command.outputOption('-c:v copy')
@ -316,7 +319,7 @@ function getLiveMuxingCommand (rtmpUrl: string, outPath: string) {
command.outputOption('-map 0:a?')
command.outputOption('-map 0:v?')
addDefaultLiveHLSParams(command, outPath)
addDefaultLiveHLSParams(command, outPath, masterPlaylistName)
return command
}
@ -371,12 +374,12 @@ function addDefaultEncoderParams (options: {
}
}
function addDefaultLiveHLSParams (command: ffmpeg.FfmpegCommand, outPath: string) {
function addDefaultLiveHLSParams (command: ffmpeg.FfmpegCommand, outPath: string, masterPlaylistName: string) {
command.outputOption('-hls_time ' + VIDEO_LIVE.SEGMENT_TIME_SECONDS)
command.outputOption('-hls_list_size ' + VIDEO_LIVE.SEGMENTS_LIST_SIZE)
command.outputOption('-hls_flags delete_segments+independent_segments')
command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%06d.ts')}`)
command.outputOption('-master_pl_name master.m3u8')
command.outputOption('-master_pl_name ' + masterPlaylistName)
command.outputOption(`-f hls`)
command.output(join(outPath, '%v.m3u8'))

5
server/helpers/webtorrent.ts
View File

@ -103,6 +103,11 @@ async function createTorrentAndSetInfoHash (
await writeFile(torrentPath, torrent)
// Remove old torrent file if it existed
if (videoFile.hasTorrent()) {
await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename))
}
const parsedTorrent = parseTorrent(torrent)
videoFile.infoHash = parsedTorrent.infoHash
videoFile.torrentFilename = torrentFilename

2
server/initializers/constants.ts
View File

@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
// ---------------------------------------------------------------------------
const LAST_MIGRATION_VERSION = 650
const LAST_MIGRATION_VERSION = 655
// ---------------------------------------------------------------------------

66
server/initializers/migrations/0655-streaming-playlist-filenames.ts Normal file
View File

@ -0,0 +1,66 @@
import * as Sequelize from 'sequelize'
async function up (utils: {
transaction: Sequelize.Transaction
queryInterface: Sequelize.QueryInterface
sequelize: Sequelize.Sequelize
db: any
}): Promise<void> {
{
for (const column of [ 'playlistUrl', 'segmentsSha256Url' ]) {
const data = {
type: Sequelize.STRING,
allowNull: true,
defaultValue: null
}
await utils.queryInterface.changeColumn('videoStreamingPlaylist', column, data)
}
}
{
await utils.sequelize.query(
`UPDATE "videoStreamingPlaylist" SET "playlistUrl" = NULL, "segmentsSha256Url" = NULL ` +
`WHERE "videoId" IN (SELECT id FROM video WHERE remote IS FALSE)`
)
}
{
for (const column of [ 'playlistFilename', 'segmentsSha256Filename' ]) {
const data = {
type: Sequelize.STRING,
allowNull: true,
defaultValue: null
}
await utils.queryInterface.addColumn('videoStreamingPlaylist', column, data)
}
}
{
await utils.sequelize.query(
`UPDATE "videoStreamingPlaylist" SET "playlistFilename" = 'master.m3u8', "segmentsSha256Filename" = 'segments-sha256.json'`
)
}
{
for (const column of [ 'playlistFilename', 'segmentsSha256Filename' ]) {
const data = {
type: Sequelize.STRING,
allowNull: false,
defaultValue: null
}
await utils.queryInterface.changeColumn('videoStreamingPlaylist', column, data)
}
}
}
function down (options) {
throw new Error('Not implemented.')
}
export {
up,
down
}

12
server/lib/activitypub/videos/shared/abstract-builder.ts
View File

@ -1,6 +1,6 @@
import { Transaction } from 'sequelize/types'
import { checkUrlsSameHost } from '@server/helpers/activitypub'
import { deleteNonExistingModels } from '@server/helpers/database-utils'
import { deleteAllModels, filterNonExistingModels } from '@server/helpers/database-utils'
import { logger, LoggerTagsFn } from '@server/helpers/logger'
import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail'
import { setVideoTags } from '@server/lib/video'
@ -111,8 +111,7 @@ export abstract class APVideoAbstractBuilder {
const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
// Remove video files that do not exist anymore
const destroyTasks = deleteNonExistingModels(video.VideoFiles || [], newVideoFiles, t)
await Promise.all(destroyTasks)
await deleteAllModels(filterNonExistingModels(video.VideoFiles || [], newVideoFiles), t)
// Update or add other one
const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t))
@ -124,13 +123,11 @@ export abstract class APVideoAbstractBuilder {
const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
// Remove video playlists that do not exist anymore
const destroyTasks = deleteNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists, t)
await Promise.all(destroyTasks)
await deleteAllModels(filterNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists), t)
video.VideoStreamingPlaylists = []
for (const playlistAttributes of streamingPlaylistAttributes) {
const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t)
streamingPlaylistModel.Video = video
@ -163,8 +160,7 @@ export abstract class APVideoAbstractBuilder {
const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(playlistModel, tagObjects).map(a => new VideoFileModel(a))
const destroyTasks = deleteNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles, t)
await Promise.all(destroyTasks)
await deleteAllModels(filterNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles), t)
// Update or add other one
const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t))

13
server/lib/activitypub/videos/shared/object-to-model-attributes.ts
View File

@ -7,10 +7,11 @@ import { logger } from '@server/helpers/logger'
import { getExtFromMimetype } from '@server/helpers/video'
import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants'
import { generateTorrentFileName } from '@server/lib/video-paths'
import { VideoCaptionModel } from '@server/models/video/video-caption'
import { VideoFileModel } from '@server/models/video/video-file'
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
import { FilteredModelAttributes } from '@server/types'
import { MChannelId, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoId } from '@server/types/models'
import { isStreamingPlaylist, MChannelId, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoId } from '@server/types/models'
import {
ActivityHashTagObject,
ActivityMagnetUrlObject,
@ -23,7 +24,6 @@ import {
VideoPrivacy,
VideoStreamingPlaylistType
} from '@shared/models'
import { VideoCaptionModel } from '@server/models/video/video-caption'
function getThumbnailFromIcons (videoObject: VideoObject) {
let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
@ -80,8 +80,8 @@ function getFileAttributesFromUrl (
const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType)
const resolution = fileUrl.height
const videoId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id
const videoStreamingPlaylistId = (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null
const videoId = isStreamingPlaylist(videoOrPlaylist) ? null : videoOrPlaylist.id
const videoStreamingPlaylistId = isStreamingPlaylist(videoOrPlaylist) ? videoOrPlaylist.id : null
const attribute = {
extname,
@ -130,8 +130,13 @@ function getStreamingPlaylistAttributesFromObject (video: MVideoId, videoObject:
const attribute = {
type: VideoStreamingPlaylistType.HLS,
playlistFilename: basename(playlistUrlObject.href),
playlistUrl: playlistUrlObject.href,
segmentsSha256Filename: basename(segmentsSha256UrlObject.href),
segmentsSha256Url: segmentsSha256UrlObject.href,
p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files),
p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
videoId: video.id,

37
server/lib/hls.ts
View File

@ -1,7 +1,7 @@
import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra'
import { flatten, uniq } from 'lodash'
import { basename, dirname, join } from 'path'
import { MVideoWithFile } from '@server/types/models'
import { MStreamingPlaylistFilesVideo, MVideoWithFile } from '@server/types/models'
import { sha256 } from '../helpers/core-utils'
import { getAudioStreamCodec, getVideoStreamCodec, getVideoStreamSize } from '../helpers/ffprobe-utils'
import { logger } from '../helpers/logger'
@ -12,7 +12,7 @@ import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from
import { sequelizeTypescript } from '../initializers/database'
import { VideoFileModel } from '../models/video/video-file'
import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
import { getVideoFilePath } from './video-paths'
import { getHlsResolutionPlaylistFilename, getVideoFilePath } from './video-paths'
async function updateStreamingPlaylistsInfohashesIfNeeded () {
const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
@ -22,27 +22,29 @@ async function updateStreamingPlaylistsInfohashesIfNeeded () {
await sequelizeTypescript.transaction(async t => {
const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t)
playlist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlist.playlistUrl, videoFiles)
playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles)
playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
await playlist.save({ transaction: t })
})
}
}
async function updateMasterHLSPlaylist (video: MVideoWithFile) {
async function updateMasterHLSPlaylist (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) {
const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
const streamingPlaylist = video.getHLSPlaylist()
for (const file of streamingPlaylist.VideoFiles) {
const playlistFilename = VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)
const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
const masterPlaylistPath = join(directory, playlist.playlistFilename)
for (const file of playlist.VideoFiles) {
const playlistFilename = getHlsResolutionPlaylistFilename(file.filename)
// If we did not generated a playlist for this resolution, skip
const filePlaylistPath = join(directory, playlistFilename)
if (await pathExists(filePlaylistPath) === false) continue
const videoFilePath = getVideoFilePath(streamingPlaylist, file)
const videoFilePath = getVideoFilePath(playlist, file)
const size = await getVideoStreamSize(videoFilePath)
@ -66,23 +68,22 @@ async function updateMasterHLSPlaylist (video: MVideoWithFile) {
await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
}
async function updateSha256VODSegments (video: MVideoWithFile) {
async function updateSha256VODSegments (video: MVideoWithFile, playlist: MStreamingPlaylistFilesVideo) {
const json: { [filename: string]: { [range: string]: string } } = {}
const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
const hlsPlaylist = video.getHLSPlaylist()
// For all the resolutions available for this video
for (const file of hlsPlaylist.VideoFiles) {
for (const file of playlist.VideoFiles) {
const rangeHashes: { [range: string]: string } = {}
const videoPath = getVideoFilePath(hlsPlaylist, file)
const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
const videoPath = getVideoFilePath(playlist, file)
const resolutionPlaylistPath = join(playlistDirectory, getHlsResolutionPlaylistFilename(file.filename))
// Maybe the playlist is not generated for this resolution yet
if (!await pathExists(playlistPath)) continue
if (!await pathExists(resolutionPlaylistPath)) continue
const playlistContent = await readFile(playlistPath)
const playlistContent = await readFile(resolutionPlaylistPath)
const ranges = getRangesFromPlaylist(playlistContent.toString())
const fd = await open(videoPath, 'r')
@ -98,7 +99,7 @@ async function updateSha256VODSegments (video: MVideoWithFile) {
json[videoFilename] = rangeHashes
}
const outputPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
const outputPath = join(playlistDirectory, playlist.segmentsSha256Filename)
await outputJSON(outputPath, json)
}

3
server/lib/job-queue/handlers/video-file-import.ts
View File

@ -61,8 +61,7 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) {
if (currentVideoFile) {
// Remove old file and old torrent
await video.removeFile(currentVideoFile)
await currentVideoFile.removeTorrent()
await video.removeFileAndTorrent(currentVideoFile)
// Remove the old video file from the array
video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile)

15
server/lib/job-queue/handlers/video-live-ending.ts
View File

@ -7,12 +7,12 @@ import { buildConcatenatedName, cleanupLive, LiveSegmentShaStore } from '@server
import { generateVideoMiniature } from '@server/lib/thumbnail'
import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/video-transcoding'
import { publishAndFederateIfNeeded } from '@server/lib/video'
import { getHLSDirectory } from '@server/lib/video-paths'
import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHLSDirectory } from '@server/lib/video-paths'
import { VideoModel } from '@server/models/video/video'
import { VideoFileModel } from '@server/models/video/video-file'
import { VideoLiveModel } from '@server/models/video/video-live'
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
import { MVideo, MVideoLive } from '@server/types/models'
import { MStreamingPlaylist, MVideo, MVideoLive } from '@server/types/models'
import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
import { logger } from '../../../helpers/logger'
@ -43,7 +43,7 @@ async function processVideoLiveEnding (job: Bull.Job) {
return cleanupLive(video, streamingPlaylist)
}
return saveLive(video, live)
return saveLive(video, live, streamingPlaylist)
}
// ---------------------------------------------------------------------------
@ -54,14 +54,14 @@ export {
// ---------------------------------------------------------------------------
async function saveLive (video: MVideo, live: MVideoLive) {
async function saveLive (video: MVideo, live: MVideoLive, streamingPlaylist: MStreamingPlaylist) {
const hlsDirectory = getHLSDirectory(video, false)
const replayDirectory = join(hlsDirectory, VIDEO_LIVE.REPLAY_DIRECTORY)
const rootFiles = await readdir(hlsDirectory)
const playlistFiles = rootFiles.filter(file => {
return file.endsWith('.m3u8') && file !== 'master.m3u8'
return file.endsWith('.m3u8') && file !== streamingPlaylist.playlistFilename
})
await cleanupLiveFiles(hlsDirectory)
@ -80,7 +80,12 @@ async function saveLive (video: MVideo, live: MVideoLive) {
const hlsPlaylist = videoWithFiles.getHLSPlaylist()
await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id)
// Reset playlist
hlsPlaylist.VideoFiles = []
hlsPlaylist.playlistFilename = generateHLSMasterPlaylistFilename()
hlsPlaylist.segmentsSha256Filename = generateHlsSha256SegmentsFilename()
await hlsPlaylist.save()
let durationDone = false

3
server/lib/job-queue/handlers/video-transcoding.ts
View File

@ -125,8 +125,7 @@ async function onHlsPlaylistGeneration (video: MVideoFullLight, user: MUser, pay
if (payload.isMaxQuality && CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) {
// Remove webtorrent files if not enabled
for (const file of video.VideoFiles) {
await video.removeFile(file)
await file.removeTorrent()
await video.removeFileAndTorrent(file)
await file.destroy()
}

26
server/lib/live/live-manager.ts
View File

@ -4,16 +4,17 @@ import { isTestInstance } from '@server/helpers/core-utils'
import { computeResolutionsToTranscode, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config'
import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME, WEBSERVER } from '@server/initializers/constants'
import { P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE, VIEW_LIFETIME } from '@server/initializers/constants'
import { UserModel } from '@server/models/user/user'
import { VideoModel } from '@server/models/video/video'
import { VideoLiveModel } from '@server/models/video/video-live'
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoLiveVideo } from '@server/types/models'
import { MStreamingPlaylistVideo, MVideo, MVideoLiveVideo } from '@server/types/models'
import { VideoState, VideoStreamingPlaylistType } from '@shared/models'
import { federateVideoIfNeeded } from '../activitypub/videos'
import { JobQueue } from '../job-queue'
import { PeerTubeSocket } from '../peertube-socket'
import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '../video-paths'
import { LiveQuotaStore } from './live-quota-store'
import { LiveSegmentShaStore } from './live-segment-sha-store'
import { cleanupLive } from './live-utils'
@ -392,19 +393,18 @@ class LiveManager {
return resolutionsEnabled.concat([ originResolution ])
}
private async createLivePlaylist (video: MVideo, allResolutions: number[]) {
const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({
videoId: video.id,
playlistUrl,
segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive),
p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, allResolutions),
p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
private async createLivePlaylist (video: MVideo, allResolutions: number[]): Promise<MStreamingPlaylistVideo> {
const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(video)
type: VideoStreamingPlaylistType.HLS
}, { returning: true }) as [ MStreamingPlaylist, boolean ]
playlist.playlistFilename = generateHLSMasterPlaylistFilename(true)
playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(true)
return Object.assign(videoStreamingPlaylist, { Video: video })
playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
playlist.type = VideoStreamingPlaylistType.HLS
playlist.assignP2PMediaLoaderInfoHashes(video, allResolutions)
return playlist.save()
}
static get Instance () {

7
server/lib/live/shared/muxing-session.ts
View File

@ -112,13 +112,16 @@ class MuxingSession extends EventEmitter {
this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED
? await getLiveTranscodingCommand({
rtmpUrl: this.rtmpUrl,
outPath,
masterPlaylistName: this.streamingPlaylist.playlistFilename,
resolutions: this.allResolutions,
fps: this.fps,
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
profile: CONFIG.LIVE.TRANSCODING.PROFILE
})
: getLiveMuxingCommand(this.rtmpUrl, outPath)
: getLiveMuxingCommand(this.rtmpUrl, outPath, this.streamingPlaylist.playlistFilename)
logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags)
@ -182,7 +185,7 @@ class MuxingSession extends EventEmitter {
}
private watchMasterFile (outPath: string) {
this.masterWatcher = chokidar.watch(outPath + '/master.m3u8')
this.masterWatcher = chokidar.watch(outPath + '/' + this.streamingPlaylist.playlistFilename)
this.masterWatcher.on('add', async () => {
this.emit('master-playlist-created', { videoId: this.videoId })

7
server/lib/schedulers/videos-redundancy-scheduler.ts
View File

@ -267,7 +267,8 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy)
const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video)
await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({
expiresOn,
@ -282,7 +283,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
await sendCreateCacheFile(serverActor, video, createdModel)
logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url)
logger.info('Duplicated playlist %s -> %s.', masterPlaylistUrl, createdModel.url)
}
private async extendsExpirationOf (redundancy: MVideoRedundancyVideo, expiresAfterMs: number) {
@ -330,7 +331,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
private buildEntryLogId (object: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo) {
if (isMVideoRedundancyFileVideo(object)) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}`
return `${object.VideoStreamingPlaylist.playlistUrl}`
return `${object.VideoStreamingPlaylist.getMasterPlaylistUrl(object.VideoStreamingPlaylist.Video)}`
}
private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]) {

71
server/lib/transcoding/video-transcoding.ts
View File

@ -10,11 +10,18 @@ import { transcode, TranscodeOptions, TranscodeOptionsType } from '../../helpers
import { canDoQuickTranscode, getDurationFromVideoFile, getMetadataFromFile, getVideoFileFPS } from '../../helpers/ffprobe-utils'
import { logger } from '../../helpers/logger'
import { CONFIG } from '../../initializers/config'
import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, WEBSERVER } from '../../initializers/constants'
import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers/constants'
import { VideoFileModel } from '../../models/video/video-file'
import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
import { updateMasterHLSPlaylist, updateSha256VODSegments } from '../hls'
import { generateHLSVideoFilename, generateWebTorrentVideoFilename, getVideoFilePath } from '../video-paths'
import {
generateHLSMasterPlaylistFilename,
generateHlsSha256SegmentsFilename,
generateHLSVideoFilename,
generateWebTorrentVideoFilename,
getHlsResolutionPlaylistFilename,
getVideoFilePath
} from '../video-paths'
import { VideoTranscodingProfilesManager } from './video-transcoding-profiles'
/**
@ -272,14 +279,14 @@ async function generateHlsPlaylistCommon (options: {
await ensureDir(videoTranscodedBasePath)
const videoFilename = generateHLSVideoFilename(resolution)
const playlistFilename = VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution)
const playlistFileTranscodePath = join(videoTranscodedBasePath, playlistFilename)
const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFilename)
const resolutionPlaylistFileTranscodePath = join(videoTranscodedBasePath, resolutionPlaylistFilename)
const transcodeOptions = {
type,
inputPath,
outputPath: playlistFileTranscodePath,
outputPath: resolutionPlaylistFileTranscodePath,
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
profile: CONFIG.TRANSCODING.PROFILE,
@ -299,19 +306,23 @@ async function generateHlsPlaylistCommon (options: {
await transcode(transcodeOptions)
const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
// Create or update the playlist
const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({
videoId: video.id,
playlistUrl,
segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid, video.isLive),
p2pMediaLoaderInfohashes: [],
p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(video)
type: VideoStreamingPlaylistType.HLS
}, { returning: true }) as [ MStreamingPlaylistFilesVideo, boolean ]
videoStreamingPlaylist.Video = video
if (!playlist.playlistFilename) {
playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive)
}
if (!playlist.segmentsSha256Filename) {
playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive)
}
playlist.p2pMediaLoaderInfohashes = []
playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
playlist.type = VideoStreamingPlaylistType.HLS
await playlist.save()
// Build the new playlist file
const extname = extnameUtil(videoFilename)
@ -321,18 +332,18 @@ async function generateHlsPlaylistCommon (options: {
size: 0,
filename: videoFilename,
fps: -1,
videoStreamingPlaylistId: videoStreamingPlaylist.id
videoStreamingPlaylistId: playlist.id
})
const videoFilePath = getVideoFilePath(videoStreamingPlaylist, newVideoFile)
const videoFilePath = getVideoFilePath(playlist, newVideoFile)
// Move files from tmp transcoded directory to the appropriate place
const baseHlsDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
await ensureDir(baseHlsDirectory)
// Move playlist file
const playlistPath = join(baseHlsDirectory, playlistFilename)
await move(playlistFileTranscodePath, playlistPath, { overwrite: true })
const resolutionPlaylistPath = join(baseHlsDirectory, resolutionPlaylistFilename)
await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true })
// Move video file
await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
@ -342,20 +353,20 @@ async function generateHlsPlaylistCommon (options: {
newVideoFile.fps = await getVideoFileFPS(videoFilePath)
newVideoFile.metadata = await getMetadataFromFile(videoFilePath)
await createTorrentAndSetInfoHash(videoStreamingPlaylist, newVideoFile)
await createTorrentAndSetInfoHash(playlist, newVideoFile)
await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
videoStreamingPlaylist.VideoFiles = await videoStreamingPlaylist.$get('VideoFiles')
videoStreamingPlaylist.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(
playlistUrl, videoStreamingPlaylist.VideoFiles
)
await videoStreamingPlaylist.save()
const playlistWithFiles = playlist as MStreamingPlaylistFilesVideo
playlistWithFiles.VideoFiles = await playlist.$get('VideoFiles')
playlist.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles)
video.setHLSPlaylist(videoStreamingPlaylist)
await playlist.save()
await updateMasterHLSPlaylist(video)
await updateSha256VODSegments(video)
video.setHLSPlaylist(playlist)
return playlistPath
await updateMasterHLSPlaylist(video, playlistWithFiles)
await updateSha256VODSegments(video, playlistWithFiles)
return resolutionPlaylistPath
}

29
server/lib/video-paths.ts
View File

@ -4,19 +4,16 @@ import { CONFIG } from '@server/initializers/config'
import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_PATHS, WEBSERVER } from '@server/initializers/constants'
import { isStreamingPlaylist, MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
import { buildUUID } from '@server/helpers/uuid'
import { removeFragmentedMP4Ext } from '@shared/core-utils'
// ################## Video file name ##################
function generateWebTorrentVideoFilename (resolution: number, extname: string) {
const uuid = buildUUID()
return uuid + '-' + resolution + extname
return buildUUID() + '-' + resolution + extname
}
function generateHLSVideoFilename (resolution: number) {
const uuid = buildUUID()
return `${uuid}-${resolution}-fragmented.mp4`
return `${buildUUID()}-${resolution}-fragmented.mp4`
}
function getVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, isRedundancy = false) {
@ -54,6 +51,23 @@ function getHLSDirectory (video: MVideoUUID, isRedundancy = false) {
return join(baseDir, video.uuid)
}
function getHlsResolutionPlaylistFilename (videoFilename: string) {
// Video file name already contain resolution
return removeFragmentedMP4Ext(videoFilename) + '.m3u8'
}
function generateHLSMasterPlaylistFilename (isLive = false) {
if (isLive) return 'master.m3u8'
return buildUUID() + '-master.m3u8'
}
function generateHlsSha256SegmentsFilename (isLive = false) {
if (isLive) return 'segments-sha256.json'
return buildUUID() + '-segments-sha256.json'
}
// ################## Torrents ##################
function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, resolution: number) {
@ -91,6 +105,9 @@ export {
getTorrentFilePath,
getHLSDirectory,
generateHLSMasterPlaylistFilename,
generateHlsSha256SegmentsFilename,
getHlsResolutionPlaylistFilename,
getLocalVideoFileMetadataUrl,

4
server/lib/video.ts
View File

@ -5,7 +5,7 @@ import { sequelizeTypescript } from '@server/initializers/database'
import { TagModel } from '@server/models/video/tag'
import { VideoModel } from '@server/models/video/video'
import { FilteredModelAttributes } from '@server/types'
import { MThumbnail, MUserId, MVideo, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models'
import { federateVideoIfNeeded } from './activitypub/videos'
import { JobQueue } from './job-queue/job-queue'
@ -105,7 +105,7 @@ async function publishAndFederateIfNeeded (video: MVideoUUID, wasLive = false) {
}
}
async function addOptimizeOrMergeAudioJob (video: MVideo, videoFile: MVideoFile, user: MUserId) {
async function addOptimizeOrMergeAudioJob (video: MVideoUUID, videoFile: MVideoFile, user: MUserId) {
let dataInput: VideoTranscodingPayload
if (videoFile.isAudio()) {

10
server/models/actor/actor-follow.ts
View File

@ -19,8 +19,8 @@ import {
UpdatedAt
} from 'sequelize-typescript'
import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc'
import { doesExist } from '@server/helpers/database-utils'
import { getServerActor } from '@server/models/application/application'
import { VideoModel } from '@server/models/video/video'
import {
MActorFollowActorsDefault,
MActorFollowActorsDefaultSubscription,
@ -166,14 +166,8 @@ export class ActorFollowModel extends Model<Partial<AttributesOnly<ActorFollowMo
static isFollowedBy (actorId: number, followerActorId: number) {
const query = 'SELECT 1 FROM "actorFollow" WHERE "actorId" = $followerActorId AND "targetActorId" = $actorId LIMIT 1'
const options = {
type: QueryTypes.SELECT as QueryTypes.SELECT,
bind: { actorId, followerActorId },
raw: true
}
return VideoModel.sequelize.query(query, options)
.then(results => results.length === 1)
return doesExist(query, { actorId, followerActorId })
}
static loadByActorAndTarget (actorId: number, targetActorId: number, t?: Transaction): Promise<MActorFollowActorsDefault> {

4
server/models/redundancy/video-redundancy.ts
View File

@ -160,8 +160,8 @@ export class VideoRedundancyModel extends Model<Partial<AttributesOnly<VideoRedu
const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
logger.info('Removing duplicated video file %s.', logIdentifier)
videoFile.Video.removeFile(videoFile, true)
.catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
videoFile.Video.removeFileAndTorrent(videoFile, true)
.catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
}
if (instance.videoStreamingPlaylistId) {

8
server/models/video/formatter/video-format-utils.ts
View File

@ -182,8 +182,8 @@ function streamingPlaylistsModelToFormattedJSON (
return {
id: playlist.id,
type: playlist.type,
playlistUrl: playlist.playlistUrl,
segmentsSha256Url: playlist.segmentsSha256Url,
playlistUrl: playlist.getMasterPlaylistUrl(video),
segmentsSha256Url: playlist.getSha256SegmentsUrl(video),
redundancies,
files
}
@ -331,7 +331,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
type: 'Link',
name: 'sha256',
mediaType: 'application/json' as 'application/json',
href: playlist.segmentsSha256Url
href: playlist.getSha256SegmentsUrl(video)
})
addVideoFilesInAPAcc(tag, video, playlist.VideoFiles || [])
@ -339,7 +339,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
url.push({
type: 'Link',
mediaType: 'application/x-mpegURL' as 'application/x-mpegURL',
href: playlist.playlistUrl,
href: playlist.getMasterPlaylistUrl(video),
tag
})
}

3
server/models/video/sql/shared/video-tables.ts
View File

@ -92,12 +92,13 @@ export class VideoTables {
}
getStreamingPlaylistAttributes () {
let playlistKeys = [ 'id', 'playlistUrl', 'type' ]
let playlistKeys = [ 'id', 'playlistUrl', 'playlistFilename', 'type' ]
if (this.mode === 'get') {
playlistKeys = playlistKeys.concat([
'p2pMediaLoaderInfohashes',
'p2pMediaLoaderPeerVersion',
'segmentsSha256Filename',
'segmentsSha256Url',
'videoId',
'createdAt',

45
server/models/video/video-file.ts
View File

@ -1,7 +1,7 @@
import { remove } from 'fs-extra'
import * as memoizee from 'memoizee'
import { join } from 'path'
import { FindOptions, Op, QueryTypes, Transaction } from 'sequelize'
import { FindOptions, Op, Transaction } from 'sequelize'
import {
AllowNull,
BelongsTo,
@ -21,6 +21,7 @@ import {
import { Where } from 'sequelize/types/lib/utils'
import validator from 'validator'
import { buildRemoteVideoBaseUrl } from '@server/helpers/activitypub'
import { doesExist } from '@server/helpers/database-utils'
import { logger } from '@server/helpers/logger'
import { extractVideo } from '@server/helpers/video'
import { getTorrentFilePath } from '@server/lib/video-paths'
@ -250,14 +251,8 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
static doesInfohashExist (infoHash: string) {
const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
const options = {
type: QueryTypes.SELECT as QueryTypes.SELECT,
bind: { infoHash },
raw: true
}
return VideoModel.sequelize.query(query, options)
.then(results => results.length === 1)
return doesExist(query, { infoHash })
}
static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
@ -266,6 +261,33 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
return !!videoFile
}
static async doesOwnedTorrentFileExist (filename: string) {
const query = 'SELECT 1 FROM "videoFile" ' +
'LEFT JOIN "video" "webtorrent" ON "webtorrent"."id" = "videoFile"."videoId" AND "webtorrent"."remote" IS FALSE ' +
'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' +
'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' +
'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webtorrent"."id" IS NOT NULL) LIMIT 1'
return doesExist(query, { filename })
}
static async doesOwnedWebTorrentVideoFileExist (filename: string) {
const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' +
'WHERE "filename" = $filename LIMIT 1'
return doesExist(query, { filename })
}
static loadByFilename (filename: string) {
const query = {
where: {
filename
}
}
return VideoFileModel.findOne(query)
}
static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
const query = {
where: {
@ -443,10 +465,9 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
}
getFileDownloadUrl (video: MVideoWithHost) {
const basePath = this.isHLS()
? STATIC_DOWNLOAD_PATHS.HLS_VIDEOS
: STATIC_DOWNLOAD_PATHS.VIDEOS
const path = join(basePath, this.filename)
const path = this.isHLS()
? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`)
: join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`)
if (video.isOwned()) return WEBSERVER.URL + path

89
server/models/video/video-streaming-playlist.ts
View File

@ -1,19 +1,27 @@
import * as memoizee from 'memoizee'
import { join } from 'path'
import { Op, QueryTypes } from 'sequelize'
import { Op } from 'sequelize'
import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
import { doesExist } from '@server/helpers/database-utils'
import { VideoFileModel } from '@server/models/video/video-file'
import { MStreamingPlaylist } from '@server/types/models'
import { MStreamingPlaylist, MVideo } from '@server/types/models'
import { AttributesOnly } from '@shared/core-utils'
import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
import { sha1 } from '../../helpers/core-utils'
import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
import { isArrayOf } from '../../helpers/custom-validators/misc'
import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
import { CONSTRAINTS_FIELDS, MEMOIZE_LENGTH, MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, STATIC_PATHS } from '../../initializers/constants'
import {
CONSTRAINTS_FIELDS,
MEMOIZE_LENGTH,
MEMOIZE_TTL,
P2P_MEDIA_LOADER_PEER_VERSION,
STATIC_PATHS,
WEBSERVER
} from '../../initializers/constants'
import { VideoRedundancyModel } from '../redundancy/video-redundancy'
import { throwIfNotValid } from '../utils'
import { VideoModel } from './video'
import { AttributesOnly } from '@shared/core-utils'
@Table({
tableName: 'videoStreamingPlaylist',
@ -43,7 +51,11 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
type: VideoStreamingPlaylistType
@AllowNull(false)
@Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url'))
@Column
playlistFilename: string
@AllowNull(true)
@Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url', true))
@Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max))
playlistUrl: string
@ -57,7 +69,11 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
p2pMediaLoaderPeerVersion: number
@AllowNull(false)
@Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url'))
@Column
segmentsSha256Filename: string
@AllowNull(true)
@Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url', true))
@Column
segmentsSha256Url: string
@ -98,14 +114,8 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
static doesInfohashExist (infoHash: string) {
const query = 'SELECT 1 FROM "videoStreamingPlaylist" WHERE $infoHash = ANY("p2pMediaLoaderInfohashes") LIMIT 1'
const options = {
type: QueryTypes.SELECT as QueryTypes.SELECT,
bind: { infoHash },
raw: true
}
return VideoModel.sequelize.query<object>(query, options)
.then(results => results.length === 1)
return doesExist(query, { infoHash })
}
static buildP2PMediaLoaderInfoHashes (playlistUrl: string, files: unknown[]) {
@ -125,7 +135,13 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
p2pMediaLoaderPeerVersion: {
[Op.ne]: P2P_MEDIA_LOADER_PEER_VERSION
}
}
},
include: [
{
model: VideoModel.unscoped(),
required: true
}
]
}
return VideoStreamingPlaylistModel.findAll(query)
@ -144,7 +160,7 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
return VideoStreamingPlaylistModel.findByPk(id, options)
}
static loadHLSPlaylistByVideo (videoId: number) {
static loadHLSPlaylistByVideo (videoId: number): Promise<MStreamingPlaylist> {
const options = {
where: {
type: VideoStreamingPlaylistType.HLS,
@ -155,30 +171,29 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
return VideoStreamingPlaylistModel.findOne(options)
}
static getHlsPlaylistFilename (resolution: number) {
return resolution + '.m3u8'
static async loadOrGenerate (video: MVideo) {
let playlist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
if (!playlist) playlist = new VideoStreamingPlaylistModel()
return Object.assign(playlist, { videoId: video.id, Video: video })
}
static getMasterHlsPlaylistFilename () {
return 'master.m3u8'
assignP2PMediaLoaderInfoHashes (video: MVideo, files: unknown[]) {
const masterPlaylistUrl = this.getMasterPlaylistUrl(video)
this.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(masterPlaylistUrl, files)
}
static getHlsSha256SegmentsFilename () {
return 'segments-sha256.json'
getMasterPlaylistUrl (video: MVideo) {
if (video.isOwned()) return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid)
return this.playlistUrl
}
static getHlsMasterPlaylistStaticPath (videoUUID: string) {
return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
}
getSha256SegmentsUrl (video: MVideo) {
if (video.isOwned()) return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid, video.isLive)
static getHlsPlaylistStaticPath (videoUUID: string, resolution: number) {
return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution))
}
static getHlsSha256SegmentsStaticPath (videoUUID: string, isLive: boolean) {
if (isLive) return join('/live', 'segments-sha256', videoUUID)
return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
return this.segmentsSha256Url
}
getStringType () {
@ -195,4 +210,14 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
return this.type === other.type &&
this.videoId === other.videoId
}
private getMasterPlaylistStaticPath (videoUUID: string) {
return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.playlistFilename)
}
private getSha256SegmentsStaticPath (videoUUID: string, isLive: boolean) {
if (isLive) return join('/live', 'segments-sha256', videoUUID)
return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.segmentsSha256Filename)
}
}

12
server/models/video/video.ts
View File

@ -762,8 +762,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
// Remove physical files and torrents
instance.VideoFiles.forEach(file => {
tasks.push(instance.removeFile(file))
tasks.push(file.removeTorrent())
tasks.push(instance.removeFileAndTorrent(file))
})
// Remove playlists file
@ -1670,10 +1669,13 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
.concat(toAdd)
}
removeFile (videoFile: MVideoFile, isRedundancy = false) {
removeFileAndTorrent (videoFile: MVideoFile, isRedundancy = false) {
const filePath = getVideoFilePath(this, videoFile, isRedundancy)
return remove(filePath)
.catch(err => logger.warn('Cannot delete file %s.', filePath, { err }))
const promises: Promise<any>[] = [ remove(filePath) ]
if (!isRedundancy) promises.push(videoFile.removeTorrent())
return Promise.all(promises)
}
async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) {

4
server/tests/api/live/live-constraints.ts
View File

@ -4,7 +4,7 @@ import 'mocha'
import * as chai from 'chai'
import { VideoPrivacy } from '@shared/models'
import {
checkLiveCleanup,
checkLiveCleanupAfterSave,
cleanupTests,
ConfigCommand,
createMultipleServers,
@ -43,7 +43,7 @@ describe('Test live constraints', function () {
expect(video.duration).to.be.greaterThan(0)
}
await checkLiveCleanup(servers[0], videoId, resolutions)
await checkLiveCleanupAfterSave(servers[0], videoId, resolutions)
}
async function waitUntilLivePublishedOnAllServers (videoId: string) {

14
server/tests/api/live/live-save-replay.ts
View File

@ -4,7 +4,7 @@ import 'mocha'
import * as chai from 'chai'
import { FfmpegCommand } from 'fluent-ffmpeg'
import {
checkLiveCleanup,
checkLiveCleanupAfterSave,
cleanupTests,
ConfigCommand,
createMultipleServers,
@ -150,7 +150,7 @@ describe('Save replay setting', function () {
await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED)
// No resolutions saved since we did not save replay
await checkLiveCleanup(servers[0], liveVideoUUID, [])
await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [])
})
it('Should correctly terminate the stream on blacklist and delete the live', async function () {
@ -179,7 +179,7 @@ describe('Save replay setting', function () {
await wait(5000)
await waitJobs(servers)
await checkLiveCleanup(servers[0], liveVideoUUID, [])
await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [])
})
it('Should correctly terminate the stream on delete and delete the video', async function () {
@ -203,7 +203,7 @@ describe('Save replay setting', function () {
await waitJobs(servers)
await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
await checkLiveCleanup(servers[0], liveVideoUUID, [])
await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [])
})
})
@ -259,7 +259,7 @@ describe('Save replay setting', function () {
})
it('Should have cleaned up the live files', async function () {
await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ])
await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [ 720 ])
})
it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
@ -287,7 +287,7 @@ describe('Save replay setting', function () {
await wait(5000)
await waitJobs(servers)
await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ])
await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [ 720 ])
})
it('Should correctly terminate the stream on delete and delete the video', async function () {
@ -310,7 +310,7 @@ describe('Save replay setting', function () {
await waitJobs(servers)
await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
await checkLiveCleanup(servers[0], liveVideoUUID, [])
await checkLiveCleanupAfterSave(servers[0], liveVideoUUID, [])
})
})

14
server/tests/api/live/live.ts
View File

@ -2,10 +2,10 @@
import 'mocha'
import * as chai from 'chai'
import { join } from 'path'
import { basename, join } from 'path'
import { ffprobePromise, getVideoStreamFromFile } from '@server/helpers/ffprobe-utils'
import {
checkLiveCleanup,
checkLiveCleanupAfterSave,
checkLiveSegmentHash,
checkResolutionsInMasterPlaylist,
cleanupTests,
@ -506,6 +506,10 @@ describe('Test live', function () {
await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200)
await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200)
// We should have generated random filenames
expect(basename(hlsPlaylist.playlistUrl)).to.not.equal('master.m3u8')
expect(basename(hlsPlaylist.segmentsSha256Url)).to.not.equal('segments-sha256.json')
expect(hlsPlaylist.files).to.have.lengthOf(resolutions.length)
for (const resolution of resolutions) {
@ -520,7 +524,9 @@ describe('Test live', function () {
expect(file.fps).to.be.approximately(30, 2)
}
const filename = `${video.uuid}-${resolution}-fragmented.mp4`
const filename = basename(file.fileUrl)
expect(filename).to.not.contain(video.uuid)
const segmentPath = servers[0].servers.buildDirectory(join('streaming-playlists', 'hls', video.uuid, filename))
const probe = await ffprobePromise(segmentPath)
@ -537,7 +543,7 @@ describe('Test live', function () {
it('Should correctly have cleaned up the live files', async function () {
this.timeout(30000)
await checkLiveCleanup(servers[0], liveVideoId, [ 240, 360, 720 ])
await checkLiveCleanupAfterSave(servers[0], liveVideoId, [ 240, 360, 720 ])
})
})

4
server/tests/api/users/users-multiple-servers.ts
View File

@ -58,10 +58,10 @@ describe('Test users with multiple servers', function () {
const { uuid } = await servers[0].videos.upload({ token: userAccessToken })
videoUUID = uuid
await waitJobs(servers)
await saveVideoInServers(servers, videoUUID)
}
await waitJobs(servers)
})
it('Should be able to update my display name', async function () {

7
server/tests/api/videos/resumable-upload.ts
View File

@ -170,8 +170,13 @@ describe('Test resumable upload', function () {
const size = 1000
// Content length check seems to have changed in v16
const expectedStatus = process.version.startsWith('v16')
? HttpStatusCode.CONFLICT_409
: HttpStatusCode.BAD_REQUEST_400
const contentRangeBuilder = (start: number) => `bytes ${start}-${start + size - 1}/${size}`
await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.CONFLICT_409, contentRangeBuilder, contentLength: size })
await sendChunks({ pathUploadId: uploadId, expectedStatus, contentRangeBuilder, contentLength: size })
await checkFileSize(uploadId, 0)
})
})

11
server/tests/api/videos/video-hls.ts
View File

@ -2,7 +2,8 @@
import 'mocha'
import * as chai from 'chai'
import { join } from 'path'
import { basename, join } from 'path'
import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils'
import {
checkDirectoryIsEmpty,
checkResolutionsInMasterPlaylist,
@ -19,8 +20,6 @@ import {
} from '@shared/extra-utils'
import { HttpStatusCode, VideoStreamingPlaylistType } from '@shared/models'
import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants'
import { uuidRegex } from '@shared/core-utils'
import { basename } from 'path/posix'
const expect = chai.expect
@ -78,11 +77,13 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h
// Check resolution playlists
{
for (const resolution of resolutions) {
const file = hlsFiles.find(f => f.resolution.id === resolution)
const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8'
const subPlaylist = await server.streamingPlaylists.get({
url: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${resolution}.m3u8`
url: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}`
})
const file = hlsFiles.find(f => f.resolution.id === resolution)
expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`))
expect(subPlaylist).to.contain(basename(file.fileUrl))
}

3
server/tests/cli/optimize-old-videos.ts
View File

@ -2,7 +2,6 @@
import 'mocha'
import * as chai from 'chai'
import { join } from 'path'
import {
cleanupTests,
createMultipleServers,
@ -86,7 +85,7 @@ describe('Test optimize old videos', function () {
expect(file.size).to.be.below(8000000)
const path = servers[0].servers.buildDirectory(join('videos', video.uuid + '-' + file.resolution.id + '.mp4'))
const path = servers[0].servers.buildWebTorrentFilePath(file.fileUrl)
const bitrate = await getVideoFileBitrate(path)
const fps = await getVideoFileFPS(path)
const resolution = await getVideoFileResolution(path)

41
server/tests/cli/prune-storage.ts
View File

@ -36,7 +36,7 @@ async function assertNotExists (server: PeerTubeServer, directory: string, subst
}
}
async function assertCountAreOkay (servers: PeerTubeServer[]) {
async function assertCountAreOkay (servers: PeerTubeServer[], videoServer2UUID: string) {
for (const server of servers) {
const videosCount = await countFiles(server, 'videos')
expect(videosCount).to.equal(8)
@ -53,12 +53,21 @@ async function assertCountAreOkay (servers: PeerTubeServer[]) {
const avatarsCount = await countFiles(server, 'avatars')
expect(avatarsCount).to.equal(2)
}
// When we'll prune HLS directories too
// const hlsRootCount = await countFiles(servers[1], 'streaming-playlists/hls/')
// expect(hlsRootCount).to.equal(2)
// const hlsCount = await countFiles(servers[1], 'streaming-playlists/hls/' + videoServer2UUID)
// expect(hlsCount).to.equal(10)
}
describe('Test prune storage scripts', function () {
let servers: PeerTubeServer[]
const badNames: { [directory: string]: string[] } = {}
let videoServer2UUID: string
before(async function () {
this.timeout(120000)
@ -68,7 +77,9 @@ describe('Test prune storage scripts', function () {
for (const server of servers) {
await server.videos.upload({ attributes: { name: 'video 1' } })
await server.videos.upload({ attributes: { name: 'video 2' } })
const { uuid } = await server.videos.upload({ attributes: { name: 'video 2' } })
if (server.serverNumber === 2) videoServer2UUID = uuid
await server.users.updateMyAvatar({ fixture: 'avatar.png' })
@ -112,7 +123,7 @@ describe('Test prune storage scripts', function () {
})
it('Should have the files on the disk', async function () {
await assertCountAreOkay(servers)
await assertCountAreOkay(servers, videoServer2UUID)
})
it('Should create some dirty files', async function () {
@ -176,6 +187,28 @@ describe('Test prune storage scripts', function () {
badNames['avatars'] = [ n1, n2 ]
}
// When we'll prune HLS directories too
// {
// const directory = join('streaming-playlists', 'hls')
// const base = servers[1].servers.buildDirectory(directory)
// const n1 = buildUUID()
// await createFile(join(base, n1))
// badNames[directory] = [ n1 ]
// }
// {
// const directory = join('streaming-playlists', 'hls', videoServer2UUID)
// const base = servers[1].servers.buildDirectory(directory)
// const n1 = buildUUID() + '-240-fragmented-.mp4'
// const n2 = buildUUID() + '-master.m3u8'
// await createFile(join(base, n1))
// await createFile(join(base, n2))
// badNames[directory] = [ n1, n2 ]
// }
}
})
@ -187,7 +220,7 @@ describe('Test prune storage scripts', function () {
})
it('Should have removed files', async function () {
await assertCountAreOkay(servers)
await assertCountAreOkay(servers, videoServer2UUID)
for (const directory of Object.keys(badNames)) {
for (const name of badNames[directory]) {

11
server/tests/cli/update-host.ts
View File

@ -108,21 +108,22 @@ describe('Test update host scripts', function () {
for (const video of data) {
const videoDetails = await server.videos.get({ id: video.id })
const files = videoDetails.files.concat(videoDetails.streamingPlaylists[0].files)
expect(videoDetails.files).to.have.lengthOf(4)
expect(files).to.have.lengthOf(8)
for (const file of videoDetails.files) {
for (const file of files) {
expect(file.magnetUri).to.contain('localhost%3A9002%2Ftracker%2Fsocket')
expect(file.magnetUri).to.contain('localhost%3A9002%2Fstatic%2Fwebseed%2F')
expect(file.magnetUri).to.contain('localhost%3A9002%2Fstatic%2F')
const torrent = await parseTorrentVideo(server, videoDetails.uuid, file.resolution.id)
const torrent = await parseTorrentVideo(server, file)
const announceWS = torrent.announce.find(a => a === 'ws://localhost:9002/tracker/socket')
expect(announceWS).to.not.be.undefined
const announceHttp = torrent.announce.find(a => a === 'http://localhost:9002/tracker/announce')
expect(announceHttp).to.not.be.undefined
expect(torrent.urlList[0]).to.contain('http://localhost:9002/static/webseed')
expect(torrent.urlList[0]).to.contain('http://localhost:9002/static/')
}
}
})

5
server/tests/plugins/plugin-transcoding.ts
View File

@ -2,7 +2,6 @@
import 'mocha'
import { expect } from 'chai'
import { join } from 'path'
import { getAudioStream, getVideoFileFPS, getVideoStreamFromFile } from '@server/helpers/ffprobe-utils'
import {
cleanupTests,
@ -247,7 +246,9 @@ describe('Test transcoding plugins', function () {
const videoUUID = (await server.videos.quickUpload({ name: 'video', fixture: 'video_very_short_240p.mp4' })).uuid
await waitJobs([ server ])
const path = server.servers.buildDirectory(join('videos', videoUUID + '-240.mp4'))
const video = await server.videos.get({ id: videoUUID })
const path = server.servers.buildWebTorrentFilePath(video.files[0].fileUrl)
const audioProbe = await getAudioStream(path)
expect(audioProbe.audioStream.codec_name).to.equal('opus')

2
server/types/models/video/video-streaming-playlist.ts
View File

@ -39,5 +39,5 @@ export type MStreamingPlaylistRedundanciesOpt =
PickWithOpt<VideoStreamingPlaylistModel, 'RedundancyVideos', MVideoRedundancyFileUrl[]>
export function isStreamingPlaylist (value: MVideo | MStreamingPlaylistVideo): value is MStreamingPlaylistVideo {
return !!(value as MStreamingPlaylist).playlistUrl
return !!(value as MStreamingPlaylist).videoId
}

4
shared/core-utils/miscs/regexp.ts
View File

@ -1 +1,5 @@
export const uuidRegex = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
export function removeFragmentedMP4Ext (path: string) {
return path.replace(/-fragmented.mp4$/i, '')
}

7
shared/extra-utils/miscs/webtorrent.ts
View File

@ -1,7 +1,8 @@
import { readFile } from 'fs-extra'
import * as parseTorrent from 'parse-torrent'
import { join } from 'path'
import { basename, join } from 'path'
import * as WebTorrent from 'webtorrent'
import { VideoFile } from '@shared/models'
import { PeerTubeServer } from '../server'
let webtorrent: WebTorrent.Instance
@ -15,8 +16,8 @@ function webtorrentAdd (torrent: string, refreshWebTorrent = false) {
return new Promise<WebTorrent.Torrent>(res => webtorrent.add(torrent, res))
}
async function parseTorrentVideo (server: PeerTubeServer, videoUUID: string, resolution: number) {
const torrentName = videoUUID + '-' + resolution + '.torrent'
async function parseTorrentVideo (server: PeerTubeServer, file: VideoFile) {
const torrentName = basename(file.torrentUrl)
const torrentPath = server.servers.buildDirectory(join('torrents', torrentName))
const data = await readFile(torrentPath)

3
shared/extra-utils/server/servers-command.ts
View File

@ -1,7 +1,6 @@
import { exec } from 'child_process'
import { copy, ensureDir, readFile, remove } from 'fs-extra'
import { join } from 'path'
import { basename } from 'path/posix'
import { basename, join } from 'path'
import { root } from '@server/helpers/core-utils'
import { HttpStatusCode } from '@shared/models'
import { getFileSize, isGithubCI, wait } from '../miscs'

18
shared/extra-utils/videos/live.ts
View File

@ -76,7 +76,7 @@ async function waitUntilLivePublishedOnAllServers (servers: PeerTubeServer[], vi
}
}
async function checkLiveCleanup (server: PeerTubeServer, videoUUID: string, resolutions: number[] = []) {
async function checkLiveCleanupAfterSave (server: PeerTubeServer, videoUUID: string, resolutions: number[] = []) {
const basePath = server.servers.buildDirectory('streaming-playlists')
const hlsPath = join(basePath, 'hls', videoUUID)
@ -93,12 +93,18 @@ async function checkLiveCleanup (server: PeerTubeServer, videoUUID: string, reso
expect(files).to.have.lengthOf(resolutions.length * 2 + 2)
for (const resolution of resolutions) {
expect(files).to.contain(`${videoUUID}-${resolution}-fragmented.mp4`)
expect(files).to.contain(`${resolution}.m3u8`)
const fragmentedFile = files.find(f => f.endsWith(`-${resolution}-fragmented.mp4`))
expect(fragmentedFile).to.exist
const playlistFile = files.find(f => f.endsWith(`${resolution}.m3u8`))
expect(playlistFile).to.exist
}
expect(files).to.contain('master.m3u8')
expect(files).to.contain('segments-sha256.json')
const masterPlaylistFile = files.find(f => f.endsWith('-master.m3u8'))
expect(masterPlaylistFile).to.exist
const shaFile = files.find(f => f.endsWith('-segments-sha256.json'))
expect(shaFile).to.exist
}
export {
@ -107,5 +113,5 @@ export {
testFfmpegStreamError,
stopFfmpeg,
waitUntilLivePublishedOnAllServers,
checkLiveCleanup
checkLiveCleanupAfterSave
}

5
shared/extra-utils/videos/streaming-playlists.ts
View File

@ -1,6 +1,7 @@
import { expect } from 'chai'
import { basename } from 'path'
import { sha256 } from '@server/helpers/core-utils'
import { removeFragmentedMP4Ext } from '@shared/core-utils'
import { HttpStatusCode, VideoStreamingPlaylist } from '@shared/models'
import { PeerTubeServer } from '../server'
@ -15,11 +16,11 @@ async function checkSegmentHash (options: {
const { server, baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist } = options
const command = server.streamingPlaylists
const playlist = await command.get({ url: `${baseUrlPlaylist}/${videoUUID}/${resolution}.m3u8` })
const file = hlsPlaylist.files.find(f => f.resolution.id === resolution)
const videoName = basename(file.fileUrl)
const playlist = await command.get({ url: `${baseUrlPlaylist}/${videoUUID}/${removeFragmentedMP4Ext(videoName)}.m3u8` })
const matches = /#EXT-X-BYTERANGE:(\d+)@(\d+)/.exec(playlist)
const length = parseInt(matches[1], 10)