Merge branch 'release/4.2.0' into develop

pull/5004/head
Chocobozzz 2022-05-25 15:22:28 +02:00
commit ab623c0e0b
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
13 changed files with 173 additions and 71 deletions

View File

@ -65,7 +65,7 @@
<span class="badge" [ngClass]="getJobStateClass(job.state)">{{ job.state }}</span>
</td>
<td class="job-progress c-hand" [pRowToggler]="job">
<td *ngIf="hasGlobalProgress()" class="job-progress c-hand" [pRowToggler]="job">
<ng-container *ngIf="hasProgress(job)">{{ getProgress(job) }}</ng-container>
</td>

View File

@ -16,10 +16,12 @@
}
.actor-img-edit-button {
@include peertube-button-file(21px);
@include button-with-icon(19px);
@include peertube-button-file(30px);
@include orange-button;
display: flex;
justify-content: center;
padding: 0;
margin-top: 10px;
margin-bottom: 5px;
cursor: pointer;
@ -30,6 +32,6 @@
}
my-global-icon {
right: 7px;
width: 19px;
}
}

View File

@ -21,11 +21,11 @@ export abstract class Actor implements ServerActor {
isLocal: boolean
static GET_ACTOR_AVATAR_URL (actor: { avatars: { width: number, url?: string, path: string }[] }, size?: number) {
const avatars = actor.avatars.sort((a, b) => b.width - a.width)
const avatarsAscWidth = actor.avatars.sort((a, b) => a.width - b.width)
const avatar = size
? avatars.find(a => a.width >= size)
: avatars[0]
? avatarsAscWidth.find(a => a.width >= size)
: avatarsAscWidth[avatarsAscWidth.length - 1] // Bigger one
if (!avatar) return ''
if (avatar.url) return avatar.url

View File

@ -1,10 +1,10 @@
import { Job } from 'bull'
import { pathExists, readdir, remove } from 'fs-extra'
import { readdir, remove } from 'fs-extra'
import { join } from 'path'
import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamDuration } from '@server/helpers/ffmpeg'
import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
import { cleanupLive, LiveSegmentShaStore } from '@server/lib/live'
import { cleanupNormalLive, cleanupPermanentLive, cleanupTMPLiveFiles, LiveSegmentShaStore } from '@server/lib/live'
import {
generateHLSMasterPlaylistFilename,
generateHlsSha256SegmentsFilename,
@ -45,13 +45,13 @@ async function processVideoLiveEnding (job: Job) {
LiveSegmentShaStore.Instance.cleanupShaSegments(liveVideo.uuid)
if (live.saveReplay !== true) {
return cleanupLiveAndFederate({ liveVideo })
return cleanupLiveAndFederate({ live, video: liveVideo })
}
if (live.permanentLive) {
await saveReplayToExternalVideo({ liveVideo, liveSession, publishedAt: payload.publishedAt, replayDirectory: payload.replayDirectory })
return cleanupLiveAndFederate({ liveVideo })
return cleanupLiveAndFederate({ live, video: liveVideo })
}
return replaceLiveByReplay({ liveVideo, live, liveSession, replayDirectory: payload.replayDirectory })
@ -164,7 +164,11 @@ async function replaceLiveByReplay (options: {
await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory })
await remove(getLiveReplayBaseDirectory(videoWithFiles))
if (live.permanentLive) { // Remove session replay
await remove(replayDirectory)
} else { // We won't stream again in this live, we can delete the base replay directory
await remove(getLiveReplayBaseDirectory(videoWithFiles))
}
// Regenerate the thumbnail & preview?
if (videoWithFiles.getMiniature().automaticallyGenerated === true) {
@ -227,34 +231,23 @@ async function assignReplayFilesToVideo (options: {
}
async function cleanupLiveAndFederate (options: {
liveVideo: MVideo
live: MVideoLive
video: MVideo
}) {
const { liveVideo } = options
const { live, video } = options
const streamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(liveVideo.id)
await cleanupLive(liveVideo, streamingPlaylist)
const streamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
const fullVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(liveVideo.id)
return federateVideoIfNeeded(fullVideo, false, undefined)
}
if (live.permanentLive) {
await cleanupPermanentLive(video, streamingPlaylist)
} else {
await cleanupNormalLive(video, streamingPlaylist)
}
async function cleanupTMPLiveFiles (hlsDirectory: string) {
if (!await pathExists(hlsDirectory)) return
const files = await readdir(hlsDirectory)
for (const filename of files) {
if (
filename.endsWith('.ts') ||
filename.endsWith('.m3u8') ||
filename.endsWith('.mpd') ||
filename.endsWith('.m4s') ||
filename.endsWith('.tmp')
) {
const p = join(hlsDirectory, filename)
remove(p)
.catch(err => logger.error('Cannot remove %s.', p, { err }))
}
try {
const fullVideo = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.id)
return federateVideoIfNeeded(fullVideo, false, undefined)
} catch (err) {
logger.warn('Cannot federate live after cleanup', { videoId: video.id, err })
}
}

View File

@ -28,7 +28,7 @@ import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, g
import { PeerTubeSocket } from '../peertube-socket'
import { LiveQuotaStore } from './live-quota-store'
import { LiveSegmentShaStore } from './live-segment-sha-store'
import { cleanupLive } from './live-utils'
import { cleanupPermanentLive } from './live-utils'
import { MuxingSession } from './shared'
const NodeRtmpSession = require('node-media-server/src/node_rtmp_session')
@ -224,7 +224,9 @@ class LiveManager {
const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id)
if (oldStreamingPlaylist) {
await cleanupLive(video, oldStreamingPlaylist)
if (!videoLive.permanentLive) throw new Error('Found previous session in a non permanent live: ' + video.uuid)
await cleanupPermanentLive(video, oldStreamingPlaylist)
}
this.videoSessions.set(video.id, sessionId)

View File

@ -1,5 +1,6 @@
import { remove } from 'fs-extra'
import { basename } from 'path'
import { pathExists, readdir, remove } from 'fs-extra'
import { basename, join } from 'path'
import { logger } from '@server/helpers/logger'
import { MStreamingPlaylist, MVideo } from '@server/types/models'
import { getLiveDirectory } from '../paths'
@ -9,7 +10,15 @@ function buildConcatenatedName (segmentOrPlaylistPath: string) {
return 'concat-' + num[1] + '.ts'
}
async function cleanupLive (video: MVideo, streamingPlaylist?: MStreamingPlaylist) {
async function cleanupPermanentLive (video: MVideo, streamingPlaylist?: MStreamingPlaylist) {
const hlsDirectory = getLiveDirectory(video)
await cleanupTMPLiveFiles(hlsDirectory)
if (streamingPlaylist) await streamingPlaylist.destroy()
}
async function cleanupNormalLive (video: MVideo, streamingPlaylist?: MStreamingPlaylist) {
const hlsDirectory = getLiveDirectory(video)
await remove(hlsDirectory)
@ -17,7 +26,30 @@ async function cleanupLive (video: MVideo, streamingPlaylist?: MStreamingPlaylis
if (streamingPlaylist) await streamingPlaylist.destroy()
}
async function cleanupTMPLiveFiles (hlsDirectory: string) {
if (!await pathExists(hlsDirectory)) return
const files = await readdir(hlsDirectory)
for (const filename of files) {
if (
filename.endsWith('.ts') ||
filename.endsWith('.m3u8') ||
filename.endsWith('.mpd') ||
filename.endsWith('.m4s') ||
filename.endsWith('.tmp')
) {
const p = join(hlsDirectory, filename)
remove(p)
.catch(err => logger.error('Cannot remove %s.', p, { err }))
}
}
}
export {
cleanupLive,
cleanupPermanentLive,
cleanupNormalLive,
cleanupTMPLiveFiles,
buildConcatenatedName
}

View File

@ -150,8 +150,8 @@ class MuxingSession extends EventEmitter {
logger.info('Running live muxing/transcoding for %s.', this.videoUUID, this.lTags())
this.watchTSFiles(this.outDirectory)
this.watchMasterFile(this.outDirectory)
this.watchTSFiles()
this.watchMasterFile()
let ffmpegShellCommand: string
this.ffmpegCommand.on('start', cmdline => {
@ -161,13 +161,13 @@ class MuxingSession extends EventEmitter {
})
this.ffmpegCommand.on('error', (err, stdout, stderr) => {
this.onFFmpegError({ err, stdout, stderr, outPath: this.outDirectory, ffmpegShellCommand })
this.onFFmpegError({ err, stdout, stderr, ffmpegShellCommand })
})
this.ffmpegCommand.on('end', () => {
this.emit('ffmpeg-end', ({ videoId: this.videoId }))
this.onFFmpegEnded(this.outDirectory)
this.onFFmpegEnded()
})
this.ffmpegCommand.run()
@ -189,12 +189,11 @@ class MuxingSession extends EventEmitter {
err: any
stdout: string
stderr: string
outPath: string
ffmpegShellCommand: string
}) {
const { err, stdout, stderr, outPath, ffmpegShellCommand } = options
const { err, stdout, stderr, ffmpegShellCommand } = options
this.onFFmpegEnded(outPath)
this.onFFmpegEnded()
// Don't care that we killed the ffmpeg process
if (err?.message?.includes('Exiting normally')) return
@ -204,7 +203,7 @@ class MuxingSession extends EventEmitter {
this.emit('ffmpeg-error', ({ videoId: this.videoId }))
}
private onFFmpegEnded (outPath: string) {
private onFFmpegEnded () {
logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.inputUrl, this.lTags())
setTimeout(() => {
@ -214,12 +213,12 @@ class MuxingSession extends EventEmitter {
.then(() => {
// Process remaining segments hash
for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) {
this.processSegments(outPath, this.segmentsToProcessPerPlaylist[key])
this.processSegments(this.segmentsToProcessPerPlaylist[key])
}
})
.catch(err => {
logger.error(
'Cannot close watchers of %s or process remaining hash segments.', outPath,
'Cannot close watchers of %s or process remaining hash segments.', this.outDirectory,
{ err, ...this.lTags() }
)
})
@ -228,21 +227,21 @@ class MuxingSession extends EventEmitter {
}, 1000)
}
private watchMasterFile (outPath: string) {
this.masterWatcher = watch(outPath + '/' + this.streamingPlaylist.playlistFilename)
private watchMasterFile () {
this.masterWatcher = watch(this.outDirectory + '/' + this.streamingPlaylist.playlistFilename)
this.masterWatcher.on('add', () => {
this.emit('master-playlist-created', { videoId: this.videoId })
this.masterWatcher.close()
.catch(err => logger.error('Cannot close master watcher of %s.', outPath, { err, ...this.lTags() }))
.catch(err => logger.error('Cannot close master watcher of %s.', this.outDirectory, { err, ...this.lTags() }))
})
}
private watchTSFiles (outPath: string) {
private watchTSFiles () {
const startStreamDateTime = new Date().getTime()
this.tsWatcher = watch(outPath + '/*.ts')
this.tsWatcher = watch(this.outDirectory + '/*.ts')
const playlistIdMatcher = /^([\d+])-/
@ -252,7 +251,7 @@ class MuxingSession extends EventEmitter {
const playlistId = basename(segmentPath).match(playlistIdMatcher)[0]
const segmentsToProcess = this.segmentsToProcessPerPlaylist[playlistId] || []
this.processSegments(outPath, segmentsToProcess)
this.processSegments(segmentsToProcess)
this.segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ]
@ -273,7 +272,7 @@ class MuxingSession extends EventEmitter {
}
}
const deleteHandler = segmentPath => LiveSegmentShaStore.Instance.removeSegmentSha(this.videoUUID, segmentPath)
const deleteHandler = (segmentPath: string) => LiveSegmentShaStore.Instance.removeSegmentSha(this.videoUUID, segmentPath)
this.tsWatcher.on('add', p => addHandler(p))
this.tsWatcher.on('unlink', p => deleteHandler(p))
@ -332,15 +331,15 @@ class MuxingSession extends EventEmitter {
return now <= max
}
private processSegments (hlsVideoPath: string, segmentPaths: string[]) {
private processSegments (segmentPaths: string[]) {
mapSeries(segmentPaths, async previousSegment => {
// Add sha hash of previous segments, because ffmpeg should have finished generating them
await LiveSegmentShaStore.Instance.addSegmentSha(this.videoUUID, previousSegment)
if (this.saveReplay) {
await this.addSegmentToReplay(hlsVideoPath, previousSegment)
await this.addSegmentToReplay(previousSegment)
}
}).catch(err => logger.error('Cannot process segments in %s', hlsVideoPath, { err, ...this.lTags() }))
}).catch(err => logger.error('Cannot process segments', { err, ...this.lTags() }))
}
private hasClientSocketInBadHealth (sessionId: string) {
@ -367,7 +366,7 @@ class MuxingSession extends EventEmitter {
return false
}
private async addSegmentToReplay (hlsVideoPath: string, segmentPath: string) {
private async addSegmentToReplay (segmentPath: string) {
const segmentName = basename(segmentPath)
const dest = join(this.replayDirectory, buildConcatenatedName(segmentName))

View File

@ -138,6 +138,9 @@ export class ActorImageModel extends Model<Partial<AttributesOnly<ActorImageMode
case ActorImageType.BANNER:
return join(LAZY_STATIC_PATHS.BANNERS, this.filename)
default:
throw new Error('Unknown actor image type: ' + this.type)
}
}

View File

@ -86,6 +86,7 @@ export class UserNotificationListQueryBuilder extends AbstractRunQuery {
"Video->VideoChannel->Actor"."preferredUsername" AS "Video.VideoChannel.Actor.preferredUsername",
"Video->VideoChannel->Actor->Avatars"."id" AS "Video.VideoChannel.Actor.Avatars.id",
"Video->VideoChannel->Actor->Avatars"."width" AS "Video.VideoChannel.Actor.Avatars.width",
"Video->VideoChannel->Actor->Avatars"."type" AS "Video.VideoChannel.Actor.Avatars.type",
"Video->VideoChannel->Actor->Avatars"."filename" AS "Video.VideoChannel.Actor.Avatars.filename",
"Video->VideoChannel->Actor->Server"."id" AS "Video.VideoChannel.Actor.Server.id",
"Video->VideoChannel->Actor->Server"."host" AS "Video.VideoChannel.Actor.Server.host",
@ -97,6 +98,7 @@ export class UserNotificationListQueryBuilder extends AbstractRunQuery {
"VideoComment->Account->Actor"."preferredUsername" AS "VideoComment.Account.Actor.preferredUsername",
"VideoComment->Account->Actor->Avatars"."id" AS "VideoComment.Account.Actor.Avatars.id",
"VideoComment->Account->Actor->Avatars"."width" AS "VideoComment.Account.Actor.Avatars.width",
"VideoComment->Account->Actor->Avatars"."type" AS "VideoComment.Account.Actor.Avatars.type",
"VideoComment->Account->Actor->Avatars"."filename" AS "VideoComment.Account.Actor.Avatars.filename",
"VideoComment->Account->Actor->Server"."id" AS "VideoComment.Account.Actor.Server.id",
"VideoComment->Account->Actor->Server"."host" AS "VideoComment.Account.Actor.Server.host",
@ -127,6 +129,7 @@ export class UserNotificationListQueryBuilder extends AbstractRunQuery {
"Abuse->FlaggedAccount->Actor"."preferredUsername" AS "Abuse.FlaggedAccount.Actor.preferredUsername",
"Abuse->FlaggedAccount->Actor->Avatars"."id" AS "Abuse.FlaggedAccount.Actor.Avatars.id",
"Abuse->FlaggedAccount->Actor->Avatars"."width" AS "Abuse.FlaggedAccount.Actor.Avatars.width",
"Abuse->FlaggedAccount->Actor->Avatars"."type" AS "Abuse.FlaggedAccount.Actor.Avatars.type",
"Abuse->FlaggedAccount->Actor->Avatars"."filename" AS "Abuse.FlaggedAccount.Actor.Avatars.filename",
"Abuse->FlaggedAccount->Actor->Server"."id" AS "Abuse.FlaggedAccount.Actor.Server.id",
"Abuse->FlaggedAccount->Actor->Server"."host" AS "Abuse.FlaggedAccount.Actor.Server.host",
@ -155,6 +158,7 @@ export class UserNotificationListQueryBuilder extends AbstractRunQuery {
"ActorFollow->ActorFollower->Account"."name" AS "ActorFollow.ActorFollower.Account.name",
"ActorFollow->ActorFollower->Avatars"."id" AS "ActorFollow.ActorFollower.Avatars.id",
"ActorFollow->ActorFollower->Avatars"."width" AS "ActorFollow.ActorFollower.Avatars.width",
"ActorFollow->ActorFollower->Avatars"."type" AS "ActorFollow.ActorFollower.Avatars.type",
"ActorFollow->ActorFollower->Avatars"."filename" AS "ActorFollow.ActorFollower.Avatars.filename",
"ActorFollow->ActorFollower->Server"."id" AS "ActorFollow.ActorFollower.Server.id",
"ActorFollow->ActorFollower->Server"."host" AS "ActorFollow.ActorFollower.Server.host",
@ -173,6 +177,7 @@ export class UserNotificationListQueryBuilder extends AbstractRunQuery {
"Account->Actor"."preferredUsername" AS "Account.Actor.preferredUsername",
"Account->Actor->Avatars"."id" AS "Account.Actor.Avatars.id",
"Account->Actor->Avatars"."width" AS "Account.Actor.Avatars.width",
"Account->Actor->Avatars"."type" AS "Account.Actor.Avatars.type",
"Account->Actor->Avatars"."filename" AS "Account.Actor.Avatars.filename",
"Account->Actor->Server"."id" AS "Account.Actor.Server.id",
"Account->Actor->Server"."host" AS "Account.Actor.Server.host"`

View File

@ -441,6 +441,40 @@ describe('Save replay setting', function () {
await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
await checkLiveCleanup(servers[0], liveVideoUUID, [])
})
it('Should correctly save replays with multiple sessions', async function () {
this.timeout(120000)
liveVideoUUID = await createLiveWrapper({ permanent: true, replay: true })
await waitJobs(servers)
// Streaming session #1
ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
await stopFfmpeg(ffmpegCommand)
await servers[0].live.waitUntilWaiting({ videoId: liveVideoUUID })
// Streaming session #2
ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
await stopFfmpeg(ffmpegCommand)
await waitUntilLiveWaitingOnAllServers(servers, liveVideoUUID)
// Wait for replays
await waitJobs(servers)
const { total, data: sessions } = await servers[0].live.listSessions({ videoId: liveVideoUUID })
expect(total).to.equal(2)
expect(sessions).to.have.lengthOf(2)
for (const session of sessions) {
expect(session.error).to.be.null
expect(session.replayVideo).to.exist
await servers[0].videos.get({ id: session.replayVideo.uuid })
}
})
})
after(async function () {

View File

@ -18,7 +18,7 @@ import {
const expect = chai.expect
describe('Test live', function () {
describe('Test live socket messages', function () {
let servers: PeerTubeServer[] = []
before(async function () {

View File

@ -3,15 +3,35 @@
import { expect } from 'chai'
import { pathExists, readdir } from 'fs-extra'
import { join } from 'path'
import { LiveVideo } from '@shared/models'
import { PeerTubeServer } from '@shared/server-commands'
async function checkLiveCleanup (server: PeerTubeServer, videoUUID: string, savedResolutions: number[] = []) {
let live: LiveVideo
try {
live = await server.live.get({ videoId: videoUUID })
} catch {}
const basePath = server.servers.buildDirectory('streaming-playlists')
const hlsPath = join(basePath, 'hls', videoUUID)
if (savedResolutions.length === 0) {
const result = await pathExists(hlsPath)
expect(result).to.be.false
if (live?.permanentLive) {
expect(await pathExists(hlsPath)).to.be.true
const hlsFiles = await readdir(hlsPath)
expect(hlsFiles).to.have.lengthOf(1) // Only replays directory
const replayDir = join(hlsPath, 'replay')
expect(await pathExists(replayDir)).to.be.true
const replayFiles = await readdir(join(hlsPath, 'replay'))
expect(replayFiles).to.have.lengthOf(0)
} else {
expect(await pathExists(hlsPath)).to.be.false
}
return
}

View File

@ -185,7 +185,7 @@ async function checkUserRegistered (options: CheckerBaseParams & {
expect(notification).to.not.be.undefined
expect(notification.type).to.equal(notificationType)
checkActor(notification.account)
checkActor(notification.account, { withAvatar: false })
expect(notification.account.name).to.equal(username)
} else {
expect(notification).to.satisfy(n => n.type !== notificationType || n.account.name !== username)
@ -253,7 +253,7 @@ async function checkNewInstanceFollower (options: CheckerBaseParams & {
expect(notification).to.not.be.undefined
expect(notification.type).to.equal(notificationType)
checkActor(notification.actorFollow.follower)
checkActor(notification.actorFollow.follower, { withAvatar: false })
expect(notification.actorFollow.follower.name).to.equal('peertube')
expect(notification.actorFollow.follower.host).to.equal(followerHost)
@ -288,7 +288,8 @@ async function checkAutoInstanceFollowing (options: CheckerBaseParams & {
expect(notification.type).to.equal(notificationType)
const following = notification.actorFollow.following
checkActor(following)
checkActor(following, { withAvatar: false })
expect(following.name).to.equal('peertube')
expect(following.host).to.equal(followingHost)
@ -701,6 +702,9 @@ async function prepareNotificationsTest (serversCount = 3, overrideConfigArg: an
const userAccessToken = await servers[0].login.getAccessToken(user)
await servers[0].notifications.updateMySettings({ token: userAccessToken, settings: getAllNotificationsSettings() })
await servers[0].users.updateMyAvatar({ token: userAccessToken, fixture: 'avatar.png' })
await servers[0].channels.updateImage({ channelName: 'user_1_channel', token: userAccessToken, fixture: 'avatar.png', type: 'avatar' })
await servers[0].notifications.updateMySettings({ settings: getAllNotificationsSettings() })
if (serversCount > 1) {
@ -832,10 +836,18 @@ function checkVideo (video: any, videoName?: string, shortUUID?: string) {
expect(video.id).to.be.a('number')
}
function checkActor (actor: any) {
function checkActor (actor: any, options: { withAvatar?: boolean } = {}) {
const { withAvatar = true } = options
expect(actor.displayName).to.be.a('string')
expect(actor.displayName).to.not.be.empty
expect(actor.host).to.not.be.undefined
if (withAvatar) {
expect(actor.avatars).to.be.an('array')
expect(actor.avatars).to.have.lengthOf(2)
expect(actor.avatars[0].path).to.exist.and.not.empty
}
}
function checkComment (comment: any, commentId: number, threadId: number) {