OpenCloud
/
PeerTube
mirror of https://github.com/Chocobozzz/PeerTube
1
0
Fork 0
Code Issues Releases Wiki Activity

Improve real world script

pull/97/head
Chocobozzz 2017-09-12 12:53:55 +02:00
parent 7ca86c864e
commit 6d33593a08
6 changed files with 111 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
client/src/app/videos/shared/video.model.ts
View File

@ -5,6 +5,7 @@ export class Video implements VideoServerModel {
author: string
by: string
createdAt: Date
updatedAt: Date
categoryLabel: string
category: number
licenceLabel: string

30
server/controllers/api/remote/videos.ts
View File

@ -17,7 +17,7 @@ import {
} from '../../../middlewares'
import { logger, retryTransactionWrapper } from '../../../helpers'
import { quickAndDirtyUpdatesVideoToFriends } from '../../../lib'
import { PodInstance } from '../../../models'
import { PodInstance, VideoFileInstance } from '../../../models'
import {
RemoteVideoRequest,
RemoteVideoCreateData,
@ -81,7 +81,7 @@ function remoteVideos (req: express.Request, res: express.Response, next: expres
// Get the function we need to call in order to process the request
const fun = functionsHash[request.type]
if (fun === undefined) {
logger.error('Unkown remote request type %s.', request.type)
logger.error('Unknown remote request type %s.', request.type)
return
}
@ -176,7 +176,7 @@ function processVideosEvents (eventData: RemoteVideoEventData, fromPod: PodInsta
return quickAndDirtyUpdatesVideoToFriends(qadusParams, t)
})
})
.then(() => logger.info('Remote video event processed for video %s.', eventData.uuid))
.then(() => logger.info('Remote video event processed for video with uuid %s.', eventData.uuid))
.catch(err => {
logger.debug('Cannot process a video event.', err)
throw err
@ -193,14 +193,14 @@ function quickAndDirtyUpdateVideoRetryWrapper (videoData: RemoteQaduVideoData, f
}
function quickAndDirtyUpdateVideo (videoData: RemoteQaduVideoData, fromPod: PodInstance) {
let videoName
let videoUUID = ''
return db.sequelize.transaction(t => {
return fetchVideoByHostAndUUID(fromPod.host, videoData.uuid)
.then(videoInstance => {
const options = { transaction: t }
videoName = videoInstance.name
videoUUID = videoInstance.uuid
if (videoData.views) {
videoInstance.set('views', videoData.views)
@ -217,7 +217,7 @@ function quickAndDirtyUpdateVideo (videoData: RemoteQaduVideoData, fromPod: PodI
return videoInstance.save(options)
})
})
.then(() => logger.info('Remote video %s quick and dirty updated', videoName))
.then(() => logger.info('Remote video with uuid %s quick and dirty updated', videoUUID))
.catch(err => logger.debug('Cannot quick and dirty update the remote video.', err))
}
@ -315,7 +315,7 @@ function addRemoteVideo (videoToCreateData: RemoteVideoCreateData, fromPod: PodI
return videoCreated.setTags(tagInstances, options)
})
})
.then(() => logger.info('Remote video %s inserted.', videoToCreateData.name))
.then(() => logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid))
.catch(err => {
logger.debug('Cannot insert the remote video.', err)
throw err
@ -361,7 +361,17 @@ function updateRemoteVideo (videoAttributesToUpdate: RemoteVideoUpdateData, from
return videoInstance.save(options).then(() => ({ videoInstance, tagInstances }))
})
.then(({ tagInstances, videoInstance }) => {
const tasks = []
const tasks: Promise<void>[] = []
// Remove old video files
videoInstance.VideoFiles.forEach(videoFile => {
tasks.push(videoFile.destroy())
})
return Promise.all(tasks).then(() => ({ tagInstances, videoInstance }))
})
.then(({ tagInstances, videoInstance }) => {
const tasks: Promise<VideoFileInstance>[] = []
const options = {
transaction: t
}
@ -386,7 +396,7 @@ function updateRemoteVideo (videoAttributesToUpdate: RemoteVideoUpdateData, from
return videoInstance.setTags(tagInstances, options)
})
})
.then(() => logger.info('Remote video %s updated', videoAttributesToUpdate.name))
.then(() => logger.info('Remote video with uuid %s updated', videoAttributesToUpdate.uuid))
.catch(err => {
// This is just a debug because we will retry the insert
logger.debug('Cannot update the remote video.', err)
@ -398,7 +408,7 @@ function removeRemoteVideo (videoToRemoveData: RemoteVideoRemoveData, fromPod: P
// We need the instance because we have to remove some other stuffs (thumbnail etc)
return fetchVideoByHostAndUUID(fromPod.host, videoToRemoveData.uuid)
.then(video => {
logger.debug('Removing remote video %s.', video.uuid)
logger.debug('Removing remote video with uuid %s.', video.uuid)
return video.destroy()
})
.catch(err => {

11
server/controllers/api/videos/index.ts
View File

@ -157,6 +157,7 @@ function addVideoRetryWrapper (req: express.Request, res: express.Response, next
function addVideo (req: express.Request, res: express.Response, videoPhysicalFile: Express.Multer.File) {
const videoInfo: VideoCreate = req.body
let videoUUID = ''
return db.sequelize.transaction(t => {
const user = res.locals.oauth.token.User
@ -241,6 +242,7 @@ function addVideo (req: express.Request, res: express.Response, videoPhysicalFil
.then(videoCreated => {
// Do not forget to add Author information to the created video
videoCreated.Author = author
videoUUID = videoCreated.uuid
return { tagInstances, video: videoCreated, videoFile }
})
@ -274,7 +276,7 @@ function addVideo (req: express.Request, res: express.Response, videoPhysicalFil
})
})
})
.then(() => logger.info('Video with name %s created.', videoInfo.name))
.then(() => logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoUUID))
.catch((err: Error) => {
logger.debug('Cannot insert the video.', err)
throw err
@ -342,7 +344,7 @@ function updateVideo (req: express.Request, res: express.Response) {
})
})
.then(() => {
logger.info('Video with name %s updated.', videoInstance.name)
logger.info('Video with name %s and uuid %s updated.', videoInstance.name, videoInstance.uuid)
})
.catch(err => {
logger.debug('Cannot update the video.', err)
@ -398,7 +400,10 @@ function removeVideo (req: express.Request, res: express.Response, next: express
const videoInstance = res.locals.video
videoInstance.destroy()
.then(() => res.type('json').status(204).end())
.then(() => {
logger.info('Video with name %s and uuid %s deleted.', videoInstance.name, videoInstance.uuid)
res.type('json').status(204).end()
})
.catch(err => {
logger.error('Errors when removed the video.', err)
return next(err)

16
server/middlewares/validators/videos.ts
View File

@ -109,8 +109,6 @@ function videosRemoveValidator (req: express.Request, res: express.Response, nex
checkErrors(req, res, () => {
checkVideoExists(req.params.id, res, () => {
// We need to make additional checks
// Check if the user who did the request is able to delete the video
checkUserCanDeleteVideo(res.locals.oauth.token.User.id, res, () => {
next()
@ -205,17 +203,15 @@ function checkUserCanDeleteVideo (userId: number, res: express.Response, callbac
// Retrieve the user who did the request
db.User.loadById(userId)
.then(user => {
if (res.locals.video.isOwned() === false) {
return res.status(403).send('Cannot remove video of another pod, blacklist it')
}
// Check if the user can delete the video
// The user can delete it if s/he is an admin
// Or if s/he is the video's author
if (user.isAdmin() === false) {
if (res.locals.video.isOwned() === false) {
return res.status(403).send('Cannot remove video of another pod')
}
if (res.locals.video.Author.userId !== res.locals.oauth.token.User.id) {
return res.status(403).send('Cannot remove video of another user')
}
if (user.isAdmin() === false && res.locals.video.Author.userId !== res.locals.oauth.token.User.id) {
return res.status(403).send('Cannot remove video of another user')
}
// If we reach this comment, we can delete the video

102
server/tests/real-world/real-world.ts
View File

@ -1,11 +1,10 @@
import * as program from 'commander'
import { isEqual, differenceWith } from 'lodash'
// /!\ Before imports /!\
process.env.NODE_ENV = 'test'
import { REQUESTS_INTERVAL } from '../../initializers/constants'
import { Video, VideoRateType } from '../../../shared'
import { Video, VideoRateType, VideoFile } from '../../../shared'
import {
ServerInfo as DefaultServerInfo,
flushAndRunMultipleServers,
@ -121,12 +120,14 @@ async function start () {
checking = true
const waitingInterval = setInterval(async () => {
const awaitingRequests = await isThereAwaitingRequests(servers)
if (awaitingRequests === true) {
console.log('A server has awaiting requests, waiting...')
const pendingRequests = await isTherePendingRequests(servers)
if (pendingRequests === true) {
console.log('A server has pending requests, waiting...')
return
}
// Even if there are no pending request, wait some potential processes
await wait(2000)
await checkIntegrity(servers)
initializeRequestsPerServer(servers)
@ -212,7 +213,7 @@ async function update (servers: ServerInfo[], numServer: number) {
async function remove (servers: ServerInfo[], numServer: number) {
const res = await getVideosList(servers[numServer].url)
const videos = res.body.data
const videos = res.body.data.filter(video => video.isLocal === true)
if (videos.length === 0) return undefined
const toRemove = videos[getRandomInt(0, videos.length)].id
@ -259,19 +260,7 @@ async function checkIntegrity (servers: ServerInfo[]) {
// Fetch all videos and remove some fields that can differ between pods
for (const server of servers) {
const p = getAllVideosListBy(server.url).then(res => {
const serverVideos = res.body.data
for (const serverVideo of serverVideos) {
delete serverVideo.id
delete serverVideo.isLocal
delete serverVideo.thumbnailPath
delete serverVideo.updatedAt
delete serverVideo.views
}
videos.push(serverVideos)
})
const p = getAllVideosListBy(server.url).then(res => videos.push(res.body.data))
tasks.push(p)
}
@ -279,12 +268,12 @@ async function checkIntegrity (servers: ServerInfo[]) {
let i = 0
for (const video of videos) {
if (!isEqual(video, videos[0])) {
const differences = areDifferences(video, videos[0])
if (differences !== undefined) {
console.error('Integrity not ok with server %d!', i + 1)
if (displayDiffOnFail) {
console.log(differenceWith(videos[0], video, isEqual))
console.log(differenceWith(video, videos[0], isEqual))
console.log(differences)
}
process.exit(-1)
@ -296,15 +285,74 @@ async function checkIntegrity (servers: ServerInfo[]) {
console.log('Integrity ok.')
}
function areDifferences (videos1: Video[], videos2: Video[]) {
// Remove some keys we don't want to compare
videos1.concat(videos2).forEach(video => {
delete video.id
delete video.isLocal
delete video.thumbnailPath
delete video.updatedAt
delete video.views
})
if (videos1.length !== videos2.length) {
return `Videos length are different (${videos1.length}/${videos2.length}).`
}
for (const video1 of videos1) {
const video2 = videos2.find(video => video.uuid === video1.uuid)
if (!video2) return 'Video ' + video1.uuid + ' is missing.'
for (const videoKey of Object.keys(video1)) {
const attribute1 = video1[videoKey]
const attribute2 = video2[videoKey]
if (videoKey === 'tags') {
if (attribute1.length !== attribute2.length) {
return 'Tags are different.'
}
attribute1.forEach(tag1 => {
if (attribute2.indexOf(tag1) === -1) {
return 'Tag ' + tag1 + ' is missing.'
}
})
} else if (videoKey === 'files') {
if (attribute1.length !== attribute2.length) {
return 'Video files are different.'
}
attribute1.forEach((videoFile1: VideoFile) => {
const videoFile2: VideoFile = attribute2.find(videoFile => videoFile.magnetUri === videoFile1.magnetUri)
if (!videoFile2) {
return `Video ${video1.uuid} has missing video file ${videoFile1.magnetUri}.`
}
if (videoFile1.size !== videoFile2.size || videoFile1.resolutionLabel !== videoFile2.resolutionLabel) {
return `Video ${video1.uuid} has different video file ${videoFile1.magnetUri}.`
}
})
} else {
if (attribute1 !== attribute2) {
return `Video ${video1.uuid} has different value for attribute ${videoKey}.`
}
}
}
}
return undefined
}
function goodbye () {
return process.exit(-1)
}
async function isThereAwaitingRequests (servers: ServerInfo[]) {
async function isTherePendingRequests (servers: ServerInfo[]) {
const tasks: Promise<any>[] = []
let awaitingRequests = false
let pendingRequests = false
// Check if each server has awaiting request
// Check if each server has pending request
for (const server of servers) {
const p = getRequestsStats(server).then(res => {
const stats = res.body
@ -314,7 +362,7 @@ async function isThereAwaitingRequests (servers: ServerInfo[]) {
stats.requestVideoEventScheduler.totalRequests !== 0 ||
stats.requestVideoQaduScheduler.totalRequests !== 0
) {
awaitingRequests = true
pendingRequests = true
}
})
@ -323,5 +371,5 @@ async function isThereAwaitingRequests (servers: ServerInfo[]) {
await Promise.all(tasks)
return awaitingRequests
return pendingRequests
}

1
shared/models/videos/video.model.ts
View File

@ -10,6 +10,7 @@ export interface Video {
uuid: string
author: string
createdAt: Date
updatedAt: Date
categoryLabel: string
category: number
licenceLabel: string