Don't show videos of remote instance after unfollow

pull/232/head
Chocobozzz 2018-01-18 10:53:54 +01:00
parent 1174a8479a
commit f05a1c30c1
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
38 changed files with 402 additions and 211 deletions

2
.gitignore vendored
View File

@ -7,7 +7,7 @@
/test6/
/storage/
/config/production.yaml
/config/local.json
/config/local*.json
/ffmpeg/
/*.sublime-project
/*.sublime-workspace

View File

@ -81,7 +81,7 @@
"rimraf": "^2.5.4",
"safe-buffer": "^5.0.1",
"scripty": "^1.5.0",
"sequelize": "4.25.2",
"sequelize": "4.31.2",
"sequelize-typescript": "^0.6.1",
"sharp": "^0.18.4",
"ts-node": "^3.3.0",

View File

@ -21,6 +21,12 @@ if [ -z $GITHUB_TOKEN ]; then
exit -1
fi
branch=$(git symbolic-ref --short -q HEAD)
if [ "$branch" != "develop" ]; then
echo "Need to be on develop branch."
exit -1
fi
version="v$1"
directory_name="peertube-$version"
zip_name="peertube-$version.zip"
@ -56,3 +62,10 @@ git push origin --tag
github-release release --user chocobozzz --repo peertube --tag "$version" --name "$version"
github-release upload --user chocobozzz --repo peertube --tag "$version" --name "$zip_name" --file "$zip_name"
# Update master branch
git checkout master
git rebase develop
git git push origin master
git checkout develop

View File

@ -1,6 +1,6 @@
import * as express from 'express'
import { getFormattedObjects } from '../../helpers/utils'
import { asyncMiddleware, paginationValidator, setDefaultSort, setPagination } from '../../middlewares'
import { asyncMiddleware, paginationValidator, setDefaultSort, setDefaultPagination } from '../../middlewares'
import { accountsGetValidator, accountsSortValidator } from '../../middlewares/validators'
import { AccountModel } from '../../models/account/account'
@ -10,7 +10,7 @@ accountsRouter.get('/',
paginationValidator,
accountsSortValidator,
setDefaultSort,
setPagination,
setDefaultPagination,
asyncMiddleware(listAccounts)
)

View File

@ -1,7 +1,10 @@
import * as express from 'express'
import { UserRight } from '../../../shared/models/users'
import { getFormattedObjects } from '../../helpers/utils'
import { asyncMiddleware, authenticate, ensureUserHasRight, jobsSortValidator, setDefaultSort, setPagination } from '../../middlewares'
import {
asyncMiddleware, authenticate, ensureUserHasRight, jobsSortValidator, setDefaultPagination,
setDefaultSort
} from '../../middlewares'
import { paginationValidator } from '../../middlewares/validators'
import { JobModel } from '../../models/job/job'
@ -13,7 +16,7 @@ jobsRouter.get('/',
paginationValidator,
jobsSortValidator,
setDefaultSort,
setPagination,
setDefaultPagination,
asyncMiddleware(listJobs)
)

View File

@ -10,7 +10,7 @@ import { getOrCreateActorAndServerAndModel } from '../../../lib/activitypub/acto
import { sendFollow, sendUndoFollow } from '../../../lib/activitypub/send'
import {
asyncMiddleware, authenticate, ensureUserHasRight, paginationValidator, removeFollowingValidator, setBodyHostsPort, setDefaultSort,
setPagination
setDefaultPagination
} from '../../../middlewares'
import { followersSortValidator, followingSortValidator, followValidator } from '../../../middlewares/validators'
import { ActorModel } from '../../../models/activitypub/actor'
@ -22,7 +22,7 @@ serverFollowsRouter.get('/following',
paginationValidator,
followingSortValidator,
setDefaultSort,
setPagination,
setDefaultPagination,
asyncMiddleware(listFollowing)
)
@ -45,7 +45,7 @@ serverFollowsRouter.get('/followers',
paginationValidator,
followersSortValidator,
setDefaultSort,
setPagination,
setDefaultPagination,
asyncMiddleware(listFollowers)
)

View File

@ -13,7 +13,7 @@ import { sendUpdateUser } from '../../lib/activitypub/send'
import { createUserAccountAndChannel } from '../../lib/user'
import {
asyncMiddleware, authenticate, ensureUserHasRight, ensureUserRegistrationAllowed, paginationValidator, setDefaultSort,
setPagination, token, usersAddValidator, usersGetValidator, usersRegisterValidator, usersRemoveValidator, usersSortValidator,
setDefaultPagination, token, usersAddValidator, usersGetValidator, usersRegisterValidator, usersRemoveValidator, usersSortValidator,
usersUpdateMeValidator, usersUpdateValidator, usersVideoRatingValidator
} from '../../middlewares'
import { usersUpdateMyAvatarValidator, videosSortValidator } from '../../middlewares/validators'
@ -40,7 +40,7 @@ usersRouter.get('/me/videos',
paginationValidator,
videosSortValidator,
setDefaultSort,
setPagination,
setDefaultPagination,
asyncMiddleware(getUserVideos)
)
@ -56,7 +56,7 @@ usersRouter.get('/',
paginationValidator,
usersSortValidator,
setDefaultSort,
setPagination,
setDefaultPagination,
asyncMiddleware(listUsers)
)
@ -129,15 +129,19 @@ async function createUserRetryWrapper (req: express.Request, res: express.Respon
errorMessage: 'Cannot insert the user with many retries.'
}
await retryTransactionWrapper(createUser, options)
const { user, account } = await retryTransactionWrapper(createUser, options)
// TODO : include Location of the new user -> 201
return res.type('json').status(204).end()
return res.json({
user: {
id: user.id,
uuid: account.uuid
}
}).end()
}
async function createUser (req: express.Request) {
const body: UserCreate = req.body
const user = new UserModel({
const userToCreate = new UserModel({
username: body.username,
password: body.password,
email: body.email,
@ -147,9 +151,11 @@ async function createUser (req: express.Request) {
videoQuota: body.videoQuota
})
await createUserAccountAndChannel(user)
const { user, account } = await createUserAccountAndChannel(userToCreate)
logger.info('User %s with its channel and account created.', body.username)
return { user, account }
}
async function registerUserRetryWrapper (req: express.Request, res: express.Response, next: express.NextFunction) {

View File

@ -6,7 +6,7 @@ import { getFormattedObjects } from '../../../helpers/utils'
import { sequelizeTypescript } from '../../../initializers'
import { sendVideoAbuse } from '../../../lib/activitypub/send'
import {
asyncMiddleware, authenticate, ensureUserHasRight, paginationValidator, setDefaultSort, setPagination, videoAbuseReportValidator,
asyncMiddleware, authenticate, ensureUserHasRight, paginationValidator, setDefaultSort, setDefaultPagination, videoAbuseReportValidator,
videoAbusesSortValidator
} from '../../../middlewares'
import { AccountModel } from '../../../models/account/account'
@ -21,7 +21,7 @@ abuseVideoRouter.get('/abuse',
paginationValidator,
videoAbusesSortValidator,
setDefaultSort,
setPagination,
setDefaultPagination,
asyncMiddleware(listVideoAbuses)
)
abuseVideoRouter.post('/:id/abuse',

View File

@ -3,7 +3,7 @@ import { BlacklistedVideo, UserRight } from '../../../../shared'
import { logger } from '../../../helpers/logger'
import { getFormattedObjects } from '../../../helpers/utils'
import {
asyncMiddleware, authenticate, blacklistSortValidator, ensureUserHasRight, paginationValidator, setBlacklistSort, setPagination,
asyncMiddleware, authenticate, blacklistSortValidator, ensureUserHasRight, paginationValidator, setBlacklistSort, setDefaultPagination,
videosBlacklistAddValidator, videosBlacklistRemoveValidator
} from '../../../middlewares'
import { VideoBlacklistModel } from '../../../models/video/video-blacklist'
@ -23,7 +23,7 @@ blacklistRouter.get('/blacklist',
paginationValidator,
blacklistSortValidator,
setBlacklistSort,
setPagination,
setDefaultPagination,
asyncMiddleware(listBlacklist)
)

View File

@ -7,7 +7,7 @@ import { sequelizeTypescript } from '../../../initializers'
import { setAsyncActorKeys } from '../../../lib/activitypub'
import { createVideoChannel } from '../../../lib/video-channel'
import {
asyncMiddleware, authenticate, listVideoAccountChannelsValidator, paginationValidator, setDefaultSort, setPagination,
asyncMiddleware, authenticate, listVideoAccountChannelsValidator, paginationValidator, setDefaultSort, setDefaultPagination,
videoChannelsAddValidator, videoChannelsGetValidator, videoChannelsRemoveValidator, videoChannelsSortValidator,
videoChannelsUpdateValidator
} from '../../../middlewares'
@ -20,7 +20,7 @@ videoChannelRouter.get('/channels',
paginationValidator,
videoChannelsSortValidator,
setDefaultSort,
setPagination,
setDefaultPagination,
asyncMiddleware(listVideoChannels)
)

View File

@ -6,7 +6,7 @@ import { logger } from '../../../helpers/logger'
import { getFormattedObjects } from '../../../helpers/utils'
import { sequelizeTypescript } from '../../../initializers'
import { buildFormattedCommentTree, createVideoComment } from '../../../lib/video-comment'
import { asyncMiddleware, authenticate, paginationValidator, setDefaultSort, setPagination } from '../../../middlewares'
import { asyncMiddleware, authenticate, paginationValidator, setDefaultSort, setDefaultPagination } from '../../../middlewares'
import { videoCommentThreadsSortValidator } from '../../../middlewares/validators'
import {
addVideoCommentReplyValidator, addVideoCommentThreadValidator, listVideoCommentThreadsValidator, listVideoThreadCommentsValidator,
@ -21,7 +21,7 @@ videoCommentRouter.get('/:videoId/comment-threads',
paginationValidator,
videoCommentThreadsSortValidator,
setDefaultSort,
setPagination,
setDefaultPagination,
asyncMiddleware(listVideoCommentThreadsValidator),
asyncMiddleware(listVideoThreads)
)

View File

@ -14,7 +14,7 @@ import { fetchRemoteVideoDescription, getVideoActivityPubUrl, shareVideoByServer
import { sendCreateVideo, sendCreateViewToOrigin, sendCreateViewToVideoFollowers, sendUpdateVideo } from '../../../lib/activitypub/send'
import { transcodingJobScheduler } from '../../../lib/jobs/transcoding-job-scheduler'
import {
asyncMiddleware, authenticate, paginationValidator, setDefaultSort, setPagination, videosAddValidator, videosGetValidator,
asyncMiddleware, authenticate, paginationValidator, setDefaultSort, setDefaultPagination, videosAddValidator, videosGetValidator,
videosRemoveValidator, videosSearchValidator, videosSortValidator, videosUpdateValidator
} from '../../../middlewares'
import { TagModel } from '../../../models/video/tag'
@ -45,7 +45,7 @@ videosRouter.get('/',
paginationValidator,
videosSortValidator,
setDefaultSort,
setPagination,
setDefaultPagination,
asyncMiddleware(listVideos)
)
videosRouter.get('/search',
@ -53,7 +53,7 @@ videosRouter.get('/search',
paginationValidator,
videosSortValidator,
setDefaultSort,
setPagination,
setDefaultPagination,
asyncMiddleware(searchVideos)
)
videosRouter.put('/:id',

View File

@ -283,7 +283,7 @@ const ACTIVITY_PUB = {
FETCH_PAGE_LIMIT: 100,
MAX_HTTP_ATTEMPT: 5,
URL_MIME_TYPES: {
VIDEO: [ 'video/mp4', 'video/webm', 'video/ogg' ], // TODO: Merge with VIDEO_MIMETYPE_EXT
VIDEO: Object.keys(VIDEO_MIMETYPE_EXT),
TORRENT: [ 'application/x-bittorrent' ],
MAGNET: [ 'application/x-bittorrent;x-scheme-handler/magnet' ]
},

View File

@ -309,7 +309,10 @@ async function refreshActorIfNeeded (actor: ActorModel) {
const actorUrl = await getUrlFromWebfinger(actor.preferredUsername, actor.getHost())
const result = await fetchRemoteActor(actorUrl)
if (result === undefined) throw new Error('Cannot fetch remote actor in refresh actor.')
if (result === undefined) {
logger.warn('Cannot fetch remote actor in refresh actor.')
return actor
}
return sequelizeTypescript.transaction(async t => {
updateInstanceWithAnother(actor, result.actor)

View File

@ -10,21 +10,26 @@ import { VideoCommentModel } from '../../../models/video/video-comment'
import { getOrCreateActorAndServerAndModel } from '../actor'
async function processDeleteActivity (activity: ActivityDelete) {
const actor = await getOrCreateActorAndServerAndModel(activity.actor)
const objectUrl = typeof activity.object === 'string' ? activity.object : activity.object.id
if (actor.url === objectUrl) {
if (activity.actor === objectUrl) {
let actor = await ActorModel.loadByUrl(activity.actor)
if (!actor) return
if (actor.type === 'Person') {
if (!actor.Account) throw new Error('Actor ' + actor.url + ' is a person but we cannot find it in database.')
actor.Account.Actor = await actor.Account.$get('Actor') as ActorModel
return processDeleteAccount(actor.Account)
} else if (actor.type === 'Group') {
if (!actor.VideoChannel) throw new Error('Actor ' + actor.url + ' is a group but we cannot find it in database.')
actor.VideoChannel.Actor = await actor.VideoChannel.$get('Actor') as ActorModel
return processDeleteVideoChannel(actor.VideoChannel)
}
}
const actor = await getOrCreateActorAndServerAndModel(activity.actor)
{
const videoCommentInstance = await VideoCommentModel.loadByUrlAndPopulateAccount(objectUrl)
if (videoCommentInstance) {

View File

@ -23,7 +23,10 @@ async function sendDeleteActor (byActor: ActorModel, t: Transaction) {
const url = getDeleteActivityPubUrl(byActor.url)
const data = deleteActivityData(url, byActor.url, byActor)
return broadcastToFollowers(data, byActor, [ byActor ], t)
const actorsInvolved = await VideoShareModel.loadActorsByVideoOwner(byActor.id, t)
actorsInvolved.push(byActor)
return broadcastToFollowers(data, byActor, actorsInvolved, t)
}
async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Transaction) {

View File

@ -20,17 +20,16 @@ import { VideoShareModel } from '../../models/video/video-share'
import { getOrCreateActorAndServerAndModel } from './actor'
function fetchRemoteVideoPreview (video: VideoModel, reject: Function) {
// FIXME: use url
const host = video.VideoChannel.Account.Actor.Server.host
const path = join(STATIC_PATHS.PREVIEWS, video.getPreviewName())
// We need to provide a callback, if no we could have an uncaught exception
return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => {
if (err) reject(err)
})
}
async function fetchRemoteVideoDescription (video: VideoModel) {
// FIXME: use url
const host = video.VideoChannel.Account.Actor.Server.host
const path = video.getDescriptionPath()
const options = {

View File

@ -6,15 +6,15 @@ import { UserModel } from '../models/account/user'
import { buildActorInstance, getAccountActivityPubUrl, setAsyncActorKeys } from './activitypub'
import { createVideoChannel } from './video-channel'
async function createUserAccountAndChannel (user: UserModel, validateUser = true) {
const { account, videoChannel } = await sequelizeTypescript.transaction(async t => {
async function createUserAccountAndChannel (userToCreate: UserModel, validateUser = true) {
const { user, account, videoChannel } = await sequelizeTypescript.transaction(async t => {
const userOptions = {
transaction: t,
validate: validateUser
}
const userCreated = await user.save(userOptions)
const accountCreated = await createLocalAccountWithoutKeys(user.username, user.id, null, t)
const userCreated = await userToCreate.save(userOptions)
const accountCreated = await createLocalAccountWithoutKeys(userToCreate.username, userToCreate.id, null, t)
const videoChannelName = `Default ${userCreated.username} channel`
const videoChannelInfo = {
@ -22,13 +22,13 @@ async function createUserAccountAndChannel (user: UserModel, validateUser = true
}
const videoChannel = await createVideoChannel(videoChannelInfo, accountCreated, t)
return { account: accountCreated, videoChannel }
return { user: userCreated, account: accountCreated, videoChannel }
})
account.Actor = await setAsyncActorKeys(account.Actor)
videoChannel.Actor = await setAsyncActorKeys(videoChannel.Actor)
return { account, videoChannel }
return { user, account, videoChannel }
}
async function createLocalAccountWithoutKeys (

View File

@ -3,7 +3,7 @@ import * as express from 'express'
import { PAGINATION_COUNT_DEFAULT } from '../initializers'
function setPagination (req: express.Request, res: express.Response, next: express.NextFunction) {
function setDefaultPagination (req: express.Request, res: express.Response, next: express.NextFunction) {
if (!req.query.start) req.query.start = 0
else req.query.start = parseInt(req.query.start, 10)
@ -16,5 +16,5 @@ function setPagination (req: express.Request, res: express.Response, next: expre
// ---------------------------------------------------------------------------
export {
setPagination
setDefaultPagination
}

View File

@ -1,9 +1,10 @@
import * as Sequelize from 'sequelize'
import {
AfterDestroy, AllowNull, BelongsTo, Column, CreatedAt, DefaultScope, ForeignKey, HasMany, Model, Table,
AllowNull, BeforeDestroy, BelongsTo, Column, CreatedAt, DefaultScope, ForeignKey, HasMany, Model, Table,
UpdatedAt
} from 'sequelize-typescript'
import { Account } from '../../../shared/models/actors'
import { logger } from '../../helpers/logger'
import { sendDeleteActor } from '../../lib/activitypub/send'
import { ActorModel } from '../activitypub/actor'
import { ApplicationModel } from '../application/application'
@ -11,6 +12,7 @@ import { AvatarModel } from '../avatar/avatar'
import { ServerModel } from '../server/server'
import { getSort } from '../utils'
import { VideoChannelModel } from '../video/video-channel'
import { VideoCommentModel } from '../video/video-comment'
import { UserModel } from './user'
@DefaultScope({
@ -80,7 +82,7 @@ export class AccountModel extends Model<AccountModel> {
},
onDelete: 'cascade'
})
Account: ApplicationModel
Application: ApplicationModel
@HasMany(() => VideoChannelModel, {
foreignKey: {
@ -91,10 +93,24 @@ export class AccountModel extends Model<AccountModel> {
})
VideoChannels: VideoChannelModel[]
@AfterDestroy
static sendDeleteIfOwned (instance: AccountModel) {
@HasMany(() => VideoCommentModel, {
foreignKey: {
allowNull: false
},
onDelete: 'cascade',
hooks: true
})
VideoComments: VideoCommentModel[]
@BeforeDestroy
static async sendDeleteIfOwned (instance: AccountModel, options) {
if (!instance.Actor) {
instance.Actor = await instance.$get('Actor', { transaction: options.transaction }) as ActorModel
}
if (instance.isOwned()) {
return sendDeleteActor(instance.Actor, undefined)
logger.debug('Sending delete of actor of account %s.', instance.Actor.url)
return sendDeleteActor(instance.Actor, options.transaction)
}
return undefined

View File

@ -94,7 +94,8 @@ export class UserModel extends Model<UserModel> {
@HasOne(() => AccountModel, {
foreignKey: 'userId',
onDelete: 'cascade'
onDelete: 'cascade',
hooks: true
})
Account: AccountModel

View File

@ -155,7 +155,8 @@ export class ActorModel extends Model<ActorModel> {
foreignKey: {
allowNull: true
},
onDelete: 'set null'
onDelete: 'set null',
hooks: true
})
Avatar: AvatarModel
@ -194,7 +195,8 @@ export class ActorModel extends Model<ActorModel> {
foreignKey: {
allowNull: true
},
onDelete: 'cascade'
onDelete: 'cascade',
hooks: true
})
Account: AccountModel
@ -202,7 +204,8 @@ export class ActorModel extends Model<ActorModel> {
foreignKey: {
allowNull: true
},
onDelete: 'cascade'
onDelete: 'cascade',
hooks: true
})
VideoChannel: VideoChannelModel

View File

@ -1,5 +1,6 @@
import { AllowNull, Column, CreatedAt, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
import { AllowNull, Column, CreatedAt, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
import { isHostValid } from '../../helpers/custom-validators/servers'
import { ActorModel } from '../activitypub/actor'
import { throwIfNotValid } from '../utils'
@Table({
@ -23,4 +24,14 @@ export class ServerModel extends Model<ServerModel> {
@UpdatedAt
updatedAt: Date
@HasMany(() => ActorModel, {
foreignKey: {
name: 'serverId',
allowNull: true
},
onDelete: 'CASCADE',
hooks: true
})
Actors: ActorModel[]
}

View File

@ -1,20 +1,10 @@
import {
AfterDestroy,
AllowNull,
BelongsTo,
Column,
CreatedAt,
DefaultScope,
ForeignKey,
HasMany,
Is,
Model,
Scopes,
Table,
AllowNull, BeforeDestroy, BelongsTo, Column, CreatedAt, DefaultScope, ForeignKey, HasMany, Is, Model, Scopes, Table,
UpdatedAt
} from 'sequelize-typescript'
import { ActivityPubActor } from '../../../shared/models/activitypub'
import { isVideoChannelDescriptionValid, isVideoChannelNameValid } from '../../helpers/custom-validators/video-channels'
import { logger } from '../../helpers/logger'
import { sendDeleteActor } from '../../lib/activitypub/send'
import { AccountModel } from '../account/account'
import { ActorModel } from '../activitypub/actor'
@ -116,14 +106,21 @@ export class VideoChannelModel extends Model<VideoChannelModel> {
name: 'channelId',
allowNull: false
},
onDelete: 'CASCADE'
onDelete: 'CASCADE',
hooks: true
})
Videos: VideoModel[]
@AfterDestroy
static sendDeleteIfOwned (instance: VideoChannelModel) {
@BeforeDestroy
static async sendDeleteIfOwned (instance: VideoChannelModel, options) {
if (!instance.Actor) {
instance.Actor = await instance.$get('Actor', { transaction: options.transaction }) as ActorModel
}
if (instance.Actor.isOwned()) {
return sendDeleteActor(instance.Actor, undefined)
logger.debug('Sending delete of actor of video channel %s.', instance.Actor.url)
return sendDeleteActor(instance.Actor, options.transaction)
}
return undefined

View File

@ -1,6 +1,6 @@
import * as Sequelize from 'sequelize'
import {
AfterDestroy, AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, IFindOptions, Is, Model, Scopes, Table,
AllowNull, BeforeDestroy, BelongsTo, Column, CreatedAt, DataType, ForeignKey, IFindOptions, Is, Model, Scopes, Table,
UpdatedAt
} from 'sequelize-typescript'
import { ActivityTagObject } from '../../../shared/models/activitypub/objects/common-objects'
@ -175,10 +175,17 @@ export class VideoCommentModel extends Model<VideoCommentModel> {
})
Account: AccountModel
@AfterDestroy
static async sendDeleteIfOwned (instance: VideoCommentModel) {
@BeforeDestroy
static async sendDeleteIfOwned (instance: VideoCommentModel, options) {
if (!instance.Account || !instance.Account.Actor) {
instance.Account = await instance.$get('Account', {
include: [ ActorModel ],
transaction: options.transaction
}) as AccountModel
}
if (instance.isOwned()) {
await sendDeleteVideoComment(instance, undefined)
await sendDeleteVideoComment(instance, options.transaction)
}
}

View File

@ -5,10 +5,9 @@ import * as parseTorrent from 'parse-torrent'
import { join } from 'path'
import * as Sequelize from 'sequelize'
import {
AfterDestroy, AllowNull, BelongsTo, BelongsToMany, Column, CreatedAt, DataType, Default, ForeignKey, HasMany, IFindOptions, Is,
IsInt, IsUUID, Min, Model, Scopes, Table, UpdatedAt
AfterDestroy, AllowNull, BeforeDestroy, BelongsTo, BelongsToMany, Column, CreatedAt, DataType, Default, ForeignKey, HasMany,
IFindOptions, Is, IsInt, IsUUID, Min, Model, Scopes, Table, UpdatedAt
} from 'sequelize-typescript'
import { IIncludeOptions } from 'sequelize-typescript/lib/interfaces/IIncludeOptions'
import { VideoPrivacy, VideoResolution } from '../../../shared'
import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
import { Video, VideoDetails } from '../../../shared/models/videos'
@ -22,6 +21,7 @@ import {
} from '../../helpers/custom-validators/videos'
import { generateImageFromVideoFile, getVideoFileHeight, transcode } from '../../helpers/ffmpeg-utils'
import { logger } from '../../helpers/logger'
import { getServerActor } from '../../helpers/utils'
import {
API_VERSION, CONFIG, CONSTRAINTS_FIELDS, PREVIEWS_SIZE, REMOTE_SCHEME, STATIC_PATHS, THUMBNAILS_SIZE, VIDEO_CATEGORIES,
VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES
@ -31,6 +31,7 @@ import { sendDeleteVideo } from '../../lib/activitypub/send'
import { AccountModel } from '../account/account'
import { AccountVideoRateModel } from '../account/account-video-rate'
import { ActorModel } from '../activitypub/actor'
import { ActorFollowModel } from '../activitypub/actor-follow'
import { ServerModel } from '../server/server'
import { getSort, throwIfNotValid } from '../utils'
import { TagModel } from './tag'
@ -43,7 +44,6 @@ import { VideoTagModel } from './video-tag'
enum ScopeNames {
AVAILABLE_FOR_LIST = 'AVAILABLE_FOR_LIST',
WITH_ACCOUNT_API = 'WITH_ACCOUNT_API',
WITH_ACCOUNT_DETAILS = 'WITH_ACCOUNT_DETAILS',
WITH_TAGS = 'WITH_TAGS',
WITH_FILES = 'WITH_FILES',
@ -53,34 +53,60 @@ enum ScopeNames {
}
@Scopes({
[ScopeNames.AVAILABLE_FOR_LIST]: {
[ScopeNames.AVAILABLE_FOR_LIST]: (actorId: number) => ({
subQuery: false,
where: {
id: {
[Sequelize.Op.notIn]: Sequelize.literal(
'(SELECT "videoBlacklist"."videoId" FROM "videoBlacklist")'
)
},
privacy: VideoPrivacy.PUBLIC
}
},
[ScopeNames.WITH_ACCOUNT_API]: {
privacy: VideoPrivacy.PUBLIC,
[Sequelize.Op.or]: [
{
'$VideoChannel.Account.Actor.serverId$': null
},
{
'$VideoChannel.Account.Actor.followers.actorId$': actorId
},
{
id: {
[ Sequelize.Op.in ]: Sequelize.literal(
'(' +
'SELECT "videoShare"."videoId" FROM "videoShare" ' +
'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "videoShare"."actorId" ' +
'WHERE "actorFollow"."actorId" = ' + parseInt(actorId.toString(), 10) +
')'
)
}
}
]
},
include: [
{
model: () => VideoChannelModel.unscoped(),
attributes: [ 'name', 'description' ],
model: VideoChannelModel.unscoped(),
required: true,
include: [
{
attributes: [ 'name' ],
model: () => AccountModel.unscoped(),
model: AccountModel.unscoped(),
required: true,
include: [
{
attributes: [ 'serverId' ],
model: () => ActorModel.unscoped(),
model: ActorModel.unscoped(),
required: true,
include: [
{
model: () => ServerModel.unscoped(),
attributes: [ 'host' ],
model: ServerModel.unscoped(),
required: false
},
{
attributes: [ ],
model: ActorFollowModel.unscoped(),
as: 'followers',
required: false
}
]
@ -90,7 +116,7 @@ enum ScopeNames {
]
}
]
},
}),
[ScopeNames.WITH_ACCOUNT_DETAILS]: {
include: [
{
@ -347,23 +373,46 @@ export class VideoModel extends Model<VideoModel> {
name: 'videoId',
allowNull: false
},
onDelete: 'cascade'
onDelete: 'cascade',
hooks: true
})
VideoComments: VideoCommentModel[]
@AfterDestroy
static removeFilesAndSendDelete (instance: VideoModel) {
const tasks = []
@BeforeDestroy
static async sendDelete (instance: VideoModel, options) {
if (instance.isOwned()) {
if (!instance.VideoChannel) {
instance.VideoChannel = await instance.$get('VideoChannel', {
include: [
{
model: AccountModel,
include: [ ActorModel ]
}
],
transaction: options.transaction
}) as VideoChannelModel
}
tasks.push(
instance.removeThumbnail()
)
logger.debug('Sending delete of video %s.', instance.url)
return sendDeleteVideo(instance, options.transaction)
}
return undefined
}
@AfterDestroy
static async removeFilesAndSendDelete (instance: VideoModel) {
const tasks: Promise<any>[] = []
tasks.push(instance.removeThumbnail())
if (instance.isOwned()) {
tasks.push(
instance.removePreview(),
sendDeleteVideo(instance, undefined)
)
if (!Array.isArray(instance.VideoFiles)) {
instance.VideoFiles = await instance.$get('VideoFiles') as VideoFileModel[]
}
tasks.push(instance.removePreview())
// Remove physical files and torrents
instance.VideoFiles.forEach(file => {
@ -500,14 +549,16 @@ export class VideoModel extends Model<VideoModel> {
})
}
static listForApi (start: number, count: number, sort: string) {
static async listForApi (start: number, count: number, sort: string) {
const query = {
offset: start,
limit: count,
order: [ getSort(sort) ]
}
return VideoModel.scope([ ScopeNames.AVAILABLE_FOR_LIST, ScopeNames.WITH_ACCOUNT_API ])
const serverActor = await getServerActor()
return VideoModel.scope({ method: [ ScopeNames.AVAILABLE_FOR_LIST, serverActor.id ] })
.findAndCountAll(query)
.then(({ rows, count }) => {
return {
@ -517,6 +568,29 @@ export class VideoModel extends Model<VideoModel> {
})
}
static async searchAndPopulateAccountAndServerAndTags (value: string, start: number, count: number, sort: string) {
const query: IFindOptions<VideoModel> = {
offset: start,
limit: count,
order: [ getSort(sort) ],
where: {
name: {
[Sequelize.Op.iLike]: '%' + value + '%'
}
}
}
const serverActor = await getServerActor()
return VideoModel.scope({ method: [ ScopeNames.AVAILABLE_FOR_LIST, serverActor.id ] })
.findAndCountAll(query).then(({ rows, count }) => {
return {
data: rows,
total: count
}
})
}
static load (id: number) {
return VideoModel.findById(id)
}
@ -603,74 +677,6 @@ export class VideoModel extends Model<VideoModel> {
.findOne(options)
}
static searchAndPopulateAccountAndServerAndTags (value: string, start: number, count: number, sort: string) {
const serverInclude: IIncludeOptions = {
model: ServerModel,
required: false
}
const accountInclude: IIncludeOptions = {
model: AccountModel,
include: [
{
model: ActorModel,
required: true,
include: [ serverInclude ]
}
]
}
const videoChannelInclude: IIncludeOptions = {
model: VideoChannelModel,
include: [ accountInclude ],
required: true
}
const tagInclude: IIncludeOptions = {
model: TagModel
}
const query: IFindOptions<VideoModel> = {
distinct: true, // Because we have tags
offset: start,
limit: count,
order: [ getSort(sort) ],
where: {}
}
// TODO: search on tags too
// const escapedValue = Video['sequelize'].escape('%' + value + '%')
// query.where['id'][Sequelize.Op.in] = Video['sequelize'].literal(
// `(SELECT "VideoTags"."videoId"
// FROM "Tags"
// INNER JOIN "VideoTags" ON "Tags"."id" = "VideoTags"."tagId"
// WHERE name ILIKE ${escapedValue}
// )`
// )
// TODO: search on account too
// accountInclude.where = {
// name: {
// [Sequelize.Op.iLike]: '%' + value + '%'
// }
// }
query.where['name'] = {
[Sequelize.Op.iLike]: '%' + value + '%'
}
query.include = [
videoChannelInclude, tagInclude
]
return VideoModel.scope([ ScopeNames.AVAILABLE_FOR_LIST ])
.findAndCountAll(query).then(({ rows, count }) => {
return {
data: rows,
total: count
}
})
}
getOriginalFile () {
if (Array.isArray(this.VideoFiles) === false) return undefined

View File

@ -185,7 +185,7 @@ describe('Test users API validators', function () {
path,
token: server.accessToken,
fields: baseCorrectParams,
statusCodeExpected: 204
statusCodeExpected: 200
})
})

View File

@ -4,7 +4,7 @@ import * as chai from 'chai'
import 'mocha'
import { Video, VideoPrivacy } from '../../../../shared/models/videos'
import { VideoComment, VideoCommentThreadTree } from '../../../../shared/models/videos/video-comment.model'
import { completeVideoCheck } from '../../utils'
import { checkVideoFilesWereRemoved, completeVideoCheck, getVideoChannelsList } from '../../utils'
import {
flushAndRunMultipleServers, flushTests, getVideosList, killallServers, ServerInfo, setAccessTokensToServers, uploadVideo,
@ -12,7 +12,7 @@ import {
} from '../../utils/index'
import { dateIsValid } from '../../utils/miscs/miscs'
import { follow, getFollowersListPaginationAndSort, getFollowingListPaginationAndSort, unfollow } from '../../utils/server/follows'
import { expectAccountFollows } from '../../utils/users/accounts'
import { expectAccountFollows, getAccountsList } from '../../utils/users/accounts'
import { userLogin } from '../../utils/users/login'
import { createUser } from '../../utils/users/users'
import {
@ -343,6 +343,26 @@ describe('Test follows', function () {
expect(secondChild.comment.text).to.equal('my second answer to thread 1')
expect(secondChild.children).to.have.lengthOf(0)
})
it('Should unfollow server 3 on server 1 and does not list server 3 videos', async function () {
this.timeout(5000)
await unfollow(servers[0].url, servers[0].accessToken, servers[2])
await wait(3000)
let res = await getVideosList(servers[ 0 ].url)
expect(res.body.total).to.equal(1)
res = await getVideoChannelsList(servers[0].url, 0, 1)
expect(res.body.total).to.equal(2)
res = await getAccountsList(servers[0].url)
expect(res.body.total).to.equal(2)
await checkVideoFilesWereRemoved(video4.uuid, servers[0].serverNumber)
})
})
after(async function () {

View File

@ -3,18 +3,20 @@
import * as chai from 'chai'
import 'mocha'
import { Account } from '../../../../shared/models/actors'
import { doubleFollow, flushAndRunMultipleServers, wait } from '../../utils'
import {
flushTests, getMyUserInformation, killallServers, ServerInfo, testVideoImage, updateMyAvatar,
uploadVideo
} from '../../utils/index'
import { getAccount, getAccountsList } from '../../utils/users/accounts'
import { checkVideoFilesWereRemoved, createUser, doubleFollow, flushAndRunMultipleServers, removeUser, userLogin, wait } from '../../utils'
import { flushTests, getMyUserInformation, killallServers, ServerInfo, testImage, updateMyAvatar, uploadVideo } from '../../utils/index'
import { checkActorFilesWereRemoved, getAccount, getAccountsList } from '../../utils/users/accounts'
import { setAccessTokensToServers } from '../../utils/users/login'
const expect = chai.expect
describe('Test users with multiple servers', function () {
let servers: ServerInfo[] = []
let user
let userUUID
let userId
let videoUUID
let userAccessToken
before(async function () {
this.timeout(120000)
@ -34,6 +36,18 @@ describe('Test users with multiple servers', function () {
// The root user of server 1 is propagated to servers 2 and 3
await uploadVideo(servers[0].url, servers[0].accessToken, {})
const user = {
username: 'user1',
password: 'password'
}
const resUser = await createUser(servers[0].url, servers[0].accessToken, user.username, user.password)
userUUID = resUser.body.user.uuid
userId = resUser.body.user.id
userAccessToken = await userLogin(servers[0], user)
const resVideo = await uploadVideo(servers[0].url, userAccessToken, {})
videoUUID = resVideo.body.uuid
await wait(5000)
})
@ -49,9 +63,9 @@ describe('Test users with multiple servers', function () {
})
const res = await getMyUserInformation(servers[0].url, servers[0].accessToken)
const user = res.body
user = res.body
const test = await testVideoImage(servers[0].url, 'avatar2-resized', user.account.avatar.path, '.png')
const test = await testImage(servers[0].url, 'avatar2-resized', user.account.avatar.path, '.png')
expect(test).to.equal(true)
await wait(5000)
@ -69,11 +83,45 @@ describe('Test users with multiple servers', function () {
expect(rootServer1Get.name).to.equal('root')
expect(rootServer1Get.host).to.equal('localhost:9001')
const test = await testVideoImage(server.url, 'avatar2-resized', rootServer1Get.avatar.path, '.png')
const test = await testImage(server.url, 'avatar2-resized', rootServer1Get.avatar.path, '.png')
expect(test).to.equal(true)
}
})
it('Should remove the user', async function () {
this.timeout(10000)
for (const server of servers) {
const resAccounts = await getAccountsList(server.url, '-createdAt')
const userServer1List = resAccounts.body.data.find(a => a.name === 'user1' && a.host === 'localhost:9001') as Account
expect(userServer1List).not.to.be.undefined
}
await removeUser(servers[0].url, userId, servers[0].accessToken)
await wait(5000)
for (const server of servers) {
const resAccounts = await getAccountsList(server.url, '-createdAt')
const userServer1List = resAccounts.body.data.find(a => a.name === 'user1' && a.host === 'localhost:9001') as Account
expect(userServer1List).to.be.undefined
}
})
it('Should not have actor files', async () => {
for (const server of servers) {
await checkActorFilesWereRemoved(userUUID, server.serverNumber)
}
})
it('Should not have video files', async () => {
for (const server of servers) {
await checkVideoFilesWereRemoved(videoUUID, server.serverNumber)
}
})
after(async function () {
killallServers(servers)

View File

@ -7,7 +7,7 @@ import {
createUser, flushTests, getBlacklistedVideosList, getMyUserInformation, getMyUserVideoQuotaUsed, getMyUserVideoRating, getUserInformation,
getUsersList,
getUsersListPaginationAndSort, getVideosList, killallServers, login, makePutBodyRequest, rateVideo, registerUser, removeUser, removeVideo,
runServer, ServerInfo, serverLogin, testVideoImage, updateMyAvatar, updateMyUser, updateUser, uploadVideo
runServer, ServerInfo, serverLogin, testImage, updateMyAvatar, updateMyUser, updateUser, uploadVideo
} from '../../utils/index'
import { follow } from '../../utils/server/follows'
import { setAccessTokensToServers } from '../../utils/users/login'
@ -361,7 +361,7 @@ describe('Test users', function () {
const res = await getMyUserInformation(server.url, accessTokenUser)
const user = res.body
const test = await testVideoImage(server.url, 'avatar-resized', user.account.avatar.path, '.png')
const test = await testImage(server.url, 'avatar-resized', user.account.avatar.path, '.png')
expect(test).to.equal(true)
})

View File

@ -8,8 +8,9 @@ import { VideoPrivacy } from '../../../../shared/models/videos'
import { VideoComment, VideoCommentThreadTree } from '../../../../shared/models/videos/video-comment.model'
import {
addVideoChannel, completeVideoCheck, createUser, dateIsValid, doubleFollow, flushAndRunMultipleServers, flushTests, getVideo,
getVideoChannelsList, getVideosList, killallServers, rateVideo, removeVideo, ServerInfo, setAccessTokensToServers, testVideoImage,
addVideoChannel, checkVideoFilesWereRemoved, completeVideoCheck, createUser, dateIsValid, doubleFollow, flushAndRunMultipleServers,
flushTests, getVideo,
getVideoChannelsList, getVideosList, killallServers, rateVideo, removeVideo, ServerInfo, setAccessTokensToServers, testImage,
updateVideo, uploadVideo, userLogin, viewVideo, wait, webtorrentAdd
} from '../../utils'
import {
@ -578,6 +579,13 @@ describe('Test multiple servers', function () {
await wait(5000)
})
it('Should not have files of videos 3 and 3-2 on each server', async function () {
for (const server of servers) {
await checkVideoFilesWereRemoved(toRemove[0].uuid, server.serverNumber)
await checkVideoFilesWereRemoved(toRemove[1].uuid, server.serverNumber)
}
})
it('Should have videos 1 and 3 on each server', async function () {
for (const server of servers) {
const res = await getVideosList(server.url)
@ -624,7 +632,7 @@ describe('Test multiple servers', function () {
const res = await getVideo(server.url, videoUUID)
const video = res.body
const test = await testVideoImage(server.url, 'video_short1-preview.webm', video.previewPath)
const test = await testImage(server.url, 'video_short1-preview.webm', video.previewPath)
expect(test).to.equal(true)
}
})

View File

@ -3,13 +3,12 @@
import * as chai from 'chai'
import { keyBy } from 'lodash'
import 'mocha'
import { join } from 'path'
import { VideoPrivacy } from '../../../../shared/models/videos'
import { readdirPromise } from '../../../helpers/core-utils'
import {
completeVideoCheck, flushTests, getVideo, getVideoCategories, getVideoLanguages, getVideoLicences, getVideoPrivacies,
getVideosList, getVideosListPagination, getVideosListSort, killallServers, rateVideo, removeVideo, runServer, searchVideo,
searchVideoWithPagination, searchVideoWithSort, ServerInfo, setAccessTokensToServers, testVideoImage, updateVideo, uploadVideo, viewVideo
checkVideoFilesWereRemoved, completeVideoCheck, flushTests, getVideo, getVideoCategories, getVideoLanguages, getVideoLicences,
getVideoPrivacies, getVideosList, getVideosListPagination, getVideosListSort, killallServers, rateVideo, removeVideo, runServer,
searchVideo, searchVideoWithPagination, searchVideoWithSort, ServerInfo, setAccessTokensToServers, testImage, updateVideo, uploadVideo,
viewVideo
} from '../../utils'
const expect = chai.expect
@ -277,11 +276,7 @@ describe('Test a single server', function () {
it('Should remove the video', async function () {
await removeVideo(server.url, server.accessToken, videoId)
const files1 = await readdirPromise(join(__dirname, '..', '..', '..', '..', 'test1', 'videos'))
expect(files1).to.have.lengthOf(0)
const files2 = await readdirPromise(join(__dirname, '..', '..', '..', '..', 'test1', 'thumbnails'))
expect(files2).to.have.lengthOf(0)
await checkVideoFilesWereRemoved(videoUUID, 1)
})
it('Should not have videos', async function () {
@ -346,7 +341,7 @@ describe('Test a single server', function () {
for (const video of videos) {
const videoName = video.name.replace(' name', '')
const test = await testVideoImage(server.url, videoName, video.thumbnailPath)
const test = await testImage(server.url, videoName, video.thumbnailPath)
expect(test).to.equal(true)
}

View File

@ -3,7 +3,7 @@
import * as chai from 'chai'
import 'mocha'
import { VideoComment, VideoCommentThreadTree } from '../../../../shared/models/videos/video-comment.model'
import { testVideoImage } from '../../utils'
import { testImage } from '../../utils'
import {
dateIsValid, flushTests, killallServers, runServer, ServerInfo, setAccessTokensToServers, updateMyAvatar,
uploadVideo
@ -83,7 +83,7 @@ describe('Test video comments', function () {
expect(comment.account.name).to.equal('root')
expect(comment.account.host).to.equal('localhost:9001')
const test = await testVideoImage(server.url, 'avatar-resized', comment.account.avatar.path, '.png')
const test = await testImage(server.url, 'avatar-resized', comment.account.avatar.path, '.png')
expect(test).to.equal(true)
expect(comment.totalReplies).to.equal(0)

View File

@ -1,3 +1,4 @@
import { join } from 'path'
import * as WebTorrent from 'webtorrent'
let webtorrent = new WebTorrent()
@ -24,11 +25,17 @@ function webtorrentAdd (torrent: string, refreshWebTorrent = false) {
return new Promise<WebTorrent.Torrent>(res => webtorrent.add(torrent, res))
}
function root () {
// We are in server/tests/utils/miscs
return join(__dirname, '..', '..', '..', '..')
}
// ---------------------------------------------------------------------------
export {
dateIsValid,
wait,
webtorrentAdd,
immutableAssign
immutableAssign,
root
}

View File

@ -1,5 +1,11 @@
/* tslint:disable:no-unused-expression */
import { expect } from 'chai'
import { existsSync } from 'fs'
import { join } from 'path'
import { Account } from '../../../../shared/models/actors'
import { readdirPromise } from '../../../helpers/core-utils'
import { root } from '../index'
import { makeGetRequest } from '../requests/requests'
function getAccountsList (url: string, sort = '-createdAt', statusCodeExpected = 200) {
@ -32,10 +38,27 @@ async function expectAccountFollows (url: string, nameWithDomain: string, follow
expect(account.followingCount).to.equal(followingCount, message)
}
async function checkActorFilesWereRemoved (actorUUID: string, serverNumber: number) {
const testDirectory = 'test' + serverNumber
for (const directory of [ 'avatars' ]) {
const directoryPath = join(root(), testDirectory, directory)
const directoryExists = existsSync(directoryPath)
expect(directoryExists).to.be.true
const files = await readdirPromise(directoryPath)
for (const file of files) {
expect(file).to.not.contain(actorUUID)
}
}
}
// ---------------------------------------------------------------------------
export {
getAccount,
expectAccountFollows,
getAccountsList
getAccountsList,
checkActorFilesWereRemoved
}

View File

@ -11,7 +11,7 @@ function createUser (
password: string,
videoQuota = 1000000,
role: UserRole = UserRole.USER,
specialStatus = 204
specialStatus = 200
) {
const path = '/api/v1/users'
const body = {

View File

@ -1,13 +1,13 @@
/* tslint:disable:no-unused-expression */
import { expect } from 'chai'
import { readFile } from 'fs'
import { existsSync, readFile } from 'fs'
import * as parseTorrent from 'parse-torrent'
import { extname, isAbsolute, join } from 'path'
import * as request from 'supertest'
import { getMyUserInformation, makeGetRequest, ServerInfo } from '../'
import { getMyUserInformation, makeGetRequest, root, ServerInfo } from '../'
import { VideoPrivacy } from '../../../../shared/models/videos'
import { readFileBufferPromise } from '../../../helpers/core-utils'
import { readdirPromise, readFileBufferPromise } from '../../../helpers/core-utils'
import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../initializers'
import { dateIsValid, webtorrentAdd } from '../index'
@ -203,7 +203,23 @@ function searchVideoWithSort (url: string, search: string, sort: string) {
.expect('Content-Type', /json/)
}
async function testVideoImage (url: string, imageName: string, imagePath: string, extension = '.jpg') {
async function checkVideoFilesWereRemoved (videoUUID: string, serverNumber: number) {
const testDirectory = 'test' + serverNumber
for (const directory of [ 'videos', 'thumbnails', 'torrents', 'previews' ]) {
const directoryPath = join(root(), testDirectory, directory)
const directoryExists = existsSync(directoryPath)
expect(directoryExists).to.be.true
const files = await readdirPromise(directoryPath)
for (const file of files) {
expect(file).to.not.contain(videoUUID)
}
}
}
async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') {
// Don't test images if the node env is not set
// Because we need a special ffmpeg version for this test
if (process.env['NODE_TEST_IMAGE']) {
@ -409,7 +425,7 @@ async function completeVideoCheck (
const maxSize = attributeFile.size + ((10 * attributeFile.size) / 100)
expect(file.size).to.be.above(minSize).and.below(maxSize)
const test = await testVideoImage(url, attributes.fixture, videoDetails.thumbnailPath)
const test = await testImage(url, attributes.fixture, videoDetails.thumbnailPath)
expect(test).to.equal(true)
const torrent = await webtorrentAdd(magnetUri, true)
@ -437,11 +453,12 @@ export {
searchVideo,
searchVideoWithPagination,
searchVideoWithSort,
testVideoImage,
testImage,
uploadVideo,
updateVideo,
rateVideo,
viewVideo,
parseTorrentVideo,
completeVideoCheck
completeVideoCheck,
checkVideoFilesWereRemoved
}

View File

@ -3796,9 +3796,9 @@ sequelize-typescript@^0.6.1:
es6-shim "0.35.3"
glob "7.1.2"
sequelize@4.25.2:
version "4.25.2"
resolved "https://registry.yarnpkg.com/sequelize/-/sequelize-4.25.2.tgz#fa4a95b9ec3cefb948ecb2dc5965ccf716f98c68"
sequelize@4.31.2:
version "4.31.2"
resolved "https://registry.yarnpkg.com/sequelize/-/sequelize-4.31.2.tgz#4b414c39bac18ae74946ed49b300f5bc7e423462"
dependencies:
bluebird "^3.4.6"
cls-bluebird "^2.0.1"