From 687d638c2bee0d223f206168173b1b95adbad983 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 24 Aug 2018 15:36:50 +0200 Subject: [PATCH] Fetch outbox when searching an actor --- client/src/app/search/search.component.scss | 2 - client/src/assets/images/menu/podcasts.svg | 26 -------- server/controllers/api/accounts.ts | 4 +- server/controllers/api/search.ts | 11 +--- server/controllers/api/video-channel.ts | 4 +- server/helpers/express-utils.ts | 8 +++ server/lib/activitypub/actor.ts | 31 ++++++--- server/models/video/video.ts | 63 ++++++++++--------- .../search-activitypub-video-channels.ts | 52 +++++++++++++-- 9 files changed, 118 insertions(+), 83 deletions(-) delete mode 100644 client/src/assets/images/menu/podcasts.svg diff --git a/client/src/app/search/search.component.scss b/client/src/app/search/search.component.scss index be7dd39cf..e5dfddcc5 100644 --- a/client/src/app/search/search.component.scss +++ b/client/src/app/search/search.component.scss @@ -113,8 +113,6 @@ } .video-channel-info { - - flex-grow: 1; width: fit-content; diff --git a/client/src/assets/images/menu/podcasts.svg b/client/src/assets/images/menu/podcasts.svg deleted file mode 100644 index cd6efc54e..000000000 --- a/client/src/assets/images/menu/podcasts.svg +++ /dev/null @@ -1,26 +0,0 @@ - - - - podcasts - Created with Sketch. - - - - - - - - - - - - - - - - - - - - - diff --git a/server/controllers/api/accounts.ts b/server/controllers/api/accounts.ts index 7b7e5e740..b7691ccba 100644 --- a/server/controllers/api/accounts.ts +++ b/server/controllers/api/accounts.ts @@ -11,7 +11,7 @@ import { import { accountsNameWithHostGetValidator, accountsSortValidator, videosSortValidator } from '../../middlewares/validators' import { AccountModel } from '../../models/account/account' import { VideoModel } from '../../models/video/video' -import { buildNSFWFilter } from '../../helpers/express-utils' +import { buildNSFWFilter, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils' import { VideoChannelModel } from '../../models/video/video-channel' const accountsRouter = express.Router() @@ -73,8 +73,10 @@ async function listVideoAccountChannels (req: express.Request, res: express.Resp async function listAccountVideos (req: express.Request, res: express.Response, next: express.NextFunction) { const account: AccountModel = res.locals.account + const actorId = isUserAbleToSearchRemoteURI(res) ? null : undefined const resultList = await VideoModel.listForApi({ + actorId, start: req.query.start, count: req.query.count, sort: req.query.sort, diff --git a/server/controllers/api/search.ts b/server/controllers/api/search.ts index 959d79855..bb7174891 100644 --- a/server/controllers/api/search.ts +++ b/server/controllers/api/search.ts @@ -1,5 +1,5 @@ import * as express from 'express' -import { buildNSFWFilter } from '../../helpers/express-utils' +import { buildNSFWFilter, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils' import { getFormattedObjects, getServerActor } from '../../helpers/utils' import { VideoModel } from '../../models/video/video' import { @@ -88,7 +88,7 @@ async function searchVideoChannelURI (search: string, isWebfingerSearch: boolean if (isUserAbleToSearchRemoteURI(res)) { try { - const actor = await getOrCreateActorAndServerAndModel(uri) + const actor = await getOrCreateActorAndServerAndModel(uri, true, true) videoChannel = actor.VideoChannel } catch (err) { logger.info('Cannot search remote video channel %s.', uri, { err }) @@ -152,10 +152,3 @@ async function searchVideoURI (url: string, res: express.Response) { data: video ? [ video.toFormattedJSON() ] : [] }) } - -function isUserAbleToSearchRemoteURI (res: express.Response) { - const user: User = res.locals.oauth ? res.locals.oauth.token.User : undefined - - return CONFIG.SEARCH.REMOTE_URI.ANONYMOUS === true || - (CONFIG.SEARCH.REMOTE_URI.USERS === true && user !== undefined) -} diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts index bd08d7a08..a7a36080b 100644 --- a/server/controllers/api/video-channel.ts +++ b/server/controllers/api/video-channel.ts @@ -19,7 +19,7 @@ import { videoChannelsNameWithHostValidator, videosSortValidator } from '../../m import { sendUpdateActor } from '../../lib/activitypub/send' import { VideoChannelCreate, VideoChannelUpdate } from '../../../shared' import { createVideoChannel } from '../../lib/video-channel' -import { buildNSFWFilter, createReqFiles } from '../../helpers/express-utils' +import { buildNSFWFilter, createReqFiles, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils' import { setAsyncActorKeys } from '../../lib/activitypub' import { AccountModel } from '../../models/account/account' import { CONFIG, IMAGE_MIMETYPE_EXT, sequelizeTypescript } from '../../initializers' @@ -210,8 +210,10 @@ async function getVideoChannel (req: express.Request, res: express.Response, nex async function listVideoChannelVideos (req: express.Request, res: express.Response, next: express.NextFunction) { const videoChannelInstance: VideoChannelModel = res.locals.videoChannel + const actorId = isUserAbleToSearchRemoteURI(res) ? null : undefined const resultList = await VideoModel.listForApi({ + actorId, start: req.query.start, count: req.query.count, sort: req.query.sort, diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts index 1d7bee87e..b715fb7d0 100644 --- a/server/helpers/express-utils.ts +++ b/server/helpers/express-utils.ts @@ -95,11 +95,19 @@ function createReqFiles ( return multer({ storage }).fields(fields) } +function isUserAbleToSearchRemoteURI (res: express.Response) { + const user: User = res.locals.oauth ? res.locals.oauth.token.User : undefined + + return CONFIG.SEARCH.REMOTE_URI.ANONYMOUS === true || + (CONFIG.SEARCH.REMOTE_URI.USERS === true && user !== undefined) +} + // --------------------------------------------------------------------------- export { buildNSFWFilter, getHostWithPort, + isUserAbleToSearchRemoteURI, badRequest, createReqFiles, cleanUpReqFiles diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index 22e1c9f19..1657262d7 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -36,8 +36,13 @@ function setAsyncActorKeys (actor: ActorModel) { }) } -async function getOrCreateActorAndServerAndModel (activityActor: string | ActivityPubActor, recurseIfNeeded = true) { +async function getOrCreateActorAndServerAndModel ( + activityActor: string | ActivityPubActor, + recurseIfNeeded = true, + updateCollections = false +) { const actorUrl = getActorUrl(activityActor) + let created = false let actor = await ActorModel.loadByUrl(actorUrl) // Orphan actor (not associated to an account of channel) so recreate it @@ -68,15 +73,21 @@ async function getOrCreateActorAndServerAndModel (activityActor: string | Activi } actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, result, ownerActor) + created = true } if (actor.Account) actor.Account.Actor = actor if (actor.VideoChannel) actor.VideoChannel.Actor = actor - actor = await retryTransactionWrapper(refreshActorIfNeeded, actor) - if (!actor) throw new Error('Actor ' + actor.url + ' does not exist anymore.') + const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor) + if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.') - return actor + if ((created === true || refreshed === true) && updateCollections === true) { + const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' } + await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) + } + + return actorRefreshed } function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) { @@ -359,8 +370,8 @@ async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResu return videoChannelCreated } -async function refreshActorIfNeeded (actor: ActorModel): Promise { - if (!actor.isOutdated()) return actor +async function refreshActorIfNeeded (actor: ActorModel): Promise<{ actor: ActorModel, refreshed: boolean }> { + if (!actor.isOutdated()) return { actor, refreshed: false } try { const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) @@ -369,12 +380,12 @@ async function refreshActorIfNeeded (actor: ActorModel): Promise { if (statusCode === 404) { logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url) actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy() - return undefined + return { actor: undefined, refreshed: false } } if (result === undefined) { logger.warn('Cannot fetch remote actor in refresh actor.') - return actor + return { actor, refreshed: false } } return sequelizeTypescript.transaction(async t => { @@ -403,10 +414,10 @@ async function refreshActorIfNeeded (actor: ActorModel): Promise { await actor.VideoChannel.save({ transaction: t }) } - return actor + return { refreshed: true, actor } }) } catch (err) { logger.warn('Cannot refresh actor.', { err }) - return actor + return { actor, refreshed: false } } } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 7acbc60f7..a956da16e 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -201,39 +201,12 @@ type AvailableForListOptions = { ] } - // Force actorId to be a number to avoid SQL injections - const actorIdNumber = parseInt(options.actorId.toString(), 10) - let localVideosReq = '' - if (options.includeLocalVideos === true) { - localVideosReq = ' UNION ALL ' + - 'SELECT "video"."id" AS "id" FROM "video" ' + - 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' + - 'INNER JOIN "account" ON "account"."id" = "videoChannel"."accountId" ' + - 'INNER JOIN "actor" ON "account"."actorId" = "actor"."id" ' + - 'WHERE "actor"."serverId" IS NULL' - } - // FIXME: It would be more efficient to use a CTE so we join AFTER the filters, but sequelize does not support it... const query: IFindOptions = { where: { id: { [Sequelize.Op.notIn]: Sequelize.literal( '(SELECT "videoBlacklist"."videoId" FROM "videoBlacklist")' - ), - [ Sequelize.Op.in ]: Sequelize.literal( - '(' + - 'SELECT "videoShare"."videoId" AS "id" FROM "videoShare" ' + - 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "videoShare"."actorId" ' + - 'WHERE "actorFollow"."actorId" = ' + actorIdNumber + - ' UNION ALL ' + - 'SELECT "video"."id" AS "id" FROM "video" ' + - 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' + - 'INNER JOIN "account" ON "account"."id" = "videoChannel"."accountId" ' + - 'INNER JOIN "actor" ON "account"."actorId" = "actor"."id" ' + - 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "actor"."id" ' + - 'WHERE "actorFollow"."actorId" = ' + actorIdNumber + - localVideosReq + - ')' ) }, // Always list public videos @@ -254,6 +227,36 @@ type AvailableForListOptions = { include: [ videoChannelInclude ] } + if (options.actorId) { + let localVideosReq = '' + if (options.includeLocalVideos === true) { + localVideosReq = ' UNION ALL ' + + 'SELECT "video"."id" AS "id" FROM "video" ' + + 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' + + 'INNER JOIN "account" ON "account"."id" = "videoChannel"."accountId" ' + + 'INNER JOIN "actor" ON "account"."actorId" = "actor"."id" ' + + 'WHERE "actor"."serverId" IS NULL' + } + + // Force actorId to be a number to avoid SQL injections + const actorIdNumber = parseInt(options.actorId.toString(), 10) + query.where['id'][ Sequelize.Op.in ] = Sequelize.literal( + '(' + + 'SELECT "videoShare"."videoId" AS "id" FROM "videoShare" ' + + 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "videoShare"."actorId" ' + + 'WHERE "actorFollow"."actorId" = ' + actorIdNumber + + ' UNION ALL ' + + 'SELECT "video"."id" AS "id" FROM "video" ' + + 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' + + 'INNER JOIN "account" ON "account"."id" = "videoChannel"."accountId" ' + + 'INNER JOIN "actor" ON "account"."actorId" = "actor"."id" ' + + 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "actor"."id" ' + + 'WHERE "actorFollow"."actorId" = ' + actorIdNumber + + localVideosReq + + ')' + ) + } + if (options.withFiles === true) { query.include.push({ model: VideoFileModel.unscoped(), @@ -849,7 +852,8 @@ export class VideoModel extends Model { order: getSort(options.sort) } - const actorId = options.actorId || (await getServerActor()).id + // actorId === null has a meaning, so just check undefined + const actorId = options.actorId !== undefined ? options.actorId : (await getServerActor()).id const scopes = { method: [ @@ -926,7 +930,8 @@ export class VideoModel extends Model { id: { [ Sequelize.Op.in ]: Sequelize.literal( '(' + - 'SELECT "video"."id" FROM "video" WHERE ' + + 'SELECT "video"."id" FROM "video" ' + + 'WHERE ' + 'lower(immutable_unaccent("video"."name")) % lower(immutable_unaccent(' + escapedSearch + ')) OR ' + 'lower(immutable_unaccent("video"."name")) LIKE lower(immutable_unaccent(' + escapedLikeSearch + '))' + 'UNION ALL ' + diff --git a/server/tests/api/search/search-activitypub-video-channels.ts b/server/tests/api/search/search-activitypub-video-channels.ts index 512cb32fd..a287c5bdf 100644 --- a/server/tests/api/search/search-activitypub-video-channels.ts +++ b/server/tests/api/search/search-activitypub-video-channels.ts @@ -8,11 +8,11 @@ import { deleteVideoChannel, flushAndRunMultipleServers, flushTests, - getVideoChannelsList, + getVideoChannelsList, getVideoChannelVideos, killallServers, ServerInfo, setAccessTokensToServers, - updateMyUser, + updateMyUser, updateVideo, updateVideoChannel, uploadVideo, userLogin, @@ -27,6 +27,8 @@ const expect = chai.expect describe('Test a ActivityPub video channels search', function () { let servers: ServerInfo[] let userServer2Token: string + let videoServer2UUID: string + let channelIdServer2: number before(async function () { this.timeout(120000) @@ -56,10 +58,10 @@ describe('Test a ActivityPub video channels search', function () { displayName: 'Channel 1 server 2' } const resChannel = await addVideoChannel(servers[1].url, userServer2Token, channel) - const channelId = resChannel.body.videoChannel.id + channelIdServer2 = resChannel.body.videoChannel.id - await uploadVideo(servers[1].url, userServer2Token, { name: 'video 1 server 2', channelId }) - await uploadVideo(servers[1].url, userServer2Token, { name: 'video 2 server 2', channelId }) + const res = await uploadVideo(servers[1].url, userServer2Token, { name: 'video 1 server 2', channelId: channelIdServer2 }) + videoServer2UUID = res.body.video.uuid } await waitJobs(servers) @@ -129,6 +131,23 @@ describe('Test a ActivityPub video channels search', function () { expect(res.body.data[2].name).to.equal('root_channel') }) + it('Should list video channel videos of server 2 without token', async function () { + this.timeout(30000) + + await waitJobs(servers) + + const res = await getVideoChannelVideos(servers[0].url, null, 'channel1_server2@localhost:9002', 0, 5) + expect(res.body.total).to.equal(0) + expect(res.body.data).to.have.lengthOf(0) + }) + + it('Should list video channel videos of server 2 with token', async function () { + const res = await getVideoChannelVideos(servers[0].url, servers[0].accessToken, 'channel1_server2@localhost:9002', 0, 5) + + expect(res.body.total).to.equal(1) + expect(res.body.data[0].name).to.equal('video 1 server 2') + }) + it('Should update video channel of server 2, and refresh it on server 1', async function () { this.timeout(60000) @@ -151,6 +170,29 @@ describe('Test a ActivityPub video channels search', function () { // expect(videoChannel.ownerAccount.displayName).to.equal('user updated') }) + it('Should update and add a video on server 2, and update it on server 1 after a search', async function () { + this.timeout(60000) + + await updateVideo(servers[1].url, userServer2Token, videoServer2UUID, { name: 'video 1 updated' }) + await uploadVideo(servers[1].url, userServer2Token, { name: 'video 2 server 2', channelId: channelIdServer2 }) + + await waitJobs(servers) + + // Expire video channel + await wait(10000) + + const search = 'http://localhost:9002/video-channels/channel1_server2' + await searchVideoChannel(servers[0].url, search, servers[0].accessToken) + + await waitJobs(servers) + + const res = await getVideoChannelVideos(servers[0].url, servers[0].accessToken, 'channel1_server2@localhost:9002', 0, 5, '-createdAt') + + expect(res.body.total).to.equal(2) + expect(res.body.data[0].name).to.equal('video 2 server 2') + expect(res.body.data[1].name).to.equal('video 1 updated') + }) + it('Should delete video channel of server 2, and delete it on server 1', async function () { this.timeout(60000)