OpenCloud
/
PeerTube
mirror of https://github.com/Chocobozzz/PeerTube
1
0
Fork 0
Code Issues Releases Wiki Activity

Fix 404 AP status codes

pull/3888/head
Chocobozzz 2021-03-09 14:01:44 +01:00
parent 2d5a469427
commit b5c361089f
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
7 changed files with 119 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
server/helpers/requests.ts
View File

@ -1,5 +1,5 @@
import { createWriteStream, remove } from 'fs-extra'
import got, { CancelableRequest, Options as GotOptions } from 'got'
import got, { CancelableRequest, Options as GotOptions, RequestError } from 'got'
import { join } from 'path'
import { CONFIG } from '../initializers/config'
import { ACTIVITY_PUB, PEERTUBE_VERSION, WEBSERVER } from '../initializers/constants'
@ -7,6 +7,11 @@ import { pipelinePromise } from './core-utils'
import { processImage } from './image-utils'
import { logger } from './logger'
export interface PeerTubeRequestError extends Error {
statusCode?: number
responseBody?: any
}
const httpSignature = require('http-signature')
type PeerTubeRequestOptions = {
@ -180,14 +185,15 @@ function buildGotOptions (options: PeerTubeRequestOptions) {
}
}
function buildRequestError (error: any) {
const newError = new Error(error.message)
function buildRequestError (error: RequestError) {
const newError: PeerTubeRequestError = new Error(error.message)
newError.name = error.name
newError.stack = error.stack
if (error.response?.body) {
error.responseBody = error.response.body
if (error.response) {
newError.responseBody = error.response.body
newError.statusCode = error.response.statusCode
}
return error
return newError
}

22
server/lib/activitypub/actor.ts
View File

@ -14,7 +14,7 @@ import { isActivityPubUrlValid } from '../../helpers/custom-validators/activityp
import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
import { logger } from '../../helpers/logger'
import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
import { doJSONRequest } from '../../helpers/requests'
import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests'
import { getUrlFromWebfinger } from '../../helpers/webfinger'
import { MIMETYPES, WEBSERVER } from '../../initializers/constants'
import { sequelizeTypescript } from '../../initializers/database'
@ -279,16 +279,7 @@ async function refreshActorIfNeeded <T extends MActorFull | MActorAccountChannel
actorUrl = actor.url
}
const { result, statusCode } = await fetchRemoteActor(actorUrl)
if (statusCode === HttpStatusCode.NOT_FOUND_404) {
logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
actor.Account
? await actor.Account.destroy()
: await actor.VideoChannel.destroy()
return { actor: undefined, refreshed: false }
}
const { result } = await fetchRemoteActor(actorUrl)
if (result === undefined) {
logger.warn('Cannot fetch remote actor in refresh actor.')
@ -328,6 +319,15 @@ async function refreshActorIfNeeded <T extends MActorFull | MActorAccountChannel
return { refreshed: true, actor }
})
} catch (err) {
if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url)
actor.Account
? await actor.Account.destroy()
: await actor.VideoChannel.destroy()
return { actor: undefined, refreshed: false }
}
logger.warn('Cannot refresh actor %s.', actor.url, { err })
return { actor, refreshed: false }
}

17
server/lib/activitypub/playlist.ts
View File

@ -7,7 +7,7 @@ import { checkUrlsSameHost } from '../../helpers/activitypub'
import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
import { isArray } from '../../helpers/custom-validators/misc'
import { logger } from '../../helpers/logger'
import { doJSONRequest } from '../../helpers/requests'
import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests'
import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
import { sequelizeTypescript } from '../../initializers/database'
import { VideoPlaylistModel } from '../../models/video/video-playlist'
@ -116,13 +116,7 @@ async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner)
if (!videoPlaylist.isOutdated()) return videoPlaylist
try {
const { statusCode, playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
if (statusCode === HttpStatusCode.NOT_FOUND_404) {
logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
await videoPlaylist.destroy()
return undefined
}
const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
if (playlistObject === undefined) {
logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url)
@ -136,6 +130,13 @@ async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner)
return videoPlaylist
} catch (err) {
if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
await videoPlaylist.destroy()
return undefined
}
logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err })
await videoPlaylist.setAsRefreshed()

19
server/lib/activitypub/videos.ts
View File

@ -30,7 +30,7 @@ import { isArray } from '../../helpers/custom-validators/misc'
import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
import { logger } from '../../helpers/logger'
import { doJSONRequest } from '../../helpers/requests'
import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests'
import { fetchVideoByUrl, getExtFromMimetype, VideoFetchByUrlType } from '../../helpers/video'
import {
ACTIVITY_PUB,
@ -523,14 +523,7 @@ async function refreshVideoIfNeeded (options: {
: await VideoModel.loadByUrlAndPopulateAccount(options.video.url)
try {
const { statusCode, videoObject } = await fetchRemoteVideo(video.url)
if (statusCode === HttpStatusCode.NOT_FOUND_404) {
logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
// Video does not exist anymore
await video.destroy()
return undefined
}
const { videoObject } = await fetchRemoteVideo(video.url)
if (videoObject === undefined) {
logger.warn('Cannot refresh remote video %s: invalid body.', video.url)
@ -554,6 +547,14 @@ async function refreshVideoIfNeeded (options: {
return video
} catch (err) {
if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url)
// Video does not exist anymore
await video.destroy()
return undefined
}
logger.warn('Cannot refresh video %s.', options.video.url, { err })
ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId)

51
server/lib/job-queue/handlers/activitypub-cleaner.ts
View File

@ -7,7 +7,7 @@ import {
isLikeActivityValid
} from '@server/helpers/custom-validators/activitypub/activity'
import { sanitizeAndCheckVideoCommentObject } from '@server/helpers/custom-validators/activitypub/video-comments'
import { doJSONRequest } from '@server/helpers/requests'
import { doJSONRequest, PeerTubeRequestError } from '@server/helpers/requests'
import { AP_CLEANER_CONCURRENCY } from '@server/initializers/constants'
import { VideoModel } from '@server/models/video/video'
import { VideoCommentModel } from '@server/models/video/video-comment'
@ -81,39 +81,44 @@ async function updateObjectIfNeeded <T> (
updater: (url: string, newUrl: string) => Promise<T>,
deleter: (url: string) => Promise<T>
): Promise<{ data: T, status: 'deleted' | 'updated' } | null> {
const { statusCode, body } = await doJSONRequest<any>(url, { activityPub: true })
// Does not exist anymore, remove entry
if (statusCode === HttpStatusCode.NOT_FOUND_404) {
const on404OrTombstone = async () => {
logger.info('Removing remote AP object %s.', url)
const data = await deleter(url)
return { status: 'deleted', data }
return { status: 'deleted' as 'deleted', data }
}
// If not same id, check same host and update
if (!body || !body.id || !bodyValidator(body)) throw new Error(`Body or body id of ${url} is invalid`)
try {
const { body } = await doJSONRequest<any>(url, { activityPub: true })
if (body.type === 'Tombstone') {
logger.info('Removing remote AP object %s.', url)
const data = await deleter(url)
// If not same id, check same host and update
if (!body || !body.id || !bodyValidator(body)) throw new Error(`Body or body id of ${url} is invalid`)
return { status: 'deleted', data }
}
const newUrl = body.id
if (newUrl !== url) {
if (checkUrlsSameHost(newUrl, url) !== true) {
throw new Error(`New url ${newUrl} has not the same host than old url ${url}`)
if (body.type === 'Tombstone') {
return on404OrTombstone()
}
logger.info('Updating remote AP object %s.', url)
const data = await updater(url, newUrl)
const newUrl = body.id
if (newUrl !== url) {
if (checkUrlsSameHost(newUrl, url) !== true) {
throw new Error(`New url ${newUrl} has not the same host than old url ${url}`)
}
return { status: 'updated', data }
logger.info('Updating remote AP object %s.', url)
const data = await updater(url, newUrl)
return { status: 'updated', data }
}
return null
} catch (err) {
// Does not exist anymore, remove entry
if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
return on404OrTombstone()
}
throw err
}
return null
}
function rateOptionsFactory () {

73
server/tests/api/activitypub/security.ts
View File

@ -79,9 +79,12 @@ describe('Test ActivityPub security', function () {
Digest: buildDigest({ hello: 'coucou' })
}
const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
try {
await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
expect(true, 'Did not throw').to.be.false
} catch (err) {
expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
}
})
it('Should fail with an invalid date', async function () {
@ -89,9 +92,12 @@ describe('Test ActivityPub security', function () {
const headers = buildGlobalHeaders(body)
headers['date'] = 'Wed, 21 Oct 2015 07:28:00 GMT'
const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
try {
await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
expect(true, 'Did not throw').to.be.false
} catch (err) {
expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
}
})
it('Should fail with bad keys', async function () {
@ -101,9 +107,12 @@ describe('Test ActivityPub security', function () {
const body = activityPubContextify(getAnnounceWithoutContext(servers[1]))
const headers = buildGlobalHeaders(body)
const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
try {
await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
expect(true, 'Did not throw').to.be.false
} catch (err) {
expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
}
})
it('Should reject requests without appropriate signed headers', async function () {
@ -123,8 +132,12 @@ describe('Test ActivityPub security', function () {
for (const badHeaders of badHeadersMatrix) {
signatureOptions.headers = badHeaders
const { statusCode } = await makePOSTAPRequest(url, body, signatureOptions, headers)
expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
try {
await makePOSTAPRequest(url, body, signatureOptions, headers)
expect(true, 'Did not throw').to.be.false
} catch (err) {
expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
}
}
})
@ -133,7 +146,6 @@ describe('Test ActivityPub security', function () {
const headers = buildGlobalHeaders(body)
const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
expect(statusCode).to.equal(HttpStatusCode.NO_CONTENT_204)
})
@ -150,9 +162,12 @@ describe('Test ActivityPub security', function () {
const body = activityPubContextify(getAnnounceWithoutContext(servers[1]))
const headers = buildGlobalHeaders(body)
const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
try {
await makePOSTAPRequest(url, body, baseHttpSignature(), headers)
expect(true, 'Did not throw').to.be.false
} catch (err) {
expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
}
})
})
@ -183,9 +198,12 @@ describe('Test ActivityPub security', function () {
const headers = buildGlobalHeaders(signedBody)
const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
try {
await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
expect(true, 'Did not throw').to.be.false
} catch (err) {
expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
}
})
it('Should fail with an altered body', async function () {
@ -204,9 +222,12 @@ describe('Test ActivityPub security', function () {
const headers = buildGlobalHeaders(signedBody)
const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
try {
await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
expect(true, 'Did not throw').to.be.false
} catch (err) {
expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
}
})
it('Should succeed with a valid signature', async function () {
@ -221,7 +242,6 @@ describe('Test ActivityPub security', function () {
const headers = buildGlobalHeaders(signedBody)
const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
expect(statusCode).to.equal(HttpStatusCode.NO_CONTENT_204)
})
@ -243,9 +263,12 @@ describe('Test ActivityPub security', function () {
const headers = buildGlobalHeaders(signedBody)
const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
try {
await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers)
expect(true, 'Did not throw').to.be.false
} catch (err) {
expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)
}
})
})

5
server/tools/peertube-import-videos.ts
View File

@ -202,10 +202,7 @@ async function uploadVideoOnPeerTube (parameters: {
if (videoInfo.thumbnail) {
thumbnailfile = join(cwd, sha256(videoInfo.thumbnail) + '.jpg')
await doRequestAndSaveToFile({
method: 'GET',
uri: videoInfo.thumbnail
}, thumbnailfile)
await doRequestAndSaveToFile(videoInfo.thumbnail, thumbnailfile)
}
const originallyPublishedAt = buildOriginallyPublishedAt(videoInfo)