From 3acc50844047a37698f0618fa235c138e386a053 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 23 Apr 2019 09:50:57 +0200 Subject: [PATCH] Upgrade sequelize --- .gitignore | 2 +- package.json | 6 +- server/controllers/api/video-playlist.ts | 20 +- server/controllers/api/videos/import.ts | 16 +- server/controllers/api/videos/index.ts | 31 ++- server/controllers/bots.ts | 2 +- server/controllers/feeds.ts | 2 +- server/controllers/static.ts | 12 +- server/initializers/database.ts | 6 +- server/lib/activitypub/playlist.ts | 14 +- server/lib/activitypub/video-comments.ts | 3 +- server/lib/activitypub/videos.ts | 34 ++-- .../abstract-video-static-file-cache.ts | 18 +- .../lib/files-cache/videos-caption-cache.ts | 9 +- .../lib/files-cache/videos-preview-cache.ts | 6 +- server/lib/job-queue/handlers/video-import.ts | 20 +- server/lib/oauth-model.ts | 2 + server/lib/thumbnail.ts | 26 +-- server/middlewares/oauth.ts | 2 + .../middlewares/validators/videos/videos.ts | 1 + server/models/account/account-blocklist.ts | 10 +- server/models/account/account.ts | 10 +- server/models/account/user-notification.ts | 44 ++--- server/models/account/user.ts | 58 +++--- server/models/activitypub/actor.ts | 47 ++--- server/models/application/application.ts | 6 +- server/models/oauth/oauth-client.ts | 4 +- server/models/oauth/oauth-token.ts | 22 ++- server/models/redundancy/video-redundancy.ts | 56 +++--- server/models/server/server-blocklist.ts | 8 +- server/models/utils.ts | 12 +- server/models/video/tag.ts | 2 +- server/models/video/thumbnail.ts | 4 +- server/models/video/video-caption.ts | 13 +- server/models/video/video-change-ownership.ts | 14 +- server/models/video/video-channel.ts | 16 +- server/models/video/video-comment.ts | 33 ++-- server/models/video/video-file.ts | 27 ++- server/models/video/video-format-utils.ts | 10 +- server/models/video/video-import.ts | 8 +- server/models/video/video-playlist.ts | 47 ++--- server/models/video/video-share.ts | 10 +- .../models/video/video-streaming-playlist.ts | 6 +- server/models/video/video.ts | 179 +++++++++--------- server/typings/sequelize.ts | 18 ++ shared/extra-utils/miscs/sql.ts | 1 - shared/models/videos/thumbnail.type.ts | 2 +- yarn.lock | 24 +-- 48 files changed, 457 insertions(+), 466 deletions(-) create mode 100644 server/typings/sequelize.ts diff --git a/.gitignore b/.gitignore index 681004527..8cd04eea1 100644 --- a/.gitignore +++ b/.gitignore @@ -16,8 +16,8 @@ /config/production.yaml /config/local* /ffmpeg/ -/ffmpeg-4/ /ffmpeg-3/ +/ffmpeg-4/ /thumbnails/ /torrents/ /videos/ diff --git a/package.json b/package.json index 20cbd555a..41a040c59 100644 --- a/package.json +++ b/package.json @@ -142,8 +142,8 @@ "reflect-metadata": "^0.1.12", "request": "^2.81.0", "scripty": "^1.5.0", - "sequelize": "5.6.1", - "sequelize-typescript": "^1.0.0-beta.1", + "sequelize": "5.7.4", + "sequelize-typescript": "1.0.0-beta.2", "sharp": "^0.22.0", "sitemap": "^2.1.0", "socket.io": "^2.2.0", @@ -212,7 +212,7 @@ "ts-node": "8.0.3", "tslint": "^5.7.0", "tslint-config-standard": "^8.0.1", - "typescript": "^3.1.6", + "typescript": "^3.4.3", "xliff": "^4.0.0" }, "scripty": { diff --git a/server/controllers/api/video-playlist.ts b/server/controllers/api/video-playlist.ts index 99325aa9d..6a1d23529 100644 --- a/server/controllers/api/video-playlist.ts +++ b/server/controllers/api/video-playlist.ts @@ -41,7 +41,7 @@ import { VideoPlaylistReorder } from '../../../shared/models/videos/playlist/vid import { JobQueue } from '../../lib/job-queue' import { CONFIG } from '../../initializers/config' import { sequelizeTypescript } from '../../initializers/database' -import { createPlaylistThumbnailFromExisting } from '../../lib/thumbnail' +import { createPlaylistMiniatureFromExisting } from '../../lib/thumbnail' const reqThumbnailFile = createReqFiles([ 'thumbnailfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, { thumbnailfile: CONFIG.STORAGE.TMP_DIR }) @@ -174,16 +174,13 @@ async function addVideoPlaylist (req: express.Request, res: express.Response) { const thumbnailField = req.files['thumbnailfile'] const thumbnailModel = thumbnailField - ? await createPlaylistThumbnailFromExisting(thumbnailField[0].path, videoPlaylist) + ? await createPlaylistMiniatureFromExisting(thumbnailField[0].path, videoPlaylist) : undefined const videoPlaylistCreated: VideoPlaylistModel = await sequelizeTypescript.transaction(async t => { const videoPlaylistCreated = await videoPlaylist.save({ transaction: t }) - if (thumbnailModel) { - thumbnailModel.videoPlaylistId = videoPlaylistCreated.id - videoPlaylistCreated.setThumbnail(await thumbnailModel.save({ transaction: t })) - } + if (thumbnailModel) await videoPlaylistCreated.setAndSaveThumbnail(thumbnailModel, t) // We need more attributes for the federation videoPlaylistCreated.OwnerAccount = await AccountModel.load(user.Account.id, t) @@ -210,7 +207,7 @@ async function updateVideoPlaylist (req: express.Request, res: express.Response) const thumbnailField = req.files['thumbnailfile'] const thumbnailModel = thumbnailField - ? await createPlaylistThumbnailFromExisting(thumbnailField[0].path, videoPlaylistInstance) + ? await createPlaylistMiniatureFromExisting(thumbnailField[0].path, videoPlaylistInstance) : undefined try { @@ -239,10 +236,7 @@ async function updateVideoPlaylist (req: express.Request, res: express.Response) const playlistUpdated = await videoPlaylistInstance.save(sequelizeOptions) - if (thumbnailModel) { - thumbnailModel.videoPlaylistId = playlistUpdated.id - playlistUpdated.setThumbnail(await thumbnailModel.save({ transaction: t })) - } + if (thumbnailModel) await playlistUpdated.setAndSaveThumbnail(thumbnailModel, t) const isNewPlaylist = wasPrivatePlaylist && playlistUpdated.privacy !== VideoPlaylistPrivacy.PRIVATE @@ -313,8 +307,8 @@ async function addVideoInPlaylist (req: express.Request, res: express.Response) if (playlistElement.position === 1 && videoPlaylist.hasThumbnail() === false) { logger.info('Generating default thumbnail to playlist %s.', videoPlaylist.url) - const inputPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnail().filename) - const thumbnailModel = await createPlaylistThumbnailFromExisting(inputPath, videoPlaylist, true) + const inputPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getMiniature().filename) + const thumbnailModel = await createPlaylistMiniatureFromExisting(inputPath, videoPlaylist, true) thumbnailModel.videoPlaylistId = videoPlaylist.id diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index a4ec41d44..bfb690906 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts @@ -23,7 +23,7 @@ import { move, readFile } from 'fs-extra' import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist' import { CONFIG } from '../../../initializers/config' import { sequelizeTypescript } from '../../../initializers/database' -import { createVideoThumbnailFromExisting } from '../../../lib/thumbnail' +import { createVideoMiniatureFromExisting } from '../../../lib/thumbnail' import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type' import { ThumbnailModel } from '../../../models/video/thumbnail' @@ -204,7 +204,7 @@ async function processThumbnail (req: express.Request, video: VideoModel) { if (thumbnailField) { const thumbnailPhysicalFile = thumbnailField[ 0 ] - return createVideoThumbnailFromExisting(thumbnailPhysicalFile.path, video, ThumbnailType.THUMBNAIL) + return createVideoMiniatureFromExisting(thumbnailPhysicalFile.path, video, ThumbnailType.MINIATURE) } return undefined @@ -215,7 +215,7 @@ async function processPreview (req: express.Request, video: VideoModel) { if (previewField) { const previewPhysicalFile = previewField[0] - return createVideoThumbnailFromExisting(previewPhysicalFile.path, video, ThumbnailType.PREVIEW) + return createVideoMiniatureFromExisting(previewPhysicalFile.path, video, ThumbnailType.PREVIEW) } return undefined @@ -238,14 +238,8 @@ function insertIntoDB (parameters: { const videoCreated = await video.save(sequelizeOptions) videoCreated.VideoChannel = videoChannel - if (thumbnailModel) { - thumbnailModel.videoId = videoCreated.id - videoCreated.addThumbnail(await thumbnailModel.save({ transaction: t })) - } - if (previewModel) { - previewModel.videoId = videoCreated.id - videoCreated.addThumbnail(await previewModel.save({ transaction: t })) - } + if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) + if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t) await autoBlacklistVideoIfNeeded(video, videoChannel.Account.User, t) diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index ad2fe958c..5bbce11b4 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -52,7 +52,7 @@ import { Notifier } from '../../../lib/notifier' import { sendView } from '../../../lib/activitypub/send/send-view' import { CONFIG } from '../../../initializers/config' import { sequelizeTypescript } from '../../../initializers/database' -import { createVideoThumbnailFromExisting, generateVideoThumbnail } from '../../../lib/thumbnail' +import { createVideoMiniatureFromExisting, generateVideoMiniature } from '../../../lib/thumbnail' import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type' const auditLogger = auditLoggerFactory('videos') @@ -214,14 +214,14 @@ async function addVideo (req: express.Request, res: express.Response) { // Process thumbnail or create it from the video const thumbnailField = req.files['thumbnailfile'] const thumbnailModel = thumbnailField - ? await createVideoThumbnailFromExisting(thumbnailField[0].path, video, ThumbnailType.THUMBNAIL) - : await generateVideoThumbnail(video, videoFile, ThumbnailType.THUMBNAIL) + ? await createVideoMiniatureFromExisting(thumbnailField[0].path, video, ThumbnailType.MINIATURE) + : await generateVideoMiniature(video, videoFile, ThumbnailType.MINIATURE) // Process preview or create it from the video const previewField = req.files['previewfile'] const previewModel = previewField - ? await createVideoThumbnailFromExisting(previewField[0].path, video, ThumbnailType.PREVIEW) - : await generateVideoThumbnail(video, videoFile, ThumbnailType.PREVIEW) + ? await createVideoMiniatureFromExisting(previewField[0].path, video, ThumbnailType.PREVIEW) + : await generateVideoMiniature(video, videoFile, ThumbnailType.PREVIEW) // Create the torrent file await video.createTorrentAndSetInfoHash(videoFile) @@ -231,11 +231,8 @@ async function addVideo (req: express.Request, res: express.Response) { const videoCreated = await video.save(sequelizeOptions) - thumbnailModel.videoId = videoCreated.id - previewModel.videoId = videoCreated.id - - videoCreated.addThumbnail(await thumbnailModel.save({ transaction: t })) - videoCreated.addThumbnail(await previewModel.save({ transaction: t })) + await videoCreated.addAndSaveThumbnail(thumbnailModel, t) + await videoCreated.addAndSaveThumbnail(previewModel, t) // Do not forget to add video channel information to the created video videoCreated.VideoChannel = res.locals.videoChannel @@ -308,11 +305,11 @@ async function updateVideo (req: express.Request, res: express.Response) { // Process thumbnail or create it from the video const thumbnailModel = req.files && req.files['thumbnailfile'] - ? await createVideoThumbnailFromExisting(req.files['thumbnailfile'][0].path, videoInstance, ThumbnailType.THUMBNAIL) + ? await createVideoMiniatureFromExisting(req.files['thumbnailfile'][0].path, videoInstance, ThumbnailType.MINIATURE) : undefined const previewModel = req.files && req.files['previewfile'] - ? await createVideoThumbnailFromExisting(req.files['previewfile'][0].path, videoInstance, ThumbnailType.PREVIEW) + ? await createVideoMiniatureFromExisting(req.files['previewfile'][0].path, videoInstance, ThumbnailType.PREVIEW) : undefined try { @@ -346,14 +343,8 @@ async function updateVideo (req: express.Request, res: express.Response) { const videoInstanceUpdated = await videoInstance.save(sequelizeOptions) - if (thumbnailModel) { - thumbnailModel.videoId = videoInstanceUpdated.id - videoInstanceUpdated.addThumbnail(await thumbnailModel.save({ transaction: t })) - } - if (previewModel) { - previewModel.videoId = videoInstanceUpdated.id - videoInstanceUpdated.addThumbnail(await previewModel.save({ transaction: t })) - } + if (thumbnailModel) await videoInstanceUpdated.addAndSaveThumbnail(thumbnailModel, t) + if (previewModel) await videoInstanceUpdated.addAndSaveThumbnail(previewModel, t) // Video tags update? if (videoInfoToUpdate.tags !== undefined) { diff --git a/server/controllers/bots.ts b/server/controllers/bots.ts index 7e8e6eff6..e25d9c21b 100644 --- a/server/controllers/bots.ts +++ b/server/controllers/bots.ts @@ -85,7 +85,7 @@ async function getSitemapLocalVideoUrls () { // Sitemap description should be < 2000 characters description: truncate(v.description || v.name, { length: 2000, omission: '...' }), player_loc: WEBSERVER.URL + '/videos/embed/' + v.uuid, - thumbnail_loc: WEBSERVER.URL + v.getThumbnailStaticPath() + thumbnail_loc: WEBSERVER.URL + v.getMiniatureStaticPath() } ] })) diff --git a/server/controllers/feeds.ts b/server/controllers/feeds.ts index 5064097cd..d3f581615 100644 --- a/server/controllers/feeds.ts +++ b/server/controllers/feeds.ts @@ -137,7 +137,7 @@ async function generateVideoFeed (req: express.Request, res: express.Response) { torrent: torrents, thumbnail: [ { - url: WEBSERVER.URL + video.getThumbnailStaticPath(), + url: WEBSERVER.URL + video.getMiniatureStaticPath(), height: THUMBNAILS_SIZE.height, width: THUMBNAILS_SIZE.width } diff --git a/server/controllers/static.ts b/server/controllers/static.ts index d75b95f52..05019fcc2 100644 --- a/server/controllers/static.ts +++ b/server/controllers/static.ts @@ -165,20 +165,20 @@ export { // --------------------------------------------------------------------------- async function getPreview (req: express.Request, res: express.Response) { - const path = await VideosPreviewCache.Instance.getFilePath(req.params.uuid) - if (!path) return res.sendStatus(404) + const result = await VideosPreviewCache.Instance.getFilePath(req.params.uuid) + if (!result) return res.sendStatus(404) - return res.sendFile(path, { maxAge: STATIC_MAX_AGE }) + return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE }) } async function getVideoCaption (req: express.Request, res: express.Response) { - const path = await VideosCaptionCache.Instance.getFilePath({ + const result = await VideosCaptionCache.Instance.getFilePath({ videoId: req.params.videoId, language: req.params.captionLanguage }) - if (!path) return res.sendStatus(404) + if (!result) return res.sendStatus(404) - return res.sendFile(path, { maxAge: STATIC_MAX_AGE }) + return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE }) } async function generateNodeinfo (req: express.Request, res: express.Response, next: express.NextFunction) { diff --git a/server/initializers/database.ts b/server/initializers/database.ts index d1744d21f..d9a265e7a 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts @@ -140,15 +140,15 @@ async function checkPostgresExtensions () { } async function checkPostgresExtension (extension: string) { - const query = `SELECT true AS enabled FROM pg_available_extensions WHERE name = '${extension}' AND installed_version IS NOT NULL;` + const query = `SELECT 1 FROM pg_available_extensions WHERE name = '${extension}' AND installed_version IS NOT NULL;` const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, raw: true } - const res = await sequelizeTypescript.query<{ enabled: boolean }>(query, options) + const res = await sequelizeTypescript.query(query, options) - if (!res || res.length === 0 || res[ 0 ][ 'enabled' ] !== true) { + if (!res || res.length === 0) { // Try to create the extension ourselves try { await sequelizeTypescript.query(`CREATE EXTENSION ${extension};`, { raw: true }) diff --git a/server/lib/activitypub/playlist.ts b/server/lib/activitypub/playlist.ts index 721c19603..36a91faec 100644 --- a/server/lib/activitypub/playlist.ts +++ b/server/lib/activitypub/playlist.ts @@ -16,7 +16,8 @@ import { VideoPlaylistElementModel } from '../../models/video/video-playlist-ele import { VideoModel } from '../../models/video/video' import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model' import { sequelizeTypescript } from '../../initializers/database' -import { createPlaylistThumbnailFromUrl } from '../thumbnail' +import { createPlaylistMiniatureFromUrl } from '../thumbnail' +import { FilteredModelAttributes } from '../../typings/sequelize' function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: AccountModel, to: string[]) { const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPlaylistPrivacy.PUBLIC : VideoPlaylistPrivacy.UNLISTED @@ -86,8 +87,7 @@ async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAc } } - // FIXME: sequelize typings - const [ playlist ] = (await VideoPlaylistModel.upsert(playlistAttributes, { returning: true }) as any) + const [ playlist ] = await VideoPlaylistModel.upsert(playlistAttributes, { returning: true }) let accItems: string[] = [] await crawlCollectionPage(playlistObject.id, items => { @@ -100,10 +100,8 @@ async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAc if (playlistObject.icon) { try { - const thumbnailModel = await createPlaylistThumbnailFromUrl(playlistObject.icon.url, refreshedPlaylist) - thumbnailModel.videoPlaylistId = refreshedPlaylist.id - - refreshedPlaylist.setThumbnail(await thumbnailModel.save()) + const thumbnailModel = await createPlaylistMiniatureFromUrl(playlistObject.icon.url, refreshedPlaylist) + await refreshedPlaylist.setAndSaveThumbnail(thumbnailModel, undefined) } catch (err) { logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err }) } @@ -156,7 +154,7 @@ export { // --------------------------------------------------------------------------- async function resetVideoPlaylistElements (elementUrls: string[], playlist: VideoPlaylistModel) { - const elementsToCreate: object[] = [] // FIXME: sequelize typings + const elementsToCreate: FilteredModelAttributes[] = [] await Bluebird.map(elementUrls, async elementUrl => { try { diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index cb67bf9a4..18f44d50e 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts @@ -73,8 +73,7 @@ async function addVideoComment (videoInstance: VideoModel, commentUrl: string) { const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body) if (!entry) return { created: false } - // FIXME: sequelize typings - const [ comment, created ] = (await VideoCommentModel.upsert(entry, { returning: true }) as any) + const [ comment, created ] = await VideoCommentModel.upsert(entry, { returning: true }) comment.Account = actor.Account comment.Video = videoInstance diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 5a56942a9..63bb07ec1 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -49,10 +49,11 @@ import { AccountVideoRateModel } from '../../models/account/account-video-rate' import { VideoShareModel } from '../../models/video/video-share' import { VideoCommentModel } from '../../models/video/video-comment' import { sequelizeTypescript } from '../../initializers/database' -import { createPlaceholderThumbnail, createVideoThumbnailFromUrl } from '../thumbnail' +import { createPlaceholderThumbnail, createVideoMiniatureFromUrl } from '../thumbnail' import { ThumbnailModel } from '../../models/video/thumbnail' import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' import { join } from 'path' +import { FilteredModelAttributes } from '../../typings/sequelize' async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { // If the video is not private and is published, we federate it @@ -247,7 +248,7 @@ async function updateVideoFromAP (options: { let thumbnailModel: ThumbnailModel try { - thumbnailModel = await createVideoThumbnailFromUrl(options.videoObject.icon.url, options.video, ThumbnailType.THUMBNAIL) + thumbnailModel = await createVideoMiniatureFromUrl(options.videoObject.icon.url, options.video, ThumbnailType.MINIATURE) } catch (err) { logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err }) } @@ -288,16 +289,12 @@ async function updateVideoFromAP (options: { await options.video.save(sequelizeOptions) - if (thumbnailModel) { - thumbnailModel.videoId = options.video.id - options.video.addThumbnail(await thumbnailModel.save({ transaction: t })) - } + if (thumbnailModel) if (thumbnailModel) await options.video.addAndSaveThumbnail(thumbnailModel, t) // FIXME: use icon URL instead const previewUrl = buildRemoteBaseUrl(options.video, join(STATIC_PATHS.PREVIEWS, options.video.getPreview().filename)) const previewModel = createPlaceholderThumbnail(previewUrl, options.video, ThumbnailType.PREVIEW, PREVIEWS_SIZE) - - options.video.addThumbnail(await previewModel.save({ transaction: t })) + await options.video.addAndSaveThumbnail(previewModel, t) { const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject) @@ -311,7 +308,7 @@ async function updateVideoFromAP (options: { // Update or add other one const upsertTasks = videoFileAttributes.map(a => { - return (VideoFileModel.upsert(a, { returning: true, transaction: t }) as any) // FIXME: sequelize typings + return VideoFileModel.upsert(a, { returning: true, transaction: t }) .then(([ file ]) => file) }) @@ -334,8 +331,7 @@ async function updateVideoFromAP (options: { // Update or add other one const upsertTasks = streamingPlaylistAttributes.map(a => { - // FIXME: sequelize typings - return (VideoStreamingPlaylistModel.upsert(a, { returning: true, transaction: t }) as any) + return VideoStreamingPlaylistModel.upsert(a, { returning: true, transaction: t }) .then(([ streamingPlaylist ]) => streamingPlaylist) }) @@ -464,7 +460,7 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to) const video = VideoModel.build(videoData) - const promiseThumbnail = createVideoThumbnailFromUrl(videoObject.icon.url, video, ThumbnailType.THUMBNAIL) + const promiseThumbnail = createVideoMiniatureFromUrl(videoObject.icon.url, video, ThumbnailType.MINIATURE) let thumbnailModel: ThumbnailModel if (waitThumbnail === true) { @@ -477,18 +473,12 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor const videoCreated = await video.save(sequelizeOptions) videoCreated.VideoChannel = channelActor.VideoChannel - if (thumbnailModel) { - thumbnailModel.videoId = videoCreated.id - - videoCreated.addThumbnail(await thumbnailModel.save({ transaction: t })) - } + if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) // FIXME: use icon URL instead const previewUrl = buildRemoteBaseUrl(videoCreated, join(STATIC_PATHS.PREVIEWS, video.generatePreviewName())) const previewModel = createPlaceholderThumbnail(previewUrl, video, ThumbnailType.PREVIEW, PREVIEWS_SIZE) - previewModel.videoId = videoCreated.id - - videoCreated.addThumbnail(await previewModel.save({ transaction: t })) + if (thumbnailModel) await videoCreated.addAndSaveThumbnail(previewModel, t) // Process files const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject) @@ -594,7 +584,7 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid throw new Error('Cannot find video files for ' + video.url) } - const attributes: object[] = [] // FIXME: add typings + const attributes: FilteredModelAttributes[] = [] for (const fileUrl of fileUrls) { // Fetch associated magnet uri const magnet = videoObject.url.find(u => { @@ -629,7 +619,7 @@ function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObj const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[] if (playlistUrls.length === 0) return [] - const attributes: object[] = [] // FIXME: add typings + const attributes: FilteredModelAttributes[] = [] for (const playlistUrlObject of playlistUrls) { const segmentsSha256UrlObject = playlistUrlObject.tag .find(t => { diff --git a/server/lib/files-cache/abstract-video-static-file-cache.ts b/server/lib/files-cache/abstract-video-static-file-cache.ts index 61837e0f8..84ed74c98 100644 --- a/server/lib/files-cache/abstract-video-static-file-cache.ts +++ b/server/lib/files-cache/abstract-video-static-file-cache.ts @@ -4,24 +4,28 @@ import { VideoModel } from '../../models/video/video' import { fetchRemoteVideoStaticFile } from '../activitypub' import * as memoizee from 'memoizee' +type GetFilePathResult = { isOwned: boolean, path: string } | undefined + export abstract class AbstractVideoStaticFileCache { - getFilePath: (params: T) => Promise + getFilePath: (params: T) => Promise - abstract getFilePathImpl (params: T): Promise + abstract getFilePathImpl (params: T): Promise // Load and save the remote file, then return the local path from filesystem - protected abstract loadRemoteFile (key: string): Promise + protected abstract loadRemoteFile (key: string): Promise init (max: number, maxAge: number) { this.getFilePath = memoizee(this.getFilePathImpl, { maxAge, max, promise: true, - dispose: (value: string) => { - remove(value) - .then(() => logger.debug('%s evicted from %s', value, this.constructor.name)) - .catch(err => logger.error('Cannot remove %s from cache %s.', value, this.constructor.name, { err })) + dispose: (result: GetFilePathResult) => { + if (result.isOwned !== true) { + remove(result.path) + .then(() => logger.debug('%s removed from %s', result.path, this.constructor.name)) + .catch(err => logger.error('Cannot remove %s from cache %s.', result.path, this.constructor.name, { err })) + } } }) } diff --git a/server/lib/files-cache/videos-caption-cache.ts b/server/lib/files-cache/videos-caption-cache.ts index d4a0a3345..305e39c35 100644 --- a/server/lib/files-cache/videos-caption-cache.ts +++ b/server/lib/files-cache/videos-caption-cache.ts @@ -4,6 +4,7 @@ import { VideoModel } from '../../models/video/video' import { VideoCaptionModel } from '../../models/video/video-caption' import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' import { CONFIG } from '../../initializers/config' +import { logger } from '../../helpers/logger' type GetPathParam = { videoId: string, language: string } @@ -24,13 +25,15 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache { const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(params.videoId, params.language) if (!videoCaption) return undefined - if (videoCaption.isOwned()) return join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.getCaptionName()) + if (videoCaption.isOwned()) return { isOwned: true, path: join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.getCaptionName()) } const key = params.videoId + VideosCaptionCache.KEY_DELIMITER + params.language return this.loadRemoteFile(key) } protected async loadRemoteFile (key: string) { + logger.debug('Loading remote caption file %s.', key) + const [ videoId, language ] = key.split(VideosCaptionCache.KEY_DELIMITER) const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(videoId, language) @@ -46,7 +49,9 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache { const remoteStaticPath = videoCaption.getCaptionStaticPath() const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName()) - return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath) + const path = await this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath) + + return { isOwned: false, path } } } diff --git a/server/lib/files-cache/videos-preview-cache.ts b/server/lib/files-cache/videos-preview-cache.ts index fc0d92c78..c117ae426 100644 --- a/server/lib/files-cache/videos-preview-cache.ts +++ b/server/lib/files-cache/videos-preview-cache.ts @@ -20,7 +20,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache { const video = await VideoModel.loadByUUIDWithFile(videoUUID) if (!video) return undefined - if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreview().filename) + if (video.isOwned()) return { isOwned: true, path: join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreview().filename) } return this.loadRemoteFile(videoUUID) } @@ -35,7 +35,9 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache { const remoteStaticPath = join(STATIC_PATHS.PREVIEWS, video.getPreview().filename) const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, video.getPreview().filename) - return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath) + const path = await this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath) + + return { isOwned: false, path } } } diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 3fa0dd65d..1650916a6 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -18,7 +18,7 @@ import { Notifier } from '../../notifier' import { CONFIG } from '../../../initializers/config' import { sequelizeTypescript } from '../../../initializers/database' import { ThumbnailModel } from '../../../models/video/thumbnail' -import { createVideoThumbnailFromUrl, generateVideoThumbnail } from '../../thumbnail' +import { createVideoMiniatureFromUrl, generateVideoMiniature } from '../../thumbnail' import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type' type VideoImportYoutubeDLPayload = { @@ -150,17 +150,17 @@ async function processFile (downloader: () => Promise, videoImport: Vide // Process thumbnail let thumbnailModel: ThumbnailModel if (options.downloadThumbnail && options.thumbnailUrl) { - thumbnailModel = await createVideoThumbnailFromUrl(options.thumbnailUrl, videoImport.Video, ThumbnailType.THUMBNAIL) + thumbnailModel = await createVideoMiniatureFromUrl(options.thumbnailUrl, videoImport.Video, ThumbnailType.MINIATURE) } else if (options.generateThumbnail || options.downloadThumbnail) { - thumbnailModel = await generateVideoThumbnail(videoImport.Video, videoFile, ThumbnailType.THUMBNAIL) + thumbnailModel = await generateVideoMiniature(videoImport.Video, videoFile, ThumbnailType.MINIATURE) } // Process preview let previewModel: ThumbnailModel if (options.downloadPreview && options.thumbnailUrl) { - previewModel = await createVideoThumbnailFromUrl(options.thumbnailUrl, videoImport.Video, ThumbnailType.PREVIEW) + previewModel = await createVideoMiniatureFromUrl(options.thumbnailUrl, videoImport.Video, ThumbnailType.PREVIEW) } else if (options.generatePreview || options.downloadPreview) { - previewModel = await generateVideoThumbnail(videoImport.Video, videoFile, ThumbnailType.PREVIEW) + previewModel = await generateVideoMiniature(videoImport.Video, videoFile, ThumbnailType.PREVIEW) } // Create torrent @@ -180,14 +180,8 @@ async function processFile (downloader: () => Promise, videoImport: Vide video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED await video.save({ transaction: t }) - if (thumbnailModel) { - thumbnailModel.videoId = video.id - video.addThumbnail(await thumbnailModel.save({ transaction: t })) - } - if (previewModel) { - previewModel.videoId = video.id - video.addThumbnail(await previewModel.save({ transaction: t })) - } + if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t) + if (previewModel) await video.addAndSaveThumbnail(previewModel, t) // Now we can federate the video (reload from database, we need more attributes) const videoForFederation = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) diff --git a/server/lib/oauth-model.ts b/server/lib/oauth-model.ts index eb0e63bc8..45ac3e7c4 100644 --- a/server/lib/oauth-model.ts +++ b/server/lib/oauth-model.ts @@ -39,6 +39,8 @@ function clearCacheByToken (token: string) { function getAccessToken (bearerToken: string) { logger.debug('Getting access token (bearerToken: ' + bearerToken + ').') + if (!bearerToken) return Bluebird.resolve(undefined) + if (accessTokenCache[bearerToken] !== undefined) return Bluebird.resolve(accessTokenCache[bearerToken]) return OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts index 344c28566..8ad82ee80 100644 --- a/server/lib/thumbnail.ts +++ b/server/lib/thumbnail.ts @@ -12,37 +12,37 @@ import { VideoPlaylistModel } from '../models/video/video-playlist' type ImageSize = { height: number, width: number } -function createPlaylistThumbnailFromExisting (inputPath: string, playlist: VideoPlaylistModel, keepOriginal = false, size?: ImageSize) { +function createPlaylistMiniatureFromExisting (inputPath: string, playlist: VideoPlaylistModel, keepOriginal = false, size?: ImageSize) { const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size) - const type = ThumbnailType.THUMBNAIL + const type = ThumbnailType.MINIATURE const thumbnailCreator = () => processImage({ path: inputPath }, outputPath, { width, height }, keepOriginal) return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail }) } -function createPlaylistThumbnailFromUrl (url: string, playlist: VideoPlaylistModel, size?: ImageSize) { +function createPlaylistMiniatureFromUrl (url: string, playlist: VideoPlaylistModel, size?: ImageSize) { const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size) - const type = ThumbnailType.THUMBNAIL + const type = ThumbnailType.MINIATURE const thumbnailCreator = () => downloadImage(url, basePath, filename, { width, height }) return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, url }) } -function createVideoThumbnailFromUrl (url: string, video: VideoModel, type: ThumbnailType, size?: ImageSize) { +function createVideoMiniatureFromUrl (url: string, video: VideoModel, type: ThumbnailType, size?: ImageSize) { const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) const thumbnailCreator = () => downloadImage(url, basePath, filename, { width, height }) return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, url }) } -function createVideoThumbnailFromExisting (inputPath: string, video: VideoModel, type: ThumbnailType, size?: ImageSize) { +function createVideoMiniatureFromExisting (inputPath: string, video: VideoModel, type: ThumbnailType, size?: ImageSize) { const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) const thumbnailCreator = () => processImage({ path: inputPath }, outputPath, { width, height }) return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail }) } -function generateVideoThumbnail (video: VideoModel, videoFile: VideoFileModel, type: ThumbnailType) { +function generateVideoMiniature (video: VideoModel, videoFile: VideoFileModel, type: ThumbnailType) { const input = video.getVideoFilePath(videoFile) const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type) @@ -68,12 +68,12 @@ function createPlaceholderThumbnail (url: string, video: VideoModel, type: Thumb // --------------------------------------------------------------------------- export { - generateVideoThumbnail, - createVideoThumbnailFromUrl, - createVideoThumbnailFromExisting, + generateVideoMiniature, + createVideoMiniatureFromUrl, + createVideoMiniatureFromExisting, createPlaceholderThumbnail, - createPlaylistThumbnailFromUrl, - createPlaylistThumbnailFromExisting + createPlaylistMiniatureFromUrl, + createPlaylistMiniatureFromExisting } function buildMetadataFromPlaylist (playlist: VideoPlaylistModel, size: ImageSize) { @@ -95,7 +95,7 @@ function buildMetadataFromVideo (video: VideoModel, type: ThumbnailType, size?: ? video.Thumbnails.find(t => t.type === type) : undefined - if (type === ThumbnailType.THUMBNAIL) { + if (type === ThumbnailType.MINIATURE) { const filename = video.generateThumbnailName() const basePath = CONFIG.STORAGE.THUMBNAILS_DIR diff --git a/server/middlewares/oauth.ts b/server/middlewares/oauth.ts index de736e593..2b4e300e4 100644 --- a/server/middlewares/oauth.ts +++ b/server/middlewares/oauth.ts @@ -35,6 +35,8 @@ function authenticateSocket (socket: Socket, next: (err?: any) => void) { logger.debug('Checking socket access token %s.', accessToken) + if (!accessToken) return next(new Error('No access token provided')) + getAccessToken(accessToken) .then(tokenDB => { const now = new Date() diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts index e9b036a02..2b01f108d 100644 --- a/server/middlewares/validators/videos/videos.ts +++ b/server/middlewares/validators/videos/videos.ts @@ -68,6 +68,7 @@ const videosAddValidator = getCommonVideoEditAttributes().concat([ if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) const isAble = await user.isAbleToUploadVideo(videoFile) + if (isAble === false) { res.status(403) .json({ error: 'The user video quota is exceeded with this video.' }) diff --git a/server/models/account/account-blocklist.ts b/server/models/account/account-blocklist.ts index efd6ed59e..d5746ad76 100644 --- a/server/models/account/account-blocklist.ts +++ b/server/models/account/account-blocklist.ts @@ -8,22 +8,22 @@ enum ScopeNames { WITH_ACCOUNTS = 'WITH_ACCOUNTS' } -@Scopes({ +@Scopes(() => ({ [ScopeNames.WITH_ACCOUNTS]: { include: [ { - model: () => AccountModel, + model: AccountModel, required: true, as: 'ByAccount' }, { - model: () => AccountModel, + model: AccountModel, required: true, as: 'BlockedAccount' } ] } -}) +})) @Table({ tableName: 'accountBlocklist', @@ -83,7 +83,7 @@ export class AccountBlocklistModel extends Model { attributes: [ 'accountId', 'id' ], where: { accountId: { - [Op.any]: accountIds + [Op.in]: accountIds // FIXME: sequelize ANY seems broken }, targetAccountId }, diff --git a/server/models/account/account.ts b/server/models/account/account.ts index bf2ed0a61..c53312990 100644 --- a/server/models/account/account.ts +++ b/server/models/account/account.ts @@ -33,15 +33,15 @@ export enum ScopeNames { SUMMARY = 'SUMMARY' } -@DefaultScope({ +@DefaultScope(() => ({ include: [ { - model: () => ActorModel, // Default scope includes avatar and server + model: ActorModel, // Default scope includes avatar and server required: true } ] -}) -@Scopes({ +})) +@Scopes(() => ({ [ ScopeNames.SUMMARY ]: (whereActor?: WhereOptions) => { return { attributes: [ 'id', 'name' ], @@ -66,7 +66,7 @@ export enum ScopeNames { ] } } -}) +})) @Table({ tableName: 'account', indexes: [ diff --git a/server/models/account/user-notification.ts b/server/models/account/user-notification.ts index 08388f268..a4f97037b 100644 --- a/server/models/account/user-notification.ts +++ b/server/models/account/user-notification.ts @@ -6,7 +6,7 @@ import { isUserNotificationTypeValid } from '../../helpers/custom-validators/use import { UserModel } from './user' import { VideoModel } from '../video/video' import { VideoCommentModel } from '../video/video-comment' -import { FindOptions, Op } from 'sequelize' +import { FindOptions, ModelIndexesOptions, Op, WhereOptions } from 'sequelize' import { VideoChannelModel } from '../video/video-channel' import { AccountModel } from './account' import { VideoAbuseModel } from '../video/video-abuse' @@ -24,17 +24,17 @@ enum ScopeNames { function buildActorWithAvatarInclude () { return { attributes: [ 'preferredUsername' ], - model: () => ActorModel.unscoped(), + model: ActorModel.unscoped(), required: true, include: [ { attributes: [ 'filename' ], - model: () => AvatarModel.unscoped(), + model: AvatarModel.unscoped(), required: false }, { attributes: [ 'host' ], - model: () => ServerModel.unscoped(), + model: ServerModel.unscoped(), required: false } ] @@ -44,7 +44,7 @@ function buildActorWithAvatarInclude () { function buildVideoInclude (required: boolean) { return { attributes: [ 'id', 'uuid', 'name' ], - model: () => VideoModel.unscoped(), + model: VideoModel.unscoped(), required } } @@ -53,7 +53,7 @@ function buildChannelInclude (required: boolean, withActor = false) { return { required, attributes: [ 'id', 'name' ], - model: () => VideoChannelModel.unscoped(), + model: VideoChannelModel.unscoped(), include: withActor === true ? [ buildActorWithAvatarInclude() ] : [] } } @@ -62,12 +62,12 @@ function buildAccountInclude (required: boolean, withActor = false) { return { required, attributes: [ 'id', 'name' ], - model: () => AccountModel.unscoped(), + model: AccountModel.unscoped(), include: withActor === true ? [ buildActorWithAvatarInclude() ] : [] } } -@Scopes({ +@Scopes(() => ({ [ScopeNames.WITH_ALL]: { include: [ Object.assign(buildVideoInclude(false), { @@ -76,7 +76,7 @@ function buildAccountInclude (required: boolean, withActor = false) { { attributes: [ 'id', 'originCommentId' ], - model: () => VideoCommentModel.unscoped(), + model: VideoCommentModel.unscoped(), required: false, include: [ buildAccountInclude(true, true), @@ -86,56 +86,56 @@ function buildAccountInclude (required: boolean, withActor = false) { { attributes: [ 'id' ], - model: () => VideoAbuseModel.unscoped(), + model: VideoAbuseModel.unscoped(), required: false, include: [ buildVideoInclude(true) ] }, { attributes: [ 'id' ], - model: () => VideoBlacklistModel.unscoped(), + model: VideoBlacklistModel.unscoped(), required: false, include: [ buildVideoInclude(true) ] }, { attributes: [ 'id', 'magnetUri', 'targetUrl', 'torrentName' ], - model: () => VideoImportModel.unscoped(), + model: VideoImportModel.unscoped(), required: false, include: [ buildVideoInclude(false) ] }, { attributes: [ 'id', 'state' ], - model: () => ActorFollowModel.unscoped(), + model: ActorFollowModel.unscoped(), required: false, include: [ { attributes: [ 'preferredUsername' ], - model: () => ActorModel.unscoped(), + model: ActorModel.unscoped(), required: true, as: 'ActorFollower', include: [ { attributes: [ 'id', 'name' ], - model: () => AccountModel.unscoped(), + model: AccountModel.unscoped(), required: true }, { attributes: [ 'filename' ], - model: () => AvatarModel.unscoped(), + model: AvatarModel.unscoped(), required: false }, { attributes: [ 'host' ], - model: () => ServerModel.unscoped(), + model: ServerModel.unscoped(), required: false } ] }, { attributes: [ 'preferredUsername' ], - model: () => ActorModel.unscoped(), + model: ActorModel.unscoped(), required: true, as: 'ActorFollowing', include: [ @@ -147,9 +147,9 @@ function buildAccountInclude (required: boolean, withActor = false) { }, buildAccountInclude(false, true) - ] as any // FIXME: sequelize typings + ] } -}) +})) @Table({ tableName: 'userNotification', indexes: [ @@ -212,7 +212,7 @@ function buildAccountInclude (required: boolean, withActor = false) { } } } - ] as any // FIXME: sequelize typings + ] as (ModelIndexesOptions & { where?: WhereOptions })[] }) export class UserNotificationModel extends Model { @@ -357,7 +357,7 @@ export class UserNotificationModel extends Model { where: { userId, id: { - [Op.any]: notificationIds + [Op.in]: notificationIds // FIXME: sequelize ANY seems broken } } } diff --git a/server/models/account/user.ts b/server/models/account/user.ts index 8bd0397dd..4a9acd703 100644 --- a/server/models/account/user.ts +++ b/server/models/account/user.ts @@ -1,4 +1,4 @@ -import * as Sequelize from 'sequelize' +import { FindOptions, literal, Op, QueryTypes } from 'sequelize' import { AfterDestroy, AfterUpdate, @@ -56,33 +56,33 @@ enum ScopeNames { WITH_VIDEO_CHANNEL = 'WITH_VIDEO_CHANNEL' } -@DefaultScope({ +@DefaultScope(() => ({ include: [ { - model: () => AccountModel, + model: AccountModel, required: true }, { - model: () => UserNotificationSettingModel, + model: UserNotificationSettingModel, required: true } ] -}) -@Scopes({ +})) +@Scopes(() => ({ [ScopeNames.WITH_VIDEO_CHANNEL]: { include: [ { - model: () => AccountModel, + model: AccountModel, required: true, - include: [ () => VideoChannelModel ] + include: [ VideoChannelModel ] }, { - model: () => UserNotificationSettingModel, + model: UserNotificationSettingModel, required: true } - ] as any // FIXME: sequelize typings + ] } -}) +})) @Table({ tableName: 'user', indexes: [ @@ -233,26 +233,26 @@ export class UserModel extends Model { let where = undefined if (search) { where = { - [Sequelize.Op.or]: [ + [Op.or]: [ { email: { - [Sequelize.Op.iLike]: '%' + search + '%' + [Op.iLike]: '%' + search + '%' } }, { username: { - [ Sequelize.Op.iLike ]: '%' + search + '%' + [ Op.iLike ]: '%' + search + '%' } } ] } } - const query = { + const query: FindOptions = { attributes: { include: [ [ - Sequelize.literal( + literal( '(' + 'SELECT COALESCE(SUM("size"), 0) ' + 'FROM (' + @@ -265,7 +265,7 @@ export class UserModel extends Model { ')' ), 'videoQuotaUsed' - ] as any // FIXME: typings + ] ] }, offset: start, @@ -291,7 +291,7 @@ export class UserModel extends Model { const query = { where: { role: { - [Sequelize.Op.in]: roles + [Op.in]: roles } } } @@ -387,7 +387,7 @@ export class UserModel extends Model { const query = { where: { - [ Sequelize.Op.or ]: [ { username }, { email } ] + [ Op.or ]: [ { username }, { email } ] } } @@ -510,7 +510,7 @@ export class UserModel extends Model { const query = { where: { username: { - [ Sequelize.Op.like ]: `%${search}%` + [ Op.like ]: `%${search}%` } }, limit: 10 @@ -591,15 +591,11 @@ export class UserModel extends Model { const uploadedTotal = videoFile.size + totalBytes const uploadedDaily = videoFile.size + totalBytesDaily - if (this.videoQuotaDaily === -1) { - return uploadedTotal < this.videoQuota - } - if (this.videoQuota === -1) { - return uploadedDaily < this.videoQuotaDaily - } - return (uploadedTotal < this.videoQuota) && - (uploadedDaily < this.videoQuotaDaily) + if (this.videoQuotaDaily === -1) return uploadedTotal < this.videoQuota + if (this.videoQuota === -1) return uploadedDaily < this.videoQuotaDaily + + return uploadedTotal < this.videoQuota && uploadedDaily < this.videoQuotaDaily } private static generateUserQuotaBaseSQL (where?: string) { @@ -619,14 +615,14 @@ export class UserModel extends Model { private static getTotalRawQuery (query: string, userId: number) { const options = { bind: { userId }, - type: Sequelize.QueryTypes.SELECT as Sequelize.QueryTypes.SELECT + type: QueryTypes.SELECT as QueryTypes.SELECT } - return UserModel.sequelize.query<{ total: number }>(query, options) + return UserModel.sequelize.query<{ total: string }>(query, options) .then(([ { total } ]) => { if (total === null) return 0 - return parseInt(total + '', 10) + return parseInt(total, 10) }) } } diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts index 1ebee8df5..4a466441c 100644 --- a/server/models/activitypub/actor.ts +++ b/server/models/activitypub/actor.ts @@ -56,46 +56,46 @@ export const unusedActorAttributesForAPI = [ 'updatedAt' ] -@DefaultScope({ +@DefaultScope(() => ({ include: [ { - model: () => ServerModel, + model: ServerModel, required: false }, { - model: () => AvatarModel, + model: AvatarModel, required: false } ] -}) -@Scopes({ +})) +@Scopes(() => ({ [ScopeNames.FULL]: { include: [ { - model: () => AccountModel.unscoped(), + model: AccountModel.unscoped(), required: false }, { - model: () => VideoChannelModel.unscoped(), + model: VideoChannelModel.unscoped(), required: false, include: [ { - model: () => AccountModel, + model: AccountModel, required: true } ] }, { - model: () => ServerModel, + model: ServerModel, required: false }, { - model: () => AvatarModel, + model: AvatarModel, required: false } - ] as any // FIXME: sequelize typings + ] } -}) +})) @Table({ tableName: 'actor', indexes: [ @@ -131,7 +131,7 @@ export const unusedActorAttributesForAPI = [ export class ActorModel extends Model { @AllowNull(false) - @Column({ type: DataType.ENUM(...values(ACTIVITY_PUB_ACTOR_TYPES)) }) // FIXME: sequelize typings + @Column(DataType.ENUM(...values(ACTIVITY_PUB_ACTOR_TYPES))) type: ActivityPubActorType @AllowNull(false) @@ -280,14 +280,16 @@ export class ActorModel extends Model { attributes: [ 'id' ], model: VideoChannelModel.unscoped(), required: true, - include: { - attributes: [ 'id' ], - model: VideoModel.unscoped(), - required: true, - where: { - id: videoId + include: [ + { + attributes: [ 'id' ], + model: VideoModel.unscoped(), + required: true, + where: { + id: videoId + } } - } + ] } ] } @@ -295,7 +297,7 @@ export class ActorModel extends Model { transaction } - return ActorModel.unscoped().findOne(query as any) // FIXME: typings + return ActorModel.unscoped().findOne(query) } static isActorUrlExist (url: string) { @@ -389,8 +391,7 @@ export class ActorModel extends Model { } static incrementFollows (id: number, column: 'followersCount' | 'followingCount', by: number) { - // FIXME: typings - return (ActorModel as any).increment(column, { + return ActorModel.increment(column, { by, where: { id diff --git a/server/models/application/application.ts b/server/models/application/application.ts index 854a5fb36..a02208b4e 100644 --- a/server/models/application/application.ts +++ b/server/models/application/application.ts @@ -1,14 +1,14 @@ import { AllowNull, Column, Default, DefaultScope, HasOne, IsInt, Model, Table } from 'sequelize-typescript' import { AccountModel } from '../account/account' -@DefaultScope({ +@DefaultScope(() => ({ include: [ { - model: () => AccountModel, + model: AccountModel, required: true } ] -}) +})) @Table({ tableName: 'application' }) diff --git a/server/models/oauth/oauth-client.ts b/server/models/oauth/oauth-client.ts index b4a841edd..42c59bb79 100644 --- a/server/models/oauth/oauth-client.ts +++ b/server/models/oauth/oauth-client.ts @@ -24,10 +24,10 @@ export class OAuthClientModel extends Model { @Column clientSecret: string - @Column({ type: DataType.ARRAY(DataType.STRING) }) // FIXME: sequelize typings + @Column(DataType.ARRAY(DataType.STRING)) grants: string[] - @Column({ type: DataType.ARRAY(DataType.STRING) }) // FIXME: sequelize typings + @Column(DataType.ARRAY(DataType.STRING)) redirectUris: string[] @CreatedAt diff --git a/server/models/oauth/oauth-token.ts b/server/models/oauth/oauth-token.ts index 3f41ee63b..903d551df 100644 --- a/server/models/oauth/oauth-token.ts +++ b/server/models/oauth/oauth-token.ts @@ -34,30 +34,30 @@ enum ScopeNames { WITH_USER = 'WITH_USER' } -@Scopes({ +@Scopes(() => ({ [ScopeNames.WITH_USER]: { include: [ { - model: () => UserModel.unscoped(), + model: UserModel.unscoped(), required: true, include: [ { attributes: [ 'id' ], - model: () => AccountModel.unscoped(), + model: AccountModel.unscoped(), required: true, include: [ { attributes: [ 'id', 'url' ], - model: () => ActorModel.unscoped(), + model: ActorModel.unscoped(), required: true } ] } ] } - ] as any // FIXME: sequelize typings + ] } -}) +})) @Table({ tableName: 'oAuthToken', indexes: [ @@ -167,11 +167,13 @@ export class OAuthTokenModel extends Model { } } - return OAuthTokenModel.scope(ScopeNames.WITH_USER).findOne(query).then(token => { - if (token) token['user'] = token.User + return OAuthTokenModel.scope(ScopeNames.WITH_USER) + .findOne(query) + .then(token => { + if (token) token[ 'user' ] = token.User - return token - }) + return token + }) } static getByRefreshTokenAndPopulateUser (refreshToken: string) { diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index cbeaa662b..eb2222256 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -13,7 +13,7 @@ import { UpdatedAt } from 'sequelize-typescript' import { ActorModel } from '../activitypub/actor' -import { getVideoSort, throwIfNotValid } from '../utils' +import { getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils' import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc' import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants' import { VideoFileModel } from '../video/video-file' @@ -27,7 +27,7 @@ import { ServerModel } from '../server/server' import { sample } from 'lodash' import { isTestInstance } from '../../helpers/core-utils' import * as Bluebird from 'bluebird' -import * as Sequelize from 'sequelize' +import { col, FindOptions, fn, literal, Op, Transaction } from 'sequelize' import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist' import { CONFIG } from '../../initializers/config' @@ -35,32 +35,32 @@ export enum ScopeNames { WITH_VIDEO = 'WITH_VIDEO' } -@Scopes({ +@Scopes(() => ({ [ ScopeNames.WITH_VIDEO ]: { include: [ { - model: () => VideoFileModel, + model: VideoFileModel, required: false, include: [ { - model: () => VideoModel, + model: VideoModel, required: true } ] }, { - model: () => VideoStreamingPlaylistModel, + model: VideoStreamingPlaylistModel, required: false, include: [ { - model: () => VideoModel, + model: VideoModel, required: true } ] } - ] as any // FIXME: sequelize typings + ] } -}) +})) @Table({ tableName: 'videoRedundancy', @@ -192,7 +192,7 @@ export class VideoRedundancyModel extends Model { return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) } - static loadByUrl (url: string, transaction?: Sequelize.Transaction) { + static loadByUrl (url: string, transaction?: Transaction) { const query = { where: { url @@ -292,7 +292,7 @@ export class VideoRedundancyModel extends Model { where: { privacy: VideoPrivacy.PUBLIC, views: { - [ Sequelize.Op.gte ]: minViews + [ Op.gte ]: minViews } }, include: [ @@ -315,7 +315,7 @@ export class VideoRedundancyModel extends Model { actorId: actor.id, strategy, createdAt: { - [ Sequelize.Op.lt ]: expiredDate + [ Op.lt ]: expiredDate } } } @@ -326,7 +326,7 @@ export class VideoRedundancyModel extends Model { static async getTotalDuplicated (strategy: VideoRedundancyStrategy) { const actor = await getServerActor() - const options = { + const query: FindOptions = { include: [ { attributes: [], @@ -340,12 +340,8 @@ export class VideoRedundancyModel extends Model { ] } - return VideoFileModel.sum('size', options as any) // FIXME: typings - .then(v => { - if (!v || isNaN(v)) return 0 - - return v - }) + return VideoFileModel.aggregate('size', 'SUM', query) + .then(result => parseAggregateResult(result)) } static async listLocalExpired () { @@ -355,7 +351,7 @@ export class VideoRedundancyModel extends Model { where: { actorId: actor.id, expiresOn: { - [ Sequelize.Op.lt ]: new Date() + [ Op.lt ]: new Date() } } } @@ -369,10 +365,10 @@ export class VideoRedundancyModel extends Model { const query = { where: { actorId: { - [Sequelize.Op.ne]: actor.id + [Op.ne]: actor.id }, expiresOn: { - [ Sequelize.Op.lt ]: new Date() + [ Op.lt ]: new Date() } } } @@ -428,12 +424,12 @@ export class VideoRedundancyModel extends Model { static async getStats (strategy: VideoRedundancyStrategy) { const actor = await getServerActor() - const query = { + const query: FindOptions = { raw: true, attributes: [ - [ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoFile.size')), '0'), 'totalUsed' ], - [ Sequelize.fn('COUNT', Sequelize.fn('DISTINCT', Sequelize.col('videoId'))), 'totalVideos' ], - [ Sequelize.fn('COUNT', Sequelize.col('videoFileId')), 'totalVideoFiles' ] + [ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ], + [ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ], + [ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ] ], where: { strategy, @@ -448,9 +444,9 @@ export class VideoRedundancyModel extends Model { ] } - return VideoRedundancyModel.findOne(query as any) // FIXME: typings + return VideoRedundancyModel.findOne(query) .then((r: any) => ({ - totalUsed: parseInt(r.totalUsed.toString(), 10), + totalUsed: parseAggregateResult(r.totalUsed), totalVideos: r.totalVideos, totalVideoFiles: r.totalVideoFiles })) @@ -503,7 +499,7 @@ export class VideoRedundancyModel extends Model { private static async buildVideoFileForDuplication () { const actor = await getServerActor() - const notIn = Sequelize.literal( + const notIn = literal( '(' + `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` + ')' @@ -515,7 +511,7 @@ export class VideoRedundancyModel extends Model { required: true, where: { id: { - [ Sequelize.Op.notIn ]: notIn + [ Op.notIn ]: notIn } } } diff --git a/server/models/server/server-blocklist.ts b/server/models/server/server-blocklist.ts index 450f27152..92c01f642 100644 --- a/server/models/server/server-blocklist.ts +++ b/server/models/server/server-blocklist.ts @@ -9,11 +9,11 @@ enum ScopeNames { WITH_SERVER = 'WITH_SERVER' } -@Scopes({ +@Scopes(() => ({ [ScopeNames.WITH_ACCOUNT]: { include: [ { - model: () => AccountModel, + model: AccountModel, required: true } ] @@ -21,12 +21,12 @@ enum ScopeNames { [ScopeNames.WITH_SERVER]: { include: [ { - model: () => ServerModel, + model: ServerModel, required: true } ] } -}) +})) @Table({ tableName: 'serverBlocklist', diff --git a/server/models/utils.ts b/server/models/utils.ts index 98170a00e..2b172f608 100644 --- a/server/models/utils.ts +++ b/server/models/utils.ts @@ -118,6 +118,15 @@ function buildWhereIdOrUUID (id: number | string) { return validator.isInt('' + id) ? { id } : { uuid: id } } +function parseAggregateResult (result: any) { + if (!result) return 0 + + const total = parseInt(result + '', 10) + if (isNaN(total)) return 0 + + return total +} + // --------------------------------------------------------------------------- export { @@ -131,7 +140,8 @@ export { buildServerIdsFollowedBy, buildTrigramSearchIndex, buildWhereIdOrUUID, - isOutdated + isOutdated, + parseAggregateResult } // --------------------------------------------------------------------------- diff --git a/server/models/video/tag.ts b/server/models/video/tag.ts index 048b47613..0fc3cfd4c 100644 --- a/server/models/video/tag.ts +++ b/server/models/video/tag.ts @@ -75,7 +75,7 @@ export class TagModel extends Model { type: QueryTypes.SELECT as QueryTypes.SELECT } - return TagModel.sequelize.query<{ name }>(query, options) + return TagModel.sequelize.query<{ name: string }>(query, options) .then(data => data.map(d => d.name)) } } diff --git a/server/models/video/thumbnail.ts b/server/models/video/thumbnail.ts index baa5533ac..ec945893f 100644 --- a/server/models/video/thumbnail.ts +++ b/server/models/video/thumbnail.ts @@ -75,8 +75,8 @@ export class ThumbnailModel extends Model { updatedAt: Date private static types: { [ id in ThumbnailType ]: { label: string, directory: string, staticPath: string } } = { - [ThumbnailType.THUMBNAIL]: { - label: 'thumbnail', + [ThumbnailType.MINIATURE]: { + label: 'miniature', directory: CONFIG.STORAGE.THUMBNAILS_DIR, staticPath: STATIC_PATHS.THUMBNAILS }, diff --git a/server/models/video/video-caption.ts b/server/models/video/video-caption.ts index 45c60e26b..76243bf48 100644 --- a/server/models/video/video-caption.ts +++ b/server/models/video/video-caption.ts @@ -12,7 +12,7 @@ import { Table, UpdatedAt } from 'sequelize-typescript' -import { throwIfNotValid } from '../utils' +import { buildWhereIdOrUUID, throwIfNotValid } from '../utils' import { VideoModel } from './video' import { isVideoCaptionLanguageValid } from '../../helpers/custom-validators/video-captions' import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model' @@ -26,17 +26,17 @@ export enum ScopeNames { WITH_VIDEO_UUID_AND_REMOTE = 'WITH_VIDEO_UUID_AND_REMOTE' } -@Scopes({ +@Scopes(() => ({ [ScopeNames.WITH_VIDEO_UUID_AND_REMOTE]: { include: [ { attributes: [ 'uuid', 'remote' ], - model: () => VideoModel.unscoped(), + model: VideoModel.unscoped(), required: true } ] } -}) +})) @Table({ tableName: 'videoCaption', @@ -97,12 +97,9 @@ export class VideoCaptionModel extends Model { const videoInclude = { model: VideoModel.unscoped(), attributes: [ 'id', 'remote', 'uuid' ], - where: { } + where: buildWhereIdOrUUID(videoId) } - if (typeof videoId === 'string') videoInclude.where['uuid'] = videoId - else videoInclude.where['id'] = videoId - const query = { where: { language diff --git a/server/models/video/video-change-ownership.ts b/server/models/video/video-change-ownership.ts index a4f4d53f1..171d4574d 100644 --- a/server/models/video/video-change-ownership.ts +++ b/server/models/video/video-change-ownership.ts @@ -23,29 +23,29 @@ enum ScopeNames { } ] }) -@Scopes({ +@Scopes(() => ({ [ScopeNames.FULL]: { include: [ { - model: () => AccountModel, + model: AccountModel, as: 'Initiator', required: true }, { - model: () => AccountModel, + model: AccountModel, as: 'NextOwner', required: true }, { - model: () => VideoModel, + model: VideoModel, required: true, include: [ - { model: () => VideoFileModel } + { model: VideoFileModel } ] } - ] as any // FIXME: sequelize typings + ] } -}) +})) export class VideoChangeOwnershipModel extends Model { @CreatedAt createdAt: Date diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts index 901006dea..fb70e6625 100644 --- a/server/models/video/video-channel.ts +++ b/server/models/video/video-channel.ts @@ -58,15 +58,15 @@ type AvailableForListOptions = { actorId: number } -@DefaultScope({ +@DefaultScope(() => ({ include: [ { - model: () => ActorModel, + model: ActorModel, required: true } ] -}) -@Scopes({ +})) +@Scopes(() => ({ [ScopeNames.SUMMARY]: (withAccount = false) => { const base: FindOptions = { attributes: [ 'name', 'description', 'id', 'actorId' ], @@ -142,22 +142,22 @@ type AvailableForListOptions = { [ScopeNames.WITH_ACCOUNT]: { include: [ { - model: () => AccountModel, + model: AccountModel, required: true } ] }, [ScopeNames.WITH_VIDEOS]: { include: [ - () => VideoModel + VideoModel ] }, [ScopeNames.WITH_ACTOR]: { include: [ - () => ActorModel + ActorModel ] } -}) +})) @Table({ tableName: 'videoChannel', indexes diff --git a/server/models/video/video-comment.ts b/server/models/video/video-comment.ts index 5f7cd3671..fee11ec5f 100644 --- a/server/models/video/video-comment.ts +++ b/server/models/video/video-comment.ts @@ -30,7 +30,7 @@ import { UserModel } from '../account/user' import { actorNameAlphabet } from '../../helpers/custom-validators/activitypub/actor' import { regexpCapture } from '../../helpers/regexp' import { uniq } from 'lodash' -import { FindOptions, Op, Order, Sequelize, Transaction } from 'sequelize' +import { FindOptions, Op, Order, ScopeOptions, Sequelize, Transaction } from 'sequelize' enum ScopeNames { WITH_ACCOUNT = 'WITH_ACCOUNT', @@ -39,7 +39,7 @@ enum ScopeNames { ATTRIBUTES_FOR_API = 'ATTRIBUTES_FOR_API' } -@Scopes({ +@Scopes(() => ({ [ScopeNames.ATTRIBUTES_FOR_API]: (serverAccountId: number, userAccountId?: number) => { return { attributes: { @@ -63,34 +63,34 @@ enum ScopeNames { ] ] } - } + } as FindOptions }, [ScopeNames.WITH_ACCOUNT]: { include: [ { - model: () => AccountModel, + model: AccountModel, include: [ { - model: () => ActorModel, + model: ActorModel, include: [ { - model: () => ServerModel, + model: ServerModel, required: false }, { - model: () => AvatarModel, + model: AvatarModel, required: false } ] } ] } - ] as any // FIXME: sequelize typings + ] }, [ScopeNames.WITH_IN_REPLY_TO]: { include: [ { - model: () => VideoCommentModel, + model: VideoCommentModel, as: 'InReplyToVideoComment' } ] @@ -98,19 +98,19 @@ enum ScopeNames { [ScopeNames.WITH_VIDEO]: { include: [ { - model: () => VideoModel, + model: VideoModel, required: true, include: [ { - model: () => VideoChannelModel.unscoped(), + model: VideoChannelModel.unscoped(), required: true, include: [ { - model: () => AccountModel, + model: AccountModel, required: true, include: [ { - model: () => ActorModel, + model: ActorModel, required: true } ] @@ -119,9 +119,9 @@ enum ScopeNames { } ] } - ] as any // FIXME: sequelize typings + ] } -}) +})) @Table({ tableName: 'videoComment', indexes: [ @@ -313,8 +313,7 @@ export class VideoCommentModel extends Model { } } - // FIXME: typings - const scopes: any[] = [ + const scopes: (string | ScopeOptions)[] = [ ScopeNames.WITH_ACCOUNT, { method: [ ScopeNames.ATTRIBUTES_FOR_API, serverAccountId, userAccountId ] diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index c14d96bc5..2203a7aba 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -19,11 +19,11 @@ import { isVideoFileSizeValid, isVideoFPSResolutionValid } from '../../helpers/custom-validators/videos' -import { throwIfNotValid } from '../utils' +import { parseAggregateResult, throwIfNotValid } from '../utils' import { VideoModel } from './video' -import * as Sequelize from 'sequelize' import { VideoRedundancyModel } from '../redundancy/video-redundancy' import { VideoStreamingPlaylistModel } from './video-streaming-playlist' +import { FindOptions, QueryTypes, Transaction } from 'sequelize' @Table({ tableName: 'videoFile', @@ -97,15 +97,13 @@ export class VideoFileModel extends Model { static doesInfohashExist (infoHash: string) { const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1' const options = { - type: Sequelize.QueryTypes.SELECT, + type: QueryTypes.SELECT, bind: { infoHash }, raw: true } return VideoModel.sequelize.query(query, options) - .then(results => { - return results.length === 1 - }) + .then(results => results.length === 1) } static loadWithVideo (id: number) { @@ -121,7 +119,7 @@ export class VideoFileModel extends Model { return VideoFileModel.findByPk(id, options) } - static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Sequelize.Transaction) { + static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) { const query = { include: [ { @@ -144,8 +142,8 @@ export class VideoFileModel extends Model { return VideoFileModel.findAll(query) } - static async getStats () { - let totalLocalVideoFilesSize = await VideoFileModel.sum('size', { + static getStats () { + const query: FindOptions = { include: [ { attributes: [], @@ -155,13 +153,12 @@ export class VideoFileModel extends Model { } } ] - } as any) - // Sequelize could return null... - if (!totalLocalVideoFilesSize) totalLocalVideoFilesSize = 0 - - return { - totalLocalVideoFilesSize } + + return VideoFileModel.aggregate('size', 'SUM', query) + .then(result => ({ + totalLocalVideoFilesSize: parseAggregateResult(result) + })) } hasSameUniqueKeysThan (other: VideoFileModel) { diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts index 89992a5a8..877fcbc57 100644 --- a/server/models/video/video-format-utils.ts +++ b/server/models/video/video-format-utils.ts @@ -59,7 +59,7 @@ function videoModelToFormattedJSON (video: VideoModel, options?: VideoFormatting views: video.views, likes: video.likes, dislikes: video.dislikes, - thumbnailPath: video.getThumbnailStaticPath(), + thumbnailPath: video.getMiniatureStaticPath(), previewPath: video.getPreviewStaticPath(), embedPath: video.getEmbedStaticPath(), createdAt: video.createdAt, @@ -301,6 +301,8 @@ function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject { }) } + const miniature = video.getMiniature() + return { type: 'Video' as 'Video', id: video.url, @@ -326,10 +328,10 @@ function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject { subtitleLanguage, icon: { type: 'Image', - url: video.getThumbnail().getUrl(), + url: miniature.getUrl(), mediaType: 'image/jpeg', - width: video.getThumbnail().width, - height: video.getThumbnail().height + width: miniature.width, + height: miniature.height }, url, likes: getVideoLikesActivityPubUrl(video), diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts index 588a13a4f..480a671c8 100644 --- a/server/models/video/video-import.ts +++ b/server/models/video/video-import.ts @@ -21,18 +21,18 @@ import { VideoImport, VideoImportState } from '../../../shared' import { isVideoMagnetUriValid } from '../../helpers/custom-validators/videos' import { UserModel } from '../account/user' -@DefaultScope({ +@DefaultScope(() => ({ include: [ { - model: () => UserModel.unscoped(), + model: UserModel.unscoped(), required: true }, { - model: () => VideoModel.scope([ VideoModelScopeNames.WITH_ACCOUNT_DETAILS, VideoModelScopeNames.WITH_TAGS]), + model: VideoModel.scope([ VideoModelScopeNames.WITH_ACCOUNT_DETAILS, VideoModelScopeNames.WITH_TAGS]), required: false } ] -}) +})) @Table({ tableName: 'videoImport', diff --git a/server/models/video/video-playlist.ts b/server/models/video/video-playlist.ts index 3e436acfc..63b4a0715 100644 --- a/server/models/video/video-playlist.ts +++ b/server/models/video/video-playlist.ts @@ -42,7 +42,7 @@ import { activityPubCollectionPagination } from '../../helpers/activitypub' import { VideoPlaylistType } from '../../../shared/models/videos/playlist/video-playlist-type.model' import { ThumbnailModel } from './thumbnail' import { ActivityIconObject } from '../../../shared/models/activitypub/objects' -import { fn, literal, Op, Transaction } from 'sequelize' +import { FindOptions, literal, Op, ScopeOptions, Transaction, WhereOptions } from 'sequelize' enum ScopeNames { AVAILABLE_FOR_LIST = 'AVAILABLE_FOR_LIST', @@ -61,11 +61,11 @@ type AvailableForListOptions = { privateAndUnlisted?: boolean } -@Scopes({ +@Scopes(() => ({ [ ScopeNames.WITH_THUMBNAIL ]: { include: [ { - model: () => ThumbnailModel, + model: ThumbnailModel, required: false } ] @@ -73,21 +73,17 @@ type AvailableForListOptions = { [ ScopeNames.WITH_VIDEOS_LENGTH ]: { attributes: { include: [ - [ - fn('COUNT', 'toto'), - 'coucou' - ], [ literal('(SELECT COUNT("id") FROM "videoPlaylistElement" WHERE "videoPlaylistId" = "VideoPlaylistModel"."id")'), 'videosLength' ] ] } - }, + } as FindOptions, [ ScopeNames.WITH_ACCOUNT ]: { include: [ { - model: () => AccountModel, + model: AccountModel, required: true } ] @@ -95,11 +91,11 @@ type AvailableForListOptions = { [ ScopeNames.WITH_ACCOUNT_AND_CHANNEL_SUMMARY ]: { include: [ { - model: () => AccountModel.scope(AccountScopeNames.SUMMARY), + model: AccountModel.scope(AccountScopeNames.SUMMARY), required: true }, { - model: () => VideoChannelModel.scope(VideoChannelScopeNames.SUMMARY), + model: VideoChannelModel.scope(VideoChannelScopeNames.SUMMARY), required: false } ] @@ -107,11 +103,11 @@ type AvailableForListOptions = { [ ScopeNames.WITH_ACCOUNT_AND_CHANNEL ]: { include: [ { - model: () => AccountModel, + model: AccountModel, required: true }, { - model: () => VideoChannelModel, + model: VideoChannelModel, required: false } ] @@ -132,7 +128,7 @@ type AvailableForListOptions = { ] } - const whereAnd: any[] = [] + const whereAnd: WhereOptions[] = [] if (options.privateAndUnlisted !== true) { whereAnd.push({ @@ -178,9 +174,9 @@ type AvailableForListOptions = { required: false } ] - } + } as FindOptions } -}) +})) @Table({ tableName: 'videoPlaylist', @@ -269,6 +265,7 @@ export class VideoPlaylistModel extends Model { VideoPlaylistElements: VideoPlaylistElementModel[] @HasOne(() => ThumbnailModel, { + foreignKey: { name: 'videoPlaylistId', allowNull: true @@ -294,7 +291,7 @@ export class VideoPlaylistModel extends Model { order: getSort(options.sort) } - const scopes = [ + const scopes: (string | ScopeOptions)[] = [ { method: [ ScopeNames.AVAILABLE_FOR_LIST, @@ -306,7 +303,7 @@ export class VideoPlaylistModel extends Model { privateAndUnlisted: options.privateAndUnlisted } as AvailableForListOptions ] - } as any, // FIXME: typings + }, ScopeNames.WITH_VIDEOS_LENGTH, ScopeNames.WITH_THUMBNAIL ] @@ -348,7 +345,7 @@ export class VideoPlaylistModel extends Model { model: VideoPlaylistElementModel.unscoped(), where: { videoId: { - [Op.any]: videoIds + [Op.in]: videoIds // FIXME: sequelize ANY seems broken } }, required: true @@ -427,12 +424,10 @@ export class VideoPlaylistModel extends Model { return VideoPlaylistModel.update({ privacy: VideoPlaylistPrivacy.PRIVATE, videoChannelId: null }, query) } - setThumbnail (thumbnail: ThumbnailModel) { - this.Thumbnail = thumbnail - } + async setAndSaveThumbnail (thumbnail: ThumbnailModel, t: Transaction) { + thumbnail.videoPlaylistId = this.id - getThumbnail () { - return this.Thumbnail + this.Thumbnail = await thumbnail.save({ transaction: t }) } hasThumbnail () { @@ -448,13 +443,13 @@ export class VideoPlaylistModel extends Model { getThumbnailUrl () { if (!this.hasThumbnail()) return null - return WEBSERVER.URL + STATIC_PATHS.THUMBNAILS + this.getThumbnail().filename + return WEBSERVER.URL + STATIC_PATHS.THUMBNAILS + this.Thumbnail.filename } getThumbnailStaticPath () { if (!this.hasThumbnail()) return null - return join(STATIC_PATHS.THUMBNAILS, this.getThumbnail().filename) + return join(STATIC_PATHS.THUMBNAILS, this.Thumbnail.filename) } setAsRefreshed () { diff --git a/server/models/video/video-share.ts b/server/models/video/video-share.ts index c83f6c5b0..fda2d7cea 100644 --- a/server/models/video/video-share.ts +++ b/server/models/video/video-share.ts @@ -14,15 +14,15 @@ enum ScopeNames { WITH_ACTOR = 'WITH_ACTOR' } -@Scopes({ +@Scopes(() => ({ [ScopeNames.FULL]: { include: [ { - model: () => ActorModel, + model: ActorModel, required: true }, { - model: () => VideoModel, + model: VideoModel, required: true } ] @@ -30,12 +30,12 @@ enum ScopeNames { [ScopeNames.WITH_ACTOR]: { include: [ { - model: () => ActorModel, + model: ActorModel, required: true } ] } -}) +})) @Table({ tableName: 'videoShare', indexes: [ diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts index b30267e09..31dc82c54 100644 --- a/server/models/video/video-streaming-playlist.ts +++ b/server/models/video/video-streaming-playlist.ts @@ -26,7 +26,7 @@ import { QueryTypes, Op } from 'sequelize' fields: [ 'p2pMediaLoaderInfohashes' ], using: 'gin' } - ] as any // FIXME: sequelize typings + ] }) export class VideoStreamingPlaylistModel extends Model { @CreatedAt @@ -46,7 +46,7 @@ export class VideoStreamingPlaylistModel extends Model throwIfNotValid(value, v => isArrayOf(v, isVideoFileInfoHashValid), 'info hashes')) - @Column({ type: DataType.ARRAY(DataType.STRING) }) // FIXME: typings + @Column(DataType.ARRAY(DataType.STRING)) p2pMediaLoaderInfohashes: string[] @AllowNull(false) @@ -87,7 +87,7 @@ export class VideoStreamingPlaylistModel extends Model(query, options) + return VideoModel.sequelize.query(query, options) .then(results => results.length === 1) } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 329cebd28..18f18795e 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -227,12 +227,12 @@ type AvailableForListIDsOptions = { historyOfUser?: UserModel } -@Scopes({ +@Scopes(() => ({ [ ScopeNames.FOR_API ]: (options: ForAPIOptions) => { const query: FindOptions = { where: { id: { - [ Op.in ]: options.ids // FIXME: sequelize any seems broken + [ Op.in ]: options.ids // FIXME: sequelize ANY seems broken } }, include: [ @@ -486,7 +486,7 @@ type AvailableForListIDsOptions = { [ ScopeNames.WITH_THUMBNAILS ]: { include: [ { - model: () => ThumbnailModel, + model: ThumbnailModel, required: false } ] @@ -495,48 +495,48 @@ type AvailableForListIDsOptions = { include: [ { attributes: [ 'accountId' ], - model: () => VideoChannelModel.unscoped(), + model: VideoChannelModel.unscoped(), required: true, include: [ { attributes: [ 'userId' ], - model: () => AccountModel.unscoped(), + model: AccountModel.unscoped(), required: true } ] } - ] as any // FIXME: sequelize typings + ] }, [ ScopeNames.WITH_ACCOUNT_DETAILS ]: { include: [ { - model: () => VideoChannelModel.unscoped(), + model: VideoChannelModel.unscoped(), required: true, include: [ { attributes: { exclude: [ 'privateKey', 'publicKey' ] }, - model: () => ActorModel.unscoped(), + model: ActorModel.unscoped(), required: true, include: [ { attributes: [ 'host' ], - model: () => ServerModel.unscoped(), + model: ServerModel.unscoped(), required: false }, { - model: () => AvatarModel.unscoped(), + model: AvatarModel.unscoped(), required: false } ] }, { - model: () => AccountModel.unscoped(), + model: AccountModel.unscoped(), required: true, include: [ { - model: () => ActorModel.unscoped(), + model: ActorModel.unscoped(), attributes: { exclude: [ 'privateKey', 'publicKey' ] }, @@ -544,11 +544,11 @@ type AvailableForListIDsOptions = { include: [ { attributes: [ 'host' ], - model: () => ServerModel.unscoped(), + model: ServerModel.unscoped(), required: false }, { - model: () => AvatarModel.unscoped(), + model: AvatarModel.unscoped(), required: false } ] @@ -557,16 +557,16 @@ type AvailableForListIDsOptions = { } ] } - ] as any // FIXME: sequelize typings + ] }, [ ScopeNames.WITH_TAGS ]: { - include: [ () => TagModel ] + include: [ TagModel ] }, [ ScopeNames.WITH_BLACKLISTED ]: { include: [ { attributes: [ 'id', 'reason' ], - model: () => VideoBlacklistModel, + model: VideoBlacklistModel, required: false } ] @@ -588,8 +588,7 @@ type AvailableForListIDsOptions = { include: [ { model: VideoFileModel.unscoped(), - // FIXME: typings - [ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join + separate: true, // We may have multiple files, having multiple redundancies so let's separate this join required: false, include: subInclude } @@ -613,8 +612,7 @@ type AvailableForListIDsOptions = { include: [ { model: VideoStreamingPlaylistModel.unscoped(), - // FIXME: typings - [ 'separate' as any ]: true, // We may have multiple streaming playlists, having multiple redundancies so let's separate this join + separate: true, // We may have multiple streaming playlists, having multiple redundancies so let's separate this join required: false, include: subInclude } @@ -624,7 +622,7 @@ type AvailableForListIDsOptions = { [ ScopeNames.WITH_SCHEDULED_UPDATE ]: { include: [ { - model: () => ScheduleVideoUpdateModel.unscoped(), + model: ScheduleVideoUpdateModel.unscoped(), required: false } ] @@ -643,7 +641,7 @@ type AvailableForListIDsOptions = { ] } } -}) +})) @Table({ tableName: 'video', indexes @@ -1075,15 +1073,14 @@ export class VideoModel extends Model { } return Bluebird.all([ - // FIXME: typing issue - VideoModel.scope(ScopeNames.WITH_THUMBNAILS).findAll(query as any), - VideoModel.sequelize.query<{ total: number }>(rawCountQuery, { type: QueryTypes.SELECT }) + VideoModel.scope(ScopeNames.WITH_THUMBNAILS).findAll(query), + VideoModel.sequelize.query<{ total: string }>(rawCountQuery, { type: QueryTypes.SELECT }) ]).then(([ rows, totals ]) => { // totals: totalVideos + totalVideoShares let totalVideos = 0 let totalVideoShares = 0 - if (totals[ 0 ]) totalVideos = parseInt(totals[ 0 ].total + '', 10) - if (totals[ 1 ]) totalVideoShares = parseInt(totals[ 1 ].total + '', 10) + if (totals[ 0 ]) totalVideos = parseInt(totals[ 0 ].total, 10) + if (totals[ 1 ]) totalVideoShares = parseInt(totals[ 1 ].total, 10) const total = totalVideos + totalVideoShares return { @@ -1094,50 +1091,58 @@ export class VideoModel extends Model { } static listUserVideosForApi (accountId: number, start: number, count: number, sort: string, withFiles = false) { - const query: FindOptions = { - offset: start, - limit: count, - order: getVideoSort(sort), - include: [ - { - model: VideoChannelModel, - required: true, - include: [ - { - model: AccountModel, - where: { - id: accountId - }, - required: true - } - ] - }, - { - model: ScheduleVideoUpdateModel, - required: false - }, - { - model: VideoBlacklistModel, - required: false - } - ] + function buildBaseQuery (): FindOptions { + return { + offset: start, + limit: count, + order: getVideoSort(sort), + include: [ + { + model: VideoChannelModel, + required: true, + include: [ + { + model: AccountModel, + where: { + id: accountId + }, + required: true + } + ] + } + ] + } } + const countQuery = buildBaseQuery() + const findQuery = buildBaseQuery() + + findQuery.include.push({ + model: ScheduleVideoUpdateModel, + required: false + }) + + findQuery.include.push({ + model: VideoBlacklistModel, + required: false + }) + if (withFiles === true) { - query.include.push({ + findQuery.include.push({ model: VideoFileModel.unscoped(), required: true }) } - return VideoModel.scope(ScopeNames.WITH_THUMBNAILS) - .findAndCountAll(query) - .then(({ rows, count }) => { - return { - data: rows, - total: count - } - }) + return Promise.all([ + VideoModel.count(countQuery), + VideoModel.findAll(findQuery) + ]).then(([ count, rows ]) => { + return { + data: rows, + total: count + } + }) } static async listForApi (options: { @@ -1404,12 +1409,12 @@ export class VideoModel extends Model { const where = buildWhereIdOrUUID(id) const options = { - order: [ [ 'Tags', 'name', 'ASC' ] ] as any, // FIXME: sequelize typings + order: [ [ 'Tags', 'name', 'ASC' ] ] as any, where, transaction: t } - const scopes = [ + const scopes: (string | ScopeOptions)[] = [ ScopeNames.WITH_TAGS, ScopeNames.WITH_BLACKLISTED, ScopeNames.WITH_ACCOUNT_DETAILS, @@ -1420,7 +1425,7 @@ export class VideoModel extends Model { ] if (userId) { - scopes.push({ method: [ ScopeNames.WITH_USER_HISTORY, userId ] } as any) // FIXME: typings + scopes.push({ method: [ ScopeNames.WITH_USER_HISTORY, userId ] }) } return VideoModel @@ -1437,18 +1442,18 @@ export class VideoModel extends Model { transaction: t } - const scopes = [ + const scopes: (string | ScopeOptions)[] = [ ScopeNames.WITH_TAGS, ScopeNames.WITH_BLACKLISTED, ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_SCHEDULED_UPDATE, ScopeNames.WITH_THUMBNAILS, - { method: [ ScopeNames.WITH_FILES, true ] } as any, // FIXME: typings - { method: [ ScopeNames.WITH_STREAMING_PLAYLISTS, true ] } as any // FIXME: typings + { method: [ ScopeNames.WITH_FILES, true ] }, + { method: [ ScopeNames.WITH_STREAMING_PLAYLISTS, true ] } ] if (userId) { - scopes.push({ method: [ ScopeNames.WITH_USER_HISTORY, userId ] } as any) // FIXME: typings + scopes.push({ method: [ ScopeNames.WITH_USER_HISTORY, userId ] }) } return VideoModel @@ -1520,9 +1525,9 @@ export class VideoModel extends Model { attributes: [ field ], limit: count, group: field, - having: Sequelize.where(Sequelize.fn('COUNT', Sequelize.col(field)), { - [ Op.gte ]: threshold - }) as any, // FIXME: typings + having: Sequelize.where( + Sequelize.fn('COUNT', Sequelize.col(field)), { [ Op.gte ]: threshold } + ), order: [ (this.sequelize as any).random() ] } @@ -1594,16 +1599,10 @@ export class VideoModel extends Model { ] } - // FIXME: typing - const apiScope: any[] = [ ScopeNames.WITH_THUMBNAILS ] + const apiScope: (string | ScopeOptions)[] = [ ScopeNames.WITH_THUMBNAILS ] if (options.user) { apiScope.push({ method: [ ScopeNames.WITH_USER_HISTORY, options.user.id ] }) - - // Even if the relation is n:m, we know that a user only have 0..1 video history - // So we won't have multiple rows for the same video - // A subquery adds some bugs in our query so disable it - secondQuery.subQuery = false } apiScope.push({ @@ -1651,13 +1650,17 @@ export class VideoModel extends Model { return maxBy(this.VideoFiles, file => file.resolution) } - addThumbnail (thumbnail: ThumbnailModel) { + async addAndSaveThumbnail (thumbnail: ThumbnailModel, transaction: Transaction) { + thumbnail.videoId = this.id + + const savedThumbnail = await thumbnail.save({ transaction }) + if (Array.isArray(this.Thumbnails) === false) this.Thumbnails = [] // Already have this thumbnail, skip - if (this.Thumbnails.find(t => t.id === thumbnail.id)) return + if (this.Thumbnails.find(t => t.id === savedThumbnail.id)) return - this.Thumbnails.push(thumbnail) + this.Thumbnails.push(savedThumbnail) } getVideoFilename (videoFile: VideoFileModel) { @@ -1668,10 +1671,10 @@ export class VideoModel extends Model { return this.uuid + '.jpg' } - getThumbnail () { + getMiniature () { if (Array.isArray(this.Thumbnails) === false) return undefined - return this.Thumbnails.find(t => t.type === ThumbnailType.THUMBNAIL) + return this.Thumbnails.find(t => t.type === ThumbnailType.MINIATURE) } generatePreviewName () { @@ -1732,8 +1735,8 @@ export class VideoModel extends Model { return '/videos/embed/' + this.uuid } - getThumbnailStaticPath () { - const thumbnail = this.getThumbnail() + getMiniatureStaticPath () { + const thumbnail = this.getMiniature() if (!thumbnail) return null return join(STATIC_PATHS.THUMBNAILS, thumbnail.filename) diff --git a/server/typings/sequelize.ts b/server/typings/sequelize.ts new file mode 100644 index 000000000..9cd83612d --- /dev/null +++ b/server/typings/sequelize.ts @@ -0,0 +1,18 @@ +import { Model } from 'sequelize-typescript' + +// Thanks to sequelize-typescript: https://github.com/RobinBuschmann/sequelize-typescript + +export type Diff = + ({ [P in T]: P } & { [P in U]: never } & { [ x: string ]: never })[T] + +export type Omit = { [P in Diff]: T[P] } + +export type RecursivePartial = { [P in keyof T]?: RecursivePartial } + +export type FilteredModelAttributes> = RecursivePartial>> & { + id?: number | any + createdAt?: Date | any + updatedAt?: Date | any + deletedAt?: Date | any + version?: number | any +} diff --git a/shared/extra-utils/miscs/sql.ts b/shared/extra-utils/miscs/sql.ts index 7aebffc32..3cfae5c23 100644 --- a/shared/extra-utils/miscs/sql.ts +++ b/shared/extra-utils/miscs/sql.ts @@ -15,7 +15,6 @@ function getSequelize (serverNumber: number) { dialect: 'postgres', host, port, - operatorsAliases: false, logging: false }) diff --git a/shared/models/videos/thumbnail.type.ts b/shared/models/videos/thumbnail.type.ts index 317b4db43..d6c2bef7b 100644 --- a/shared/models/videos/thumbnail.type.ts +++ b/shared/models/videos/thumbnail.type.ts @@ -1,4 +1,4 @@ export enum ThumbnailType { - THUMBNAIL = 1, + MINIATURE = 1, PREVIEW = 2 } diff --git a/yarn.lock b/yarn.lock index dd1de4fa0..aa616f4a0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7461,17 +7461,17 @@ sequelize-pool@^1.0.2: dependencies: bluebird "^3.5.3" -sequelize-typescript@^1.0.0-beta.1: - version "1.0.0-beta.1" - resolved "https://registry.yarnpkg.com/sequelize-typescript/-/sequelize-typescript-1.0.0-beta.1.tgz#402279fec52669cbd78ecbf50e189638483a7360" - integrity sha512-xD28kqa1rIKujlmgA4hWQgtwFfRM6tLv1/mnZOrOFEZxvSWazUbTzqGB7OZydZDNj3iJnyrV1l6i6HOfvrpvEw== +sequelize-typescript@1.0.0-beta.2: + version "1.0.0-beta.2" + resolved "https://registry.yarnpkg.com/sequelize-typescript/-/sequelize-typescript-1.0.0-beta.2.tgz#fd9ae47ecf8b159e32e19c1298426cc9773cebd8" + integrity sha512-Iu67kF/RunoeBQBsU5llViJkxAHBVmeS9DBP+eC63hkEwxeDGZgxOkodyW5v5k3h2DJ0MBO+clRURXoDb+/OHg== dependencies: glob "7.1.2" -sequelize@5.6.1: - version "5.6.1" - resolved "https://registry.yarnpkg.com/sequelize/-/sequelize-5.6.1.tgz#fc22306109fb2504a6573edfb3c469ec86fae873" - integrity sha512-QsXUDar6ow0HrF9BtnHRaNumu6qRYb97dfwvez/Z5guH3i6w6k8+bp6gP3VCiDC+2qX+jQIyrYohKg9evy8GFg== +sequelize@5.7.4: + version "5.7.4" + resolved "https://registry.yarnpkg.com/sequelize/-/sequelize-5.7.4.tgz#1631faadff65f3a345b9757fca60429c65ba8e57" + integrity sha512-CaVYpAgZQEsGDuZ+Oq6uIZy4pxQxscotuh5UGIaFRa0VkTIgV0IiF7vAhSv+1Wn+NvhKCvgJJ85M34BP3AdGNg== dependencies: bluebird "^3.5.0" cls-bluebird "^2.1.0" @@ -8678,10 +8678,10 @@ typedarray@^0.0.6: resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= -typescript@^3.1.6: - version "3.4.1" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.4.1.tgz#b6691be11a881ffa9a05765a205cb7383f3b63c6" - integrity sha512-3NSMb2VzDQm8oBTLH6Nj55VVtUEpe/rgkIzMir0qVoLyjDZlnMBva0U6vDiV3IH+sl/Yu6oP5QwsAQtHPmDd2Q== +typescript@^3.4.3: + version "3.4.3" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.4.3.tgz#0eb320e4ace9b10eadf5bc6103286b0f8b7c224f" + integrity sha512-FFgHdPt4T/duxx6Ndf7hwgMZZjZpB+U0nMNGVCYPq0rEzWKjEDobm4J6yb3CS7naZ0yURFqdw9Gwc7UOh/P9oQ== uid-number@0.0.6: version "0.0.6"