From 8f0bc73d7d5f4c88cbc5588a0ece12b3855c8f98 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 11 Apr 2019 15:38:53 +0200 Subject: [PATCH] Add ability to limit videos history size --- config/default.yaml | 7 ++++ config/production.yaml.example | 6 ++++ server.ts | 2 ++ server/controllers/api/users/my-history.ts | 2 +- server/helpers/core-utils.ts | 2 +- server/helpers/custom-validators/videos.ts | 3 +- server/initializers/config.ts | 11 ++++-- server/initializers/constants.ts | 9 ++--- server/lib/schedulers/abstract-scheduler.ts | 3 +- .../remove-old-history-scheduler.ts | 32 +++++++++++++++++ server/middlewares/cache.ts | 4 +-- server/models/account/user-video-history.ts | 14 +++++++- server/tests/api/videos/videos-history.ts | 34 +++++++++++++++++-- shared/utils/videos/videos.ts | 4 ++- 14 files changed, 116 insertions(+), 17 deletions(-) create mode 100644 server/lib/schedulers/remove-old-history-scheduler.ts diff --git a/config/default.yaml b/config/default.yaml index 617159c2c..d45d84b90 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -111,6 +111,13 @@ tracker: # Reject peers that do a lot of announces (could improve privacy of TCP/UDP peers) reject_too_many_announces: false +history: + videos: + # If you want to limit users videos history + # -1 means there is no limitations + # Other values could be '6 months' or '30 days' etc (PeerTube will periodically delete old entries from database) + max_age: -1 + cache: previews: size: 500 # Max number of previews you want to cache diff --git a/config/production.yaml.example b/config/production.yaml.example index dd5c9769b..b813a65e9 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -112,6 +112,12 @@ tracker: # Reject peers that do a lot of announces (could improve privacy of TCP/UDP peers) reject_too_many_announces: false +history: + videos: + # If you want to limit users videos history + # -1 means there is no limitations + # Other values could be '6 months' or '30 days' etc (PeerTube will periodically delete old entries from database) + max_age: -1 ############################################################################### # diff --git a/server.ts b/server.ts index 110ae1ab8..f4f0c4d68 100644 --- a/server.ts +++ b/server.ts @@ -105,6 +105,7 @@ import { RemoveOldJobsScheduler } from './server/lib/schedulers/remove-old-jobs- import { UpdateVideosScheduler } from './server/lib/schedulers/update-videos-scheduler' import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-update-scheduler' import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler' +import { RemoveOldHistoryScheduler } from './server/lib/schedulers/remove-old-history-scheduler' import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto' import { PeerTubeSocket } from './server/lib/peertube-socket' import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls' @@ -240,6 +241,7 @@ async function startApplication () { UpdateVideosScheduler.Instance.enable() YoutubeDlUpdateScheduler.Instance.enable() VideosRedundancyScheduler.Instance.enable() + RemoveOldHistoryScheduler.Instance.enable() // Redis initialization Redis.Instance.init() diff --git a/server/controllers/api/users/my-history.ts b/server/controllers/api/users/my-history.ts index b30d3aec2..7025c0ff1 100644 --- a/server/controllers/api/users/my-history.ts +++ b/server/controllers/api/users/my-history.ts @@ -48,7 +48,7 @@ async function removeUserHistory (req: express.Request, res: express.Response) { const beforeDate = req.body.beforeDate || null await sequelizeTypescript.transaction(t => { - return UserVideoHistoryModel.removeHistoryBefore(user, beforeDate, t) + return UserVideoHistoryModel.removeUserHistoryBefore(user, beforeDate, t) }) // Do not send the delete to other instances, we delete OUR copy of this video abuse diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index f6d90bfca..305d3b71e 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts @@ -40,7 +40,7 @@ const timeTable = { month: 3600000 * 24 * 30 } -export function parseDuration (duration: number | string): number { +export function parseDurationToMs (duration: number | string): number { if (typeof duration === 'number') return duration if (typeof duration === 'string') { diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts index eb08ae4ad..214db17a1 100644 --- a/server/helpers/custom-validators/videos.ts +++ b/server/helpers/custom-validators/videos.ts @@ -5,7 +5,8 @@ import 'multer' import * as validator from 'validator' import { UserRight, VideoFilter, VideoPrivacy, VideoRateType } from '../../../shared' import { - CONSTRAINTS_FIELDS, MIMETYPES, + CONSTRAINTS_FIELDS, + MIMETYPES, VIDEO_CATEGORIES, VIDEO_LICENCES, VIDEO_PRIVACIES, diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 8dd62cba8..1f374dea9 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -2,7 +2,7 @@ import { IConfig } from 'config' import { dirname, join } from 'path' import { VideosRedundancy } from '../../shared/models' // Do not use barrels, remain constants as independent as possible -import { buildPath, parseBytes, parseDuration, root } from '../helpers/core-utils' +import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils' import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' import * as bytes from 'bytes' @@ -80,7 +80,7 @@ const CONFIG = { }, REDUNDANCY: { VIDEOS: { - CHECK_INTERVAL: parseDuration(config.get('redundancy.videos.check_interval')), + CHECK_INTERVAL: parseDurationToMs(config.get('redundancy.videos.check_interval')), STRATEGIES: buildVideosRedundancy(config.get('redundancy.videos.strategies')) } }, @@ -94,6 +94,11 @@ const CONFIG = { PRIVATE: config.get('tracker.private'), REJECT_TOO_MANY_ANNOUNCES: config.get('tracker.reject_too_many_announces') }, + HISTORY: { + VIDEOS: { + MAX_AGE: parseDurationToMs(config.get('history.videos.max_age')) + } + }, ADMIN: { get EMAIL () { return config.get('admin.email') } }, @@ -216,7 +221,7 @@ function buildVideosRedundancy (objs: any[]): VideosRedundancy[] { return objs.map(obj => { return Object.assign({}, obj, { - minLifetime: parseDuration(obj.min_lifetime), + minLifetime: parseDurationToMs(obj.min_lifetime), size: bytes.parse(obj.size), minViews: obj.min_views }) diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index a0609d7cd..f008cd291 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -158,12 +158,12 @@ const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days const VIDEO_IMPORT_TIMEOUT = 1000 * 3600 // 1 hour -// 1 hour -let SCHEDULER_INTERVALS_MS = { +const SCHEDULER_INTERVALS_MS = { actorFollowScores: 60000 * 60, // 1 hour removeOldJobs: 60000 * 60, // 1 hour updateVideos: 60000, // 1 minute - youtubeDLUpdate: 60000 * 60 * 24 // 1 day + youtubeDLUpdate: 60000 * 60 * 24, // 1 day + removeOldHistory: 60000 * 60 * 24 // 1 day } // --------------------------------------------------------------------------- @@ -591,6 +591,7 @@ if (isTestInstance() === true) { SCHEDULER_INTERVALS_MS.actorFollowScores = 1000 SCHEDULER_INTERVALS_MS.removeOldJobs = 10000 + SCHEDULER_INTERVALS_MS.removeOldHistory = 5000 SCHEDULER_INTERVALS_MS.updateVideos = 5000 REPEAT_JOBS[ 'videos-views' ] = { every: 5000 } @@ -734,7 +735,7 @@ function buildVideosExtname () { } function loadLanguages () { - VIDEO_LANGUAGES = buildLanguages() + Object.assign(VIDEO_LANGUAGES, buildLanguages()) } function buildLanguages () { diff --git a/server/lib/schedulers/abstract-scheduler.ts b/server/lib/schedulers/abstract-scheduler.ts index 86ea7aa38..0e6088911 100644 --- a/server/lib/schedulers/abstract-scheduler.ts +++ b/server/lib/schedulers/abstract-scheduler.ts @@ -1,4 +1,5 @@ import { logger } from '../../helpers/logger' +import * as Bluebird from 'bluebird' export abstract class AbstractScheduler { @@ -30,5 +31,5 @@ export abstract class AbstractScheduler { } } - protected abstract internalExecute (): Promise + protected abstract internalExecute (): Promise | Bluebird } diff --git a/server/lib/schedulers/remove-old-history-scheduler.ts b/server/lib/schedulers/remove-old-history-scheduler.ts new file mode 100644 index 000000000..1b5ff8394 --- /dev/null +++ b/server/lib/schedulers/remove-old-history-scheduler.ts @@ -0,0 +1,32 @@ +import { logger } from '../../helpers/logger' +import { AbstractScheduler } from './abstract-scheduler' +import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' +import { UserVideoHistoryModel } from '../../models/account/user-video-history' +import { CONFIG } from '../../initializers/config' +import { isTestInstance } from '../../helpers/core-utils' + +export class RemoveOldHistoryScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.removeOldHistory + + private constructor () { + super() + } + + protected internalExecute () { + if (CONFIG.HISTORY.VIDEOS.MAX_AGE === -1) return + + logger.info('Removing old videos history.') + + const now = new Date() + const beforeDate = new Date(now.getTime() - CONFIG.HISTORY.VIDEOS.MAX_AGE).toISOString() + + return UserVideoHistoryModel.removeOldHistory(beforeDate) + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/middlewares/cache.ts b/server/middlewares/cache.ts index 8ffe75700..e83d8d569 100644 --- a/server/middlewares/cache.ts +++ b/server/middlewares/cache.ts @@ -1,6 +1,6 @@ import * as express from 'express' import * as AsyncLock from 'async-lock' -import { parseDuration } from '../helpers/core-utils' +import { parseDurationToMs } from '../helpers/core-utils' import { Redis } from '../lib/redis' import { logger } from '../helpers/logger' @@ -24,7 +24,7 @@ function cacheRoute (lifetimeArg: string | number) { res.send = (body) => { if (res.statusCode >= 200 && res.statusCode < 400) { const contentType = res.get('content-type') - const lifetime = parseDuration(lifetimeArg) + const lifetime = parseDurationToMs(lifetimeArg) Redis.Instance.setCachedRoute(req, body, lifetime, contentType, res.statusCode) .then(() => done()) diff --git a/server/models/account/user-video-history.ts b/server/models/account/user-video-history.ts index 15cb399c9..49d2def81 100644 --- a/server/models/account/user-video-history.ts +++ b/server/models/account/user-video-history.ts @@ -67,7 +67,7 @@ export class UserVideoHistoryModel extends Model { }) } - static removeHistoryBefore (user: UserModel, beforeDate: string, t: Transaction) { + static removeUserHistoryBefore (user: UserModel, beforeDate: string, t: Transaction) { const query: DestroyOptions = { where: { userId: user.id @@ -83,4 +83,16 @@ export class UserVideoHistoryModel extends Model { return UserVideoHistoryModel.destroy(query) } + + static removeOldHistory (beforeDate: string) { + const query: DestroyOptions = { + where: { + updatedAt: { + [Op.lt]: beforeDate + } + } + } + + return UserVideoHistoryModel.destroy(query) + } } diff --git a/server/tests/api/videos/videos-history.ts b/server/tests/api/videos/videos-history.ts index f654a422b..f7d3a6aeb 100644 --- a/server/tests/api/videos/videos-history.ts +++ b/server/tests/api/videos/videos-history.ts @@ -7,14 +7,15 @@ import { flushTests, getVideosListWithToken, getVideoWithToken, - killallServers, + killallServers, reRunServer, runServer, searchVideoWithToken, ServerInfo, setAccessTokensToServers, updateMyUser, uploadVideo, - userLogin + userLogin, + wait } from '../../../../shared/utils' import { Video, VideoDetails } from '../../../../shared/models/videos' import { listMyVideosHistory, removeMyVideosHistory, userWatchVideo } from '../../../../shared/utils/videos/video-history' @@ -192,6 +193,35 @@ describe('Test videos history', function () { expect(videos[1].name).to.equal('video 3') }) + it('Should not clean old history', async function () { + this.timeout(50000) + + killallServers([ server ]) + + await reRunServer(server, { history: { videos: { max_age: '10 days' } } }) + + await wait(6000) + + // Should still have history + + const res = await listMyVideosHistory(server.url, server.accessToken) + + expect(res.body.total).to.equal(2) + }) + + it('Should clean old history', async function () { + this.timeout(50000) + + killallServers([ server ]) + + await reRunServer(server, { history: { videos: { max_age: '5 seconds' } } }) + + await wait(6000) + + const res = await listMyVideosHistory(server.url, server.accessToken) + expect(res.body.total).to.equal(0) + }) + after(async function () { killallServers([ server ]) diff --git a/shared/utils/videos/videos.ts b/shared/utils/videos/videos.ts index 54c6bccec..b5a07b792 100644 --- a/shared/utils/videos/videos.ts +++ b/shared/utils/videos/videos.ts @@ -18,9 +18,11 @@ import { } from '../' import * as validator from 'validator' import { VideoDetails, VideoPrivacy } from '../../models/videos' -import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants' +import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, loadLanguages, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants' import { dateIsValid, webtorrentAdd } from '../miscs/miscs' +loadLanguages() + type VideoAttributes = { name?: string category?: number