Put config redundancy strategies in "strategies" subkey

pull/1093/head
Chocobozzz 2018-09-19 16:12:07 +02:00
parent 12ba460e9e
commit d9bdd007d7
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
9 changed files with 46 additions and 40 deletions

View File

@ -71,16 +71,17 @@ trending:
# Once you have defined your strategies, choose which instances you want to cache in admin -> manage follows -> following # Once you have defined your strategies, choose which instances you want to cache in admin -> manage follows -> following
redundancy: redundancy:
videos: videos:
# - strategies:
# size: '10GB' # -
# strategy: 'most-views' # Cache videos that have the most views # size: '10GB'
# - # strategy: 'most-views' # Cache videos that have the most views
# size: '10GB' # -
# strategy: 'trending' # Cache trending videos # size: '10GB'
# - # strategy: 'trending' # Cache trending videos
# size: '10GB' # -
# strategy: 'recently-added' # Cache recently added videos # size: '10GB'
# minViews: 10 # Having at least x views # strategy: 'recently-added' # Cache recently added videos
# minViews: 10 # Having at least x views
cache: cache:
previews: previews:

View File

@ -72,16 +72,17 @@ trending:
# Once you have defined your strategies, choose which instances you want to cache in admin -> manage follows -> following # Once you have defined your strategies, choose which instances you want to cache in admin -> manage follows -> following
redundancy: redundancy:
videos: videos:
# - strategies:
# size: '10GB' # -
# strategy: 'most-views' # Cache videos that have the most views # size: '10GB'
# - # strategy: 'most-views' # Cache videos that have the most views
# size: '10GB' # -
# strategy: 'trending' # Cache trending videos # size: '10GB'
# - # strategy: 'trending' # Cache trending videos
# size: '10GB' # -
# strategy: 'recently-added' # Cache recently added videos # size: '10GB'
# minViews: 10 # Having at least x views # strategy: 'recently-added' # Cache recently added videos
# minViews: 10 # Having at least x views
############################################################################### ###############################################################################
# #

View File

@ -23,16 +23,17 @@ log:
redundancy: redundancy:
videos: videos:
- strategies:
size: '10MB' -
strategy: 'most-views' size: '10MB'
- strategy: 'most-views'
size: '10MB' -
strategy: 'trending' size: '10MB'
- strategy: 'trending'
size: '10MB' -
strategy: 'recently-added' size: '10MB'
minViews: 1 strategy: 'recently-added'
minViews: 1
cache: cache:
previews: previews:

View File

@ -23,7 +23,7 @@ async function getStats (req: express.Request, res: express.Response, next: expr
const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats()
const videosRedundancyStats = await Promise.all( const videosRedundancyStats = await Promise.all(
CONFIG.REDUNDANCY.VIDEOS.map(r => { CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.map(r => {
return VideoRedundancyModel.getStats(r.strategy) return VideoRedundancyModel.getStats(r.strategy)
.then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size })) .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size }))
}) })

View File

@ -40,7 +40,7 @@ function checkConfig () {
} }
// Redundancies // Redundancies
const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos') const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos.strategies')
if (isArray(redundancyVideos)) { if (isArray(redundancyVideos)) {
for (const r of redundancyVideos) { for (const r of redundancyVideos) {
if ([ 'most-views', 'trending', 'recently-added' ].indexOf(r.strategy) === -1) { if ([ 'most-views', 'trending', 'recently-added' ].indexOf(r.strategy) === -1) {
@ -75,6 +75,7 @@ function checkMissedConfig () {
'cache.previews.size', 'admin.email', 'cache.previews.size', 'admin.email',
'signup.enabled', 'signup.limit', 'signup.requires_email_verification', 'signup.enabled', 'signup.limit', 'signup.requires_email_verification',
'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist', 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist',
'redundancy.videos.strategies',
'transcoding.enabled', 'transcoding.threads', 'transcoding.enabled', 'transcoding.threads',
'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.http.enabled', 'import.videos.torrent.enabled',
'trending.videos.interval_days', 'trending.videos.interval_days',

View File

@ -212,7 +212,9 @@ const CONFIG = {
} }
}, },
REDUNDANCY: { REDUNDANCY: {
VIDEOS: buildVideosRedundancy(config.get<any[]>('redundancy.videos')) VIDEOS: {
STRATEGIES: buildVideosRedundancy(config.get<any[]>('redundancy.videos.strategies'))
}
}, },
ADMIN: { ADMIN: {
get EMAIL () { return config.get<string>('admin.email') } get EMAIL () { return config.get<string>('admin.email') }

View File

@ -78,6 +78,9 @@ async function getOrCreateActorAndServerAndModel (
created = true created = true
} }
if (actor.Account) actor.Account.Actor = actor
if (actor.VideoChannel) actor.VideoChannel.Actor = actor
const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType) const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType)
if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.') if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.')

View File

@ -1,10 +1,9 @@
import { AbstractScheduler } from './abstract-scheduler' import { AbstractScheduler } from './abstract-scheduler'
import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers' import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers'
import { logger } from '../../helpers/logger' import { logger } from '../../helpers/logger'
import { RecentlyAddedStrategy, VideoRedundancyStrategy, VideosRedundancy } from '../../../shared/models/redundancy' import { VideoRedundancyStrategy, VideosRedundancy } from '../../../shared/models/redundancy'
import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
import { VideoFileModel } from '../../models/video/video-file' import { VideoFileModel } from '../../models/video/video-file'
import { sortBy } from 'lodash'
import { downloadWebTorrentVideo } from '../../helpers/webtorrent' import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
import { join } from 'path' import { join } from 'path'
import { rename } from 'fs-extra' import { rename } from 'fs-extra'
@ -12,7 +11,6 @@ import { getServerActor } from '../../helpers/utils'
import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
import { VideoModel } from '../../models/video/video' import { VideoModel } from '../../models/video/video'
import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' import { getVideoCacheFileActivityPubUrl } from '../activitypub/url'
import { removeVideoRedundancy } from '../redundancy'
import { isTestInstance } from '../../helpers/core-utils' import { isTestInstance } from '../../helpers/core-utils'
export class VideosRedundancyScheduler extends AbstractScheduler { export class VideosRedundancyScheduler extends AbstractScheduler {
@ -31,7 +29,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
this.executing = true this.executing = true
for (const obj of CONFIG.REDUNDANCY.VIDEOS) { for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
try { try {
const videoToDuplicate = await this.findVideoToDuplicate(obj) const videoToDuplicate = await this.findVideoToDuplicate(obj)
if (!videoToDuplicate) continue if (!videoToDuplicate) continue

View File

@ -48,11 +48,10 @@ export class TagModel extends Model<TagModel> {
}, },
defaults: { defaults: {
name: tag name: tag
} },
transaction
} }
if (transaction) query['transaction'] = transaction
const promise = TagModel.findOrCreate(query) const promise = TagModel.findOrCreate(query)
.then(([ tagInstance ]) => tagInstance) .then(([ tagInstance ]) => tagInstance)
tasks.push(promise) tasks.push(promise)