diff --git a/server/helpers/promise-cache.ts b/server/helpers/promise-cache.ts
index 07e8a9962..303bab976 100644
--- a/server/helpers/promise-cache.ts
+++ b/server/helpers/promise-cache.ts
@@ -1,4 +1,4 @@
-export class PromiseCache <A, R> {
+export class CachePromiseFactory <A, R> {
   private readonly running = new Map<string, Promise<R>>()
 
   constructor (
@@ -8,14 +8,32 @@ export class PromiseCache <A, R> {
   }
 
   run (arg: A) {
+    return this.runWithContext(null, arg)
+  }
+
+  runWithContext (ctx: any, arg: A) {
     const key = this.keyBuilder(arg)
 
     if (this.running.has(key)) return this.running.get(key)
 
-    const p = this.fn(arg)
+    const p = this.fn.apply(ctx || this, [ arg ])
 
     this.running.set(key, p)
 
     return p.finally(() => this.running.delete(key))
   }
 }
+
+export function CachePromise (options: {
+  keyBuilder: (...args: any[]) => string
+}) {
+  return function (_target, _key, descriptor: PropertyDescriptor) {
+    const promiseCache = new CachePromiseFactory(descriptor.value, options.keyBuilder)
+
+    descriptor.value = function () {
+      if (arguments.length !== 1) throw new Error('Cache promise only support methods with 1 argument')
+
+      return promiseCache.runWithContext(this, arguments[0])
+    }
+  }
+}
diff --git a/server/lib/activitypub/actors/refresh.ts b/server/lib/activitypub/actors/refresh.ts
index 6d8428d66..d15cb5e90 100644
--- a/server/lib/activitypub/actors/refresh.ts
+++ b/server/lib/activitypub/actors/refresh.ts
@@ -1,5 +1,5 @@
 import { logger, loggerTagsFactory } from '@server/helpers/logger'
-import { PromiseCache } from '@server/helpers/promise-cache'
+import { CachePromiseFactory } from '@server/helpers/promise-cache'
 import { PeerTubeRequestError } from '@server/helpers/requests'
 import { ActorLoadByUrlType } from '@server/lib/model-loaders'
 import { ActorModel } from '@server/models/actor/actor'
@@ -16,7 +16,7 @@ type RefreshOptions <T> = {
   fetchedType: ActorLoadByUrlType
 }
 
-const promiseCache = new PromiseCache(doRefresh, (options: RefreshOptions<MActorFull | MActorAccountChannelId>) => options.actor.url)
+const promiseCache = new CachePromiseFactory(doRefresh, (options: RefreshOptions<MActorFull | MActorAccountChannelId>) => options.actor.url)
 
 function refreshActorIfNeeded <T extends MActorFull | MActorAccountChannelId> (options: RefreshOptions<T>): RefreshResult <T> {
   const actorArg = options.actor
diff --git a/server/lib/files-cache/shared/abstract-permanent-file-cache.ts b/server/lib/files-cache/shared/abstract-permanent-file-cache.ts
index 297461035..f990e9872 100644
--- a/server/lib/files-cache/shared/abstract-permanent-file-cache.ts
+++ b/server/lib/files-cache/shared/abstract-permanent-file-cache.ts
@@ -1,10 +1,11 @@
 import express from 'express'
 import { LRUCache } from 'lru-cache'
+import { Model } from 'sequelize'
 import { logger } from '@server/helpers/logger'
+import { CachePromise } from '@server/helpers/promise-cache'
 import { LRU_CACHE, STATIC_MAX_AGE } from '@server/initializers/constants'
 import { downloadImageFromWorker } from '@server/lib/worker/parent-process'
 import { HttpStatusCode } from '@shared/models'
-import { Model } from 'sequelize'
 
 type ImageModel = {
   fileUrl: string
@@ -41,21 +42,9 @@ export abstract class AbstractPermanentFileCache <M extends ImageModel> {
       return res.sendFile(this.filenameToPathUnsafeCache.get(filename), { maxAge: STATIC_MAX_AGE.SERVER })
     }
 
-    const image = await this.loadModel(filename)
+    const image = await this.lazyLoadIfNeeded(filename)
     if (!image) return res.status(HttpStatusCode.NOT_FOUND_404).end()
 
-    if (image.onDisk === false) {
-      if (!image.fileUrl) return res.status(HttpStatusCode.NOT_FOUND_404).end()
-
-      try {
-        await this.downloadRemoteFile(image)
-      } catch (err) {
-        logger.warn('Cannot process remote image %s.', image.fileUrl, { err })
-
-        return res.status(HttpStatusCode.NOT_FOUND_404).end()
-      }
-    }
-
     const path = image.getPath()
     this.filenameToPathUnsafeCache.set(filename, path)
 
@@ -66,6 +55,28 @@ export abstract class AbstractPermanentFileCache <M extends ImageModel> {
     })
   }
 
+  @CachePromise({
+    keyBuilder: filename => filename
+  })
+  private async lazyLoadIfNeeded (filename: string) {
+    const image = await this.loadModel(filename)
+    if (!image) return undefined
+
+    if (image.onDisk === false) {
+      if (!image.fileUrl) return undefined
+
+      try {
+        await this.downloadRemoteFile(image)
+      } catch (err) {
+        logger.warn('Cannot process remote image %s.', image.fileUrl, { err })
+
+        return undefined
+      }
+    }
+
+    return image
+  }
+
   async downloadRemoteFile (image: M) {
     logger.info('Download remote image %s lazily.', image.fileUrl)