diff --git a/packages/tests/src/api/server/config.ts b/packages/tests/src/api/server/config.ts
index f867c9ee8..85d40e232 100644
--- a/packages/tests/src/api/server/config.ts
+++ b/packages/tests/src/api/server/config.ts
@@ -7,7 +7,9 @@ import {
   cleanupTests,
   createSingleServer,
   killallServers,
+  makeActivityPubGetRequest,
   makeGetRequest,
+  makeRawRequest,
   PeerTubeServer,
   setAccessTokensToServers
 } from '@peertube/peertube-server-commands'
@@ -747,7 +749,18 @@ describe('Test config', function () {
         }
       })
 
-      it('Should remove instance banner', async function () {
+      it('Should have the avatars in the AP representation of the instance', async function () {
+        const res = await makeActivityPubGetRequest(server.url, '/accounts/peertube')
+        const object = res.body
+
+        expect(object.icon).to.have.lengthOf(4)
+
+        for (const icon of object.icon) {
+          await makeRawRequest({ url: icon.url, expectedStatus: HttpStatusCode.OK_200 })
+        }
+      })
+
+      it('Should remove instance avatar', async function () {
         await server.config.deleteInstanceImage({ type: ActorImageType.AVATAR })
 
         const { avatars } = await checkAndGetServerImages()
@@ -755,6 +768,13 @@ describe('Test config', function () {
 
         await testFileExistsOrNot(server, 'avatars', basename(avatarPath), false)
       })
+
+      it('Should not have the avatars anymore in the AP representation of the instance', async function () {
+        const res = await makeActivityPubGetRequest(server.url, '/accounts/peertube')
+        const object = res.body
+
+        expect(object.icon).to.not.exist
+      })
     })
   })
 
diff --git a/server/core/controllers/api/config.ts b/server/core/controllers/api/config.ts
index 3d9d9959b..a04524714 100644
--- a/server/core/controllers/api/config.ts
+++ b/server/core/controllers/api/config.ts
@@ -23,6 +23,7 @@ import { MIMETYPES } from '@server/initializers/constants.js'
 import { deleteLocalActorImageFile, updateLocalActorImageFiles } from '@server/lib/local-actor.js'
 import { getServerActor } from '@server/models/application/application.js'
 import { ActorImageModel } from '@server/models/actor/actor-image.js'
+import { ModelCache } from '@server/models/shared/model-cache.js'
 
 const configRouter = express.Router()
 
@@ -193,6 +194,7 @@ function updateInstanceImageFactory (imageType: ActorImageType_Type) {
     })
 
     ClientHtml.invalidateCache()
+    ModelCache.Instance.clearCache('server-account')
 
     return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
   }
@@ -203,6 +205,7 @@ function deleteInstanceImageFactory (imageType: ActorImageType_Type) {
     await deleteLocalActorImageFile((await getServerActorWithUpdatedImages(imageType)).Account, imageType)
 
     ClientHtml.invalidateCache()
+    ModelCache.Instance.clearCache('server-account')
 
     return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
   }
diff --git a/server/core/models/account/account.ts b/server/core/models/account/account.ts
index 99b151b44..0611fb25f 100644
--- a/server/core/models/account/account.ts
+++ b/server/core/models/account/account.ts
@@ -301,7 +301,7 @@ export class AccountModel extends SequelizeModel<AccountModel> {
     }
 
     return ModelCache.Instance.doCache({
-      cacheType: 'local-account-name',
+      cacheType: 'server-account',
       key: name,
       fun,
       // The server actor never change, so we can easily cache it
diff --git a/server/core/models/shared/model-cache.ts b/server/core/models/shared/model-cache.ts
index 273d8e608..54fad6b21 100644
--- a/server/core/models/shared/model-cache.ts
+++ b/server/core/models/shared/model-cache.ts
@@ -2,7 +2,7 @@ import { Model } from 'sequelize-typescript'
 import { logger } from '@server/helpers/logger.js'
 
 type ModelCacheType =
-  'local-account-name'
+  'server-account'
   | 'local-actor-name'
   | 'local-actor-url'
   | 'load-video-immutable-id'
@@ -16,7 +16,7 @@ class ModelCache {
   private static instance: ModelCache
 
   private readonly localCache: { [id in ModelCacheType]: Map<string, any> } = {
-    'local-account-name': new Map(),
+    'server-account': new Map(),
     'local-actor-name': new Map(),
     'local-actor-url': new Map(),
     'load-video-immutable-id': new Map(),
@@ -83,6 +83,10 @@ class ModelCache {
 
     map.delete(modelId)
   }
+
+  clearCache (cacheType: ModelCacheType) {
+    this.localCache[cacheType] = new Map()
+  }
 }
 
 export {