diff --git a/synapse/config/cache.py b/synapse/config/cache.py
index 1d65c64e1a..d6dadc5fde 100644
--- a/synapse/config/cache.py
+++ b/synapse/config/cache.py
@@ -17,15 +17,11 @@ import os
 from collections import defaultdict
 from typing import DefaultDict
 
-from twisted.logger import Logger
-
 from ._base import Config, ConfigError
 
-log = Logger()
-
 _CACHES = {}
-DEFAULT_CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.5))
-
+_CACHE_PREFIX = "SYNAPSE_CACHE_FACTOR"
+DEFAULT_CACHE_SIZE_FACTOR = float(os.environ.get(_CACHE_PREFIX, 0.5))
 
 _DEFAULT_CONFIG = """\
 # Cache configuration
@@ -51,6 +47,7 @@ def add_resizable_cache(cache_name, cache_resize_callback):
 
 class CacheConfig(Config):
     section = "caches"
+    _environ = os.environ
 
     def read_config(self, config, **kwargs):
         self.event_cache_size = self.parse_size(config.get("event_cache_size", "10K"))
@@ -68,10 +65,20 @@ class CacheConfig(Config):
         # Set the global one so that it's reflected in new caches
         DEFAULT_CACHE_SIZE_FACTOR = self.global_factor
 
-        individual_factors = cache_config.get("per_cache_factors", {}) or {}
-        if not isinstance(individual_factors, dict):
+        # Load cache factors from the environment, but override them with the
+        # ones in the config file if they exist
+        individual_factors = {
+            key[len(_CACHE_PREFIX) + 1 :].lower(): float(val)
+            for key, val in self._environ.items()
+            if key.startswith(_CACHE_PREFIX + "_")
+        }
+
+        individual_factors_config = cache_config.get("per_cache_factors", {}) or {}
+        if not isinstance(individual_factors_config, dict):
             raise ConfigError("caches.per_cache_factors must be a dictionary")
 
+        individual_factors.update(individual_factors_config)
+
         self.cache_factors = defaultdict(
             lambda: self.global_factor
         )  # type: DefaultDict[str, float]
@@ -82,22 +89,3 @@ class CacheConfig(Config):
                     "caches.per_cache_factors.%s must be a number" % (cache.lower(),)
                 )
             self.cache_factors[cache.lower()] = factor
-
-    def resize_caches(self):
-        for cache_name, cache_resize_callback in _CACHES.items():
-            cache_factor = self.cache_factors[cache_name]
-            log.debug(
-                "Setting cache factor for {cache_name} to {new_cache_factor}",
-                cache_name=cache_name,
-                new_cache_factor=cache_factor,
-            )
-            changed = cache_resize_callback(cache_factor)
-            if changed:
-                log.info(
-                    "Cache factor for {cache_name} set to {new_cache_factor}",
-                    cache_name=cache_name,
-                    new_cache_factor=cache_factor,
-                )
-
-    def get_factor_for(self, cache_name):
-        return self.cache_factors[cache_name.lower()]
diff --git a/tests/config/test_cache.py b/tests/config/test_cache.py
new file mode 100644
index 0000000000..3b3a49c779
--- /dev/null
+++ b/tests/config/test_cache.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+# Copyright 2020 Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from synapse.config._base import Config, RootConfig
+from synapse.config.cache import CacheConfig
+
+from tests.unittest import TestCase
+
+
+class FakeServer(Config):
+    section = "server"
+
+
+class TestConfig(RootConfig):
+    config_classes = [FakeServer, CacheConfig]
+
+
+class CacheConfigTests(TestCase):
+    def test_individual_caches_from_environ(self):
+        """
+        Individual cache factors will be loaded from the environment.
+        """
+        config = {}
+        t = TestConfig()
+        t.caches._environ = {
+            "SYNAPSE_CACHE_FACTOR_SOMETHING_OR_OTHER": "2",
+            "SYNAPSE_NOT_CACHE": "BLAH",
+        }
+        t.read_config(config, config_dir_path="", data_dir_path="")
+
+        self.assertEqual(dict(t.caches.cache_factors), {"something_or_other": 2.0})
+
+    def test_config_overrides_environ(self):
+        """
+        Individual cache factors defined in config will take precedence over
+        ones in the environment.
+        """
+        config = {"caches": {"per_cache_factors": {"foo": 2, "bar": 3}}}
+        t = TestConfig()
+        t.caches._environ = {
+            "SYNAPSE_CACHE_FACTOR_SOMETHING_OR_OTHER": "2",
+            "SYNAPSE_CACHE_FACTOR_FOO": 1,
+        }
+        t.read_config(config, config_dir_path="", data_dir_path="")
+
+        self.assertEqual(
+            dict(t.caches.cache_factors),
+            {"foo": 2.0, "bar": 3.0, "something_or_other": 2.0},
+        )