diff --git a/changelog.d/15425.bugfix b/changelog.d/15425.bugfix
new file mode 100644
index 0000000000..fd104a63b3
--- /dev/null
+++ b/changelog.d/15425.bugfix
@@ -0,0 +1 @@
+Synapse now correctly fails to start if the config option `app_service_config_files` is not a list.
\ No newline at end of file
diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py
index 00182090b2..fd89960e72 100644
--- a/synapse/config/appservice.py
+++ b/synapse/config/appservice.py
@@ -33,6 +33,16 @@ class AppServiceConfig(Config):
 
     def read_config(self, config: JsonDict, **kwargs: Any) -> None:
         self.app_service_config_files = config.get("app_service_config_files", [])
+        if not isinstance(self.app_service_config_files, list) or not all(
+            type(x) is str for x in self.app_service_config_files
+        ):
+            # type-ignore: this function gets arbitrary json value; we do use this path.
+            raise ConfigError(
+                "Expected '%s' to be a list of AS config files:"
+                % (self.app_service_config_files),
+                "app_service_config_files",
+            )
+
         self.track_appservice_user_ips = config.get("track_appservice_user_ips", False)
 
 
@@ -40,10 +50,6 @@ def load_appservices(
     hostname: str, config_files: List[str]
 ) -> List[ApplicationService]:
     """Returns a list of Application Services from the config files."""
-    if not isinstance(config_files, list):
-        # type-ignore: this function gets arbitrary json value; we do use this path.
-        logger.warning("Expected %s to be a list of AS config files.", config_files)  # type: ignore[unreachable]
-        return []
 
     # Dicts of value -> filename
     seen_as_tokens: Dict[str, str] = {}
diff --git a/tests/config/test_appservice.py b/tests/config/test_appservice.py
new file mode 100644
index 0000000000..d2d1a40dfc
--- /dev/null
+++ b/tests/config/test_appservice.py
@@ -0,0 +1,40 @@
+# Copyright 2023 Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse.config.appservice import AppServiceConfig, ConfigError
+
+from tests.unittest import TestCase
+
+
+class AppServiceConfigTest(TestCase):
+    def test_invalid_app_service_config_files(self) -> None:
+        for invalid_value in [
+            "foobar",
+            1,
+            None,
+            True,
+            False,
+            {},
+            ["foo", "bar", False],
+        ]:
+            with self.assertRaises(ConfigError):
+                AppServiceConfig().read_config(
+                    {"app_service_config_files": invalid_value}
+                )
+
+    def test_valid_app_service_config_files(self) -> None:
+        AppServiceConfig().read_config({"app_service_config_files": []})
+        AppServiceConfig().read_config(
+            {"app_service_config_files": ["/not/a/real/path", "/not/a/real/path/2"]}
+        )