diff --git a/changelog.d/4616.misc b/changelog.d/4616.misc
new file mode 100644
index 0000000000..ee79e208e3
--- /dev/null
+++ b/changelog.d/4616.misc
@@ -0,0 +1 @@
+Fail cleanly if listener config lacks a 'port'
diff --git a/synapse/config/server.py b/synapse/config/server.py
index ce0458195c..eed9d7c81e 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -129,6 +129,11 @@ class ServerConfig(Config):
         self.listeners = config.get("listeners", [])
 
         for listener in self.listeners:
+            if not isinstance(listener.get("port", None), int):
+                raise ConfigError(
+                    "Listener configuration is lacking a valid 'port' option"
+                )
+
             bind_address = listener.pop("bind_address", None)
             bind_addresses = listener.setdefault("bind_addresses", [])