diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index 06250d2d96..5abab80e67 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -136,7 +136,7 @@ class BulkPushRuleEvaluator:
                 )
                 if matches:
                     actions = [x for x in rule['actions'] if x != 'dont_notify']
-                    if actions:
+                    if actions and 'notify' in actions:
                         actions_by_user[uid] = actions
                     break
         defer.returnValue(actions_by_user)