diff --git a/src/components/views/dialogs/NewSessionReviewDialog.js b/src/components/views/dialogs/NewSessionReviewDialog.js
index 0019e0644f..125811115f 100644
--- a/src/components/views/dialogs/NewSessionReviewDialog.js
+++ b/src/components/views/dialogs/NewSessionReviewDialog.js
@@ -23,6 +23,7 @@ import VerificationRequestDialog from './VerificationRequestDialog';
 import BaseDialog from './BaseDialog';
 import DialogButtons from '../elements/DialogButtons';
 import {MatrixClientPeg} from "../../../MatrixClientPeg";
+import * as sdk from '../../../index';
 
 @replaceableComponent("views.dialogs.NewSessionReviewDialog")
 export default class NewSessionReviewDialog extends React.PureComponent {
@@ -33,7 +34,24 @@ export default class NewSessionReviewDialog extends React.PureComponent {
     }
 
     onCancelClick = () => {
-        this.props.onFinished(false);
+        const ErrorDialog = sdk.getComponent("dialogs.ErrorDialog");
+        Modal.createTrackedDialog("Verification failed", "insecure", ErrorDialog, {
+            headerImage: require("../../../../res/img/e2e/warning.svg"),
+            title: _t("Your account is not secure"),
+            description: <div>
+                {_t("One of the following may be compromised:")}
+                <ul>
+                    <li>{_t("Your password")}</li>
+                    <li>{_t("Your homeserver")}</li>
+                    <li>{_t("This session, or the other session")}</li>
+                    <li>{_t("The internet connection either session is using")}</li>
+                </ul>
+                <div>
+                    {_t("We recommend you change your password and recovery key in Settings immediately")}
+                </div>
+            </div>,
+            onFinished: () => this.props.onFinished(false),
+        });
     }
 
     onContinueClick = async () => {
diff --git a/src/components/views/toasts/UnverifiedSessionToast.js b/src/components/views/toasts/UnverifiedSessionToast.js
index 15cf18f47d..cb0cadcdc8 100644
--- a/src/components/views/toasts/UnverifiedSessionToast.js
+++ b/src/components/views/toasts/UnverifiedSessionToast.js
@@ -42,6 +42,12 @@ export default class UnverifiedSessionToast extends React.PureComponent {
         Modal.createTrackedDialog('New Session Review', 'Starting dialog', NewSessionReviewDialog, {
             userId: MatrixClientPeg.get().getUserId(),
             device,
+            onFinished: (r) => {
+                if (!r) {
+                    /* This'll come back false if the user clicks "this wasn't me" and saw a warning dialog */
+                    this._onLaterClick();
+                }
+            },
         }, null, /* priority = */ false, /* static = */ true);
     };
 
diff --git a/src/i18n/strings/en_EN.json b/src/i18n/strings/en_EN.json
index 1d030f5118..f702da6b4c 100644
--- a/src/i18n/strings/en_EN.json
+++ b/src/i18n/strings/en_EN.json
@@ -1572,6 +1572,11 @@
     "Are you sure you want to sign out?": "Are you sure you want to sign out?",
     "Your homeserver doesn't seem to support this feature.": "Your homeserver doesn't seem to support this feature.",
     "Message edits": "Message edits",
+    "Your account is not secure": "Your account is not secure",
+    "Your password": "Your password",
+    "This session, or the other session": "This session, or the other session",
+    "The internet connection either session is using": "The internet connection either session is using",
+    "We recommend you change your password and recovery key in Settings immediately": "We recommend you change your password and recovery key in Settings immediately",
     "New session": "New session",
     "Use this session to verify your new one, granting it access to encrypted messages:": "Use this session to verify your new one, granting it access to encrypted messages:",
     "If you didn’t sign in to this session, your account may be compromised.": "If you didn’t sign in to this session, your account may be compromised.",