From 3cc852d339ad1cdcb0a435c76b44182bdb81dfe9 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Mon, 9 Oct 2017 17:44:42 +0100
Subject: [PATCH 1/2] Fancy logformatter to format exceptions better

This is a bit of an experimental change at this point; the idea is to see if it
helps us track down where our stack overflows are coming from by logging the
stack when the exception was caught and turned into a Failure. (We'll also need
https://github.com/richvdh/twisted/commit/edf27044200e74680ea67c525768e36dc9d9af2b).

If we deploy this, we'll be able to enable it via the log config yaml.
---
 synapse/util/logformatter.py | 43 ++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100644 synapse/util/logformatter.py

diff --git a/synapse/util/logformatter.py b/synapse/util/logformatter.py
new file mode 100644
index 0000000000..60504162e9
--- /dev/null
+++ b/synapse/util/logformatter.py
@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import StringIO
+import logging
+import traceback
+
+
+class LogFormatter(logging.Formatter):
+    """Log formatter which gives more detail for exceptions
+
+    This is the same as the standard log formatter, except that when logging
+    exceptions [typically via log.foo("msg", exc_info=1)], it prints the
+    sequence that led up to the point at which the exception was caught.
+    (Normally only stack frames between the point the exception was raised and
+    where it was caught are logged).
+    """
+    def __init__(self, *args, **kwargs):
+        super(LogFormatter, self).__init__(*args, **kwargs)
+
+    def formatException(self, ei):
+        sio = StringIO.StringIO()
+        sio.write("Capture point (most recent call last):\n")
+        traceback.print_stack(ei[2].tb_frame.f_back, None, sio)
+        traceback.print_exception(ei[0], ei[1], ei[2], None, sio)
+        s = sio.getvalue()
+        sio.close()
+        if s[-1:] == "\n":
+            s = s[:-1]
+        return s

From f30c4ed2bc2255dc7182bd026fb6437afec735a5 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <richard@matrix.org>
Date: Wed, 11 Oct 2017 17:26:17 +0100
Subject: [PATCH 2/2] logformatter: fix AttributeError

make sure we have the relevant fields before we try to log them.
---
 synapse/util/logformatter.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/synapse/util/logformatter.py b/synapse/util/logformatter.py
index 60504162e9..cdbc4bffd7 100644
--- a/synapse/util/logformatter.py
+++ b/synapse/util/logformatter.py
@@ -33,9 +33,17 @@ class LogFormatter(logging.Formatter):
 
     def formatException(self, ei):
         sio = StringIO.StringIO()
-        sio.write("Capture point (most recent call last):\n")
-        traceback.print_stack(ei[2].tb_frame.f_back, None, sio)
-        traceback.print_exception(ei[0], ei[1], ei[2], None, sio)
+        (typ, val, tb) = ei
+
+        # log the stack above the exception capture point if possible, but
+        # check that we actually have an f_back attribute to work around
+        # https://twistedmatrix.com/trac/ticket/9305
+
+        if tb and hasattr(tb.tb_frame, 'f_back'):
+            sio.write("Capture point (most recent call last):\n")
+            traceback.print_stack(tb.tb_frame.f_back, None, sio)
+
+        traceback.print_exception(typ, val, tb, None, sio)
         s = sio.getvalue()
         sio.close()
         if s[-1:] == "\n":