Log the details of background update failures (#16212)

pull/16220/head
David Robertson 2023-09-01 12:41:56 +01:00 committed by GitHub
parent ed5e8a77ca
commit 6525fd65ee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 26 additions and 3 deletions

1
changelog.d/16212.misc Normal file
View File

@ -0,0 +1 @@
Log the details of background update failures.

View File

@ -405,14 +405,14 @@ class BackgroundUpdater:
try: try:
result = await self.do_next_background_update(sleep) result = await self.do_next_background_update(sleep)
back_to_back_failures = 0 back_to_back_failures = 0
except Exception: except Exception as e:
logger.exception("Error doing update: %s", e)
back_to_back_failures += 1 back_to_back_failures += 1
if back_to_back_failures >= 5: if back_to_back_failures >= 5:
self._aborted = True self._aborted = True
raise RuntimeError( raise RuntimeError(
"5 back-to-back background update failures; aborting." "5 back-to-back background update failures; aborting."
) )
logger.exception("Error doing update")
else: else:
if result: if result:
logger.info( logger.info(

View File

@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging
from unittest.mock import AsyncMock, Mock from unittest.mock import AsyncMock, Mock
import yaml import yaml
@ -330,6 +330,28 @@ class BackgroundUpdateTestCase(unittest.HomeserverTestCase):
self.update_handler.side_effect = update_short self.update_handler.side_effect = update_short
self.get_success(self.updates.do_next_background_update(False)) self.get_success(self.updates.do_next_background_update(False))
def test_failed_update_logs_exception_details(self) -> None:
needle = "RUH ROH RAGGY"
def failing_update(progress: JsonDict, count: int) -> int:
raise Exception(needle)
self.update_handler.side_effect = failing_update
self.update_handler.reset_mock()
self.get_success(
self.store.db_pool.simple_insert(
"background_updates",
values={"update_name": "test_update", "progress_json": "{}"},
)
)
with self.assertLogs(level=logging.ERROR) as logs:
# Expect a back-to-back RuntimeError to be raised
self.get_failure(self.updates.run_background_updates(False), RuntimeError)
self.assertTrue(any(needle in log for log in logs.output), logs.output)
class BackgroundUpdateControllerTestCase(unittest.HomeserverTestCase): class BackgroundUpdateControllerTestCase(unittest.HomeserverTestCase):
def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: