Log the details of background update failures (#16212)
parent
ed5e8a77ca
commit
6525fd65ee
|
@ -0,0 +1 @@
|
|||
Log the details of background update failures.
|
|
@ -405,14 +405,14 @@ class BackgroundUpdater:
|
|||
try:
|
||||
result = await self.do_next_background_update(sleep)
|
||||
back_to_back_failures = 0
|
||||
except Exception:
|
||||
except Exception as e:
|
||||
logger.exception("Error doing update: %s", e)
|
||||
back_to_back_failures += 1
|
||||
if back_to_back_failures >= 5:
|
||||
self._aborted = True
|
||||
raise RuntimeError(
|
||||
"5 back-to-back background update failures; aborting."
|
||||
)
|
||||
logger.exception("Error doing update")
|
||||
else:
|
||||
if result:
|
||||
logger.info(
|
||||
|
|
|
@ -11,7 +11,7 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import yaml
|
||||
|
@ -330,6 +330,28 @@ class BackgroundUpdateTestCase(unittest.HomeserverTestCase):
|
|||
self.update_handler.side_effect = update_short
|
||||
self.get_success(self.updates.do_next_background_update(False))
|
||||
|
||||
def test_failed_update_logs_exception_details(self) -> None:
|
||||
needle = "RUH ROH RAGGY"
|
||||
|
||||
def failing_update(progress: JsonDict, count: int) -> int:
|
||||
raise Exception(needle)
|
||||
|
||||
self.update_handler.side_effect = failing_update
|
||||
self.update_handler.reset_mock()
|
||||
|
||||
self.get_success(
|
||||
self.store.db_pool.simple_insert(
|
||||
"background_updates",
|
||||
values={"update_name": "test_update", "progress_json": "{}"},
|
||||
)
|
||||
)
|
||||
|
||||
with self.assertLogs(level=logging.ERROR) as logs:
|
||||
# Expect a back-to-back RuntimeError to be raised
|
||||
self.get_failure(self.updates.run_background_updates(False), RuntimeError)
|
||||
|
||||
self.assertTrue(any(needle in log for log in logs.output), logs.output)
|
||||
|
||||
|
||||
class BackgroundUpdateControllerTestCase(unittest.HomeserverTestCase):
|
||||
def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
|
||||
|
|
Loading…
Reference in New Issue