Log the details of background update failures (#16212)
parent
ed5e8a77ca
commit
6525fd65ee
|
@ -0,0 +1 @@
|
||||||
|
Log the details of background update failures.
|
|
@ -405,14 +405,14 @@ class BackgroundUpdater:
|
||||||
try:
|
try:
|
||||||
result = await self.do_next_background_update(sleep)
|
result = await self.do_next_background_update(sleep)
|
||||||
back_to_back_failures = 0
|
back_to_back_failures = 0
|
||||||
except Exception:
|
except Exception as e:
|
||||||
|
logger.exception("Error doing update: %s", e)
|
||||||
back_to_back_failures += 1
|
back_to_back_failures += 1
|
||||||
if back_to_back_failures >= 5:
|
if back_to_back_failures >= 5:
|
||||||
self._aborted = True
|
self._aborted = True
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
"5 back-to-back background update failures; aborting."
|
"5 back-to-back background update failures; aborting."
|
||||||
)
|
)
|
||||||
logger.exception("Error doing update")
|
|
||||||
else:
|
else:
|
||||||
if result:
|
if result:
|
||||||
logger.info(
|
logger.info(
|
||||||
|
|
|
@ -11,7 +11,7 @@
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
import logging
|
||||||
from unittest.mock import AsyncMock, Mock
|
from unittest.mock import AsyncMock, Mock
|
||||||
|
|
||||||
import yaml
|
import yaml
|
||||||
|
@ -330,6 +330,28 @@ class BackgroundUpdateTestCase(unittest.HomeserverTestCase):
|
||||||
self.update_handler.side_effect = update_short
|
self.update_handler.side_effect = update_short
|
||||||
self.get_success(self.updates.do_next_background_update(False))
|
self.get_success(self.updates.do_next_background_update(False))
|
||||||
|
|
||||||
|
def test_failed_update_logs_exception_details(self) -> None:
|
||||||
|
needle = "RUH ROH RAGGY"
|
||||||
|
|
||||||
|
def failing_update(progress: JsonDict, count: int) -> int:
|
||||||
|
raise Exception(needle)
|
||||||
|
|
||||||
|
self.update_handler.side_effect = failing_update
|
||||||
|
self.update_handler.reset_mock()
|
||||||
|
|
||||||
|
self.get_success(
|
||||||
|
self.store.db_pool.simple_insert(
|
||||||
|
"background_updates",
|
||||||
|
values={"update_name": "test_update", "progress_json": "{}"},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
with self.assertLogs(level=logging.ERROR) as logs:
|
||||||
|
# Expect a back-to-back RuntimeError to be raised
|
||||||
|
self.get_failure(self.updates.run_background_updates(False), RuntimeError)
|
||||||
|
|
||||||
|
self.assertTrue(any(needle in log for log in logs.output), logs.output)
|
||||||
|
|
||||||
|
|
||||||
class BackgroundUpdateControllerTestCase(unittest.HomeserverTestCase):
|
class BackgroundUpdateControllerTestCase(unittest.HomeserverTestCase):
|
||||||
def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
|
def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
|
||||||
|
|
Loading…
Reference in New Issue