diff --git a/.cursor/context/timelord.md b/.cursor/context/timelord.md index 1772a7d079..0c1efab834 100644 --- a/.cursor/context/timelord.md +++ b/.cursor/context/timelord.md @@ -77,7 +77,7 @@ Both paths intentionally choose a random target VDF field first, then fall back - `timelord.lock` is the module's main consistency boundary. API handlers, VDF-client mapping, iteration submission, proof appending, compact queue mutation, and reset operations rely on this lock around multi-field mutations. - `_do_process_communication()` performs network reads outside many lock sections but appends validated proofs and failure records under the lock. Be careful not to hold the lock across long VDF-client reads. -- `_handle_failures()` prioritizes liveness. A current-generation VDF-client failure resets to EOS-only work; prolonged inactivity backs off the restart threshold up to a source-defined cap and resets all chains. +- `_handle_failures()` splits VDF-client crashes (`_handle_vdf_failures`, EOS-only reset) from the inactivity watchdog (`_maybe_reset_for_inactivity`). The watchdog backs off `max_allowed_inactivity_time` up to 1800s. `0` disables it (`math.inf` threshold). `restore_inactivity_timeout()` re-reads the config value; `new_peak_timelord` calls that instead of assigning the field. - Shutdown closes the executor trigger file, cancels communication/main-loop tasks, sends stop signals to VDF clients, closes idle and assigned writers, and closes the TCP server. - `timelord_launcher.py` is a separate process manager for `chiavdf`'s `vdf_client` binary. It resolves the configured host, restarts clients until stopped, suppresses early stderr noise, and kills all active subprocesses on shutdown. diff --git a/chia/_tests/timelord/test_timelord.py b/chia/_tests/timelord/test_timelord.py index b47d98c44a..c3875879f4 100644 --- a/chia/_tests/timelord/test_timelord.py +++ b/chia/_tests/timelord/test_timelord.py @@ -2,6 +2,8 @@ from __future__ import annotations import asyncio import time +from pathlib import Path +from typing import Any from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -10,6 +12,7 @@ from chia_rs.sized_bytes import bytes32 from chia_rs.sized_ints import uint32, uint64 from chia._tests.conftest import ConsensusMode +from chia.consensus.default_constants import DEFAULT_CONSTANTS from chia.timelord import iters_from_block as iters_from_block_module from chia.timelord import timelord as timelord_module from chia.timelord.iters_from_block import iters_from_block @@ -181,3 +184,52 @@ class TestHandleClient: assert len(tl.free_clients) == 3 overflow_writer.close.assert_called_once() overflow_writer.wait_closed.assert_awaited_once() + + +def _minimal_timelord_config(**overrides: object) -> dict[str, Any]: + config: dict[str, Any] = {"vdf_clients": {"ip": ["127.0.0.1"]}} + config.update(overrides) + return config + + +def test_inactivity_timeout_defaults_to_60(tmp_path: Path) -> None: + tl = Timelord(tmp_path, _minimal_timelord_config(), DEFAULT_CONSTANTS) + assert tl.max_allowed_inactivity_time == 60 + + +def test_inactivity_timeout_reads_config(tmp_path: Path) -> None: + tl = Timelord(tmp_path, _minimal_timelord_config(max_allowed_inactivity_time=600), DEFAULT_CONSTANTS) + assert tl.max_allowed_inactivity_time == 600 + + +def test_restore_inactivity_timeout_rereads_config(tmp_path: Path) -> None: + tl = Timelord(tmp_path, _minimal_timelord_config(max_allowed_inactivity_time=600), DEFAULT_CONSTANTS) + tl.max_allowed_inactivity_time = 1200 + tl.restore_inactivity_timeout() + assert tl.max_allowed_inactivity_time == 600 + + +@pytest.mark.anyio +async def test_inactivity_reset_disabled_when_configured_zero(tmp_path: Path) -> None: + tl = Timelord(tmp_path, _minimal_timelord_config(max_allowed_inactivity_time=0), DEFAULT_CONSTANTS) + tl.last_active_time = time.time() - 10_000 + with patch.object(tl, "_reset_chains", new_callable=AsyncMock) as reset: + await tl._maybe_reset_for_inactivity() + reset.assert_not_called() + + +@pytest.mark.anyio +async def test_inactivity_reset_uses_configured_threshold(tmp_path: Path) -> None: + tl = Timelord(tmp_path, _minimal_timelord_config(max_allowed_inactivity_time=600), DEFAULT_CONSTANTS) + tl.vdf_failure_time = 0 + + tl.last_active_time = time.time() - 61 + with patch.object(tl, "_reset_chains", new_callable=AsyncMock) as reset: + await tl._maybe_reset_for_inactivity() + reset.assert_not_called() + + tl.last_active_time = time.time() - 601 + with patch.object(tl, "_reset_chains", new_callable=AsyncMock) as reset: + await tl._maybe_reset_for_inactivity() + reset.assert_awaited_once() + assert tl.max_allowed_inactivity_time == 1200 diff --git a/chia/timelord/timelord.py b/chia/timelord/timelord.py index 4a98fc65af..89e45372dd 100644 --- a/chia/timelord/timelord.py +++ b/chia/timelord/timelord.py @@ -5,6 +5,7 @@ import contextlib import dataclasses import io import logging +import math import os import random import tempfile @@ -148,7 +149,7 @@ class Timelord: self.bluebox_mode = self.config.get("sanitizer_mode", False) self.pending_bluebox_info: list[tuple[float, timelord_protocol.RequestCompactProofOfTime]] = [] self.last_active_time = time.time() - self.max_allowed_inactivity_time = 60 + self.restore_inactivity_timeout() self._executor_shutdown_tempfile: IO[bytes] | None = None self.bluebox_pool: ThreadPoolExecutor | None = None @@ -888,34 +889,44 @@ class Timelord: self.total_unfinished += 1 await self._reset_chains() + def restore_inactivity_timeout(self) -> None: + self.max_allowed_inactivity_time = int(self.config.get("max_allowed_inactivity_time", 60)) + async def _handle_failures(self) -> None: - if len(self.vdf_failures) > 0: - # This can happen if one of the VDF processes has an issue. In this case, we abort all other - # infusion points and signage points, and go straight to the end of slot, so we avoid potential - # issues with the number of iterations that failed. + await self._handle_vdf_failures() + await self._maybe_reset_for_inactivity() - failed_chain, proof_label = self.vdf_failures[0] - log.error( - f"Vdf clients failed {self.vdf_failures_count} times. Last failure: {failed_chain}, " - f"label {proof_label}, current: {self.num_resets}" - ) - if proof_label == self.num_resets: - await self._reset_chains(only_eos=True) - self.vdf_failure_time = time.time() - self.vdf_failures = [] + async def _handle_vdf_failures(self) -> None: + if len(self.vdf_failures) == 0: + return + # This can happen if one of the VDF processes has an issue. In this case, we abort all other + # infusion points and signage points, and go straight to the end of slot, so we avoid potential + # issues with the number of iterations that failed. + failed_chain, proof_label = self.vdf_failures[0] + log.error( + f"Vdf clients failed {self.vdf_failures_count} times. Last failure: {failed_chain}, " + f"label {proof_label}, current: {self.num_resets}" + ) + if proof_label == self.num_resets: + await self._reset_chains(only_eos=True) + self.vdf_failure_time = time.time() + self.vdf_failures = [] + async def _maybe_reset_for_inactivity(self) -> None: # If something goes wrong in the VDF client due to a failed thread, we might get stuck in a situation where we # are waiting for that client to finish. Usually other peers will finish the VDFs and reset us. In the case that # there are no other timelords, this reset should bring the timelord back to a running state. - if time.time() - self.vdf_failure_time < self.constants.SUB_SLOT_TIME_TARGET * 3: + # 0 disables the inactivity reset so a slow EOS (up to ~SUB_SLOT_TIME_TARGET) is not aborted. + if self.max_allowed_inactivity_time <= 0: + threshold = math.inf + elif time.time() - self.vdf_failure_time < self.constants.SUB_SLOT_TIME_TARGET * 3: # If we have recently had a failure, allow some more time to finish the slot (we can be up to 3x slower) - active_time_threshold = self.constants.SUB_SLOT_TIME_TARGET * 3 + threshold = float(self.constants.SUB_SLOT_TIME_TARGET * 3) else: - # If there were no failures recently trigger a reset after 60 seconds of no activity. - # Signage points should be every 9 seconds - active_time_threshold = self.max_allowed_inactivity_time - if time.time() - self.last_active_time > active_time_threshold: - log.error(f"Not active for {active_time_threshold} seconds, restarting all chains") + # Signage points should be every 9 seconds on a healthy, calibrated slot. + threshold = float(self.max_allowed_inactivity_time) + if time.time() - self.last_active_time > threshold: + log.error(f"Not active for {threshold} seconds, restarting all chains") self.max_allowed_inactivity_time = min(self.max_allowed_inactivity_time * 2, 1800) await self._reset_chains() diff --git a/chia/timelord/timelord_api.py b/chia/timelord/timelord_api.py index d853bb7bda..dd23426d3d 100644 --- a/chia/timelord/timelord_api.py +++ b/chia/timelord/timelord_api.py @@ -63,7 +63,7 @@ class TimelordAPI: async with self.timelord.lock: if self.timelord.bluebox_mode: return None - self.timelord.max_allowed_inactivity_time = 60 + self.timelord.restore_inactivity_timeout() if self.timelord.last_state.peak is None: # no known peak diff --git a/chia/util/initial-config.yaml b/chia/util/initial-config.yaml index fffcb30e77..a89c8e02bd 100644 --- a/chia/util/initial-config.yaml +++ b/chia/util/initial-config.yaml @@ -280,6 +280,13 @@ timelord: # If `slow_bluebox` is True, launches `slow_bluebox_process_count` processes. slow_bluebox_process_count: 1 + # Seconds of no finished SP/IP/EOS before the timelord resets all VDF chains. + # Signage points are normally ~9s, but finishing a sub-slot can take ~10 minutes + # when SSI is high or blocks are sparse. 0 disables this inactivity reset + # (VDF-client failure handling is unchanged). After each reset the timeout + # doubles up to 1800s; a new peak restores this configured value. + max_allowed_inactivity_time: 60 + multiprocessing_start_method: default start_rpc_server: True