[CHIA-4423] Add a config knob for the timelord inactivity chain reset (#21401)

* Add a config knob for the timelord inactivity chain reset.

A 60s no-SP timeout can abort VDF work that still needs a full sub-slot, which is enough to stall a sparse testnet.

Co-authored-by: Cursor <cursoragent@cursor.com>

* Fix the inactivity-timeout test mock so new-peak restore can run.

NewPeakTimelord's spec mock rejects reward_chain_block, which made the restore test fail on every consensus mode.

Co-authored-by: Cursor <cursoragent@cursor.com>

* Collapse the timelord inactivity timeout to one field and one watchdog path.

Drop the shadow configured copy, restore from config on peak, and keep VDF-failure handling separate from the inactivity reset.

Co-authored-by: Cursor <cursoragent@cursor.com>

---------

Co-authored-by: Cursor <cursoragent@cursor.com>

.cursor/context/timelord.md

@@ -77,7 +77,7 @@ Both paths intentionally choose a random target VDF field first, then fall back
 
 - `timelord.lock` is the module's main consistency boundary. API handlers, VDF-client mapping, iteration submission, proof appending, compact queue mutation, and reset operations rely on this lock around multi-field mutations.
 - `_do_process_communication()` performs network reads outside many lock sections but appends validated proofs and failure records under the lock. Be careful not to hold the lock across long VDF-client reads.
-- `_handle_failures()` prioritizes liveness. A current-generation VDF-client failure resets to EOS-only work; prolonged inactivity backs off the restart threshold up to a source-defined cap and resets all chains.
+- `_handle_failures()` splits VDF-client crashes (`_handle_vdf_failures`, EOS-only reset) from the inactivity watchdog (`_maybe_reset_for_inactivity`). The watchdog backs off `max_allowed_inactivity_time` up to 1800s. `0` disables it (`math.inf` threshold). `restore_inactivity_timeout()` re-reads the config value; `new_peak_timelord` calls that instead of assigning the field.
 - Shutdown closes the executor trigger file, cancels communication/main-loop tasks, sends stop signals to VDF clients, closes idle and assigned writers, and closes the TCP server.
 - `timelord_launcher.py` is a separate process manager for `chiavdf`'s `vdf_client` binary. It resolves the configured host, restarts clients until stopped, suppresses early stderr noise, and kills all active subprocesses on shutdown.
 

chia/_tests/timelord/test_timelord.py

@@ -2,6 +2,8 @@ from __future__ import annotations
 
 import asyncio
 import time
+from pathlib import Path
+from typing import Any
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
@@ -10,6 +12,7 @@ from chia_rs.sized_bytes import bytes32
 from chia_rs.sized_ints import uint32, uint64
 
 from chia._tests.conftest import ConsensusMode
+from chia.consensus.default_constants import DEFAULT_CONSTANTS
 from chia.timelord import iters_from_block as iters_from_block_module
 from chia.timelord import timelord as timelord_module
 from chia.timelord.iters_from_block import iters_from_block
@@ -181,3 +184,52 @@ class TestHandleClient:
         assert len(tl.free_clients) == 3
         overflow_writer.close.assert_called_once()
         overflow_writer.wait_closed.assert_awaited_once()
+
+
+def _minimal_timelord_config(**overrides: object) -> dict[str, Any]:
+    config: dict[str, Any] = {"vdf_clients": {"ip": ["127.0.0.1"]}}
+    config.update(overrides)
+    return config
+
+
+def test_inactivity_timeout_defaults_to_60(tmp_path: Path) -> None:
+    tl = Timelord(tmp_path, _minimal_timelord_config(), DEFAULT_CONSTANTS)
+    assert tl.max_allowed_inactivity_time == 60
+
+
+def test_inactivity_timeout_reads_config(tmp_path: Path) -> None:
+    tl = Timelord(tmp_path, _minimal_timelord_config(max_allowed_inactivity_time=600), DEFAULT_CONSTANTS)
+    assert tl.max_allowed_inactivity_time == 600
+
+
+def test_restore_inactivity_timeout_rereads_config(tmp_path: Path) -> None:
+    tl = Timelord(tmp_path, _minimal_timelord_config(max_allowed_inactivity_time=600), DEFAULT_CONSTANTS)
+    tl.max_allowed_inactivity_time = 1200
+    tl.restore_inactivity_timeout()
+    assert tl.max_allowed_inactivity_time == 600
+
+
+@pytest.mark.anyio
+async def test_inactivity_reset_disabled_when_configured_zero(tmp_path: Path) -> None:
+    tl = Timelord(tmp_path, _minimal_timelord_config(max_allowed_inactivity_time=0), DEFAULT_CONSTANTS)
+    tl.last_active_time = time.time() - 10_000
+    with patch.object(tl, "_reset_chains", new_callable=AsyncMock) as reset:
+        await tl._maybe_reset_for_inactivity()
+    reset.assert_not_called()
+
+
+@pytest.mark.anyio
+async def test_inactivity_reset_uses_configured_threshold(tmp_path: Path) -> None:
+    tl = Timelord(tmp_path, _minimal_timelord_config(max_allowed_inactivity_time=600), DEFAULT_CONSTANTS)
+    tl.vdf_failure_time = 0
+
+    tl.last_active_time = time.time() - 61
+    with patch.object(tl, "_reset_chains", new_callable=AsyncMock) as reset:
+        await tl._maybe_reset_for_inactivity()
+    reset.assert_not_called()
+
+    tl.last_active_time = time.time() - 601
+    with patch.object(tl, "_reset_chains", new_callable=AsyncMock) as reset:
+        await tl._maybe_reset_for_inactivity()
+    reset.assert_awaited_once()
+    assert tl.max_allowed_inactivity_time == 1200

chia/timelord/timelord.py

@@ -5,6 +5,7 @@ import contextlib
 import dataclasses
 import io
 import logging
+import math
 import os
 import random
 import tempfile
@@ -148,7 +149,7 @@ class Timelord:
             self.bluebox_mode = self.config.get("sanitizer_mode", False)
         self.pending_bluebox_info: list[tuple[float, timelord_protocol.RequestCompactProofOfTime]] = []
         self.last_active_time = time.time()
-        self.max_allowed_inactivity_time = 60
+        self.restore_inactivity_timeout()
         self._executor_shutdown_tempfile: IO[bytes] | None = None
         self.bluebox_pool: ThreadPoolExecutor | None = None
 
@@ -888,34 +889,44 @@ class Timelord:
                     self.total_unfinished += 1
             await self._reset_chains()
 
+    def restore_inactivity_timeout(self) -> None:
+        self.max_allowed_inactivity_time = int(self.config.get("max_allowed_inactivity_time", 60))
+
     async def _handle_failures(self) -> None:
-        if len(self.vdf_failures) > 0:
-            # This can happen if one of the VDF processes has an issue. In this case, we abort all other
-            # infusion points and signage points, and go straight to the end of slot, so we avoid potential
-            # issues with the number of iterations that failed.
+        await self._handle_vdf_failures()
+        await self._maybe_reset_for_inactivity()
 
-            failed_chain, proof_label = self.vdf_failures[0]
-            log.error(
-                f"Vdf clients failed {self.vdf_failures_count} times. Last failure: {failed_chain}, "
-                f"label {proof_label}, current: {self.num_resets}"
-            )
-            if proof_label == self.num_resets:
-                await self._reset_chains(only_eos=True)
-            self.vdf_failure_time = time.time()
-            self.vdf_failures = []
+    async def _handle_vdf_failures(self) -> None:
+        if len(self.vdf_failures) == 0:
+            return
+        # This can happen if one of the VDF processes has an issue. In this case, we abort all other
+        # infusion points and signage points, and go straight to the end of slot, so we avoid potential
+        # issues with the number of iterations that failed.
+        failed_chain, proof_label = self.vdf_failures[0]
+        log.error(
+            f"Vdf clients failed {self.vdf_failures_count} times. Last failure: {failed_chain}, "
+            f"label {proof_label}, current: {self.num_resets}"
+        )
+        if proof_label == self.num_resets:
+            await self._reset_chains(only_eos=True)
+        self.vdf_failure_time = time.time()
+        self.vdf_failures = []
 
+    async def _maybe_reset_for_inactivity(self) -> None:
         # If something goes wrong in the VDF client due to a failed thread, we might get stuck in a situation where we
         # are waiting for that client to finish. Usually other peers will finish the VDFs and reset us. In the case that
         # there are no other timelords, this reset should bring the timelord back to a running state.
-        if time.time() - self.vdf_failure_time < self.constants.SUB_SLOT_TIME_TARGET * 3:
+        # 0 disables the inactivity reset so a slow EOS (up to ~SUB_SLOT_TIME_TARGET) is not aborted.
+        if self.max_allowed_inactivity_time <= 0:
+            threshold = math.inf
+        elif time.time() - self.vdf_failure_time < self.constants.SUB_SLOT_TIME_TARGET * 3:
             # If we have recently had a failure, allow some more time to finish the slot (we can be up to 3x slower)
-            active_time_threshold = self.constants.SUB_SLOT_TIME_TARGET * 3
+            threshold = float(self.constants.SUB_SLOT_TIME_TARGET * 3)
         else:
-            # If there were no failures recently trigger a reset after 60 seconds of no activity.
-            # Signage points should be every 9 seconds
-            active_time_threshold = self.max_allowed_inactivity_time
-        if time.time() - self.last_active_time > active_time_threshold:
-            log.error(f"Not active for {active_time_threshold} seconds, restarting all chains")
+            # Signage points should be every 9 seconds on a healthy, calibrated slot.
+            threshold = float(self.max_allowed_inactivity_time)
+        if time.time() - self.last_active_time > threshold:
+            log.error(f"Not active for {threshold} seconds, restarting all chains")
             self.max_allowed_inactivity_time = min(self.max_allowed_inactivity_time * 2, 1800)
             await self._reset_chains()
 

chia/timelord/timelord_api.py

@@ -63,7 +63,7 @@ class TimelordAPI:
         async with self.timelord.lock:
             if self.timelord.bluebox_mode:
                 return None
-            self.timelord.max_allowed_inactivity_time = 60
+            self.timelord.restore_inactivity_timeout()
 
             if self.timelord.last_state.peak is None:
                 # no known peak

chia/util/initial-config.yaml

@@ -280,6 +280,13 @@ timelord:
   # If `slow_bluebox` is True, launches `slow_bluebox_process_count` processes.
   slow_bluebox_process_count: 1
 
+  # Seconds of no finished SP/IP/EOS before the timelord resets all VDF chains.
+  # Signage points are normally ~9s, but finishing a sub-slot can take ~10 minutes
+  # when SSI is high or blocks are sparse. 0 disables this inactivity reset
+  # (VDF-client failure handling is unchanged). After each reset the timeout
+  # doubles up to 1800s; a new peak restores this configured value.
+  max_allowed_inactivity_time: 60
+
   multiprocessing_start_method: default
 
   start_rpc_server: True