Farmer now returns missing signage points (#15765)

* Farmer now returns missing signage points

* Added test

* Fixed lint error

* Ignored strange pylint behavior

* Fixed lint error

---------

Co-authored-by: Earle Lowe <e.lowe@chia.net>

chia/farmer/farmer.py

@@ -5,6 +5,7 @@ import json
 import logging
 import time
 import traceback
+from math import floor
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
@@ -160,6 +161,9 @@ class Farmer:
 
         self.all_root_sks: List[PrivateKey] = []
 
+        # Use to find missing signage points. (new_signage_point, time)
+        self.prev_signage_point: Optional[Tuple[uint64, farmer_protocol.NewSignagePoint]] = None
+
     def get_connections(self, request_node_type: Optional[NodeType]) -> List[Dict[str, Any]]:
         return default_get_connections(server=self.server, request_node_type=request_node_type)
 
@@ -743,6 +747,34 @@ class Farmer:
             raise KeyError(f"Receiver missing for {node_id}")
         return receiver
 
+    def check_missing_signage_points(
+        self, timestamp: uint64, new_signage_point: farmer_protocol.NewSignagePoint
+    ) -> Optional[Tuple[uint64, uint32]]:
+        if self.prev_signage_point is None:
+            self.prev_signage_point = (timestamp, new_signage_point)
+            return None
+
+        prev_time, prev_sp = self.prev_signage_point
+        self.prev_signage_point = (timestamp, new_signage_point)
+
+        if prev_sp.challenge_hash == new_signage_point.challenge_hash:
+            missing_sps = new_signage_point.signage_point_index - prev_sp.signage_point_index - 1
+            if missing_sps > 0:
+                return timestamp, uint32(missing_sps)
+            return None
+
+        actual_sp_interval_seconds = float(timestamp - prev_time)
+        if actual_sp_interval_seconds <= 0:
+            return None
+
+        expected_sp_interval_seconds = self.constants.SUB_SLOT_TIME_TARGET / self.constants.NUM_SPS_SUB_SLOT
+        allowance = 1.6  # Should be chosen from the range (1 <= allowance < 2)
+        if actual_sp_interval_seconds < expected_sp_interval_seconds * allowance:
+            return None
+
+        skipped_sps = uint32(floor(actual_sp_interval_seconds / expected_sp_interval_seconds))
+        return timestamp, skipped_sps
+
     async def _periodically_update_pool_state_task(self) -> None:
         time_slept = 0
         config_path: Path = config_path_for_filename(self._root_path, "config.yaml")

chia/farmer/farmer_api.py

@@ -623,9 +623,14 @@ class FarmerAPI:
             self.farmer.log.debug(f"Duplicate signage point {new_signage_point.signage_point_index}")
             return
 
+        now = uint64(int(time.time()))
         self.farmer.sps[new_signage_point.challenge_chain_sp].append(new_signage_point)
-        self.farmer.cache_add_time[new_signage_point.challenge_chain_sp] = uint64(int(time.time()))
-        self.farmer.state_changed("new_signage_point", {"sp_hash": new_signage_point.challenge_chain_sp})
+        self.farmer.cache_add_time[new_signage_point.challenge_chain_sp] = now
+        missing_signage_points = self.farmer.check_missing_signage_points(now, new_signage_point)
+        self.farmer.state_changed(
+            "new_signage_point",
+            {"sp_hash": new_signage_point.challenge_chain_sp, "missing_signage_points": missing_signage_points},
+        )
 
     @api_request()
     async def request_signed_values(self, full_node_request: farmer_protocol.RequestSignedValues) -> None:

chia/rpc/farmer_rpc_api.py

@@ -110,7 +110,9 @@ class FarmerRpcApi:
             pass
         elif change == "new_signage_point":
             sp_hash = change_data["sp_hash"]
+            missing_signage_points = change_data["missing_signage_points"]
             data = await self.get_signage_point({"sp_hash": sp_hash.hex()})
+            data["missing_signage_points"] = missing_signage_points
             payloads.append(
                 create_payload_dict(
                     "new_signage_point",

tests/farmer_harvester/test_farmer_harvester.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+from math import floor
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
@@ -13,7 +14,7 @@ from chia.farmer.farmer_api import FarmerAPI
 from chia.harvester.harvester import Harvester
 from chia.harvester.harvester_api import HarvesterAPI
 from chia.plotting.util import PlotsRefreshParameter
-from chia.protocols import harvester_protocol
+from chia.protocols import farmer_protocol, harvester_protocol
 from chia.protocols.protocol_message_types import ProtocolMessageTypes
 from chia.rpc.harvester_rpc_client import HarvesterRpcClient
 from chia.server.outbound_message import NodeType, make_msg
@@ -23,6 +24,8 @@ from chia.simulator.time_out_assert import time_out_assert
 from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.types.peer_info import UnresolvedPeerInfo
 from chia.util.config import load_config
+from chia.util.hash import std_hash
+from chia.util.ints import uint8, uint32, uint64
 from chia.util.keychain import generate_mnemonic
 from tests.conftest import HarvesterFarmerEnvironment
 
@@ -209,3 +212,75 @@ async def test_harvester_config(
     assert res is True
     new_config = load_config(harvester_service.root_path, "config.yaml")
     check_config_match(new_config, harvester_config)
+
+
+@pytest.mark.asyncio
+async def test_missing_signage_point(
+    farmer_one_harvester: Tuple[List[Service[Harvester, HarvesterAPI]], Service[Farmer, FarmerAPI], BlockTools]
+) -> None:
+    _, farmer_service, bt = farmer_one_harvester
+    farmer_api = farmer_service._api
+    farmer = farmer_api.farmer
+
+    def create_sp(index: int, challenge_hash: bytes32) -> Tuple[uint64, farmer_protocol.NewSignagePoint]:
+        time = uint64(index + 1)
+        sp = farmer_protocol.NewSignagePoint(
+            challenge_hash, std_hash(b"2"), std_hash(b"3"), uint64(1), uint64(1000000), uint8(index), uint32(1)
+        )
+        return time, sp
+
+    # First sp. No missing sps
+    time0, sp0 = create_sp(index=0, challenge_hash=std_hash(b"1"))
+    assert farmer.prev_signage_point is None
+    ret = farmer.check_missing_signage_points(time0, sp0)
+    assert ret is None
+    assert farmer.prev_signage_point == (time0, sp0)
+
+    # 2nd sp. No missing sps
+    time1, sp1 = create_sp(index=1, challenge_hash=std_hash(b"1"))
+    ret = farmer.check_missing_signage_points(time1, sp1)
+    assert ret is None
+
+    # 3rd sp. 1 missing sp
+    time3, sp3 = create_sp(index=3, challenge_hash=std_hash(b"1"))
+    ret = farmer.check_missing_signage_points(time3, sp3)
+    assert ret == (time3, uint32(1))
+
+    # New challenge hash. Not counted as missing sp
+    _, sp_new_cc1 = create_sp(index=0, challenge_hash=std_hash(b"2"))
+    time_new_cc1 = time3
+    ret = farmer.check_missing_signage_points(time_new_cc1, sp_new_cc1)
+    assert ret is None
+
+    # Another new challenge hash. Calculating missing sps by timestamp
+    _, sp_new_cc2 = create_sp(index=0, challenge_hash=std_hash(b"3"))
+    # New sp is not in 9s but 12s is allowed
+    # since allowance multiplier is 1.6. (12 < 9 * 1.6)
+    time_new_cc2 = uint64(time_new_cc1 + 12)
+    ret = farmer.check_missing_signage_points(time_new_cc2, sp_new_cc2)
+    assert ret is None
+
+    # Another new challenge hash. Calculating missing sps by timestamp
+    _, sp_new_cc3 = create_sp(index=0, challenge_hash=std_hash(b"4"))
+    time_new_cc3 = uint64(time_new_cc2 + 601)  # roughly 10 minutes passed.
+    ret = farmer.check_missing_signage_points(time_new_cc3, sp_new_cc3)
+    assert ret is not None
+    ret_time, ret_skipped_sps = ret
+    assert ret_time == time_new_cc3
+    assert ret_skipped_sps == uint32(
+        floor(601 / (farmer.constants.SUB_SLOT_TIME_TARGET / farmer.constants.NUM_SPS_SUB_SLOT))
+    )
+
+    original_state_changed_callback = farmer.state_changed_callback
+    assert original_state_changed_callback is not None
+    number_of_missing_sps: uint32 = uint32(0)
+
+    def state_changed(change: str, data: Dict[str, Any]) -> None:
+        nonlocal number_of_missing_sps
+        number_of_missing_sps = data["missing_signage_points"][1]
+        original_state_changed_callback(change, data)
+
+    farmer.state_changed_callback = state_changed  # type: ignore
+    _, sp_for_farmer_api = create_sp(index=2, challenge_hash=std_hash(b"4"))
+    await farmer_api.new_signage_point(sp_for_farmer_api)
+    assert number_of_missing_sps == uint32(1)