From 79008d11d57e9706b375f8f732f81304f11620c8 Mon Sep 17 00:00:00 2001 From: Almog De Paz Date: Fri, 1 May 2026 22:02:08 +0300 Subject: [PATCH] context for moduls and testing for cursor to use automatically (#20572) * context fo cursor * pass with docs refrence, fix line counts and some more and minor fixes * lint * pr comments * fixes --- .cursor/context/INDEX.md | 66 +++++ .cursor/context/architecture-overview.md | 145 +++++++++++ .cursor/context/clvm-execution.md | 218 ++++++++++++++++ .cursor/context/consensus.md | 280 +++++++++++++++++++++ .cursor/context/full-node.md | 249 ++++++++++++++++++ .cursor/context/global-invariants.md | 226 +++++++++++++++++ .cursor/context/mempool.md | 213 ++++++++++++++++ .cursor/context/networking.md | 244 ++++++++++++++++++ .cursor/context/wallet.md | 179 +++++++++++++ .cursor/rules/context-clvm.mdc | 21 ++ .cursor/rules/context-consensus.mdc | 15 ++ .cursor/rules/context-full-node.mdc | 21 ++ .cursor/rules/context-mempool.mdc | 17 ++ .cursor/rules/context-networking.mdc | 14 ++ .cursor/rules/context-routing.mdc | 37 +++ .cursor/rules/context-wallet.mdc | 13 + .cursor/rules/testing-guide-blockchain.mdc | 90 +++++++ .cursor/rules/testing-guide-data-layer.mdc | 132 ++++++++++ .cursor/rules/testing-guide-full-node.mdc | 133 ++++++++++ .cursor/rules/testing-guide-mempool.mdc | 142 +++++++++++ .cursor/rules/testing-guide-patterns.mdc | 119 +++++++++ .cursor/rules/testing-guide-server.mdc | 125 +++++++++ .cursor/rules/testing-guide.mdc | 34 +++ .cursor/rules/use-venv-wrappers.mdc | 13 + .gitignore | 7 + tools/py | 26 ++ tools/pytest | 12 + 27 files changed, 2791 insertions(+) create mode 100644 .cursor/context/INDEX.md create mode 100644 .cursor/context/architecture-overview.md create mode 100644 .cursor/context/clvm-execution.md create mode 100644 .cursor/context/consensus.md create mode 100644 .cursor/context/full-node.md create mode 100644 .cursor/context/global-invariants.md create mode 100644 .cursor/context/mempool.md create mode 100644 .cursor/context/networking.md create mode 100644 .cursor/context/wallet.md create mode 100644 .cursor/rules/context-clvm.mdc create mode 100644 .cursor/rules/context-consensus.mdc create mode 100644 .cursor/rules/context-full-node.mdc create mode 100644 .cursor/rules/context-mempool.mdc create mode 100644 .cursor/rules/context-networking.mdc create mode 100644 .cursor/rules/context-routing.mdc create mode 100644 .cursor/rules/context-wallet.mdc create mode 100644 .cursor/rules/testing-guide-blockchain.mdc create mode 100644 .cursor/rules/testing-guide-data-layer.mdc create mode 100644 .cursor/rules/testing-guide-full-node.mdc create mode 100644 .cursor/rules/testing-guide-mempool.mdc create mode 100644 .cursor/rules/testing-guide-patterns.mdc create mode 100644 .cursor/rules/testing-guide-server.mdc create mode 100644 .cursor/rules/testing-guide.mdc create mode 100644 .cursor/rules/use-venv-wrappers.mdc create mode 100755 tools/py create mode 100755 tools/pytest diff --git a/.cursor/context/INDEX.md b/.cursor/context/INDEX.md new file mode 100644 index 0000000000..f25b4575db --- /dev/null +++ b/.cursor/context/INDEX.md @@ -0,0 +1,66 @@ +# Chia Blockchain — Deep Context Index + +> Generated from deep context-building pass. Each subsystem file is +> self-contained: pull only the file(s) relevant to the code you're touching. + +## How to use + +1. Read **this file** first for orientation. +2. Attach the subsystem file(s) that cover the code you're working on. +3. If your change crosses subsystem boundaries, also attach + `global-invariants.md` — it documents the contracts between modules. + +--- + +## Subsystem files + +| File | Covers | When to attach | +| ---------------------------------------------------- | -------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------- | +| [architecture-overview.md](architecture-overview.md) | Module map, actors, entrypoints, key types, `chia_rs` boundary | Starting any unfamiliar work; first-time orientation | +| [consensus.md](consensus.md) | Block validation, difficulty adjustment, fork choice, VDF iterations, rewards | Touching `chia/consensus/`, block acceptance, reorgs | +| [mempool.md](mempool.md) | Transaction admission, eviction, fee logic, conflict detection, FF/DEDUP | Touching `chia/full_node/mempool*.py`, `eligible_coin_spends.py`, fee estimation | +| [full-node.md](full-node.md) | FullNode orchestration, sync, block processing pipeline, FullNodeStore, FullNodeAPI | Touching `chia/full_node/full_node.py`, `full_node_api.py`, `full_node_store.py` | +| [networking.md](networking.md) | WebSocket connections, rate limiting, peer discovery, protocol state machine | Touching `chia/server/`, `chia/protocols/`, connection handling | +| [wallet.md](wallet.md) | Coin selection, wallet state manager, wallet node sync, sub-wallets | Touching `chia/wallet/` | +| [clvm-execution.md](clvm-execution.md) | CLVM execution, condition processing, canonical serialization, cost metering | Touching puzzle execution, spend validation, generator logic | +| [global-invariants.md](global-invariants.md) | Cross-module invariants, state dependencies, trust boundaries, workflow traces, fragility clusters | Cross-cutting changes, security review, reorg-related work | + +--- + +## Quick reference — key files by size/complexity + +| File | Lines | Role | +| ------------------------------------------- | ----- | --------------------------------- | +| `chia/full_node/full_node.py` | ~3400 | Main orchestrator | +| `chia/wallet/wallet_state_manager.py` | ~3330 | Wallet state | +| `chia/wallet/wallet_rpc_api.py` | ~3610 | Wallet RPC surface | +| `chia/full_node/full_node_api.py` | ~2080 | P2P message handlers | +| `chia/full_node/full_node_rpc_api.py` | ~1170 | Full node RPC | +| `chia/consensus/blockchain.py` | ~1090 | Chain state + add_block | +| `chia/consensus/block_header_validation.py` | ~1060 | Header checks | +| `chia/full_node/weight_proof.py` | ~1740 | Weight proof validation | +| `chia/full_node/mempool_manager.py` | ~1160 | Mempool admission | +| `chia/full_node/mempool.py` | ~810 | Mempool data structure | +| `chia/consensus/block_body_validation.py` | ~580 | Body checks | +| `chia/consensus/difficulty_adjustment.py` | ~410 | Difficulty/SSI | +| `chia/full_node/coin_store.py` | ~680 | UTXO database | +| `chia/full_node/full_node_store.py` | ~1060 | Signage points, unfinished blocks | + +--- + +## Consensus constants cheat-sheet + +| Constant | Value | Note | +| ------------------------------ | -------------- | -------------------------- | +| `SLOT_BLOCKS_TARGET` | 32 | Target blocks / sub-slot | +| `NUM_SPS_SUB_SLOT` | 64 | Signage points / sub-slot | +| `SUB_SLOT_TIME_TARGET` | 600 s | ~10 min / sub-slot | +| `EPOCH_BLOCKS` | 4608 | Blocks / difficulty epoch | +| `SUB_EPOCH_BLOCKS` | 384 | Blocks / sub-epoch | +| `MAX_BLOCK_COST_CLVM` | 11 000 000 000 | Max CLVM cost / block | +| `COST_PER_BYTE` | 12 000 | Generator byte cost | +| `MAX_BLOCK_COUNT_PER_REQUESTS` | 32 | Max blocks / P2P request | +| `DIFFICULTY_CHANGE_MAX_FACTOR` | 3 | Max epoch difficulty ratio | +| `MAX_FUTURE_TIME2` | 120 s | Max timestamp drift | +| `HARD_FORK_HEIGHT` | 5 496 000 | June 2024 hard fork | +| `MEMPOOL_BLOCK_BUFFER` | 10 | Mempool = 10× block cost | diff --git a/.cursor/context/architecture-overview.md b/.cursor/context/architecture-overview.md new file mode 100644 index 0000000000..4f3adedc9d --- /dev/null +++ b/.cursor/context/architecture-overview.md @@ -0,0 +1,145 @@ +# Architecture Overview + +> Attach this file when starting unfamiliar work or needing first-time +> orientation on the chia-blockchain codebase. + +## Project shape + +Python PoST blockchain. **Not a monorepo** — this repository (`chia-blockchain`) +is the Python node implementation. It depends on several external packages from +the Chia-Network GitHub org for Rust-accelerated cryptography, proofs, and +puzzle compilation. + +## External Chia dependencies + +| Package | Repo | Role | Used by | +| ----------------- | ------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | +| `chia_rs` | [Chia-Network/chia_rs](https://github.com/Chia-Network/chia_rs) | Core Rust FFI: consensus types, BLS signatures, CLVM execution, serialization, condition validation, spend bundle validation, merkle sets, V2 proof solving | Nearly everything — consensus, mempool, wallet, types, solver | +| `chiapos` | [Chia-Network/chiapos](https://github.com/Chia-Network/chiapos) | Proof of Space: plot creation, proof verification, quality computation | `chia/plotting/`, `chia/types/blockchain_format/proof_of_space.py` | +| `chiavdf` | [Chia-Network/chiavdf](https://github.com/Chia-Network/chiavdf) | VDF computation and proof verification | `chia/timelord/`, `chia/types/blockchain_format/vdf.py`, `chia/simulator/` | +| `clvm` | [Chia-Network/clvm](https://github.com/Chia-Network/clvm) | Python CLVM interpreter (used in tooling, not consensus-hot path) | `chia/types/blockchain_format/program.py`, wallet puzzle drivers | +| `clvm_tools` | [Chia-Network/clvm_tools](https://github.com/Chia-Network/clvm_tools) | CLVM utilities: currying, `Program.to()`, disassembly | Wallet puzzle construction, tests, debugging | +| `chialisp` | [Chia-Network/chialisp](https://github.com/Chia-Network/chialisp) | Rust ChiaLisp compiler — compiles `.clsp` puzzle source to CLVM bytecode | `chia/wallet/puzzles/load_clvm.py`, puzzle compilation tooling | +| `chia-puzzles-py` | [Chia-Network/chia-puzzles-py](https://github.com/Chia-Network/chia-puzzles-py) | Pre-compiled standard puzzle bytecode (singletons, CATs, DIDs, NFTs, etc.) | Wallet puzzle drivers, pool puzzles, data layer | +| `chiabip158` | [Chia-Network/chiabip158](https://github.com/Chia-Network/chiabip158) | BIP-158 compact block filters for lightweight wallet sync | Block body validation, mempool manager, wallet sync | + +**Version pinning**: `chia_rs` is pinned to a minor range (`>=0.37.0, <0.38`). +Other Chia packages use minimum-version pins. See `pyproject.toml` for current values. + +## Module map + +| Module | Purpose | Criticality | +| ------------------ | -------------------------------------------------------------------- | ------------ | +| `chia/consensus/` | Block validation, difficulty, fork choice, VDF iters, rewards | **Critical** | +| `chia/full_node/` | Full node state, mempool, stores, fee estimation, weight proofs, RPC | **Critical** | +| `chia/server/` | Networking: WebSocket, rate limiting, peer discovery, TLS | **Critical** | +| `chia/protocols/` | Wire protocol message definitions between all node types | **Critical** | +| `chia/wallet/` | Wallet state, coin selection, spend construction, sub-wallets | **High** | +| `chia/farmer/` | Farming logic, signage point handling, proof forwarding | **High** | +| `chia/harvester/` | Plot file management, PoS lookups | **Medium** | +| `chia/timelord/` | VDF computation, infusion point management | **High** | +| `chia/types/` | Type definitions: blockchain format, mempool items, generators | **High** | +| `chia/util/` | DB wrapper, streamable, keychain, bech32m, etc. | **Medium** | +| `chia/simulator/` | Test blockchain simulator | Low | +| `chia/data_layer/` | DataLayer (data-storage singleton) | Medium | +| `chia/cmds/` | CLI command handlers | Low | + +## `chia_rs` boundary (largest external dependency) + +Nearly all core consensus types live in Rust via `chia_rs`: + +**Types**: `BlockRecord`, `FullBlock`, `ConsensusConstants`, `SpendBundleConditions`, +`CoinRecord`, `SpendBundle`, `EndOfSubSlotBundle`, `HeaderBlock`, `UnfinishedBlock`, +`SubEpochSummary`, `SubEpochChallengeSegment`, `Coin`, `CoinSpend`, `G1Element`, +`G2Element`, `AugSchemeMPL`, `BLSCache`, `PartialProof`. + +**Functions**: `validate_clvm_and_signature`, `run_block_generator`, +`run_block_generator2`, `additions_and_removals`, `check_time_locks`, +`compute_merkle_set_root`, `fast_forward_singleton`, `supports_fast_forward`, +`get_flags_for_height_and_constants`, `solution_generator_backrefs`, +`get_puzzle_and_solution_for_coin2`, `is_canonical_serialization`, +`get_conditions_from_spendbundle`, `get_spends_for_trusted_block`, +`solve_proof` (V2 plot solving). + +**Rule of thumb**: Consensus-critical _math_ (VDF iteration calculation, difficulty +adjustment, quality computation) is Python. Signature/CLVM/serialization +validation is Rust. VDF proofs are computed by `chiavdf`, PoS proofs by +`chiapos`. Puzzle bytecode comes pre-compiled from `chia-puzzles-py`. + +## Actors + +Node roles are defined by `NodeType` in `chia/protocols/outbound_message.py`: +`FULL_NODE`, `HARVESTER`, `FARMER`, `TIMELORD`, `INTRODUCER`, `WALLET`, +`DATA_LAYER`, `SOLVER`. + +### Full Node (central) + +- **P2P API**: `FullNodeAPI` in `full_node_api.py` (~2080 lines) +- **RPC API**: `FullNodeRpcApi` in `full_node_rpc_api.py` (~1170 lines) +- **State machine**: `FullNode` in `full_node.py` (~3400 lines) + +### Farmer + +- **API**: `FarmerAPI` in `farmer_api.py` — receives signage points, forwards proofs +- **RPC**: `FarmerRpcApi` — local management + +### Harvester + +- **API**: `HarvesterAPI` in `harvester_api.py` — receives challenges, checks plots + +### Timelord + +- **API**: `TimelordAPI` in `timelord_api.py` — receives peaks, produces VDFs +- **State**: `TimelordState` in `timelord_state.py` + +### Wallet + +- **P2P**: `WalletNodeAPI` in `wallet_node_api.py` — coin state updates +- **RPC**: `WalletRpcApi` in `wallet_rpc_api.py` (~3600 lines) — full wallet surface +- **State**: `WalletStateManager` in `wallet_state_manager.py` (~3300 lines) + +### Introducer + +- **Service**: `Introducer` in `introducer.py` — bootstrap peer discovery +- **API**: `IntroducerAPI` in `introducer_api.py` — serves vetted peer lists + +### Data Layer + +- **Service**: `DataLayer` in `data_layer.py` — singleton-based data store service +- **RPC**: `DataLayerRpcApi` in `data_layer_rpc_api.py` — DataLayer control surface + +### Solver + +- **Service**: `Solver` in `solver.py` — solves V2 plot partial proofs into full proofs of space +- **API**: `SolverAPI` in `solver_api.py` — receives `SolverInfo` (partial proof, plot_id, k-size) from farmer, returns full proof via `SolverResponse` + +## Wire protocol overview + +109 message types in `ProtocolMessageTypes` enum. Key flows: + +- **Full Node ↔ Full Node**: `new_peak`, `new_transaction`, `request_block(s)`, + `new_signage_point_or_end_of_sub_slot`, `request_compact_vdf` +- **Full Node ↔ Wallet**: `new_peak_wallet`, `send_transaction`, + `coin_state_update`, `request_puzzle_state`, `mempool_items_added/removed` +- **Farmer ↔ Full Node**: `new_signage_point`, `declare_proof_of_space`, + `request_signed_values` +- **Farmer ↔ Harvester**: `new_signage_point_harvester`, `new_proof_of_space`, + `request_signatures` +- **Full Node ↔ Timelord**: `new_peak_timelord`, `new_infusion_point_vdf`, + `new_signage_point_vdf` + +## Key type files + +| File | Contents | +| ---------------------------------------------------- | ------------------------------------------------------ | +| `chia/types/blockchain_format/coin.py` | `Coin` (parent_id, puzzle_hash, amount) | +| `chia/types/blockchain_format/vdf.py` | `VDFInfo`, `VDFProof` | +| `chia/types/blockchain_format/proof_of_space.py` | PoS verification | +| `chia/types/blockchain_format/program.py` | CLVM program wrappers | +| `chia/types/blockchain_format/serialized_program.py` | Lazy CLVM deserialization | +| `chia/types/mempool_item.py` | `MempoolItem`, `BundleCoinSpend`, `UnspentLineageInfo` | +| `chia/types/generator_types.py` | `BlockGenerator`, `NewBlockGenerator` | +| `chia/types/validation_state.py` | `ValidationState` | +| `chia/types/weight_proof.py` | `WeightProof` | +| `chia/consensus/block_record.py` | Re-export of `BlockRecord` from chia_rs | +| `chia/consensus/default_constants.py` | `DEFAULT_CONSTANTS` with all parameter values | diff --git a/.cursor/context/clvm-execution.md b/.cursor/context/clvm-execution.md new file mode 100644 index 0000000000..f08a3c1fa2 --- /dev/null +++ b/.cursor/context/clvm-execution.md @@ -0,0 +1,218 @@ +# CLVM / Puzzle Execution — Deep Context + +> Attach when touching puzzle execution, spend validation, generator logic, +> condition processing, or `chia/types/blockchain_format/`. + +## Key files + +| File | Role | +| ---------------------------------------------------- | --------------------------------------------------- | +| `chia/consensus/condition_tools.py` | `pkm_pairs()` — extract AGG_SIG conditions | +| `chia/consensus/generator_tools.py` | `get_block_header()` — strip generator from block | +| `chia/consensus/get_block_generator.py` | `get_block_generator()` — resolve generator refs | +| `chia/consensus/cost_calculator.py` | `NPCResult` — name/puzzle/conditions result | +| `chia/consensus/condition_costs.py` | Condition opcode costs | +| `chia/types/blockchain_format/program.py` | CLVM program wrapper | +| `chia/types/blockchain_format/serialized_program.py` | Lazy deserialization | +| `chia/types/blockchain_format/coin.py` | `Coin(parent_id, puzzle_hash, amount)` | +| `chia/types/condition_opcodes.py` | All condition opcode values | +| `chia/types/condition_with_args.py` | `ConditionWithArgs` | +| `chia/types/generator_types.py` | `BlockGenerator`, `NewBlockGenerator` | +| `chia/types/clvm_cost.py` | Cost constants | +| `chia/full_node/mempool_manager.py` | `pre_validate_spendbundle()`, `is_clvm_canonical()` | +| `chia/wallet/conditions.py` | Condition construction for wallet spends | +| `chia/wallet/puzzles/` | CLVM puzzle source files | + +--- + +## Execution paths + +### 1. Mempool validation (spend bundles) + +``` +SpendBundle → validate_clvm_and_signature() [Rust] + → SpendBundleConditions +``` + +- Runs in thread pool (`MempoolManager.pool`) +- Flags: `get_flags_for_height_and_constants(peak.height) | MEMPOOL_MODE` +- Cost limit: `max_tx_clvm_cost` (half of `MAX_BLOCK_COST_CLVM`) + +### 2. Block validation (generators) + +``` +BlockGenerator → run_block_generator() / run_block_generator2() [Rust] + → SpendBundleConditions +``` + +- Generator refs resolved via `get_block_generator()` (up to 512 refs) +- Flags: `get_flags_for_height_and_constants(prev_tx_height)` +- Cost limit: `MAX_BLOCK_COST_CLVM` (11 000 000 000) + +### 3. Block building (mempool → generator) + +``` +Mempool items → solution_generator_backrefs() [Rust] → generator program +``` + +- Combines spend bundles into a single block generator +- Uses back-references for compression +- Legacy path: `create_block_generator()` serializes with `solution_generator_backrefs()` +- Alternative path: `create_block_generator2()` uses Rust `BlockBuilder`; opt-in today via + `full_node.config["block_creation"] = 1` (see TODO in `full_node_api.py` to make it default) + +--- + +## Resource limits + +### Cost metering + +Every CLVM operation has an associated cost. Total cost per block cannot +exceed `MAX_BLOCK_COST_CLVM = 11 000 000 000`. + +### Generator byte cost + +Each byte of generator program costs `COST_PER_BYTE = 12 000` CLVM cost +units, in addition to execution cost. + +### Atom/pair bounds (mempool only) + +After CLVM execution: + +```python +if sbc.num_atoms > sbc.cost * 60_000_000 / MAX_BLOCK_COST_CLVM: + reject # too many atoms +if sbc.num_pairs > sbc.cost * 60_000_000 / MAX_BLOCK_COST_CLVM: + reject # too many pairs +``` + +This bounds memory usage relative to cost paid. At max cost, allows +60 million atoms or pairs. + +### Generator ref list + +`MAX_GENERATOR_REF_LIST_SIZE = 512` — max number of previous block generators +that can be referenced. + +--- + +## Canonical serialization + +**Location**: `mempool_manager.py:185` + +### `is_clvm_canonical(clvm_buffer)` + +Checks that a CLVM program uses shortest-form atom encoding: + +- No unnecessary length prefix bytes +- No back-references (`0xFE` byte) +- No trailing garbage + +### When enforced + +Required for DEDUP-eligible spends. Without canonical form, identical +solutions could have different serializations, breaking dedup. + +### `is_atom_canonical(clvm_buffer, offset)` + +Validates a single atom's length prefix encoding. The CLVM format uses +variable-length prefixes (1-6 bytes) based on atom size. Each prefix +length has a minimum atom size threshold. + +--- + +## Condition opcodes + +**Location**: `chia/types/condition_opcodes.py` + +Key conditions (from CLVM spend output): + +| Opcode | Name | Effect | +| ------ | ---------------------------- | ----------------------------------------- | +| 43 | `AGG_SIG_PARENT` | Require signature with parent data | +| 44 | `AGG_SIG_PUZZLE` | Require signature with puzzle hash data | +| 45 | `AGG_SIG_AMOUNT` | Require signature with amount data | +| 46 | `AGG_SIG_PUZZLE_AMOUNT` | Require signature with puzzle+amount data | +| 47 | `AGG_SIG_PARENT_AMOUNT` | Require signature with parent+amount data | +| 48 | `AGG_SIG_PARENT_PUZZLE` | Require signature with parent+puzzle data | +| 49 | `AGG_SIG_UNSAFE` | Require signature (no domain separation) | +| 50 | `AGG_SIG_ME` | Require signature with coin data | +| 51 | `CREATE_COIN` | Create a new coin | +| 52 | `RESERVE_FEE` | Declare minimum fee | +| 60 | `CREATE_COIN_ANNOUNCEMENT` | Create announcement | +| 61 | `ASSERT_COIN_ANNOUNCEMENT` | Assert announcement exists | +| 62 | `CREATE_PUZZLE_ANNOUNCEMENT` | Create puzzle announcement | +| 63 | `ASSERT_PUZZLE_ANNOUNCEMENT` | Assert puzzle announcement | +| 64 | `ASSERT_CONCURRENT_SPEND` | Assert another coin is spent | +| 65 | `ASSERT_CONCURRENT_PUZZLE` | Assert puzzle hash is spent | +| 66 | `SEND_MESSAGE` | Cross-coin messaging | +| 67 | `RECEIVE_MESSAGE` | Cross-coin messaging | +| 70 | `ASSERT_MY_COIN_ID` | Assert own coin ID | +| 71 | `ASSERT_MY_PARENT_ID` | Assert parent coin ID | +| 72 | `ASSERT_MY_PUZZLEHASH` | Assert own puzzle hash | +| 73 | `ASSERT_MY_AMOUNT` | Assert own amount | +| 74 | `ASSERT_MY_BIRTH_SECONDS` | Assert creation timestamp | +| 75 | `ASSERT_MY_BIRTH_HEIGHT` | Assert creation height | +| 76 | `ASSERT_EPHEMERAL` | Assert coin is ephemeral | +| 90 | `SOFTFORK` | Future-proof softfork condition | + +### Timelock conditions + +| Opcode | Name | Effect | +| ------ | -------------------------------- | ------------------------------ | +| 80 | `ASSERT_SECONDS_RELATIVE` | Min seconds since confirmation | +| 81 | `ASSERT_SECONDS_ABSOLUTE` | Min timestamp | +| 82 | `ASSERT_HEIGHT_RELATIVE` | Min blocks since confirmation | +| 83 | `ASSERT_HEIGHT_ABSOLUTE` | Min block height | +| 84 | `ASSERT_BEFORE_SECONDS_RELATIVE` | Max seconds since confirmation | +| 85 | `ASSERT_BEFORE_SECONDS_ABSOLUTE` | Max timestamp | +| 86 | `ASSERT_BEFORE_HEIGHT_RELATIVE` | Max blocks since confirmation | +| 87 | `ASSERT_BEFORE_HEIGHT_ABSOLUTE` | Max block height | + +--- + +## AGG_SIG conditions and replay protection + +### `AGG_SIG_ME_ADDITIONAL_DATA` + +Mainnet: `ccd5bb71183532bff220ba46c268991a3ff07eb358e8255a65c30a2dce0e5fbb` + +Each AGG_SIG variant appends different additional data: + +- `AGG_SIG_PARENT`: `hash(data + 43)` +- `AGG_SIG_PUZZLE`: `hash(data + 44)` +- `AGG_SIG_AMOUNT`: `hash(data + 45)` +- etc. + +This provides replay protection across forks. Forks MUST change +`AGG_SIG_ME_ADDITIONAL_DATA` to prevent cross-chain replays. + +--- + +## `MEMPOOL_MODE` flag + +When set during CLVM execution: + +- Enables stricter validation rules +- Rejects certain operations allowed in blocks but not in mempool +- Applied via `flags | MEMPOOL_MODE` in `pre_validate_spendbundle()` + +--- + +## Block generator construction + +### `get_block_generator()` + +**Location**: `consensus/get_block_generator.py` + +Resolves a block's generator and its references: + +1. Block has `transactions_generator` (the program) +2. Block has `transactions_generator_ref_list` (height references) +3. For each ref, fetch the generator bytes from that block height +4. Return `BlockGenerator(program, ref_generators)` + +### `solution_generator_backrefs()` [Rust] + +Creates a block generator program from spend bundles using back-references +for compression. This is used during block creation from mempool items. diff --git a/.cursor/context/consensus.md b/.cursor/context/consensus.md new file mode 100644 index 0000000000..f7ab46a2a5 --- /dev/null +++ b/.cursor/context/consensus.md @@ -0,0 +1,280 @@ +# Consensus Layer — Deep Context + +> Attach when touching `chia/consensus/`, block acceptance, reorgs, difficulty, +> or VDF iteration logic. + +## File map + +| File | Lines | Role | +| ------------------------------- | ----- | ------------------------------------------------------------------------------------- | +| `blockchain.py` | ~1090 | `Blockchain` class: chain state, `add_block()`, `_reconsider_peak()` | +| `blockchain_interface.py` | ~54 | Protocol definitions: `BlockRecordsProtocol`, `BlocksProtocol`, `BlockchainInterface` | +| `augmented_chain.py` | ~169 | `AugmentedBlockchain`: in-memory overlay for parallel validation | +| `block_header_validation.py` | ~1060 | `validate_unfinished_header_block()`: all header checks | +| `block_body_validation.py` | ~580 | `validate_block_body()`: transaction/coin checks, `ForkInfo` | +| `block_creation.py` | ~650 | `create_unfinished_block()`, `unfinished_block_to_full_block()` | +| `difficulty_adjustment.py` | ~410 | Difficulty & sub-slot-iters recalculation per epoch | +| `pot_iterations.py` | ~101 | SP/IP iteration math, quality → required_iters | +| `pos_quality.py` | ~29 | Expected plot size calculation | +| `block_rewards.py` | ~54 | Pool/farmer reward schedule (halving) | +| `coinbase.py` | ~25 | Pool/farmer coin creation from block height | +| `full_block_to_block_record.py` | ~170 | `block_to_block_record()` | +| `find_fork_point.py` | ~109 | Fork point search between two chains | +| `get_block_challenge.py` | ~170 | Challenge computation per block | +| `make_sub_epoch_summary.py` | ~250 | Sub-epoch summary creation | +| `deficit.py` | ~55 | Deficit calculation for sub-epoch boundaries | +| `multiprocess_validation.py` | ~300 | `PreValidationResult`, parallel block validation | +| `condition_tools.py` | ~200 | `pkm_pairs()` for AGG_SIG conditions | +| `signage_point.py` | ~10 | `SignagePoint` dataclass | +| `vdf_info_computation.py` | ~180 | VDF info reconstruction from block records | +| `default_constants.py` | ~119 | All consensus constant values | +| `constants.py` | ~50 | `replace_str_to_bytes()` for config overrides | + +--- + +## `Blockchain.add_block()` — Core block acceptance + +**Location**: `consensus/blockchain.py:294` + +### Purpose + +Single entry point for adding a validated block. Determines if a block +becomes the new peak, an orphan, or is rejected. + +### Inputs & assumptions + +- `block: FullBlock` — the full block to add +- `pre_validation_result: PreValidationResult` — must have valid + `required_iters` and no error (pre-validation already ran in parallel) +- `sub_slot_iters: uint64` — pre-computed for this block's epoch +- `fork_info: ForkInfo` — must correctly describe the fork context +- **Caller holds the blockchain lock** (`priority_mutex`) +- Header validation already passed via `validate_unfinished_header_block()` + +### Return + +`(AddBlockResult, Err | None, StateChangeSummary | None)` + +### Block-by-block logic + +1. **Genesis check** (L326): `height == 0` requires `prev_header_hash == +GENESIS_CHALLENGE`. + +2. **Extending main chain?** (L331): Fast path when `prev_header_hash == +peak.header_hash`. + +3. **Disconnected block** (L337-343): If prev block not in cache → + `DISCONNECTED_BLOCK`. Invariant: we only accept blocks connected to known + chain. + +4. **Pre-validation error** (L345-348): Reject immediately on any error. + +5. **ForkInfo assertions** (L354-366): Multiple assertions verify fork_info + consistency. Incorrect fork_info → assertion failure (crash, not silent + corruption). + +6. **Already-have-block** (L372-380): Even known blocks update fork_info + (important for parallel batch validation). + +7. **Body validation** (L392-403): `validate_block_body()` checks coins, + merkle roots, rewards, timelocks. + +8. **Block record creation** (L415-423): `block_to_block_record()` computes + lightweight record. + +9. **Atomic DB transaction** (L432-476): + - `add_full_block()` → `_reconsider_peak()` → `add_block_record()` + - On success: update `_peak_height` and height map + - On failure: rollback in-memory state, fork_info, block store cache + +### Invariants + +- `fork_info.peak_height == block.height - 1` before body validation +- `block.height == 0 or fork_info.peak_hash == block.prev_header_hash` +- Database transaction atomicity: no partial state updates +- `_peak_height` only updated after commit + +--- + +## `_reconsider_peak()` — Fork choice rule + +**Location**: `consensus/blockchain.py:486` + +### Fork choice criteria (in order) + +1. Higher weight wins +2. On equal weight: lower `total_iters` wins +3. Otherwise: no change (current peak stays) + +### Reorg handling + +- `coin_store.rollback_to_block(fork_info.fork_height)` removes coins above + fork point +- Replays all additions/removals from `fork_info` for the new chain +- `block_store.rollback()` clears sub-epoch summaries above fork point +- `block_store.set_in_chain()` marks new chain blocks +- `block_store.set_peak()` updates stored peak + +### Assumption + +`fork_info.additions_since_fork` and `fork_info.removals_since_fork` contain +ALL coin operations from `fork_height + 1` to the new peak. Incomplete data → +inconsistent coin store. + +--- + +## `ForkInfo` — Fork tracking state + +**Location**: `consensus/block_body_validation.py:62` + +### Fields + +- `fork_height: int` — last block shared by fork and main chain +- `peak_height: int` — height of the fork tip (-1 for genesis validation) +- `peak_hash: bytes32` +- `additions_since_fork: dict[bytes32, ForkAdd]` — all coin additions since fork +- `removals_since_fork: dict[bytes32, ForkRem]` — all coin removals since fork +- `block_hashes: list[bytes32]` — ordered header hashes from fork_height+1 + +### Critical methods + +- `reset()` — clear all fork state (used when extending main chain) +- `update_fork_peak()` — advance peak, append header hash +- `include_spends()` — record additions/removals from `SpendBundleConditions` +- `include_reward_coins()` — record coinbase additions +- `rollback()` — undo to a previous height + +### Invariant + +`len(block_hashes) == peak_height - fork_height` — always. + +--- + +## `validate_block_body()` — Block body validation + +**Location**: `consensus/block_body_validation.py:190` + +### Checks performed + +1. Non-tx blocks: foliage_transaction_block, transactions_info, generator all None +2. Tx blocks: foliage_transaction_block and transactions_info must exist +3. `transactions_info_hash` matches foliage commitment +4. `foliage_transaction_block_hash` matches foliage commitment +5. Reward claims valid (pool + farmer coins for all blocks since last tx block) +6. Previous transaction block reference is correct +7. Timestamp: `> prev_tx_block_timestamp` and `< max_future_time` +8. Transaction filter matches additions/removals +9. Generator cost ≤ `MAX_BLOCK_COST_CLVM` +10. Generator ref list size ≤ `MAX_GENERATOR_REF_LIST_SIZE` (512) +11. Merkle roots (additions and removals) match +12. `check_time_locks()` (Rust) validates absolute/relative height/seconds +13. Fees = `sum(removals) - sum(additions)` matches declared fees +14. Coins not double-spent (checked against fork_info and coin store) +15. Additions don't collide with existing coins + +--- + +## Difficulty adjustment + +**Location**: `consensus/difficulty_adjustment.py` + +### Key function: `get_next_sub_slot_iters_and_difficulty()` + +Called at epoch boundaries (every `EPOCH_BLOCKS = 4608` blocks). + +### Algorithm + +1. Find second-to-last transaction block in previous epoch +2. Compute elapsed time between reference points +3. New difficulty = `old_difficulty × target_time / actual_time` +4. Clamp to `[old / DIFFICULTY_CHANGE_MAX_FACTOR, old × DIFFICULTY_CHANGE_MAX_FACTOR]` + (factor = 3) +5. Truncate to `SIGNIFICANT_BITS` (8) + +### Same logic applies to sub-slot iterations (SSI) + +### Invariant + +Difficulty and SSI can change at most 3× per epoch. + +--- + +## VDF iteration math + +**Location**: `consensus/pot_iterations.py` + +### `calculate_iterations_quality(quality_string, size, difficulty, cc_sp_output_hash)` + +``` +sp_quality = hash(quality_string + cc_sp_output_hash) +iters = difficulty × DIFFICULTY_CONSTANT_FACTOR × sp_quality_int / (2^256 × expected_plot_size) +return max(iters, 1) +``` + +### `calculate_ip_iters(sub_slot_iters, signage_point_index, required_iters)` + +``` +ip_iters = (sp_iters + NUM_SP_INTERVALS_EXTRA × sp_interval_iters + required_iters) % sub_slot_iters +``` + +### `is_overflow_block(signage_point_index)` + +``` +overflow = signage_point_index >= NUM_SPS_SUB_SLOT - NUM_SP_INTERVALS_EXTRA +``` + +i.e., the last 3 signage points of a sub-slot are overflow blocks. + +### Constraints + +- `required_iters ∈ (0, sp_interval_iters)` +- `signage_point_index < NUM_SPS_SUB_SLOT` (64) +- `sub_slot_iters % NUM_SPS_SUB_SLOT == 0` + +--- + +## Block rewards + +**Location**: `consensus/block_rewards.py` + +### Schedule (pool = 7/8, farmer = 1/8 + fees) + +| Period | Per-block reward | +| ------------------- | ---------------- | +| Height 0 (pre-farm) | 21 000 000 XCH | +| Years 0–3 | 2 XCH | +| Years 3–6 | 1 XCH | +| Years 6–9 | 0.5 XCH | +| Years 9–12 | 0.25 XCH | +| Year 12+ | 0.125 XCH | + +`_blocks_per_year = 1 681 920` (32 × 6 × 24 × 365) + +### Coinbase parent IDs + +- Pool: `genesis_challenge[:16] + height.to_bytes(16)` +- Farmer: `genesis_challenge[16:] + height.to_bytes(16)` + +These are deterministic, not hashed. + +--- + +## `AugmentedBlockchain` — Parallel validation overlay + +**Location**: `consensus/augmented_chain.py` + +### Purpose + +Wraps a `BlocksProtocol` with an in-memory cache of extra blocks. Used during +parallel batch validation: blocks in the batch aren't committed to the DB until +all pass, but subsequent blocks in the batch need to reference earlier ones. + +### Key invariant + +Extra blocks must form a contiguous chain. `add_extra_block()` validates that +each new block's `prev_hash` matches the last added block. + +### Generator ref resolution + +`lookup_block_generators()` first checks extra blocks (walking backward via +`prev_header_hash`), then falls through to the underlying blockchain. diff --git a/.cursor/context/full-node.md b/.cursor/context/full-node.md new file mode 100644 index 0000000000..64f5c8867c --- /dev/null +++ b/.cursor/context/full-node.md @@ -0,0 +1,249 @@ +# Full Node Orchestration — Deep Context + +> Attach when touching `chia/full_node/full_node.py`, `full_node_api.py`, +> `full_node_store.py`, `full_node_rpc_api.py`, sync logic, or block +> processing pipeline. + +## File map + +| File | Lines | Role | +| -------------------------- | ----- | -------------------------------------------------------- | +| `full_node.py` | ~3400 | `FullNode`: main orchestrator, sync, block/tx processing | +| `full_node_api.py` | ~2080 | `FullNodeAPI`: all P2P message handlers | +| `full_node_rpc_api.py` | ~1170 | `FullNodeRpcApi`: HTTP/WS RPC endpoints | +| `full_node_rpc_client.py` | ~380 | RPC client (used by CLI and tests) | +| `full_node_store.py` | ~1060 | `FullNodeStore`: signage points, unfinished blocks | +| `full_node_service.py` | ~10 | Service registration | +| `start_full_node.py` | ~120 | Service startup config | +| `block_store.py` | ~700 | `BlockStore`: SQLite full block persistence | +| `coin_store.py` | ~680 | `CoinStore`: UTXO database | +| `sync_store.py` | ~140 | `SyncStore`: sync state tracking | +| `weight_proof.py` | ~1740 | `WeightProofHandler`: weight proof creation/validation | +| `subscriptions.py` | ~240 | `PeerSubscriptions`: wallet coin/puzzle subscriptions | +| `hint_store.py` | ~100 | `HintStore`: hint persistence | +| `hint_management.py` | ~60 | Hint processing from conditions | +| `tx_processing_queue.py` | ~250 | `TransactionQueue`: async tx processing | +| `check_fork_next_block.py` | ~40 | Fork-next-block utility | +| `hard_fork_utils.py` | ~55 | Hard fork flag computation | +| `full_block_utils.py` | ~370 | Block ↔ header block conversion | +| `bundle_tools.py` | ~20 | SpendBundle utilities | + +--- + +## `FullNode` — Main orchestrator + +**Location**: `full_node/full_node.py` + +### Key state + +- `blockchain: Blockchain` — chain state + UTXO +- `mempool_manager: MempoolManager` — transaction pool +- `full_node_store: FullNodeStore` — signage points, unfinished blocks +- `sync_store: SyncStore` — sync state +- `full_node_peers: FullNodePeers` — peer discovery +- `weight_proof_handler: WeightProofHandler` — weight proof logic +- `subscriptions: PeerSubscriptions` — wallet subscriptions +- `_transaction_queue: TransactionQueue` — async tx processing +- `server: ChiaServer` — networking + +### Key dataclass: `PeakPostProcessingResult` + +After a new peak is accepted: + +- `mempool_peak_added_tx_ids` — transactions re-added +- `mempool_removals` — transactions removed +- `fns_peak_result` — signage points and infusion points +- `hints` — new hints for wallet notifications +- `lookup_coin_ids` — coins to look up for wallet updates +- `signage_points` — signage points to forward to farmers after new peak + +--- + +## Block processing pipeline + +### 1. Receive block + +`FullNodeAPI.respond_block()` / `FullNodeAPI.respond_blocks()` receive blocks +from peers. + +### 2. Pre-validate + +`pre_validate_block()` runs header validation + CLVM execution in parallel +(thread pool). Returns `PreValidationResult` with `required_iters` and +`conds`. + +### 3. Add to blockchain + +Under `blockchain.priority_mutex` (high priority): + +- `Blockchain.add_block()` validates body, updates DB, reorgs if needed +- Returns `(AddBlockResult, Err, StateChangeSummary)` + +### 4. Post-processing (peak_post_processing) + +If `NEW_PEAK`: + +- Update `FullNodeStore` with new signage points +- Update `MempoolManager` with `new_peak()` +- Process hints and subscriptions +- Compute wallet notifications + +### 5. Broadcast + +- Send `new_peak` to full node peers +- Send `new_peak_wallet` to wallet peers +- Send coin state updates to subscribed wallets +- Forward new signage points to farmer + +--- + +## `FullNodeStore` — Signage point & unfinished block tracking + +**Location**: `full_node/full_node_store.py` + +### Key state + +- Signage points per challenge hash (LRU-bounded) +- End-of-sub-slot bundles per challenge hash +- Unfinished blocks indexed by `(reward_hash, foliage_hash)` +- Peers that advertised each transaction (`peers_with_tx`) +- Seen compact VDFs (dedup set) + +### Constants + +- `MAX_UNFINISHED_BLOCKS_PER_REWARD_HASH = 20` — eviction of worst foliage + +### `new_peak()` returns + +- `added_eos`: any end-of-sub-slot that becomes relevant +- `new_signage_points`: signage points that can now be released +- `new_infusion_points`: infusion points for timelord + +--- + +## Sync logic + +### Weight proof sync + +1. Peer announces `new_peak` with higher weight +2. Request `request_proof_of_weight` → `WeightProof` +3. Validate weight proof (sub-epoch summaries, VDF segments) +4. If valid: switch to batch download + +### Batch sync + +1. Download blocks in ranges via `request_blocks` (max 32 per request) +2. Pre-validate batches in parallel +3. Add blocks sequentially under blockchain lock +4. Continue until caught up to peer's peak + +### Long sync detection + +If peer peak is significantly ahead, enters long sync mode. During long sync, +transactions are not processed (mempool frozen). + +--- + +## `FullNodeAPI` — P2P message handlers + +**Location**: `full_node/full_node_api.py` + +### Key handlers + +| Handler | Trigger | Notes | +| ---------------------------------------------------- | ------------------------ | ---------------------------------------- | +| `new_peak()` | Peer has new peak | Triggers sync if heavier | +| `new_transaction()` | Peer has new tx | Adds to `peers_with_tx`, schedules fetch | +| `request_transaction()` | Peer wants a tx | Look up in mempool | +| `respond_transaction()` | Received requested tx | Pre-validate + add to mempool | +| `send_transaction()` | Wallet submits tx | Pre-validate + add to mempool | +| `respond_block()` | Received single block | Add to blockchain | +| `respond_blocks()` | Received block batch | Add batch to blockchain | +| `new_signage_point_or_end_of_sub_slot()` | New SP/EOS | Store + broadcast | +| `new_unfinished_block()` / `new_unfinished_block2()` | Farmer block | Validate + infuse | +| `request_compact_vdf()` | Peer wants compact proof | Look up + respond | + +### Transaction processing + +`respond_transaction()` and `send_transaction()` both: + +1. Check `seen_bundle_hashes` for dedup +2. Run `pre_validate_spendbundle()` in thread pool +3. Acquire blockchain lock (low priority) +4. Call `add_spend_bundle()` +5. On success: broadcast `new_transaction` to peers + +--- + +## `FullNodeRpcApi` — RPC endpoints + +**Location**: `full_node/full_node_rpc_api.py` + +### Key endpoints + +- `get_blockchain_state` — peak, sync status, mempool info, space estimate +- `get_block` / `get_blocks` — fetch by height or hash +- `get_block_record` / `get_block_records` — lightweight records +- `get_coin_record_by_name` — single UTXO lookup +- `get_coin_records_by_*` — batch lookups by puzzle hash, parent, hint +- `push_tx` — submit transaction (same as `send_transaction` P2P) +- `get_mempool_item_by_tx_id` — mempool query +- `get_fee_estimate` — fee estimation +- `get_network_space` — estimated network space + +--- + +## `CoinStore` — UTXO database + +**Location**: `full_node/coin_store.py` + +### Schema + +```sql +coin_record( + coin_name BLOB PRIMARY KEY, + confirmed_index BIGINT, + spent_index BIGINT, -- >0 spent at that height; 0 = normal unspent; -1 = FF lineage unspent + coinbase INT, + puzzle_hash BLOB, + coin_parent BLOB, + amount BLOB, -- 8-byte uint64 + timestamp BIGINT +) +``` + +### Indexes + +- `coin_confirmed_index` — reorg rollbacks +- `coin_spent_index` — spent coin queries +- `coin_puzzle_hash` — address lookups +- `coin_parent_index` — parent traversal +- `coin_record_ph_ff_unspent_idx` (partial, new DBs only) — FF singleton optimization + +### Key operations + +- `new_block()` — batch insert additions, mark removals as spent +- `rollback_to_block()` — revert coins confirmed/spent above a height +- `get_coin_records()` — fetch by coin IDs +- `get_coin_records_by_puzzle_hash()` — wallet queries +- `get_unspent_lineage_info_for_puzzle_hash()` — FF singleton lineage + +--- + +## `BlockStore` — Full block persistence + +**Location**: `full_node/block_store.py` + +### Schema + +- `full_blocks` table: header_hash, height, in_main_chain flag, block_record, full block bytes +- `sub_epoch_segments_v3` table: sub-epoch challenge segments for weight proofs + +### Key operations + +- `add_full_block()` — insert with block record +- `get_full_block()` / `get_full_blocks_at()` — fetch +- `set_in_chain()` — mark blocks as main chain +- `set_peak()` — update peak pointer +- `rollback()` — clear in_chain and sub-epoch data above height +- Transaction support via `self.db_wrapper.writer_maybe_transaction()` diff --git a/.cursor/context/global-invariants.md b/.cursor/context/global-invariants.md new file mode 100644 index 0000000000..e3dc672c59 --- /dev/null +++ b/.cursor/context/global-invariants.md @@ -0,0 +1,226 @@ +# Global Invariants, Workflows & Trust Boundaries + +> Attach for cross-cutting changes, security review, reorg-related work, +> or when your change touches multiple subsystems. + +--- + +## Global invariants + +### 1. Weight monotonicity (fork choice) + +Peak always has the heaviest weight. Equal weight resolves by lower +`total_iters`. This is enforced in `Blockchain._reconsider_peak()`. + +### 2. Coin uniqueness + +Each coin ID exists at most once in the UTXO set. Double-spends are rejected +at both mempool admission and block validation. + +### 3. Conservation of value + +`sum(removals) >= sum(additions)` for every transaction. The difference is +fees. Enforced in `validate_block_body()` and `MempoolManager.validate_spend_bundle()`. + +### 4. Block cost bound + +Every block's CLVM cost ≤ `MAX_BLOCK_COST_CLVM` (11 000 000 000). +Every mempool item's cost ≤ `MAX_BLOCK_COST_CLVM / 2`. + +### 5. Timestamp ordering + +For transaction blocks, timestamps must be: + +- Strictly greater than the previous transaction block timestamp +- At most `now + MAX_FUTURE_TIME2` (120 seconds) + +### 6. Difficulty bounds + +Next difficulty is clamped to `[prev / DIFFICULTY_CHANGE_MAX_FACTOR, prev × DIFFICULTY_CHANGE_MAX_FACTOR]` +where factor = 3. Same applies to sub-slot iterations. + +### 7. Reward schedule + +Pool gets 7/8 of block reward, farmer gets 1/8 + fees. +Halving every 3 years (~1 681 920 blocks). Pre-farm at height 0. + +### 8. Signage point ordering + +`signage_point_index < NUM_SPS_SUB_SLOT` (64). Overflow blocks use the +last `NUM_SP_INTERVALS_EXTRA` (3) indices. + +### 9. Sub-slot iteration divisibility + +`sub_slot_iters % NUM_SPS_SUB_SLOT == 0` — always. Enforced by the +starting value and adjustment algorithm. + +### 10. Fork info consistency + +During block validation: + +- `fork_info.peak_height == block.height - 1` +- `block.height == 0 or fork_info.peak_hash == block.prev_header_hash` +- `len(fork_info.block_hashes) == fork_info.peak_height - fork_info.fork_height` + +--- + +## Cross-module state dependencies + +| State | Written by | Read by | Consistency rule | +| -------------------- | ----------------------------------- | --------------------------------------------------- | ------------------------------------------ | +| `coin_record` table | `Blockchain._reconsider_peak()` | `MempoolManager`, `FullNodeRpcApi`, wallet protocol | Matches current peak chain | +| `_peak_height` | `Blockchain.add_block()` | All full node components | Only updated after DB commit | +| `mempool._items` | `MempoolManager.add_spend_bundle()` | Block creation, RPC, TX relay | All items valid at current peak | +| `fork_info` | `Blockchain.add_block()` | `validate_block_body()` | Contains all adds/removes since fork point | +| `seen_bundle_hashes` | `MempoolManager` | `FullNodeAPI.new_transaction()` | Prevents re-processing | +| `block_store.peak` | `block_store.set_peak()` | `Blockchain._load_chain_from_store()` | Matches `_peak_height` | +| `height_map` | `Blockchain.add_block()` | Height lookups throughout | Matches chain up to peak | +| `PeerSubscriptions` | `FullNodeAPI` register handlers | `FullNode.peak_post_processing_2()` | Wallet notifications | + +--- + +## Trust boundary map + +| Boundary | Trust level | Protection | +| ------------------------------------- | --------------------------- | ----------------------------------------------------------------------------------- | +| P2P messages from peers | **Untrusted** | Streamable deserialization, rate limiting, protocol state machine, ban on violation | +| RPC from localhost | **Semi-trusted** | TLS client cert required, inputs validated | +| Farmer → Full Node | **Semi-trusted** | Proofs cryptographically verified, signatures validated | +| Harvester → Farmer | **Trusted** (same operator) | Minimal validation | +| CLVM execution (arbitrary puzzles) | **Untrusted** | Sandboxed in Rust, cost-metered, atom/pair count bounded | +| Block generators | **Untrusted** | Cost limits, ref list size capped (512) | +| Weight proofs | **Untrusted** | Full VDF verification, sub-epoch summary validation | +| Wallet → Full Node (send_transaction) | **Untrusted** | Full mempool validation pipeline | + +--- + +## Key workflow traces + +### Block production + +``` +Timelord → new_signage_point_vdf → FullNode +FullNode → new_signage_point → Farmer +Farmer → new_signage_point_harvester → Harvester +Harvester → new_proof_of_space → Farmer +Farmer → declare_proof_of_space → FullNode +FullNode creates unfinished block (mempool txs included) +Timelord → new_infusion_point_vdf → FullNode +FullNode creates finished block → add_block() → broadcasts new_peak +``` + +### Transaction lifecycle + +``` +Wallet → send_transaction → FullNode +FullNode: pre_validate_spendbundle() [thread pool, CLVM + BLS] +FullNode: add_spend_bundle() [under blockchain lock] + → validate_spend_bundle() → check coins, fees, conflicts, timelocks + → if SUCCESS: add to mempool, broadcast new_transaction + → if PENDING: add to conflict/pending cache + → if FAILED: return error + +Peer receives new_transaction → request_transaction → respond_transaction + → same validation pipeline + +At block creation: + mempool.create_block_generator2() → ordered by fee/cost → block generator +``` + +### Sync + +``` +Peer → new_peak (heavier) → FullNode +FullNode → request_proof_of_weight → Peer → respond_proof_of_weight +FullNode validates weight proof +FullNode → request_blocks (batches of 32) → Peer → respond_blocks +FullNode: pre_validate batches [parallel] +FullNode: add_block() [sequential, under lock] +Repeat until caught up +``` + +### Reorg + +``` +Receive block on fork with higher weight +Blockchain._reconsider_peak() detects weight > current peak +coin_store.rollback_to_block(fork_height) +Replay additions/removals from fork_info +Update peak, height map, block store +MempoolManager.new_peak() — re-validate mempool items +Broadcast new_peak to peers and wallets +``` + +--- + +## Concurrency model + +### Blockchain lock (`priority_mutex`) + +- **High priority**: Block validation and addition +- **Low priority**: Transaction processing (mempool) +- Blocks are never starved by transactions + +### Thread pools + +- `Blockchain.pool`: Block validation (CLVM execution) +- `MempoolManager.pool`: Spend bundle validation (2 workers) +- Both use `ThreadPoolExecutor` + +### Async coordination + +- `asyncio.Lock` for compact proof dedup +- `TransactionQueue` for async transaction processing +- `LimitedSemaphore` for concurrent block requests + +--- + +## Fragility clusters + +### 1. `FullNode` (~3400 lines) + +Massive orchestration class. Sync, block processing, transaction handling, +peer management all interleaved. High coupling, hard to reason about +independently. + +### 2. Fork handling in `ForkInfo` + +Complex state tracking. `include_spends()` / `include_block()` / `rollback()` +must be called in exact sequence. Assertion-heavy (crashes on inconsistency). + +### 3. Mempool FF/DEDUP logic + +`eligible_coin_spends.py` + `check_removals()` + `new_peak()` FF rebase. +Multiple code paths for singleton chaining with subtle conflict resolution +rules. + +### 4. Weight proof validation (~1740 lines) + +Dense validation of compressed chain proofs. Many edge cases around +sub-epoch boundaries, VDF segment matching, and difficulty transitions. + +### 5. Block header validation (~1060 lines) + +~30+ numbered checks with complex interdependencies. VDF validation, +signage point verification, challenge computation. Ordering matters. + +### 6. Difficulty adjustment + +Complex epoch/sub-epoch boundary detection with lookback across multiple +block types (transaction vs non-transaction). + +--- + +## Hard fork boundaries + +| Fork | Height | What changed | +| ------------------------ | ---------- | ------------------------------------------------------------------ | +| `HARD_FORK_HEIGHT` | 5 496 000 | June 2024 — condition set changes, CLVM flags | +| `HARD_FORK2_HEIGHT` | 0xFFFFFFFA | Placeholder sentinel for v2 plots; real height is network-specific | +| `SOFT_FORK8_HEIGHT` | 8 655 000 | Soft fork conditions | +| `PLOT_FILTER_128_HEIGHT` | 10 542 000 | June 2027 — plot filter reduction | +| `PLOT_FILTER_64_HEIGHT` | 15 592 000 | June 2030 | +| `PLOT_FILTER_32_HEIGHT` | 20 643 000 | June 2033 | + +Heights are checked via `get_flags_for_height_and_constants()` which returns +the appropriate flag set for CLVM execution at a given height. diff --git a/.cursor/context/mempool.md b/.cursor/context/mempool.md new file mode 100644 index 0000000000..a10c348813 --- /dev/null +++ b/.cursor/context/mempool.md @@ -0,0 +1,213 @@ +# Mempool Subsystem — Deep Context + +> Attach when touching `chia/full_node/mempool*.py`, +> `eligible_coin_spends.py`, `pending_tx_cache.py`, or fee estimation. + +## File map + +| File | Lines | Role | +| ---------------------------- | ----- | -------------------------------------------------------------------- | +| `mempool_manager.py` | ~1160 | `MempoolManager`: admission, validation, new_peak handling | +| `mempool.py` | ~810 | `Mempool`: in-memory SQLite data structure, block generator creation | +| `eligible_coin_spends.py` | ~290 | Fast-forward and dedup singleton logic | +| `pending_tx_cache.py` | ~100 | `ConflictTxCache`, `PendingTxCache` for deferred items | +| `bitcoin_fee_estimator.py` | ~100 | Bitcoin-style fee estimation adapter | +| `fee_estimation.py` | ~80 | `MempoolInfo`, `FeeBlockInfo`, `MempoolItemInfo` | +| `fee_estimator.py` | ~110 | Fee estimator implementation | +| `fee_estimator_interface.py` | ~40 | `FeeEstimatorInterface` protocol | +| `fee_tracker.py` | ~600 | `FeeTracker`: fee bucket tracking | +| `fee_history.py` | ~20 | Fee history data | +| `fee_estimator_constants.py` | ~30 | Estimator tuning constants | +| `fee_estimate_store.py` | ~15 | Persistence (minimal) | + +## Related types + +- `chia/types/mempool_item.py` — `MempoolItem`, `BundleCoinSpend`, `UnspentLineageInfo` +- `chia/types/internal_mempool_item.py` — `InternalMempoolItem` (signature separated) +- `chia/types/mempool_inclusion_status.py` — `SUCCESS`, `FAILED`, `PENDING` +- `chia/types/fee_rate.py` — `FeeRate` +- `chia/types/clvm_cost.py` — `CLVMCost`, `QUOTE_BYTES`, `QUOTE_EXECUTION_COST` + +--- + +## Key constants + +| Constant | Value | Meaning | +| -------------------------- | ------------------------- | ------------------------------------------------------ | +| `MEMPOOL_BLOCK_BUFFER` | 10 | Mempool capacity = 10× max block cost | +| `MEMPOOL_MIN_FEE_INCREASE` | 10 000 000 | Min fee increase for replacement (0.00001 XCH) | +| `MEMPOOL_ITEM_FEE_LIMIT` | 2^50 | Max fee per item (prevents SQLite int64 overflow) | +| `nonzero_fee_minimum_fpc` | 5 | Min fee-per-cost to kick out others (~0.055 XCH/block) | +| `max_tx_clvm_cost` | `MAX_BLOCK_COST_CLVM / 2` | Single tx cost limit | +| `MAX_SKIPPED_ITEMS` | 10 | Max items skipped during block building | +| `PRIORITY_TX_THRESHOLD` | 3 | FF/DEDUP items allowed before cutoff | +| `MIN_COST_THRESHOLD` | 6 000 000 | Heuristic for block fullness | +| `seen_cache_size` | 10 000 | Seen spend bundle hash cache | + +--- + +## `MempoolManager.validate_spend_bundle()` — Admission pipeline + +**Location**: `mempool_manager.py:596` + +### Pipeline (in order) + +1. **Peak check**: mempool must be initialized + +2. **Coin spend processing** (per spend): + + - Track removal names, addition amounts + - DEDUP eligibility requires canonical CLVM serialization + - FF eligibility queries `get_unspent_lineage_info_for_puzzle_hash` + - Builds `BundleCoinSpend` per coin + +3. **FF-only rejection**: Bundles with ONLY FF spends are invalid. FF spends + must be bundled with at least one normal spend. + +4. **Coin record lookup**: Fetch from DB. Ephemeral coins (created + spent in + same bundle) get synthetic records: `confirmed_index = peak.height + 1`, + `timestamp = peak.timestamp`. + +5. **Fee = removal_amount − addition_amount** + +6. **Cost/fee limits**: + + - `cost > max_tx_clvm_cost` → reject + - `fees > MEMPOOL_ITEM_FEE_LIMIT` or would overflow → reject + +7. **Capacity check**: If mempool full: + + - `fee_per_cost < nonzero_fee_minimum_fpc (5)` → reject + - `fee_per_cost ≤ min_fee_rate` → reject + +8. **Conflict detection** via `check_removals()`: + + - Already-spent (non-FF) → `DOUBLE_SPEND` + - Mempool collision → `MEMPOOL_CONFLICT` (may be resolvable) + +9. **Puzzle hash match**: Revealed puzzle hash must match coin record + +10. **Timelock validation**: `check_time_locks()` (Rust) against peak + height/timestamp + +11. **Impossible constraints**: `assert_before_height ≤ assert_height` → reject permanently + +12. **Duration guard**: >2s validation time → reject (DoS protection) + +### Return semantics + +- `(None, item, conflicts)` → immediate add, remove conflicts +- `(MEMPOOL_CONFLICT, item, [])` → store in conflict cache for retry +- `(ASSERT_HEIGHT_*, item, [])` → store in pending cache for retry +- `(err, None, [])` → permanent failure + +--- + +## `check_removals()` — Conflict detection + +**Location**: `mempool_manager.py:229` + +### Logic per coin + +1. **Spent + non-FF** → `DOUBLE_SPEND` (immediate reject) +2. **In mempool**: look up conflicting items by coin ID + - Both FF → can chain, no conflict + - Both DEDUP + same solution → can merge, no conflict + - Otherwise → `MEMPOOL_CONFLICT` +3. Handles edge case of FF spends indexed under latest singleton coin ID + +--- + +## `MempoolManager.add_spend_bundle()` — Admission + conflict resolution + +**Location**: `mempool_manager.py:525` + +### Flow + +1. Skip if already in mempool (idempotent) +2. Call `validate_spend_bundle()` +3. On success: remove conflicts, add to mempool, return `SUCCESS` +4. On `MEMPOOL_CONFLICT`: add to `_conflict_cache`, return `PENDING` +5. On height-not-met: add to `_pending_cache`, return `PENDING` +6. Otherwise: return `FAILED` + +--- + +## `Mempool` data structure + +**Location**: `mempool.py:85` + +### Storage + +- In-memory SQLite database with table `tx`: + `name`, `cost`, `fee`, `assert_height`, `assert_before_height`, + `assert_before_seconds`, `fee_per_cost`, `seq` +- `_items: dict[bytes32, InternalMempoolItem]` — full item data (signatures + kept separate from SQLite for perf) +- Additional SQLite tables: `spends` (coin_id → item name mapping) + +### Key operations + +- `add_to_pool()` — insert, evict lowest-fee items if over capacity +- `remove_from_pool()` — remove by item names +- `new_tx_block()` — advance height/timestamp, expire items with + `assert_before_height`/`assert_before_seconds` violations +- `get_min_fee_rate()` — lowest fee/cost in pool (for admission threshold) +- `at_full_capacity()` — `total_cost + new_cost > mempool_max_total_cost` + +### Block building (`create_block_generator2()`) + +1. Query items ordered by `fee_per_cost DESC, seq ASC` +2. Process FF/DEDUP info and build transaction batches +3. Try batches through `BlockBuilder.add_spend_bundles()` to account for real compression cost +4. Keep accepted batches and skip non-fitting batches +5. Stop on timeout, or when `BlockBuilder` reports the block is full + +--- + +## `MempoolManager.new_peak()` — Reorg/new-block handling + +**Location**: `mempool_manager.py:845` + +### Optimization path (simple chain extension) + +When `new_peak.prev_transaction_block_hash == self.peak.header_hash`: + +1. Expire items violating new height/timestamp constraints +2. Find mempool items spending coins that were just spent on-chain +3. For regular spends: remove (they're included) +4. For FF spends: attempt to rebase to new singleton version +5. Re-add items from conflict/pending caches + +### Full reinit path (reorg) + +All mempool items re-validated against new chain state. Failed items removed. + +--- + +## Fast-forward (FF) and dedup logic + +**Location**: `eligible_coin_spends.py` + +### Fast-forward singletons + +FF-eligible spends can be "rebased" when the singleton they reference gets +spent. The mempool updates the coin spend to point to the latest unspent +singleton version. + +**Key function**: `perform_the_fast_forward()` — replaces the coin in a +`CoinSpend` with the latest unspent version, preserving the puzzle and +solution. + +**Tracking**: `UnspentLineageInfo` stores `coin_id`, `parent_id`, +`parent_parent_id` for the latest unspent singleton. + +### Dedup + +DEDUP-eligible spends with identical solutions can coexist in the mempool. +During block building, they're merged via `IdenticalSpendDedup`. + +### Invariant + +A bundle cannot contain ONLY FF spends. At least one normal spend is required +to ensure the bundle can eventually be invalidated. diff --git a/.cursor/context/networking.md b/.cursor/context/networking.md new file mode 100644 index 0000000000..b7b8dc6960 --- /dev/null +++ b/.cursor/context/networking.md @@ -0,0 +1,244 @@ +# Networking & Peer Protocol — Deep Context + +> Attach when touching `chia/server/`, `chia/protocols/`, connection handling, +> rate limiting, or peer discovery. + +## File map + +### `chia/server/` + +| File | Lines | Role | +| -------------------------- | ----- | ---------------------------------------------------- | +| `server.py` | ~900 | `ChiaServer`: connection management, message routing | +| `ws_connection.py` | ~790 | `WSChiaConnection`: single peer connection | +| `rate_limits.py` | ~160 | `RateLimiter`: per-connection rate enforcement | +| `rate_limit_numbers.py` | ~200 | Rate limit values per message type | +| `chia_policy.py` | ~370 | Custom asyncio event loop policy, connection limits | +| `node_discovery.py` | ~850 | `FullNodePeers`: peer discovery and management | +| `address_manager.py` | ~1050 | Address book for known peers | +| `address_manager_store.py` | ~15 | Address persistence | +| `api_protocol.py` | ~117 | `ApiProtocol`, `ApiMetadata`, `@request` decorator | +| `capabilities.py` | ~20 | Capability detection | +| `introducer_peers.py` | ~65 | Introducer peer handling | +| `resolve_peer_info.py` | ~50 | DNS resolution | +| `start_service.py` | ~350 | Service lifecycle management | +| `signal_handlers.py` | ~100 | Graceful shutdown | +| `ssl_context.py` | ~25 | TLS configuration | +| `upnp.py` | ~100 | UPnP port forwarding | + +### `chia/protocols/` + +| File | Lines | Role | +| --------------------------------------- | ----- | ----------------------------------------------- | +| `protocol_message_types.py` | ~147 | `ProtocolMessageTypes` enum (109 message types) | +| `protocol_state_machine.py` | ~88 | Valid request→response map, import-time check | +| `protocol_message_type_to_node_type.py` | ~230 | Message type → allowed node type mapping | +| `outbound_message.py` | ~25 | `Message`, `NodeType`, `make_msg()` | +| `protocol_timing.py` | ~10 | Ban duration constants | +| `shared_protocol.py` | ~80 | `Handshake`, `Capability`, `protocol_version` | +| `full_node_protocol.py` | ~217 | Full node ↔ full node message types | +| `wallet_protocol.py` | ~400 | Wallet ↔ full node message types | +| `farmer_protocol.py` | ~75 | Farmer ↔ full node message types | +| `harvester_protocol.py` | ~190 | Farmer ↔ harvester message types | +| `timelord_protocol.py` | ~80 | Full node ↔ timelord message types | +| `solver_protocol.py` | ~15 | Solver protocol | +| `pool_protocol.py` | ~110 | Pool protocol messages | +| `introducer_protocol.py` | ~15 | Introducer protocol | +| `fee_estimate.py` | ~50 | Fee estimate messages | + +--- + +## `WSChiaConnection` — Per-peer connection + +**Location**: `server/ws_connection.py` + +### Key properties + +- WebSocket-based (aiohttp) +- Mutual TLS authentication +- `local_type: NodeType` — our node type +- `peer_node_id: bytes32` — peer identity +- `is_outbound: bool` +- `peer_capabilities: list[Capability]` + +### Constants + +- `LENGTH_BYTES = 4` — message length prefix (max ~4 GiB per message) +- `MAX_VERSION_STRING_BYTES = 128` +- `MAX_PENDING_COMPACT_VDFS = 100` + +### Message flow + +1. Receive raw bytes over WebSocket +2. Parse length prefix + `Message` (type + id + data) +3. Look up handler in `ApiMetadata.message_type_to_request` +4. Deserialize data via `Streamable.from_bytes()` +5. Call handler, send reply if expected + +### Error handling & banning + +- `ApiError` from handlers is converted to an `error` response (not an automatic ban) +- `ConsensusError` from handlers → close + ban for `CONSENSUS_ERROR_BAN_SECONDS` +- Other unhandled handler exceptions → close + ban for `API_EXCEPTION_BAN_SECONDS` +- Rate limit exceeded (for full node inbound peers) → close + ban for `RATE_LIMITER_BAN_SECONDS` +- Protocol response mismatch (`message_response_ok` failure) → `ban_peer_bad_protocol()` using `INTERNAL_PROTOCOL_ERROR_BAN_SECONDS` + +### Protocol state machine validation + +On receiving a reply, `message_response_ok()` checks that the response type is +valid for the original request type (defined in `VALID_REPLY_MESSAGE_MAP`). + +--- + +## Rate limiting + +**Location**: `server/rate_limits.py`, `server/rate_limit_numbers.py` + +### Two-tier system + +**Per-message-type limits** (`RLSettings`): + +- `frequency`: max count per 60 seconds +- `max_size`: max bytes per single message +- `max_total_size`: max cumulative bytes per 60 seconds (optional) +- `aggregate_limit`: whether to count against the global aggregate + +**Aggregate limit** (across all non-tx message types): + +- 1000 messages per minute +- 100 MB per minute + +### Transaction messages exempt from aggregate + +`new_transaction`, `request_transaction`, `respond_transaction`, +`send_transaction`, `transaction_ack` — these have their own per-type limits +and do NOT count against the aggregate. + +### Key rate limits (v1) + +| Message | Freq/min | Size | Total/min | +| ------------------------- | -------- | ------ | --------- | +| `new_transaction` | 5000 | 100 B | 500 KB | +| `respond_transaction` | 5000 | 1 MB | 20 MB | +| `send_transaction` | 5000 | 1 MB | — | +| `respond_blocks` | 100 | 50 MB | — | +| `respond_proof_of_weight` | 5 | 400 MB | — | +| `new_peak` | 200 | 512 B | — | +| `request_block` | 200 | 100 B | — | +| `request_blocks` | 100 | 100 B | — | + +### V2 rate limits + +Activated when both peers have `Capability.RATE_LIMITS_V2`. Overrides/extends +v1 with additional message types (wallet sync, mempool updates, etc.). + +### `Unlimited` message types + +Some response messages use `Unlimited` instead of `RLSettings` — they have a +per-message size limit but no frequency limit and are exempt from aggregate. + +--- + +## Protocol state machine + +**Location**: `protocols/protocol_state_machine.py` + +### `VALID_REPLY_MESSAGE_MAP` + +Maps request types to valid response types. Examples: + +- `request_block` → `[respond_block, reject_block]` +- `request_blocks` → `[respond_blocks, reject_blocks]` +- `send_transaction` → `[transaction_ack]` +- `request_puzzle_state` → `[respond_puzzle_state, reject_puzzle_state]` + +### `NO_REPLY_EXPECTED` + +Fire-and-forget messages: `new_peak`, `new_transaction`, +`new_unfinished_block`, `new_signage_point_or_end_of_sub_slot`, +`request_mempool_transactions`, `new_compact_vdf`, `coin_state_update`, +`mempool_items_added`, `mempool_items_removed`. + +### Import-time check + +`static_check_sent_message_response()` verifies NO_REPLY_EXPECTED and +VALID_REPLY_MESSAGE_MAP don't overlap. Runs at module import. + +--- + +## `ApiProtocol` and `@request` decorator + +**Location**: `server/api_protocol.py` + +### Handler registration + +Each API class (e.g., `FullNodeAPI`) has a class-level `ApiMetadata` that maps +`ProtocolMessageTypes` → `ApiRequest`. + +The `@metadata.request()` decorator: + +- Registers the handler for its message type +- Auto-deserializes `bytes` → `Streamable` subclass +- Optionally passes raw bytes and/or peer reference +- `execute_task=True` means handler runs as a separate asyncio task + +### Key flags + +- `peer_required=True` — handler receives `WSChiaConnection` as parameter +- `bytes_required=True` — handler receives raw bytes (for forwarding) +- `execute_task=True` — non-blocking execution + +--- + +## Connection limits + +**Location**: `server/chia_policy.py` + +- Default `global_max_concurrent_connections = 250` +- `set_chia_policy(connection_limit)` sets effective limit to `connection_limit + 100` +- Custom event loop policy (`ChiaProactorEventLoop` on Windows, + selector-based on Unix) enforces connection limits at the socket level + +--- + +## Peer discovery + +**Location**: `server/node_discovery.py` + +### `FullNodePeers` + +- Maintains address book of known peers +- Periodic peer exchange via `request_peers` / `respond_peers` +- Connects to introducer nodes for bootstrapping +- DNS seeder support +- Preference for outbound connections to maintain network topology + +--- + +## Message type → node type mapping + +**Location**: `protocols/protocol_message_type_to_node_type.py` + +Maps each `ProtocolMessageTypes` to the set of `NodeType`s allowed to send it. +Used to reject messages from unexpected node types (e.g., a wallet trying to +send `new_peak` which is a full-node-only message). + +--- + +## `Handshake` protocol + +**Location**: `protocols/shared_protocol.py` + +### Fields + +- `network_id: str` +- `protocol_version: str` +- `software_version: str` +- `server_port: uint16` +- `node_type: NodeType` +- `capabilities: list[tuple[uint16, str]]` + +### `Capability` enum + +Key capabilities: `BASE`, `BLOCK_HEADERS`, `RATE_LIMITS_V2`, +`NONE_RESPONSE`, `MEMPOOL_UPDATES`. Used for feature negotiation. diff --git a/.cursor/context/wallet.md b/.cursor/context/wallet.md new file mode 100644 index 0000000000..4ea762db43 --- /dev/null +++ b/.cursor/context/wallet.md @@ -0,0 +1,179 @@ +# Wallet Layer — Deep Context + +> Attach when touching `chia/wallet/`. + +## File map (top-level) + +| File | Lines | Role | +| -------------------------------- | ----- | ----------------------------------------------------- | +| `wallet_state_manager.py` | ~3330 | `WalletStateManager`: all wallet state, coin tracking | +| `wallet_rpc_api.py` | ~3610 | `WalletRpcApi`: full wallet RPC surface | +| `wallet_rpc_client.py` | ~1030 | RPC client for CLI/tests | +| `wallet_node.py` | ~1750 | `WalletNode`: SPV sync, peer communication | +| `wallet_node_api.py` | ~210 | P2P message handlers | +| `wallet.py` | ~670 | `Wallet`: standard XCH wallet logic | +| `wallet_blockchain.py` | ~250 | Lightweight chain tracking for wallet | +| `conditions.py` | ~1550 | Condition parsing and construction | +| `coin_selection.py` | ~190 | Coin selection algorithm | +| `trade_manager.py` | ~1060 | Offer/trade management | +| `wallet_request_types.py` | ~2550 | RPC request type definitions | +| `wallet_coin_store.py` | ~350 | Wallet-side coin persistence | +| `wallet_transaction_store.py` | ~500 | Transaction record persistence | +| `wallet_puzzle_store.py` | ~390 | Derivation/puzzle hash store | +| `wallet_weight_proof_handler.py` | ~130 | Weight proof handling for wallet | +| `notification_manager.py` | ~120 | Notification handling | +| `wallet_action_scope.py` | ~170 | Action scope for atomic operations | +| `derive_keys.py` | ~140 | Key derivation | +| `singleton.py` | ~110 | Singleton utilities | +| `wallet_coin_record.py` | ~80 | `WalletCoinRecord` type | +| `transaction_record.py` | ~120 | `TransactionRecord` type | +| `start_wallet.py` | ~120 | Service startup | + +## Sub-wallet modules + +| Directory | Purpose | +| ------------- | ----------------------------------------------- | +| `cat_wallet/` | Chia Asset Token (CAT) wallet | +| `did_wallet/` | Decentralized Identity wallet | +| `nft_wallet/` | NFT wallet | +| `vc_wallet/` | Verifiable Credentials wallet | +| `db_wallet/` | DataLayer wallet | +| `trading/` | Offer trading utilities | +| `puzzles/` | CLVM puzzle definitions | +| `util/` | Wallet utilities, tx config, puzzle compression | + +--- + +## Coin selection + +**Location**: `wallet/coin_selection.py` + +### `select_coins(spendable_amount, config, spendable_coins, unconfirmed_removals, log, amount)` + +1. **Filter**: Remove unconfirmed removals, excluded coin IDs/amounts, + coins outside min/max amount bounds +2. **Max coins**: 500 per selection +3. **Sort + exact checks**: Exact single-coin match first, then exact sum of + all smaller coins if feasible +4. **Selection strategy**: + - If smaller coins are insufficient: select smallest coin over target + - Otherwise: run randomized knapsack search, then fallback to + `sum_largest_coins()`, then smallest over target + +### `CoinSelectionConfig` fields + +- `min_coin_amount`, `max_coin_amount` +- `excluded_coin_ids: set[bytes32]` +- `excluded_coin_amounts: set[uint64]` + +--- + +## Wallet sync model + +**Location**: `wallet/wallet_node.py` + +### Sync flow + +1. Receive `new_peak_wallet` from full node +2. If behind: request weight proof → validate +3. Subscribe to puzzle hashes via `register_for_ph_updates` +4. Subscribe to coin IDs via `register_for_coin_updates` +5. Receive `coin_state_update` pushes for subscribed items +6. Process coin state changes → update local wallet DB + +### Mempool tracking + +- Wallet send path tracks acceptance via `transaction_ack` +- Protocol supports `mempool_items_added` / `mempool_items_removed`, but the + reference wallet node API does not currently process those message types + +### Trust model + +- **Trusted mode**: Connected to own full node, skip weight proof verification +- **Untrusted mode**: Full weight proof validation required + +--- + +## Wallet state manager + +**Location**: `wallet/wallet_state_manager.py` + +### Responsibilities + +- Key management and derivation +- Coin record tracking (confirmed/unconfirmed/pending) +- Transaction creation and signing +- Sub-wallet registry and lifecycle +- Puzzle hash generation and caching +- Action scope management for atomic multi-step operations + +### Key state + +- `puzzle_store: WalletPuzzleStore` — derivation indexes and puzzle hashes +- `coin_store: WalletCoinStore` — wallet-side coin records +- `tx_store: WalletTransactionStore` — transaction history +- `user_store: WalletUserStore` — sub-wallet metadata +- `interested_store: WalletInterestedStore` — tracked coin/puzzle IDs + +### Sub-wallet types + +Each sub-wallet type handles its own puzzle construction, spend creation, +and coin tracking: + +- Standard wallet (XCH) +- CAT wallet (fungible tokens) +- DID wallet (identity) +- NFT wallet (non-fungible tokens) +- DataLayer wallet +- VC wallet (verifiable credentials) + +--- + +## Transaction construction + +### General flow + +1. Select coins via `coin_selection.py` +2. Construct puzzle reveals and solutions +3. Create `CoinSpend` objects +4. Aggregate into `SpendBundle` +5. Sign with BLS keys +6. Submit via `send_transaction` to full node + +### `WalletActionScope` + +Provides atomic operation scope for multi-step wallet operations. +Tracks additions, removals, and intermediate state. On failure, +all changes can be rolled back. + +--- + +## Wallet RPC surface + +**Location**: `wallet/wallet_rpc_api.py` + +`wallet_rpc_api.py` is one of the larger wallet modules (~3610 lines). +Key endpoint categories: + +- **Key management**: `log_in`, `get_public_keys`, `generate_mnemonic` +- **Wallet info**: `get_wallets`, `get_wallet_balance`, `get_sync_status` +- **Transactions**: `send_transaction`, `get_transactions`, `delete_unconfirmed_transactions` +- **Coin management**: `get_spendable_coins`, `select_coins`, `get_coin_records_by_names` +- **CAT**: `cat_spend`, `cat_get_asset_id`, `create_new_cat_wallet` +- **NFT**: `nft_mint_nft`, `nft_transfer_nft`, `nft_get_nfts` +- **DID**: `did_get_info`, `did_update_metadata`, `did_transfer_did` +- **Offers**: `create_offer_for_ids`, `take_offer`, `get_all_offers` +- **DataLayer**: `dl_*` endpoints +- **Fee estimation**: `get_fee_estimate` + +--- + +## Wallet blockchain + +**Location**: `wallet/wallet_blockchain.py` + +Lightweight chain state for the wallet. Tracks: + +- Peak height and header hash +- Sub-epoch summaries (for weight proof validation) +- Does NOT store full blocks — only enough for sync verification diff --git a/.cursor/rules/context-clvm.mdc b/.cursor/rules/context-clvm.mdc new file mode 100644 index 0000000000..ab95b05039 --- /dev/null +++ b/.cursor/rules/context-clvm.mdc @@ -0,0 +1,21 @@ +--- +description: Auto-attach CLVM context when working in puzzle/execution code +globs: + - chia/types/blockchain_format/** + - chia/types/condition_opcodes.py + - chia/types/generator_types.py + - chia/wallet/puzzles/** + - chia/wallet/conditions.py + - chia/consensus/condition_tools.py + - chia/consensus/generator_tools.py + - chia/consensus/get_block_generator.py + - chia/consensus/cost_calculator.py +--- + +# CLVM / Puzzle Execution Context Available + +Deep context for this subsystem exists at `.cursor/context/clvm-execution.md`. +Read it before exploring code — it documents the three CLVM execution paths +(mempool, block validation, block building), resource limits (cost, atoms, pairs), +canonical serialization, condition opcodes, AGG_SIG replay protection, and +`MEMPOOL_MODE` flag behavior. diff --git a/.cursor/rules/context-consensus.mdc b/.cursor/rules/context-consensus.mdc new file mode 100644 index 0000000000..6fa85d01d6 --- /dev/null +++ b/.cursor/rules/context-consensus.mdc @@ -0,0 +1,15 @@ +--- +description: Auto-attach consensus context when working in consensus code +globs: + - chia/consensus/** +--- + +# Consensus Context Available + +Deep context for this subsystem exists at `.cursor/context/consensus.md`. +Read it before exploring code — it documents `add_block()`, `_reconsider_peak()`, +`ForkInfo`, `validate_block_body()`, difficulty adjustment, VDF iteration math, +block rewards, and `AugmentedBlockchain` with line-level analysis and invariants. + +If the change involves reorgs or cross-module state, also read +`.cursor/context/global-invariants.md`. diff --git a/.cursor/rules/context-full-node.mdc b/.cursor/rules/context-full-node.mdc new file mode 100644 index 0000000000..68be5d094e --- /dev/null +++ b/.cursor/rules/context-full-node.mdc @@ -0,0 +1,21 @@ +--- +description: Auto-attach full node context when working in full node code +globs: + - chia/full_node/full_node.py + - chia/full_node/full_node_api.py + - chia/full_node/full_node_rpc_api.py + - chia/full_node/full_node_store.py + - chia/full_node/coin_store.py + - chia/full_node/block_store.py + - chia/full_node/sync_store.py + - chia/full_node/weight_proof.py + - chia/full_node/subscriptions.py + - chia/full_node/tx_processing_queue.py +--- + +# Full Node Context Available + +Deep context for this subsystem exists at `.cursor/context/full-node.md`. +Read it before exploring code — it documents the block processing pipeline, +sync logic, `FullNodeStore` state, `FullNodeAPI` handlers, `CoinStore` schema, +`BlockStore` operations, and the `PeakPostProcessingResult` workflow. diff --git a/.cursor/rules/context-mempool.mdc b/.cursor/rules/context-mempool.mdc new file mode 100644 index 0000000000..8a6d784e86 --- /dev/null +++ b/.cursor/rules/context-mempool.mdc @@ -0,0 +1,17 @@ +--- +description: Auto-attach mempool context when working in mempool code +globs: + - chia/full_node/mempool*.py + - chia/full_node/eligible_coin_spends.py + - chia/full_node/pending_tx_cache.py + - chia/full_node/fee_*.py + - chia/full_node/bitcoin_fee_estimator.py +--- + +# Mempool Context Available + +Deep context for this subsystem exists at `.cursor/context/mempool.md`. +Read it before exploring code — it documents the full admission pipeline, +`validate_spend_bundle()`, `check_removals()`, `add_spend_bundle()`, +`new_peak()` reorg handling, FF/DEDUP logic, conflict resolution, and all +key constants with their rationale. diff --git a/.cursor/rules/context-networking.mdc b/.cursor/rules/context-networking.mdc new file mode 100644 index 0000000000..09a927ab81 --- /dev/null +++ b/.cursor/rules/context-networking.mdc @@ -0,0 +1,14 @@ +--- +description: Auto-attach networking context when working in server/protocol code +globs: + - chia/server/** + - chia/protocols/** +--- + +# Networking Context Available + +Deep context for this subsystem exists at `.cursor/context/networking.md`. +Read it before exploring code — it documents `WSChiaConnection`, rate limiting +(per-type and aggregate), the protocol state machine, `ApiProtocol`/`@request` +decorator, connection limits, peer discovery, and the handshake/capability +negotiation flow. diff --git a/.cursor/rules/context-routing.mdc b/.cursor/rules/context-routing.mdc new file mode 100644 index 0000000000..03a3276d20 --- /dev/null +++ b/.cursor/rules/context-routing.mdc @@ -0,0 +1,37 @@ +--- +description: Route agents to pre-built context before code exploration +alwaysApply: true +--- + +# Deep Context — Read Before You Grep + +Pre-built deep context exists in `.cursor/context/`. **Before running broad +code searches** (semantic search, grep, glob) to understand how a subsystem +works, read the matching context file first. It will answer most structural +questions instantly. + +## Routing table + +Match the files or intent to the right context doc: + +| If you're touching… | Read first | +|----------------------|------------| +| `chia/consensus/**` or block validation / reorg / difficulty | `.cursor/context/consensus.md` | +| `chia/full_node/mempool*.py`, `eligible_coin_spends.py`, fee logic | `.cursor/context/mempool.md` | +| `chia/full_node/full_node.py`, `full_node_api.py`, `full_node_store.py`, sync | `.cursor/context/full-node.md` | +| `chia/server/**`, `chia/protocols/**`, connections, rate limits | `.cursor/context/networking.md` | +| `chia/wallet/**` | `.cursor/context/wallet.md` | +| CLVM, puzzles, conditions, generators, `chia/types/blockchain_format/**` | `.cursor/context/clvm-execution.md` | +| Cross-cutting / security review / unsure | `.cursor/context/global-invariants.md` | +| First time / orientation / unfamiliar area | `.cursor/context/architecture-overview.md` | + +## Protocol + +1. Identify which subsystem(s) the task touches. +2. Read the matching context file(s) via the Read tool. +3. State: `Context loaded: (area: )`. +4. Only then do targeted code reads for the specific lines you need to change. + +The context files contain: file maps, function-level analysis, invariants, +constants, assumptions, and cross-module dependencies. They replace the need +for broad exploratory searches. diff --git a/.cursor/rules/context-wallet.mdc b/.cursor/rules/context-wallet.mdc new file mode 100644 index 0000000000..68ca2d2923 --- /dev/null +++ b/.cursor/rules/context-wallet.mdc @@ -0,0 +1,13 @@ +--- +description: Auto-attach wallet context when working in wallet code +globs: + - chia/wallet/** +--- + +# Wallet Context Available + +Deep context for this subsystem exists at `.cursor/context/wallet.md`. +Read it before exploring code — it documents coin selection, the wallet sync +model (trusted vs untrusted), `WalletStateManager` responsibilities, +sub-wallet types, transaction construction flow, `WalletActionScope`, and +the RPC surface layout. diff --git a/.cursor/rules/testing-guide-blockchain.mdc b/.cursor/rules/testing-guide-blockchain.mdc new file mode 100644 index 0000000000..0c707a2e08 --- /dev/null +++ b/.cursor/rules/testing-guide-blockchain.mdc @@ -0,0 +1,90 @@ +--- +description: Blockchain consensus test patterns — block validation, reorgs, overflow slots, fork behavior +globs: + - chia/_tests/blockchain/** +--- + +# Blockchain Tests + +## Scope + +Use this for consensus-level chain behavior in `chia/_tests/blockchain/**`: + +- block validation outcomes (`NEW_PEAK`, `ADDED_AS_ORPHAN`, `INVALID_BLOCK`) +- reorgs and fork preference behavior +- overflow/slot edge cases +- block-record / fork-info invariants + +Primary references: + +- `chia/_tests/blockchain/test_blockchain.py` +- `chia/_tests/blockchain/blockchain_test_utils.py` + +## Preferred Harness + +- `empty_blockchain` fixture for most unit/integration-style blockchain tests. +- `bt` (`BlockTools`) for deterministic block construction. +- `create_blockchain(...)` when two-chain comparisons are needed. + +## Common Build/Add Pattern + +1. Build deterministic blocks with `bt.get_consecutive_blocks(...)`. +2. Add with `_validate_and_add_block(...)` (or multi-result helper variants). +3. Assert expected add result/error at each step. + +Useful options in `get_consecutive_blocks(...)`: + +- `block_list_input` to continue a chain +- `seed` for alternate fork chain +- `force_overflow` for overflow peak/slot scenarios +- `skip_slots` for sub-slot boundary behavior + +## Reorg Pattern + +For fork validation flows: + +- Initialize `fork_info = ForkInfo(...)`. +- Reuse a single `AugmentedBlockchain(b)` while validating fork blocks. +- Expect: + - `ALREADY_HAVE_BLOCK` for shared prefix blocks + - `ADDED_AS_ORPHAN` before fork wins + - default success (`NEW_PEAK`) once heavier fork overtakes + +## Overflow-Specific Guidance + +- Use `is_overflow_block(constants, block.reward_chain_block.signage_point_index)` for assertions. +- If a test requires overflow peak precondition, enforce it explicitly with `force_overflow=True`. +- Keep reorg assertions layered: precondition peak state -> fork progression results -> final peak state. + +## Starter Template + +```python +from __future__ import annotations + +import pytest +from chia_rs.sized_ints import uint32 + +from chia._tests.blockchain.blockchain_test_utils import ( + _validate_and_add_block, + _validate_and_add_block_multi_result, +) +from chia.consensus.augmented_chain import AugmentedBlockchain +from chia.consensus.block_body_validation import ForkInfo +from chia.consensus.blockchain import AddBlockResult, Blockchain +from chia.consensus.pot_iterations import is_overflow_block +from chia.simulator.block_tools import BlockTools + + +@pytest.mark.anyio +async def test_example(empty_blockchain: Blockchain, bt: BlockTools) -> None: + # Build a base chain + blocks = bt.get_consecutive_blocks(5) + for block in blocks: + await _validate_and_add_block(empty_blockchain, block) + + # Fork from earlier point with different seed + fork_blocks = bt.get_consecutive_blocks( + 3, block_list_input=blocks[:3], seed=b"fork" + ) + # Add fork blocks and assert results... +``` diff --git a/.cursor/rules/testing-guide-data-layer.mdc b/.cursor/rules/testing-guide-data-layer.mdc new file mode 100644 index 0000000000..24a716ae89 --- /dev/null +++ b/.cursor/rules/testing-guide-data-layer.mdc @@ -0,0 +1,132 @@ +--- +description: Data layer test patterns — DataStore logic, wallet-backed RPC flows, singleton lifecycle +globs: + - chia/_tests/core/data_layer/** +--- + +# Data Layer Tests + +## Scope + +Use this for data layer behavior across both local store logic and chain-backed updates: + +- `DataStore` tree/node/key behavior +- data layer RPC behavior (`create_data_store`, `batch_update`, root/value/proof endpoints) +- wallet-confirmed singleton/update lifecycle +- offer/integrity and sync-related data layer flows + +Primary references: + +- `chia/_tests/core/data_layer/conftest.py` +- `chia/_tests/core/data_layer/test_data_store.py` +- `chia/_tests/core/data_layer/test_data_rpc.py` + +## Two Distinct Test Tracks + +### Track A: Pure Store Logic (No full node topology needed) + +Use fixtures from `core/data_layer/conftest.py`: + +- `raw_data_store` +- `data_store` +- `store_id` +- `create_example` + +This is ideal for: + +- insert/upsert/delete semantics +- generation and schema behavior +- node/tree traversal and validation +- deterministic error behavior (`KeyNotFoundError`, invalid operations) + +### Track B: Wallet-Backed Data Layer RPC (Chain-confirmed lifecycle) + +Use simulator + wallet service setup: + +- `one_wallet_and_one_simulator_services` +- helper flow from `test_data_rpc.py`: + - `init_wallet_and_node(...)` + - `init_data_layer(...)` / `init_data_layer_service(...)` + +This is ideal for: + +- singleton creation confirmation +- mempool-to-block update confirmation +- RPC/client/CLI parity checks + +## Chain-Backed Update Flow (Canonical Pattern) + +1. Start wallet/full-node and farm prefund blocks. +2. Create data layer service and API client. +3. `create_data_store`. +4. Farm until singleton confirmed. +5. `batch_update` with changelist. +6. Farm tx block and wait for tx confirmation. +7. Assert value/root/local-root consistency. + +Reusable assertion helpers in practice: + +- `check_mempool_spend_count(...)` +- `check_singleton_confirmed(...)` +- `farm_block_check_singleton(...)` +- `farm_block_with_spend(...)` + +## How To Assert Steps Happened + +Layer assertions in this order: + +1. RPC response success and expected ids/tx ids exist. +2. Mempool contains expected spend count before farming. +3. Singleton/update tx confirmed after farming. +4. Wallet sync complete (`wait_for_wallet_synced` / wallet sync checks). +5. Data correctness (`get_value`, roots, proof validation where relevant). + +## CLI/Func/Client Parity Pattern + +`test_data_rpc.py` validates multiple interface layers (direct API, client, functions, CLI). +When adding behavior, prefer adding one parity test that confirms the same operation through at least two interfaces. + +## Anti-Flake Guidance + +- Prefer `time_out_assert` and sync helpers over fixed sleeps. +- Explicitly farm the include block for pending updates before asserting roots. +- Keep test-local db paths (`tmp_path`) isolated per test. +- For heavy scenarios, keep operations grouped and avoid unnecessary service restarts. + +## Quick Checklist + +- Chosen track (store-only vs chain-backed) matches the behavior under test. +- Chain-backed flow includes both mempool and confirmation assertions. +- Wallet sync is explicitly awaited before balance/state checks. +- Root/value checks are done after final confirmation, not before. + +## Starter Template (Track A — Pure Store) + +```python +from __future__ import annotations + +import pytest +from chia_rs.sized_bytes import bytes32 + +from chia.data_layer.data_layer_errors import KeyNotFoundError +from chia.data_layer.data_layer_util import OperationType, Status +from chia.data_layer.data_store import DataStore + +pytestmark = pytest.mark.data_layer + + +@pytest.mark.anyio +async def test_example(data_store: DataStore, store_id: bytes32) -> None: + key = b"\x01" * 32 + value = b"\x02" * 32 + await data_store.autoinsert( + key=key, + value=value, + store_id=store_id, + status=Status.COMMITTED, + ) + result = await data_store.get_node_by_key(store_id=store_id, key=key) + assert result.value == value +``` + +Fixtures `data_store` and `store_id` come from `chia/_tests/core/data_layer/conftest.py`. diff --git a/.cursor/rules/testing-guide-full-node.mdc b/.cursor/rules/testing-guide-full-node.mdc new file mode 100644 index 0000000000..a4d4c2b71a --- /dev/null +++ b/.cursor/rules/testing-guide-full-node.mdc @@ -0,0 +1,133 @@ +--- +description: Full node integration test patterns — sync, propagation, mempool-to-block, reorg, wallet-connected flows +globs: + - chia/_tests/core/full_node/** +--- + +# Full Node Tests + +## Scope + +Use this when testing behavior driven by full node state transitions: + +- node sync / backtrack / batch sync +- block acceptance and propagation +- mempool to block inclusion +- wallet-connected full node flows +- reorg and chain preference behavior + +Primary references: + +- `chia/_tests/core/full_node/test_full_node.py` +- `chia/_tests/core/full_node/test_transactions.py` +- `chia/_tests/core/full_node/full_sync/test_full_sync.py` + +## Go-To Fixtures and Harness + +Pick the smallest fixture that covers your case: + +- single-node state checks: `one_node_one_block`, `one_node` +- two/three/five node sync tests: `two_nodes`, `three_nodes`, `five_nodes` +- wallet + full node integration: `simulator_and_wallet`, `setup_two_nodes_and_wallet`, `three_nodes_two_wallets` +- custom service setup: `setup_simulators_and_wallets(...)` from `chia/_tests/util/setup_nodes.py` + +Typical test setup pattern: + +1. Connect peers with `start_client(...)` or `connect_and_get_peer(...)`. +2. Seed chain with deterministic blocks or farmed tx blocks. +3. Submit spend(s) and wait for mempool visibility. +4. Farm include block and assert final chain/wallet state. + +## Block Creation Patterns + +Use deterministic block construction when sequence matters: + +- `bt.get_consecutive_blocks(...)` + - Common options: `block_list_input`, `seed`, `guarantee_transaction_block`, `skip_slots`, `force_overflow`. + +Add blocks with: + +- `await add_blocks_in_batches(blocks, full_node)` for bulk chain setup. +- `await full_node.add_block(block)` when each step needs validation. + +Use simulator helpers when internals are less important: + +- `farm_new_transaction_block(FarmNewBlockProtocol(...))` +- `farm_blocks_to_puzzlehash(...)` + +## Transaction Submission Patterns + +Wallet-first path (common): + +1. Build tx in `new_action_scope(..., push=True)`. +2. `await wallet.generate_signed_transaction(...)`. +3. Read created records from `action_scope.side_effects.transactions`. + +Protocol path (for peer/mempool behavior): + +1. Wrap spend bundle in `wallet_protocol.SendTransaction(...)`. +2. Submit via `full_node_api.send_transaction(...)` using a dummy or connected peer. + +## How To Assert Steps Happened + +Use layered checks: + +1. **Mempool entered** + - `time_out_assert(..., mempool_manager.get_spendbundle, expected_bundle, tx_name)` +2. **Block inclusion** + - farm tx block, then assert mempool no longer contains bundle +3. **Heights/sync convergence** + - `time_out_assert(..., node_height_at_least|node_height_exactly, ...)` +4. **Wallet convergence** + - `wait_for_wallet_synced(...)` + balance assertions +5. **Failure path** + - `pytest.raises(...)` and explicit `Err`/status checks where applicable + +## Anti-Flake Guidance + +- Prefer `time_out_assert` and sync wait helpers over raw sleeps. +- Use deterministic seeds in `get_consecutive_blocks` for fork/reorg scenarios. +- For sync tests, assert both node height and peak equality when possible. +- Keep fixture scope narrow; large shared state increases intermittent failures. + +## Quick Checklist + +- Fixture matches topology (single node vs multi-node vs wallet-connected). +- Block sequence is deterministic if order-sensitive. +- Mempool, inclusion, and sync are asserted as separate steps. +- Negative path validates expected `Err` or status, not just generic failure. + +## Starter Template + +```python +from __future__ import annotations + +import pytest + +from chia._tests.blockchain.blockchain_test_utils import _validate_and_add_block +from chia._tests.connection_utils import add_dummy_connection, connect_and_get_peer +from chia._tests.core.node_height import node_height_at_least +from chia._tests.util.setup_nodes import OldSimulatorsAndWallets +from chia._tests.util.time_out_assert import time_out_assert +from chia.consensus.blockchain import Blockchain +from chia.full_node.full_node_api import FullNodeAPI +from chia.protocols import wallet_protocol +from chia.server.server import ChiaServer +from chia.simulator.block_tools import BlockTools + + +@pytest.mark.anyio +async def test_example( + one_node_one_block: tuple[FullNodeAPI, ChiaServer, BlockTools], +) -> None: + full_node_api, server, bt = one_node_one_block + full_node = full_node_api.full_node + + # Build and add blocks + blocks = bt.get_consecutive_blocks(3) + for block in blocks: + await full_node.add_block(block) + + # Assert height convergence + await time_out_assert(10, node_height_at_least, True, full_node, 3) +``` diff --git a/.cursor/rules/testing-guide-mempool.mdc b/.cursor/rules/testing-guide-mempool.mdc new file mode 100644 index 0000000000..2e8f2d272e --- /dev/null +++ b/.cursor/rules/testing-guide-mempool.mdc @@ -0,0 +1,142 @@ +--- +description: Mempool manager test patterns — acceptance/rejection, replacement, fee-per-cost, eviction, bundle selection +globs: + - chia/_tests/core/mempool/** +--- + +# Mempool Tests + +## Scope + +Use this for mempool rule correctness: + +- acceptance/rejection status (`SUCCESS`, `PENDING`, `FAILED`) +- replacement and conflict rules +- fee-per-cost behavior +- eviction and expiration behavior +- bundle construction from mempool (`create_bundle_from_mempool`) + +Primary references: + +- `chia/_tests/core/mempool/test_mempool_manager.py` +- `chia/_tests/core/mempool/test_mempool.py` +- `chia/_tests/core/mempool/test_mempool_fee_protocol.py` + +## Preferred Harness (Fast and Deterministic) + +For most mempool logic, avoid full network setup: + +- `instantiate_mempool_manager(...)` for isolated mempool manager tests +- `setup_mempool_with_coins(...)` for synthetic coin sets and controlled constants + +Core helper flow: + +1. Build a spend bundle from conditions (`spend_bundle_from_conditions(...)`). +2. Pre-validate/add with `add_spendbundle(...)`. +3. Assert `MempoolInclusionStatus` and `Err` exactly. +4. Validate pool content with: + - `assert_sb_in_pool(...)` + - `assert_sb_not_in_pool(...)` + +## When To Use Full Node Integration + +Use `one_node_one_block` or related node fixtures only when testing: + +- peer-facing protocol behavior (`send_transaction`, `respond_transaction`) +- pending tx cache interaction with chain advancement +- network propagation interactions not visible in isolated manager tests + +## Transaction and Bundle Patterns + +Common builder patterns: + +- `make_test_spendbundle(coin, fee=...)` +- aggregated replacements: `SpendBundle.aggregate([...])` +- generated conditions for specific policy checks + +For block assembly checks: + +- `result = mempool_manager.create_bundle_from_mempool(peak.header_hash)` +- assert which spends are included/excluded, not only that result is non-None + +## How To Assert Steps Happened + +1. **Admission status** + - assert tuple `(status, error)` from add path +2. **Pool state** + - assert exact presence/absence of old/new conflicting bundles +3. **Invariants** + - keep `invariant_check_mempool(...)` in the loop for complex mutation tests +4. **On-new-peak behavior** + - advance peak and verify expiration/retention conditions +5. **Bundle selection** + - verify selected spends for cost/priority behavior, not just count + +## Anti-Flake Guidance + +- Prefer isolated manager fixtures for policy tests; full node setup adds non-essential timing variability. +- Seed deterministic coins and fees. +- Assert precise errors (`Err.MEMPOOL_CONFLICT`, etc.), not generic failure. +- Keep each test focused on one replacement rule or one limit family. + +## Quick Checklist + +- Isolated harness chosen unless protocol/network behavior is required. +- Spend bundles constructed to isolate one rule at a time. +- Status + error + pool content all asserted. +- Peak advancement tested when rule depends on height/time. + +## Starter Template + +Key helpers are defined in `test_mempool_manager.py` itself (not importable from a separate module): + +```python +from __future__ import annotations + +from typing import Any + +import pytest +from chia_rs import CoinRecord, G2Element, SpendBundle +from chia_rs.sized_bytes import bytes32 +from chia_rs.sized_ints import uint32, uint64 + +from chia.consensus.default_constants import DEFAULT_CONSTANTS +from chia.full_node.mempool_manager import MempoolManager +from chia.types.blockchain_format.coin import Coin +from chia.types.blockchain_format.serialized_program import SerializedProgram +from chia.types.condition_opcodes import ConditionOpcode +from chia.types.mempool_inclusion_status import MempoolInclusionStatus +from chia.util.errors import Err + +IDENTITY_PUZZLE = SerializedProgram.to(1) +IDENTITY_PUZZLE_HASH = IDENTITY_PUZZLE.get_tree_hash() +TEST_COIN_AMOUNT = uint64(1000000000) +TEST_COIN = Coin(IDENTITY_PUZZLE_HASH, IDENTITY_PUZZLE_HASH, TEST_COIN_AMOUNT) +TEST_COIN_ID = TEST_COIN.name() +TEST_TIMESTAMP = uint64(10040) +TEST_COIN_RECORD = CoinRecord(TEST_COIN, uint32(0), uint32(0), False, TEST_TIMESTAMP) +TEST_HEIGHT = uint32(5) + + +# Re-use these local helpers from test_mempool_manager.py as reference: +# instantiate_mempool_manager(get_coin_records, ...) +# spend_bundle_from_conditions(conditions, coin=TEST_COIN) +# add_spendbundle(mempool_manager, sb, sb_name) +# assert_sb_in_pool(mempool_manager, sb) +# assert_sb_not_in_pool(mempool_manager, sb) + + +@pytest.mark.anyio +async def test_example() -> None: + # Coin record lookup for the mempool manager + async def get_coin_records(coin_ids): + return [TEST_COIN_RECORD] if TEST_COIN_ID in coin_ids else [] + + mempool_manager = await instantiate_mempool_manager(get_coin_records) + conditions = [[ConditionOpcode.CREATE_COIN, bytes(32), 1]] + sb = spend_bundle_from_conditions(conditions) + _, status, error = await add_spendbundle(mempool_manager, sb, sb.name()) + assert status == MempoolInclusionStatus.SUCCESS + assert error is None + assert_sb_in_pool(mempool_manager, sb) +``` diff --git a/.cursor/rules/testing-guide-patterns.mdc b/.cursor/rules/testing-guide-patterns.mdc new file mode 100644 index 0000000000..0ee93e4158 --- /dev/null +++ b/.cursor/rules/testing-guide-patterns.mdc @@ -0,0 +1,119 @@ +--- +description: Detailed block creation, transaction submission, and assertion patterns for Chia tests +globs: + - chia/_tests/** +--- + +# Chia Test Patterns + +## How We Make Blocks + +### 1) Deterministic block lists (`BlockTools`) + +Use `bt.get_consecutive_blocks(...)` when you need exact block structure, specific spend inclusion, or malformed block variants. + +Common options: +- `block_list_input=` continue from existing chain +- `transaction_data=` include a spend bundle +- `guarantee_transaction_block=True` force tx block +- `force_overflow`, `skip_slots`, `seed`, `time_per_block` + +Then add blocks with: +- `await full_node.add_block(block)` for single steps, or +- `await add_blocks_in_batches(blocks, full_node)` for larger sets + +### 2) High-level farming APIs (`FullNodeSimulator`) + +Use these for behavior tests where exact block internals do not matter: +- `farm_blocks_to_puzzlehash()` +- `farm_blocks_to_wallet()` +- `farm_rewards_to_wallet()` +- `farm_new_transaction_block()` +- `reorg_from_index_to_new_index()` +- `revert_block_height()` + +### 3) Pre-generated persistent chains + +Use fixtures like: +- `default_400_blocks`, `default_1000_blocks`, `default_10000_blocks` +- reorg variants and compact variants + +These come from `persistent_blocks(...)` and are used heavily in consensus/timelord/weight proof tests. + +## How We Submit Transactions + +### A) Wallet-internal (most common) + +1. `async with wallet.wallet_state_manager.new_action_scope(..., push=True) as action_scope:` +2. `await wallet.generate_signed_transaction(...)` +3. Records from `action_scope.side_effects.transactions` +4. Wait via `wait_transaction_records_entered_mempool` or `process_pending_states`. + +### B) Wallet RPC + +`WalletRpcClient`: `send_transaction(...)`, `create_signed_transactions(...)`, `push_transactions(...)`, `push_tx(...)`. + +### C) Full node protocol-level + +Build `wallet_protocol.SendTransaction(spend_bundle)`, send via `full_node_api.send_transaction(...)` with dummy peers from `chia/_tests/connection_utils.py`. + +### D) CLVM simulator + +`status, err = await sim_client.push_tx(spend_bundle)` + +## How We Assert Steps Happened + +Use layered assertions instead of a single final check: + +1. **Immediate invariants** — object created, response success, expected fields present. +2. **Eventual behavior** — `time_out_assert(...)` for async convergence. +3. **Mempool checks** — `mempool_manager.get_spendbundle(...)`, `assert_sb_in_pool(...)`. +4. **Wallet transitions** — `process_pending_states(...)` with `WalletStateTransition`. +5. **Failure paths** — `pytest.raises(...)` with explicit error matching. +6. **Log assertions** — `caplog` for protocol/service side effects. + +## Module-by-Module Test Setup Map + +| Module | Typical Setup | Blocks | Transaction Path | Assertion Style | +|---|---|---|---|---| +| `blockchain` | `bt`, `empty_blockchain`, `two_nodes` | `get_consecutive_blocks`, `add_block`, `add_blocks_in_batches` | `WalletTool.generate_signed_transaction`, protocol `send_transaction`, in-block `transaction_data` | direct consensus result checks, `pytest.raises`, occasional `time_out_assert` | +| `clvm` | no network harness, or `sim_and_client` | `SpendSim.farm_block` | `sim_client.push_tx` | direct CLVM/coin-store assertions, `pytest.raises` | +| `cmds` | `CliRunner`, `get_test_cli_clients`, temp config roots | usually none | mocked RPC client calls | output assertions, parse/validation errors | +| `core` | mixed: `one_node_one_block`, `simulator_and_wallet`, data-layer fixtures | heavy use of `get_consecutive_blocks`, farming APIs | wallet-generated spends, protocol `send_transaction`/`respond_transaction` | heavy `time_out_assert`, mempool/state assertions, `caplog`, `pytest.raises` | +| `db` | `DBConnection`/`PathDBConnection` fixtures | none | none | concurrency/transactionality assertions, `pytest.raises` | +| `farmer_harvester` | `farmer_one_harvester*`, `harvester_farmer_environment` | minimal | protocol message flow | service-state `time_out_assert`, `caplog` | +| `fee_estimation` | mostly mempool/unit harness | minimal farming | small generated spend bundles | direct estimator state assertions | +| `generator` | pure generator/CLVM tests | none | none | deterministic program output/cost assertions | +| `harvester` | `harvester_farmer_environment` + test plots | `default_400_blocks` for signage data | harvester protocol interactions | `time_out_assert`, mock peer assertions | +| `pools` | pure puzzle unit tests and wallet/simulator integration | farming + reorg in integration | wallet RPC and framework tx processing | `process_pending_states`, `time_out_assert`, `pytest.raises` | +| `simulation` | `simulator_and_wallet`, full system fixture | high-level simulator farming/reorg | wallet-generated spends | heavy `time_out_assert`, mempool/coin-store confirmations | +| `wallet` | `wallet_environments` (primary), simulator fixtures | frequent farming/reorg | wallet action scopes, wallet RPC | `process_pending_states`, `time_out_assert`, mempool checks | +| `weight_proof` | pre-generated block fixtures + `BlockchainMock` | `get_consecutive_blocks` for edge chains | none | proof validity/fork point assertions | + +## Agent Templates + +### Wallet transfer with robust checks + +1. Use `wallet_environments` with needed prefarm. +2. Build tx inside `new_action_scope(..., push=True)`. +3. Call `wallet_environments.process_pending_states([...])` with pre-block and post-block expected deltas. + +### Protocol/mempool acceptance test + +1. Create spend bundle (`WalletTool` or wallet action scope). +2. Submit with `full_node_api.send_transaction(...)` via dummy peer. +3. Assert mempool inclusion, farm tx block, assert eviction + coin-store updates. + +### Consensus block validation test + +1. Build base chain using `bt.get_consecutive_blocks`. +2. Mutate crafted block (or use transaction conditions). +3. Validate with `_validate_and_add_block(...)` expecting specific `Err`. + +## Checklist + +- Harness matches behavior under test. +- Async convergence uses `time_out_assert`, not raw sleeps. +- Wallet behavior uses `process_pending_states` where practical. +- Mempool and confirmation asserted as separate steps. +- Failure paths use `pytest.raises` with explicit error matching. diff --git a/.cursor/rules/testing-guide-server.mdc b/.cursor/rules/testing-guide-server.mdc new file mode 100644 index 0000000000..401ea916b8 --- /dev/null +++ b/.cursor/rules/testing-guide-server.mdc @@ -0,0 +1,125 @@ +--- +description: Server/network test patterns — connection lifecycle, API errors, DoS/ban behavior, rate limiting +globs: + - chia/_tests/core/server/** +--- + +# Server Tests + +## Scope + +Use this for network/server behavior rather than consensus correctness: + +- connection lifecycle and duplicate connection handling +- API readiness/error transport behavior between peers +- protocol error logging and compatibility behavior +- DoS protection (bad handshake, oversized messages, spam) +- rate limiter correctness (counts, bytes, periodic reset, capability versions) + +Primary references: + +- `chia/_tests/core/server/test_server.py` +- `chia/_tests/core/server/test_dos.py` +- `chia/_tests/core/server/test_rate_limits.py` + +## Harness Selection + +### Server integration behavior + +Use node fixtures: + +- `two_nodes`, `two_nodes_one_block` +- `setup_two_nodes_fixture` +- `one_wallet_and_one_simulator_services` + +Use helpers: + +- `connect_and_get_peer(...)` for explicit peer wiring +- `time_out_assert(...)` for async event/log convergence + +### Pure rate-limit logic + +Prefer isolated unit-style tests: + +- instantiate `RateLimiter(...)` +- build protocol messages with `make_msg(...)` +- assert allow/deny responses directly + +This is faster and more deterministic than full socket setup. + +## Common Test Patterns + +### 1) Connection semantics + +- Assert `start_client(...)` success/failure behavior (including duplicate connection rejection). +- Inspect both outgoing and incoming connection state where needed. + +### 2) API error transport behavior + +- Use custom API methods that raise `ApiError`. +- Assert response payload compatibility by protocol version. +- Use `caplog` to assert expected warning/debug lines were produced. + +### 3) DoS and ban behavior + +- Open websocket using real SSL context from fixture services. +- Send malformed or oversized payload. +- Assert: + - close code (`PROTOCOL_ERROR`, `MESSAGE_TOO_BIG`, etc.) + - banned peer insertion (`server.banned_peers`) + +### 4) Rate limit behavior + +- Build deterministic message workloads. +- Assert threshold crossing by count and by bytes. +- For time-window reset checks, use a simulated clock object. + +## How To Assert Steps Happened + +1. **Immediate transport result** + - returned bool/object or websocket close code +2. **Eventually visible side effect** + - ban table update, log line appearance, queue-driven behavior +3. **Protocol correctness** + - exact message/response type and payload compatibility + +## Anti-Flake Guidance + +- For localhost ban tests, patch localhost checks when needed to exercise ban paths. +- Use `time_out_assert` for log/banning events; avoid raw sleep-based checks. +- Keep direct socket tests narrow (single failure mode per test). +- Use pure `RateLimiter` tests for matrix coverage of limits. + +## Quick Checklist + +- Pick integration fixture only when socket/peer behavior is required. +- Assert close codes and ban state explicitly in DoS tests. +- For versioned API errors, assert behavior on both sides of compatibility threshold. +- For rate limits, cover both count and size paths, plus reset behavior. + +## Starter Template + +```python +from __future__ import annotations + +import pytest + +from chia._tests.connection_utils import connect_and_get_peer +from chia._tests.util.time_out_assert import time_out_assert +from chia.full_node.full_node_api import FullNodeAPI +from chia.protocols.protocol_message_types import ProtocolMessageTypes +from chia.protocols.outbound_message import make_msg +from chia.server.server import ChiaServer +from chia.simulator.block_tools import BlockTools +from chia.types.peer_info import PeerInfo + + +@pytest.mark.anyio +async def test_example( + two_nodes: tuple[FullNodeAPI, FullNodeAPI, ChiaServer, ChiaServer, BlockTools], + self_hostname: str, +) -> None: + _, _, server_1, server_2, _ = two_nodes + peer = await connect_and_get_peer(server_1, server_2, self_hostname) + # Assert connection state, send messages, check responses... +``` diff --git a/.cursor/rules/testing-guide.mdc b/.cursor/rules/testing-guide.mdc new file mode 100644 index 0000000000..dcda345be0 --- /dev/null +++ b/.cursor/rules/testing-guide.mdc @@ -0,0 +1,34 @@ +--- +description: Chia test harness selection and routing to domain-specific guides +alwaysApply: true +--- + +# Chia Test Guide + +When writing or modifying tests under `chia/_tests/`, follow this sequence: + +## 1. Route to the right guide + +Identify the area from the user-provided path or intent, then read the matching guide **before any code search**: + +- `chia/_tests/blockchain/**` or consensus/reorg/overflow intent -> `testing-guide-blockchain.mdc` +- `chia/_tests/core/full_node/**` or sync/full-node intent -> `testing-guide-full-node.mdc` +- `chia/_tests/core/mempool/**` or mempool intent -> `testing-guide-mempool.mdc` +- `chia/_tests/core/data_layer/**` or data-layer intent -> `testing-guide-data-layer.mdc` +- `chia/_tests/core/server/**` or server/network intent -> `testing-guide-server.mdc` + +After reading, state: `Guide loaded: (area: )`. +If area is ambiguous, ask one clarifying question instead of broad repository search. + +## 2. Pick the right harness + +1. **Pure logic / parsing / utility** — no simulator, standard pytest. +2. **Chain state / farming / wallets / mempool / reorg** — simulator fixtures from `chia/_tests/conftest.py`: + - `simulator_and_wallet`, `one_node`, `two_nodes`, `one_node_one_block` + - `wallet_environments` for wallet-heavy tests +3. **Service wiring (farmer/harvester/solver/timelord)** — `setup_*` in `chia/_tests/util/setup_nodes.py`. +4. **Lightweight CLVM spend sim** — `sim_and_client()` from `chia/_tests/util/spend_sim.py`. + +Key shared utilities: `chia/_tests/conftest.py`, `chia/simulator/block_tools.py`, `chia/_tests/util/time_out_assert.py`, `chia/_tests/environments/wallet.py`. + +For detailed block/tx/assertion patterns, see `testing-guide-patterns.mdc`. diff --git a/.cursor/rules/use-venv-wrappers.mdc b/.cursor/rules/use-venv-wrappers.mdc new file mode 100644 index 0000000000..1eff393558 --- /dev/null +++ b/.cursor/rules/use-venv-wrappers.mdc @@ -0,0 +1,13 @@ +--- +description: Force repo venv wrappers for Python commands +alwaysApply: true +--- + +# Python Venv Guardrail + +In this workspace, always execute Python tooling via repository wrappers: + +- Use `tools/py` for Python and pip/module commands. +- Use `tools/pytest` for tests. + +Do not use bare `python`, `python3`, `pip`, or `pytest`. diff --git a/.gitignore b/.gitignore index 073acb4b89..c1a0acf8b3 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,13 @@ venv*/ venv* activate +# Cursor rules and context (shared via git) +!.cursor/ +!.cursor/rules/ +!.cursor/rules/** +!.cursor/context/ +!.cursor/context/** + # Editors .vscode .idea diff --git a/tools/py b/tools/py new file mode 100755 index 0000000000..de09a8e224 --- /dev/null +++ b/tools/py @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_DIRECTORY=$( + cd -- "$(dirname -- "$0")" + pwd +) +REPO_ROOT=$( + cd -- "${SCRIPT_DIRECTORY}/.." + pwd +) + +PYTHON_BIN="" +if [ -x "${REPO_ROOT}/.venv/bin/python" ]; then + PYTHON_BIN="${REPO_ROOT}/.venv/bin/python" +elif [ -x "${REPO_ROOT}/venv/bin/python" ]; then + PYTHON_BIN="${REPO_ROOT}/venv/bin/python" +else + echo "error: missing virtualenv python. expected ${REPO_ROOT}/.venv/bin/python or ${REPO_ROOT}/venv/bin/python" >&2 + exit 1 +fi + +exec "${PYTHON_BIN}" "$@" diff --git a/tools/pytest b/tools/pytest new file mode 100755 index 0000000000..913571a3cc --- /dev/null +++ b/tools/pytest @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +SCRIPT_DIRECTORY=$( + cd -- "$(dirname -- "$0")" + pwd +) + +exec "${SCRIPT_DIRECTORY}/py" -m pytest "$@"