only compile clvm for developers (#13478)

* only compile clvm for developers

* retain existing behavior for backwards compatibility

* tests/check_clvm.py becomes tools/manage_clvm.py check (and add build subcommand)

* trigger recompilation in pytest runs

.pre-commit-config.yaml

@@ -33,7 +33,7 @@ repos:
     hooks:
     -   id: clvm_hex
         name: .clvm.hex files
-        entry: ./activated.py python tests/check_clvm.py
+        entry: ./activated.py python tools/manage_clvm.py check
         language: python
         pass_filenames: false
 -   repo: local

chia/clvm/singleton.py

@@ -1,5 +1,5 @@
-from chia.wallet.puzzles.load_clvm import load_clvm
+from chia.wallet.puzzles.load_clvm import load_clvm_maybe_recompile
 
-P2_SINGLETON_MOD = load_clvm("p2_singleton.clvm")
-SINGLETON_TOP_LAYER_MOD = load_clvm("singleton_top_layer.clvm")
-SINGLETON_LAUNCHER = load_clvm("singleton_launcher.clvm")
+P2_SINGLETON_MOD = load_clvm_maybe_recompile("p2_singleton.clvm")
+SINGLETON_TOP_LAYER_MOD = load_clvm_maybe_recompile("singleton_top_layer.clvm")
+SINGLETON_LAUNCHER = load_clvm_maybe_recompile("singleton_launcher.clvm")

chia/full_node/generator.py

@@ -3,16 +3,19 @@ from typing import List, Optional, Union, Tuple
 from chia.types.blockchain_format.program import Program, SerializedProgram
 from chia.types.generator_types import BlockGenerator, GeneratorBlockCacheInterface, CompressorArg
 from chia.util.ints import uint32
-from chia.wallet.puzzles.load_clvm import load_clvm
+from chia.wallet.puzzles.load_clvm import load_clvm_maybe_recompile
 from chia.wallet.puzzles.rom_bootstrap_generator import get_generator
 
 GENERATOR_MOD = get_generator()
 
-DECOMPRESS_BLOCK = load_clvm("block_program_zero.clvm", package_or_requirement="chia.wallet.puzzles")
-DECOMPRESS_PUZZLE = load_clvm("decompress_puzzle.clvm", package_or_requirement="chia.wallet.puzzles")
-# DECOMPRESS_CSE = load_clvm("decompress_coin_spend_entry.clvm", package_or_requirement="chia.wallet.puzzles")
+DECOMPRESS_BLOCK = load_clvm_maybe_recompile("block_program_zero.clvm", package_or_requirement="chia.wallet.puzzles")
+DECOMPRESS_PUZZLE = load_clvm_maybe_recompile("decompress_puzzle.clvm", package_or_requirement="chia.wallet.puzzles")
+# DECOMPRESS_CSE = load_clvm_maybe_recompile(
+#     "decompress_coin_spend_entry.clvm",
+#     package_or_requirement="chia.wallet.puzzles",
+# )
 
-DECOMPRESS_CSE_WITH_PREFIX = load_clvm(
+DECOMPRESS_CSE_WITH_PREFIX = load_clvm_maybe_recompile(
     "decompress_coin_spend_entry_with_prefix.clvm", package_or_requirement="chia.wallet.puzzles"
 )
 log = logging.getLogger(__name__)

chia/pools/pool_puzzles.py

@@ -13,17 +13,17 @@ from chia.types.blockchain_format.program import Program, SerializedProgram
 
 from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.types.coin_spend import CoinSpend
-from chia.wallet.puzzles.load_clvm import load_clvm
+from chia.wallet.puzzles.load_clvm import load_clvm_maybe_recompile
 from chia.wallet.puzzles.singleton_top_layer import puzzle_for_singleton
 
 from chia.util.ints import uint32, uint64
 
 log = logging.getLogger(__name__)
 # "Full" is the outer singleton, with the inner puzzle filled in
-SINGLETON_MOD = load_clvm("singleton_top_layer.clvm")
-POOL_WAITING_ROOM_MOD = load_clvm("pool_waitingroom_innerpuz.clvm")
-POOL_MEMBER_MOD = load_clvm("pool_member_innerpuz.clvm")
-P2_SINGLETON_MOD = load_clvm("p2_singleton_or_delayed_puzhash.clvm")
+SINGLETON_MOD = load_clvm_maybe_recompile("singleton_top_layer.clvm")
+POOL_WAITING_ROOM_MOD = load_clvm_maybe_recompile("pool_waitingroom_innerpuz.clvm")
+POOL_MEMBER_MOD = load_clvm_maybe_recompile("pool_member_innerpuz.clvm")
+P2_SINGLETON_MOD = load_clvm_maybe_recompile("p2_singleton_or_delayed_puzhash.clvm")
 POOL_OUTER_MOD = SINGLETON_MOD
 
 POOL_MEMBER_HASH = POOL_MEMBER_MOD.get_tree_hash()

chia/wallet/db_wallet/db_wallet_puzzles.py

@@ -5,17 +5,17 @@ from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.types.condition_opcodes import ConditionOpcode
 from chia.util.ints import uint64
 from chia.wallet.nft_wallet.nft_puzzles import NFT_STATE_LAYER_MOD, create_nft_layer_puzzle_with_curry_params
-from chia.wallet.puzzles.load_clvm import load_clvm
+from chia.wallet.puzzles.load_clvm import load_clvm_maybe_recompile
 
 # from chia.types.condition_opcodes import ConditionOpcode
 # from chia.wallet.util.merkle_tree import MerkleTree, TreeType
 
 ACS_MU = Program.to(11)  # returns the third argument a.k.a the full solution
 ACS_MU_PH = ACS_MU.get_tree_hash()
-SINGLETON_TOP_LAYER_MOD = load_clvm("singleton_top_layer_v1_1.clvm")
-SINGLETON_LAUNCHER = load_clvm("singleton_launcher.clvm")
-GRAFTROOT_DL_OFFERS = load_clvm("graftroot_dl_offers.clvm")
-P2_PARENT = load_clvm("p2_parent.clvm")
+SINGLETON_TOP_LAYER_MOD = load_clvm_maybe_recompile("singleton_top_layer_v1_1.clvm")
+SINGLETON_LAUNCHER = load_clvm_maybe_recompile("singleton_launcher.clvm")
+GRAFTROOT_DL_OFFERS = load_clvm_maybe_recompile("graftroot_dl_offers.clvm")
+P2_PARENT = load_clvm_maybe_recompile("p2_parent.clvm")
 
 
 def create_host_fullpuz(innerpuz: Union[Program, bytes32], current_root: bytes32, genesis_id: bytes32) -> Program:

chia/wallet/did_wallet/did_wallet_puzzles.py

@@ -5,15 +5,15 @@ from blspy import G1Element
 from chia.types.blockchain_format.coin import Coin
 from chia.types.coin_spend import CoinSpend
 from chia.util.ints import uint64
-from chia.wallet.puzzles.load_clvm import load_clvm
+from chia.wallet.puzzles.load_clvm import load_clvm_maybe_recompile
 from chia.types.condition_opcodes import ConditionOpcode
 from chia.wallet.util.curry_and_treehash import calculate_hash_of_quoted_mod_hash, curry_and_treehash
 
-SINGLETON_TOP_LAYER_MOD = load_clvm("singleton_top_layer_v1_1.clvm")
+SINGLETON_TOP_LAYER_MOD = load_clvm_maybe_recompile("singleton_top_layer_v1_1.clvm")
 SINGLETON_TOP_LAYER_MOD_HASH = SINGLETON_TOP_LAYER_MOD.get_tree_hash()
 SINGLETON_TOP_LAYER_MOD_HASH_QUOTED = calculate_hash_of_quoted_mod_hash(SINGLETON_TOP_LAYER_MOD_HASH)
-LAUNCHER_PUZZLE = load_clvm("singleton_launcher.clvm")
-DID_INNERPUZ_MOD = load_clvm("did_innerpuz.clvm")
+LAUNCHER_PUZZLE = load_clvm_maybe_recompile("singleton_launcher.clvm")
+DID_INNERPUZ_MOD = load_clvm_maybe_recompile("did_innerpuz.clvm")
 LAUNCHER_PUZZLE_HASH = LAUNCHER_PUZZLE.get_tree_hash()
 DID_INNERPUZ_MOD_HASH = DID_INNERPUZ_MOD.get_tree_hash()
 

chia/wallet/nft_wallet/metadata_outer_puzzle.py

@@ -8,10 +8,10 @@ from chia.types.blockchain_format.program import Program
 from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.util.ints import uint64
 from chia.wallet.puzzle_drivers import PuzzleInfo, Solver
-from chia.wallet.puzzles.load_clvm import load_clvm
+from chia.wallet.puzzles.load_clvm import load_clvm_maybe_recompile
 from chia.wallet.uncurried_puzzle import UncurriedPuzzle, uncurry_puzzle
 
-NFT_STATE_LAYER_MOD = load_clvm("nft_state_layer.clvm")
+NFT_STATE_LAYER_MOD = load_clvm_maybe_recompile("nft_state_layer.clvm")
 NFT_STATE_LAYER_MOD_HASH = NFT_STATE_LAYER_MOD.get_tree_hash()
 
 

chia/wallet/nft_wallet/nft_info.py

@@ -7,9 +7,9 @@ from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.util.ints import uint16, uint32, uint64
 from chia.util.streamable import Streamable, streamable
 from chia.wallet.lineage_proof import LineageProof
-from chia.wallet.puzzles.load_clvm import load_clvm
+from chia.wallet.puzzles.load_clvm import load_clvm_maybe_recompile
 
-LAUNCHER_PUZZLE = load_clvm("singleton_launcher.clvm")
+LAUNCHER_PUZZLE = load_clvm_maybe_recompile("singleton_launcher.clvm")
 IN_TRANSACTION_STATUS = "IN_TRANSACTION"
 DEFAULT_STATUS = "DEFAULT"
 

chia/wallet/nft_wallet/nft_puzzles.py

@@ -10,21 +10,23 @@ from chia.util.ints import uint16, uint64
 from chia.wallet.nft_wallet.nft_info import NFTCoinInfo, NFTInfo
 from chia.wallet.nft_wallet.nft_off_chain import get_off_chain_metadata
 from chia.wallet.nft_wallet.uncurry_nft import UncurriedNFT
-from chia.wallet.puzzles.load_clvm import load_clvm
+from chia.wallet.puzzles.load_clvm import load_clvm_maybe_recompile
 from chia.wallet.puzzles.p2_delegated_puzzle_or_hidden_puzzle import solution_for_conditions
 
 log = logging.getLogger(__name__)
-SINGLETON_TOP_LAYER_MOD = load_clvm("singleton_top_layer_v1_1.clvm")
-LAUNCHER_PUZZLE = load_clvm("singleton_launcher.clvm")
-NFT_STATE_LAYER_MOD = load_clvm("nft_state_layer.clvm")
+SINGLETON_TOP_LAYER_MOD = load_clvm_maybe_recompile("singleton_top_layer_v1_1.clvm")
+LAUNCHER_PUZZLE = load_clvm_maybe_recompile("singleton_launcher.clvm")
+NFT_STATE_LAYER_MOD = load_clvm_maybe_recompile("nft_state_layer.clvm")
 LAUNCHER_PUZZLE_HASH = LAUNCHER_PUZZLE.get_tree_hash()
 SINGLETON_MOD_HASH = SINGLETON_TOP_LAYER_MOD.get_tree_hash()
 NFT_STATE_LAYER_MOD_HASH = NFT_STATE_LAYER_MOD.get_tree_hash()
-NFT_METADATA_UPDATER = load_clvm("nft_metadata_updater_default.clvm")
-NFT_OWNERSHIP_LAYER = load_clvm("nft_ownership_layer.clvm")
+NFT_METADATA_UPDATER = load_clvm_maybe_recompile("nft_metadata_updater_default.clvm")
+NFT_OWNERSHIP_LAYER = load_clvm_maybe_recompile("nft_ownership_layer.clvm")
 NFT_OWNERSHIP_LAYER_HASH = NFT_OWNERSHIP_LAYER.get_tree_hash()
-NFT_TRANSFER_PROGRAM_DEFAULT = load_clvm("nft_ownership_transfer_program_one_way_claim_with_royalties.clvm")
-STANDARD_PUZZLE_MOD = load_clvm("p2_delegated_puzzle_or_hidden_puzzle.clvm")
+NFT_TRANSFER_PROGRAM_DEFAULT = load_clvm_maybe_recompile(
+    "nft_ownership_transfer_program_one_way_claim_with_royalties.clvm",
+)
+STANDARD_PUZZLE_MOD = load_clvm_maybe_recompile("p2_delegated_puzzle_or_hidden_puzzle.clvm")
 
 
 def create_nft_layer_puzzle_with_curry_params(

chia/wallet/nft_wallet/ownership_outer_puzzle.py

@@ -6,10 +6,10 @@ from clvm_tools.binutils import disassemble
 from chia.types.blockchain_format.program import Program
 from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.wallet.puzzle_drivers import PuzzleInfo, Solver
-from chia.wallet.puzzles.load_clvm import load_clvm
+from chia.wallet.puzzles.load_clvm import load_clvm_maybe_recompile
 from chia.wallet.uncurried_puzzle import UncurriedPuzzle, uncurry_puzzle
 
-OWNERSHIP_LAYER_MOD = load_clvm("nft_ownership_layer.clvm")
+OWNERSHIP_LAYER_MOD = load_clvm_maybe_recompile("nft_ownership_layer.clvm")
 
 
 def match_ownership_layer_puzzle(puzzle: UncurriedPuzzle) -> Tuple[bool, List[Program]]:

chia/wallet/nft_wallet/transfer_program_puzzle.py

@@ -5,11 +5,11 @@ from chia.types.blockchain_format.program import Program
 from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.util.ints import uint16
 from chia.wallet.puzzle_drivers import PuzzleInfo, Solver
-from chia.wallet.puzzles.load_clvm import load_clvm
+from chia.wallet.puzzles.load_clvm import load_clvm_maybe_recompile
 from chia.wallet.puzzles.singleton_top_layer_v1_1 import SINGLETON_LAUNCHER_HASH, SINGLETON_MOD_HASH
 from chia.wallet.uncurried_puzzle import UncurriedPuzzle
 
-TRANSFER_PROGRAM_MOD = load_clvm("nft_ownership_transfer_program_one_way_claim_with_royalties.clvm")
+TRANSFER_PROGRAM_MOD = load_clvm_maybe_recompile("nft_ownership_transfer_program_one_way_claim_with_royalties.clvm")
 
 
 def match_transfer_program_puzzle(puzzle: UncurriedPuzzle) -> Tuple[bool, List[Program]]:

chia/wallet/nft_wallet/uncurry_nft.py

@@ -7,12 +7,12 @@ from typing import Optional, Type, TypeVar
 from chia.types.blockchain_format.program import Program
 from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.util.ints import uint16
-from chia.wallet.puzzles.load_clvm import load_clvm
+from chia.wallet.puzzles.load_clvm import load_clvm_maybe_recompile
 
 log = logging.getLogger(__name__)
-SINGLETON_TOP_LAYER_MOD = load_clvm("singleton_top_layer_v1_1.clvm")
-NFT_MOD = load_clvm("nft_state_layer.clvm")
-NFT_OWNERSHIP_LAYER = load_clvm("nft_ownership_layer.clvm")
+SINGLETON_TOP_LAYER_MOD = load_clvm_maybe_recompile("singleton_top_layer_v1_1.clvm")
+NFT_MOD = load_clvm_maybe_recompile("nft_state_layer.clvm")
+NFT_OWNERSHIP_LAYER = load_clvm_maybe_recompile("nft_ownership_layer.clvm")
 
 _T_UncurriedNFT = TypeVar("_T_UncurriedNFT", bound="UncurriedNFT")
 

chia/wallet/puzzles/cat_loader.py

@@ -1,6 +1,6 @@
-from chia.wallet.puzzles.load_clvm import load_clvm
+from chia.wallet.puzzles.load_clvm import load_clvm_maybe_recompile
 
-CAT_MOD = load_clvm("cat_v2.clvm", package_or_requirement=__name__)
-LOCK_INNER_PUZZLE = load_clvm("lock.inner.puzzle.clvm", package_or_requirement=__name__)
+CAT_MOD = load_clvm_maybe_recompile("cat_v2.clvm", package_or_requirement=__name__)
+LOCK_INNER_PUZZLE = load_clvm_maybe_recompile("lock.inner.puzzle.clvm", package_or_requirement=__name__)
 
 CAT_MOD_HASH = CAT_MOD.get_tree_hash()

chia/wallet/puzzles/load_clvm.py

@@ -1,6 +1,7 @@
 import importlib
 import inspect
 import os
+import sys
 
 import tempfile
 import pathlib
@@ -14,6 +15,8 @@ from clvm_tools_rs import compile_clvm as compile_clvm_rust
 
 compile_clvm_py = None
 
+recompile_requested = (os.environ.get("CHIA_DEV_COMPILE_CLVM_ON_IMPORT", "") != "") or ("pytest" in sys.modules)
+
 
 def translate_path(p_):
     p = str(p_)
@@ -74,7 +77,7 @@ def compile_clvm(full_path: pathlib.Path, output: pathlib.Path, search_paths: Li
 
 
 def load_serialized_clvm(
-    clvm_filename, package_or_requirement=__name__, include_standard_libraries: bool = False
+    clvm_filename, package_or_requirement=__name__, include_standard_libraries: bool = False, recompile: bool = True
 ) -> SerializedProgram:
     """
     This function takes a .clvm file in the given package and compiles it to a
@@ -86,22 +89,25 @@ def load_serialized_clvm(
     """
     hex_filename = f"{clvm_filename}.hex"
 
-    try:
-        if pkg_resources.resource_exists(package_or_requirement, clvm_filename):
-            # Establish whether the size is zero on entry
-            full_path = pathlib.Path(pkg_resources.resource_filename(package_or_requirement, clvm_filename))
-            output = full_path.parent / hex_filename
-            search_paths = [full_path.parent]
-            if include_standard_libraries:
-                # we can't get the dir, but we can get a file then get its parent.
-                chia_puzzles_path = pathlib.Path(pkg_resources.resource_filename(__name__, "__init__.py")).parent
-                search_paths.append(chia_puzzles_path)
-            compile_clvm(full_path, output, search_paths=search_paths)
+    # Set the CHIA_DEV_COMPILE_CLVM_ON_IMPORT environment variable to anything except
+    # "" or "0" to trigger automatic recompilation of the Chialisp on load.
+    if recompile:
+        try:
+            if pkg_resources.resource_exists(package_or_requirement, clvm_filename):
+                # Establish whether the size is zero on entry
+                full_path = pathlib.Path(pkg_resources.resource_filename(package_or_requirement, clvm_filename))
+                output = full_path.parent / hex_filename
+                search_paths = [full_path.parent]
+                if include_standard_libraries:
+                    # we can't get the dir, but we can get a file then get its parent.
+                    chia_puzzles_path = pathlib.Path(pkg_resources.resource_filename(__name__, "__init__.py")).parent
+                    search_paths.append(chia_puzzles_path)
+                compile_clvm(full_path, output, search_paths=search_paths)
 
-    except NotImplementedError:
-        # pyinstaller doesn't support `pkg_resources.resource_exists`
-        # so we just fall through to loading the hex clvm
-        pass
+        except NotImplementedError:
+            # pyinstaller doesn't support `pkg_resources.resource_exists`
+            # so we just fall through to loading the hex clvm
+            pass
 
     clvm_hex = pkg_resources.resource_string(package_or_requirement, hex_filename).decode("utf8")
     assert len(clvm_hex.strip()) != 0
@@ -109,13 +115,33 @@ def load_serialized_clvm(
     return SerializedProgram.from_bytes(clvm_blob)
 
 
-def load_clvm(clvm_filename, package_or_requirement=__name__, include_standard_libraries: bool = False) -> Program:
+def load_clvm(
+    clvm_filename,
+    package_or_requirement=__name__,
+    include_standard_libraries: bool = False,
+    recompile: bool = True,
+) -> Program:
     return Program.from_bytes(
         bytes(
             load_serialized_clvm(
                 clvm_filename,
                 package_or_requirement=package_or_requirement,
                 include_standard_libraries=include_standard_libraries,
+                recompile=recompile,
             )
         )
     )
+
+
+def load_clvm_maybe_recompile(
+    clvm_filename,
+    package_or_requirement=__name__,
+    include_standard_libraries: bool = False,
+    recompile: bool = recompile_requested,
+) -> Program:
+    return load_clvm(
+        clvm_filename=clvm_filename,
+        package_or_requirement=package_or_requirement,
+        include_standard_libraries=include_standard_libraries,
+        recompile=recompile,
+    )

chia/wallet/puzzles/p2_conditions.py

@@ -12,9 +12,9 @@ the doctor ordered.
 
 from chia.types.blockchain_format.program import Program
 
-from .load_clvm import load_clvm
+from .load_clvm import load_clvm_maybe_recompile
 
-MOD = load_clvm("p2_conditions.clvm")
+MOD = load_clvm_maybe_recompile("p2_conditions.clvm")
 
 
 def puzzle_for_conditions(conditions) -> Program:

chia/wallet/puzzles/p2_delegated_conditions.py

@@ -8,9 +8,9 @@ is returned literally.
 
 from chia.types.blockchain_format.program import Program
 
-from .load_clvm import load_clvm
+from .load_clvm import load_clvm_maybe_recompile
 
-MOD = load_clvm("p2_delegated_conditions.clvm")
+MOD = load_clvm_maybe_recompile("p2_delegated_conditions.clvm")
 
 
 def puzzle_for_pk(public_key: Program) -> Program:

chia/wallet/puzzles/p2_delegated_puzzle.py

@@ -15,9 +15,9 @@ This roughly corresponds to bitcoin's graftroot.
 from chia.types.blockchain_format.program import Program
 
 from . import p2_conditions
-from .load_clvm import load_clvm
+from .load_clvm import load_clvm_maybe_recompile
 
-MOD = load_clvm("p2_delegated_puzzle.clvm")
+MOD = load_clvm_maybe_recompile("p2_delegated_puzzle.clvm")
 
 
 def puzzle_for_pk(public_key: bytes) -> Program:

chia/wallet/puzzles/p2_delegated_puzzle_or_hidden_puzzle.py

@@ -65,14 +65,14 @@ from chia.types.blockchain_format.program import Program
 from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.wallet.util.curry_and_treehash import calculate_hash_of_quoted_mod_hash, curry_and_treehash
 
-from .load_clvm import load_clvm
+from .load_clvm import load_clvm_maybe_recompile
 from .p2_conditions import puzzle_for_conditions
 
 DEFAULT_HIDDEN_PUZZLE = Program.from_bytes(bytes.fromhex("ff0980"))
 
 DEFAULT_HIDDEN_PUZZLE_HASH = DEFAULT_HIDDEN_PUZZLE.get_tree_hash()  # this puzzle `(x)` always fails
 
-MOD = load_clvm("p2_delegated_puzzle_or_hidden_puzzle.clvm")
+MOD = load_clvm_maybe_recompile("p2_delegated_puzzle_or_hidden_puzzle.clvm")
 
 QUOTED_MOD_HASH = calculate_hash_of_quoted_mod_hash(MOD.get_tree_hash())
 

chia/wallet/puzzles/p2_m_of_n_delegate_direct.py

@@ -7,9 +7,9 @@ it includes N public keys, any M of which needs to sign the delegated puzzle.
 
 from chia.types.blockchain_format.program import Program
 
-from .load_clvm import load_clvm
+from .load_clvm import load_clvm_maybe_recompile
 
-MOD = load_clvm("p2_m_of_n_delegate_direct.clvm")
+MOD = load_clvm_maybe_recompile("p2_m_of_n_delegate_direct.clvm")
 
 
 def puzzle_for_m_of_public_key_list(m, public_key_list) -> Program:

chia/wallet/puzzles/p2_puzzle_hash.py

@@ -8,9 +8,9 @@ hash along with its solution.
 from chia.types.blockchain_format.program import Program
 from chia.types.blockchain_format.sized_bytes import bytes32
 
-from .load_clvm import load_clvm
+from .load_clvm import load_clvm_maybe_recompile
 
-MOD = load_clvm("p2_puzzle_hash.clvm")
+MOD = load_clvm_maybe_recompile("p2_puzzle_hash.clvm")
 
 
 def puzzle_for_inner_puzzle_hash(inner_puzzle_hash: bytes32) -> Program:

chia/wallet/puzzles/singleton_top_layer.py

@@ -5,16 +5,16 @@ from chia.types.blockchain_format.program import Program
 from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.types.condition_opcodes import ConditionOpcode
 from chia.types.coin_spend import CoinSpend
-from chia.wallet.puzzles.load_clvm import load_clvm
+from chia.wallet.puzzles.load_clvm import load_clvm_maybe_recompile
 from chia.wallet.lineage_proof import LineageProof
 from chia.util.ints import uint64
 from chia.util.hash import std_hash
 
-SINGLETON_MOD = load_clvm("singleton_top_layer.clvm")
+SINGLETON_MOD = load_clvm_maybe_recompile("singleton_top_layer.clvm")
 SINGLETON_MOD_HASH = SINGLETON_MOD.get_tree_hash()
-P2_SINGLETON_MOD = load_clvm("p2_singleton.clvm")
-P2_SINGLETON_OR_DELAYED_MOD = load_clvm("p2_singleton_or_delayed_puzhash.clvm")
-SINGLETON_LAUNCHER = load_clvm("singleton_launcher.clvm")
+P2_SINGLETON_MOD = load_clvm_maybe_recompile("p2_singleton.clvm")
+P2_SINGLETON_OR_DELAYED_MOD = load_clvm_maybe_recompile("p2_singleton_or_delayed_puzhash.clvm")
+SINGLETON_LAUNCHER = load_clvm_maybe_recompile("singleton_launcher.clvm")
 SINGLETON_LAUNCHER_HASH = SINGLETON_LAUNCHER.get_tree_hash()
 ESCAPE_VALUE = -113
 MELT_CONDITION = [ConditionOpcode.CREATE_COIN, 0, ESCAPE_VALUE]

chia/wallet/puzzles/singleton_top_layer_v1_1.py

@@ -8,14 +8,14 @@ from chia.types.condition_opcodes import ConditionOpcode
 from chia.util.hash import std_hash
 from chia.util.ints import uint64
 from chia.wallet.lineage_proof import LineageProof
-from chia.wallet.puzzles.load_clvm import load_clvm
+from chia.wallet.puzzles.load_clvm import load_clvm_maybe_recompile
 from chia.wallet.uncurried_puzzle import UncurriedPuzzle
 
-SINGLETON_MOD = load_clvm("singleton_top_layer_v1_1.clvm")
+SINGLETON_MOD = load_clvm_maybe_recompile("singleton_top_layer_v1_1.clvm")
 SINGLETON_MOD_HASH = SINGLETON_MOD.get_tree_hash()
-P2_SINGLETON_MOD = load_clvm("p2_singleton.clvm")
-P2_SINGLETON_OR_DELAYED_MOD = load_clvm("p2_singleton_or_delayed_puzhash.clvm")
-SINGLETON_LAUNCHER = load_clvm("singleton_launcher.clvm")
+P2_SINGLETON_MOD = load_clvm_maybe_recompile("p2_singleton.clvm")
+P2_SINGLETON_OR_DELAYED_MOD = load_clvm_maybe_recompile("p2_singleton_or_delayed_puzhash.clvm")
+SINGLETON_LAUNCHER = load_clvm_maybe_recompile("singleton_launcher.clvm")
 SINGLETON_LAUNCHER_HASH = SINGLETON_LAUNCHER.get_tree_hash()
 ESCAPE_VALUE = -113
 MELT_CONDITION = [ConditionOpcode.CREATE_COIN, 0, ESCAPE_VALUE]

chia/wallet/puzzles/tails.py

@@ -7,7 +7,7 @@ from chia.util.ints import uint64
 from chia.util.byte_types import hexstr_to_bytes
 from chia.wallet.cat_wallet.lineage_store import CATLineageStore
 from chia.wallet.lineage_proof import LineageProof
-from chia.wallet.puzzles.load_clvm import load_clvm
+from chia.wallet.puzzles.load_clvm import load_clvm_maybe_recompile
 from chia.wallet.cat_wallet.cat_utils import (
     construct_cat_puzzle,
     unsigned_spend_bundle_for_spendable_cats,
@@ -17,10 +17,10 @@ from chia.wallet.puzzles.cat_loader import CAT_MOD
 from chia.wallet.cat_wallet.cat_info import CATInfo
 from chia.wallet.transaction_record import TransactionRecord
 
-GENESIS_BY_ID_MOD = load_clvm("genesis_by_coin_id.clvm")
-GENESIS_BY_PUZHASH_MOD = load_clvm("genesis_by_puzzle_hash.clvm")
-EVERYTHING_WITH_SIG_MOD = load_clvm("everything_with_signature.clvm")
-DELEGATED_LIMITATIONS_MOD = load_clvm("delegated_tail.clvm")
+GENESIS_BY_ID_MOD = load_clvm_maybe_recompile("genesis_by_coin_id.clvm")
+GENESIS_BY_PUZHASH_MOD = load_clvm_maybe_recompile("genesis_by_puzzle_hash.clvm")
+EVERYTHING_WITH_SIG_MOD = load_clvm_maybe_recompile("everything_with_signature.clvm")
+DELEGATED_LIMITATIONS_MOD = load_clvm_maybe_recompile("delegated_tail.clvm")
 
 
 class LimitationsProgram:

chia/wallet/trade_manager.py

@@ -23,7 +23,7 @@ from chia.wallet.nft_wallet.nft_wallet import NFTWallet
 from chia.wallet.outer_puzzles import AssetType
 from chia.wallet.payment import Payment
 from chia.wallet.puzzle_drivers import PuzzleInfo, Solver
-from chia.wallet.puzzles.load_clvm import load_clvm
+from chia.wallet.puzzles.load_clvm import load_clvm_maybe_recompile
 from chia.wallet.trade_record import TradeRecord
 from chia.wallet.trading.offer import NotarizedPayment, Offer
 from chia.wallet.trading.trade_status import TradeStatus
@@ -34,7 +34,7 @@ from chia.wallet.util.wallet_types import WalletType
 from chia.wallet.wallet import Wallet
 from chia.wallet.wallet_coin_record import WalletCoinRecord
 
-OFFER_MOD = load_clvm("settlement_payments.clvm")
+OFFER_MOD = load_clvm_maybe_recompile("settlement_payments.clvm")
 
 
 class TradeManager:

chia/wallet/trading/offer.py

@@ -23,7 +23,7 @@ from chia.wallet.outer_puzzles import (
 )
 from chia.wallet.payment import Payment
 from chia.wallet.puzzle_drivers import PuzzleInfo, Solver
-from chia.wallet.puzzles.load_clvm import load_clvm
+from chia.wallet.puzzles.load_clvm import load_clvm_maybe_recompile
 from chia.wallet.util.puzzle_compression import (
     compress_object_with_puzzles,
     decompress_object_with_puzzles,
@@ -31,7 +31,7 @@ from chia.wallet.util.puzzle_compression import (
 )
 from chia.wallet.uncurried_puzzle import UncurriedPuzzle, uncurry_puzzle
 
-OFFER_MOD = load_clvm("settlement_payments.clvm")
+OFFER_MOD = load_clvm_maybe_recompile("settlement_payments.clvm")
 OFFER_MOD_HASH = OFFER_MOD.get_tree_hash()
 ZERO_32 = bytes32([0] * 32)
 

chia/wallet/util/puzzle_compression.py

@@ -3,7 +3,7 @@ import zlib
 from typing import List
 
 from chia.types.blockchain_format.program import Program
-from chia.wallet.puzzles.load_clvm import load_clvm
+from chia.wallet.puzzles.load_clvm import load_clvm_maybe_recompile
 from chia.wallet.puzzles import p2_delegated_puzzle_or_hidden_puzzle as standard_puzzle
 from chia.wallet.puzzles.cat_loader import CAT_MOD
 from chia.wallet.nft_wallet.nft_puzzles import (
@@ -19,7 +19,7 @@ LEGACY_CAT_MOD = Program.fromhex(
     "ff02ffff01ff02ff5effff04ff02ffff04ffff04ff05ffff04ffff0bff2cff0580ffff04ff0bff80808080ffff04ffff02ff17ff2f80ffff04ff5fffff04ffff02ff2effff04ff02ffff04ff17ff80808080ffff04ffff0bff82027fff82057fff820b7f80ffff04ff81bfffff04ff82017fffff04ff8202ffffff04ff8205ffffff04ff820bffff80808080808080808080808080ffff04ffff01ffffffff81ca3dff46ff0233ffff3c04ff01ff0181cbffffff02ff02ffff03ff05ffff01ff02ff32ffff04ff02ffff04ff0dffff04ffff0bff22ffff0bff2cff3480ffff0bff22ffff0bff22ffff0bff2cff5c80ff0980ffff0bff22ff0bffff0bff2cff8080808080ff8080808080ffff010b80ff0180ffff02ffff03ff0bffff01ff02ffff03ffff09ffff02ff2effff04ff02ffff04ff13ff80808080ff820b9f80ffff01ff02ff26ffff04ff02ffff04ffff02ff13ffff04ff5fffff04ff17ffff04ff2fffff04ff81bfffff04ff82017fffff04ff1bff8080808080808080ffff04ff82017fff8080808080ffff01ff088080ff0180ffff01ff02ffff03ff17ffff01ff02ffff03ffff20ff81bf80ffff0182017fffff01ff088080ff0180ffff01ff088080ff018080ff0180ffff04ffff04ff05ff2780ffff04ffff10ff0bff5780ff778080ff02ffff03ff05ffff01ff02ffff03ffff09ffff02ffff03ffff09ff11ff7880ffff0159ff8080ff0180ffff01818f80ffff01ff02ff7affff04ff02ffff04ff0dffff04ff0bffff04ffff04ff81b9ff82017980ff808080808080ffff01ff02ff5affff04ff02ffff04ffff02ffff03ffff09ff11ff7880ffff01ff04ff78ffff04ffff02ff36ffff04ff02ffff04ff13ffff04ff29ffff04ffff0bff2cff5b80ffff04ff2bff80808080808080ff398080ffff01ff02ffff03ffff09ff11ff2480ffff01ff04ff24ffff04ffff0bff20ff2980ff398080ffff010980ff018080ff0180ffff04ffff02ffff03ffff09ff11ff7880ffff0159ff8080ff0180ffff04ffff02ff7affff04ff02ffff04ff0dffff04ff0bffff04ff17ff808080808080ff80808080808080ff0180ffff01ff04ff80ffff04ff80ff17808080ff0180ffffff02ffff03ff05ffff01ff04ff09ffff02ff26ffff04ff02ffff04ff0dffff04ff0bff808080808080ffff010b80ff0180ff0bff22ffff0bff2cff5880ffff0bff22ffff0bff22ffff0bff2cff5c80ff0580ffff0bff22ffff02ff32ffff04ff02ffff04ff07ffff04ffff0bff2cff2c80ff8080808080ffff0bff2cff8080808080ffff02ffff03ffff07ff0580ffff01ff0bffff0102ffff02ff2effff04ff02ffff04ff09ff80808080ffff02ff2effff04ff02ffff04ff0dff8080808080ffff01ff0bff2cff058080ff0180ffff04ffff04ff28ffff04ff5fff808080ffff02ff7effff04ff02ffff04ffff04ffff04ff2fff0580ffff04ff5fff82017f8080ffff04ffff02ff7affff04ff02ffff04ff0bffff04ff05ffff01ff808080808080ffff04ff17ffff04ff81bfffff04ff82017fffff04ffff0bff8204ffffff02ff36ffff04ff02ffff04ff09ffff04ff820affffff04ffff0bff2cff2d80ffff04ff15ff80808080808080ff8216ff80ffff04ff8205ffffff04ff820bffff808080808080808080808080ff02ff2affff04ff02ffff04ff5fffff04ff3bffff04ffff02ffff03ff17ffff01ff09ff2dffff0bff27ffff02ff36ffff04ff02ffff04ff29ffff04ff57ffff04ffff0bff2cff81b980ffff04ff59ff80808080808080ff81b78080ff8080ff0180ffff04ff17ffff04ff05ffff04ff8202ffffff04ffff04ffff04ff24ffff04ffff0bff7cff2fff82017f80ff808080ffff04ffff04ff30ffff04ffff0bff81bfffff0bff7cff15ffff10ff82017fffff11ff8202dfff2b80ff8202ff808080ff808080ff138080ff80808080808080808080ff018080"  # noqa
 )
 
-OFFER_MOD = load_clvm("settlement_payments.clvm")
+OFFER_MOD = load_clvm_maybe_recompile("settlement_payments.clvm")
 
 # For backwards compatibility to work, we must assume that these mods (already deployed) will not change
 # In the case that they do change and we don't support the old asset then we need to keep around the legacy module

tools/manage_clvm.py

@@ -6,7 +6,9 @@ import pathlib
 import sys
 import tempfile
 import traceback
+import typing
 
+import click
 import typing_extensions
 
 here = pathlib.Path(__file__).parent.resolve()
@@ -27,6 +29,9 @@ from chia.types.blockchain_format.program import SerializedProgram  # noqa: E402
 clvm_suffix = ".clvm"
 hex_suffix = ".clvm.hex"
 hash_suffix = ".clvm.hex.sha256tree"
+all_suffixes = {"clvm": clvm_suffix, "hex": hex_suffix, "hash": hash_suffix}
+# TODO: could be cli options
+top_levels = {"chia"}
 
 
 def generate_hash_bytes(hex_bytes: bytes) -> bytes:
@@ -77,12 +82,10 @@ class ClvmBytes:
 excludes = {"condition_codes.clvm", "create-lock-puzzlehash.clvm"}
 
 
-def main() -> int:
-    used_excludes = set()
-    overall_fail = False
-
-    suffixes = {"clvm": clvm_suffix, "hex": hex_suffix, "hash": hash_suffix}
-    top_levels = {"chia"}
+def find_stems(
+    top_levels: typing.Set[str],
+    suffixes: typing.Mapping[str, str] = all_suffixes,
+) -> typing.Dict[str, typing.Set[pathlib.Path]]:
     found_stems = {
         name: {
             path.with_name(path.name[: -len(suffix)])
@@ -91,9 +94,23 @@ def main() -> int:
         }
         for name, suffix in suffixes.items()
     }
+    return found_stems
+
+
+@click.group()
+def main() -> None:
+    pass
+
+
+@main.command()
+def check() -> int:
+    used_excludes = set()
+    overall_fail = False
+
+    found_stems = find_stems(top_levels)
     for name in ["hex", "hash"]:
         found = found_stems[name]
-        suffix = suffixes[name]
+        suffix = all_suffixes[name]
         extra = found - found_stems["clvm"]
 
         print()
@@ -164,4 +181,52 @@ def main() -> int:
     return 1 if overall_fail else 0
 
 
+@main.command()
+def build() -> int:
+    overall_fail = False
+
+    found_stems = find_stems(top_levels, suffixes={"clvm": clvm_suffix})
+
+    print(f"Building all existing {clvm_suffix} files to {hex_suffix}:")
+    for stem_path in sorted(found_stems["clvm"]):
+        clvm_path = stem_path.with_name(stem_path.name + clvm_suffix)
+        if clvm_path.name in excludes:
+            continue
+
+        file_fail = False
+        error = None
+
+        try:
+            reference_paths = ClvmPaths.from_clvm(clvm=clvm_path)
+
+            with tempfile.TemporaryDirectory() as temporary_directory:
+                generated_paths = ClvmPaths.from_clvm(
+                    clvm=pathlib.Path(temporary_directory).joinpath(f"generated{clvm_suffix}")
+                )
+
+                compile_clvm(
+                    input_path=os.fspath(reference_paths.clvm),
+                    output_path=os.fspath(generated_paths.hex),
+                    search_paths=[os.fspath(reference_paths.clvm.parent)],
+                )
+
+                generated_bytes = ClvmBytes.from_hex_bytes(hex_bytes=generated_paths.hex.read_bytes())
+                reference_paths.hex.write_bytes(generated_bytes.hex)
+        except Exception:
+            file_fail = True
+            error = traceback.format_exc()
+
+        if file_fail:
+            print(f"FAIL     : {clvm_path}")
+            if error is not None:
+                print(error)
+        else:
+            print(f"    built: {clvm_path}")
+
+        if file_fail:
+            overall_fail = True
+
+    return 1 if overall_fail else 0
+
+
 sys.exit(main())