From 02bf8489d2d9ca5350ba36f4f8bee499006d26d6 Mon Sep 17 00:00:00 2001 From: Mariano Sorgente <3069354+mariano54@users.noreply.github.com> Date: Thu, 31 Mar 2022 11:23:34 -0400 Subject: [PATCH 01/63] Ms.parallel pool t (#10966) * Try parallel pool tests * Also change workflow files * Run less combinations * Todo for bad test * Try lower n --- .github/workflows/build-test-macos-pools.yml | 2 +- .github/workflows/build-test-ubuntu-pools.yml | 2 +- tests/block_tools.py | 7 +- tests/build-workflows.py | 5 +- tests/pools/config.py | 2 + tests/pools/test_pool_rpc.py | 83 +++++++++---------- 6 files changed, 54 insertions(+), 47 deletions(-) diff --git a/.github/workflows/build-test-macos-pools.yml b/.github/workflows/build-test-macos-pools.yml index 80ff69a5db..ced34bc307 100644 --- a/.github/workflows/build-test-macos-pools.yml +++ b/.github/workflows/build-test-macos-pools.yml @@ -85,7 +85,7 @@ jobs: - name: Test pools code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/pools/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/pools/test_*.py -s -v --durations 0 -n 2 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-pools.yml b/.github/workflows/build-test-ubuntu-pools.yml index d083a12d7c..91e5d819dc 100644 --- a/.github/workflows/build-test-ubuntu-pools.yml +++ b/.github/workflows/build-test-ubuntu-pools.yml @@ -84,7 +84,7 @@ jobs: - name: Test pools code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/pools/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/pools/test_*.py -s -v --durations 0 -n 2 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/tests/block_tools.py b/tests/block_tools.py index dedd9d4212..f358b955cf 100644 --- a/tests/block_tools.py +++ b/tests/block_tools.py @@ -265,6 +265,7 @@ class BlockTools: self, pool_contract_puzzle_hash: Optional[bytes32] = None, path: Path = None, + tmp_dir: Path = None, plot_keys: Optional[PlotKeys] = None, exclude_final_dir: bool = False, ) -> Optional[bytes32]: @@ -272,14 +273,16 @@ class BlockTools: if path is not None: final_dir = path mkdir(final_dir) + if tmp_dir is None: + tmp_dir = self.temp_dir args = Namespace() # Can't go much lower than 20, since plots start having no solutions and more buggy args.size = 22 # Uses many plots for testing, in order to guarantee proofs of space at every height args.num = 1 args.buffer = 100 - args.tmp_dir = self.temp_dir - args.tmp2_dir = self.temp_dir + args.tmp_dir = tmp_dir + args.tmp2_dir = tmp_dir args.final_dir = final_dir args.plotid = None args.memo = None diff --git a/tests/build-workflows.py b/tests/build-workflows.py index b9c8e2c0d4..3898390a48 100755 --- a/tests/build-workflows.py +++ b/tests/build-workflows.py @@ -83,7 +83,10 @@ def generate_replacements(conf, dir): ] = "# Omitted checking out blocks and plots repo Chia-Network/test-cache" if not conf["install_timelord"]: replacements["INSTALL_TIMELORD"] = "# Omitted installing Timelord" - replacements["PYTEST_PARALLEL_ARGS"] = " -n 4" if conf["parallel"] else " -n 0" + if conf.get("custom_parallel_n", None): + replacements["PYTEST_PARALLEL_ARGS"] = f" -n {conf['custom_parallel_n']}" + else: + replacements["PYTEST_PARALLEL_ARGS"] = " -n 4" if conf["parallel"] else " -n 0" if conf["job_timeout"]: replacements["JOB_TIMEOUT"] = str(conf["job_timeout"]) replacements["TEST_DIR"] = "/".join([*dir.relative_to(root_path.parent).parts, "test_*.py"]) diff --git a/tests/pools/config.py b/tests/pools/config.py index d9b815b24c..bc4811c2c5 100644 --- a/tests/pools/config.py +++ b/tests/pools/config.py @@ -1 +1,3 @@ +custom_parallel_n = 2 +parallel = True job_timeout = 60 diff --git a/tests/pools/test_pool_rpc.py b/tests/pools/test_pool_rpc.py index 0a4d996859..d49b0b3103 100644 --- a/tests/pools/test_pool_rpc.py +++ b/tests/pools/test_pool_rpc.py @@ -1,5 +1,6 @@ import asyncio import logging +import tempfile from dataclasses import dataclass from pathlib import Path from shutil import rmtree @@ -64,7 +65,9 @@ class TemporaryPoolPlot: plot_id: Optional[bytes32] = None async def __aenter__(self): - plot_id: bytes32 = await self.bt.new_plot(self.p2_singleton_puzzle_hash, get_pool_plot_dir()) + self._tmpdir = tempfile.TemporaryDirectory() + dirname = self._tmpdir.__enter__() + plot_id: bytes32 = await self.bt.new_plot(self.p2_singleton_puzzle_hash, Path(dirname), tmp_dir=Path(dirname)) assert plot_id is not None await self.bt.refresh_plots() self.plot_id = plot_id @@ -72,12 +75,7 @@ class TemporaryPoolPlot: async def __aexit__(self, exc_type, exc_value, exc_traceback): await self.bt.delete_plot(self.plot_id) - - -async def create_pool_plot(bt: BlockTools, p2_singleton_puzzle_hash: bytes32) -> Optional[bytes32]: - plot_id = await bt.new_plot(p2_singleton_puzzle_hash, get_pool_plot_dir()) - await bt.refresh_plots() - return plot_id + self._tmpdir.__exit__(None, None, None) async def wallet_is_synced(wallet_node: WalletNode, full_node_api): @@ -165,9 +163,9 @@ async def setup(two_wallet_nodes, bt, self_hostname): class TestPoolWalletRpc: @pytest.mark.asyncio - @pytest.mark.parametrize("trusted", [True, False]) - @pytest.mark.parametrize("fee", [0, FEE_AMOUNT]) - async def test_create_new_pool_wallet_self_farm(self, one_wallet_node_and_rpc, fee, trusted, self_hostname): + @pytest.mark.parametrize("trusted_and_fee", [(True, FEE_AMOUNT), (False, 0)]) + async def test_create_new_pool_wallet_self_farm(self, one_wallet_node_and_rpc, trusted_and_fee, self_hostname): + trusted, fee = trusted_and_fee client, wallet_node_0, full_node_api = one_wallet_node_and_rpc wallet_0 = wallet_node_0.wallet_state_manager.main_wallet if trusted: @@ -235,9 +233,9 @@ class TestPoolWalletRpc: assert pool_config["pool_url"] == "" @pytest.mark.asyncio - @pytest.mark.parametrize("trusted", [True, False]) - @pytest.mark.parametrize("fee", [0, FEE_AMOUNT]) - async def test_create_new_pool_wallet_farm_to_pool(self, one_wallet_node_and_rpc, fee, trusted, self_hostname): + @pytest.mark.parametrize("trusted_and_fee", [(True, FEE_AMOUNT), (False, 0)]) + async def test_create_new_pool_wallet_farm_to_pool(self, one_wallet_node_and_rpc, trusted_and_fee, self_hostname): + trusted, fee = trusted_and_fee client, wallet_node_0, full_node_api = one_wallet_node_and_rpc wallet_0 = wallet_node_0.wallet_state_manager.main_wallet if trusted: @@ -308,9 +306,9 @@ class TestPoolWalletRpc: assert pool_config["pool_url"] == "http://pool.example.com" @pytest.mark.asyncio - @pytest.mark.parametrize("trusted", [True, False]) - @pytest.mark.parametrize("fee", [0, FEE_AMOUNT]) - async def test_create_multiple_pool_wallets(self, one_wallet_node_and_rpc, fee, trusted, self_hostname): + @pytest.mark.parametrize("trusted_and_fee", [(True, FEE_AMOUNT), (False, 0)]) + async def test_create_multiple_pool_wallets(self, one_wallet_node_and_rpc, trusted_and_fee, self_hostname): + trusted, fee = trusted_and_fee client, wallet_node_0, full_node_api = one_wallet_node_and_rpc if trusted: wallet_node_0.config["trusted_peers"] = { @@ -374,7 +372,7 @@ class TestPoolWalletRpc: assert len(await wallet_node_0.wallet_state_manager.tx_store.get_unconfirmed_for_wallet(3)) == 0 # Doing a reorg reverts and removes the pool wallets await full_node_api.reorg_from_index_to_new_index(ReorgProtocol(uint32(0), uint32(20), our_ph_2)) - await asyncio.sleep(5) + await time_out_assert(30, wallet_is_synced, True, wallet_node_0, full_node_api) summaries_response = await client.get_wallets() assert len(summaries_response) == 1 @@ -400,7 +398,7 @@ class TestPoolWalletRpc: # Test creation of many pool wallets. Use untrusted since that is the more complicated protocol, but don't # run this code more than once, since it's slow. - if fee == 0 and not trusted: + if not trusted: for i in range(22): await time_out_assert(10, wallet_is_synced, True, wallet_node_0, full_node_api) creation_tx_3: TransactionRecord = await client.create_new_pool_wallet( @@ -435,9 +433,9 @@ class TestPoolWalletRpc: assert owner_sk != auth_sk @pytest.mark.asyncio - @pytest.mark.parametrize("trusted", [True, False]) - @pytest.mark.parametrize("fee", [0, FEE_AMOUNT]) - async def test_absorb_self(self, one_wallet_node_and_rpc, fee, trusted, bt, self_hostname): + @pytest.mark.parametrize("trusted_and_fee", [(True, FEE_AMOUNT), (False, 0)]) + async def test_absorb_self(self, one_wallet_node_and_rpc, trusted_and_fee, bt, self_hostname): + trusted, fee = trusted_and_fee client, wallet_node_0, full_node_api = one_wallet_node_and_rpc if trusted: wallet_node_0.config["trusted_peers"] = { @@ -548,9 +546,9 @@ class TestPoolWalletRpc: # await time_out_assert(10, wallet_0.get_confirmed_balance, total_block_rewards) @pytest.mark.asyncio - @pytest.mark.parametrize("trusted", [True, False]) - @pytest.mark.parametrize("fee", [0, FEE_AMOUNT]) - async def test_absorb_pooling(self, one_wallet_node_and_rpc, fee, trusted, bt, self_hostname): + @pytest.mark.parametrize("trusted_and_fee", [(True, FEE_AMOUNT), (False, 0)]) + async def test_absorb_pooling(self, one_wallet_node_and_rpc, trusted_and_fee, bt, self_hostname): + trusted, fee = trusted_and_fee client, wallet_node_0, full_node_api = one_wallet_node_and_rpc if trusted: wallet_node_0.config["trusted_peers"] = { @@ -693,10 +691,14 @@ class TestPoolWalletRpc: assert (250000000000 + fee) in [tx.additions[0].amount for tx in tx1] @pytest.mark.asyncio - @pytest.mark.parametrize("trusted", [True]) - @pytest.mark.parametrize("fee", [0]) - async def test_self_pooling_to_pooling(self, setup, fee, trusted, self_hostname): - """This tests self-pooling -> pooling""" + @pytest.mark.parametrize("trusted_and_fee", [(True, 0), (False, 0)]) + async def test_self_pooling_to_pooling(self, setup, trusted_and_fee, self_hostname): + """ + This tests self-pooling -> pooling + TODO: Fix this test for a positive fee value + """ + + trusted, fee = trusted_and_fee num_blocks = 4 # Num blocks to farm at a time total_blocks = 0 # Total blocks farmed so far full_nodes, wallet_nodes, receive_address, client, rpc_cleanup = setup @@ -793,8 +795,8 @@ class TestPoolWalletRpc: fetched: Optional[TransactionRecord] = await client.get_transaction(wid, tx.name) return fetched is not None and fetched.is_in_mempool() - await time_out_assert(5, tx_is_in_mempool, True, wallet_id, join_pool_tx) - await time_out_assert(5, tx_is_in_mempool, True, wallet_id_2, join_pool_tx_2) + await time_out_assert(10, tx_is_in_mempool, True, wallet_id, join_pool_tx) + await time_out_assert(10, tx_is_in_mempool, True, wallet_id_2, join_pool_tx_2) assert status.current.state == PoolSingletonState.SELF_POOLING.value assert status.target is not None @@ -821,13 +823,10 @@ class TestPoolWalletRpc: await rpc_cleanup() @pytest.mark.asyncio - @pytest.mark.parametrize("trusted", [True, False]) - @pytest.mark.parametrize( - "fee", - [0, FEE_AMOUNT], - ) - async def test_leave_pool(self, setup, fee, trusted, self_hostname): + @pytest.mark.parametrize("trusted_and_fee", [(True, FEE_AMOUNT), (False, 0)]) + async def test_leave_pool(self, setup, trusted_and_fee, self_hostname): """This tests self-pooling -> pooling -> escaping -> self pooling""" + trusted, fee = trusted_and_fee full_nodes, wallet_nodes, receive_address, client, rpc_cleanup = setup our_ph = receive_address[0] wallets = [wallet_n.wallet_state_manager.main_wallet for wallet_n in wallet_nodes] @@ -942,10 +941,10 @@ class TestPoolWalletRpc: await rpc_cleanup() @pytest.mark.asyncio - @pytest.mark.parametrize("trusted", [True, False]) - @pytest.mark.parametrize("fee", [0, FEE_AMOUNT]) - async def test_change_pools(self, setup, fee, trusted, self_hostname): + @pytest.mark.parametrize("trusted_and_fee", [(True, FEE_AMOUNT), (False, 0)]) + async def test_change_pools(self, setup, trusted_and_fee, self_hostname): """This tests Pool A -> escaping -> Pool B""" + trusted, fee = trusted_and_fee full_nodes, wallet_nodes, receive_address, client, rpc_cleanup = setup our_ph = receive_address[0] pool_a_ph = receive_address[1] @@ -1042,10 +1041,10 @@ class TestPoolWalletRpc: await rpc_cleanup() @pytest.mark.asyncio - @pytest.mark.parametrize("trusted", [True, False]) - @pytest.mark.parametrize("fee", [0, FEE_AMOUNT]) - async def test_change_pools_reorg(self, setup, fee, trusted, bt, self_hostname): + @pytest.mark.parametrize("trusted_and_fee", [(True, FEE_AMOUNT), (False, 0)]) + async def test_change_pools_reorg(self, setup, trusted_and_fee, bt, self_hostname): """This tests Pool A -> escaping -> reorg -> escaping -> Pool B""" + trusted, fee = trusted_and_fee full_nodes, wallet_nodes, receive_address, client, rpc_cleanup = setup our_ph = receive_address[0] pool_a_ph = receive_address[1] From bf15087c7997f7fef83d612c6b41f37123ae5389 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Thu, 31 Mar 2022 17:23:59 +0200 Subject: [PATCH 02/63] run more tests in parallel on CI (#10960) * run more tests in parallel on CI * fix test_farmer_get_harvesters to wait for plots to be loaded before asking about them --- .../build-test-macos-core-full_node-full_sync.yml | 2 +- .github/workflows/build-test-macos-core-ssl.yml | 2 +- .github/workflows/build-test-macos-core-util.yml | 2 +- .github/workflows/build-test-macos-core.yml | 2 +- .../build-test-ubuntu-core-full_node-full_sync.yml | 2 +- .github/workflows/build-test-ubuntu-core-ssl.yml | 2 +- .github/workflows/build-test-ubuntu-core-util.yml | 2 +- .github/workflows/build-test-ubuntu-core.yml | 2 +- tests/build-workflows.py | 4 +++- tests/core/config.py | 1 + tests/core/full_node/full_sync/config.py | 1 + tests/core/ssl/config.py | 1 + tests/core/test_farmer_harvester_rpc.py | 12 +++++++++--- tests/core/util/config.py | 1 + 14 files changed, 24 insertions(+), 12 deletions(-) create mode 100644 tests/core/config.py create mode 100644 tests/core/ssl/config.py create mode 100644 tests/core/util/config.py diff --git a/.github/workflows/build-test-macos-core-full_node-full_sync.yml b/.github/workflows/build-test-macos-core-full_node-full_sync.yml index f0ae27d1d6..d8aa450771 100644 --- a/.github/workflows/build-test-macos-core-full_node-full_sync.yml +++ b/.github/workflows/build-test-macos-core-full_node-full_sync.yml @@ -85,7 +85,7 @@ jobs: - name: Test core-full_node-full_sync code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/full_node/full_sync/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/full_node/full_sync/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-core-ssl.yml b/.github/workflows/build-test-macos-core-ssl.yml index 6b4b174832..3c624b9a51 100644 --- a/.github/workflows/build-test-macos-core-ssl.yml +++ b/.github/workflows/build-test-macos-core-ssl.yml @@ -85,7 +85,7 @@ jobs: - name: Test core-ssl code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/ssl/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/ssl/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-core-util.yml b/.github/workflows/build-test-macos-core-util.yml index 2a01174dcd..40ae0bd425 100644 --- a/.github/workflows/build-test-macos-core-util.yml +++ b/.github/workflows/build-test-macos-core-util.yml @@ -85,7 +85,7 @@ jobs: - name: Test core-util code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/util/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/util/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-core.yml b/.github/workflows/build-test-macos-core.yml index db9288eda4..2924c0fe49 100644 --- a/.github/workflows/build-test-macos-core.yml +++ b/.github/workflows/build-test-macos-core.yml @@ -85,7 +85,7 @@ jobs: - name: Test core code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-core-full_node-full_sync.yml b/.github/workflows/build-test-ubuntu-core-full_node-full_sync.yml index 40f05d1bfb..f43e01760c 100644 --- a/.github/workflows/build-test-ubuntu-core-full_node-full_sync.yml +++ b/.github/workflows/build-test-ubuntu-core-full_node-full_sync.yml @@ -84,7 +84,7 @@ jobs: - name: Test core-full_node-full_sync code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/full_node/full_sync/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/full_node/full_sync/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-core-ssl.yml b/.github/workflows/build-test-ubuntu-core-ssl.yml index e5074ab76b..fe58d94a7c 100644 --- a/.github/workflows/build-test-ubuntu-core-ssl.yml +++ b/.github/workflows/build-test-ubuntu-core-ssl.yml @@ -84,7 +84,7 @@ jobs: - name: Test core-ssl code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/ssl/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/ssl/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-core-util.yml b/.github/workflows/build-test-ubuntu-core-util.yml index 0b23005959..8e845db650 100644 --- a/.github/workflows/build-test-ubuntu-core-util.yml +++ b/.github/workflows/build-test-ubuntu-core-util.yml @@ -84,7 +84,7 @@ jobs: - name: Test core-util code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/util/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/util/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-core.yml b/.github/workflows/build-test-ubuntu-core.yml index eb06af8df6..345e07b504 100644 --- a/.github/workflows/build-test-ubuntu-core.yml +++ b/.github/workflows/build-test-ubuntu-core.yml @@ -84,7 +84,7 @@ jobs: - name: Test core code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/tests/build-workflows.py b/tests/build-workflows.py index 3898390a48..51cb0dd4fa 100755 --- a/tests/build-workflows.py +++ b/tests/build-workflows.py @@ -132,7 +132,9 @@ if args.verbose: # main test_dirs = subdirs() -current_workflows: Dict[Path, str] = {file: read_file(file) for file in args.output_dir.iterdir()} +current_workflows: Dict[Path, str] = { + file: read_file(file) for file in args.output_dir.iterdir() if str(file).endswith(".yml") +} changed: bool = False for os in testconfig.oses: diff --git a/tests/core/config.py b/tests/core/config.py new file mode 100644 index 0000000000..7f9e1e1a76 --- /dev/null +++ b/tests/core/config.py @@ -0,0 +1 @@ +parallel = True diff --git a/tests/core/full_node/full_sync/config.py b/tests/core/full_node/full_sync/config.py index d9b815b24c..251dd4220d 100644 --- a/tests/core/full_node/full_sync/config.py +++ b/tests/core/full_node/full_sync/config.py @@ -1 +1,2 @@ job_timeout = 60 +parallel = True diff --git a/tests/core/ssl/config.py b/tests/core/ssl/config.py new file mode 100644 index 0000000000..7f9e1e1a76 --- /dev/null +++ b/tests/core/ssl/config.py @@ -0,0 +1 @@ +parallel = True diff --git a/tests/core/test_farmer_harvester_rpc.py b/tests/core/test_farmer_harvester_rpc.py index 226c6542be..72a16ba9e4 100644 --- a/tests/core/test_farmer_harvester_rpc.py +++ b/tests/core/test_farmer_harvester_rpc.py @@ -115,9 +115,15 @@ async def test_farmer_get_harvesters(harvester_farmer_environment): farmer_api = farmer_service._api harvester = harvester_service._node - res = await harvester_rpc_client.get_plots() - num_plots = len(res["plots"]) - assert num_plots > 0 + num_plots = 0 + + async def non_zero_plots() -> bool: + res = await harvester_rpc_client.get_plots() + nonlocal num_plots + num_plots = len(res["plots"]) + return num_plots > 0 + + await time_out_assert(10, non_zero_plots) # Reset cache and force updates cache every second to make sure the farmer gets the most recent data update_interval_before = farmer_api.farmer.update_harvester_cache_interval diff --git a/tests/core/util/config.py b/tests/core/util/config.py new file mode 100644 index 0000000000..7f9e1e1a76 --- /dev/null +++ b/tests/core/util/config.py @@ -0,0 +1 @@ +parallel = True From 6075027e6d03d9eb68daf547eb2c17b9455c06de Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Thu, 31 Mar 2022 17:24:37 +0200 Subject: [PATCH 03/63] improve error message when a block is missing from the blockchain database (#10958) * improve error message when a block is missing from the blockchain database * Update chia/full_node/block_height_map.py Co-authored-by: Kyle Altendorf Co-authored-by: Kyle Altendorf --- chia/full_node/block_height_map.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/chia/full_node/block_height_map.py b/chia/full_node/block_height_map.py index 98a4d2826d..ce9d2a3085 100644 --- a/chia/full_node/block_height_map.py +++ b/chia/full_node/block_height_map.py @@ -184,6 +184,10 @@ class BlockHeightMap: ordered[bytes32.fromhex(r[0])] = (r[2], bytes32.fromhex(r[1]), r[3]) while height > window_end: + if prev_hash not in ordered: + raise ValueError( + f"block with header hash is missing from your blockchain database: {prev_hash.hex()}" + ) entry = ordered[prev_hash] assert height == entry[0] + 1 height = entry[0] From 908f186c1e66356d97b6132f00573363a8ba5e84 Mon Sep 17 00:00:00 2001 From: Mariano Sorgente <3069354+mariano54@users.noreply.github.com> Date: Thu, 31 Mar 2022 11:25:57 -0400 Subject: [PATCH 04/63] Also throw DB error on double spending a coin (#10947) * Throw error on double spending a coin * Throw error on double spending a coin * Improve test --- chia/full_node/coin_store.py | 30 ++++++++++---- .../core/full_node/stores/test_coin_store.py | 40 +++++++++++++------ 2 files changed, 50 insertions(+), 20 deletions(-) diff --git a/chia/full_node/coin_store.py b/chia/full_node/coin_store.py index 4b91d80f73..3fec82d411 100644 --- a/chia/full_node/coin_store.py +++ b/chia/full_node/coin_store.py @@ -1,4 +1,7 @@ from typing import List, Optional, Set, Dict, Any, Tuple + +from aiosqlite import Cursor + from chia.protocols.wallet_protocol import CoinState from chia.types.blockchain_format.coin import Coin from chia.types.blockchain_format.sized_bytes import bytes32 @@ -515,16 +518,27 @@ class CoinStore: for coin_name in coin_names: r = self.coin_record_cache.get(coin_name) if r is not None: + if r.spent_block_index != uint32(0): + raise ValueError(f"Coin already spent in cache: {coin_name}") + self.coin_record_cache.put( r.name, CoinRecord(r.coin, r.confirmed_block_index, index, r.coinbase, r.timestamp) ) updates.append((index, self.maybe_to_hex(coin_name))) - if updates != []: - async with self.db_wrapper.write_db() as conn: - if self.db_wrapper.db_version == 2: - await conn.executemany("UPDATE OR FAIL coin_record SET spent_index=? WHERE coin_name=?", updates) - else: - await conn.executemany( - "UPDATE OR FAIL coin_record SET spent=1,spent_index=? WHERE coin_name=?", updates - ) + assert len(updates) == len(coin_names) + async with self.db_wrapper.write_db() as conn: + if self.db_wrapper.db_version == 2: + ret: Cursor = await conn.executemany( + "UPDATE OR FAIL coin_record SET spent_index=? WHERE coin_name=? AND spent_index=0", updates + ) + + else: + ret = await conn.executemany( + "UPDATE OR FAIL coin_record SET spent=1,spent_index=? WHERE coin_name=? AND spent_index=0", + updates, + ) + if ret.rowcount != len(coin_names): + raise ValueError( + f"Invalid operation to set spent, total updates {ret.rowcount} expected {len(coin_names)}" + ) diff --git a/tests/core/full_node/stores/test_coin_store.py b/tests/core/full_node/stores/test_coin_store.py index 883aa7d973..ca3564ebbc 100644 --- a/tests/core/full_node/stores/test_coin_store.py +++ b/tests/core/full_node/stores/test_coin_store.py @@ -161,22 +161,38 @@ class TestCoinStoreWithBlocks: if block.is_transaction_block(): removals: List[bytes32] = [] additions: List[Coin] = [] + async with db_wrapper.write_db(): + if block.is_transaction_block(): + assert block.foliage_transaction_block is not None + await coin_store.new_block( + block.height, + block.foliage_transaction_block.timestamp, + block.get_included_reward_coins(), + additions, + removals, + ) - if block.is_transaction_block(): - assert block.foliage_transaction_block is not None - await coin_store.new_block( - block.height, - block.foliage_transaction_block.timestamp, - block.get_included_reward_coins(), - additions, - removals, - ) - - coins = block.get_included_reward_coins() - records = [await coin_store.get_coin_record(coin.name()) for coin in coins] + coins = block.get_included_reward_coins() + records = [await coin_store.get_coin_record(coin.name()) for coin in coins] await coin_store._set_spent([r.name for r in records], block.height) + if len(records) > 0: + for r in records: + assert (await coin_store.get_coin_record(r.name)) is not None + + if cache_size > 0: + # Check that we can't spend a coin twice in cache + with pytest.raises(ValueError, match="Coin already spent"): + await coin_store._set_spent([r.name for r in records], block.height) + + for r in records: + coin_store.coin_record_cache.remove(r.name) + + # Check that we can't spend a coin twice in DB + with pytest.raises(ValueError, match="Invalid operation to set spent"): + await coin_store._set_spent([r.name for r in records], block.height) + records = [await coin_store.get_coin_record(coin.name()) for coin in coins] for record in records: assert record.spent From 8833cc351c49a7e9340626c7c9e4715a0ecd3626 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Thu, 31 Mar 2022 17:26:23 +0200 Subject: [PATCH 05/63] reorg fixes (#10943) * when going through a reorg, maintain all chain state until the very end, when the new fork has been fully validated and added * when rolling back the chain, also rollback the height-to-hash map * add tests --- chia/consensus/blockchain.py | 8 +- chia/full_node/block_height_map.py | 1 + tests/block_tools.py | 10 +- tests/blockchain/blockchain_test_utils.py | 5 +- tests/blockchain/test_blockchain.py | 299 ++++++++++++++++++ tests/core/full_node/test_block_height_map.py | 14 +- 6 files changed, 326 insertions(+), 11 deletions(-) diff --git a/chia/consensus/blockchain.py b/chia/consensus/blockchain.py index fbbf4cc02d..45910f8a55 100644 --- a/chia/consensus/blockchain.py +++ b/chia/consensus/blockchain.py @@ -381,10 +381,6 @@ class Blockchain(BlockchainInterface): for coin_record in roll_changes: latest_coin_state[coin_record.name] = coin_record - # Rollback sub_epoch_summaries - self.__height_map.rollback(fork_height) - await self.block_store.rollback(fork_height) - # Collect all blocks from fork point to new peak blocks_to_add: List[Tuple[FullBlock, BlockRecord]] = [] curr = block_record.header_hash @@ -445,6 +441,10 @@ class Blockchain(BlockchainInterface): hint_coin_state[key] = {} hint_coin_state[key][coin_id] = latest_coin_state[coin_id] + # we made it to the end successfully + # Rollback sub_epoch_summaries + self.__height_map.rollback(fork_height) + await self.block_store.rollback(fork_height) await self.block_store.set_in_chain([(br.header_hash,) for br in records_to_add]) # Changes the peak to be the new peak diff --git a/chia/full_node/block_height_map.py b/chia/full_node/block_height_map.py index ce9d2a3085..49e60df164 100644 --- a/chia/full_node/block_height_map.py +++ b/chia/full_node/block_height_map.py @@ -230,6 +230,7 @@ class BlockHeightMap: heights_to_delete.append(ses_included_height) for height in heights_to_delete: del self.__sub_epoch_summaries[height] + del self.__height_to_hash[(fork_height + 1) * 32 :] def get_ses(self, height: uint32) -> SubEpochSummary: return SubEpochSummary.from_bytes(self.__sub_epoch_summaries[height]) diff --git a/tests/block_tools.py b/tests/block_tools.py index f358b955cf..e2fb177bcc 100644 --- a/tests/block_tools.py +++ b/tests/block_tools.py @@ -11,7 +11,7 @@ import time from argparse import Namespace from dataclasses import replace from pathlib import Path -from typing import Callable, Dict, List, Optional, Tuple, Any +from typing import Callable, Dict, List, Optional, Tuple, Any, Union from blspy import AugSchemeMPL, G1Element, G2Element, PrivateKey from chiabip158 import PyBIP158 @@ -435,7 +435,7 @@ class BlockTools: normalized_to_identity_cc_sp: bool = False, normalized_to_identity_cc_ip: bool = False, current_time: bool = False, - previous_generator: CompressorArg = None, + previous_generator: Optional[Union[CompressorArg, List[uint32]]] = None, genesis_timestamp: Optional[uint64] = None, force_plot_id: Optional[bytes32] = None, ) -> List[FullBlock]: @@ -588,12 +588,14 @@ class BlockTools: pool_target = PoolTarget(self.pool_ph, uint32(0)) if transaction_data is not None: - if previous_generator is not None: + if type(previous_generator) is CompressorArg: block_generator: Optional[BlockGenerator] = best_solution_generator_from_template( previous_generator, transaction_data ) else: block_generator = simple_solution_generator(transaction_data) + if type(previous_generator) is list: + block_generator = BlockGenerator(block_generator.program, [], previous_generator) aggregate_signature = transaction_data.aggregated_signature else: @@ -861,7 +863,7 @@ class BlockTools: else: pool_target = PoolTarget(self.pool_ph, uint32(0)) if transaction_data is not None: - if previous_generator is not None: + if previous_generator is not None and type(previous_generator) is CompressorArg: block_generator = best_solution_generator_from_template( previous_generator, transaction_data ) diff --git a/tests/blockchain/blockchain_test_utils.py b/tests/blockchain/blockchain_test_utils.py index 93152782cf..ee991e74c7 100644 --- a/tests/blockchain/blockchain_test_utils.py +++ b/tests/blockchain/blockchain_test_utils.py @@ -4,7 +4,7 @@ from chia.consensus.blockchain import Blockchain, ReceiveBlockResult from chia.consensus.multiprocess_validation import PreValidationResult from chia.types.full_block import FullBlock from chia.util.errors import Err -from chia.util.ints import uint64 +from chia.util.ints import uint64, uint32 async def check_block_store_invariant(bc: Blockchain): @@ -42,6 +42,7 @@ async def _validate_and_add_block( expected_result: Optional[ReceiveBlockResult] = None, expected_error: Optional[Err] = None, skip_prevalidation: bool = False, + fork_point_with_peak: Optional[uint32] = None, ) -> None: # Tries to validate and add the block, and checks that there are no errors in the process and that the # block is added to the peak. @@ -74,7 +75,7 @@ async def _validate_and_add_block( await check_block_store_invariant(blockchain) return None - result, err, _, _ = await blockchain.receive_block(block, results) + result, err, _, _ = await blockchain.receive_block(block, results, fork_point_with_peak=fork_point_with_peak) await check_block_store_invariant(blockchain) if expected_error is None and expected_result != ReceiveBlockResult.INVALID_BLOCK: diff --git a/tests/blockchain/test_blockchain.py b/tests/blockchain/test_blockchain.py index 99927fc0ff..f4fb02e930 100644 --- a/tests/blockchain/test_blockchain.py +++ b/tests/blockchain/test_blockchain.py @@ -2949,3 +2949,302 @@ class TestReorgs: assert blocks assert len(blocks) == 200 assert blocks[-1].height == 199 + + +@pytest.mark.asyncio +async def test_reorg_new_ref(empty_blockchain, bt): + b = empty_blockchain + wallet_a = WalletTool(b.constants) + WALLET_A_PUZZLE_HASHES = [wallet_a.get_new_puzzlehash() for _ in range(5)] + coinbase_puzzlehash = WALLET_A_PUZZLE_HASHES[0] + receiver_puzzlehash = WALLET_A_PUZZLE_HASHES[1] + + blocks = bt.get_consecutive_blocks( + 5, + farmer_reward_puzzle_hash=coinbase_puzzlehash, + pool_reward_puzzle_hash=receiver_puzzlehash, + guarantee_transaction_block=True, + ) + + all_coins = [] + for spend_block in blocks[:5]: + for coin in list(spend_block.get_included_reward_coins()): + if coin.puzzle_hash == coinbase_puzzlehash: + all_coins.append(coin) + spend_bundle_0 = wallet_a.generate_signed_transaction(1000, receiver_puzzlehash, all_coins.pop()) + blocks = bt.get_consecutive_blocks( + 15, + block_list_input=blocks, + farmer_reward_puzzle_hash=coinbase_puzzlehash, + pool_reward_puzzle_hash=receiver_puzzlehash, + transaction_data=spend_bundle_0, + guarantee_transaction_block=True, + ) + + for block in blocks: + await _validate_and_add_block(b, block) + assert b.get_peak().height == 19 + + print("first chain done") + + # Make sure a ref back into the reorg chain itself works as expected + + blocks_reorg_chain = bt.get_consecutive_blocks( + 1, + blocks[:10], + seed=b"2", + farmer_reward_puzzle_hash=coinbase_puzzlehash, + pool_reward_puzzle_hash=receiver_puzzlehash, + ) + spend_bundle = wallet_a.generate_signed_transaction(1000, receiver_puzzlehash, all_coins.pop()) + + blocks_reorg_chain = bt.get_consecutive_blocks( + 2, + blocks_reorg_chain, + seed=b"2", + farmer_reward_puzzle_hash=coinbase_puzzlehash, + pool_reward_puzzle_hash=receiver_puzzlehash, + transaction_data=spend_bundle, + guarantee_transaction_block=True, + ) + + spend_bundle2 = wallet_a.generate_signed_transaction(1000, receiver_puzzlehash, all_coins.pop()) + blocks_reorg_chain = bt.get_consecutive_blocks( + 4, blocks_reorg_chain, seed=b"2", previous_generator=[uint32(5), uint32(11)], transaction_data=spend_bundle2 + ) + blocks_reorg_chain = bt.get_consecutive_blocks(4, blocks_reorg_chain, seed=b"2") + + for i, block in enumerate(blocks_reorg_chain): + fork_point_with_peak = None + if i < 10: + expected = ReceiveBlockResult.ALREADY_HAVE_BLOCK + elif i < 20: + expected = ReceiveBlockResult.ADDED_AS_ORPHAN + else: + expected = ReceiveBlockResult.NEW_PEAK + fork_point_with_peak = uint32(1) + await _validate_and_add_block(b, block, expected_result=expected, fork_point_with_peak=fork_point_with_peak) + assert b.get_peak().height == 20 + + +# this test doesn't reorg, but _reconsider_peak() is passed a stale +# "fork_height" to make it look like it's in a reorg, but all the same blocks +# are just added back. +@pytest.mark.asyncio +async def test_reorg_stale_fork_height(empty_blockchain, bt): + b = empty_blockchain + wallet_a = WalletTool(b.constants) + WALLET_A_PUZZLE_HASHES = [wallet_a.get_new_puzzlehash() for _ in range(5)] + coinbase_puzzlehash = WALLET_A_PUZZLE_HASHES[0] + receiver_puzzlehash = WALLET_A_PUZZLE_HASHES[1] + + blocks = bt.get_consecutive_blocks( + 5, + farmer_reward_puzzle_hash=coinbase_puzzlehash, + pool_reward_puzzle_hash=receiver_puzzlehash, + guarantee_transaction_block=True, + ) + + all_coins = [] + for spend_block in blocks: + for coin in list(spend_block.get_included_reward_coins()): + if coin.puzzle_hash == coinbase_puzzlehash: + all_coins.append(coin) + spend_bundle_0 = wallet_a.generate_signed_transaction(1000, receiver_puzzlehash, all_coins.pop()) + + # Make sure a ref back into the reorg chain itself works as expected + spend_bundle = wallet_a.generate_signed_transaction(1000, receiver_puzzlehash, all_coins.pop()) + + # make sure we have a transaction block, with at least one transaction in it + blocks = bt.get_consecutive_blocks( + 5, + blocks, + farmer_reward_puzzle_hash=coinbase_puzzlehash, + pool_reward_puzzle_hash=receiver_puzzlehash, + transaction_data=spend_bundle, + guarantee_transaction_block=True, + ) + + # this block (height 10) refers back to the generator in block 5 + spend_bundle2 = wallet_a.generate_signed_transaction(1000, receiver_puzzlehash, all_coins.pop()) + blocks = bt.get_consecutive_blocks(4, blocks, previous_generator=[uint32(5)], transaction_data=spend_bundle2) + + for block in blocks[:5]: + await _validate_and_add_block(b, block, expected_result=ReceiveBlockResult.NEW_PEAK) + + # fake the fork_height to make every new block look like a reorg + for block in blocks[5:]: + await _validate_and_add_block(b, block, expected_result=ReceiveBlockResult.NEW_PEAK, fork_point_with_peak=2) + assert b.get_peak().height == 13 + + +@pytest.mark.asyncio +async def test_chain_failed_rollback(empty_blockchain, bt): + b = empty_blockchain + wallet_a = WalletTool(b.constants) + WALLET_A_PUZZLE_HASHES = [wallet_a.get_new_puzzlehash() for _ in range(5)] + coinbase_puzzlehash = WALLET_A_PUZZLE_HASHES[0] + receiver_puzzlehash = WALLET_A_PUZZLE_HASHES[1] + + blocks = bt.get_consecutive_blocks( + 20, + farmer_reward_puzzle_hash=coinbase_puzzlehash, + pool_reward_puzzle_hash=receiver_puzzlehash, + ) + + for block in blocks: + await _validate_and_add_block(b, block) + assert b.get_peak().height == 19 + + print("first chain done") + + # Make sure a ref back into the reorg chain itself works as expected + + all_coins = [] + for spend_block in blocks[:10]: + for coin in list(spend_block.get_included_reward_coins()): + if coin.puzzle_hash == coinbase_puzzlehash: + all_coins.append(coin) + + spend_bundle = wallet_a.generate_signed_transaction(1000, receiver_puzzlehash, all_coins.pop()) + + blocks_reorg_chain = bt.get_consecutive_blocks( + 11, + blocks[:10], + seed=b"2", + farmer_reward_puzzle_hash=coinbase_puzzlehash, + pool_reward_puzzle_hash=receiver_puzzlehash, + transaction_data=spend_bundle, + guarantee_transaction_block=True, + ) + + for block in blocks_reorg_chain[10:-1]: + await _validate_and_add_block(b, block, expected_result=ReceiveBlockResult.ADDED_AS_ORPHAN) + + # Incorrectly set the height as spent in DB to trigger an error + print(f"{await b.coin_store.get_coin_record(spend_bundle.coin_spends[0].coin.name())}") + print(spend_bundle.coin_spends[0].coin.name()) + # await b.coin_store._set_spent([spend_bundle.coin_spends[0].coin.name()], 8) + await b.coin_store.rollback_to_block(2) + print(f"{await b.coin_store.get_coin_record(spend_bundle.coin_spends[0].coin.name())}") + + try: + await _validate_and_add_block(b, blocks_reorg_chain[-1]) + except AssertionError: + pass + + assert b.get_peak().height == 19 + + +@pytest.mark.asyncio +async def test_reorg_flip_flop(empty_blockchain, bt): + b = empty_blockchain + wallet_a = WalletTool(b.constants) + WALLET_A_PUZZLE_HASHES = [wallet_a.get_new_puzzlehash() for _ in range(5)] + coinbase_puzzlehash = WALLET_A_PUZZLE_HASHES[0] + receiver_puzzlehash = WALLET_A_PUZZLE_HASHES[1] + + chain_a = bt.get_consecutive_blocks( + 10, + farmer_reward_puzzle_hash=coinbase_puzzlehash, + pool_reward_puzzle_hash=receiver_puzzlehash, + guarantee_transaction_block=True, + ) + + all_coins = [] + for spend_block in chain_a: + for coin in list(spend_block.get_included_reward_coins()): + if coin.puzzle_hash == coinbase_puzzlehash: + all_coins.append(coin) + + # this is a transaction block at height 10 + spend_bundle = wallet_a.generate_signed_transaction(1000, receiver_puzzlehash, all_coins.pop()) + chain_a = bt.get_consecutive_blocks( + 5, + chain_a, + farmer_reward_puzzle_hash=coinbase_puzzlehash, + pool_reward_puzzle_hash=receiver_puzzlehash, + transaction_data=spend_bundle, + guarantee_transaction_block=True, + ) + + spend_bundle = wallet_a.generate_signed_transaction(1000, receiver_puzzlehash, all_coins.pop()) + chain_a = bt.get_consecutive_blocks(5, chain_a, previous_generator=[uint32(10)], transaction_data=spend_bundle) + + spend_bundle = wallet_a.generate_signed_transaction(1000, receiver_puzzlehash, all_coins.pop()) + chain_a = bt.get_consecutive_blocks( + 20, + chain_a, + farmer_reward_puzzle_hash=coinbase_puzzlehash, + pool_reward_puzzle_hash=receiver_puzzlehash, + transaction_data=spend_bundle, + guarantee_transaction_block=True, + ) + + # chain A is 40 blocks deep + # chain B share the first 20 blocks with chain A + + # add 5 blocks on top of the first 20, to form chain B + chain_b = bt.get_consecutive_blocks( + 5, + chain_a[:20], + seed=b"2", + farmer_reward_puzzle_hash=coinbase_puzzlehash, + pool_reward_puzzle_hash=receiver_puzzlehash, + ) + spend_bundle = wallet_a.generate_signed_transaction(1000, receiver_puzzlehash, all_coins.pop()) + + # this is a transaction block at height 15 (in Chain B) + chain_b = bt.get_consecutive_blocks( + 5, + chain_b, + seed=b"2", + farmer_reward_puzzle_hash=coinbase_puzzlehash, + pool_reward_puzzle_hash=receiver_puzzlehash, + transaction_data=spend_bundle, + guarantee_transaction_block=True, + ) + + spend_bundle = wallet_a.generate_signed_transaction(1000, receiver_puzzlehash, all_coins.pop()) + chain_b = bt.get_consecutive_blocks( + 10, chain_b, seed=b"2", previous_generator=[uint32(15)], transaction_data=spend_bundle + ) + + assert len(chain_a) == len(chain_b) + + counter = 0 + for b1, b2 in zip(chain_a, chain_b): + + # alternate the order we add blocks from the two chains, to ensure one + # chain overtakes the other one in weight every other time + if counter % 2 == 0: + block1, block2 = b2, b1 + else: + block1, block2 = b1, b2 + counter += 1 + + fork_height = 2 if counter > 3 else None + + preval: List[PreValidationResult] = await b.pre_validate_blocks_multiprocessing( + [block1], {}, validate_signatures=False + ) + result, err, _, _ = await b.receive_block(block1, preval[0], fork_point_with_peak=fork_height) + assert not err + preval: List[PreValidationResult] = await b.pre_validate_blocks_multiprocessing( + [block2], {}, validate_signatures=False + ) + result, err, _, _ = await b.receive_block(block2, preval[0], fork_point_with_peak=fork_height) + assert not err + + assert b.get_peak().height == 39 + + chain_b = bt.get_consecutive_blocks( + 10, + chain_b, + seed=b"2", + farmer_reward_puzzle_hash=coinbase_puzzlehash, + pool_reward_puzzle_hash=receiver_puzzlehash, + ) + + for block in chain_b[40:]: + await _validate_and_add_block(b, block) diff --git a/tests/core/full_node/test_block_height_map.py b/tests/core/full_node/test_block_height_map.py index ac5bf5c549..dfc34cf88c 100644 --- a/tests/core/full_node/test_block_height_map.py +++ b/tests/core/full_node/test_block_height_map.py @@ -371,7 +371,15 @@ class TestBlockHeightMap: assert height_map.get_hash(5) == gen_block_hash(5) height_map.rollback(5) - + assert height_map.contains_height(0) + assert height_map.contains_height(1) + assert height_map.contains_height(2) + assert height_map.contains_height(3) + assert height_map.contains_height(4) + assert height_map.contains_height(5) + assert not height_map.contains_height(6) + assert not height_map.contains_height(7) + assert not height_map.contains_height(8) assert height_map.get_hash(5) == gen_block_hash(5) assert height_map.get_ses(0) == gen_ses(0) @@ -401,8 +409,12 @@ class TestBlockHeightMap: assert height_map.get_hash(6) == gen_block_hash(6) height_map.rollback(6) + assert height_map.contains_height(6) + assert not height_map.contains_height(7) assert height_map.get_hash(6) == gen_block_hash(6) + with pytest.raises(AssertionError) as _: + height_map.get_hash(7) assert height_map.get_ses(0) == gen_ses(0) assert height_map.get_ses(2) == gen_ses(2) From a2490e07656d6fff95d36102dcc24ecbf6c3b9ae Mon Sep 17 00:00:00 2001 From: Mariano Sorgente <3069354+mariano54@users.noreply.github.com> Date: Thu, 31 Mar 2022 14:27:01 -0400 Subject: [PATCH 06/63] Fix the issues in main (failing tests) (#10977) * Fix one of the issues in test_blockchain * Only rollback after all async operations are finished --- chia/consensus/blockchain.py | 3 ++- tests/blockchain/test_blockchain.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/chia/consensus/blockchain.py b/chia/consensus/blockchain.py index 45910f8a55..91fcd4458e 100644 --- a/chia/consensus/blockchain.py +++ b/chia/consensus/blockchain.py @@ -275,6 +275,8 @@ class Blockchain(BlockchainInterface): # Then update the memory cache. It is important that this task is not cancelled and does not throw self.add_block_record(block_record) + if fork_height is not None: + self.__height_map.rollback(fork_height) for fetched_block_record in records: self.__height_map.update_height( fetched_block_record.height, @@ -443,7 +445,6 @@ class Blockchain(BlockchainInterface): # we made it to the end successfully # Rollback sub_epoch_summaries - self.__height_map.rollback(fork_height) await self.block_store.rollback(fork_height) await self.block_store.set_in_chain([(br.header_hash,) for br in records_to_add]) diff --git a/tests/blockchain/test_blockchain.py b/tests/blockchain/test_blockchain.py index f4fb02e930..59631b7614 100644 --- a/tests/blockchain/test_blockchain.py +++ b/tests/blockchain/test_blockchain.py @@ -3130,7 +3130,7 @@ async def test_chain_failed_rollback(empty_blockchain, bt): try: await _validate_and_add_block(b, blocks_reorg_chain[-1]) - except AssertionError: + except ValueError: pass assert b.get_peak().height == 19 From e5ab4cd842b7173a967c3a889437f139bf937836 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 1 Apr 2022 02:01:20 -0400 Subject: [PATCH 07/63] back to a single option for workflow parallel config (#10979) --- tests/build-workflows.py | 7 +++---- tests/pools/config.py | 3 +-- tests/testconfig.py | 9 +++++++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/build-workflows.py b/tests/build-workflows.py index 51cb0dd4fa..9be0e79336 100755 --- a/tests/build-workflows.py +++ b/tests/build-workflows.py @@ -77,16 +77,15 @@ def generate_replacements(conf, dir): "PYTEST_PARALLEL_ARGS": "", } + xdist_numprocesses = {False: 0, True: 4}.get(conf["parallel"], conf["parallel"]) + replacements["PYTEST_PARALLEL_ARGS"] = f" -n {xdist_numprocesses}" + if not conf["checkout_blocks_and_plots"]: replacements[ "CHECKOUT_TEST_BLOCKS_AND_PLOTS" ] = "# Omitted checking out blocks and plots repo Chia-Network/test-cache" if not conf["install_timelord"]: replacements["INSTALL_TIMELORD"] = "# Omitted installing Timelord" - if conf.get("custom_parallel_n", None): - replacements["PYTEST_PARALLEL_ARGS"] = f" -n {conf['custom_parallel_n']}" - else: - replacements["PYTEST_PARALLEL_ARGS"] = " -n 4" if conf["parallel"] else " -n 0" if conf["job_timeout"]: replacements["JOB_TIMEOUT"] = str(conf["job_timeout"]) replacements["TEST_DIR"] = "/".join([*dir.relative_to(root_path.parent).parts, "test_*.py"]) diff --git a/tests/pools/config.py b/tests/pools/config.py index bc4811c2c5..be5a59232c 100644 --- a/tests/pools/config.py +++ b/tests/pools/config.py @@ -1,3 +1,2 @@ -custom_parallel_n = 2 -parallel = True +parallel = 2 job_timeout = 60 diff --git a/tests/testconfig.py b/tests/testconfig.py index 41e1720b29..3aae8f4cea 100644 --- a/tests/testconfig.py +++ b/tests/testconfig.py @@ -1,10 +1,15 @@ -from typing import List +from __future__ import annotations + +from typing import TYPE_CHECKING, List, Union + +if TYPE_CHECKING: + from typing_extensions import Literal # Github actions template config. oses = ["ubuntu", "macos"] # Defaults are conservative. -parallel = False +parallel: Union[bool, int, Literal["auto"]] = False checkout_blocks_and_plots = True install_timelord = False check_resource_usage = False From a571c7889993ccba39f5fba96cffec87fc8f5822 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Fri, 1 Apr 2022 08:01:50 +0200 Subject: [PATCH 08/63] limit test output on CI by dropping -s and -v. Also, only print the 10 slowest tests, instead of all (#10959) --- .github/workflows/build-test-macos-blockchain.yml | 2 +- .github/workflows/build-test-macos-clvm.yml | 2 +- .github/workflows/build-test-macos-core-cmds.yml | 2 +- .github/workflows/build-test-macos-core-consensus.yml | 2 +- .github/workflows/build-test-macos-core-custom_types.yml | 2 +- .github/workflows/build-test-macos-core-daemon.yml | 2 +- .github/workflows/build-test-macos-core-full_node-full_sync.yml | 2 +- .github/workflows/build-test-macos-core-full_node-stores.yml | 2 +- .github/workflows/build-test-macos-core-full_node.yml | 2 +- .github/workflows/build-test-macos-core-server.yml | 2 +- .github/workflows/build-test-macos-core-ssl.yml | 2 +- .github/workflows/build-test-macos-core-util.yml | 2 +- .github/workflows/build-test-macos-core.yml | 2 +- .github/workflows/build-test-macos-farmer_harvester.yml | 2 +- .github/workflows/build-test-macos-generator.yml | 2 +- .github/workflows/build-test-macos-plotting.yml | 2 +- .github/workflows/build-test-macos-pools.yml | 2 +- .github/workflows/build-test-macos-simulation.yml | 2 +- .github/workflows/build-test-macos-tools.yml | 2 +- .github/workflows/build-test-macos-util.yml | 2 +- .github/workflows/build-test-macos-wallet-cat_wallet.yml | 2 +- .github/workflows/build-test-macos-wallet-did_wallet.yml | 2 +- .github/workflows/build-test-macos-wallet-rl_wallet.yml | 2 +- .github/workflows/build-test-macos-wallet-rpc.yml | 2 +- .github/workflows/build-test-macos-wallet-simple_sync.yml | 2 +- .github/workflows/build-test-macos-wallet-sync.yml | 2 +- .github/workflows/build-test-macos-wallet.yml | 2 +- .github/workflows/build-test-macos-weight_proof.yml | 2 +- .github/workflows/build-test-ubuntu-blockchain.yml | 2 +- .github/workflows/build-test-ubuntu-clvm.yml | 2 +- .github/workflows/build-test-ubuntu-core-cmds.yml | 2 +- .github/workflows/build-test-ubuntu-core-consensus.yml | 2 +- .github/workflows/build-test-ubuntu-core-custom_types.yml | 2 +- .github/workflows/build-test-ubuntu-core-daemon.yml | 2 +- .../workflows/build-test-ubuntu-core-full_node-full_sync.yml | 2 +- .github/workflows/build-test-ubuntu-core-full_node-stores.yml | 2 +- .github/workflows/build-test-ubuntu-core-full_node.yml | 2 +- .github/workflows/build-test-ubuntu-core-server.yml | 2 +- .github/workflows/build-test-ubuntu-core-ssl.yml | 2 +- .github/workflows/build-test-ubuntu-core-util.yml | 2 +- .github/workflows/build-test-ubuntu-core.yml | 2 +- .github/workflows/build-test-ubuntu-farmer_harvester.yml | 2 +- .github/workflows/build-test-ubuntu-generator.yml | 2 +- .github/workflows/build-test-ubuntu-plotting.yml | 2 +- .github/workflows/build-test-ubuntu-pools.yml | 2 +- .github/workflows/build-test-ubuntu-simulation.yml | 2 +- .github/workflows/build-test-ubuntu-tools.yml | 2 +- .github/workflows/build-test-ubuntu-util.yml | 2 +- .github/workflows/build-test-ubuntu-wallet-cat_wallet.yml | 2 +- .github/workflows/build-test-ubuntu-wallet-did_wallet.yml | 2 +- .github/workflows/build-test-ubuntu-wallet-rl_wallet.yml | 2 +- .github/workflows/build-test-ubuntu-wallet-rpc.yml | 2 +- .github/workflows/build-test-ubuntu-wallet-simple_sync.yml | 2 +- .github/workflows/build-test-ubuntu-wallet-sync.yml | 2 +- .github/workflows/build-test-ubuntu-wallet.yml | 2 +- .github/workflows/build-test-ubuntu-weight_proof.yml | 2 +- tests/runner_templates/build-test-macos | 2 +- tests/runner_templates/build-test-ubuntu | 2 +- 58 files changed, 58 insertions(+), 58 deletions(-) diff --git a/.github/workflows/build-test-macos-blockchain.yml b/.github/workflows/build-test-macos-blockchain.yml index 6edf41d2d4..b39022abb9 100644 --- a/.github/workflows/build-test-macos-blockchain.yml +++ b/.github/workflows/build-test-macos-blockchain.yml @@ -85,7 +85,7 @@ jobs: - name: Test blockchain code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/blockchain/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/blockchain/test_*.py --durations=10 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-clvm.yml b/.github/workflows/build-test-macos-clvm.yml index 414ff342c7..f2d496da93 100644 --- a/.github/workflows/build-test-macos-clvm.yml +++ b/.github/workflows/build-test-macos-clvm.yml @@ -79,7 +79,7 @@ jobs: - name: Test clvm code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/clvm/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/clvm/test_*.py --durations=10 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-core-cmds.yml b/.github/workflows/build-test-macos-core-cmds.yml index 6b5548a526..6d64dfaa4f 100644 --- a/.github/workflows/build-test-macos-core-cmds.yml +++ b/.github/workflows/build-test-macos-core-cmds.yml @@ -85,7 +85,7 @@ jobs: - name: Test core-cmds code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/cmds/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/cmds/test_*.py --durations=10 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-core-consensus.yml b/.github/workflows/build-test-macos-core-consensus.yml index 0a95110a98..d85ee8a281 100644 --- a/.github/workflows/build-test-macos-core-consensus.yml +++ b/.github/workflows/build-test-macos-core-consensus.yml @@ -85,7 +85,7 @@ jobs: - name: Test core-consensus code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/consensus/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/consensus/test_*.py --durations=10 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-core-custom_types.yml b/.github/workflows/build-test-macos-core-custom_types.yml index d08ed4e638..b6f1aa512e 100644 --- a/.github/workflows/build-test-macos-core-custom_types.yml +++ b/.github/workflows/build-test-macos-core-custom_types.yml @@ -85,7 +85,7 @@ jobs: - name: Test core-custom_types code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/custom_types/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/custom_types/test_*.py --durations=10 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-core-daemon.yml b/.github/workflows/build-test-macos-core-daemon.yml index b7c94c8743..043fb03b91 100644 --- a/.github/workflows/build-test-macos-core-daemon.yml +++ b/.github/workflows/build-test-macos-core-daemon.yml @@ -97,7 +97,7 @@ jobs: - name: Test core-daemon code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/daemon/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/daemon/test_*.py --durations=10 -n 0 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-core-full_node-full_sync.yml b/.github/workflows/build-test-macos-core-full_node-full_sync.yml index d8aa450771..bd45bb6dc2 100644 --- a/.github/workflows/build-test-macos-core-full_node-full_sync.yml +++ b/.github/workflows/build-test-macos-core-full_node-full_sync.yml @@ -85,7 +85,7 @@ jobs: - name: Test core-full_node-full_sync code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/full_node/full_sync/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/full_node/full_sync/test_*.py --durations=10 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-core-full_node-stores.yml b/.github/workflows/build-test-macos-core-full_node-stores.yml index c971b17215..65a81bb923 100644 --- a/.github/workflows/build-test-macos-core-full_node-stores.yml +++ b/.github/workflows/build-test-macos-core-full_node-stores.yml @@ -85,7 +85,7 @@ jobs: - name: Test core-full_node-stores code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/full_node/stores/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/full_node/stores/test_*.py --durations=10 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-core-full_node.yml b/.github/workflows/build-test-macos-core-full_node.yml index 4c13a79b5d..5e833ac6df 100644 --- a/.github/workflows/build-test-macos-core-full_node.yml +++ b/.github/workflows/build-test-macos-core-full_node.yml @@ -85,7 +85,7 @@ jobs: - name: Test core-full_node code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/full_node/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/full_node/test_*.py --durations=10 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-core-server.yml b/.github/workflows/build-test-macos-core-server.yml index e4bceaf93d..f81fc0de33 100644 --- a/.github/workflows/build-test-macos-core-server.yml +++ b/.github/workflows/build-test-macos-core-server.yml @@ -85,7 +85,7 @@ jobs: - name: Test core-server code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/server/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/server/test_*.py --durations=10 -n 0 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-core-ssl.yml b/.github/workflows/build-test-macos-core-ssl.yml index 3c624b9a51..95f52c10ec 100644 --- a/.github/workflows/build-test-macos-core-ssl.yml +++ b/.github/workflows/build-test-macos-core-ssl.yml @@ -85,7 +85,7 @@ jobs: - name: Test core-ssl code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/ssl/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/ssl/test_*.py --durations=10 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-core-util.yml b/.github/workflows/build-test-macos-core-util.yml index 40ae0bd425..984c43b7f0 100644 --- a/.github/workflows/build-test-macos-core-util.yml +++ b/.github/workflows/build-test-macos-core-util.yml @@ -85,7 +85,7 @@ jobs: - name: Test core-util code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/util/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/util/test_*.py --durations=10 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-core.yml b/.github/workflows/build-test-macos-core.yml index 2924c0fe49..52a592a174 100644 --- a/.github/workflows/build-test-macos-core.yml +++ b/.github/workflows/build-test-macos-core.yml @@ -85,7 +85,7 @@ jobs: - name: Test core code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/test_*.py --durations=10 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-farmer_harvester.yml b/.github/workflows/build-test-macos-farmer_harvester.yml index d22814b660..60bcf838f1 100644 --- a/.github/workflows/build-test-macos-farmer_harvester.yml +++ b/.github/workflows/build-test-macos-farmer_harvester.yml @@ -85,7 +85,7 @@ jobs: - name: Test farmer_harvester code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/farmer_harvester/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/farmer_harvester/test_*.py --durations=10 -n 0 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-generator.yml b/.github/workflows/build-test-macos-generator.yml index 339d88dcaa..77c89e44d9 100644 --- a/.github/workflows/build-test-macos-generator.yml +++ b/.github/workflows/build-test-macos-generator.yml @@ -85,7 +85,7 @@ jobs: - name: Test generator code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/generator/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/generator/test_*.py --durations=10 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-plotting.yml b/.github/workflows/build-test-macos-plotting.yml index 0d614d0428..83ef742b82 100644 --- a/.github/workflows/build-test-macos-plotting.yml +++ b/.github/workflows/build-test-macos-plotting.yml @@ -85,7 +85,7 @@ jobs: - name: Test plotting code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/plotting/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/plotting/test_*.py --durations=10 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-pools.yml b/.github/workflows/build-test-macos-pools.yml index ced34bc307..e83532a8ff 100644 --- a/.github/workflows/build-test-macos-pools.yml +++ b/.github/workflows/build-test-macos-pools.yml @@ -85,7 +85,7 @@ jobs: - name: Test pools code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/pools/test_*.py -s -v --durations 0 -n 2 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/pools/test_*.py --durations=10 -n 2 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-simulation.yml b/.github/workflows/build-test-macos-simulation.yml index 3859585c05..5a1f7f7c3e 100644 --- a/.github/workflows/build-test-macos-simulation.yml +++ b/.github/workflows/build-test-macos-simulation.yml @@ -97,7 +97,7 @@ jobs: - name: Test simulation code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/simulation/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/simulation/test_*.py --durations=10 -n 0 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-tools.yml b/.github/workflows/build-test-macos-tools.yml index 3e4e981a37..d304aca53d 100644 --- a/.github/workflows/build-test-macos-tools.yml +++ b/.github/workflows/build-test-macos-tools.yml @@ -85,7 +85,7 @@ jobs: - name: Test tools code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/tools/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/tools/test_*.py --durations=10 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-util.yml b/.github/workflows/build-test-macos-util.yml index 3fa4a0c2ee..8661fbff15 100644 --- a/.github/workflows/build-test-macos-util.yml +++ b/.github/workflows/build-test-macos-util.yml @@ -85,7 +85,7 @@ jobs: - name: Test util code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/util/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/util/test_*.py --durations=10 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-wallet-cat_wallet.yml b/.github/workflows/build-test-macos-wallet-cat_wallet.yml index 16be450bd0..32bc83d1ba 100644 --- a/.github/workflows/build-test-macos-wallet-cat_wallet.yml +++ b/.github/workflows/build-test-macos-wallet-cat_wallet.yml @@ -85,7 +85,7 @@ jobs: - name: Test wallet-cat_wallet code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/cat_wallet/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/cat_wallet/test_*.py --durations=10 -n 0 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-wallet-did_wallet.yml b/.github/workflows/build-test-macos-wallet-did_wallet.yml index 29efd583d7..fbb4fcb432 100644 --- a/.github/workflows/build-test-macos-wallet-did_wallet.yml +++ b/.github/workflows/build-test-macos-wallet-did_wallet.yml @@ -85,7 +85,7 @@ jobs: - name: Test wallet-did_wallet code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/did_wallet/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/did_wallet/test_*.py --durations=10 -n 0 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-wallet-rl_wallet.yml b/.github/workflows/build-test-macos-wallet-rl_wallet.yml index af3cb0d9d1..5d74c4e6bc 100644 --- a/.github/workflows/build-test-macos-wallet-rl_wallet.yml +++ b/.github/workflows/build-test-macos-wallet-rl_wallet.yml @@ -85,7 +85,7 @@ jobs: - name: Test wallet-rl_wallet code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/rl_wallet/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/rl_wallet/test_*.py --durations=10 -n 0 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-wallet-rpc.yml b/.github/workflows/build-test-macos-wallet-rpc.yml index 9629b8d628..63de5e94b0 100644 --- a/.github/workflows/build-test-macos-wallet-rpc.yml +++ b/.github/workflows/build-test-macos-wallet-rpc.yml @@ -85,7 +85,7 @@ jobs: - name: Test wallet-rpc code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/rpc/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/rpc/test_*.py --durations=10 -n 0 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-wallet-simple_sync.yml b/.github/workflows/build-test-macos-wallet-simple_sync.yml index 7c6ae1eee6..0b5ee26a52 100644 --- a/.github/workflows/build-test-macos-wallet-simple_sync.yml +++ b/.github/workflows/build-test-macos-wallet-simple_sync.yml @@ -85,7 +85,7 @@ jobs: - name: Test wallet-simple_sync code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/simple_sync/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/simple_sync/test_*.py --durations=10 -n 0 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-wallet-sync.yml b/.github/workflows/build-test-macos-wallet-sync.yml index ad293617eb..af8635d600 100644 --- a/.github/workflows/build-test-macos-wallet-sync.yml +++ b/.github/workflows/build-test-macos-wallet-sync.yml @@ -85,7 +85,7 @@ jobs: - name: Test wallet-sync code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/sync/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/sync/test_*.py --durations=10 -n 0 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-wallet.yml b/.github/workflows/build-test-macos-wallet.yml index 8073f21328..a756ecb9fa 100644 --- a/.github/workflows/build-test-macos-wallet.yml +++ b/.github/workflows/build-test-macos-wallet.yml @@ -85,7 +85,7 @@ jobs: - name: Test wallet code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/test_*.py --durations=10 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-macos-weight_proof.yml b/.github/workflows/build-test-macos-weight_proof.yml index 2c45969587..019df26033 100644 --- a/.github/workflows/build-test-macos-weight_proof.yml +++ b/.github/workflows/build-test-macos-weight_proof.yml @@ -85,7 +85,7 @@ jobs: - name: Test weight_proof code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/weight_proof/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/weight_proof/test_*.py --durations=10 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-blockchain.yml b/.github/workflows/build-test-ubuntu-blockchain.yml index 94109d800d..fda7296d91 100644 --- a/.github/workflows/build-test-ubuntu-blockchain.yml +++ b/.github/workflows/build-test-ubuntu-blockchain.yml @@ -84,7 +84,7 @@ jobs: - name: Test blockchain code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/blockchain/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/blockchain/test_*.py --durations=10 -n 4 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-clvm.yml b/.github/workflows/build-test-ubuntu-clvm.yml index de7660a488..35275d29e8 100644 --- a/.github/workflows/build-test-ubuntu-clvm.yml +++ b/.github/workflows/build-test-ubuntu-clvm.yml @@ -78,7 +78,7 @@ jobs: - name: Test clvm code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/clvm/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/clvm/test_*.py --durations=10 -n 4 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-core-cmds.yml b/.github/workflows/build-test-ubuntu-core-cmds.yml index 7939d45b1f..88ee7ab3cb 100644 --- a/.github/workflows/build-test-ubuntu-core-cmds.yml +++ b/.github/workflows/build-test-ubuntu-core-cmds.yml @@ -84,7 +84,7 @@ jobs: - name: Test core-cmds code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/cmds/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/cmds/test_*.py --durations=10 -n 4 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-core-consensus.yml b/.github/workflows/build-test-ubuntu-core-consensus.yml index 8083c05ad5..307f85716f 100644 --- a/.github/workflows/build-test-ubuntu-core-consensus.yml +++ b/.github/workflows/build-test-ubuntu-core-consensus.yml @@ -84,7 +84,7 @@ jobs: - name: Test core-consensus code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/consensus/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/consensus/test_*.py --durations=10 -n 4 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-core-custom_types.yml b/.github/workflows/build-test-ubuntu-core-custom_types.yml index 4684f2c25d..94f493f641 100644 --- a/.github/workflows/build-test-ubuntu-core-custom_types.yml +++ b/.github/workflows/build-test-ubuntu-core-custom_types.yml @@ -84,7 +84,7 @@ jobs: - name: Test core-custom_types code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/custom_types/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/custom_types/test_*.py --durations=10 -n 4 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-core-daemon.yml b/.github/workflows/build-test-ubuntu-core-daemon.yml index 524ab9ad30..9b69c941e9 100644 --- a/.github/workflows/build-test-ubuntu-core-daemon.yml +++ b/.github/workflows/build-test-ubuntu-core-daemon.yml @@ -96,7 +96,7 @@ jobs: - name: Test core-daemon code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/daemon/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/daemon/test_*.py --durations=10 -n 0 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-core-full_node-full_sync.yml b/.github/workflows/build-test-ubuntu-core-full_node-full_sync.yml index f43e01760c..8e5769be2d 100644 --- a/.github/workflows/build-test-ubuntu-core-full_node-full_sync.yml +++ b/.github/workflows/build-test-ubuntu-core-full_node-full_sync.yml @@ -84,7 +84,7 @@ jobs: - name: Test core-full_node-full_sync code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/full_node/full_sync/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/full_node/full_sync/test_*.py --durations=10 -n 4 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-core-full_node-stores.yml b/.github/workflows/build-test-ubuntu-core-full_node-stores.yml index d28a1294e9..d9cc30278c 100644 --- a/.github/workflows/build-test-ubuntu-core-full_node-stores.yml +++ b/.github/workflows/build-test-ubuntu-core-full_node-stores.yml @@ -84,7 +84,7 @@ jobs: - name: Test core-full_node-stores code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/full_node/stores/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/full_node/stores/test_*.py --durations=10 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-core-full_node.yml b/.github/workflows/build-test-ubuntu-core-full_node.yml index d19fb71f32..9e703a06b3 100644 --- a/.github/workflows/build-test-ubuntu-core-full_node.yml +++ b/.github/workflows/build-test-ubuntu-core-full_node.yml @@ -84,7 +84,7 @@ jobs: - name: Test core-full_node code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/full_node/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/full_node/test_*.py --durations=10 -n 4 -m "not benchmark" - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-core-server.yml b/.github/workflows/build-test-ubuntu-core-server.yml index 6a24118aa7..47bb7e7f4f 100644 --- a/.github/workflows/build-test-ubuntu-core-server.yml +++ b/.github/workflows/build-test-ubuntu-core-server.yml @@ -84,7 +84,7 @@ jobs: - name: Test core-server code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/server/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/server/test_*.py --durations=10 -n 0 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-core-ssl.yml b/.github/workflows/build-test-ubuntu-core-ssl.yml index fe58d94a7c..ba3c7945f5 100644 --- a/.github/workflows/build-test-ubuntu-core-ssl.yml +++ b/.github/workflows/build-test-ubuntu-core-ssl.yml @@ -84,7 +84,7 @@ jobs: - name: Test core-ssl code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/ssl/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/ssl/test_*.py --durations=10 -n 4 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-core-util.yml b/.github/workflows/build-test-ubuntu-core-util.yml index 8e845db650..938fd0f403 100644 --- a/.github/workflows/build-test-ubuntu-core-util.yml +++ b/.github/workflows/build-test-ubuntu-core-util.yml @@ -84,7 +84,7 @@ jobs: - name: Test core-util code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/util/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/util/test_*.py --durations=10 -n 4 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-core.yml b/.github/workflows/build-test-ubuntu-core.yml index 345e07b504..01be4c7181 100644 --- a/.github/workflows/build-test-ubuntu-core.yml +++ b/.github/workflows/build-test-ubuntu-core.yml @@ -84,7 +84,7 @@ jobs: - name: Test core code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/core/test_*.py --durations=10 -n 4 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-farmer_harvester.yml b/.github/workflows/build-test-ubuntu-farmer_harvester.yml index 37eaa651f7..6cd3a81af0 100644 --- a/.github/workflows/build-test-ubuntu-farmer_harvester.yml +++ b/.github/workflows/build-test-ubuntu-farmer_harvester.yml @@ -84,7 +84,7 @@ jobs: - name: Test farmer_harvester code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/farmer_harvester/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/farmer_harvester/test_*.py --durations=10 -n 0 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-generator.yml b/.github/workflows/build-test-ubuntu-generator.yml index 707deb5adb..1aa74d5daf 100644 --- a/.github/workflows/build-test-ubuntu-generator.yml +++ b/.github/workflows/build-test-ubuntu-generator.yml @@ -84,7 +84,7 @@ jobs: - name: Test generator code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/generator/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/generator/test_*.py --durations=10 -n 4 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-plotting.yml b/.github/workflows/build-test-ubuntu-plotting.yml index 9561180be1..26c186f954 100644 --- a/.github/workflows/build-test-ubuntu-plotting.yml +++ b/.github/workflows/build-test-ubuntu-plotting.yml @@ -84,7 +84,7 @@ jobs: - name: Test plotting code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/plotting/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/plotting/test_*.py --durations=10 -n 4 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-pools.yml b/.github/workflows/build-test-ubuntu-pools.yml index 91e5d819dc..9ae59262f9 100644 --- a/.github/workflows/build-test-ubuntu-pools.yml +++ b/.github/workflows/build-test-ubuntu-pools.yml @@ -84,7 +84,7 @@ jobs: - name: Test pools code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/pools/test_*.py -s -v --durations 0 -n 2 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/pools/test_*.py --durations=10 -n 2 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-simulation.yml b/.github/workflows/build-test-ubuntu-simulation.yml index 385fce0ed9..be0a0f55c3 100644 --- a/.github/workflows/build-test-ubuntu-simulation.yml +++ b/.github/workflows/build-test-ubuntu-simulation.yml @@ -96,7 +96,7 @@ jobs: - name: Test simulation code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/simulation/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/simulation/test_*.py --durations=10 -n 0 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-tools.yml b/.github/workflows/build-test-ubuntu-tools.yml index 4a738a726e..8b8ff6cfbe 100644 --- a/.github/workflows/build-test-ubuntu-tools.yml +++ b/.github/workflows/build-test-ubuntu-tools.yml @@ -84,7 +84,7 @@ jobs: - name: Test tools code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/tools/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/tools/test_*.py --durations=10 -n 4 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-util.yml b/.github/workflows/build-test-ubuntu-util.yml index 9637c59bc4..6f4768ccee 100644 --- a/.github/workflows/build-test-ubuntu-util.yml +++ b/.github/workflows/build-test-ubuntu-util.yml @@ -84,7 +84,7 @@ jobs: - name: Test util code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/util/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/util/test_*.py --durations=10 -n 4 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-wallet-cat_wallet.yml b/.github/workflows/build-test-ubuntu-wallet-cat_wallet.yml index b83b9aa953..b52dfde0c5 100644 --- a/.github/workflows/build-test-ubuntu-wallet-cat_wallet.yml +++ b/.github/workflows/build-test-ubuntu-wallet-cat_wallet.yml @@ -84,7 +84,7 @@ jobs: - name: Test wallet-cat_wallet code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/cat_wallet/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/cat_wallet/test_*.py --durations=10 -n 0 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-wallet-did_wallet.yml b/.github/workflows/build-test-ubuntu-wallet-did_wallet.yml index 084dca1d9d..f7e9f1012f 100644 --- a/.github/workflows/build-test-ubuntu-wallet-did_wallet.yml +++ b/.github/workflows/build-test-ubuntu-wallet-did_wallet.yml @@ -84,7 +84,7 @@ jobs: - name: Test wallet-did_wallet code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/did_wallet/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/did_wallet/test_*.py --durations=10 -n 0 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-wallet-rl_wallet.yml b/.github/workflows/build-test-ubuntu-wallet-rl_wallet.yml index 75200f1423..3d241ad1bb 100644 --- a/.github/workflows/build-test-ubuntu-wallet-rl_wallet.yml +++ b/.github/workflows/build-test-ubuntu-wallet-rl_wallet.yml @@ -84,7 +84,7 @@ jobs: - name: Test wallet-rl_wallet code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/rl_wallet/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/rl_wallet/test_*.py --durations=10 -n 0 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-wallet-rpc.yml b/.github/workflows/build-test-ubuntu-wallet-rpc.yml index 6f9a5e2277..05b4cef7a9 100644 --- a/.github/workflows/build-test-ubuntu-wallet-rpc.yml +++ b/.github/workflows/build-test-ubuntu-wallet-rpc.yml @@ -84,7 +84,7 @@ jobs: - name: Test wallet-rpc code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/rpc/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/rpc/test_*.py --durations=10 -n 0 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-wallet-simple_sync.yml b/.github/workflows/build-test-ubuntu-wallet-simple_sync.yml index 85fd306fc8..fdaa81b6dd 100644 --- a/.github/workflows/build-test-ubuntu-wallet-simple_sync.yml +++ b/.github/workflows/build-test-ubuntu-wallet-simple_sync.yml @@ -84,7 +84,7 @@ jobs: - name: Test wallet-simple_sync code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/simple_sync/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/simple_sync/test_*.py --durations=10 -n 0 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-wallet-sync.yml b/.github/workflows/build-test-ubuntu-wallet-sync.yml index 924485cb7e..797731d3f3 100644 --- a/.github/workflows/build-test-ubuntu-wallet-sync.yml +++ b/.github/workflows/build-test-ubuntu-wallet-sync.yml @@ -84,7 +84,7 @@ jobs: - name: Test wallet-sync code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/sync/test_*.py -s -v --durations 0 -n 0 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/sync/test_*.py --durations=10 -n 0 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-wallet.yml b/.github/workflows/build-test-ubuntu-wallet.yml index 88c1c1bfa5..28f2b1da0a 100644 --- a/.github/workflows/build-test-ubuntu-wallet.yml +++ b/.github/workflows/build-test-ubuntu-wallet.yml @@ -84,7 +84,7 @@ jobs: - name: Test wallet code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/wallet/test_*.py --durations=10 -n 4 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-weight_proof.yml b/.github/workflows/build-test-ubuntu-weight_proof.yml index 3a43ec6b3c..34f33582e2 100644 --- a/.github/workflows/build-test-ubuntu-weight_proof.yml +++ b/.github/workflows/build-test-ubuntu-weight_proof.yml @@ -84,7 +84,7 @@ jobs: - name: Test weight_proof code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/weight_proof/test_*.py -s -v --durations 0 -n 4 -m "not benchmark" -p no:monitor + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/weight_proof/test_*.py --durations=10 -n 4 -m "not benchmark" -p no:monitor - name: Process coverage data run: | diff --git a/tests/runner_templates/build-test-macos b/tests/runner_templates/build-test-macos index 6031c07ef4..536ab47fcf 100644 --- a/tests/runner_templates/build-test-macos +++ b/tests/runner_templates/build-test-macos @@ -79,7 +79,7 @@ INSTALL_TIMELORD - name: Test TEST_NAME code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test TEST_DIR -s -v --durations 0PYTEST_PARALLEL_ARGS -m "not benchmark" + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test TEST_DIR --durations=10 PYTEST_PARALLEL_ARGS -m "not benchmark" - name: Process coverage data run: | diff --git a/tests/runner_templates/build-test-ubuntu b/tests/runner_templates/build-test-ubuntu index 512d848740..bc587a89ae 100644 --- a/tests/runner_templates/build-test-ubuntu +++ b/tests/runner_templates/build-test-ubuntu @@ -78,7 +78,7 @@ INSTALL_TIMELORD - name: Test TEST_NAME code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test TEST_DIR -s -v --durations 0PYTEST_PARALLEL_ARGS -m "not benchmark" DISABLE_PYTEST_MONITOR + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test TEST_DIR --durations=10 PYTEST_PARALLEL_ARGS -m "not benchmark" DISABLE_PYTEST_MONITOR - name: Process coverage data run: | From 685744218e9f89044525e2209d29ecd5e8764a26 Mon Sep 17 00:00:00 2001 From: Mariano Sorgente <3069354+mariano54@users.noreply.github.com> Date: Fri, 1 Apr 2022 02:03:22 -0400 Subject: [PATCH 09/63] Ms.flaky gen speed (#10965) * Flaky test sometimes goes slower than 1 second * Add sleep to reduce flakiness * Increase timeout instead of sleeping to hopefully reduce flakiness --- tests/core/test_cost_calculation.py | 2 +- tests/wallet/test_wallet.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/core/test_cost_calculation.py b/tests/core/test_cost_calculation.py index af62892f70..2dfdc59c49 100644 --- a/tests/core/test_cost_calculation.py +++ b/tests/core/test_cost_calculation.py @@ -207,7 +207,7 @@ class TestCostCalculation: assert len(npc_result.npc_list) == LARGE_BLOCK_COIN_CONSUMED_COUNT log.info(f"Time spent: {duration}") - assert duration < 1 + assert duration < 2 @pytest.mark.asyncio async def test_clvm_max_cost(self, softfork_height): diff --git a/tests/wallet/test_wallet.py b/tests/wallet/test_wallet.py index 02467334ac..89dd3dd8bf 100644 --- a/tests/wallet/test_wallet.py +++ b/tests/wallet/test_wallet.py @@ -733,17 +733,17 @@ class TestWalletSimulator: await full_node_api.farm_new_transaction_block(FarmNewBlockProtocol(puzzle_hashes[114])) await full_node_api.farm_new_transaction_block(FarmNewBlockProtocol(32 * b"0")) - await time_out_assert(15, wallet.get_confirmed_balance, 2 * 10 ** 12) + await time_out_assert(60, wallet.get_confirmed_balance, 2 * 10 ** 12) await full_node_api.farm_new_transaction_block(FarmNewBlockProtocol(puzzle_hashes[50])) await full_node_api.farm_new_transaction_block(FarmNewBlockProtocol(32 * b"0")) - await time_out_assert(15, wallet.get_confirmed_balance, 8 * 10 ** 12) + await time_out_assert(60, wallet.get_confirmed_balance, 8 * 10 ** 12) await full_node_api.farm_new_transaction_block(FarmNewBlockProtocol(puzzle_hashes[113])) await full_node_api.farm_new_transaction_block(FarmNewBlockProtocol(puzzle_hashes[209])) await full_node_api.farm_new_transaction_block(FarmNewBlockProtocol(32 * b"0")) - await time_out_assert(15, wallet.get_confirmed_balance, 12 * 10 ** 12) + await time_out_assert(60, wallet.get_confirmed_balance, 12 * 10 ** 12) @pytest.mark.parametrize( "trusted", From 8ca0633d9f5f9cfe5cbe4bc634973ea026c4188d Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Fri, 1 Apr 2022 08:03:59 +0200 Subject: [PATCH 10/63] fix test_full_sync.py to only feed the blocks in the main chain to the node (#10974) --- chia/cmds/init_funcs.py | 1 + tools/test_full_sync.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/chia/cmds/init_funcs.py b/chia/cmds/init_funcs.py index 1bf2dea4f2..70dd0c1ce3 100644 --- a/chia/cmds/init_funcs.py +++ b/chia/cmds/init_funcs.py @@ -502,6 +502,7 @@ def chia_init( db_path_replaced = new_db_path.replace("CHALLENGE", config["selected_network"]) db_path = path_from_root(root_path, db_path_replaced) + mkdir(db_path.parent) with sqlite3.connect(db_path) as connection: set_db_version(connection, 1) diff --git a/tools/test_full_sync.py b/tools/test_full_sync.py index afd3197d96..e28a9d2f1f 100755 --- a/tools/test_full_sync.py +++ b/tools/test_full_sync.py @@ -64,7 +64,7 @@ async def run_sync_test(file: Path, db_version, profile: bool, single_thread: bo with tempfile.TemporaryDirectory() as root_dir: root_path = Path(root_dir) - chia_init(root_path, should_check_keys=False) + chia_init(root_path, should_check_keys=False, v1_db=(db_version == 1)) config = load_config(root_path, "config.yaml") overrides = config["network_overrides"]["constants"][config["selected_network"]] @@ -85,7 +85,9 @@ async def run_sync_test(file: Path, db_version, profile: bool, single_thread: bo counter = 0 async with aiosqlite.connect(file) as in_db: - rows = await in_db.execute("SELECT header_hash, height, block FROM full_blocks ORDER BY height") + rows = await in_db.execute( + "SELECT header_hash, height, block FROM full_blocks WHERE in_main_chain=1 ORDER BY height" + ) block_batch = [] @@ -122,7 +124,7 @@ def main() -> None: @main.command("run", short_help="run simulated full sync from an existing blockchain db") @click.argument("file", type=click.Path(), required=True) -@click.option("--db-version", type=int, required=False, default=2, help="the version of the specified db file") +@click.option("--db-version", type=int, required=False, default=2, help="the DB version to use in simulated node") @click.option("--profile", is_flag=True, required=False, default=False, help="dump CPU profiles for slow batches") @click.option( "--single-thread", @@ -132,6 +134,9 @@ def main() -> None: help="run node in a single process, to include validation in profiles", ) def run(file: Path, db_version: int, profile: bool, single_thread: bool) -> None: + """ + The FILE parameter should point to an existing blockchain database file (in v2 format) + """ asyncio.run(run_sync_test(Path(file), db_version, profile, single_thread)) From fc1d52de6a8b1ec49bbbe0910fa1ee04c9e89cef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 31 Mar 2022 23:04:48 -0700 Subject: [PATCH 11/63] Bump peter-evans/create-pull-request from 3 to 4 (#10950) Bumps [peter-evans/create-pull-request](https://github.com/peter-evans/create-pull-request) from 3 to 4. - [Release notes](https://github.com/peter-evans/create-pull-request/releases) - [Commits](https://github.com/peter-evans/create-pull-request/compare/v3...v4) --- updated-dependencies: - dependency-name: peter-evans/create-pull-request dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/mozilla-ca-cert.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mozilla-ca-cert.yml b/.github/workflows/mozilla-ca-cert.yml index 73edbaa78a..a637f430e1 100644 --- a/.github/workflows/mozilla-ca-cert.yml +++ b/.github/workflows/mozilla-ca-cert.yml @@ -20,7 +20,7 @@ jobs: cd ./mozilla-ca git pull origin main - name: "Create Pull Request" - uses: peter-evans/create-pull-request@v3 + uses: peter-evans/create-pull-request@v4 with: base: main body: "Newest Mozilla CA cert" From 441823f0da6dd15ea8570fc80d478293f1bf41d2 Mon Sep 17 00:00:00 2001 From: Adam Kelly <338792+aqk@users.noreply.github.com> Date: Thu, 31 Mar 2022 23:05:30 -0700 Subject: [PATCH 12/63] normalized_to_identity_cc_ip from get_consecutive_blocks was being passed in as overflow_cc_challenge in get_full_block_and_block_record (#10941) --- tests/block_tools.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/block_tools.py b/tests/block_tools.py index e2fb177bcc..f67a5d5c93 100644 --- a/tests/block_tools.py +++ b/tests/block_tools.py @@ -602,9 +602,6 @@ class BlockTools: block_generator = None aggregate_signature = G2Element() - # TODO: address hint error and remove ignore - # error: Argument 27 to "get_full_block_and_block_record" has incompatible type "bool"; - # expected "Optional[bytes32]" [arg-type] full_block, block_record = get_full_block_and_block_record( constants, blocks, @@ -632,7 +629,7 @@ class BlockTools: signage_point, latest_block, seed, - normalized_to_identity_cc_ip, # type: ignore[arg-type] + normalized_to_identity_cc_ip=normalized_to_identity_cc_ip, current_time=current_time, ) if block_record.is_transaction_block: @@ -1504,6 +1501,7 @@ def get_full_block_and_block_record( signage_point: SignagePoint, prev_block: BlockRecord, seed: bytes = b"", + *, overflow_cc_challenge: bytes32 = None, overflow_rc_challenge: bytes32 = None, normalized_to_identity_cc_ip: bool = False, From 647cf6b52b3b85415b9ba3804be3d67644693938 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Fri, 1 Apr 2022 16:12:10 +0200 Subject: [PATCH 13/63] fix performance tests (#10983) --- tests/core/full_node/test_mempool_performance.py | 2 +- tests/core/test_cost_calculation.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/core/full_node/test_mempool_performance.py b/tests/core/full_node/test_mempool_performance.py index b887863e55..4e8ed60496 100644 --- a/tests/core/full_node/test_mempool_performance.py +++ b/tests/core/full_node/test_mempool_performance.py @@ -77,4 +77,4 @@ class TestMempoolPerformance: if idx >= len(blocks) - 3: assert duration < 0.1 else: - assert duration < 0.0003 + assert duration < 0.001 diff --git a/tests/core/test_cost_calculation.py b/tests/core/test_cost_calculation.py index 2dfdc59c49..ac24ddf55c 100644 --- a/tests/core/test_cost_calculation.py +++ b/tests/core/test_cost_calculation.py @@ -187,6 +187,7 @@ class TestCostCalculation: assert npc_result.error is None @pytest.mark.asyncio + @pytest.mark.benchmark async def test_tx_generator_speed(self, softfork_height): LARGE_BLOCK_COIN_CONSUMED_COUNT = 687 generator_bytes = large_block_generator(LARGE_BLOCK_COIN_CONSUMED_COUNT) @@ -207,7 +208,7 @@ class TestCostCalculation: assert len(npc_result.npc_list) == LARGE_BLOCK_COIN_CONSUMED_COUNT log.info(f"Time spent: {duration}") - assert duration < 2 + assert duration < 0.5 @pytest.mark.asyncio async def test_clvm_max_cost(self, softfork_height): @@ -246,6 +247,7 @@ class TestCostCalculation: assert npc_result.cost > 10000000 @pytest.mark.asyncio + @pytest.mark.benchmark async def test_standard_tx(self): # this isn't a real public key, but we don't care public_key = bytes.fromhex( @@ -270,4 +272,4 @@ class TestCostCalculation: duration = time_end - time_start log.info(f"Time spent: {duration}") - assert duration < 3 + assert duration < 0.1 From 58abb351145ab940390dc84c44c51425597561fb Mon Sep 17 00:00:00 2001 From: Gene Hoffman <30377676+hoffmang9@users.noreply.github.com> Date: Fri, 1 Apr 2022 13:20:29 -0700 Subject: [PATCH 14/63] Check for vulnerable openssl (#10988) * Check for vulnerable openssl * Update OpenSSL on MacOS * First attempt - openssl Ubuntu 18.04 and 20.04 * place local/bin ahead in PATH * specify install openssl * correct path * run ldconfig * stop building and check for patched openssl * spell sudo right by removing it * Remove openssl building - 1st attempt RHs * Test Windows OpenSSL version HT @AmineKhaldi --- Install.ps1 | 8 ++++++++ install.sh | 31 ++++++++++++++++++++++--------- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/Install.ps1 b/Install.ps1 index ca69e4c41e..c97d6d737e 100644 --- a/Install.ps1 +++ b/Install.ps1 @@ -41,6 +41,14 @@ if ([version]$pythonVersion -lt [version]"3.7.0") } Write-Output "Python version is:" $pythonVersion +$openSSLVersionStr = (py -c 'import ssl; print(ssl.OPENSSL_VERSION)') +$openSSLVersion = (py -c 'import ssl; print(ssl.OPENSSL_VERSION_NUMBER)') +if ($openSSLVersion -lt 269488367) +{ + Write-Output "Found Python with OpenSSL version:" $openSSLVersionStr + Write-Output "Anything before 1.1.1n is vulnerable to CVE-2022-0778." +} + py -m venv venv venv\scripts\python -m pip install --upgrade pip setuptools wheel diff --git a/install.sh b/install.sh index 3dc27ae13f..e9a4c728ab 100755 --- a/install.sh +++ b/install.sh @@ -75,8 +75,8 @@ install_python3_and_sqlite3_from_source_with_yum() { # Preparing installing Python echo 'yum groupinstall -y "Development Tools"' sudo yum groupinstall -y "Development Tools" - echo "sudo yum install -y openssl-devel libffi-devel bzip2-devel wget" - sudo yum install -y openssl-devel libffi-devel bzip2-devel wget + echo "sudo yum install -y openssl-devel openssl libffi-devel bzip2-devel wget" + sudo yum install -y openssl-devel openssl libffi-devel bzip2-devel wget echo "cd $TMP_PATH" cd "$TMP_PATH" @@ -111,7 +111,6 @@ install_python3_and_sqlite3_from_source_with_yum() { cd "$CURRENT_WD" } - # Manage npm and other install requirements on an OS specific basis if [ "$(uname)" = "Linux" ]; then #LINUX=1 @@ -119,19 +118,21 @@ if [ "$(uname)" = "Linux" ]; then # Ubuntu echo "Installing on Ubuntu pre 20.04 LTS." sudo apt-get update - sudo apt-get install -y python3.7-venv python3.7-distutils + sudo apt-get install -y python3.7-venv python3.7-distutils openssl + apt show openssl elif [ "$UBUNTU" = "true" ] && [ "$UBUNTU_PRE_2004" = "0" ] && [ "$UBUNTU_2100" = "0" ]; then echo "Installing on Ubuntu 20.04 LTS." sudo apt-get update - sudo apt-get install -y python3.8-venv python3-distutils + sudo apt-get install -y python3.8-venv python3-distutils openssl + apt show openssl elif [ "$UBUNTU" = "true" ] && [ "$UBUNTU_2100" = "1" ]; then echo "Installing on Ubuntu 21.04 or newer." sudo apt-get update - sudo apt-get install -y python3.9-venv python3-distutils + sudo apt-get install -y python3.9-venv python3-distutils openssl elif [ "$DEBIAN" = "true" ]; then echo "Installing on Debian." sudo apt-get update - sudo apt-get install -y python3-venv + sudo apt-get install -y python3-venv openssl elif type pacman >/dev/null 2>&1 && [ -f "/etc/arch-release" ]; then # Arch Linux # Arch provides latest python version. User will need to manually install python 3.9 if it is not present @@ -160,16 +161,17 @@ if [ "$(uname)" = "Linux" ]; then elif type yum >/dev/null 2>&1 && [ -f "/etc/redhat-release" ] && grep Rocky /etc/redhat-release; then echo "Installing on Rocky." # TODO: make this smarter about getting the latest version - sudo yum install --assumeyes python39 + sudo yum install --assumeyes python39 openssl elif type yum >/dev/null 2>&1 && [ -f "/etc/redhat-release" ] || [ -f "/etc/fedora-release" ]; then # Redhat or Fedora echo "Installing on Redhat/Fedora." if ! command -v python3.9 >/dev/null 2>&1; then - sudo yum install -y python39 + sudo yum install -y python39 openssl fi fi elif [ "$(uname)" = "Darwin" ] && ! type brew >/dev/null 2>&1; then echo "Installation currently requires brew on MacOS - https://brew.sh/" + brew install openssl elif [ "$(uname)" = "OpenBSD" ]; then export MAKE=${MAKE:-gmake} export BUILD_VDF_CLIENT=${BUILD_VDF_CLIENT:-N} @@ -231,6 +233,17 @@ if [ "$SQLITE_MAJOR_VER" -lt "3" ] || [ "$SQLITE_MAJOR_VER" = "3" ] && [ "$SQLIT exit 1 fi +# Check openssl version python will use +OPENSSL_VERSION_STRING=$($INSTALL_PYTHON_PATH -c 'import ssl; print(ssl.OPENSSL_VERSION)') +OPENSSL_VERSION_INT=$($INSTALL_PYTHON_PATH -c 'import ssl; print(ssl.OPENSSL_VERSION_NUMBER)') +# There is also ssl.OPENSSL_VERSION_INFO returning a tuple +# 1.1.1n corresponds to 269488367 as an integer +echo "OpenSSL version for Python is ${OPENSSL_VERSION_STRING}" +if [ "$OPENSSL_VERSION_INT" -lt "269488367" ]; then + echo "WARNING: OpenSSL versions before 3.0.2, 1.1.1n, or 1.0.2zd are vulnerable to CVE-2022-0778" + echo "Your OS may have patched OpenSSL and not updated the version to 1.1.1n" +fi + # If version of `python` and "$INSTALL_PYTHON_VERSION" does not match, clear old version VENV_CLEAR="" if [ -e venv/bin/python ]; then From 8cbf96d73cdb00d81b4bcd334fabd71422b43efe Mon Sep 17 00:00:00 2001 From: Gene Hoffman <30377676+hoffmang9@users.noreply.github.com> Date: Fri, 1 Apr 2022 16:31:00 -0700 Subject: [PATCH 15/63] Non Hobo patch the winstaller for CVE-2022-0778 (#10995) --- .github/workflows/build-windows-installer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-windows-installer.yml b/.github/workflows/build-windows-installer.yml index 7f7c6bd8cb..67ee617b5e 100644 --- a/.github/workflows/build-windows-installer.yml +++ b/.github/workflows/build-windows-installer.yml @@ -62,7 +62,7 @@ jobs: - uses: actions/setup-python@v2 name: Install Python 3.9 with: - python-version: "3.9" + python-version: "3.9.11" - name: Setup Node 16.x uses: actions/setup-node@v3 From 14627d8a6efe5515393948acfbd740e6f7bdf58b Mon Sep 17 00:00:00 2001 From: Gene Hoffman <30377676+hoffmang9@users.noreply.github.com> Date: Fri, 1 Apr 2022 18:01:30 -0700 Subject: [PATCH 16/63] apt show not needed (#10997) --- install.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/install.sh b/install.sh index e9a4c728ab..9d73724a9d 100755 --- a/install.sh +++ b/install.sh @@ -119,12 +119,10 @@ if [ "$(uname)" = "Linux" ]; then echo "Installing on Ubuntu pre 20.04 LTS." sudo apt-get update sudo apt-get install -y python3.7-venv python3.7-distutils openssl - apt show openssl elif [ "$UBUNTU" = "true" ] && [ "$UBUNTU_PRE_2004" = "0" ] && [ "$UBUNTU_2100" = "0" ]; then echo "Installing on Ubuntu 20.04 LTS." sudo apt-get update sudo apt-get install -y python3.8-venv python3-distutils openssl - apt show openssl elif [ "$UBUNTU" = "true" ] && [ "$UBUNTU_2100" = "1" ]; then echo "Installing on Ubuntu 21.04 or newer." sudo apt-get update From 3e2b979751b52bb43b13c57a579d2c2484b451b0 Mon Sep 17 00:00:00 2001 From: Gene Hoffman <30377676+hoffmang9@users.noreply.github.com> Date: Fri, 1 Apr 2022 19:28:20 -0700 Subject: [PATCH 17/63] install/upgrade openssl on Arch Linux also (#10999) --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 9d73724a9d..0bd6c7266d 100755 --- a/install.sh +++ b/install.sh @@ -137,7 +137,7 @@ if [ "$(uname)" = "Linux" ]; then echo "Installing on Arch Linux." case $(uname -m) in x86_64|aarch64) - sudo pacman ${PACMAN_AUTOMATED} -S --needed git + sudo pacman ${PACMAN_AUTOMATED} -S --needed git openssl ;; *) echo "Incompatible CPU architecture. Must be x86_64 or aarch64." From e3e814b9ecdbc63c711ad9f2ca6b0b5fc1b76d8d Mon Sep 17 00:00:00 2001 From: Gene Hoffman <30377676+hoffmang9@users.noreply.github.com> Date: Fri, 1 Apr 2022 21:06:32 -0700 Subject: [PATCH 18/63] Compile python 3.9.11 which is aware of the openssl issue (#11001) --- install.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/install.sh b/install.sh index 0bd6c7266d..1a1b0afc4a 100755 --- a/install.sh +++ b/install.sh @@ -96,11 +96,11 @@ install_python3_and_sqlite3_from_source_with_yum() { sudo make install | stdbuf -o0 cut -b1-"$(tput cols)" | sed -u 'i\\o033[2K' | stdbuf -o0 tr '\n' '\r'; echo # yum install python3 brings Python3.6 which is not supported by chia cd .. - echo "wget https://www.python.org/ftp/python/3.9.9/Python-3.9.9.tgz" - wget https://www.python.org/ftp/python/3.9.9/Python-3.9.9.tgz - tar xf Python-3.9.9.tgz - echo "cd Python-3.9.9" - cd Python-3.9.9 + echo "wget https://www.python.org/ftp/python/3.9.11/Python-3.9.11.tgz" + wget https://www.python.org/ftp/python/3.9.11/Python-3.9.11.tgz + tar xf Python-3.9.11.tgz + echo "cd Python-3.9.11" + cd Python-3.9.11 echo "LD_RUN_PATH=/usr/local/lib ./configure --prefix=/usr/local" # '| stdbuf ...' seems weird but this makes command outputs stay in single line. LD_RUN_PATH=/usr/local/lib ./configure --prefix=/usr/local | stdbuf -o0 cut -b1-"$(tput cols)" | sed -u 'i\\o033[2K' | stdbuf -o0 tr '\n' '\r'; echo From 131bd4e2f32c0ac30af9638e85e2c3455c1b5381 Mon Sep 17 00:00:00 2001 From: Gene Hoffman <30377676+hoffmang9@users.noreply.github.com> Date: Fri, 1 Apr 2022 21:22:56 -0700 Subject: [PATCH 19/63] install.sh is not upgrading OpenSSL on MacOS (#11003) * MacOS isn't updating OpenSSL in install.sh * Exit if no brew on MacOS * Code the if tree like a pro instead. Co-authored-by: Kyle Altendorf Co-authored-by: Kyle Altendorf --- install.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index 1a1b0afc4a..5ce4095fed 100755 --- a/install.sh +++ b/install.sh @@ -167,8 +167,13 @@ if [ "$(uname)" = "Linux" ]; then sudo yum install -y python39 openssl fi fi -elif [ "$(uname)" = "Darwin" ] && ! type brew >/dev/null 2>&1; then - echo "Installation currently requires brew on MacOS - https://brew.sh/" +elif [ "$(uname)" = "Darwin" ]; then + echo "Installing on macOS." + if ! type brew >/dev/null 2>&1; then + echo "Installation currently requires brew on macOS - https://brew.sh/" + exit 1 + fi + echo "Installing OpenSSL" brew install openssl elif [ "$(uname)" = "OpenBSD" ]; then export MAKE=${MAKE:-gmake} From 8b5c7012dc01dac1842651db3b0e0f3e98f7612e Mon Sep 17 00:00:00 2001 From: roseiliend <90035993+roseiliend@users.noreply.github.com> Date: Sun, 3 Apr 2022 04:21:54 +0800 Subject: [PATCH 20/63] force index in get_coin_records_by_names (#10987) * force index in get_coin_records_by_names * fix lint --- chia/full_node/coin_store.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/chia/full_node/coin_store.py b/chia/full_node/coin_store.py index 3fec82d411..9f24e68e45 100644 --- a/chia/full_node/coin_store.py +++ b/chia/full_node/coin_store.py @@ -288,7 +288,8 @@ class CoinStore: async with self.db_wrapper.read_db() as conn: async with conn.execute( f"SELECT confirmed_index, spent_index, coinbase, puzzle_hash, " - f'coin_parent, amount, timestamp FROM coin_record WHERE coin_name in ({"?," * (len(names) - 1)}?) ' + f"coin_parent, amount, timestamp FROM coin_record INDEXED BY sqlite_autoindex_coin_record_1 " + f'WHERE coin_name in ({"?," * (len(names) - 1)}?) ' f"AND confirmed_index>=? AND confirmed_index Date: Sat, 2 Apr 2022 16:22:55 -0400 Subject: [PATCH 21/63] Fix remaining linting issues (#10962) * FIx remaining linting issues * Revert type:ignore * Revert token_bytes change --- benchmarks/utils.py | 10 +- chia/consensus/block_body_validation.py | 12 +- chia/consensus/block_creation.py | 22 +--- chia/consensus/blockchain.py | 49 ++++---- chia/consensus/multiprocess_validation.py | 14 +-- chia/full_node/block_store.py | 7 +- chia/full_node/full_node.py | 7 +- chia/full_node/full_node_api.py | 107 ++++++++---------- chia/rpc/full_node_rpc_api.py | 7 +- chia/server/server.py | 10 +- chia/simulator/simulator_constants.py | 5 +- chia/timelord/timelord_state.py | 9 +- .../types/blockchain_format/proof_of_space.py | 6 +- chia/util/block_cache.py | 11 +- chia/util/generator_tools.py | 12 +- tests/block_tools.py | 73 ++++-------- .../full_node/stores/test_full_node_store.py | 6 +- tests/core/full_node/test_block_height_map.py | 4 +- tests/core/full_node/test_mempool.py | 6 +- tests/core/make_block_generator.py | 20 ++-- tests/pools/test_wallet_pool_store.py | 5 +- tests/util/key_tool.py | 12 +- tests/util/network_protocol_data.py | 4 +- tests/wallet_tools.py | 34 +++--- tests/weight_proof/test_weight_proof.py | 5 +- 25 files changed, 170 insertions(+), 287 deletions(-) diff --git a/benchmarks/utils.py b/benchmarks/utils.py index 3646db14f3..af9db5e23e 100644 --- a/benchmarks/utils.py +++ b/benchmarks/utils.py @@ -3,7 +3,7 @@ from chia.util.ints import uint64, uint32, uint8 from chia.consensus.coinbase import create_farmer_coin, create_pool_coin from chia.types.blockchain_format.classgroup import ClassgroupElement from chia.types.blockchain_format.coin import Coin -from chia.types.blockchain_format.sized_bytes import bytes32 +from chia.types.blockchain_format.sized_bytes import bytes32, bytes100 from chia.types.blockchain_format.vdf import VDFInfo, VDFProof from chia.types.blockchain_format.foliage import Foliage, FoliageBlockData, FoliageTransactionBlock, TransactionsInfo from chia.types.blockchain_format.pool_target import PoolTarget @@ -56,9 +56,7 @@ def rand_bytes(num) -> bytes: def rand_hash() -> bytes32: - # TODO: address hint errors and remove ignores - # error: Incompatible return value type (got "bytes", expected "bytes32") [return-value] - return rand_bytes(32) # type: ignore[return-value] + return bytes32(rand_bytes(32)) def rand_g1() -> G1Element: @@ -72,9 +70,7 @@ def rand_g2() -> G2Element: def rand_class_group_element() -> ClassgroupElement: - # TODO: address hint errors and remove ignores - # error: Argument 1 to "ClassgroupElement" has incompatible type "bytes"; expected "bytes100" [arg-type] - return ClassgroupElement(rand_bytes(100)) # type: ignore[arg-type] + return ClassgroupElement(bytes100(rand_bytes(100))) def rand_vdf() -> VDFInfo: diff --git a/chia/consensus/block_body_validation.py b/chia/consensus/block_body_validation.py index 5ce7cad6f5..9bb704c730 100644 --- a/chia/consensus/block_body_validation.py +++ b/chia/consensus/block_body_validation.py @@ -245,18 +245,12 @@ async def validate_block_body( return root_error, None # 12. The additions and removals must result in the correct filter - byte_array_tx: List[bytes32] = [] + byte_array_tx: List[bytearray] = [] for coin in additions + coinbase_additions: - # TODO: address hint error and remove ignore - # error: Argument 1 to "append" of "list" has incompatible type "bytearray"; expected "bytes32" - # [arg-type] - byte_array_tx.append(bytearray(coin.puzzle_hash)) # type: ignore[arg-type] + byte_array_tx.append(bytearray(coin.puzzle_hash)) for coin_name in removals: - # TODO: address hint error and remove ignore - # error: Argument 1 to "append" of "list" has incompatible type "bytearray"; expected "bytes32" - # [arg-type] - byte_array_tx.append(bytearray(coin_name)) # type: ignore[arg-type] + byte_array_tx.append(bytearray(coin_name)) bip158: PyBIP158 = PyBIP158(byte_array_tx) encoded_filter = bytes(bip158.GetEncoded()) diff --git a/chia/consensus/block_creation.py b/chia/consensus/block_creation.py index 1b57abce0e..a308551274 100644 --- a/chia/consensus/block_creation.py +++ b/chia/consensus/block_creation.py @@ -35,9 +35,6 @@ from chia.util.recursive_replace import recursive_replace log = logging.getLogger(__name__) -# TODO: address hint error and remove ignore -# error: Incompatible default for argument "seed" (default has type "bytes", argument has type "bytes32") -# [assignment] def create_foliage( constants: ConsensusConstants, reward_block_unfinished: RewardChainBlockUnfinished, @@ -53,7 +50,7 @@ def create_foliage( pool_target: PoolTarget, get_plot_signature: Callable[[bytes32, G1Element], G2Element], get_pool_signature: Callable[[PoolTarget, Optional[G1Element]], Optional[G2Element]], - seed: bytes32 = b"", # type: ignore[assignment] + seed: bytes = b"", ) -> Tuple[Foliage, Optional[FoliageTransactionBlock], Optional[TransactionsInfo]]: """ Creates a foliage for a given reward chain block. This may or may not be a tx block. In the case of a tx block, @@ -95,7 +92,7 @@ def create_foliage( height = uint32(prev_block.height + 1) # Create filter - byte_array_tx: List[bytes32] = [] + byte_array_tx: List[bytearray] = [] tx_additions: List[Coin] = [] tx_removals: List[bytes32] = [] @@ -192,16 +189,10 @@ def create_foliage( additions.extend(reward_claims_incorporated.copy()) for coin in additions: tx_additions.append(coin) - # TODO: address hint error and remove ignore - # error: Argument 1 to "append" of "list" has incompatible type "bytearray"; expected "bytes32" - # [arg-type] - byte_array_tx.append(bytearray(coin.puzzle_hash)) # type: ignore[arg-type] + byte_array_tx.append(bytearray(coin.puzzle_hash)) for coin in removals: tx_removals.append(coin.name()) - # TODO: address hint error and remove ignore - # error: Argument 1 to "append" of "list" has incompatible type "bytearray"; expected "bytes32" - # [arg-type] - byte_array_tx.append(bytearray(coin.name())) # type: ignore[arg-type] + byte_array_tx.append(bytearray(coin.name())) bip158: PyBIP158 = PyBIP158(byte_array_tx) encoded = bytes(bip158.GetEncoded()) @@ -289,9 +280,6 @@ def create_foliage( return foliage, foliage_transaction_block, transactions_info -# TODO: address hint error and remove ignore -# error: Incompatible default for argument "seed" (default has type "bytes", argument has type "bytes32") -# [assignment] def create_unfinished_block( constants: ConsensusConstants, sub_slot_start_total_iters: uint128, @@ -308,7 +296,7 @@ def create_unfinished_block( signage_point: SignagePoint, timestamp: uint64, blocks: BlockchainInterface, - seed: bytes32 = b"", # type: ignore[assignment] + seed: bytes = b"", block_generator: Optional[BlockGenerator] = None, aggregate_sig: G2Element = G2Element(), additions: Optional[List[Coin]] = None, diff --git a/chia/consensus/blockchain.py b/chia/consensus/blockchain.py index 91fcd4458e..4f8dccfc50 100644 --- a/chia/consensus/blockchain.py +++ b/chia/consensus/blockchain.py @@ -182,10 +182,9 @@ class Blockchain(BlockchainInterface): if self._peak_height is None: return None """ Return list of FullBlocks that are peaks""" - # TODO: address hint error and remove ignore - # error: Argument 1 to "get_full_block" of "BlockStore" has incompatible type "Optional[bytes32]"; - # expected "bytes32" [arg-type] - block = await self.block_store.get_full_block(self.height_to_hash(self._peak_height)) # type: ignore[arg-type] + peak_hash: Optional[bytes32] = self.height_to_hash(self._peak_height) + assert peak_hash is not None # Since we must have the peak block + block = await self.block_store.get_full_block(peak_hash) assert block is not None return block @@ -308,12 +307,9 @@ class Blockchain(BlockchainInterface): if opcode == ConditionOpcode.CREATE_COIN: for condition in conditions: if len(condition.vars) > 2 and condition.vars[2] != b"": - puzzle_hash, amount_bin = condition.vars[0], condition.vars[1] - amount = int_from_bytes(amount_bin) - # TODO: address hint error and remove ignore - # error: Argument 2 to "Coin" has incompatible type "bytes"; expected "bytes32" - # [arg-type] - coin_id = Coin(npc.coin_name, puzzle_hash, amount).name() # type: ignore[arg-type] + puzzle_hash, amount_bin = bytes32(condition.vars[0]), condition.vars[1] + amount: uint64 = uint64(int_from_bytes(amount_bin)) + coin_id: bytes32 = Coin(npc.coin_name, puzzle_hash, amount).name() h_list.append((coin_id, condition.vars[2])) return h_list @@ -679,11 +675,11 @@ class Blockchain(BlockchainInterface): return self.__block_records[header_hash] def height_to_block_record(self, height: uint32) -> BlockRecord: - header_hash = self.height_to_hash(height) - # TODO: address hint error and remove ignore - # error: Argument 1 to "block_record" of "Blockchain" has incompatible type "Optional[bytes32]"; expected - # "bytes32" [arg-type] - return self.block_record(header_hash) # type: ignore[arg-type] + # Precondition: height is in the blockchain + header_hash: Optional[bytes32] = self.height_to_hash(height) + if header_hash is None: + raise ValueError(f"Height is not in blockchain: {height}") + return self.block_record(header_hash) def get_ses_heights(self) -> List[uint32]: return self.__height_map.get_ses_heights() @@ -762,10 +758,8 @@ class Blockchain(BlockchainInterface): hashes = [] for height in range(start, stop + 1): if self.contains_height(uint32(height)): - # TODO: address hint error and remove ignore - # error: Incompatible types in assignment (expression has type "Optional[bytes32]", variable has - # type "bytes32") [assignment] - header_hash: bytes32 = self.height_to_hash(uint32(height)) # type: ignore[assignment] + header_hash: Optional[bytes32] = self.height_to_hash(uint32(height)) + assert header_hash is not None hashes.append(header_hash) blocks: List[FullBlock] = [] @@ -810,23 +804,20 @@ class Blockchain(BlockchainInterface): gets block records by height (only blocks that are part of the chain) """ records: List[BlockRecord] = [] - hashes = [] + hashes: List[bytes32] = [] assert batch_size < 999 # sqlite in python 3.7 has a limit on 999 variables in queries for height in heights: - hashes.append(self.height_to_hash(height)) + header_hash: Optional[bytes32] = self.height_to_hash(height) + if header_hash is None: + raise ValueError(f"Do not have block at height {height}") + hashes.append(header_hash) if len(hashes) > batch_size: - # TODO: address hint error and remove ignore - # error: Argument 1 to "get_block_records_by_hash" of "BlockStore" has incompatible type - # "List[Optional[bytes32]]"; expected "List[bytes32]" [arg-type] - res = await self.block_store.get_block_records_by_hash(hashes) # type: ignore[arg-type] + res = await self.block_store.get_block_records_by_hash(hashes) records.extend(res) hashes = [] if len(hashes) > 0: - # TODO: address hint error and remove ignore - # error: Argument 1 to "get_block_records_by_hash" of "BlockStore" has incompatible type - # "List[Optional[bytes32]]"; expected "List[bytes32]" [arg-type] - res = await self.block_store.get_block_records_by_hash(hashes) # type: ignore[arg-type] + res = await self.block_store.get_block_records_by_hash(hashes) records.extend(res) return records diff --git a/chia/consensus/multiprocess_validation.py b/chia/consensus/multiprocess_validation.py index 9fe5485829..0e0ad3ff85 100644 --- a/chia/consensus/multiprocess_validation.py +++ b/chia/consensus/multiprocess_validation.py @@ -56,9 +56,9 @@ def batch_pre_validate_blocks( expected_sub_slot_iters: List[uint64], validate_signatures: bool, ) -> List[bytes]: - blocks: Dict[bytes, BlockRecord] = {} + blocks: Dict[bytes32, BlockRecord] = {} for k, v in blocks_pickled.items(): - blocks[k] = BlockRecord.from_bytes(v) + blocks[bytes32(k)] = BlockRecord.from_bytes(v) results: List[PreValidationResult] = [] constants: ConsensusConstants = dataclass_from_dict(ConsensusConstants, constants_dict) if full_blocks_pickled is not None and header_blocks_pickled is not None: @@ -99,12 +99,9 @@ def batch_pre_validate_blocks( continue header_block = get_block_header(block, tx_additions, removals) - # TODO: address hint error and remove ignore - # error: Argument 1 to "BlockCache" has incompatible type "Dict[bytes, BlockRecord]"; expected - # "Dict[bytes32, BlockRecord]" [arg-type] required_iters, error = validate_finished_header_block( constants, - BlockCache(blocks), # type: ignore[arg-type] + BlockCache(blocks), header_block, check_filter, expected_difficulty[i], @@ -144,12 +141,9 @@ def batch_pre_validate_blocks( for i in range(len(header_blocks_pickled)): try: header_block = HeaderBlock.from_bytes(header_blocks_pickled[i]) - # TODO: address hint error and remove ignore - # error: Argument 1 to "BlockCache" has incompatible type "Dict[bytes, BlockRecord]"; expected - # "Dict[bytes32, BlockRecord]" [arg-type] required_iters, error = validate_finished_header_block( constants, - BlockCache(blocks), # type: ignore[arg-type] + BlockCache(blocks), header_block, check_filter, expected_difficulty[i], diff --git a/chia/full_node/block_store.py b/chia/full_node/block_store.py index 3da53d27ad..aad306cf62 100644 --- a/chia/full_node/block_store.py +++ b/chia/full_node/block_store.py @@ -421,12 +421,9 @@ class BlockStore: async with self.db_wrapper.read_db() as conn: async with conn.execute(formatted_str, header_hashes_db) as cursor: for row in await cursor.fetchall(): - header_hash = self.maybe_from_hex(row[0]) + header_hash = bytes32(self.maybe_from_hex(row[0])) full_block: FullBlock = self.maybe_decompress(row[1]) - # TODO: address hint error and remove ignore - # error: Invalid index type "bytes" for "Dict[bytes32, FullBlock]"; - # expected type "bytes32" [index] - all_blocks[header_hash] = full_block # type: ignore[index] + all_blocks[header_hash] = full_block self.block_cache.put(header_hash, full_block) ret: List[FullBlock] = [] for hh in header_hashes: diff --git a/chia/full_node/full_node.py b/chia/full_node/full_node.py index 0eeeede688..5ac784804f 100644 --- a/chia/full_node/full_node.py +++ b/chia/full_node/full_node.py @@ -1297,10 +1297,9 @@ class FullNode: fork_block: Optional[BlockRecord] = None if fork_height != block.height - 1 and block.height != 0: # This is a reorg - # TODO: address hint error and remove ignore - # error: Argument 1 to "block_record" of "Blockchain" has incompatible type "Optional[bytes32]"; - # expected "bytes32" [arg-type] - fork_block = self.blockchain.block_record(self.blockchain.height_to_hash(fork_height)) # type: ignore[arg-type] # noqa: E501 + fork_hash: Optional[bytes32] = self.blockchain.height_to_hash(fork_height) + assert fork_hash is not None + fork_block = self.blockchain.block_record(fork_hash) fns_peak_result: FullNodeStorePeakResult = self.full_node_store.new_peak( record, diff --git a/chia/full_node/full_node_api.py b/chia/full_node/full_node_api.py index 38e2debdda..0811784806 100644 --- a/chia/full_node/full_node_api.py +++ b/chia/full_node/full_node_api.py @@ -199,13 +199,11 @@ class FullNodeAPI: if task_id in full_node.full_node_store.tx_fetch_tasks: full_node.full_node_store.tx_fetch_tasks.pop(task_id) - task_id = token_bytes() + task_id: bytes32 = bytes32(token_bytes(32)) fetch_task = asyncio.create_task( tx_request_and_timeout(self.full_node, transaction.transaction_id, task_id) ) - # TODO: address hint error and remove ignore - # error: Invalid index type "bytes" for "Dict[bytes32, Task[Any]]"; expected type "bytes32" [index] - self.full_node.full_node_store.tx_fetch_tasks[task_id] = fetch_task # type: ignore[index] + self.full_node.full_node_store.tx_fetch_tasks[task_id] = fetch_task return None return None @@ -311,18 +309,16 @@ class FullNodeAPI: reject = RejectBlock(request.height) msg = make_msg(ProtocolMessageTypes.reject_block, reject) return msg - header_hash = self.full_node.blockchain.height_to_hash(request.height) - # TODO: address hint error and remove ignore - # error: Argument 1 to "get_full_block" of "BlockStore" has incompatible type "Optional[bytes32]"; - # expected "bytes32" [arg-type] - block: Optional[FullBlock] = await self.full_node.block_store.get_full_block(header_hash) # type: ignore[arg-type] # noqa: E501 + header_hash: Optional[bytes32] = self.full_node.blockchain.height_to_hash(request.height) + if header_hash is None: + return make_msg(ProtocolMessageTypes.reject_block, RejectBlock(request.height)) + + block: Optional[FullBlock] = await self.full_node.block_store.get_full_block(header_hash) if block is not None: if not request.include_transaction_block and block.transactions_generator is not None: block = dataclasses.replace(block, transactions_generator=None) return make_msg(ProtocolMessageTypes.respond_block, full_node_protocol.RespondBlock(block)) - reject = RejectBlock(request.height) - msg = make_msg(ProtocolMessageTypes.reject_block, reject) - return msg + return make_msg(ProtocolMessageTypes.reject_block, RejectBlock(request.height)) @api_request @reply_type([ProtocolMessageTypes.respond_blocks, ProtocolMessageTypes.reject_blocks]) @@ -340,16 +336,15 @@ class FullNodeAPI: if not request.include_transaction_block: blocks: List[FullBlock] = [] for i in range(request.start_height, request.end_height + 1): - # TODO: address hint error and remove ignore - # error: Argument 1 to "get_full_block" of "BlockStore" has incompatible type "Optional[bytes32]"; - # expected "bytes32" [arg-type] - block: Optional[FullBlock] = await self.full_node.block_store.get_full_block( - self.full_node.blockchain.height_to_hash(uint32(i)) # type: ignore[arg-type] - ) + header_hash_i: Optional[bytes32] = self.full_node.blockchain.height_to_hash(uint32(i)) + if header_hash_i is None: + reject = RejectBlocks(request.start_height, request.end_height) + return make_msg(ProtocolMessageTypes.reject_blocks, reject) + + block: Optional[FullBlock] = await self.full_node.block_store.get_full_block(header_hash_i) if block is None: reject = RejectBlocks(request.start_height, request.end_height) - msg = make_msg(ProtocolMessageTypes.reject_blocks, reject) - return msg + return make_msg(ProtocolMessageTypes.reject_blocks, reject) block = dataclasses.replace(block, transactions_generator=None) blocks.append(block) msg = make_msg( @@ -359,12 +354,11 @@ class FullNodeAPI: else: blocks_bytes: List[bytes] = [] for i in range(request.start_height, request.end_height + 1): - # TODO: address hint error and remove ignore - # error: Argument 1 to "get_full_block_bytes" of "BlockStore" has incompatible type - # "Optional[bytes32]"; expected "bytes32" [arg-type] - block_bytes: Optional[bytes] = await self.full_node.block_store.get_full_block_bytes( - self.full_node.blockchain.height_to_hash(uint32(i)) # type: ignore[arg-type] - ) + header_hash_i = self.full_node.blockchain.height_to_hash(uint32(i)) + if header_hash_i is None: + reject = RejectBlocks(request.start_height, request.end_height) + return make_msg(ProtocolMessageTypes.reject_blocks, reject) + block_bytes: Optional[bytes] = await self.full_node.block_store.get_full_block_bytes(header_hash_i) if block_bytes is None: reject = RejectBlocks(request.start_height, request.end_height) msg = make_msg(ProtocolMessageTypes.reject_blocks, reject) @@ -898,9 +892,6 @@ class FullNodeAPI: timestamp = uint64(int(curr.timestamp + 1)) self.log.info("Starting to make the unfinished block") - # TODO: address hint error and remove ignore - # error: Argument 16 to "create_unfinished_block" has incompatible type "bytes"; expected "bytes32" - # [arg-type] unfinished_block: UnfinishedBlock = create_unfinished_block( self.full_node.constants, total_iters_pos_slot, @@ -917,7 +908,7 @@ class FullNodeAPI: sp_vdfs, timestamp, self.full_node.blockchain, - b"", # type: ignore[arg-type] + b"", block_generator, aggregate_signature, additions, @@ -937,22 +928,17 @@ class FullNodeAPI: foliage_transaction_block_hash = unfinished_block.foliage.foliage_transaction_block_hash else: foliage_transaction_block_hash = bytes32([0] * 32) + assert foliage_transaction_block_hash is not None - # TODO: address hint error and remove ignore - # error: Argument 3 to "RequestSignedValues" has incompatible type "Optional[bytes32]"; expected - # "bytes32" [arg-type] message = farmer_protocol.RequestSignedValues( quality_string, foliage_sb_data_hash, - foliage_transaction_block_hash, # type: ignore[arg-type] + foliage_transaction_block_hash, ) await peer.send_message(make_msg(ProtocolMessageTypes.request_signed_values, message)) # Adds backup in case the first one fails if unfinished_block.is_transaction_block() and unfinished_block.transactions_generator is not None: - # TODO: address hint error and remove ignore - # error: Argument 16 to "create_unfinished_block" has incompatible type "bytes"; expected - # "bytes32" [arg-type] unfinished_block_backup = create_unfinished_block( self.full_node.constants, total_iters_pos_slot, @@ -969,7 +955,7 @@ class FullNodeAPI: sp_vdfs, timestamp, self.full_node.blockchain, - b"", # type: ignore[arg-type] + b"", None, G2Element(), None, @@ -1039,13 +1025,12 @@ class FullNodeAPI: self.full_node.full_node_store.add_candidate_block( farmer_request.quality_string, height, unfinished_block, False ) - # TODO: address hint error and remove ignore - # error: Argument 3 to "RequestSignedValues" has incompatible type "Optional[bytes32]"; expected - # "bytes32" [arg-type] + # All unfinished blocks that we create will have the foliage transaction block and hash + assert unfinished_block.foliage.foliage_transaction_block_hash is not None message = farmer_protocol.RequestSignedValues( farmer_request.quality_string, unfinished_block.foliage.foliage_block_data.get_hash(), - unfinished_block.foliage.foliage_transaction_block_hash, # type: ignore[arg-type] + unfinished_block.foliage.foliage_transaction_block_hash, ) await peer.send_message(make_msg(ProtocolMessageTypes.request_signed_values, message)) return None @@ -1124,10 +1109,14 @@ class FullNodeAPI: @api_request async def request_additions(self, request: wallet_protocol.RequestAdditions) -> Optional[Message]: - # TODO: address hint error and remove ignore - # error: Argument 1 to "get_full_block" of "BlockStore" has incompatible type "Optional[bytes32]"; - # expected "bytes32" [arg-type] - block: Optional[FullBlock] = await self.full_node.block_store.get_full_block(request.header_hash) # type: ignore[arg-type] # noqa: E501 + if request.header_hash is None: + header_hash: Optional[bytes32] = self.full_node.blockchain.height_to_hash(request.height) + else: + header_hash = request.header_hash + if header_hash is None: + raise ValueError(f"Block at height {request.height} not found") + + block: Optional[FullBlock] = await self.full_node.block_store.get_full_block(header_hash) # We lock so that the coin store does not get modified if ( @@ -1135,10 +1124,7 @@ class FullNodeAPI: or block.is_transaction_block() is False or self.full_node.blockchain.height_to_hash(block.height) != request.header_hash ): - # TODO: address hint error and remove ignore - # error: Argument 2 to "RejectAdditionsRequest" has incompatible type "Optional[bytes32]"; expected - # "bytes32" [arg-type] - reject = wallet_protocol.RejectAdditionsRequest(request.height, request.header_hash) # type: ignore[arg-type] # noqa: E501 + reject = wallet_protocol.RejectAdditionsRequest(request.height, header_hash) msg = make_msg(ProtocolMessageTypes.reject_additions_request, reject) return msg @@ -1300,11 +1286,11 @@ class FullNodeAPI: if coin_record is None or coin_record.spent_block_index != height: return reject_msg - header_hash = self.full_node.blockchain.height_to_hash(height) - # TODO: address hint error and remove ignore - # error: Argument 1 to "get_full_block" of "BlockStore" has incompatible type "Optional[bytes32]"; - # expected "bytes32" [arg-type] - block: Optional[FullBlock] = await self.full_node.block_store.get_full_block(header_hash) # type: ignore[arg-type] # noqa: E501 + header_hash: Optional[bytes32] = self.full_node.blockchain.height_to_hash(height) + if header_hash is None: + return reject_msg + + block: Optional[FullBlock] = await self.full_node.block_store.get_full_block(header_hash) if block is None or block.transactions_generator is None: return reject_msg @@ -1331,18 +1317,17 @@ class FullNodeAPI: if request.end_height < request.start_height or request.end_height - request.start_height > 32: return None - header_hashes = [] + header_hashes: List[bytes32] = [] for i in range(request.start_height, request.end_height + 1): if not self.full_node.blockchain.contains_height(uint32(i)): reject = RejectHeaderBlocks(request.start_height, request.end_height) msg = make_msg(ProtocolMessageTypes.reject_header_blocks, reject) return msg - header_hashes.append(self.full_node.blockchain.height_to_hash(uint32(i))) + header_hash: Optional[bytes32] = self.full_node.blockchain.height_to_hash(uint32(i)) + assert header_hash is not None + header_hashes.append(header_hash) - # TODO: address hint error and remove ignore - # error: Argument 1 to "get_blocks_by_hash" of "BlockStore" has incompatible type - # "List[Optional[bytes32]]"; expected "List[bytes32]" [arg-type] - blocks: List[FullBlock] = await self.full_node.block_store.get_blocks_by_hash(header_hashes) # type: ignore[arg-type] # noqa: E501 + blocks: List[FullBlock] = await self.full_node.block_store.get_blocks_by_hash(header_hashes) header_blocks = [] for block in blocks: added_coins_records = await self.full_node.coin_store.get_coins_added_at_height(block.height) diff --git a/chia/rpc/full_node_rpc_api.py b/chia/rpc/full_node_rpc_api.py index 4b843fa0bb..04e84a9bf5 100644 --- a/chia/rpc/full_node_rpc_api.py +++ b/chia/rpc/full_node_rpc_api.py @@ -385,10 +385,9 @@ class FullNodeRpcApi: if peak_height < uint32(a): self.service.log.warning("requested block is higher than known peak ") break - # TODO: address hint error and remove ignore - # error: Incompatible types in assignment (expression has type "Optional[bytes32]", variable has type - # "bytes32") [assignment] - header_hash: bytes32 = self.service.blockchain.height_to_hash(uint32(a)) # type: ignore[assignment] + header_hash: Optional[bytes32] = self.service.blockchain.height_to_hash(uint32(a)) + if header_hash is None: + raise ValueError(f"Height not in blockchain: {a}") record: Optional[BlockRecord] = self.service.blockchain.try_block_record(header_hash) if record is None: # Fetch from DB diff --git a/chia/server/server.py b/chia/server/server.py index 0a8983951c..948985165c 100644 --- a/chia/server/server.py +++ b/chia/server/server.py @@ -632,16 +632,12 @@ class ChiaServer: if task_id in self.execute_tasks: self.execute_tasks.remove(task_id) - task_id = token_bytes() + task_id: bytes32 = bytes32(token_bytes(32)) api_task = asyncio.create_task(api_call(payload_inc, connection_inc, task_id)) - # TODO: address hint error and remove ignore - # error: Invalid index type "bytes" for "Dict[bytes32, Task[Any]]"; expected type "bytes32" [index] - self.api_tasks[task_id] = api_task # type: ignore[index] + self.api_tasks[task_id] = api_task if connection_inc.peer_node_id not in self.tasks_from_peer: self.tasks_from_peer[connection_inc.peer_node_id] = set() - # TODO: address hint error and remove ignore - # error: Argument 1 to "add" of "set" has incompatible type "bytes"; expected "bytes32" [arg-type] - self.tasks_from_peer[connection_inc.peer_node_id].add(task_id) # type: ignore[arg-type] + self.tasks_from_peer[connection_inc.peer_node_id].add(task_id) async def send_to_others( self, diff --git a/chia/simulator/simulator_constants.py b/chia/simulator/simulator_constants.py index 34c854a16e..ae09b6a348 100644 --- a/chia/simulator/simulator_constants.py +++ b/chia/simulator/simulator_constants.py @@ -6,9 +6,6 @@ if __name__ == "__main__": with TempKeyring() as keychain: # TODO: mariano: fix this with new consensus bt = create_block_tools(root_path=DEFAULT_ROOT_PATH, keychain=keychain) - # TODO: address hint error and remove ignore - # error: Argument 2 to "create_genesis_block" of "BlockTools" has incompatible type "bytes"; expected - # "bytes32" [arg-type] - new_genesis_block = bt.create_genesis_block(test_constants, b"0") # type: ignore[arg-type] + new_genesis_block = bt.create_genesis_block(test_constants, b"0") print(bytes(new_genesis_block)) diff --git a/chia/timelord/timelord_state.py b/chia/timelord/timelord_state.py index 72e6c7e68d..6431d336b8 100644 --- a/chia/timelord/timelord_state.py +++ b/chia/timelord/timelord_state.py @@ -104,11 +104,10 @@ class LastState: else: assert False - # TODO: address hint error and remove ignore - # error: Argument 1 to "append" of "list" has incompatible type "Tuple[Optional[bytes32], uint128]"; - # expected "Tuple[bytes32, uint128]" [arg-type] - self.reward_challenge_cache.append((self.get_challenge(Chain.REWARD_CHAIN), self.total_iters)) # type: ignore[arg-type] # noqa: E501 - log.info(f"Updated timelord peak to {self.get_challenge(Chain.REWARD_CHAIN)}, total iters: {self.total_iters}") + reward_challenge: Optional[bytes32] = self.get_challenge(Chain.REWARD_CHAIN) + assert reward_challenge is not None # Reward chain always has VDFs + self.reward_challenge_cache.append((reward_challenge, self.total_iters)) + log.info(f"Updated timelord peak to {reward_challenge}, total iters: {self.total_iters}") while len(self.reward_challenge_cache) > 2 * self.constants.MAX_SUB_SLOT_BLOCKS: self.reward_challenge_cache.pop(0) diff --git a/chia/types/blockchain_format/proof_of_space.py b/chia/types/blockchain_format/proof_of_space.py index 1b4617c8a3..72d90a7a9f 100644 --- a/chia/types/blockchain_format/proof_of_space.py +++ b/chia/types/blockchain_format/proof_of_space.py @@ -28,10 +28,8 @@ class ProofOfSpace(Streamable): def get_plot_id(self) -> bytes32: assert self.pool_public_key is None or self.pool_contract_puzzle_hash is None if self.pool_public_key is None: - # TODO: address hint error and remove ignore - # error: Argument 1 to "calculate_plot_id_ph" of "ProofOfSpace" has incompatible type - # "Optional[bytes32]"; expected "bytes32" [arg-type] - return self.calculate_plot_id_ph(self.pool_contract_puzzle_hash, self.plot_public_key) # type: ignore[arg-type] # noqa: E501 + assert self.pool_contract_puzzle_hash is not None + return self.calculate_plot_id_ph(self.pool_contract_puzzle_hash, self.plot_public_key) return self.calculate_plot_id_pk(self.pool_public_key, self.plot_public_key) def verify_and_get_quality_string( diff --git a/chia/util/block_cache.py b/chia/util/block_cache.py index 17d4c74a48..a07dc20810 100644 --- a/chia/util/block_cache.py +++ b/chia/util/block_cache.py @@ -35,11 +35,12 @@ class BlockCache(BlockchainInterface): return self._block_records[header_hash] def height_to_block_record(self, height: uint32, check_db: bool = False) -> BlockRecord: - header_hash = self.height_to_hash(height) - # TODO: address hint error and remove ignore - # error: Argument 1 to "block_record" of "BlockCache" has incompatible type "Optional[bytes32]"; expected - # "bytes32" [arg-type] - return self.block_record(header_hash) # type: ignore[arg-type] + # Precondition: height is < peak height + + header_hash: Optional[bytes32] = self.height_to_hash(height) + assert header_hash is not None + + return self.block_record(header_hash) def get_ses_heights(self) -> List[uint32]: return sorted(self._sub_epoch_summaries.keys()) diff --git a/chia/util/generator_tools.py b/chia/util/generator_tools.py index 9c2780a2ff..3034c9ec64 100644 --- a/chia/util/generator_tools.py +++ b/chia/util/generator_tools.py @@ -11,19 +11,13 @@ from chia.util.condition_tools import created_outputs_for_conditions_dict def get_block_header(block: FullBlock, tx_addition_coins: List[Coin], removals_names: List[bytes32]) -> HeaderBlock: # Create filter - byte_array_tx: List[bytes32] = [] + byte_array_tx: List[bytearray] = [] addition_coins = tx_addition_coins + list(block.get_included_reward_coins()) if block.is_transaction_block(): for coin in addition_coins: - # TODO: address hint error and remove ignore - # error: Argument 1 to "append" of "list" has incompatible type "bytearray"; expected "bytes32" - # [arg-type] - byte_array_tx.append(bytearray(coin.puzzle_hash)) # type: ignore[arg-type] + byte_array_tx.append(bytearray(coin.puzzle_hash)) for name in removals_names: - # TODO: address hint error and remove ignore - # error: Argument 1 to "append" of "list" has incompatible type "bytearray"; expected "bytes32" - # [arg-type] - byte_array_tx.append(bytearray(name)) # type: ignore[arg-type] + byte_array_tx.append(bytearray(name)) bip158: PyBIP158 = PyBIP158(byte_array_tx) encoded_filter: bytes = bytes(bip158.GetEncoded()) diff --git a/tests/block_tools.py b/tests/block_tools.py index f67a5d5c93..6e480fdd25 100644 --- a/tests/block_tools.py +++ b/tests/block_tools.py @@ -314,7 +314,7 @@ class BlockTools: self.created_plots += 1 plot_id_new: Optional[bytes32] = None - path_new: Path = Path() + path_new: Optional[Path] = None if len(created): assert len(existed) == 0 @@ -323,12 +323,11 @@ class BlockTools: if len(existed): assert len(created) == 0 plot_id_new, path_new = list(existed.items())[0] + assert plot_id_new is not None + assert path_new is not None if not exclude_final_dir: - # TODO: address hint error and remove ignore - # error: Invalid index type "Optional[bytes32]" for "Dict[bytes32, Path]"; expected type "bytes32" - # [index] - self.expected_plots[plot_id_new] = path_new # type: ignore[index] + self.expected_plots[plot_id_new] = path_new # create_plots() updates plot_directories. Ensure we refresh our config to reflect the updated value self._config["harvester"]["plot_directories"] = load_config(self.root_path, "config.yaml", "harvester")[ @@ -456,12 +455,9 @@ class BlockTools: if force_plot_id is not None: raise ValueError("Cannot specify plot_id for genesis block") initial_block_list_len = 0 - # TODO: address hint error and remove ignore - # error: Argument 2 to "create_genesis_block" of "BlockTools" has incompatible type "bytes"; expected - # "bytes32" [arg-type] genesis = self.create_genesis_block( constants, - seed, # type: ignore[arg-type] + seed, force_overflow=force_overflow, skip_slots=skip_slots, timestamp=(uint64(int(time.time())) if genesis_timestamp is None else genesis_timestamp), @@ -477,6 +473,7 @@ class BlockTools: if num_blocks == 0: return block_list + blocks: Dict[bytes32, BlockRecord] height_to_hash, difficulty, blocks = load_block_list(block_list, constants) latest_block: BlockRecord = blocks[block_list[-1].header_hash] @@ -709,14 +706,9 @@ class BlockTools: if pending_ses: sub_epoch_summary: Optional[SubEpochSummary] = None else: - # TODO: address hint error and remove ignore - # error: Argument 1 to "BlockCache" has incompatible type "Dict[uint32, BlockRecord]"; expected - # "Dict[bytes32, BlockRecord]" [arg-type] - # error: Argument 2 to "BlockCache" has incompatible type "Dict[uint32, bytes32]"; expected - # "Optional[Dict[bytes32, HeaderBlock]]" [arg-type] sub_epoch_summary = next_sub_epoch_summary( constants, - BlockCache(blocks, height_to_hash), # type: ignore[arg-type] + BlockCache(blocks, height_to_hash=height_to_hash), latest_block.required_iters, block_list[-1], False, @@ -940,13 +932,10 @@ class BlockTools: sub_slot_iters = new_sub_slot_iters difficulty = new_difficulty - # TODO: address hint error and remove ignore - # error: Incompatible default for argument "seed" (default has type "bytes", argument has type "bytes32") - # [assignment] def create_genesis_block( self, constants: ConsensusConstants, - seed: bytes32 = b"", # type: ignore[assignment] + seed: bytes = b"", timestamp: Optional[uint64] = None, force_overflow: bool = False, skip_slots: int = 0, @@ -1395,12 +1384,10 @@ def load_block_list( quality_str = full_block.reward_chain_block.proof_of_space.verify_and_get_quality_string( constants, challenge, sp_hash ) - # TODO: address hint error and remove ignore - # error: Argument 2 to "calculate_iterations_quality" has incompatible type "Optional[bytes32]"; expected - # "bytes32" [arg-type] + assert quality_str is not None required_iters: uint64 = calculate_iterations_quality( constants.DIFFICULTY_CONSTANT_FACTOR, - quality_str, # type: ignore[arg-type] + quality_str, full_block.reward_chain_block.proof_of_space.size, uint64(difficulty), sp_hash, @@ -1533,7 +1520,7 @@ def get_full_block_and_block_record( signage_point, timestamp, BlockCache(blocks), - seed, # type: ignore[arg-type] + seed, block_generator, aggregate_signature, additions, @@ -1589,9 +1576,6 @@ def compute_cost_test(generator: BlockGenerator, cost_per_byte: int) -> Tuple[Op return uint16(Err.GENERATOR_RUNTIME_ERROR.value), uint64(0) -# TODO: address hint error and remove ignore -# error: Incompatible default for argument "seed" (default has type "bytes", argument has type "bytes32") -# [assignment] def create_test_foliage( constants: ConsensusConstants, reward_block_unfinished: RewardChainBlockUnfinished, @@ -1607,7 +1591,7 @@ def create_test_foliage( pool_target: PoolTarget, get_plot_signature: Callable[[bytes32, G1Element], G2Element], get_pool_signature: Callable[[PoolTarget, Optional[G1Element]], Optional[G2Element]], - seed: bytes32 = b"", # type: ignore[assignment] + seed: bytes = b"", ) -> Tuple[Foliage, Optional[FoliageTransactionBlock], Optional[TransactionsInfo]]: """ Creates a foliage for a given reward chain block. This may or may not be a tx block. In the case of a tx block, @@ -1642,17 +1626,14 @@ def create_test_foliage( random.seed(seed) # Use the extension data to create different blocks based on header hash - # TODO: address hint error and remove ignore - # error: Incompatible types in assignment (expression has type "bytes", variable has type "bytes32") - # [assignment] - extension_data: bytes32 = random.randint(0, 100000000).to_bytes(32, "big") # type: ignore[assignment] + extension_data: bytes32 = bytes32(random.randint(0, 100000000).to_bytes(32, "big")) if prev_block is None: height: uint32 = uint32(0) else: height = uint32(prev_block.height + 1) # Create filter - byte_array_tx: List[bytes32] = [] + byte_array_tx: List[bytearray] = [] tx_additions: List[Coin] = [] tx_removals: List[bytes32] = [] @@ -1746,16 +1727,10 @@ def create_test_foliage( additions.extend(reward_claims_incorporated.copy()) for coin in additions: tx_additions.append(coin) - # TODO: address hint error and remove ignore - # error: Argument 1 to "append" of "list" has incompatible type "bytearray"; expected "bytes32" - # [arg-type] - byte_array_tx.append(bytearray(coin.puzzle_hash)) # type: ignore[arg-type] + byte_array_tx.append(bytearray(coin.puzzle_hash)) for coin in removals: tx_removals.append(coin.name()) - # TODO: address hint error and remove ignore - # error: Argument 1 to "append" of "list" has incompatible type "bytearray"; expected "bytes32" - # [arg-type] - byte_array_tx.append(bytearray(coin.name())) # type: ignore[arg-type] + byte_array_tx.append(bytearray(coin.name())) bip158: PyBIP158 = PyBIP158(byte_array_tx) encoded = bytes(bip158.GetEncoded()) @@ -1820,10 +1795,9 @@ def create_test_foliage( assert foliage_transaction_block is not None foliage_transaction_block_hash: Optional[bytes32] = foliage_transaction_block.get_hash() - # TODO: address hint error and remove ignore - # error: Argument 1 has incompatible type "Optional[bytes32]"; expected "bytes32" [arg-type] + assert foliage_transaction_block_hash is not None foliage_transaction_block_signature: Optional[G2Element] = get_plot_signature( - foliage_transaction_block_hash, # type: ignore[arg-type] + foliage_transaction_block_hash, reward_block_unfinished.proof_of_space.plot_public_key, ) assert foliage_transaction_block_signature is not None @@ -1846,9 +1820,6 @@ def create_test_foliage( return foliage, foliage_transaction_block, transactions_info -# TODO: address hint error and remove ignore -# error: Incompatible default for argument "seed" (default has type "bytes", argument has type "bytes32") -# [assignment] def create_test_unfinished_block( constants: ConsensusConstants, sub_slot_start_total_iters: uint128, @@ -1865,7 +1836,7 @@ def create_test_unfinished_block( signage_point: SignagePoint, timestamp: uint64, blocks: BlockchainInterface, - seed: bytes32 = b"", # type: ignore[assignment] + seed: bytes = b"", block_generator: Optional[BlockGenerator] = None, aggregate_sig: G2Element = G2Element(), additions: Optional[List[Coin]] = None, @@ -1914,7 +1885,7 @@ def create_test_unfinished_block( new_sub_slot: bool = len(finished_sub_slots) > 0 - cc_sp_hash: Optional[bytes32] = slot_cc_challenge + cc_sp_hash: bytes32 = slot_cc_challenge # Only enters this if statement if we are in testing mode (making VDF proofs here) if signage_point.cc_vdf is not None: @@ -1937,10 +1908,8 @@ def create_test_unfinished_block( rc_sp_hash = curr.finished_reward_slot_hashes[-1] signage_point = SignagePoint(None, None, None, None) - # TODO: address hint error and remove ignore - # error: Argument 1 has incompatible type "Optional[bytes32]"; expected "bytes32" [arg-type] cc_sp_signature: Optional[G2Element] = get_plot_signature( - cc_sp_hash, # type: ignore[arg-type] + cc_sp_hash, proof_of_space.plot_public_key, ) rc_sp_signature: Optional[G2Element] = get_plot_signature(rc_sp_hash, proof_of_space.plot_public_key) diff --git a/tests/core/full_node/stores/test_full_node_store.py b/tests/core/full_node/stores/test_full_node_store.py index ca6c14cb14..f3d6c441aa 100644 --- a/tests/core/full_node/stores/test_full_node_store.py +++ b/tests/core/full_node/stores/test_full_node_store.py @@ -422,10 +422,8 @@ class TestFullNodeStore: ) # Get signage point by hash - # TODO: address hint error and remove ignore - # error: Argument 1 to "get_signage_point" of "FullNodeStore" has incompatible type "Optional[bytes32]"; - # expected "bytes32" [arg-type] - assert store.get_signage_point(saved_sp_hash) is not None # type: ignore[arg-type] + assert saved_sp_hash is not None + assert store.get_signage_point(saved_sp_hash) is not None assert store.get_signage_point(std_hash(b"2")) is None # Test adding signage points before genesis diff --git a/tests/core/full_node/test_block_height_map.py b/tests/core/full_node/test_block_height_map.py index dfc34cf88c..835023c0fa 100644 --- a/tests/core/full_node/test_block_height_map.py +++ b/tests/core/full_node/test_block_height_map.py @@ -12,9 +12,7 @@ from chia.util.files import write_file_async def gen_block_hash(height: int) -> bytes32: - # TODO: address hint errors and remove ignores - # error: Incompatible return value type (got "bytes", expected "bytes32") [return-value] - return struct.pack(">I", height + 1) * (32 // 4) # type: ignore[return-value] + return bytes32(struct.pack(">I", height + 1) * (32 // 4)) def gen_ses(height: int) -> SubEpochSummary: diff --git a/tests/core/full_node/test_mempool.py b/tests/core/full_node/test_mempool.py index 0961a61532..ad5ea9cce7 100644 --- a/tests/core/full_node/test_mempool.py +++ b/tests/core/full_node/test_mempool.py @@ -111,15 +111,13 @@ async def two_nodes_mempool(bt, wallet_a): def make_item(idx: int, cost: uint64 = uint64(80)) -> MempoolItem: - spend_bundle_name = bytes([idx] * 32) - # TODO: address hint error and remove ignore - # error: Argument 5 to "MempoolItem" has incompatible type "bytes"; expected "bytes32" [arg-type] + spend_bundle_name = bytes32([idx] * 32) return MempoolItem( SpendBundle([], G2Element()), uint64(0), NPCResult(None, [], cost), cost, - spend_bundle_name, # type: ignore[arg-type] + spend_bundle_name, [], [], SerializedProgram(), diff --git a/tests/core/make_block_generator.py b/tests/core/make_block_generator.py index cea8464366..274b623843 100644 --- a/tests/core/make_block_generator.py +++ b/tests/core/make_block_generator.py @@ -5,6 +5,7 @@ import blspy from chia.full_node.bundle_tools import simple_solution_generator from chia.types.blockchain_format.coin import Coin from chia.types.blockchain_format.program import Program +from chia.types.blockchain_format.sized_bytes import bytes32 from chia.types.coin_spend import CoinSpend from chia.types.condition_opcodes import ConditionOpcode from chia.types.generator_types import BlockGenerator @@ -21,10 +22,10 @@ def int_to_public_key(index: int) -> blspy.G1Element: return private_key_from_int.get_g1() -def puzzle_hash_for_index(index: int, puzzle_hash_db: dict) -> bytes: - public_key = bytes(int_to_public_key(index)) - puzzle = puzzle_for_pk(public_key) - puzzle_hash = puzzle.get_tree_hash() +def puzzle_hash_for_index(index: int, puzzle_hash_db: dict) -> bytes32: + public_key: blspy.G1Element = int_to_public_key(index) + puzzle: Program = puzzle_for_pk(public_key) + puzzle_hash: bytes32 = puzzle.get_tree_hash() puzzle_hash_db[puzzle_hash] = puzzle return puzzle_hash @@ -34,13 +35,10 @@ def make_fake_coin(index: int, puzzle_hash_db: dict) -> Coin: Make a fake coin with parent id equal to the index (ie. a genesis block coin) """ - parent = index.to_bytes(32, "big") - puzzle_hash = puzzle_hash_for_index(index, puzzle_hash_db) - amount = 100000 - # TODO: address hint error and remove ignore - # error: Argument 1 to "Coin" has incompatible type "bytes"; expected "bytes32" [arg-type] - # error: Argument 2 to "Coin" has incompatible type "bytes"; expected "bytes32" [arg-type] - return Coin(parent, puzzle_hash, uint64(amount)) # type: ignore[arg-type] + parent: bytes32 = bytes32(index.to_bytes(32, "big")) + puzzle_hash: bytes32 = puzzle_hash_for_index(index, puzzle_hash_db) + amount: uint64 = uint64(100000) + return Coin(parent, puzzle_hash, amount) def conditions_for_payment(coin) -> Program: diff --git a/tests/pools/test_wallet_pool_store.py b/tests/pools/test_wallet_pool_store.py index 8d5c43c7ae..27c3152a2c 100644 --- a/tests/pools/test_wallet_pool_store.py +++ b/tests/pools/test_wallet_pool_store.py @@ -17,10 +17,7 @@ from chia.wallet.wallet_pool_store import WalletPoolStore def make_child_solution(coin_spend: CoinSpend, new_coin: Optional[Coin] = None) -> CoinSpend: - # TODO: address hint error and remove ignore - # error: Incompatible types in assignment (expression has type "bytes", variable has type "bytes32") - # [assignment] - new_puzzle_hash: bytes32 = token_bytes(32) # type: ignore[assignment] + new_puzzle_hash: bytes32 = bytes32(token_bytes(32)) solution = "()" puzzle = f"(q . ((51 0x{new_puzzle_hash.hex()} 1)))" puzzle_prog = Program.to(binutils.assemble(puzzle)) diff --git a/tests/util/key_tool.py b/tests/util/key_tool.py index 157223d8e1..8b74ff6ef6 100644 --- a/tests/util/key_tool.py +++ b/tests/util/key_tool.py @@ -2,7 +2,6 @@ from typing import List from blspy import AugSchemeMPL, G2Element, PrivateKey -from chia.types.blockchain_format.sized_bytes import bytes32 from chia.types.coin_spend import CoinSpend from chia.util.condition_tools import conditions_by_opcode, conditions_for_solution, pkm_pairs_for_conditions_dict from tests.core.make_block_generator import GROUP_ORDER, int_to_public_key @@ -18,12 +17,12 @@ class KeyTool(dict): for _ in secret_exponents: self[bytes(int_to_public_key(_))] = _ % GROUP_ORDER - def sign(self, public_key: bytes, message_hash: bytes32) -> G2Element: + def sign(self, public_key: bytes, message: bytes) -> G2Element: secret_exponent = self.get(public_key) if not secret_exponent: raise ValueError("unknown pubkey %s" % public_key.hex()) bls_private_key = PrivateKey.from_bytes(secret_exponent.to_bytes(32, "big")) - return AugSchemeMPL.sign(bls_private_key, message_hash) + return AugSchemeMPL.sign(bls_private_key, message) def signature_for_solution(self, coin_spend: CoinSpend, additional_data: bytes) -> AugSchemeMPL: signatures = [] @@ -32,12 +31,9 @@ class KeyTool(dict): ) assert conditions is not None conditions_dict = conditions_by_opcode(conditions) - for public_key, message_hash in pkm_pairs_for_conditions_dict( + for public_key, message in pkm_pairs_for_conditions_dict( conditions_dict, coin_spend.coin.name(), additional_data ): - # TODO: address hint error and remove ignore - # error: Argument 2 to "sign" of "KeyTool" has incompatible type "bytes"; expected "bytes32" - # [arg-type] - signature = self.sign(public_key, message_hash) # type: ignore[arg-type] + signature = self.sign(public_key, message) signatures.append(signature) return AugSchemeMPL.aggregate(signatures) diff --git a/tests/util/network_protocol_data.py b/tests/util/network_protocol_data.py index 8d8c818ec3..0de8ddf766 100644 --- a/tests/util/network_protocol_data.py +++ b/tests/util/network_protocol_data.py @@ -66,10 +66,8 @@ proof_of_space = ProofOfSpace( ), ) -# TODO: address hint error and remove ignore -# error: Argument 1 to "PoolTarget" has incompatible type "bytes"; expected "bytes32" [arg-type] pool_target = PoolTarget( - bytes.fromhex("d23da14695a188ae5708dd152263c4db883eb27edeb936178d4d988b8f3ce5fc"), # type: ignore[arg-type] + bytes32.from_hexstr("d23da14695a188ae5708dd152263c4db883eb27edeb936178d4d988b8f3ce5fc"), uint32(421941852), ) g2_element = G2Element( diff --git a/tests/wallet_tools.py b/tests/wallet_tools.py index e1df8bb879..6600dee0e3 100644 --- a/tests/wallet_tools.py +++ b/tests/wallet_tools.py @@ -1,13 +1,13 @@ from typing import Dict, List, Optional, Tuple, Any -from blspy import AugSchemeMPL, G2Element, PrivateKey +from blspy import AugSchemeMPL, G2Element, PrivateKey, G1Element from clvm.casts import int_from_bytes, int_to_bytes from chia.consensus.constants import ConsensusConstants from chia.util.hash import std_hash from chia.types.announcement import Announcement from chia.types.blockchain_format.coin import Coin -from chia.types.blockchain_format.program import Program +from chia.types.blockchain_format.program import Program, SerializedProgram from chia.types.blockchain_format.sized_bytes import bytes32 from chia.types.coin_spend import CoinSpend from chia.types.condition_opcodes import ConditionOpcode @@ -63,21 +63,19 @@ class WalletTool: def puzzle_for_pk(self, pubkey: bytes) -> Program: return puzzle_for_pk(pubkey) - def get_new_puzzle(self) -> bytes32: + def get_new_puzzle(self) -> Program: next_address_index: uint32 = self.get_next_address_index() - pubkey = master_sk_to_wallet_sk(self.private_key, next_address_index).get_g1() + pubkey: G1Element = master_sk_to_wallet_sk(self.private_key, next_address_index).get_g1() self.pubkey_num_lookup[bytes(pubkey)] = next_address_index - puzzle = puzzle_for_pk(bytes(pubkey)) + puzzle: Program = puzzle_for_pk(pubkey) self.puzzle_pk_cache[puzzle.get_tree_hash()] = next_address_index return puzzle def get_new_puzzlehash(self) -> bytes32: puzzle = self.get_new_puzzle() - # TODO: address hint error and remove ignore - # error: "bytes32" has no attribute "get_tree_hash" [attr-defined] - return puzzle.get_tree_hash() # type: ignore[attr-defined] + return puzzle.get_tree_hash() def sign(self, value: bytes, pubkey: bytes) -> G2Element: privatekey: PrivateKey = master_sk_to_wallet_sk(self.private_key, self.pubkey_num_lookup[pubkey]) @@ -141,15 +139,13 @@ class WalletTool: if secret_key is None: secret_key = self.get_private_key_for_puzzle_hash(puzzle_hash) pubkey = secret_key.get_g1() - puzzle = puzzle_for_pk(bytes(pubkey)) + puzzle: Program = puzzle_for_pk(pubkey) if n == 0: message_list = [c.name() for c in coins] for outputs in condition_dic[ConditionOpcode.CREATE_COIN]: - # TODO: address hint error and remove ignore - # error: Argument 2 to "Coin" has incompatible type "bytes"; expected "bytes32" [arg-type] coin_to_append = Coin( coin.name(), - outputs.vars[0], # type: ignore[arg-type] + bytes32(outputs.vars[0]), int_from_bytes(outputs.vars[1]), ) message_list.append(coin_to_append.name()) @@ -162,9 +158,19 @@ class WalletTool: ConditionWithArgs(ConditionOpcode.ASSERT_COIN_ANNOUNCEMENT, [primary_announcement_hash]) ) main_solution = self.make_solution(condition_dic) - spends.append(CoinSpend(coin, puzzle, main_solution)) + spends.append( + CoinSpend( + coin, SerializedProgram.from_program(puzzle), SerializedProgram.from_program(main_solution) + ) + ) else: - spends.append(CoinSpend(coin, puzzle, self.make_solution(secondary_coins_cond_dic))) + spends.append( + CoinSpend( + coin, + SerializedProgram.from_program(puzzle), + SerializedProgram.from_program(self.make_solution(secondary_coins_cond_dic)), + ) + ) return spends def sign_transaction(self, coin_spends: List[CoinSpend]) -> SpendBundle: diff --git a/tests/weight_proof/test_weight_proof.py b/tests/weight_proof/test_weight_proof.py index e7e7b86e6c..843016f9cb 100644 --- a/tests/weight_proof/test_weight_proof.py +++ b/tests/weight_proof/test_weight_proof.py @@ -107,12 +107,9 @@ async def load_blocks_dont_validate( cc_sp, ) - # TODO: address hint error and remove ignore - # error: Argument 2 to "BlockCache" has incompatible type "Dict[uint32, bytes32]"; expected - # "Optional[Dict[bytes32, HeaderBlock]]" [arg-type] sub_block = block_to_block_record( test_constants, - BlockCache(sub_blocks, height_to_hash), # type: ignore[arg-type] + BlockCache(sub_blocks, height_to_hash=height_to_hash), required_iters, block, None, From 62de4a883cdca69a63498b3c03e15872c2cb206b Mon Sep 17 00:00:00 2001 From: dustinface <35775977+xdustinface@users.noreply.github.com> Date: Mon, 4 Apr 2022 20:50:59 +0200 Subject: [PATCH 22/63] streamable|pools: Fix `Optional` parsing in `dataclass_from_dict` (#10573) * Test more `Optional` parsing in `dataclass_from_dict` * Fix optional parsing in `dataclass_from_dict` * Fix pool wallet / tests --- chia/pools/pool_wallet.py | 2 +- chia/util/streamable.py | 2 +- tests/core/util/test_streamable.py | 27 +++++++++++++++++++++++++++ tests/pools/test_pool_rpc.py | 4 ++-- 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/chia/pools/pool_wallet.py b/chia/pools/pool_wallet.py index a2df612349..341cb758d9 100644 --- a/chia/pools/pool_wallet.py +++ b/chia/pools/pool_wallet.py @@ -127,7 +127,7 @@ class PoolWallet: @classmethod def _verify_self_pooled(cls, state) -> Optional[str]: err = "" - if state.pool_url != "": + if state.pool_url not in [None, ""]: err += " Unneeded pool_url for self-pooling" if state.relative_lock_height != 0: diff --git a/chia/util/streamable.py b/chia/util/streamable.py index 73584bac7e..2602709600 100644 --- a/chia/util/streamable.py +++ b/chia/util/streamable.py @@ -55,7 +55,7 @@ def dataclass_from_dict(klass, d): """ if is_type_SpecificOptional(klass): # Type is optional, data is either None, or Any - if not d: + if d is None: return None return dataclass_from_dict(get_args(klass)[0], d) elif is_type_Tuple(klass): diff --git a/tests/core/util/test_streamable.py b/tests/core/util/test_streamable.py index 7b945d502c..5562d03d15 100644 --- a/tests/core/util/test_streamable.py +++ b/tests/core/util/test_streamable.py @@ -1,6 +1,7 @@ from dataclasses import dataclass from typing import List, Optional, Tuple import io +import pytest from clvm_tools import binutils from pytest import raises @@ -71,6 +72,32 @@ def test_json(bt): assert FullBlock.from_json_dict(dict_block) == block +@dataclass(frozen=True) +@streamable +class OptionalTestClass(Streamable): + a: Optional[str] + b: Optional[bool] + c: Optional[List[Optional[str]]] + + +@pytest.mark.parametrize( + "a, b, c", + [ + ("", True, ["1"]), + ("1", False, ["1"]), + ("1", True, []), + ("1", True, [""]), + ("1", True, ["1"]), + (None, None, None), + ], +) +def test_optional_json(a: Optional[str], b: Optional[bool], c: Optional[List[Optional[str]]]): + obj: OptionalTestClass = OptionalTestClass.from_json_dict({"a": a, "b": b, "c": c}) + assert obj.a == a + assert obj.b == b + assert obj.c == c + + def test_recursive_json(): @dataclass(frozen=True) @streamable diff --git a/tests/pools/test_pool_rpc.py b/tests/pools/test_pool_rpc.py index d49b0b3103..504431f5e9 100644 --- a/tests/pools/test_pool_rpc.py +++ b/tests/pools/test_pool_rpc.py @@ -211,7 +211,7 @@ class TestPoolWalletRpc: "b286bbf7a10fa058d2a2a758921377ef00bb7f8143e1bd40dd195ae918dbef42cfc481140f01b9eae13b430a0c8fe304" ) ) - assert status.current.pool_url is None + assert status.current.pool_url == "" assert status.current.relative_lock_height == 0 assert status.current.version == 1 # Check that config has been written properly @@ -893,7 +893,7 @@ class TestPoolWalletRpc: status: PoolWalletInfo = (await client.pw_status(wallet_id))[0] assert status.current.state == PoolSingletonState.SELF_POOLING.value - assert status.current.pool_url is None + assert status.current.pool_url == "" assert status.current.relative_lock_height == 0 assert status.current.state == 1 assert status.current.version == 1 From fe77c690182e97f7ef13d1fb383481f32efe2e87 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Mon, 4 Apr 2022 20:53:13 +0200 Subject: [PATCH 23/63] run_generator2 rust call and compact conditions data structure (#8862) * use run_generator2 rust call and compact spend bundle conditions data structure pervasively. * address review comments --- chia/consensus/block_body_validation.py | 59 ++-- chia/consensus/blockchain.py | 23 +- chia/consensus/cost_calculator.py | 6 +- chia/consensus/multiprocess_validation.py | 13 +- chia/full_node/full_node.py | 5 +- chia/full_node/mempool_check_conditions.py | 187 ++----------- chia/full_node/mempool_manager.py | 80 +++--- chia/types/blockchain_format/program.py | 32 ++- chia/types/name_puzzle_condition.py | 23 -- chia/types/spend_bundle_conditions.py | 30 ++ chia/util/condition_tools.py | 73 +---- chia/util/generator_tools.py | 28 +- tests/clvm/coin_store.py | 32 +-- .../core/full_node/stores/test_coin_store.py | 2 +- tests/core/full_node/test_mempool.py | 263 +++++++----------- tests/core/test_cost_calculation.py | 9 +- tests/generator/test_rom.py | 20 +- tests/util/generator_tools_testing.py | 9 +- tools/run_block.py | 9 +- 19 files changed, 345 insertions(+), 558 deletions(-) delete mode 100644 chia/types/name_puzzle_condition.py create mode 100644 chia/types/spend_bundle_conditions.py diff --git a/chia/consensus/block_body_validation.py b/chia/consensus/block_body_validation.py index 9bb704c730..7ba5510ff5 100644 --- a/chia/consensus/block_body_validation.py +++ b/chia/consensus/block_body_validation.py @@ -3,7 +3,6 @@ import logging from typing import Awaitable, Callable, Dict, List, Optional, Set, Tuple, Union from chiabip158 import PyBIP158 -from clvm.casts import int_from_bytes from chia.consensus.block_record import BlockRecord from chia.consensus.block_rewards import calculate_base_farmer_reward, calculate_pool_reward @@ -15,23 +14,20 @@ from chia.consensus.cost_calculator import NPCResult from chia.consensus.find_fork_point import find_fork_point_in_chain from chia.full_node.block_store import BlockStore from chia.full_node.coin_store import CoinStore -from chia.full_node.mempool_check_conditions import get_name_puzzle_conditions, mempool_check_conditions_dict +from chia.full_node.mempool_check_conditions import get_name_puzzle_conditions, mempool_check_time_locks from chia.types.block_protocol import BlockInfo from chia.types.blockchain_format.coin import Coin -from chia.types.blockchain_format.sized_bytes import bytes32 +from chia.types.blockchain_format.sized_bytes import bytes32, bytes48 from chia.types.coin_record import CoinRecord -from chia.types.condition_opcodes import ConditionOpcode -from chia.types.condition_with_args import ConditionWithArgs from chia.types.full_block import FullBlock from chia.types.generator_types import BlockGenerator -from chia.types.name_puzzle_condition import NPC from chia.types.unfinished_block import UnfinishedBlock from chia.util import cached_bls from chia.util.condition_tools import pkm_pairs from chia.util.errors import Err -from chia.util.generator_tools import additions_for_npc, tx_removals_and_additions +from chia.util.generator_tools import tx_removals_and_additions from chia.util.hash import std_hash -from chia.util.ints import uint32, uint64, uint128 +from chia.util.ints import uint32, uint64 log = logging.getLogger(__name__) @@ -153,7 +149,6 @@ async def validate_block_body( removals: List[bytes32] = [] coinbase_additions: List[Coin] = list(expected_reward_coins) additions: List[Coin] = [] - npc_list: List[NPC] = [] removals_puzzle_dic: Dict[bytes32, bytes32] = {} cost: uint64 = uint64(0) @@ -196,7 +191,6 @@ async def validate_block_body( assert npc_result is not None cost = npc_result.cost - npc_list = npc_result.npc_list # 7. Check that cost <= MAX_BLOCK_COST_CLVM log.debug( @@ -210,11 +204,13 @@ async def validate_block_body( if npc_result.error is not None: return Err(npc_result.error), None - for npc in npc_list: - removals.append(npc.coin_name) - removals_puzzle_dic[npc.coin_name] = npc.puzzle_hash + assert npc_result.conds is not None - additions = additions_for_npc(npc_list) + for spend in npc_result.conds.spends: + removals.append(spend.coin_id) + removals_puzzle_dic[spend.coin_id] = spend.puzzle_hash + for puzzle_hash, amount, _ in spend.create_coin: + additions.append(Coin(spend.coin_id, puzzle_hash, uint64(amount))) else: assert npc_result is None @@ -317,7 +313,7 @@ async def validate_block_body( mempool_mode=False, height=curr.height, ) - removals_in_curr, additions_in_curr = tx_removals_and_additions(curr_npc_result.npc_list) + removals_in_curr, additions_in_curr = tx_removals_and_additions(curr_npc_result.conds) else: removals_in_curr = [] additions_in_curr = [] @@ -401,16 +397,13 @@ async def validate_block_body( fees = removed - added assert fees >= 0 - assert_fee_sum: uint128 = uint128(0) - for npc in npc_list: - if ConditionOpcode.RESERVE_FEE in npc.condition_dict: - fee_list: List[ConditionWithArgs] = npc.condition_dict[ConditionOpcode.RESERVE_FEE] - for cvp in fee_list: - fee = int_from_bytes(cvp.vars[0]) - if fee < 0: - return Err.RESERVE_FEE_CONDITION_FAILED, None - assert_fee_sum = uint128(assert_fee_sum + fee) + # reserve fee cannot be greater than UINT64_MAX per consensus rule. + # run_generator() would fail + assert_fee_sum: uint64 = uint64(0) + if npc_result: + assert npc_result.conds is not None + assert_fee_sum = npc_result.conds.reserve_fee # 17. Check that the assert fee sum <= fees, and that each reserved fee is non-negative if fees < assert_fee_sum: @@ -430,12 +423,12 @@ async def validate_block_body( return Err.WRONG_PUZZLE_HASH, None # 21. Verify conditions - for npc in npc_list: - assert height is not None - unspent = removal_coin_records[npc.coin_name] - error = mempool_check_conditions_dict( - unspent, - npc.condition_dict, + # verify absolute/relative height/time conditions + if npc_result is not None: + assert npc_result.conds is not None + error = mempool_check_time_locks( + removal_coin_records, + npc_result.conds, prev_transaction_block_height, block.foliage_transaction_block.timestamp, ) @@ -443,7 +436,11 @@ async def validate_block_body( return error, None # create hash_key list for aggsig check - pairs_pks, pairs_msgs = pkm_pairs(npc_list, constants.AGG_SIG_ME_ADDITIONAL_DATA) + pairs_pks: List[bytes48] = [] + pairs_msgs: List[bytes] = [] + if npc_result: + assert npc_result.conds is not None + pairs_pks, pairs_msgs = pkm_pairs(npc_result.conds, constants.AGG_SIG_ME_ADDITIONAL_DATA) # 22. Verify aggregated signature # TODO: move this to pre_validate_blocks_multiprocessing so we can sync faster diff --git a/chia/consensus/blockchain.py b/chia/consensus/blockchain.py index 4f8dccfc50..f436572bc7 100644 --- a/chia/consensus/blockchain.py +++ b/chia/consensus/blockchain.py @@ -10,8 +10,6 @@ from multiprocessing.context import BaseContext from pathlib import Path from typing import Dict, List, Optional, Set, Tuple -from clvm.casts import int_from_bytes - from chia.consensus.block_body_validation import validate_block_body from chia.consensus.block_header_validation import validate_unfinished_header_block from chia.consensus.block_record import BlockRecord @@ -38,7 +36,6 @@ from chia.types.blockchain_format.sized_bytes import bytes32 from chia.types.blockchain_format.sub_epoch_summary import SubEpochSummary from chia.types.blockchain_format.vdf import VDFInfo from chia.types.coin_record import CoinRecord -from chia.types.condition_opcodes import ConditionOpcode from chia.types.end_of_slot_bundle import EndOfSubSlotBundle from chia.types.full_block import FullBlock from chia.types.generator_types import BlockGenerator @@ -301,16 +298,14 @@ class Blockchain(BlockchainInterface): return ReceiveBlockResult.ADDED_AS_ORPHAN, None, None, ([], {}) def get_hint_list(self, npc_result: NPCResult) -> List[Tuple[bytes32, bytes]]: + if npc_result.conds is None: + return [] h_list = [] - for npc in npc_result.npc_list: - for opcode, conditions in npc.conditions: - if opcode == ConditionOpcode.CREATE_COIN: - for condition in conditions: - if len(condition.vars) > 2 and condition.vars[2] != b"": - puzzle_hash, amount_bin = bytes32(condition.vars[0]), condition.vars[1] - amount: uint64 = uint64(int_from_bytes(amount_bin)) - coin_id: bytes32 = Coin(npc.coin_name, puzzle_hash, amount).name() - h_list.append((coin_id, condition.vars[2])) + for spend in npc_result.conds.spends: + for puzzle_hash, amount, hint in spend.create_coin: + if hint != b"": + coin_id = Coin(spend.coin_id, puzzle_hash, amount).name() + h_list.append((coin_id, hint)) return h_list async def _reconsider_peak( @@ -341,7 +336,7 @@ class Blockchain(BlockchainInterface): assert block is not None if npc_result is not None: - tx_removals, tx_additions = tx_removals_and_additions(npc_result.npc_list) + tx_removals, tx_additions = tx_removals_and_additions(npc_result.conds) else: tx_removals, tx_additions = [], [] if block.is_transaction_block(): @@ -473,7 +468,7 @@ class Blockchain(BlockchainInterface): mempool_mode=False, height=block.height, ) - tx_removals, tx_additions = tx_removals_and_additions(npc_result.npc_list) + tx_removals, tx_additions = tx_removals_and_additions(npc_result.conds) return tx_removals, tx_additions, npc_result def get_next_difficulty(self, header_hash: bytes32, new_slot: bool) -> uint64: diff --git a/chia/consensus/cost_calculator.py b/chia/consensus/cost_calculator.py index d090116d56..c207ccf561 100644 --- a/chia/consensus/cost_calculator.py +++ b/chia/consensus/cost_calculator.py @@ -1,7 +1,7 @@ from dataclasses import dataclass -from typing import List, Optional +from typing import Optional -from chia.types.name_puzzle_condition import NPC +from chia.types.spend_bundle_conditions import SpendBundleConditions from chia.util.ints import uint16, uint64 from chia.util.streamable import Streamable, streamable @@ -10,6 +10,6 @@ from chia.util.streamable import Streamable, streamable @streamable class NPCResult(Streamable): error: Optional[uint16] - npc_list: List[NPC] + conds: Optional[SpendBundleConditions] cost: uint64 # The total cost of the block, including CLVM cost, cost of # conditions and cost of bytes diff --git a/chia/consensus/multiprocess_validation.py b/chia/consensus/multiprocess_validation.py index 0e0ad3ff85..d908ab5808 100644 --- a/chia/consensus/multiprocess_validation.py +++ b/chia/consensus/multiprocess_validation.py @@ -75,8 +75,8 @@ def batch_pre_validate_blocks( if block.height in npc_results: npc_result = NPCResult.from_bytes(npc_results[block.height]) assert npc_result is not None - if npc_result.npc_list is not None: - removals, tx_additions = tx_removals_and_additions(npc_result.npc_list) + if npc_result.conds is not None: + removals, tx_additions = tx_removals_and_additions(npc_result.conds) else: removals, tx_additions = [], [] @@ -93,7 +93,7 @@ def batch_pre_validate_blocks( mempool_mode=False, height=block.height, ) - removals, tx_additions = tx_removals_and_additions(npc_result.npc_list) + removals, tx_additions = tx_removals_and_additions(npc_result.conds) if npc_result is not None and npc_result.error is not None: results.append(PreValidationResult(uint16(npc_result.error), None, npc_result, False)) continue @@ -120,7 +120,8 @@ def batch_pre_validate_blocks( # validate it later. receive_block will attempt to validate the signature later. if validate_signatures: if npc_result is not None and block.transactions_info is not None: - pairs_pks, pairs_msgs = pkm_pairs(npc_result.npc_list, constants.AGG_SIG_ME_ADDITIONAL_DATA) + assert npc_result.conds + pairs_pks, pairs_msgs = pkm_pairs(npc_result.conds, constants.AGG_SIG_ME_ADDITIONAL_DATA) pks_objects: List[G1Element] = [G1Element.from_bytes(pk) for pk in pairs_pks] if not AugSchemeMPL.aggregate_verify( pks_objects, pairs_msgs, block.transactions_info.aggregated_signature @@ -387,6 +388,6 @@ def _run_generator( ) return bytes(npc_result) except ValidationError as e: - return bytes(NPCResult(uint16(e.code.value), [], uint64(0))) + return bytes(NPCResult(uint16(e.code.value), None, uint64(0))) except Exception: - return bytes(NPCResult(uint16(Err.UNKNOWN.value), [], uint64(0))) + return bytes(NPCResult(uint16(Err.UNKNOWN.value), None, uint64(0))) diff --git a/chia/full_node/full_node.py b/chia/full_node/full_node.py index 5ac784804f..4c5ebbe541 100644 --- a/chia/full_node/full_node.py +++ b/chia/full_node/full_node.py @@ -1736,7 +1736,10 @@ class FullNode: npc_result = await self.blockchain.run_generator(block_bytes, block_generator, height) pre_validation_time = time.time() - pre_validation_start - pairs_pks, pairs_msgs = pkm_pairs(npc_result.npc_list, self.constants.AGG_SIG_ME_ADDITIONAL_DATA) + # blockchain.run_generator throws on errors, so npc_result is + # guaranteed to represent a successful run + assert npc_result.conds is not None + pairs_pks, pairs_msgs = pkm_pairs(npc_result.conds, self.constants.AGG_SIG_ME_ADDITIONAL_DATA) if not cached_bls.aggregate_verify( pairs_pks, pairs_msgs, block.transactions_info.aggregated_signature, True ): diff --git a/chia/full_node/mempool_check_conditions.py b/chia/full_node/mempool_check_conditions.py index d14500ffbb..10c44cd1e3 100644 --- a/chia/full_node/mempool_check_conditions.py +++ b/chia/full_node/mempool_check_conditions.py @@ -1,17 +1,15 @@ import logging -import time -from typing import Dict, List, Optional + +from typing import Dict, Optional from clvm_rs import MEMPOOL_MODE, COND_CANON_INTS, NO_NEG_DIV -from clvm.casts import int_from_bytes, int_to_bytes -from chia.consensus.cost_calculator import NPCResult from chia.consensus.default_constants import DEFAULT_CONSTANTS +from chia.consensus.cost_calculator import NPCResult +from chia.types.spend_bundle_conditions import SpendBundleConditions from chia.full_node.generator import create_generator_args, setup_generator_args from chia.types.coin_record import CoinRecord -from chia.types.condition_opcodes import ConditionOpcode -from chia.types.condition_with_args import ConditionWithArgs from chia.types.generator_types import BlockGenerator -from chia.types.name_puzzle_condition import NPC +from chia.types.blockchain_format.sized_bytes import bytes32 from chia.util.errors import Err from chia.util.ints import uint32, uint64, uint16 from chia.wallet.puzzles.generator_loader import GENERATOR_FOR_SINGLE_COIN_MOD @@ -22,91 +20,6 @@ GENERATOR_MOD = get_generator() log = logging.getLogger(__name__) -def mempool_assert_absolute_block_height_exceeds( - condition: ConditionWithArgs, prev_transaction_block_height: uint32 -) -> Optional[Err]: - """ - Checks if the next block index exceeds the block index from the condition - """ - try: - block_index_exceeds_this = int_from_bytes(condition.vars[0]) - except ValueError: - return Err.INVALID_CONDITION - if prev_transaction_block_height < block_index_exceeds_this: - return Err.ASSERT_HEIGHT_ABSOLUTE_FAILED - return None - - -def mempool_assert_relative_block_height_exceeds( - condition: ConditionWithArgs, unspent: CoinRecord, prev_transaction_block_height: uint32 -) -> Optional[Err]: - """ - Checks if the coin age exceeds the age from the condition - """ - try: - expected_block_age = int_from_bytes(condition.vars[0]) - block_index_exceeds_this = expected_block_age + unspent.confirmed_block_index - except ValueError: - return Err.INVALID_CONDITION - if prev_transaction_block_height < block_index_exceeds_this: - return Err.ASSERT_HEIGHT_RELATIVE_FAILED - return None - - -def mempool_assert_absolute_time_exceeds(condition: ConditionWithArgs, timestamp: uint64) -> Optional[Err]: - """ - Check if the current time in seconds exceeds the time specified by condition - """ - try: - expected_seconds = int_from_bytes(condition.vars[0]) - except ValueError: - return Err.INVALID_CONDITION - - if timestamp is None: - timestamp = uint64(int(time.time())) - if timestamp < expected_seconds: - return Err.ASSERT_SECONDS_ABSOLUTE_FAILED - return None - - -def mempool_assert_relative_time_exceeds( - condition: ConditionWithArgs, unspent: CoinRecord, timestamp: uint64 -) -> Optional[Err]: - """ - Check if the current time in seconds exceeds the time specified by condition - """ - try: - expected_seconds = int_from_bytes(condition.vars[0]) - except ValueError: - return Err.INVALID_CONDITION - - if timestamp is None: - timestamp = uint64(int(time.time())) - if timestamp < expected_seconds + unspent.timestamp: - return Err.ASSERT_SECONDS_RELATIVE_FAILED - return None - - -def add_int_cond( - conds: Dict[ConditionOpcode, List[ConditionWithArgs]], - op: ConditionOpcode, - arg: int, -): - if op not in conds: - conds[op] = [] - conds[op].append(ConditionWithArgs(op, [int_to_bytes(arg)])) - - -def add_cond( - conds: Dict[ConditionOpcode, List[ConditionWithArgs]], - op: ConditionOpcode, - args: List[bytes], -): - if op not in conds: - conds[op] = [] - conds[op].append(ConditionWithArgs(op, args)) - - def unwrap(x: Optional[uint32]) -> uint32: assert x is not None return x @@ -119,7 +32,7 @@ def get_name_puzzle_conditions( size_cost = len(bytes(generator.program)) * cost_per_byte max_cost -= size_cost if max_cost < 0: - return NPCResult(uint16(Err.INVALID_BLOCK_COST.value), [], uint64(0)) + return NPCResult(uint16(Err.INVALID_BLOCK_COST.value), None, uint64(0)) # in mempool mode, the height doesn't matter, because it's always strict. # But otherwise, height must be specified to know which rules to apply @@ -141,46 +54,14 @@ def get_name_puzzle_conditions( try: err, result = GENERATOR_MOD.run_as_generator(max_cost, flags, block_program, block_program_args) - + assert (err is None) != (result is None) if err is not None: - assert err != 0 - return NPCResult(uint16(err), [], uint64(0)) - - first = True - npc_list = [] - for r in result.spends: - conditions: Dict[ConditionOpcode, List[ConditionWithArgs]] = {} - if r.height_relative is not None: - add_int_cond(conditions, ConditionOpcode.ASSERT_HEIGHT_RELATIVE, r.height_relative) - if r.seconds_relative > 0: - add_int_cond(conditions, ConditionOpcode.ASSERT_SECONDS_RELATIVE, r.seconds_relative) - for cc in r.create_coin: - if cc[2] == b"": - add_cond(conditions, ConditionOpcode.CREATE_COIN, [cc[0], int_to_bytes(cc[1])]) - else: - add_cond(conditions, ConditionOpcode.CREATE_COIN, [cc[0], int_to_bytes(cc[1]), cc[2]]) - for sig in r.agg_sig_me: - add_cond(conditions, ConditionOpcode.AGG_SIG_ME, [sig[0], sig[1]]) - - # all conditions that aren't tied to a specific spent coin, we roll into the first one - if first: - first = False - if result.reserve_fee > 0: - add_int_cond(conditions, ConditionOpcode.RESERVE_FEE, result.reserve_fee) - if result.height_absolute > 0: - add_int_cond(conditions, ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE, result.height_absolute) - if result.seconds_absolute > 0: - add_int_cond(conditions, ConditionOpcode.ASSERT_SECONDS_ABSOLUTE, result.seconds_absolute) - for sig in result.agg_sig_unsafe: - add_cond(conditions, ConditionOpcode.AGG_SIG_UNSAFE, [sig[0], sig[1]]) - - npc_list.append(NPC(r.coin_id, r.puzzle_hash, [(op, cond) for op, cond in conditions.items()])) - - return NPCResult(None, npc_list, uint64(result.cost + size_cost)) - + return NPCResult(uint16(err), None, uint64(0)) + else: + return NPCResult(None, result, uint64(result.cost + size_cost)) except BaseException as e: log.debug(f"get_name_puzzle_condition failed: {e}") - return NPCResult(uint16(Err.GENERATOR_RUNTIME_ERROR.value), [], uint64(0)) + return NPCResult(uint16(Err.GENERATOR_RUNTIME_ERROR.value), None, uint64(0)) def get_puzzle_and_solution_for_coin(generator: BlockGenerator, coin_name: bytes, max_cost: int): @@ -198,40 +79,26 @@ def get_puzzle_and_solution_for_coin(generator: BlockGenerator, coin_name: bytes return e, None, None -def mempool_check_conditions_dict( - unspent: CoinRecord, - conditions_dict: Dict[ConditionOpcode, List[ConditionWithArgs]], +def mempool_check_time_locks( + removal_coin_records: Dict[bytes32, CoinRecord], + bundle_conds: SpendBundleConditions, prev_transaction_block_height: uint32, timestamp: uint64, ) -> Optional[Err]: """ - Check all conditions against current state. + Check all time and height conditions against current state. """ - for con_list in conditions_dict.values(): - cvp: ConditionWithArgs - for cvp in con_list: - error: Optional[Err] = None - if cvp.opcode is ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE: - error = mempool_assert_absolute_block_height_exceeds(cvp, prev_transaction_block_height) - elif cvp.opcode is ConditionOpcode.ASSERT_HEIGHT_RELATIVE: - error = mempool_assert_relative_block_height_exceeds(cvp, unspent, prev_transaction_block_height) - elif cvp.opcode is ConditionOpcode.ASSERT_SECONDS_ABSOLUTE: - error = mempool_assert_absolute_time_exceeds(cvp, timestamp) - elif cvp.opcode is ConditionOpcode.ASSERT_SECONDS_RELATIVE: - error = mempool_assert_relative_time_exceeds(cvp, unspent, timestamp) - elif cvp.opcode is ConditionOpcode.ASSERT_MY_COIN_ID: - assert False - elif cvp.opcode is ConditionOpcode.ASSERT_COIN_ANNOUNCEMENT: - assert False - elif cvp.opcode is ConditionOpcode.ASSERT_PUZZLE_ANNOUNCEMENT: - assert False - elif cvp.opcode is ConditionOpcode.ASSERT_MY_PARENT_ID: - assert False - elif cvp.opcode is ConditionOpcode.ASSERT_MY_PUZZLEHASH: - assert False - elif cvp.opcode is ConditionOpcode.ASSERT_MY_AMOUNT: - assert False - if error: - return error + if prev_transaction_block_height < bundle_conds.height_absolute: + return Err.ASSERT_HEIGHT_ABSOLUTE_FAILED + if timestamp < bundle_conds.seconds_absolute: + return Err.ASSERT_SECONDS_ABSOLUTE_FAILED + + for spend in bundle_conds.spends: + unspent = removal_coin_records[spend.coin_id] + if spend.height_relative is not None: + if prev_transaction_block_height < unspent.confirmed_block_index + spend.height_relative: + return Err.ASSERT_HEIGHT_RELATIVE_FAILED + if timestamp < unspent.timestamp + spend.seconds_relative: + return Err.ASSERT_SECONDS_RELATIVE_FAILED return None diff --git a/chia/full_node/mempool_manager.py b/chia/full_node/mempool_manager.py index 6c5f7aa49e..f1a4ba514b 100644 --- a/chia/full_node/mempool_manager.py +++ b/chia/full_node/mempool_manager.py @@ -10,7 +10,6 @@ from chia.util.inline_executor import InlineExecutor from typing import Dict, List, Optional, Set, Tuple from blspy import GTElement from chiabip158 import PyBIP158 -from clvm.casts import int_from_bytes from chia.util import cached_bls from chia.consensus.block_record import BlockRecord @@ -19,14 +18,12 @@ from chia.consensus.cost_calculator import NPCResult from chia.full_node.bundle_tools import simple_solution_generator from chia.full_node.coin_store import CoinStore from chia.full_node.mempool import Mempool -from chia.full_node.mempool_check_conditions import mempool_check_conditions_dict, get_name_puzzle_conditions +from chia.full_node.mempool_check_conditions import get_name_puzzle_conditions from chia.full_node.pending_tx_cache import PendingTxCache from chia.types.blockchain_format.coin import Coin from chia.types.blockchain_format.program import SerializedProgram from chia.types.blockchain_format.sized_bytes import bytes32, bytes48 from chia.types.coin_record import CoinRecord -from chia.types.condition_opcodes import ConditionOpcode -from chia.types.condition_with_args import ConditionWithArgs from chia.types.mempool_inclusion_status import MempoolInclusionStatus from chia.types.mempool_item import MempoolItem from chia.types.spend_bundle import SpendBundle @@ -38,6 +35,7 @@ from chia.util.ints import uint32, uint64 from chia.util.lru_cache import LRUCache from chia.util.setproctitle import getproctitle, setproctitle from chia.util.streamable import recurse_jsonify +from chia.full_node.mempool_check_conditions import mempool_check_time_locks log = logging.getLogger(__name__) @@ -61,9 +59,10 @@ def validate_clvm_and_signature( if result.error is not None: return Err(result.error), b"", {} - pks: List[bytes48] - msgs: List[bytes] - pks, msgs = pkm_pairs(result.npc_list, additional_data) + pks: List[bytes48] = [] + msgs: List[bytes] = [] + assert result.conds is not None + pks, msgs = pkm_pairs(result.conds, additional_data) # Verify aggregated signature cache: LRUCache = LRUCache(10000) @@ -301,8 +300,10 @@ class MempoolManager: if self.peak is None: return None, MempoolInclusionStatus.FAILED, Err.MEMPOOL_NOT_INITIALIZED - npc_list = npc_result.npc_list assert npc_result.error is None + if npc_result.error is not None: + return None, MempoolInclusionStatus.FAILED, Err(npc_result.error) + if program is None: program = simple_solution_generator(new_spend).program cost = npc_result.cost @@ -314,12 +315,13 @@ class MempoolManager: # execute the CLVM program. return None, MempoolInclusionStatus.FAILED, Err.BLOCK_COST_EXCEEDS_MAX + assert npc_result.conds is not None # build removal list - removal_names: List[bytes32] = [npc.coin_name for npc in npc_list] + removal_names: List[bytes32] = [spend.coin_id for spend in npc_result.conds.spends] if set(removal_names) != set([s.name() for s in new_spend.removals()]): return None, MempoolInclusionStatus.FAILED, Err.INVALID_SPEND_BUNDLE - additions = additions_for_npc(npc_list) + additions = additions_for_npc(npc_result) additions_dict: Dict[bytes32, Coin] = {} for add in additions: @@ -372,7 +374,7 @@ class MempoolManager: assert self.peak.timestamp is not None removal_record = CoinRecord( removal_coin, - uint32(self.peak.height + 1), # In mempool, so will be included in next height + uint32(self.peak.height + 1), uint32(0), False, self.peak.timestamp, @@ -388,16 +390,8 @@ class MempoolManager: return None, MempoolInclusionStatus.FAILED, Err.MINTING_COIN fees = uint64(removal_amount - addition_amount) - assert_fee_sum: uint64 = uint64(0) + assert_fee_sum: uint64 = uint64(npc_result.conds.reserve_fee) - for npc in npc_list: - if ConditionOpcode.RESERVE_FEE in npc.condition_dict: - fee_list: List[ConditionWithArgs] = npc.condition_dict[ConditionOpcode.RESERVE_FEE] - for cvp in fee_list: - fee = int_from_bytes(cvp.vars[0]) - if fee < 0: - return None, MempoolInclusionStatus.FAILED, Err.RESERVE_FEE_CONDITION_FAILED - assert_fee_sum = assert_fee_sum + fee if fees < assert_fee_sum: return ( None, @@ -439,37 +433,35 @@ class MempoolManager: return None, MempoolInclusionStatus.FAILED, fail_reason # Verify conditions, create hash_key list for aggsig check - error: Optional[Err] = None - for npc in npc_list: - coin_record: CoinRecord = removal_record_dict[npc.coin_name] + for spend in npc_result.conds.spends: + coin_record: CoinRecord = removal_record_dict[spend.coin_id] # Check that the revealed removal puzzles actually match the puzzle hash - if npc.puzzle_hash != coin_record.coin.puzzle_hash: + if spend.puzzle_hash != coin_record.coin.puzzle_hash: log.warning("Mempool rejecting transaction because of wrong puzzle_hash") - log.warning(f"{npc.puzzle_hash} != {coin_record.coin.puzzle_hash}") + log.warning(f"{spend.puzzle_hash} != {coin_record.coin.puzzle_hash}") return None, MempoolInclusionStatus.FAILED, Err.WRONG_PUZZLE_HASH - chialisp_height = ( - self.peak.prev_transaction_block_height if not self.peak.is_transaction_block else self.peak.height - ) - assert self.peak.timestamp is not None - error = mempool_check_conditions_dict( - coin_record, - npc.condition_dict, - uint32(chialisp_height), - self.peak.timestamp, - ) + chialisp_height = ( + self.peak.prev_transaction_block_height if not self.peak.is_transaction_block else self.peak.height + ) - if error: - if error is Err.ASSERT_HEIGHT_ABSOLUTE_FAILED or error is Err.ASSERT_HEIGHT_RELATIVE_FAILED: - potential = MempoolItem( - new_spend, uint64(fees), npc_result, cost, spend_name, additions, removals, program - ) - self.potential_cache.add(potential) - return uint64(cost), MempoolInclusionStatus.PENDING, error - break + assert self.peak.timestamp is not None + error: Optional[Err] = mempool_check_time_locks( + removal_record_dict, + npc_result.conds, + uint32(chialisp_height), + self.peak.timestamp, + ) if error: - return None, MempoolInclusionStatus.FAILED, error + if error is Err.ASSERT_HEIGHT_ABSOLUTE_FAILED or error is Err.ASSERT_HEIGHT_RELATIVE_FAILED: + potential = MempoolItem( + new_spend, uint64(fees), npc_result, cost, spend_name, additions, removals, program + ) + self.potential_cache.add(potential) + return uint64(cost), MempoolInclusionStatus.PENDING, error + else: + return None, MempoolInclusionStatus.FAILED, error # Remove all conflicting Coins and SpendBundles if fail_reason: diff --git a/chia/types/blockchain_format/program.py b/chia/types/blockchain_format/program.py index 3b29e6cd78..36278ef7ec 100644 --- a/chia/types/blockchain_format/program.py +++ b/chia/types/blockchain_format/program.py @@ -1,5 +1,5 @@ import io -from typing import List, Set, Tuple, Optional, Any +from typing import List, Set, Tuple, Optional from clvm import SExp from clvm.casts import int_from_bytes @@ -10,8 +10,8 @@ from clvm_tools.curry import curry, uncurry from chia.types.blockchain_format.sized_bytes import bytes32 from chia.util.hash import std_hash -from chia.util.ints import uint16 from chia.util.byte_types import hexstr_to_bytes +from chia.types.spend_bundle_conditions import SpendBundleConditions, Spend from .tree_hash import sha256_treehash @@ -222,9 +222,12 @@ class SerializedProgram: def run_with_cost(self, max_cost: int, *args) -> Tuple[int, Program]: return self._run(max_cost, 0, *args) - # returns an optional error code and an optional PySpendBundleConditions (from clvm_rs) + # returns an optional error code and an optional SpendBundleConditions # exactly one of those will hold a value - def run_as_generator(self, max_cost: int, flags: int, *args) -> Tuple[Optional[uint16], Optional[Any]]: + def run_as_generator( + self, max_cost: int, flags: int, *args + ) -> Tuple[Optional[int], Optional[SpendBundleConditions]]: + serialized_args = b"" if len(args) > 1: # when we have more than one argument, serialize them into a list @@ -235,12 +238,31 @@ class SerializedProgram: else: serialized_args += _serialize(args[0]) - return run_generator2( + err, conds = run_generator2( self._buf, serialized_args, max_cost, flags, ) + if err is not None: + assert err != 0 + return err, None + + # for now, we need to copy this data into python objects, in order to + # support streamable. This will become simpler and faster once we can + # implement streamable in rust + spends = [] + for s in conds.spends: + spends.append( + Spend(s.coin_id, s.puzzle_hash, s.height_relative, s.seconds_relative, s.create_coin, s.agg_sig_me) + ) + + ret = SpendBundleConditions( + spends, conds.reserve_fee, conds.height_absolute, conds.seconds_absolute, conds.agg_sig_unsafe, conds.cost + ) + + assert ret is not None + return None, ret def _run(self, max_cost: int, flags, *args) -> Tuple[int, Program]: # when multiple arguments are passed, concatenate them into a serialized diff --git a/chia/types/name_puzzle_condition.py b/chia/types/name_puzzle_condition.py deleted file mode 100644 index e3ff30f903..0000000000 --- a/chia/types/name_puzzle_condition.py +++ /dev/null @@ -1,23 +0,0 @@ -from dataclasses import dataclass -from typing import Dict, List, Tuple - -from chia.types.blockchain_format.sized_bytes import bytes32 -from chia.types.condition_with_args import ConditionWithArgs -from chia.types.condition_opcodes import ConditionOpcode -from chia.util.streamable import Streamable, streamable - - -@dataclass(frozen=True) -@streamable -class NPC(Streamable): - coin_name: bytes32 - puzzle_hash: bytes32 - conditions: List[Tuple[ConditionOpcode, List[ConditionWithArgs]]] - - @property - def condition_dict(self): - d: Dict[ConditionOpcode, List[ConditionWithArgs]] = {} - for opcode, l in self.conditions: - assert opcode not in d - d[opcode] = l - return d diff --git a/chia/types/spend_bundle_conditions.py b/chia/types/spend_bundle_conditions.py new file mode 100644 index 0000000000..0c59fb732e --- /dev/null +++ b/chia/types/spend_bundle_conditions.py @@ -0,0 +1,30 @@ +from dataclasses import dataclass +from typing import List, Optional, Tuple + +from chia.types.blockchain_format.sized_bytes import bytes32, bytes48 +from chia.util.ints import uint32, uint64 +from chia.util.streamable import Streamable, streamable + + +# the Spend and SpendBundleConditions classes are mirrors of native types, returned by +# run_generator2 +@dataclass(frozen=True) +@streamable +class Spend(Streamable): + coin_id: bytes32 + puzzle_hash: bytes32 + height_relative: Optional[uint32] + seconds_relative: uint64 + create_coin: List[Tuple[bytes32, uint64, bytes]] + agg_sig_me: List[Tuple[bytes48, bytes]] + + +@dataclass(frozen=True) +@streamable +class SpendBundleConditions(Streamable): + spends: List[Spend] + reserve_fee: uint64 + height_absolute: uint32 + seconds_absolute: uint64 + agg_sig_unsafe: List[Tuple[bytes48, bytes]] + cost: uint64 diff --git a/chia/util/condition_tools.py b/chia/util/condition_tools.py index 74a1e5fda7..8d756ccd9f 100644 --- a/chia/util/condition_tools.py +++ b/chia/util/condition_tools.py @@ -1,9 +1,7 @@ -from typing import Dict, List, Optional, Tuple, Set +from typing import Dict, List, Optional, Tuple from clvm.casts import int_from_bytes -from chia.types.announcement import Announcement -from chia.types.name_puzzle_condition import NPC from chia.types.blockchain_format.coin import Coin from chia.types.blockchain_format.program import Program, SerializedProgram from chia.types.blockchain_format.sized_bytes import bytes32, bytes48 @@ -11,6 +9,7 @@ from chia.types.condition_opcodes import ConditionOpcode from chia.types.condition_with_args import ConditionWithArgs from chia.util.errors import ConsensusError, Err from chia.util.ints import uint64 +from chia.types.spend_bundle_conditions import SpendBundleConditions # TODO: review each `assert` and consider replacing with explicit checks # since asserts can be stripped with python `-OO` flag @@ -65,25 +64,17 @@ def conditions_by_opcode( return d -def pkm_pairs(npc_list: List[NPC], additional_data: bytes) -> Tuple[List[bytes48], List[bytes]]: +def pkm_pairs(conditions: SpendBundleConditions, additional_data: bytes) -> Tuple[List[bytes48], List[bytes]]: ret: Tuple[List[bytes48], List[bytes]] = ([], []) - for npc in npc_list: - for opcode, l in npc.conditions: - if opcode == ConditionOpcode.AGG_SIG_UNSAFE: - for cwa in l: - assert len(cwa.vars) == 2 - assert len(cwa.vars[0]) == 48 and len(cwa.vars[1]) <= 1024 - assert cwa.vars[0] is not None and cwa.vars[1] is not None - ret[0].append(bytes48(cwa.vars[0])) - ret[1].append(cwa.vars[1]) - elif opcode == ConditionOpcode.AGG_SIG_ME: - for cwa in l: - assert len(cwa.vars) == 2 - assert len(cwa.vars[0]) == 48 and len(cwa.vars[1]) <= 1024 - assert cwa.vars[0] is not None and cwa.vars[1] is not None - ret[0].append(bytes48(cwa.vars[0])) - ret[1].append(cwa.vars[1] + npc.coin_name + additional_data) + for pk, msg in conditions.agg_sig_unsafe: + ret[0].append(bytes48(pk)) + ret[1].append(msg) + + for spend in conditions.spends: + for pk, msg in spend.agg_sig_me: + ret[0].append(bytes48(pk)) + ret[1].append(msg + spend.coin_id + additional_data) return ret @@ -120,48 +111,6 @@ def created_outputs_for_conditions_dict( return output_coins -def coin_announcements_for_conditions_dict( - conditions_dict: Dict[ConditionOpcode, List[ConditionWithArgs]], - input_coin: Coin, -) -> Set[Announcement]: - output_announcements: Set[Announcement] = set() - for cvp in conditions_dict.get(ConditionOpcode.CREATE_COIN_ANNOUNCEMENT, []): - message = cvp.vars[0] - assert len(message) <= 1024 - announcement = Announcement(input_coin.name(), message) - output_announcements.add(announcement) - return output_announcements - - -def puzzle_announcements_for_conditions_dict( - conditions_dict: Dict[ConditionOpcode, List[ConditionWithArgs]], - input_coin: Coin, -) -> Set[Announcement]: - output_announcements: Set[Announcement] = set() - for cvp in conditions_dict.get(ConditionOpcode.CREATE_PUZZLE_ANNOUNCEMENT, []): - message = cvp.vars[0] - assert len(message) <= 1024 - announcement = Announcement(input_coin.puzzle_hash, message) - output_announcements.add(announcement) - return output_announcements - - -def coin_announcement_names_for_conditions_dict( - conditions_dict: Dict[ConditionOpcode, List[ConditionWithArgs]], - input_coin: Coin, -) -> List[bytes32]: - output = [an.name() for an in coin_announcements_for_conditions_dict(conditions_dict, input_coin)] - return output - - -def puzzle_announcement_names_for_conditions_dict( - conditions_dict: Dict[ConditionOpcode, List[ConditionWithArgs]], - input_coin: Coin, -) -> List[bytes32]: - output = [an.name() for an in puzzle_announcements_for_conditions_dict(conditions_dict, input_coin)] - return output - - def conditions_dict_for_solution( puzzle_reveal: SerializedProgram, solution: SerializedProgram, diff --git a/chia/util/generator_tools.py b/chia/util/generator_tools.py index 3034c9ec64..526051022c 100644 --- a/chia/util/generator_tools.py +++ b/chia/util/generator_tools.py @@ -1,12 +1,13 @@ -from typing import Any, Iterator, List, Tuple +from typing import Any, Iterator, List, Tuple, Optional from chiabip158 import PyBIP158 from chia.types.blockchain_format.coin import Coin from chia.types.blockchain_format.sized_bytes import bytes32 from chia.types.full_block import FullBlock from chia.types.header_block import HeaderBlock -from chia.types.name_puzzle_condition import NPC -from chia.util.condition_tools import created_outputs_for_conditions_dict +from chia.types.spend_bundle_conditions import SpendBundleConditions +from chia.consensus.cost_calculator import NPCResult +from chia.util.ints import uint64 def get_block_header(block: FullBlock, tx_addition_coins: List[Coin], removals_names: List[bytes32]) -> HeaderBlock: @@ -37,17 +38,20 @@ def get_block_header(block: FullBlock, tx_addition_coins: List[Coin], removals_n ) -def additions_for_npc(npc_list: List[NPC]) -> List[Coin]: +def additions_for_npc(npc_result: NPCResult) -> List[Coin]: additions: List[Coin] = [] - for npc in npc_list: - for coin in created_outputs_for_conditions_dict(npc.condition_dict, npc.coin_name): + if npc_result.conds is None: + return [] + for spend in npc_result.conds.spends: + for puzzle_hash, amount, _ in spend.create_coin: + coin = Coin(spend.coin_id, puzzle_hash, uint64(amount)) additions.append(coin) return additions -def tx_removals_and_additions(npc_list: List[NPC]) -> Tuple[List[bytes32], List[Coin]]: +def tx_removals_and_additions(results: Optional[SpendBundleConditions]) -> Tuple[List[bytes32], List[Coin]]: """ Doesn't return farmer and pool reward. """ @@ -56,12 +60,12 @@ def tx_removals_and_additions(npc_list: List[NPC]) -> Tuple[List[bytes32], List[ additions: List[Coin] = [] # build removals list - if npc_list is None: + if results is None: return [], [] - for npc in npc_list: - removals.append(npc.coin_name) - - additions.extend(additions_for_npc(npc_list)) + for spend in results.spends: + removals.append(spend.coin_id) + for puzzle_hash, amount, _ in spend.create_coin: + additions.append(Coin(spend.coin_id, puzzle_hash, uint64(amount))) return removals, additions diff --git a/tests/clvm/coin_store.py b/tests/clvm/coin_store.py index afe8ccc9a9..7e60981228 100644 --- a/tests/clvm/coin_store.py +++ b/tests/clvm/coin_store.py @@ -2,8 +2,7 @@ from collections import defaultdict from dataclasses import dataclass, replace from typing import Dict, Iterator, Optional -from chia.util.condition_tools import created_outputs_for_conditions_dict -from chia.full_node.mempool_check_conditions import mempool_check_conditions_dict, get_name_puzzle_conditions +from chia.full_node.mempool_check_conditions import mempool_check_time_locks, get_name_puzzle_conditions from chia.types.blockchain_format.coin import Coin from chia.types.blockchain_format.sized_bytes import bytes32 from chia.types.coin_record import CoinRecord @@ -71,8 +70,10 @@ class CoinStore: raise BadSpendBundleError(f"condition validation failure {Err(result.error)}") ephemeral_db = dict(self._db) - for npc in result.npc_list: - for coin in created_outputs_for_conditions_dict(npc.condition_dict, npc.coin_name): + assert result.conds is not None + for spend in result.conds.spends: + for puzzle_hash, amount, hint in spend.create_coin: + coin = Coin(spend.coin_id, puzzle_hash, amount) name = coin.name() ephemeral_db[name] = CoinRecord( coin, @@ -82,20 +83,15 @@ class CoinStore: uint64(now.seconds), ) - for npc in result.npc_list: - prev_transaction_block_height = uint32(now.height) - timestamp = uint64(now.seconds) - coin_record = ephemeral_db.get(npc.coin_name) - if coin_record is None: - raise BadSpendBundleError(f"coin not found for id 0x{npc.coin_name.hex()}") # noqa - err = mempool_check_conditions_dict( - coin_record, - npc.condition_dict, - prev_transaction_block_height, - timestamp, - ) - if err is not None: - raise BadSpendBundleError(f"condition validation failure {Err(err)}") + err = mempool_check_time_locks( + ephemeral_db, + result.conds, + uint32(now.height), + uint64(now.seconds), + ) + + if err is not None: + raise BadSpendBundleError(f"condition validation failure {Err(err)}") return 0 diff --git a/tests/core/full_node/stores/test_coin_store.py b/tests/core/full_node/stores/test_coin_store.py index ca3564ebbc..169ce66210 100644 --- a/tests/core/full_node/stores/test_coin_store.py +++ b/tests/core/full_node/stores/test_coin_store.py @@ -103,7 +103,7 @@ class TestCoinStoreWithBlocks: mempool_mode=False, height=softfork_height, ) - tx_removals, tx_additions = tx_removals_and_additions(npc_result.npc_list) + tx_removals, tx_additions = tx_removals_and_additions(npc_result.conds) else: tx_removals, tx_additions = [], [] diff --git a/tests/core/full_node/test_mempool.py b/tests/core/full_node/test_mempool.py index ad5ea9cce7..0fc7f504bd 100644 --- a/tests/core/full_node/test_mempool.py +++ b/tests/core/full_node/test_mempool.py @@ -18,7 +18,7 @@ from chia.server.outbound_message import Message from chia.simulator.simulator_protocol import FarmNewBlockProtocol from chia.types.announcement import Announcement from chia.types.blockchain_format.coin import Coin -from chia.types.blockchain_format.sized_bytes import bytes32 +from chia.types.blockchain_format.sized_bytes import bytes32, bytes48 from chia.types.coin_spend import CoinSpend from chia.types.condition_opcodes import ConditionOpcode from chia.types.condition_with_args import ConditionWithArgs @@ -31,7 +31,6 @@ from chia.util.hash import std_hash from chia.types.mempool_inclusion_status import MempoolInclusionStatus from chia.util.api_decorators import api_request, peer_required, bytes_required from chia.full_node.mempool_check_conditions import get_name_puzzle_conditions -from chia.types.name_puzzle_condition import NPC from chia.full_node.pending_tx_cache import PendingTxCache from blspy import G2Element @@ -47,8 +46,8 @@ from chia.consensus.condition_costs import ConditionCost from chia.types.blockchain_format.program import SerializedProgram from clvm_tools import binutils from chia.types.generator_types import BlockGenerator -from clvm.casts import int_from_bytes from blspy import G1Element +from chia.types.spend_bundle_conditions import SpendBundleConditions, Spend from tests.wallet_tools import WalletTool @@ -115,7 +114,7 @@ def make_item(idx: int, cost: uint64 = uint64(80)) -> MempoolItem: return MempoolItem( SpendBundle([], G2Element()), uint64(0), - NPCResult(None, [], cost), + NPCResult(None, None, cost), cost, spend_bundle_name, [], @@ -1829,13 +1828,8 @@ class TestGeneratorConditions: # at are ignored, including the termination of the list npc_result = generator_condition_tester("(80 50 . 1)", height=softfork_height) assert npc_result.error is None - assert len(npc_result.npc_list) == 1 - opcode = ConditionOpcode(bytes([80])) - assert len(npc_result.npc_list[0].conditions) == 1 - assert npc_result.npc_list[0].conditions[0][0] == opcode - assert len(npc_result.npc_list[0].conditions[0][1]) == 1 - c = npc_result.npc_list[0].conditions[0][1][0] - assert c == ConditionWithArgs(opcode=ConditionOpcode.ASSERT_SECONDS_RELATIVE, vars=[bytes([50])]) + assert len(npc_result.conds.spends) == 1 + assert npc_result.conds.spends[0].seconds_relative == 50 @pytest.mark.parametrize( "mempool,height,operand,expected", @@ -1881,13 +1875,17 @@ class TestGeneratorConditions: ) print(npc_result) assert npc_result.error is None - assert len(npc_result.npc_list) == 1 - max_arg = 0 - assert npc_result.npc_list[0].conditions[0][0] == opcode - for c in npc_result.npc_list[0].conditions[0][1]: - assert c.opcode == opcode - max_arg = max(max_arg, int_from_bytes(c.vars[0])) - assert max_arg == 100 + assert len(npc_result.conds.spends) == 1 + + assert len(npc_result.conds.spends) == 1 + if opcode == ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE: + assert npc_result.conds.height_absolute == 100 + elif opcode == ConditionOpcode.ASSERT_HEIGHT_RELATIVE: + assert npc_result.conds.spends[0].height_relative == 100 + elif opcode == ConditionOpcode.ASSERT_SECONDS_ABSOLUTE: + assert npc_result.conds.seconds_absolute == 100 + elif opcode == ConditionOpcode.ASSERT_SECONDS_RELATIVE: + assert npc_result.conds.spends[0].seconds_relative == 100 @pytest.mark.parametrize( "opcode", @@ -1902,10 +1900,9 @@ class TestGeneratorConditions: # back. They are either satisified or cause an immediate failure npc_result = generator_condition_tester(f'({opcode.value[0]} "{message}") ' * 50, height=softfork_height) assert npc_result.error is None - assert len(npc_result.npc_list) == 1 + assert len(npc_result.conds.spends) == 1 # create-announcements and assert-announcements are dropped once # validated - assert npc_result.npc_list[0].conditions == [] @pytest.mark.parametrize( "opcode", @@ -1923,7 +1920,6 @@ class TestGeneratorConditions: npc_result = generator_condition_tester(f'({opcode.value[0]} "{message}") ', height=softfork_height) print(npc_result) assert npc_result.error == Err.ASSERT_ANNOUNCE_CONSUMED_FAILED.value - assert npc_result.npc_list == [] def test_multiple_reserve_fee(self, softfork_height): # RESERVE_FEE @@ -1932,17 +1928,8 @@ class TestGeneratorConditions: # with all the fees accumulated npc_result = generator_condition_tester(f"({cond} 100) " * 3, height=softfork_height) assert npc_result.error is None - assert len(npc_result.npc_list) == 1 - opcode = ConditionOpcode(bytes([cond])) - reserve_fee = 0 - assert len(npc_result.npc_list[0].conditions) == 1 - assert npc_result.npc_list[0].conditions[0][0] == opcode - for c in npc_result.npc_list[0].conditions[0][1]: - assert c.opcode == opcode - reserve_fee += int_from_bytes(c.vars[0]) - - assert reserve_fee == 300 - assert len(npc_result.npc_list[0].conditions[0][1]) == 1 + assert npc_result.conds.reserve_fee == 300 + assert len(npc_result.conds.spends) == 1 def test_duplicate_outputs(self, softfork_height): # CREATE_COIN @@ -1952,7 +1939,6 @@ class TestGeneratorConditions: puzzle_hash = "abababababababababababababababab" npc_result = generator_condition_tester(f'(51 "{puzzle_hash}" 10) ' * 2, height=softfork_height) assert npc_result.error == Err.DUPLICATE_OUTPUT.value - assert npc_result.npc_list == [] def test_create_coin_cost(self, softfork_height): # CREATE_COIN @@ -1966,7 +1952,8 @@ class TestGeneratorConditions: ) assert npc_result.error is None assert npc_result.cost == 20470 + 95 * COST_PER_BYTE + ConditionCost.CREATE_COIN.value - assert len(npc_result.npc_list) == 1 + assert len(npc_result.conds.spends) == 1 + assert len(npc_result.conds.spends[0].create_coin) == 1 # if we subtract one from max cost, this should fail npc_result = generator_condition_tester( @@ -1988,7 +1975,7 @@ class TestGeneratorConditions: ) assert npc_result.error is None assert npc_result.cost == 20512 + 117 * COST_PER_BYTE + ConditionCost.AGG_SIG.value - assert len(npc_result.npc_list) == 1 + assert len(npc_result.conds.spends) == 1 # if we subtract one from max cost, this should fail npc_result = generator_condition_tester( @@ -2014,15 +2001,9 @@ class TestGeneratorConditions: generator, MAX_BLOCK_COST_CLVM, cost_per_byte=COST_PER_BYTE, mempool_mode=False, height=softfork_height ) assert npc_result.error is None - assert len(npc_result.npc_list) == 2 - opcode = ConditionOpcode.CREATE_COIN - for c in npc_result.npc_list: - assert c.conditions == [ - ( - opcode.value, - [ConditionWithArgs(opcode, [puzzle_hash.encode("ascii"), bytes([10])])], - ) - ] + assert len(npc_result.conds.spends) == 2 + for s in npc_result.conds.spends: + assert s.create_coin == [(puzzle_hash.encode("ascii"), 10, b"")] def test_create_coin_different_puzzhash(self, softfork_height): # CREATE_COIN @@ -2033,16 +2014,9 @@ class TestGeneratorConditions: f'(51 "{puzzle_hash_1}" 5) (51 "{puzzle_hash_2}" 5)', height=softfork_height ) assert npc_result.error is None - assert len(npc_result.npc_list) == 1 - opcode = ConditionOpcode.CREATE_COIN - assert ( - ConditionWithArgs(opcode, [puzzle_hash_1.encode("ascii"), bytes([5])]) - in npc_result.npc_list[0].conditions[0][1] - ) - assert ( - ConditionWithArgs(opcode, [puzzle_hash_2.encode("ascii"), bytes([5])]) - in npc_result.npc_list[0].conditions[0][1] - ) + assert len(npc_result.conds.spends) == 1 + assert (puzzle_hash_1.encode("ascii"), 5, b"") in npc_result.conds.spends[0].create_coin + assert (puzzle_hash_2.encode("ascii"), 5, b"") in npc_result.conds.spends[0].create_coin def test_create_coin_different_amounts(self, softfork_height): # CREATE_COIN @@ -2052,16 +2026,10 @@ class TestGeneratorConditions: f'(51 "{puzzle_hash}" 5) (51 "{puzzle_hash}" 4)', height=softfork_height ) assert npc_result.error is None - assert len(npc_result.npc_list) == 1 - opcode = ConditionOpcode.CREATE_COIN - assert ( - ConditionWithArgs(opcode, [puzzle_hash.encode("ascii"), bytes([5])]) - in npc_result.npc_list[0].conditions[0][1] - ) - assert ( - ConditionWithArgs(opcode, [puzzle_hash.encode("ascii"), bytes([4])]) - in npc_result.npc_list[0].conditions[0][1] - ) + assert len(npc_result.conds.spends) == 1 + coins = npc_result.conds.spends[0].create_coin + assert (puzzle_hash.encode("ascii"), 5, b"") in coins + assert (puzzle_hash.encode("ascii"), 4, b"") in coins def test_create_coin_with_hint(self, softfork_height): # CREATE_COIN @@ -2069,11 +2037,9 @@ class TestGeneratorConditions: hint = "12341234123412341234213421341234" npc_result = generator_condition_tester(f'(51 "{puzzle_hash_1}" 5 ("{hint}"))', height=softfork_height) assert npc_result.error is None - assert len(npc_result.npc_list) == 1 - opcode = ConditionOpcode.CREATE_COIN - assert npc_result.npc_list[0].conditions[0][1][0] == ConditionWithArgs( - opcode, [puzzle_hash_1.encode("ascii"), bytes([5]), hint.encode("ascii")] - ) + assert len(npc_result.conds.spends) == 1 + coins = npc_result.conds.spends[0].create_coin + assert coins == [(puzzle_hash_1.encode("ascii"), 5, hint.encode("ascii"))] @pytest.mark.parametrize( "mempool,height", @@ -2089,10 +2055,8 @@ class TestGeneratorConditions: print(npc_result) if mempool: assert npc_result.error == Err.INVALID_CONDITION.value - assert npc_result.npc_list == [] else: assert npc_result.error is None - assert npc_result.npc_list[0].conditions == [] # the tests below are malicious generator programs @@ -2231,13 +2195,16 @@ class TestMaliciousGenerators: assert npc_result.error == error_for_condition(opcode) else: assert npc_result.error is None - assert len(npc_result.npc_list) == 1 - assert npc_result.npc_list[0].conditions == [ - ( - opcode, - [ConditionWithArgs(opcode, [int_to_bytes(28)])], - ) - ] + assert len(npc_result.conds.spends) == 1 + if opcode == ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE: + assert npc_result.conds.height_absolute == 28 + elif opcode == ConditionOpcode.ASSERT_HEIGHT_RELATIVE: + assert npc_result.conds.spends[0].height_relative == 28 + elif opcode == ConditionOpcode.ASSERT_SECONDS_ABSOLUTE: + assert npc_result.conds.seconds_absolute == 28 + elif opcode == ConditionOpcode.ASSERT_SECONDS_RELATIVE: + assert npc_result.conds.spends[0].seconds_relative == 28 + print(f"run time:{run_time}") assert run_time < 0.7 @@ -2260,13 +2227,17 @@ class TestMaliciousGenerators: assert npc_result.error == error_for_condition(opcode) else: assert npc_result.error is None - assert len(npc_result.npc_list) == 1 - assert npc_result.npc_list[0].conditions == [ - ( - opcode, - [ConditionWithArgs(opcode, [bytes([100])])], - ) - ] + assert len(npc_result.conds.spends) == 1 + + if opcode == ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE: + assert npc_result.conds.height_absolute == 100 + elif opcode == ConditionOpcode.ASSERT_HEIGHT_RELATIVE: + assert npc_result.conds.spends[0].height_relative == 100 + elif opcode == ConditionOpcode.ASSERT_SECONDS_ABSOLUTE: + assert npc_result.conds.seconds_absolute == 100 + elif opcode == ConditionOpcode.ASSERT_SECONDS_RELATIVE: + assert npc_result.conds.spends[0].seconds_relative == 100 + print(f"run time:{run_time}") assert run_time < 1.1 @@ -2289,13 +2260,17 @@ class TestMaliciousGenerators: assert npc_result.error == error_for_condition(opcode) else: assert npc_result.error is None - assert len(npc_result.npc_list) == 1 - assert npc_result.npc_list[0].conditions == [ - ( - opcode, - [ConditionWithArgs(opcode, [bytes([100])])], - ) - ] + assert len(npc_result.conds.spends) == 1 + + if opcode == ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE: + assert npc_result.conds.height_absolute == 100 + elif opcode == ConditionOpcode.ASSERT_HEIGHT_RELATIVE: + assert npc_result.conds.spends[0].height_relative == 100 + elif opcode == ConditionOpcode.ASSERT_SECONDS_ABSOLUTE: + assert npc_result.conds.seconds_absolute == 100 + elif opcode == ConditionOpcode.ASSERT_SECONDS_RELATIVE: + assert npc_result.conds.spends[0].seconds_relative == 100 + print(f"run time:{run_time}") assert run_time < 1.1 @@ -2320,10 +2295,17 @@ class TestMaliciousGenerators: assert npc_result.error == error_for_condition(opcode) else: assert npc_result.error is None - assert len(npc_result.npc_list) == 1 + assert len(npc_result.conds.spends) == 1 + + if opcode == ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE: + assert npc_result.conds.height_absolute == 0xFFFFFFFF + elif opcode == ConditionOpcode.ASSERT_HEIGHT_RELATIVE: + assert npc_result.conds.spends[0].height_relative == 0xFFFFFFFF + elif opcode == ConditionOpcode.ASSERT_SECONDS_ABSOLUTE: + assert npc_result.conds.seconds_absolute == 0xFFFFFFFF + elif opcode == ConditionOpcode.ASSERT_SECONDS_RELATIVE: + assert npc_result.conds.spends[0].seconds_relative == 0xFFFFFFFF - print(npc_result.npc_list[0].conditions[0][1]) - assert ConditionWithArgs(opcode, [int_to_bytes(0xFFFFFFFF)]) in npc_result.npc_list[0].conditions[0][1] print(f"run time:{run_time}") assert run_time < 0.3 @@ -2343,8 +2325,7 @@ class TestMaliciousGenerators: npc_result = generator_condition_tester(condition, quote=False, height=softfork_height) run_time = time() - start_time assert npc_result.error is None - assert len(npc_result.npc_list) == 1 - assert npc_result.npc_list[0].conditions == [] + assert len(npc_result.conds.spends) == 1 print(f"run time:{run_time}") assert run_time < 1 @@ -2359,13 +2340,9 @@ class TestMaliciousGenerators: assert npc_result.error == error_for_condition(opcode) else: assert npc_result.error is None - assert len(npc_result.npc_list) == 1 - assert npc_result.npc_list[0].conditions == [ - ( - opcode.value, - [ConditionWithArgs(opcode, [int_to_bytes(100 * 280000)])], - ) - ] + assert len(npc_result.conds.spends) == 1 + assert npc_result.conds.reserve_fee == 100 * 280000 + print(f"run time:{run_time}") assert run_time < 1 @@ -2379,7 +2356,7 @@ class TestMaliciousGenerators: # RESERVE_FEE conditions fail unconditionally if they have a negative # amount assert npc_result.error == Err.RESERVE_FEE_CONDITION_FAILED.value - assert len(npc_result.npc_list) == 0 + assert npc_result.conds is None print(f"run time:{run_time}") assert run_time < 0.8 @@ -2393,9 +2370,8 @@ class TestMaliciousGenerators: npc_result = generator_condition_tester(condition, quote=False, height=softfork_height) run_time = time() - start_time assert npc_result.error is None - assert len(npc_result.npc_list) == 1 + assert len(npc_result.conds.spends) == 1 # coin announcements are not propagated to python, but validated in rust - assert len(npc_result.npc_list[0].conditions) == 0 # TODO: optimize clvm to make this run in < 1 second print(f"run time:{run_time}") assert run_time < 7 @@ -2411,7 +2387,7 @@ class TestMaliciousGenerators: npc_result = generator_condition_tester(condition, quote=False, height=softfork_height) run_time = time() - start_time assert npc_result.error == Err.DUPLICATE_OUTPUT.value - assert len(npc_result.npc_list) == 0 + assert npc_result.conds is None print(f"run time:{run_time}") assert run_time < 0.8 @@ -2426,10 +2402,9 @@ class TestMaliciousGenerators: npc_result = generator_condition_tester(condition, quote=False, height=softfork_height) run_time = time() - start_time assert npc_result.error is None - assert len(npc_result.npc_list) == 1 - assert len(npc_result.npc_list[0].conditions) == 1 - assert npc_result.npc_list[0].conditions[0][0] == ConditionOpcode.CREATE_COIN.value - assert len(npc_result.npc_list[0].conditions[0][1]) == 6094 + assert len(npc_result.conds.spends) == 1 + spend = npc_result.conds.spends[0] + assert len(spend.create_coin) == 6094 print(f"run time:{run_time}") assert run_time < 0.2 @@ -2473,65 +2448,37 @@ class TestPkmPairs: ASU = ConditionOpcode.AGG_SIG_UNSAFE def test_empty_list(self): - npc_list = [] - pks, msgs = pkm_pairs(npc_list, b"foobar") + conds = SpendBundleConditions([], 0, 0, 0, [], 0) + pks, msgs = pkm_pairs(conds, b"foobar") assert pks == [] assert msgs == [] def test_no_agg_sigs(self): - npc_list = [ - NPC(self.h1, self.h2, [(self.CCA, [ConditionWithArgs(self.CCA, [b"msg"])])]), - NPC(self.h3, self.h4, [(self.CC, [ConditionWithArgs(self.CCA, [self.h1, bytes([1])])])]), - ] - pks, msgs = pkm_pairs(npc_list, b"foobar") + # one create coin: h1 amount: 1 and not hint + spends = [Spend(self.h3, self.h4, None, 0, [(self.h1, 1, b"")], [])] + conds = SpendBundleConditions(spends, 0, 0, 0, [], 0) + pks, msgs = pkm_pairs(conds, b"foobar") assert pks == [] assert msgs == [] def test_agg_sig_me(self): - npc_list = [ - NPC( - self.h1, - self.h2, - [ - ( - self.ASM, - [ - ConditionWithArgs(self.ASM, [bytes(self.pk1), b"msg1"]), - ConditionWithArgs(self.ASM, [bytes(self.pk2), b"msg2"]), - ], - ) - ], - ) - ] - pks, msgs = pkm_pairs(npc_list, b"foobar") + + spends = [Spend(self.h1, self.h2, None, 0, [], [(bytes48(self.pk1), b"msg1"), (bytes48(self.pk2), b"msg2")])] + conds = SpendBundleConditions(spends, 0, 0, 0, [], 0) + pks, msgs = pkm_pairs(conds, b"foobar") assert [bytes(pk) for pk in pks] == [bytes(self.pk1), bytes(self.pk2)] assert msgs == [b"msg1" + self.h1 + b"foobar", b"msg2" + self.h1 + b"foobar"] def test_agg_sig_unsafe(self): - npc_list = [ - NPC( - self.h1, - self.h2, - [ - ( - self.ASU, - [ - ConditionWithArgs(self.ASU, [bytes(self.pk1), b"msg1"]), - ConditionWithArgs(self.ASU, [bytes(self.pk2), b"msg2"]), - ], - ) - ], - ) - ] - pks, msgs = pkm_pairs(npc_list, b"foobar") + conds = SpendBundleConditions([], 0, 0, 0, [(bytes48(self.pk1), b"msg1"), (bytes48(self.pk2), b"msg2")], 0) + pks, msgs = pkm_pairs(conds, b"foobar") assert [bytes(pk) for pk in pks] == [bytes(self.pk1), bytes(self.pk2)] assert msgs == [b"msg1", b"msg2"] def test_agg_sig_mixed(self): - npc_list = [ - NPC(self.h1, self.h2, [(self.ASM, [ConditionWithArgs(self.ASM, [bytes(self.pk1), b"msg1"])])]), - NPC(self.h1, self.h2, [(self.ASU, [ConditionWithArgs(self.ASU, [bytes(self.pk2), b"msg2"])])]), - ] - pks, msgs = pkm_pairs(npc_list, b"foobar") - assert [bytes(pk) for pk in pks] == [bytes(self.pk1), bytes(self.pk2)] - assert msgs == [b"msg1" + self.h1 + b"foobar", b"msg2"] + + spends = [Spend(self.h1, self.h2, None, 0, [], [(bytes48(self.pk1), b"msg1")])] + conds = SpendBundleConditions(spends, 0, 0, 0, [(bytes48(self.pk2), b"msg2")], 0) + pks, msgs = pkm_pairs(conds, b"foobar") + assert [bytes(pk) for pk in pks] == [bytes(self.pk2), bytes(self.pk1)] + assert msgs == [b"msg2", b"msg1" + self.h1 + b"foobar"] diff --git a/tests/core/test_cost_calculation.py b/tests/core/test_cost_calculation.py index ac24ddf55c..164a3f12a9 100644 --- a/tests/core/test_cost_calculation.py +++ b/tests/core/test_cost_calculation.py @@ -79,12 +79,15 @@ class TestCostCalculation: assert npc_result.error is None assert len(bytes(program.program)) == 433 - coin_name = npc_result.npc_list[0].coin_name + coin_name = npc_result.conds.spends[0].coin_id error, puzzle, solution = get_puzzle_and_solution_for_coin( program, coin_name, test_constants.MAX_BLOCK_COST_CLVM ) assert error is None + assert npc_result.conds.cost == ConditionCost.CREATE_COIN.value + ConditionCost.AGG_SIG.value + 404560 + + # Create condition + agg_sig_condition + length + cpu_cost assert ( npc_result.cost == 404560 @@ -154,7 +157,7 @@ class TestCostCalculation: ) assert npc_result.error is None - coin_name = npc_result.npc_list[0].coin_name + coin_name = npc_result.conds.spends[0].coin_id error, puzzle, solution = get_puzzle_and_solution_for_coin( generator, coin_name, test_constants.MAX_BLOCK_COST_CLVM ) @@ -205,7 +208,7 @@ class TestCostCalculation: end_time = time.time() duration = end_time - start_time assert npc_result.error is None - assert len(npc_result.npc_list) == LARGE_BLOCK_COIN_CONSUMED_COUNT + assert len(npc_result.conds.spends) == LARGE_BLOCK_COIN_CONSUMED_COUNT log.info(f"Time spent: {duration}") assert duration < 0.5 diff --git a/tests/generator/test_rom.py b/tests/generator/test_rom.py index 36e7b74a29..955ae2564b 100644 --- a/tests/generator/test_rom.py +++ b/tests/generator/test_rom.py @@ -1,4 +1,3 @@ -from clvm.casts import int_to_bytes from clvm_tools import binutils from clvm_tools.clvmc import compile_clvm_text @@ -6,13 +5,11 @@ from chia.full_node.generator import run_generator_unsafe from chia.full_node.mempool_check_conditions import get_name_puzzle_conditions from chia.types.blockchain_format.program import Program, SerializedProgram from chia.types.blockchain_format.sized_bytes import bytes32 -from chia.types.condition_opcodes import ConditionOpcode -from chia.types.condition_with_args import ConditionWithArgs -from chia.types.name_puzzle_condition import NPC from chia.types.generator_types import BlockGenerator from chia.util.ints import uint32 from chia.wallet.puzzles.load_clvm import load_clvm from chia.consensus.condition_costs import ConditionCost +from chia.types.spend_bundle_conditions import Spend MAX_COST = int(1e15) COST_PER_BYTE = int(12000) @@ -109,18 +106,17 @@ class TestROM: assert npc_result.cost == EXPECTED_COST + ConditionCost.CREATE_COIN.value + ( len(bytes(gen.program)) * COST_PER_BYTE ) - cond_1 = ConditionWithArgs(ConditionOpcode.CREATE_COIN, [bytes([0] * 31 + [1]), int_to_bytes(500)]) - CONDITIONS = [ - (ConditionOpcode.CREATE_COIN, [cond_1]), - ] - npc = NPC( - coin_name=bytes32.fromhex("e8538c2d14f2a7defae65c5c97f5d4fae7ee64acef7fec9d28ad847a0880fd03"), + spend = Spend( + coin_id=bytes32.fromhex("e8538c2d14f2a7defae65c5c97f5d4fae7ee64acef7fec9d28ad847a0880fd03"), puzzle_hash=bytes32.fromhex("9dcf97a184f32623d11a73124ceb99a5709b083721e878a16d78f596718ba7b2"), - conditions=CONDITIONS, + height_relative=None, + seconds_relative=0, + create_coin=[(bytes([0] * 31 + [1]), 500, b"")], + agg_sig_me=[], ) - assert npc_result.npc_list == [npc] + assert npc_result.conds.spends == [spend] def test_coin_extras(self): # the ROM supports extra data after a coin. This test checks that it actually gets passed through diff --git a/tests/util/generator_tools_testing.py b/tests/util/generator_tools_testing.py index 3bb3c024e4..a4eefdf518 100644 --- a/tests/util/generator_tools_testing.py +++ b/tests/util/generator_tools_testing.py @@ -5,7 +5,7 @@ from chia.types.blockchain_format.coin import Coin from chia.types.blockchain_format.sized_bytes import bytes32 from chia.types.full_block import FullBlock from chia.types.generator_types import BlockGenerator -from chia.util.generator_tools import additions_for_npc +from chia.util.generator_tools import tx_removals_and_additions from chia.util.ints import uint32 @@ -27,10 +27,11 @@ def run_and_get_removals_and_additions( mempool_mode=mempool_mode, height=height, ) + assert npc_result.error is None + rem, add = tx_removals_and_additions(npc_result.conds) # build removals list - for npc in npc_result.npc_list: - removals.append(npc.coin_name) - additions.extend(additions_for_npc(npc_result.npc_list)) + removals.extend(rem) + additions.extend(add) rewards = block.get_included_reward_coins() additions.extend(rewards) diff --git a/tools/run_block.py b/tools/run_block.py index 90956da0ce..c8fe416adf 100644 --- a/tools/run_block.py +++ b/tools/run_block.py @@ -49,10 +49,10 @@ from chia.consensus.default_constants import DEFAULT_CONSTANTS from chia.full_node.generator import create_generator_args from chia.types.blockchain_format.program import SerializedProgram from chia.types.blockchain_format.coin import Coin +from chia.types.blockchain_format.sized_bytes import bytes32 from chia.types.condition_opcodes import ConditionOpcode from chia.types.condition_with_args import ConditionWithArgs from chia.types.generator_types import BlockGenerator -from chia.types.name_puzzle_condition import NPC from chia.util.config import load_config from chia.util.default_root import DEFAULT_ROOT_PATH from chia.util.ints import uint32, uint64 @@ -60,6 +60,13 @@ from chia.wallet.cat_wallet.cat_utils import match_cat_puzzle from clvm.casts import int_from_bytes +@dataclass +class NPC: + coin_name: bytes32 + puzzle_hash: bytes32 + conditions: List[Tuple[ConditionOpcode, List[ConditionWithArgs]]] + + @dataclass class CAT: asset_id: str From 35fb7d341c9dba897cc594ff76eae1351dd993ab Mon Sep 17 00:00:00 2001 From: Mariano Sorgente <3069354+mariano54@users.noreply.github.com> Date: Mon, 4 Apr 2022 16:49:33 -0400 Subject: [PATCH 24/63] Faster full node tests (#10986) * Start fast full node tests * Perf improvement on send_transaction * Major performance improvement for mempool test * Speed up another test * Speed up mempool tests startup * Lint * Debug tests * Try function scope for wallet_nodes * Update comment --- chia/full_node/full_node_api.py | 5 +- tests/conftest.py | 92 +++-- tests/core/full_node/test_full_node.py | 141 ++++--- tests/core/full_node/test_mempool.py | 501 +++++++++++++------------ 4 files changed, 391 insertions(+), 348 deletions(-) diff --git a/chia/full_node/full_node_api.py b/chia/full_node/full_node_api.py index 0811784806..ee60e44657 100644 --- a/chia/full_node/full_node_api.py +++ b/chia/full_node/full_node_api.py @@ -1250,8 +1250,9 @@ class FullNodeAPI: ) # Waits for the transaction to go into the mempool, times out after 45 seconds. status, error = None, None - for i in range(450): - await asyncio.sleep(0.1) + sleep_time = 0.01 + for i in range(int(45 / sleep_time)): + await asyncio.sleep(sleep_time) for potential_name, potential_status, potential_error in self.full_node.transaction_responses: if spend_name == potential_name: status = potential_status diff --git a/tests/conftest.py b/tests/conftest.py index f2d3933a5f..9dd58b313a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -206,9 +206,9 @@ async def five_nodes(db_version, self_hostname): yield _ -@pytest_asyncio.fixture(scope="module") +@pytest_asyncio.fixture(scope="function") async def wallet_nodes(bt): - async_gen = setup_simulators_and_wallets(2, 1, {"MEMPOOL_BLOCK_BUFFER": 2, "MAX_BLOCK_COST_CLVM": 400000000}) + async_gen = setup_simulators_and_wallets(2, 1, {"MEMPOOL_BLOCK_BUFFER": 1, "MAX_BLOCK_COST_CLVM": 400000000}) nodes, wallets = await async_gen.__anext__() full_node_1 = nodes[0] full_node_2 = nodes[1] @@ -343,6 +343,66 @@ async def wallet_and_node(): yield _ +@pytest_asyncio.fixture(scope="function") +async def one_node_one_block(bt, wallet_a): + async_gen = setup_simulators_and_wallets(1, 0, {}) + nodes, _ = await async_gen.__anext__() + full_node_1 = nodes[0] + server_1 = full_node_1.full_node.server + + reward_ph = wallet_a.get_new_puzzlehash() + blocks = bt.get_consecutive_blocks( + 1, + guarantee_transaction_block=True, + farmer_reward_puzzle_hash=reward_ph, + pool_reward_puzzle_hash=reward_ph, + genesis_timestamp=10000, + time_per_block=10, + ) + assert blocks[0].height == 0 + + for block in blocks: + await full_node_1.full_node.respond_block(full_node_protocol.RespondBlock(block)) + + await time_out_assert(60, node_height_at_least, True, full_node_1, blocks[-1].height) + + yield full_node_1, server_1 + + async for _ in async_gen: + yield _ + + +@pytest_asyncio.fixture(scope="function") +async def two_nodes_one_block(bt, wallet_a): + async_gen = setup_simulators_and_wallets(2, 0, {}) + nodes, _ = await async_gen.__anext__() + full_node_1 = nodes[0] + full_node_2 = nodes[1] + server_1 = full_node_1.full_node.server + server_2 = full_node_2.full_node.server + + reward_ph = wallet_a.get_new_puzzlehash() + blocks = bt.get_consecutive_blocks( + 1, + guarantee_transaction_block=True, + farmer_reward_puzzle_hash=reward_ph, + pool_reward_puzzle_hash=reward_ph, + genesis_timestamp=10000, + time_per_block=10, + ) + assert blocks[0].height == 0 + + for block in blocks: + await full_node_1.full_node.respond_block(full_node_protocol.RespondBlock(block)) + + await time_out_assert(60, node_height_at_least, True, full_node_1, blocks[-1].height) + + yield full_node_1, full_node_2, server_1, server_2 + + async for _ in async_gen: + yield _ + + # TODO: Ideally, the db_version should be the (parameterized) db_version # fixture, to test all versions of the database schema. This doesn't work # because of a hack in shutting down the full node, which means you cannot run @@ -451,34 +511,6 @@ async def timelord(bt): yield _ -@pytest_asyncio.fixture(scope="module") -async def two_nodes_mempool(bt, wallet_a): - async_gen = setup_simulators_and_wallets(2, 1, {}) - nodes, _ = await async_gen.__anext__() - full_node_1 = nodes[0] - full_node_2 = nodes[1] - server_1 = full_node_1.full_node.server - server_2 = full_node_2.full_node.server - - reward_ph = wallet_a.get_new_puzzlehash() - blocks = bt.get_consecutive_blocks( - 3, - guarantee_transaction_block=True, - farmer_reward_puzzle_hash=reward_ph, - pool_reward_puzzle_hash=reward_ph, - ) - - for block in blocks: - await full_node_1.full_node.respond_block(full_node_protocol.RespondBlock(block)) - - await time_out_assert(60, node_height_at_least, True, full_node_1, blocks[-1].height) - - yield full_node_1, full_node_2, server_1, server_2 - - async for _ in async_gen: - yield _ - - @pytest_asyncio.fixture(scope="function") async def setup_sim(): sim = await SpendSim.create() diff --git a/tests/core/full_node/test_full_node.py b/tests/core/full_node/test_full_node.py index 5c324c3540..c71e8b0758 100644 --- a/tests/core/full_node/test_full_node.py +++ b/tests/core/full_node/test_full_node.py @@ -1,6 +1,5 @@ import asyncio import dataclasses -import logging import random import time from secrets import token_bytes @@ -52,8 +51,6 @@ from tests.pools.test_pool_rpc import wallet_is_synced from tests.setup_nodes import test_constants from tests.time_out_assert import time_out_assert, time_out_assert_custom_interval, time_out_messages -log = logging.getLogger(__name__) - async def new_transaction_not_requested(incoming, new_spend): await asyncio.sleep(3) @@ -97,7 +94,7 @@ async def get_block_path(full_node: FullNodeAPI): class TestFullNodeBlockCompression: @pytest.mark.asyncio - @pytest.mark.parametrize("tx_size", [10000, 3000000000000]) + @pytest.mark.parametrize("tx_size", [3000000000000]) async def test_block_compression(self, setup_two_nodes_and_wallet, empty_blockchain, tx_size, bt, self_hostname): nodes, wallets = setup_two_nodes_and_wallet server_1 = nodes[0].full_node.server @@ -153,7 +150,6 @@ class TestFullNodeBlockCompression: return tx.confirmed await time_out_assert(30, check_transaction_confirmed, True, tr) - await asyncio.sleep(2) # Confirm generator is not compressed program: Optional[SerializedProgram] = (await full_node_1.get_all_full_blocks())[-1].transactions_generator @@ -182,7 +178,6 @@ class TestFullNodeBlockCompression: await time_out_assert(30, wallet_is_synced, True, wallet_node_1, full_node_1) await time_out_assert(10, check_transaction_confirmed, True, tr) - await asyncio.sleep(2) # Confirm generator is compressed program: Optional[SerializedProgram] = (await full_node_1.get_all_full_blocks())[-1].transactions_generator @@ -254,7 +249,6 @@ class TestFullNodeBlockCompression: await time_out_assert(30, wallet_is_synced, True, wallet_node_1, full_node_1) await time_out_assert(10, check_transaction_confirmed, True, tr) - await asyncio.sleep(2) # Confirm generator is compressed program: Optional[SerializedProgram] = (await full_node_1.get_all_full_blocks())[-1].transactions_generator @@ -300,7 +294,6 @@ class TestFullNodeBlockCompression: await time_out_assert(30, wallet_is_synced, True, wallet_node_1, full_node_1) await time_out_assert(10, check_transaction_confirmed, True, new_tr) - await asyncio.sleep(2) # Confirm generator is not compressed, #CAT creation has a cat spend all_blocks = await full_node_1.get_all_full_blocks() @@ -356,7 +349,6 @@ class TestFullNodeBlockCompression: blockchain = empty_blockchain all_blocks: List[FullBlock] = await full_node_1.get_all_full_blocks() assert height == len(all_blocks) - 1 - assert full_node_1.full_node.full_node_store.previous_generator is not None if test_reorgs: reog_blocks = bt.get_consecutive_blocks(14) @@ -364,7 +356,7 @@ class TestFullNodeBlockCompression: for reorg_block in reog_blocks[:r]: await _validate_and_add_block_no_error(blockchain, reorg_block) for i in range(1, height): - for batch_size in range(1, height): + for batch_size in range(1, height, 3): results = await blockchain.pre_validate_blocks_multiprocessing( all_blocks[:i], {}, batch_size, validate_signatures=False ) @@ -376,7 +368,7 @@ class TestFullNodeBlockCompression: for block in all_blocks[:r]: await _validate_and_add_block_no_error(blockchain, block) for i in range(1, height): - for batch_size in range(1, height): + for batch_size in range(1, height, 3): results = await blockchain.pre_validate_blocks_multiprocessing( all_blocks[:i], {}, batch_size, validate_signatures=False ) @@ -785,12 +777,9 @@ class TestFullNodeProtocol: @pytest.mark.asyncio async def test_new_transaction_and_mempool(self, wallet_nodes, bt, self_hostname): full_node_1, full_node_2, server_1, server_2, wallet_a, wallet_receiver = wallet_nodes - blocks = await full_node_1.get_all_full_blocks() - wallet_ph = wallet_a.get_new_puzzlehash() blocks = bt.get_consecutive_blocks( - 10, - block_list_input=blocks, + 3, guarantee_transaction_block=True, farmer_reward_puzzle_hash=wallet_ph, pool_reward_puzzle_hash=wallet_ph, @@ -806,55 +795,48 @@ class TestFullNodeProtocol: peer = await connect_and_get_peer(server_1, server_2, self_hostname) incoming_queue, node_id = await add_dummy_connection(server_1, self_hostname, 12312) fake_peer = server_1.all_connections[node_id] - # Mempool has capacity of 100, make 110 unspent coins that we can use puzzle_hashes = [] # Makes a bunch of coins - for i in range(5): - conditions_dict: Dict = {ConditionOpcode.CREATE_COIN: []} - # This should fit in one transaction - for _ in range(100): - receiver_puzzlehash = wallet_receiver.get_new_puzzlehash() - puzzle_hashes.append(receiver_puzzlehash) - output = ConditionWithArgs(ConditionOpcode.CREATE_COIN, [receiver_puzzlehash, int_to_bytes(100000000)]) + conditions_dict: Dict = {ConditionOpcode.CREATE_COIN: []} + # This should fit in one transaction + for _ in range(100): + receiver_puzzlehash = wallet_receiver.get_new_puzzlehash() + puzzle_hashes.append(receiver_puzzlehash) + output = ConditionWithArgs(ConditionOpcode.CREATE_COIN, [receiver_puzzlehash, int_to_bytes(10000000000)]) - conditions_dict[ConditionOpcode.CREATE_COIN].append(output) + conditions_dict[ConditionOpcode.CREATE_COIN].append(output) - spend_bundle = wallet_a.generate_signed_transaction( - 100, - puzzle_hashes[0], - get_future_reward_coins(blocks[1 + i])[0], - condition_dic=conditions_dict, - ) - assert spend_bundle is not None - cost_result = await full_node_1.full_node.mempool_manager.pre_validate_spendbundle( - spend_bundle, None, spend_bundle.name() - ) - log.info(f"Cost result: {cost_result.cost}") + spend_bundle = wallet_a.generate_signed_transaction( + 100, + puzzle_hashes[0], + get_future_reward_coins(blocks[1])[0], + condition_dic=conditions_dict, + ) + assert spend_bundle is not None + new_transaction = fnp.NewTransaction(spend_bundle.get_hash(), uint64(100), uint64(100)) - new_transaction = fnp.NewTransaction(spend_bundle.get_hash(), uint64(100), uint64(100)) + await full_node_1.new_transaction(new_transaction, fake_peer) + await time_out_assert(10, new_transaction_requested, True, incoming_queue, new_transaction) - await full_node_1.new_transaction(new_transaction, fake_peer) - await time_out_assert(10, new_transaction_requested, True, incoming_queue, new_transaction) + respond_transaction_2 = fnp.RespondTransaction(spend_bundle) + await full_node_1.respond_transaction(respond_transaction_2, peer) - respond_transaction_2 = fnp.RespondTransaction(spend_bundle) - await full_node_1.respond_transaction(respond_transaction_2, peer) + blocks = bt.get_consecutive_blocks( + 1, + block_list_input=blocks, + guarantee_transaction_block=True, + transaction_data=spend_bundle, + ) + await full_node_1.full_node.respond_block(fnp.RespondBlock(blocks[-1]), None) - blocks = bt.get_consecutive_blocks( - 1, - block_list_input=blocks, - guarantee_transaction_block=True, - transaction_data=spend_bundle, - ) - await full_node_1.full_node.respond_block(fnp.RespondBlock(blocks[-1]), peer) + # Already seen + await full_node_1.new_transaction(new_transaction, fake_peer) + await time_out_assert(10, new_transaction_not_requested, True, incoming_queue, new_transaction) - # Already seen - await full_node_1.new_transaction(new_transaction, fake_peer) - await time_out_assert(10, new_transaction_not_requested, True, incoming_queue, new_transaction) - - await time_out_assert(10, node_height_at_least, True, full_node_1, start_height + 5) - - spend_bundles = [] + print(f"FULL NODE HEIGHT: {start_height + 1} {full_node_1.full_node.blockchain.get_peak_height()}") + await time_out_assert(10, node_height_at_least, True, full_node_1, start_height + 1) + await time_out_assert(10, node_height_at_least, True, full_node_2, start_height + 1) included_tx = 0 not_included_tx = 0 @@ -862,18 +844,32 @@ class TestFullNodeProtocol: successful_bundle: Optional[SpendBundle] = None # Fill mempool - for puzzle_hash in puzzle_hashes[1:]: - coin_record = (await full_node_1.full_node.coin_store.get_coin_records_by_puzzle_hash(True, puzzle_hash))[0] - receiver_puzzlehash = wallet_receiver.get_new_puzzlehash() - if puzzle_hash == puzzle_hashes[-1]: + receiver_puzzlehash = wallet_receiver.get_new_puzzlehash() + group_size = 3 # We will generate transaction bundles of this size (* standard transaction of around 3-4M cost) + for i in range(1, len(puzzle_hashes), group_size): + phs_to_use = [puzzle_hashes[i + j] for j in range(group_size) if (i + j) < len(puzzle_hashes)] + coin_records = [ + (await full_node_1.full_node.coin_store.get_coin_records_by_puzzle_hash(True, puzzle_hash))[0] + for puzzle_hash in phs_to_use + ] + + last_iteration = (i == len(puzzle_hashes) - group_size) or len(phs_to_use) < group_size + if last_iteration: force_high_fee = True - fee = 100000000 # 100 million (20 fee per cost) + fee = 100000000 * group_size # 100 million * group_size (20 fee per cost) else: force_high_fee = False - fee = random.randint(1, 100000000) - spend_bundle = wallet_receiver.generate_signed_transaction( - uint64(500), receiver_puzzlehash, coin_record.coin, fee=fee - ) + fee = random.randint(1, 100000000 * group_size) + spend_bundles = [ + wallet_receiver.generate_signed_transaction(uint64(500), receiver_puzzlehash, coin_record.coin, fee=0) + for coin_record in coin_records[1:] + ] + [ + wallet_receiver.generate_signed_transaction( + uint64(500), receiver_puzzlehash, coin_records[0].coin, fee=fee + ) + ] + spend_bundle = SpendBundle.aggregate(spend_bundles) + assert spend_bundle.fees() == fee respond_transaction = wallet_protocol.SendTransaction(spend_bundle) await full_node_1.send_transaction(respond_transaction) @@ -881,12 +877,8 @@ class TestFullNodeProtocol: request = fnp.RequestTransaction(spend_bundle.get_hash()) req = await full_node_1.request_transaction(request) - fee_rate_for_small = full_node_1.full_node.mempool_manager.mempool.get_min_fee_rate(10) fee_rate_for_med = full_node_1.full_node.mempool_manager.mempool.get_min_fee_rate(5000000) fee_rate_for_large = full_node_1.full_node.mempool_manager.mempool.get_min_fee_rate(50000000) - log.info(f"Min fee rate (10): {fee_rate_for_small}") - log.info(f"Min fee rate (5000000): {fee_rate_for_med}") - log.info(f"Min fee rate (50000000): {fee_rate_for_large}") if fee_rate_for_large > fee_rate_for_med: seen_bigger_transaction_has_high_fee = True @@ -898,11 +890,11 @@ class TestFullNodeProtocol: if force_high_fee: successful_bundle = spend_bundle else: - assert full_node_1.full_node.mempool_manager.mempool.at_full_capacity(10000000) - assert full_node_1.full_node.mempool_manager.mempool.get_min_fee_rate(10000000) > 0 + assert full_node_1.full_node.mempool_manager.mempool.at_full_capacity(5000000 * group_size) + assert full_node_1.full_node.mempool_manager.mempool.get_min_fee_rate(5000000 * group_size) > 0 assert not force_high_fee not_included_tx += 1 - log.info(f"Included: {included_tx}, not included: {not_included_tx}") + assert full_node_1.full_node.mempool_manager.mempool.at_full_capacity(10000000 * group_size) assert included_tx > 0 assert not_included_tx > 0 @@ -911,6 +903,8 @@ class TestFullNodeProtocol: # Mempool is full new_transaction = fnp.NewTransaction(token_bytes(32), 10000000, uint64(1)) await full_node_1.new_transaction(new_transaction, fake_peer) + assert full_node_1.full_node.mempool_manager.mempool.at_full_capacity(10000000 * group_size) + assert full_node_2.full_node.mempool_manager.mempool.at_full_capacity(10000000 * group_size) await time_out_assert(10, new_transaction_not_requested, True, incoming_queue, new_transaction) @@ -940,11 +934,11 @@ class TestFullNodeProtocol: # Reorg the blockchain blocks = await full_node_1.get_all_full_blocks() blocks = bt.get_consecutive_blocks( - 3, + 2, block_list_input=blocks[:-1], guarantee_transaction_block=True, ) - for block in blocks[-3:]: + for block in blocks[-2:]: await full_node_1.full_node.respond_block(fnp.RespondBlock(block), peer) # Can now resubmit a transaction after the reorg @@ -1228,9 +1222,10 @@ class TestFullNodeProtocol: block_2 = recursive_replace(block_2, "foliage.foliage_transaction_block_signature", new_fbh_sig) block_2 = recursive_replace(block_2, "transactions_generator", None) - await full_node_2.full_node.respond_block(fnp.RespondBlock(block_2), dummy_peer) + rb_task = asyncio.create_task(full_node_2.full_node.respond_block(fnp.RespondBlock(block_2), dummy_peer)) await time_out_assert(10, time_out_messages(incoming_queue, "request_block", 1)) + rb_task.cancel() @pytest.mark.asyncio async def test_request_unfinished_block(self, wallet_nodes, bt, self_hostname): diff --git a/tests/core/full_node/test_mempool.py b/tests/core/full_node/test_mempool.py index 0fc7f504bd..a05f5b4e33 100644 --- a/tests/core/full_node/test_mempool.py +++ b/tests/core/full_node/test_mempool.py @@ -6,7 +6,6 @@ from typing import Dict, List, Optional, Tuple, Callable from clvm.casts import int_to_bytes import pytest -import pytest_asyncio import chia.server.ws_connection as ws @@ -36,9 +35,8 @@ from blspy import G2Element from chia.util.recursive_replace import recursive_replace from tests.blockchain.blockchain_test_utils import _validate_and_add_block -from tests.connection_utils import connect_and_get_peer +from tests.connection_utils import connect_and_get_peer, add_dummy_connection from tests.core.node_height import node_height_at_least -from tests.setup_nodes import setup_simulators_and_wallets from tests.time_out_assert import time_out_assert from chia.types.blockchain_format.program import Program, INFINITE_COST from chia.consensus.cost_calculator import NPCResult @@ -78,37 +76,6 @@ def generate_test_spend_bundle( return transaction -@pytest_asyncio.fixture(scope="function") -async def two_nodes_mempool(bt, wallet_a): - async_gen = setup_simulators_and_wallets(2, 1, {}) - nodes, _ = await async_gen.__anext__() - full_node_1 = nodes[0] - full_node_2 = nodes[1] - server_1 = full_node_1.full_node.server - server_2 = full_node_2.full_node.server - - reward_ph = wallet_a.get_new_puzzlehash() - blocks = bt.get_consecutive_blocks( - 3, - guarantee_transaction_block=True, - farmer_reward_puzzle_hash=reward_ph, - pool_reward_puzzle_hash=reward_ph, - genesis_timestamp=10000, - time_per_block=10, - ) - assert blocks[0].height == 0 - - for block in blocks: - await full_node_1.full_node.respond_block(full_node_protocol.RespondBlock(block)) - - await time_out_assert(60, node_height_at_least, True, full_node_1, blocks[-1].height) - - yield full_node_1, full_node_2, server_1, server_2 - - async for _ in async_gen: - yield _ - - def make_item(idx: int, cost: uint64 = uint64(80)) -> MempoolItem: spend_bundle_name = bytes32([idx] * 32) return MempoolItem( @@ -184,9 +151,12 @@ class TestPendingTxCache: class TestMempool: @pytest.mark.asyncio - async def test_basic_mempool(self, bt, two_nodes_mempool, wallet_a): + async def test_basic_mempool(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block + + _ = await next_block(full_node_1, wallet_a, bt) + _ = await next_block(full_node_1, wallet_a, bt) max_mempool_cost = 40000000 * 5 mempool = Mempool(max_mempool_cost) @@ -248,11 +218,12 @@ async def next_block(full_node_1, wallet_a, bt) -> Coin: class TestMempoolManager: @pytest.mark.asyncio - async def test_basic_mempool_manager(self, bt, two_nodes_mempool, wallet_a, self_hostname): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + async def test_basic_mempool_manager(self, bt, two_nodes_one_block, wallet_a, self_hostname): + full_node_1, full_node_2, server_1, server_2 = two_nodes_one_block peer = await connect_and_get_peer(server_1, server_2, self_hostname) + _ = await next_block(full_node_1, wallet_a, bt) coin = await next_block(full_node_1, wallet_a, bt) spend_bundle = generate_test_spend_bundle(wallet_a, coin) assert spend_bundle is not None @@ -279,7 +250,7 @@ class TestMempoolManager: (ConditionOpcode.ASSERT_HEIGHT_RELATIVE, 0, MempoolInclusionStatus.PENDING), (ConditionOpcode.ASSERT_HEIGHT_RELATIVE, 1, MempoolInclusionStatus.PENDING), # the absolute height and seconds tests require fresh full nodes to - # run the test on. The fixture (two_nodes_mempool) creates 3 blocks, + # run the test on. The fixture (one_node_one_block) creates a block, # then condition_tester2 creates another 3 blocks (ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE, 4, MempoolInclusionStatus.SUCCESS), (ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE, 5, MempoolInclusionStatus.SUCCESS), @@ -292,7 +263,7 @@ class TestMempoolManager: (ConditionOpcode.ASSERT_SECONDS_ABSOLUTE, 10052, MempoolInclusionStatus.FAILED), ], ) - async def test_ephemeral_timelock(self, bt, two_nodes_mempool, wallet_a, opcode, lock_value, expected): + async def test_ephemeral_timelock(self, bt, one_node_one_block, wallet_a, opcode, lock_value, expected): def test_fun(coin_1: Coin, coin_2: Coin) -> SpendBundle: conditions = {opcode: [ConditionWithArgs(opcode, [int_to_bytes(lock_value)])]} @@ -308,8 +279,10 @@ class TestMempoolManager: bundle = SpendBundle.aggregate([tx1, tx2]) return bundle - full_node_1, _, server_1, _ = two_nodes_mempool - blocks, bundle, status, err = await self.condition_tester2(bt, two_nodes_mempool, wallet_a, test_fun) + full_node_1, server_1 = one_node_one_block + _ = await next_block(full_node_1, wallet_a, bt) + _ = await next_block(full_node_1, wallet_a, bt) + blocks, bundle, status, err = await self.condition_tester2(bt, one_node_one_block, wallet_a, test_fun) mempool_bundle = full_node_1.full_node.mempool_manager.get_spendbundle(bundle.name()) print(f"status: {status}") @@ -326,7 +299,7 @@ class TestMempoolManager: # this test makes sure that one spend successfully asserts the announce from # another spend, even though the assert condition is duplicated 100 times @pytest.mark.asyncio - async def test_coin_announcement_duplicate_consumed(self, bt, two_nodes_mempool, wallet_a): + async def test_coin_announcement_duplicate_consumed(self, bt, one_node_one_block, wallet_a): def test_fun(coin_1: Coin, coin_2: Coin) -> SpendBundle: announce = Announcement(coin_2.name(), b"test") cvp = ConditionWithArgs(ConditionOpcode.ASSERT_COIN_ANNOUNCEMENT, [announce.name()]) @@ -339,8 +312,8 @@ class TestMempoolManager: bundle = SpendBundle.aggregate([spend_bundle1, spend_bundle2]) return bundle - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool - blocks, bundle, status, err = await self.condition_tester2(bt, two_nodes_mempool, wallet_a, test_fun) + full_node_1, server_1 = one_node_one_block + blocks, bundle, status, err = await self.condition_tester2(bt, one_node_one_block, wallet_a, test_fun) mempool_bundle = full_node_1.full_node.mempool_manager.get_spendbundle(bundle.name()) assert err is None @@ -350,7 +323,7 @@ class TestMempoolManager: # this test makes sure that one spend successfully asserts the announce from # another spend, even though the create announcement is duplicated 100 times @pytest.mark.asyncio - async def test_coin_duplicate_announcement_consumed(self, bt, two_nodes_mempool, wallet_a): + async def test_coin_duplicate_announcement_consumed(self, bt, one_node_one_block, wallet_a): def test_fun(coin_1: Coin, coin_2: Coin) -> SpendBundle: announce = Announcement(coin_2.name(), b"test") cvp = ConditionWithArgs(ConditionOpcode.ASSERT_COIN_ANNOUNCEMENT, [announce.name()]) @@ -363,8 +336,8 @@ class TestMempoolManager: bundle = SpendBundle.aggregate([spend_bundle1, spend_bundle2]) return bundle - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool - blocks, bundle, status, err = await self.condition_tester2(bt, two_nodes_mempool, wallet_a, test_fun) + full_node_1, server_1 = one_node_one_block + blocks, bundle, status, err = await self.condition_tester2(bt, one_node_one_block, wallet_a, test_fun) mempool_bundle = full_node_1.full_node.mempool_manager.get_spendbundle(bundle.name()) assert err is None @@ -372,9 +345,9 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_double_spend(self, bt, two_nodes_mempool, wallet_a, self_hostname): + async def test_double_spend(self, bt, two_nodes_one_block, wallet_a, self_hostname): reward_ph = wallet_a.get_new_puzzlehash() - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, full_node_2, server_1, server_2 = two_nodes_one_block blocks = await full_node_1.get_all_full_blocks() start_height = blocks[-1].height blocks = bt.get_consecutive_blocks( @@ -433,10 +406,10 @@ class TestMempoolManager: assert node.full_node.mempool_manager.get_spendbundle(sb.name()) is None @pytest.mark.asyncio - async def test_double_spend_with_higher_fee(self, bt, two_nodes_mempool, wallet_a, self_hostname): + async def test_double_spend_with_higher_fee(self, bt, two_nodes_one_block, wallet_a, self_hostname): reward_ph = wallet_a.get_new_puzzlehash() - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, full_node_2, server_1, server_2 = two_nodes_one_block blocks = await full_node_1.get_all_full_blocks() start_height = blocks[-1].height if len(blocks) > 0 else -1 blocks = bt.get_consecutive_blocks( @@ -510,10 +483,10 @@ class TestMempoolManager: self.assert_sb_not_in_pool(full_node_1, sb3) @pytest.mark.asyncio - async def test_invalid_signature(self, bt, two_nodes_mempool, wallet_a): + async def test_invalid_signature(self, bt, one_node_one_block, wallet_a): reward_ph = wallet_a.get_new_puzzlehash() - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block blocks = await full_node_1.get_all_full_blocks() start_height = blocks[-1].height if len(blocks) > 0 else -1 blocks = bt.get_consecutive_blocks( @@ -544,7 +517,7 @@ class TestMempoolManager: async def condition_tester( self, bt, - two_nodes_mempool, + one_node_one_block, wallet_a, dic: Dict[ConditionOpcode, List[ConditionWithArgs]], fee: int = 0, @@ -552,7 +525,7 @@ class TestMempoolManager: coin: Optional[Coin] = None, ): reward_ph = wallet_a.get_new_puzzlehash() - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block blocks = await full_node_1.get_all_full_blocks() start_height = blocks[-1].height blocks = bt.get_consecutive_blocks( @@ -562,7 +535,12 @@ class TestMempoolManager: farmer_reward_puzzle_hash=reward_ph, pool_reward_puzzle_hash=reward_ph, ) - peer = await connect_and_get_peer(server_1, server_2, bt.config["self_hostname"]) + _, dummy_node_id = await add_dummy_connection(server_1, bt.config["self_hostname"], 100) + dummy_peer = None + for node_id, wsc in server_1.all_connections.items(): + if node_id == dummy_node_id: + dummy_peer = wsc + break for block in blocks: await full_node_1.full_node.respond_block(full_node_protocol.RespondBlock(block)) @@ -577,13 +555,13 @@ class TestMempoolManager: tx1: full_node_protocol.RespondTransaction = full_node_protocol.RespondTransaction(spend_bundle1) - status, err = await respond_transaction(full_node_1, tx1, peer, test=True) - return blocks, spend_bundle1, peer, status, err + status, err = await respond_transaction(full_node_1, tx1, dummy_peer, test=True) + return blocks, spend_bundle1, dummy_peer, status, err @pytest.mark.asyncio - async def condition_tester2(self, bt, two_nodes_mempool, wallet_a, test_fun: Callable[[Coin, Coin], SpendBundle]): + async def condition_tester2(self, bt, one_node_one_block, wallet_a, test_fun: Callable[[Coin, Coin], SpendBundle]): reward_ph = wallet_a.get_new_puzzlehash() - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block blocks = await full_node_1.get_all_full_blocks() start_height = blocks[-1].height if len(blocks) > 0 else -1 blocks = bt.get_consecutive_blocks( @@ -594,7 +572,12 @@ class TestMempoolManager: pool_reward_puzzle_hash=reward_ph, time_per_block=10, ) - peer = await connect_and_get_peer(server_1, server_2, bt.config["self_hostname"]) + _, dummy_node_id = await add_dummy_connection(server_1, bt.config["self_hostname"], 100) + dummy_peer = None + for node_id, wsc in server_1.all_connections.items(): + if node_id == dummy_node_id: + dummy_peer = wsc + break for block in blocks: await full_node_1.full_node.respond_block(full_node_protocol.RespondBlock(block)) @@ -607,14 +590,14 @@ class TestMempoolManager: bundle = test_fun(coin_1, coin_2) tx1: full_node_protocol.RespondTransaction = full_node_protocol.RespondTransaction(bundle) - status, err = await respond_transaction(full_node_1, tx1, peer, test=True) + status, err = await respond_transaction(full_node_1, tx1, dummy_peer, test=True) return blocks, bundle, status, err @pytest.mark.asyncio - async def test_invalid_block_index(self, bt, two_nodes_mempool, wallet_a): + async def test_invalid_block_index(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block blocks = await full_node_1.get_all_full_blocks() start_height = blocks[-1].height cvp = ConditionWithArgs( @@ -622,7 +605,7 @@ class TestMempoolManager: [int_to_bytes(start_height + 5)], ) dic = {ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) assert sb1 is None # the transaction may become valid later @@ -630,13 +613,13 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.PENDING @pytest.mark.asyncio - async def test_block_index_missing_arg(self, bt, two_nodes_mempool, wallet_a): + async def test_block_index_missing_arg(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block blocks = await full_node_1.get_all_full_blocks() cvp = ConditionWithArgs(ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE, []) dic = {ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) assert sb1 is None # the transaction may become valid later @@ -644,49 +627,49 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_correct_block_index(self, bt, two_nodes_mempool, wallet_a): + async def test_correct_block_index(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block cvp = ConditionWithArgs(ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE, [int_to_bytes(1)]) dic = {ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) assert err is None assert sb1 is spend_bundle1 assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_block_index_garbage(self, bt, two_nodes_mempool, wallet_a): + async def test_block_index_garbage(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block # garbage at the end of the argument list is ignored cvp = ConditionWithArgs(ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE, [int_to_bytes(1), b"garbage"]) dic = {ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) assert err is None assert sb1 is spend_bundle1 assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_negative_block_index(self, bt, two_nodes_mempool, wallet_a): + async def test_negative_block_index(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block cvp = ConditionWithArgs(ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE, [int_to_bytes(-1)]) dic = {ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) assert err is None assert sb1 is spend_bundle1 assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_invalid_block_age(self, bt, two_nodes_mempool, wallet_a): + async def test_invalid_block_age(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block cvp = ConditionWithArgs(ConditionOpcode.ASSERT_HEIGHT_RELATIVE, [int_to_bytes(5)]) dic = {cvp.opcode: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) assert err == Err.ASSERT_HEIGHT_RELATIVE_FAILED assert sb1 is None @@ -694,12 +677,12 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.PENDING @pytest.mark.asyncio - async def test_block_age_missing_arg(self, bt, two_nodes_mempool, wallet_a): + async def test_block_age_missing_arg(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block cvp = ConditionWithArgs(ConditionOpcode.ASSERT_HEIGHT_RELATIVE, []) dic = {cvp.opcode: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) assert err == Err.INVALID_CONDITION assert sb1 is None @@ -707,13 +690,13 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_correct_block_age(self, bt, two_nodes_mempool, wallet_a): + async def test_correct_block_age(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block cvp = ConditionWithArgs(ConditionOpcode.ASSERT_HEIGHT_RELATIVE, [int_to_bytes(1)]) dic = {cvp.opcode: [cvp]} blocks, spend_bundle1, peer, status, err = await self.condition_tester( - bt, two_nodes_mempool, wallet_a, dic, num_blocks=4 + bt, one_node_one_block, wallet_a, dic, num_blocks=4 ) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -722,14 +705,14 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_block_age_garbage(self, bt, two_nodes_mempool, wallet_a): + async def test_block_age_garbage(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block # garbage at the end of the argument list is ignored cvp = ConditionWithArgs(ConditionOpcode.ASSERT_HEIGHT_RELATIVE, [int_to_bytes(1), b"garbage"]) dic = {cvp.opcode: [cvp]} blocks, spend_bundle1, peer, status, err = await self.condition_tester( - bt, two_nodes_mempool, wallet_a, dic, num_blocks=4 + bt, one_node_one_block, wallet_a, dic, num_blocks=4 ) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -738,13 +721,13 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_negative_block_age(self, bt, two_nodes_mempool, wallet_a): + async def test_negative_block_age(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block cvp = ConditionWithArgs(ConditionOpcode.ASSERT_HEIGHT_RELATIVE, [int_to_bytes(-1)]) dic = {cvp.opcode: [cvp]} blocks, spend_bundle1, peer, status, err = await self.condition_tester( - bt, two_nodes_mempool, wallet_a, dic, num_blocks=4 + bt, one_node_one_block, wallet_a, dic, num_blocks=4 ) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -753,14 +736,17 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_correct_my_id(self, bt, two_nodes_mempool, wallet_a): + async def test_correct_my_id(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block + + _ = await next_block(full_node_1, wallet_a, bt) + _ = await next_block(full_node_1, wallet_a, bt) coin = await next_block(full_node_1, wallet_a, bt) cvp = ConditionWithArgs(ConditionOpcode.ASSERT_MY_COIN_ID, [coin.name()]) dic = {cvp.opcode: [cvp]} blocks, spend_bundle1, peer, status, err = await self.condition_tester( - bt, two_nodes_mempool, wallet_a, dic, coin=coin + bt, one_node_one_block, wallet_a, dic, coin=coin ) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -769,15 +755,18 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_my_id_garbage(self, bt, two_nodes_mempool, wallet_a): + async def test_my_id_garbage(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block + + _ = await next_block(full_node_1, wallet_a, bt) + _ = await next_block(full_node_1, wallet_a, bt) coin = await next_block(full_node_1, wallet_a, bt) # garbage at the end of the argument list is ignored cvp = ConditionWithArgs(ConditionOpcode.ASSERT_MY_COIN_ID, [coin.name(), b"garbage"]) dic = {cvp.opcode: [cvp]} blocks, spend_bundle1, peer, status, err = await self.condition_tester( - bt, two_nodes_mempool, wallet_a, dic, coin=coin + bt, one_node_one_block, wallet_a, dic, coin=coin ) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -786,15 +775,18 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_invalid_my_id(self, bt, two_nodes_mempool, wallet_a): + async def test_invalid_my_id(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block + + _ = await next_block(full_node_1, wallet_a, bt) + _ = await next_block(full_node_1, wallet_a, bt) coin = await next_block(full_node_1, wallet_a, bt) coin_2 = await next_block(full_node_1, wallet_a, bt) cvp = ConditionWithArgs(ConditionOpcode.ASSERT_MY_COIN_ID, [coin_2.name()]) dic = {cvp.opcode: [cvp]} blocks, spend_bundle1, peer, status, err = await self.condition_tester( - bt, two_nodes_mempool, wallet_a, dic, coin=coin + bt, one_node_one_block, wallet_a, dic, coin=coin ) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -803,13 +795,13 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_my_id_missing_arg(self, bt, two_nodes_mempool, wallet_a): + async def test_my_id_missing_arg(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block blocks = await full_node_1.get_all_full_blocks() cvp = ConditionWithArgs(ConditionOpcode.ASSERT_MY_COIN_ID, []) dic = {cvp.opcode: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) assert err == Err.INVALID_CONDITION @@ -817,100 +809,100 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_assert_time_exceeds(self, bt, two_nodes_mempool, wallet_a): + async def test_assert_time_exceeds(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block # 5 seconds should be before the next block time_now = full_node_1.full_node.blockchain.get_peak().timestamp + 5 cvp = ConditionWithArgs(ConditionOpcode.ASSERT_SECONDS_ABSOLUTE, [int_to_bytes(time_now)]) dic = {cvp.opcode: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) assert err is None assert sb1 is spend_bundle1 assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_assert_time_fail(self, bt, two_nodes_mempool, wallet_a): + async def test_assert_time_fail(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block time_now = full_node_1.full_node.blockchain.get_peak().timestamp + 1000 cvp = ConditionWithArgs(ConditionOpcode.ASSERT_SECONDS_ABSOLUTE, [int_to_bytes(time_now)]) dic = {cvp.opcode: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) assert err == Err.ASSERT_SECONDS_ABSOLUTE_FAILED assert sb1 is None assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_assert_height_pending(self, bt, two_nodes_mempool, wallet_a): + async def test_assert_height_pending(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block print(full_node_1.full_node.blockchain.get_peak()) current_height = full_node_1.full_node.blockchain.get_peak().height cvp = ConditionWithArgs(ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE, [int_to_bytes(current_height + 4)]) dic = {cvp.opcode: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) assert err == Err.ASSERT_HEIGHT_ABSOLUTE_FAILED assert sb1 is None assert status == MempoolInclusionStatus.PENDING @pytest.mark.asyncio - async def test_assert_time_negative(self, bt, two_nodes_mempool, wallet_a): + async def test_assert_time_negative(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block time_now = -1 cvp = ConditionWithArgs(ConditionOpcode.ASSERT_SECONDS_ABSOLUTE, [int_to_bytes(time_now)]) dic = {cvp.opcode: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) assert err is None assert sb1 is spend_bundle1 assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_assert_time_missing_arg(self, bt, two_nodes_mempool, wallet_a): + async def test_assert_time_missing_arg(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block cvp = ConditionWithArgs(ConditionOpcode.ASSERT_SECONDS_ABSOLUTE, []) dic = {cvp.opcode: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) assert err == Err.INVALID_CONDITION assert sb1 is None assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_assert_time_garbage(self, bt, two_nodes_mempool, wallet_a): + async def test_assert_time_garbage(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block time_now = full_node_1.full_node.blockchain.get_peak().timestamp + 5 # garbage at the end of the argument list is ignored cvp = ConditionWithArgs(ConditionOpcode.ASSERT_SECONDS_ABSOLUTE, [int_to_bytes(time_now), b"garbage"]) dic = {cvp.opcode: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) assert err is None assert sb1 is spend_bundle1 assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_assert_time_relative_exceeds(self, bt, two_nodes_mempool, wallet_a): + async def test_assert_time_relative_exceeds(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block time_relative = 3 cvp = ConditionWithArgs(ConditionOpcode.ASSERT_SECONDS_RELATIVE, [int_to_bytes(time_relative)]) dic = {cvp.opcode: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) assert err == Err.ASSERT_SECONDS_RELATIVE_FAILED @@ -930,15 +922,15 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_assert_time_relative_garbage(self, bt, two_nodes_mempool, wallet_a): + async def test_assert_time_relative_garbage(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block time_relative = 0 # garbage at the end of the arguments is ignored cvp = ConditionWithArgs(ConditionOpcode.ASSERT_SECONDS_RELATIVE, [int_to_bytes(time_relative), b"garbage"]) dic = {cvp.opcode: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) assert err is None @@ -946,13 +938,13 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_assert_time_relative_missing_arg(self, bt, two_nodes_mempool, wallet_a): + async def test_assert_time_relative_missing_arg(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block cvp = ConditionWithArgs(ConditionOpcode.ASSERT_SECONDS_RELATIVE, []) dic = {cvp.opcode: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) assert err == Err.INVALID_CONDITION @@ -960,14 +952,14 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_assert_time_relative_negative(self, bt, two_nodes_mempool, wallet_a): + async def test_assert_time_relative_negative(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block time_relative = -3 cvp = ConditionWithArgs(ConditionOpcode.ASSERT_SECONDS_RELATIVE, [int_to_bytes(time_relative)]) dic = {cvp.opcode: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) assert err is None @@ -976,7 +968,7 @@ class TestMempoolManager: # ensure one spend can assert a coin announcement from another spend @pytest.mark.asyncio - async def test_correct_coin_announcement_consumed(self, bt, two_nodes_mempool, wallet_a): + async def test_correct_coin_announcement_consumed(self, bt, one_node_one_block, wallet_a): def test_fun(coin_1: Coin, coin_2: Coin) -> SpendBundle: announce = Announcement(coin_2.name(), b"test") cvp = ConditionWithArgs(ConditionOpcode.ASSERT_COIN_ANNOUNCEMENT, [announce.name()]) @@ -989,8 +981,8 @@ class TestMempoolManager: bundle = SpendBundle.aggregate([spend_bundle1, spend_bundle2]) return bundle - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool - blocks, bundle, status, err = await self.condition_tester2(bt, two_nodes_mempool, wallet_a, test_fun) + full_node_1, server_1 = one_node_one_block + blocks, bundle, status, err = await self.condition_tester2(bt, one_node_one_block, wallet_a, test_fun) mempool_bundle = full_node_1.full_node.mempool_manager.get_spendbundle(bundle.name()) assert err is None @@ -1000,7 +992,7 @@ class TestMempoolManager: # ensure one spend can assert a coin announcement from another spend, even # though the conditions have garbage (ignored) at the end @pytest.mark.asyncio - async def test_coin_announcement_garbage(self, bt, two_nodes_mempool, wallet_a): + async def test_coin_announcement_garbage(self, bt, one_node_one_block, wallet_a): def test_fun(coin_1: Coin, coin_2: Coin) -> SpendBundle: announce = Announcement(coin_2.name(), b"test") # garbage at the end is ignored @@ -1015,8 +1007,8 @@ class TestMempoolManager: bundle = SpendBundle.aggregate([spend_bundle1, spend_bundle2]) return bundle - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool - blocks, bundle, status, err = await self.condition_tester2(bt, two_nodes_mempool, wallet_a, test_fun) + full_node_1, server_1 = one_node_one_block + blocks, bundle, status, err = await self.condition_tester2(bt, one_node_one_block, wallet_a, test_fun) mempool_bundle = full_node_1.full_node.mempool_manager.get_spendbundle(bundle.name()) assert err is None @@ -1024,8 +1016,8 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_coin_announcement_missing_arg(self, bt, two_nodes_mempool, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + async def test_coin_announcement_missing_arg(self, bt, one_node_one_block, wallet_a): + full_node_1, server_1 = one_node_one_block def test_fun(coin_1: Coin, coin_2: Coin): # missing arg here @@ -1038,15 +1030,15 @@ class TestMempoolManager: return SpendBundle.aggregate([spend_bundle1, spend_bundle2]) - blocks, bundle, status, err = await self.condition_tester2(bt, two_nodes_mempool, wallet_a, test_fun) + blocks, bundle, status, err = await self.condition_tester2(bt, one_node_one_block, wallet_a, test_fun) assert err == Err.INVALID_CONDITION assert full_node_1.full_node.mempool_manager.get_spendbundle(bundle.name()) is None assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_coin_announcement_missing_arg2(self, bt, two_nodes_mempool, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + async def test_coin_announcement_missing_arg2(self, bt, one_node_one_block, wallet_a): + full_node_1, server_1 = one_node_one_block def test_fun(coin_1: Coin, coin_2: Coin): announce = Announcement(coin_2.name(), b"test") @@ -1060,15 +1052,15 @@ class TestMempoolManager: return SpendBundle.aggregate([spend_bundle1, spend_bundle2]) - blocks, bundle, status, err = await self.condition_tester2(bt, two_nodes_mempool, wallet_a, test_fun) + blocks, bundle, status, err = await self.condition_tester2(bt, one_node_one_block, wallet_a, test_fun) assert err == Err.INVALID_CONDITION assert full_node_1.full_node.mempool_manager.get_spendbundle(bundle.name()) is None assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_coin_announcement_too_big(self, bt, two_nodes_mempool, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + async def test_coin_announcement_too_big(self, bt, one_node_one_block, wallet_a): + full_node_1, server_1 = one_node_one_block def test_fun(coin_1: Coin, coin_2: Coin): announce = Announcement(coin_2.name(), bytes([1] * 10000)) @@ -1084,7 +1076,7 @@ class TestMempoolManager: return SpendBundle.aggregate([spend_bundle1, spend_bundle2]) - blocks, bundle, status, err = await self.condition_tester2(bt, two_nodes_mempool, wallet_a, test_fun) + blocks, bundle, status, err = await self.condition_tester2(bt, one_node_one_block, wallet_a, test_fun) assert err == Err.ASSERT_ANNOUNCE_CONSUMED_FAILED assert full_node_1.full_node.mempool_manager.get_spendbundle(bundle.name()) is None @@ -1102,8 +1094,8 @@ class TestMempoolManager: # ensure an assert coin announcement is rejected if it doesn't match the # create announcement @pytest.mark.asyncio - async def test_invalid_coin_announcement_rejected(self, bt, two_nodes_mempool, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + async def test_invalid_coin_announcement_rejected(self, bt, one_node_one_block, wallet_a): + full_node_1, server_1 = one_node_one_block def test_fun(coin_1: Coin, coin_2: Coin): announce = Announcement(coin_2.name(), b"test") @@ -1122,7 +1114,7 @@ class TestMempoolManager: return SpendBundle.aggregate([spend_bundle1, spend_bundle2]) - blocks, bundle, status, err = await self.condition_tester2(bt, two_nodes_mempool, wallet_a, test_fun) + blocks, bundle, status, err = await self.condition_tester2(bt, one_node_one_block, wallet_a, test_fun) mempool_bundle = full_node_1.full_node.mempool_manager.get_spendbundle(bundle.name()) @@ -1131,8 +1123,8 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_invalid_coin_announcement_rejected_two(self, bt, two_nodes_mempool, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + async def test_invalid_coin_announcement_rejected_two(self, bt, one_node_one_block, wallet_a): + full_node_1, server_1 = one_node_one_block def test_fun(coin_1: Coin, coin_2: Coin): announce = Announcement(coin_1.name(), b"test") @@ -1149,7 +1141,7 @@ class TestMempoolManager: return SpendBundle.aggregate([spend_bundle1, spend_bundle2]) - blocks, bundle, status, err = await self.condition_tester2(bt, two_nodes_mempool, wallet_a, test_fun) + blocks, bundle, status, err = await self.condition_tester2(bt, one_node_one_block, wallet_a, test_fun) mempool_bundle = full_node_1.full_node.mempool_manager.get_spendbundle(bundle.name()) assert err == Err.ASSERT_ANNOUNCE_CONSUMED_FAILED @@ -1157,8 +1149,8 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_correct_puzzle_announcement(self, bt, two_nodes_mempool, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + async def test_correct_puzzle_announcement(self, bt, one_node_one_block, wallet_a): + full_node_1, server_1 = one_node_one_block def test_fun(coin_1: Coin, coin_2: Coin): announce = Announcement(coin_2.puzzle_hash, bytes(0x80)) @@ -1174,7 +1166,7 @@ class TestMempoolManager: return SpendBundle.aggregate([spend_bundle1, spend_bundle2]) - blocks, bundle, status, err = await self.condition_tester2(bt, two_nodes_mempool, wallet_a, test_fun) + blocks, bundle, status, err = await self.condition_tester2(bt, one_node_one_block, wallet_a, test_fun) mempool_bundle = full_node_1.full_node.mempool_manager.get_spendbundle(bundle.name()) @@ -1183,8 +1175,8 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_puzzle_announcement_garbage(self, bt, two_nodes_mempool, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + async def test_puzzle_announcement_garbage(self, bt, one_node_one_block, wallet_a): + full_node_1, server_1 = one_node_one_block def test_fun(coin_1: Coin, coin_2: Coin): announce = Announcement(coin_2.puzzle_hash, bytes(0x80)) @@ -1200,7 +1192,7 @@ class TestMempoolManager: return SpendBundle.aggregate([spend_bundle1, spend_bundle2]) - blocks, bundle, status, err = await self.condition_tester2(bt, two_nodes_mempool, wallet_a, test_fun) + blocks, bundle, status, err = await self.condition_tester2(bt, one_node_one_block, wallet_a, test_fun) mempool_bundle = full_node_1.full_node.mempool_manager.get_spendbundle(bundle.name()) assert err is None @@ -1208,8 +1200,8 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_puzzle_announcement_missing_arg(self, bt, two_nodes_mempool, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + async def test_puzzle_announcement_missing_arg(self, bt, one_node_one_block, wallet_a): + full_node_1, server_1 = one_node_one_block def test_fun(coin_1: Coin, coin_2: Coin): # missing arg here @@ -1225,7 +1217,7 @@ class TestMempoolManager: return SpendBundle.aggregate([spend_bundle1, spend_bundle2]) - blocks, bundle, status, err = await self.condition_tester2(bt, two_nodes_mempool, wallet_a, test_fun) + blocks, bundle, status, err = await self.condition_tester2(bt, one_node_one_block, wallet_a, test_fun) mempool_bundle = full_node_1.full_node.mempool_manager.get_spendbundle(bundle.name()) @@ -1234,8 +1226,8 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_puzzle_announcement_missing_arg2(self, bt, two_nodes_mempool, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + async def test_puzzle_announcement_missing_arg2(self, bt, one_node_one_block, wallet_a): + full_node_1, server_1 = one_node_one_block def test_fun(coin_1: Coin, coin_2: Coin): announce = Announcement(coin_2.puzzle_hash, b"test") @@ -1253,7 +1245,7 @@ class TestMempoolManager: return SpendBundle.aggregate([spend_bundle1, spend_bundle2]) - blocks, bundle, status, err = await self.condition_tester2(bt, two_nodes_mempool, wallet_a, test_fun) + blocks, bundle, status, err = await self.condition_tester2(bt, one_node_one_block, wallet_a, test_fun) mempool_bundle = full_node_1.full_node.mempool_manager.get_spendbundle(bundle.name()) @@ -1262,8 +1254,8 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_invalid_puzzle_announcement_rejected(self, bt, two_nodes_mempool, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + async def test_invalid_puzzle_announcement_rejected(self, bt, one_node_one_block, wallet_a): + full_node_1, server_1 = one_node_one_block def test_fun(coin_1: Coin, coin_2: Coin): announce = Announcement(coin_2.puzzle_hash, bytes("test", "utf-8")) @@ -1282,7 +1274,7 @@ class TestMempoolManager: return SpendBundle.aggregate([spend_bundle1, spend_bundle2]) - blocks, bundle, status, err = await self.condition_tester2(bt, two_nodes_mempool, wallet_a, test_fun) + blocks, bundle, status, err = await self.condition_tester2(bt, one_node_one_block, wallet_a, test_fun) mempool_bundle = full_node_1.full_node.mempool_manager.get_spendbundle(bundle.name()) @@ -1291,8 +1283,8 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_invalid_puzzle_announcement_rejected_two(self, bt, two_nodes_mempool, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + async def test_invalid_puzzle_announcement_rejected_two(self, bt, one_node_one_block, wallet_a): + full_node_1, server_1 = one_node_one_block def test_fun(coin_1: Coin, coin_2: Coin): announce = Announcement(coin_2.puzzle_hash, b"test") @@ -1311,7 +1303,7 @@ class TestMempoolManager: return SpendBundle.aggregate([spend_bundle1, spend_bundle2]) - blocks, bundle, status, err = await self.condition_tester2(bt, two_nodes_mempool, wallet_a, test_fun) + blocks, bundle, status, err = await self.condition_tester2(bt, one_node_one_block, wallet_a, test_fun) mempool_bundle = full_node_1.full_node.mempool_manager.get_spendbundle(bundle.name()) @@ -1320,13 +1312,13 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_assert_fee_condition(self, bt, two_nodes_mempool, wallet_a): + async def test_assert_fee_condition(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block cvp = ConditionWithArgs(ConditionOpcode.RESERVE_FEE, [int_to_bytes(10)]) dic = {cvp.opcode: [cvp]} blocks, spend_bundle1, peer, status, err = await self.condition_tester( - bt, two_nodes_mempool, wallet_a, dic, fee=10 + bt, one_node_one_block, wallet_a, dic, fee=10 ) mempool_bundle = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -1335,14 +1327,14 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_assert_fee_condition_garbage(self, bt, two_nodes_mempool, wallet_a): + async def test_assert_fee_condition_garbage(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block # garbage at the end of the arguments is ignored cvp = ConditionWithArgs(ConditionOpcode.RESERVE_FEE, [int_to_bytes(10), b"garbage"]) dic = {cvp.opcode: [cvp]} blocks, spend_bundle1, peer, status, err = await self.condition_tester( - bt, two_nodes_mempool, wallet_a, dic, fee=10 + bt, one_node_one_block, wallet_a, dic, fee=10 ) mempool_bundle = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -1351,23 +1343,23 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_assert_fee_condition_missing_arg(self, bt, two_nodes_mempool, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + async def test_assert_fee_condition_missing_arg(self, bt, one_node_one_block, wallet_a): + full_node_1, server_1 = one_node_one_block cvp = ConditionWithArgs(ConditionOpcode.RESERVE_FEE, []) dic = {cvp.opcode: [cvp]} blocks, spend_bundle1, peer, status, err = await self.condition_tester( - bt, two_nodes_mempool, wallet_a, dic, fee=10 + bt, one_node_one_block, wallet_a, dic, fee=10 ) assert err == Err.INVALID_CONDITION assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_assert_fee_condition_negative_fee(self, bt, two_nodes_mempool, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + async def test_assert_fee_condition_negative_fee(self, bt, one_node_one_block, wallet_a): + full_node_1, server_1 = one_node_one_block cvp = ConditionWithArgs(ConditionOpcode.RESERVE_FEE, [int_to_bytes(-1)]) dic = {cvp.opcode: [cvp]} blocks, spend_bundle1, peer, status, err = await self.condition_tester( - bt, two_nodes_mempool, wallet_a, dic, fee=10 + bt, one_node_one_block, wallet_a, dic, fee=10 ) assert err == Err.RESERVE_FEE_CONDITION_FAILED assert status == MempoolInclusionStatus.FAILED @@ -1380,12 +1372,12 @@ class TestMempoolManager: ) @pytest.mark.asyncio - async def test_assert_fee_condition_fee_too_large(self, bt, two_nodes_mempool, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + async def test_assert_fee_condition_fee_too_large(self, bt, one_node_one_block, wallet_a): + full_node_1, server_1 = one_node_one_block cvp = ConditionWithArgs(ConditionOpcode.RESERVE_FEE, [int_to_bytes(2 ** 64)]) dic = {cvp.opcode: [cvp]} blocks, spend_bundle1, peer, status, err = await self.condition_tester( - bt, two_nodes_mempool, wallet_a, dic, fee=10 + bt, one_node_one_block, wallet_a, dic, fee=10 ) assert err == Err.RESERVE_FEE_CONDITION_FAILED assert status == MempoolInclusionStatus.FAILED @@ -1398,14 +1390,14 @@ class TestMempoolManager: ) @pytest.mark.asyncio - async def test_assert_fee_condition_wrong_fee(self, bt, two_nodes_mempool, wallet_a): + async def test_assert_fee_condition_wrong_fee(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block cvp = ConditionWithArgs(ConditionOpcode.RESERVE_FEE, [int_to_bytes(10)]) dic = {cvp.opcode: [cvp]} blocks, spend_bundle1, peer, status, err = await self.condition_tester( - bt, two_nodes_mempool, wallet_a, dic, fee=9 + bt, one_node_one_block, wallet_a, dic, fee=9 ) mempool_bundle = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -1414,9 +1406,9 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_stealing_fee(self, bt, two_nodes_mempool, wallet_a): + async def test_stealing_fee(self, bt, two_nodes_one_block, wallet_a): reward_ph = wallet_a.get_new_puzzlehash() - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, full_node_2, server_1, server_2 = two_nodes_one_block blocks = await full_node_1.get_all_full_blocks() start_height = blocks[-1].height blocks = bt.get_consecutive_blocks( @@ -1427,7 +1419,6 @@ class TestMempoolManager: pool_reward_puzzle_hash=reward_ph, ) - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool peer = await connect_and_get_peer(server_1, server_2, bt.config["self_hostname"]) for block in blocks: @@ -1471,9 +1462,9 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_double_spend_same_bundle(self, bt, two_nodes_mempool, wallet_a): + async def test_double_spend_same_bundle(self, bt, two_nodes_one_block, wallet_a): reward_ph = wallet_a.get_new_puzzlehash() - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, full_node_2, server_1, server_2 = two_nodes_one_block blocks = await full_node_1.get_all_full_blocks() start_height = blocks[-1].height blocks = bt.get_consecutive_blocks( @@ -1517,9 +1508,9 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_agg_sig_condition(self, bt, two_nodes_mempool, wallet_a): + async def test_agg_sig_condition(self, bt, one_node_one_block, wallet_a): reward_ph = wallet_a.get_new_puzzlehash() - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block blocks = await full_node_1.get_all_full_blocks() start_height = blocks[-1].height blocks = bt.get_consecutive_blocks( @@ -1565,14 +1556,17 @@ class TestMempoolManager: # assert sb is spend_bundle @pytest.mark.asyncio - async def test_correct_my_parent(self, bt, two_nodes_mempool, wallet_a): + async def test_correct_my_parent(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block + + _ = await next_block(full_node_1, wallet_a, bt) + _ = await next_block(full_node_1, wallet_a, bt) coin = await next_block(full_node_1, wallet_a, bt) cvp = ConditionWithArgs(ConditionOpcode.ASSERT_MY_PARENT_ID, [coin.parent_coin_info]) dic = {cvp.opcode: [cvp]} blocks, spend_bundle1, peer, status, err = await self.condition_tester( - bt, two_nodes_mempool, wallet_a, dic, coin=coin + bt, one_node_one_block, wallet_a, dic, coin=coin ) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -1582,15 +1576,18 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_my_parent_garbage(self, bt, two_nodes_mempool, wallet_a): + async def test_my_parent_garbage(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block + + _ = await next_block(full_node_1, wallet_a, bt) + _ = await next_block(full_node_1, wallet_a, bt) coin = await next_block(full_node_1, wallet_a, bt) # garbage at the end of the arguments list is allowed but stripped cvp = ConditionWithArgs(ConditionOpcode.ASSERT_MY_PARENT_ID, [coin.parent_coin_info, b"garbage"]) dic = {cvp.opcode: [cvp]} blocks, spend_bundle1, peer, status, err = await self.condition_tester( - bt, two_nodes_mempool, wallet_a, dic, coin=coin + bt, one_node_one_block, wallet_a, dic, coin=coin ) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -1600,13 +1597,13 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_my_parent_missing_arg(self, bt, two_nodes_mempool, wallet_a): + async def test_my_parent_missing_arg(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block blocks = await full_node_1.get_all_full_blocks() cvp = ConditionWithArgs(ConditionOpcode.ASSERT_MY_PARENT_ID, []) dic = {cvp.opcode: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -1615,15 +1612,18 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_invalid_my_parent(self, bt, two_nodes_mempool, wallet_a): + async def test_invalid_my_parent(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block + + _ = await next_block(full_node_1, wallet_a, bt) + _ = await next_block(full_node_1, wallet_a, bt) coin = await next_block(full_node_1, wallet_a, bt) coin_2 = await next_block(full_node_1, wallet_a, bt) cvp = ConditionWithArgs(ConditionOpcode.ASSERT_MY_PARENT_ID, [coin_2.parent_coin_info]) dic = {cvp.opcode: [cvp]} blocks, spend_bundle1, peer, status, err = await self.condition_tester( - bt, two_nodes_mempool, wallet_a, dic, coin=coin + bt, one_node_one_block, wallet_a, dic, coin=coin ) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -1633,14 +1633,17 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_correct_my_puzhash(self, bt, two_nodes_mempool, wallet_a): + async def test_correct_my_puzhash(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block + + _ = await next_block(full_node_1, wallet_a, bt) + _ = await next_block(full_node_1, wallet_a, bt) coin = await next_block(full_node_1, wallet_a, bt) cvp = ConditionWithArgs(ConditionOpcode.ASSERT_MY_PUZZLEHASH, [coin.puzzle_hash]) dic = {cvp.opcode: [cvp]} blocks, spend_bundle1, peer, status, err = await self.condition_tester( - bt, two_nodes_mempool, wallet_a, dic, coin=coin + bt, one_node_one_block, wallet_a, dic, coin=coin ) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -1650,15 +1653,18 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_my_puzhash_garbage(self, bt, two_nodes_mempool, wallet_a): + async def test_my_puzhash_garbage(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block + + _ = await next_block(full_node_1, wallet_a, bt) + _ = await next_block(full_node_1, wallet_a, bt) coin = await next_block(full_node_1, wallet_a, bt) # garbage at the end of the arguments list is allowed but stripped cvp = ConditionWithArgs(ConditionOpcode.ASSERT_MY_PUZZLEHASH, [coin.puzzle_hash, b"garbage"]) dic = {cvp.opcode: [cvp]} blocks, spend_bundle1, peer, status, err = await self.condition_tester( - bt, two_nodes_mempool, wallet_a, dic, coin=coin + bt, one_node_one_block, wallet_a, dic, coin=coin ) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -1668,13 +1674,13 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_my_puzhash_missing_arg(self, bt, two_nodes_mempool, wallet_a): + async def test_my_puzhash_missing_arg(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block blocks = await full_node_1.get_all_full_blocks() cvp = ConditionWithArgs(ConditionOpcode.ASSERT_MY_PUZZLEHASH, []) dic = {cvp.opcode: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -1683,14 +1689,17 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_invalid_my_puzhash(self, bt, two_nodes_mempool, wallet_a): + async def test_invalid_my_puzhash(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block + + _ = await next_block(full_node_1, wallet_a, bt) + _ = await next_block(full_node_1, wallet_a, bt) coin = await next_block(full_node_1, wallet_a, bt) cvp = ConditionWithArgs(ConditionOpcode.ASSERT_MY_PUZZLEHASH, [Program.to([]).get_tree_hash()]) dic = {cvp.opcode: [cvp]} blocks, spend_bundle1, peer, status, err = await self.condition_tester( - bt, two_nodes_mempool, wallet_a, dic, coin=coin + bt, one_node_one_block, wallet_a, dic, coin=coin ) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -1700,14 +1709,17 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_correct_my_amount(self, bt, two_nodes_mempool, wallet_a): + async def test_correct_my_amount(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block + + _ = await next_block(full_node_1, wallet_a, bt) + _ = await next_block(full_node_1, wallet_a, bt) coin = await next_block(full_node_1, wallet_a, bt) cvp = ConditionWithArgs(ConditionOpcode.ASSERT_MY_AMOUNT, [int_to_bytes(coin.amount)]) dic = {cvp.opcode: [cvp]} blocks, spend_bundle1, peer, status, err = await self.condition_tester( - bt, two_nodes_mempool, wallet_a, dic, coin=coin + bt, one_node_one_block, wallet_a, dic, coin=coin ) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -1717,15 +1729,18 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_my_amount_garbage(self, bt, two_nodes_mempool, wallet_a): + async def test_my_amount_garbage(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block + + _ = await next_block(full_node_1, wallet_a, bt) + _ = await next_block(full_node_1, wallet_a, bt) coin = await next_block(full_node_1, wallet_a, bt) # garbage at the end of the arguments list is allowed but stripped cvp = ConditionWithArgs(ConditionOpcode.ASSERT_MY_AMOUNT, [int_to_bytes(coin.amount), b"garbage"]) dic = {cvp.opcode: [cvp]} blocks, spend_bundle1, peer, status, err = await self.condition_tester( - bt, two_nodes_mempool, wallet_a, dic, coin=coin + bt, one_node_one_block, wallet_a, dic, coin=coin ) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -1735,13 +1750,13 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.SUCCESS @pytest.mark.asyncio - async def test_my_amount_missing_arg(self, bt, two_nodes_mempool, wallet_a): + async def test_my_amount_missing_arg(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block blocks = await full_node_1.get_all_full_blocks() cvp = ConditionWithArgs(ConditionOpcode.ASSERT_MY_AMOUNT, []) dic = {cvp.opcode: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -1750,13 +1765,13 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_invalid_my_amount(self, bt, two_nodes_mempool, wallet_a): + async def test_invalid_my_amount(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block blocks = await full_node_1.get_all_full_blocks() cvp = ConditionWithArgs(ConditionOpcode.ASSERT_MY_AMOUNT, [int_to_bytes(1000)]) dic = {cvp.opcode: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -1765,13 +1780,13 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_negative_my_amount(self, bt, two_nodes_mempool, wallet_a): + async def test_negative_my_amount(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block blocks = await full_node_1.get_all_full_blocks() cvp = ConditionWithArgs(ConditionOpcode.ASSERT_MY_AMOUNT, [int_to_bytes(-1)]) dic = {cvp.opcode: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -1780,13 +1795,13 @@ class TestMempoolManager: assert status == MempoolInclusionStatus.FAILED @pytest.mark.asyncio - async def test_my_amount_too_large(self, bt, two_nodes_mempool, wallet_a): + async def test_my_amount_too_large(self, bt, one_node_one_block, wallet_a): - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block blocks = await full_node_1.get_all_full_blocks() cvp = ConditionWithArgs(ConditionOpcode.ASSERT_MY_AMOUNT, [int_to_bytes(2 ** 64)]) dic = {cvp.opcode: [cvp]} - blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, two_nodes_mempool, wallet_a, dic) + blocks, spend_bundle1, peer, status, err = await self.condition_tester(bt, one_node_one_block, wallet_a, dic) sb1 = full_node_1.full_node.mempool_manager.get_spendbundle(spend_bundle1.name()) @@ -2409,7 +2424,7 @@ class TestMaliciousGenerators: assert run_time < 0.2 @pytest.mark.asyncio - async def test_invalid_coin_spend_coin(self, bt, two_nodes_mempool, wallet_a): + async def test_invalid_coin_spend_coin(self, bt, one_node_one_block, wallet_a): reward_ph = wallet_a.get_new_puzzlehash() blocks = bt.get_consecutive_blocks( 5, @@ -2417,7 +2432,7 @@ class TestMaliciousGenerators: farmer_reward_puzzle_hash=reward_ph, pool_reward_puzzle_hash=reward_ph, ) - full_node_1, full_node_2, server_1, server_2 = two_nodes_mempool + full_node_1, server_1 = one_node_one_block for block in blocks: await full_node_1.full_node.respond_block(full_node_protocol.RespondBlock(block)) From 22edd733313cdd1b679abf64400c5d52801a38e8 Mon Sep 17 00:00:00 2001 From: Adam Kelly <338792+aqk@users.noreply.github.com> Date: Mon, 4 Apr 2022 17:43:31 -0700 Subject: [PATCH 25/63] Force apt to install the things we asked it to (#11047) * Force apt to install the things we asked it to * Update .github/workflows/benchmarks.yml Co-authored-by: dustinface <35775977+xdustinface@users.noreply.github.com> Co-authored-by: dustinface <35775977+xdustinface@users.noreply.github.com> --- .github/workflows/benchmarks.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 90bea4ef84..0dc6ff3bbe 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -51,11 +51,13 @@ jobs: ${{ runner.os }}-pip- - name: Install ubuntu dependencies + env: + DEBIAN_FRONTEND: noninteractive run: | - sudo apt-get install software-properties-common + sudo apt-get install -y software-properties-common sudo add-apt-repository ppa:deadsnakes/ppa sudo apt-get update - sudo apt-get install python${{ matrix.python-version }}-venv python${{ matrix.python-version }}-distutils git -y + sudo apt-get install -y python${{ matrix.python-version }}-venv python${{ matrix.python-version }}-distutils git - name: Run install script env: From 4a4b14b78b17773566cf4afd21fca98daa2981e3 Mon Sep 17 00:00:00 2001 From: dustinface <35775977+xdustinface@users.noreply.github.com> Date: Tue, 5 Apr 2022 03:43:51 +0200 Subject: [PATCH 26/63] github: Drop unused `BUILD_VDF_CLIENT` variables (#11050) From my understanding this is only used by `chiavdf` source builds which happen only if `install-timelord.sh` gets called but it doesn't in the addressed cases. --- .github/workflows/build-linux-arm64-installer.yml | 1 - .github/workflows/build-linux-installer-deb.yml | 1 - .github/workflows/build-linux-installer-rpm.yml | 1 - .github/workflows/build-macos-installer.yml | 1 - .github/workflows/build-macos-m1-installer.yml | 1 - .github/workflows/test-install-scripts.yml | 2 -- 6 files changed, 7 deletions(-) diff --git a/.github/workflows/build-linux-arm64-installer.yml b/.github/workflows/build-linux-arm64-installer.yml index fad14fa525..6b05a2560e 100644 --- a/.github/workflows/build-linux-arm64-installer.yml +++ b/.github/workflows/build-linux-arm64-installer.yml @@ -105,7 +105,6 @@ jobs: - name: Run install script env: INSTALL_PYTHON_VERSION: ${{ matrix.python-version }} - BUILD_VDF_CLIENT: "N" run: | sh install.sh diff --git a/.github/workflows/build-linux-installer-deb.yml b/.github/workflows/build-linux-installer-deb.yml index 5734d6204f..e9690e8bfa 100644 --- a/.github/workflows/build-linux-installer-deb.yml +++ b/.github/workflows/build-linux-installer-deb.yml @@ -139,7 +139,6 @@ jobs: - name: Run install script env: INSTALL_PYTHON_VERSION: ${{ matrix.python-version }} - BUILD_VDF_CLIENT: "N" run: | sh install.sh diff --git a/.github/workflows/build-linux-installer-rpm.yml b/.github/workflows/build-linux-installer-rpm.yml index 5564b472e5..aa9c85ed83 100644 --- a/.github/workflows/build-linux-installer-rpm.yml +++ b/.github/workflows/build-linux-installer-rpm.yml @@ -108,7 +108,6 @@ jobs: - name: Run install script env: INSTALL_PYTHON_VERSION: ${{ matrix.python-version }} - BUILD_VDF_CLIENT: "N" run: | sh install.sh diff --git a/.github/workflows/build-macos-installer.yml b/.github/workflows/build-macos-installer.yml index f9d3fc3fae..da60f6f20a 100644 --- a/.github/workflows/build-macos-installer.yml +++ b/.github/workflows/build-macos-installer.yml @@ -127,7 +127,6 @@ jobs: - name: Run install script env: INSTALL_PYTHON_VERSION: ${{ matrix.python-version }} - BUILD_VDF_CLIENT: "N" run: | sh install.sh diff --git a/.github/workflows/build-macos-m1-installer.yml b/.github/workflows/build-macos-m1-installer.yml index 7a178382e3..18d08db16d 100644 --- a/.github/workflows/build-macos-m1-installer.yml +++ b/.github/workflows/build-macos-m1-installer.yml @@ -101,7 +101,6 @@ jobs: - name: Run install script env: INSTALL_PYTHON_VERSION: ${{ matrix.python-version }} - BUILD_VDF_CLIENT: "N" run: | arch -arm64 sh install.sh diff --git a/.github/workflows/test-install-scripts.yml b/.github/workflows/test-install-scripts.yml index 90917ac0f4..9b2a7e8596 100644 --- a/.github/workflows/test-install-scripts.yml +++ b/.github/workflows/test-install-scripts.yml @@ -40,7 +40,6 @@ jobs: - name: Run install script env: INSTALL_PYTHON_VERSION: ${{ matrix.python-version }} - BUILD_VDF_CLIENT: "N" run: sh install.sh - name: Run install-gui script @@ -188,7 +187,6 @@ jobs: - name: Run install script env: INSTALL_PYTHON_VERSION: ${{ matrix.python-version }} - BUILD_VDF_CLIENT: "N" run: sh install.sh -a - name: Run chia --help From 261f5baa6ffecb62130e4118f01dd0329378c076 Mon Sep 17 00:00:00 2001 From: wjblanke Date: Mon, 4 Apr 2022 23:35:28 -0700 Subject: [PATCH 27/63] bump up to 2.1.7 to fix inotify issue resolved by 848 (#11042) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2c55afe278..c746a4cf26 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ dependencies = [ # TODO: when moving to click 8 remove the pinning of black noted below "click==7.1.2", # For the CLI "dnspythonchia==2.2.0", # Query DNS seeds - "watchdog==2.1.6", # Filesystem event watching - watches keyring.yaml + "watchdog==2.1.7", # Filesystem event watching - watches keyring.yaml "dnslib==0.9.17", # dns lib "typing-extensions==4.0.1", # typing backports like Protocol and TypedDict "zstd==1.5.0.4", From b377339372bc95241b40d4e513e61cf4a9f6db2f Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Tue, 5 Apr 2022 17:53:12 +0200 Subject: [PATCH 28/63] fix memory leak in test_full_sync (#11004) --- tools/test_full_sync.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/test_full_sync.py b/tools/test_full_sync.py index e28a9d2f1f..f895647479 100755 --- a/tools/test_full_sync.py +++ b/tools/test_full_sync.py @@ -103,6 +103,8 @@ async def run_sync_test(file: Path, db_version, profile: bool, single_thread: bo success, advanced_peak, fork_height, coin_changes = await full_node.receive_block_batch( block_batch, None, None # type: ignore[arg-type] ) + end_height = block_batch[-1].height + full_node.blockchain.clean_block_record(end_height - full_node.constants.BLOCKS_CACHE_SIZE) assert success assert advanced_peak From 9311e58ac4d6c2e7561088f4f66c5352f1347a94 Mon Sep 17 00:00:00 2001 From: dustinface <35775977+xdustinface@users.noreply.github.com> Date: Tue, 5 Apr 2022 17:53:48 +0200 Subject: [PATCH 29/63] full_node: Drop unused `MempoolManager.constants_json` (#11046) --- chia/full_node/mempool_manager.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/chia/full_node/mempool_manager.py b/chia/full_node/mempool_manager.py index f1a4ba514b..821e2aa1a3 100644 --- a/chia/full_node/mempool_manager.py +++ b/chia/full_node/mempool_manager.py @@ -1,6 +1,5 @@ import asyncio import collections -import dataclasses import logging from concurrent.futures import Executor from multiprocessing.context import BaseContext @@ -34,7 +33,6 @@ from chia.util.generator_tools import additions_for_npc from chia.util.ints import uint32, uint64 from chia.util.lru_cache import LRUCache from chia.util.setproctitle import getproctitle, setproctitle -from chia.util.streamable import recurse_jsonify from chia.full_node.mempool_check_conditions import mempool_check_time_locks log = logging.getLogger(__name__) @@ -91,7 +89,6 @@ class MempoolManager: single_threaded: bool = False, ): self.constants: ConsensusConstants = consensus_constants - self.constants_json = recurse_jsonify(dataclasses.asdict(self.constants)) # Keep track of seen spend_bundles self.seen_bundle_hashes: Dict[bytes32, bytes32] = {} From d87b8ac08766c9cbb4f4577a4819bb592f63123c Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 5 Apr 2022 11:54:38 -0400 Subject: [PATCH 30/63] simplify some header hash getting and assertions (#11007) --- chia/consensus/blockchain.py | 5 ++--- chia/full_node/full_node_api.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/chia/consensus/blockchain.py b/chia/consensus/blockchain.py index f436572bc7..258537d01c 100644 --- a/chia/consensus/blockchain.py +++ b/chia/consensus/blockchain.py @@ -752,9 +752,8 @@ class Blockchain(BlockchainInterface): ) -> Dict[bytes32, HeaderBlock]: hashes = [] for height in range(start, stop + 1): - if self.contains_height(uint32(height)): - header_hash: Optional[bytes32] = self.height_to_hash(uint32(height)) - assert header_hash is not None + header_hash: Optional[bytes32] = self.height_to_hash(uint32(height)) + if header_hash is not None: hashes.append(header_hash) blocks: List[FullBlock] = [] diff --git a/chia/full_node/full_node_api.py b/chia/full_node/full_node_api.py index ee60e44657..ff91b11858 100644 --- a/chia/full_node/full_node_api.py +++ b/chia/full_node/full_node_api.py @@ -1320,12 +1320,11 @@ class FullNodeAPI: header_hashes: List[bytes32] = [] for i in range(request.start_height, request.end_height + 1): - if not self.full_node.blockchain.contains_height(uint32(i)): + header_hash: Optional[bytes32] = self.full_node.blockchain.height_to_hash(uint32(i)) + if header_hash is None: reject = RejectHeaderBlocks(request.start_height, request.end_height) msg = make_msg(ProtocolMessageTypes.reject_header_blocks, reject) return msg - header_hash: Optional[bytes32] = self.full_node.blockchain.height_to_hash(uint32(i)) - assert header_hash is not None header_hashes.append(header_hash) blocks: List[FullBlock] = await self.full_node.block_store.get_blocks_by_hash(header_hashes) From 9b7d7d2555d3728f178ec81bdae3a7d138942389 Mon Sep 17 00:00:00 2001 From: Jack Nelson Date: Tue, 5 Apr 2022 13:19:09 -0400 Subject: [PATCH 31/63] Remove websockets dependency & do some refactoring (#10611) * remove old ws --- chia/daemon/client.py | 66 +++++++--- chia/daemon/server.py | 220 +++++++++++----------------------- chia/farmer/farmer.py | 3 + chia/rpc/rpc_server.py | 70 +++++------ chia/server/server.py | 6 +- chia/wallet/wallet_node.py | 47 ++++---- setup.py | 1 - tests/core/test_daemon_rpc.py | 4 +- tests/setup_nodes.py | 29 +++-- tests/setup_services.py | 6 +- 10 files changed, 205 insertions(+), 247 deletions(-) diff --git a/chia/daemon/client.py b/chia/daemon/client.py index c8ca93019a..88677d6890 100644 --- a/chia/daemon/client.py +++ b/chia/daemon/client.py @@ -5,7 +5,7 @@ from contextlib import asynccontextmanager from pathlib import Path from typing import Any, Dict, Optional -import websockets +import aiohttp from chia.util.config import load_config from chia.util.json_util import dict_to_json_str @@ -13,31 +13,52 @@ from chia.util.ws_message import WsRpcMessage, create_payload_dict class DaemonProxy: - def __init__(self, uri: str, ssl_context: Optional[ssl.SSLContext]): + def __init__( + self, + uri: str, + ssl_context: Optional[ssl.SSLContext], + max_message_size: Optional[int] = 50 * 1000 * 1000, + ): self._uri = uri self._request_dict: Dict[str, asyncio.Event] = {} self.response_dict: Dict[str, Any] = {} self.ssl_context = ssl_context + self.client_session: Optional[aiohttp.ClientSession] = None + self.websocket: Optional[aiohttp.ClientWebSocketResponse] = None + self.max_message_size = max_message_size def format_request(self, command: str, data: Dict[str, Any]) -> WsRpcMessage: request = create_payload_dict(command, data, "client", "daemon") return request async def start(self): - self.websocket = await websockets.connect(self._uri, max_size=None, ssl=self.ssl_context) + try: + self.client_session = aiohttp.ClientSession() + self.websocket = await self.client_session.ws_connect( + self._uri, + autoclose=True, + autoping=True, + heartbeat=60, + ssl_context=self.ssl_context, + max_msg_size=self.max_message_size, + ) + except Exception: + await self.close() + raise async def listener(): while True: - try: - message = await self.websocket.recv() - except websockets.exceptions.ConnectionClosedOK: - return None - decoded = json.loads(message) - id = decoded["request_id"] + message = await self.websocket.receive() + if message.type == aiohttp.WSMsgType.TEXT: + decoded = json.loads(message.data) + request_id = decoded["request_id"] - if id in self._request_dict: - self.response_dict[id] = decoded - self._request_dict[id].set() + if request_id in self._request_dict: + self.response_dict[request_id] = decoded + self._request_dict[request_id].set() + else: + await self.close() + return None asyncio.create_task(listener()) await asyncio.sleep(1) @@ -46,7 +67,9 @@ class DaemonProxy: request_id = request["request_id"] self._request_dict[request_id] = asyncio.Event() string = dict_to_json_str(request) - asyncio.create_task(self.websocket.send(string)) + if self.websocket is None: + raise Exception("Websocket is not connected") + asyncio.create_task(self.websocket.send_str(string)) async def timeout(): await asyncio.sleep(30) @@ -117,19 +140,24 @@ class DaemonProxy: return response async def close(self) -> None: - await self.websocket.close() + if self.websocket is not None: + await self.websocket.close() + if self.client_session is not None: + await self.client_session.close() async def exit(self) -> WsRpcMessage: request = self.format_request("exit", {}) return await self._get(request) -async def connect_to_daemon(self_hostname: str, daemon_port: int, ssl_context: ssl.SSLContext) -> DaemonProxy: +async def connect_to_daemon( + self_hostname: str, daemon_port: int, max_message_size: int, ssl_context: ssl.SSLContext +) -> DaemonProxy: """ Connect to the local daemon. """ - client = DaemonProxy(f"wss://{self_hostname}:{daemon_port}", ssl_context) + client = DaemonProxy(f"wss://{self_hostname}:{daemon_port}", ssl_context, max_message_size) await client.start() return client @@ -143,12 +171,15 @@ async def connect_to_daemon_and_validate(root_path: Path, quiet: bool = False) - try: net_config = load_config(root_path, "config.yaml") + daemon_max_message_size = net_config.get("daemon_max_message_size", 50 * 1000 * 1000) crt_path = root_path / net_config["daemon_ssl"]["private_crt"] key_path = root_path / net_config["daemon_ssl"]["private_key"] ca_crt_path = root_path / net_config["private_ssl_ca"]["crt"] ca_key_path = root_path / net_config["private_ssl_ca"]["key"] ssl_context = ssl_context_for_client(ca_crt_path, ca_key_path, crt_path, key_path) - connection = await connect_to_daemon(net_config["self_hostname"], net_config["daemon_port"], ssl_context) + connection = await connect_to_daemon( + net_config["self_hostname"], net_config["daemon_port"], daemon_max_message_size, ssl_context + ) r = await connection.ping() if "value" in r["data"] and r["data"]["value"] == "pong": @@ -168,7 +199,6 @@ async def acquire_connection_to_daemon(root_path: Path, quiet: bool = False): block exits scope, execution resumes in this function, wherein the connection is closed. """ - from chia.daemon.client import connect_to_daemon_and_validate daemon: Optional[DaemonProxy] = None try: diff --git a/chia/daemon/server.py b/chia/daemon/server.py index c1216c3f12..5281279dbe 100644 --- a/chia/daemon/server.py +++ b/chia/daemon/server.py @@ -14,8 +14,7 @@ from enum import Enum from pathlib import Path from typing import Any, Dict, List, Optional, TextIO, Tuple, cast -from websockets import ConnectionClosedOK, WebSocketException, WebSocketServerProtocol, serve - +from chia import __version__ from chia.cmds.init_funcs import check_keys, chia_init from chia.cmds.passphrase_funcs import default_passphrase, using_default_passphrase from chia.daemon.keychain_server import KeychainServer, keychain_commands @@ -39,12 +38,12 @@ from chia.util.path import mkdir from chia.util.service_groups import validate_service from chia.util.setproctitle import setproctitle from chia.util.ws_message import WsRpcMessage, create_payload, format_response -from chia import __version__ io_pool_exc = ThreadPoolExecutor() try: - from aiohttp import ClientSession, web + from aiohttp import ClientSession, WSMsgType, web + from aiohttp.web_ws import WebSocketResponse except ModuleNotFoundError: print("Error: Make sure to run . ./activate from the project folder before starting Chia.") quit() @@ -134,24 +133,24 @@ class WebSocketServer: ca_key_path: Path, crt_path: Path, key_path: Path, + shutdown_event: asyncio.Event, run_check_keys_on_unlock: bool = False, ): self.root_path = root_path self.log = log self.services: Dict = dict() self.plots_queue: List[Dict] = [] - self.connections: Dict[str, List[WebSocketServerProtocol]] = dict() # service_name : [WebSocket] - self.remote_address_map: Dict[WebSocketServerProtocol, str] = dict() # socket: service_name - self.ping_job: Optional[asyncio.Task] = None + self.connections: Dict[str, List[WebSocketResponse]] = dict() # service_name : [WebSocket] + self.remote_address_map: Dict[WebSocketResponse, str] = dict() # socket: service_name self.net_config = load_config(root_path, "config.yaml") self.self_hostname = self.net_config["self_hostname"] self.daemon_port = self.net_config["daemon_port"] self.daemon_max_message_size = self.net_config.get("daemon_max_message_size", 50 * 1000 * 1000) - self.websocket_server = None + self.websocket_runner: Optional[web.AppRunner] = None self.ssl_context = ssl_context_for_server(ca_crt_path, ca_key_path, crt_path, key_path, log=self.log) - self.shut_down = False self.keychain_server = KeychainServer() self.run_check_keys_on_unlock = run_check_keys_on_unlock + self.shutdown_event = shutdown_event async def start(self): self.log.info("Starting Daemon Server") @@ -184,16 +183,19 @@ class WebSocketServer: except NotImplementedError: self.log.info("Not implemented") - self.websocket_server = await serve( - self.safe_handle, - self.self_hostname, - self.daemon_port, - max_size=self.daemon_max_message_size, - ping_interval=500, - ping_timeout=300, - ssl=self.ssl_context, + app = web.Application(client_max_size=self.daemon_max_message_size) + app.add_routes([web.get("/", self.incoming_connection)]) + self.websocket_runner = web.AppRunner(app, access_log=None, logger=self.log, keepalive_timeout=300) + await self.websocket_runner.setup() + + site = web.TCPSite( + self.websocket_runner, + host=self.self_hostname, + port=self.daemon_port, + shutdown_timeout=3, + ssl_context=self.ssl_context, ) - self.log.info("Waiting Daemon WebSocketServer closure") + await site.start() def cancel_task_safe(self, task: Optional[asyncio.Task]): if task is not None: @@ -203,22 +205,28 @@ class WebSocketServer: self.log.error(f"Error while canceling task.{e} {task}") async def stop(self) -> Dict[str, Any]: - self.shut_down = True - self.cancel_task_safe(self.ping_job) - await self.exit() - if self.websocket_server is not None: - self.websocket_server.close() + jobs = [] + for service_name in self.services.keys(): + jobs.append(kill_service(self.root_path, self.services, service_name)) + if jobs: + await asyncio.wait(jobs) + self.services.clear() + asyncio.create_task(self.exit()) return {"success": True} - async def safe_handle(self, websocket: WebSocketServerProtocol, path: str): - service_name = "" - try: - async for message in websocket: + async def incoming_connection(self, request): + ws: WebSocketResponse = web.WebSocketResponse(max_msg_size=self.daemon_max_message_size, heartbeat=30) + await ws.prepare(request) + + while True: + msg = await ws.receive() + self.log.debug(f"Received message: {msg}") + if msg.type == WSMsgType.TEXT: try: - decoded = json.loads(message) + decoded = json.loads(msg.data) if "data" not in decoded: decoded["data"] = {} - response, sockets_to_use = await self.handle_message(websocket, decoded) + response, sockets_to_use = await self.handle_message(ws, decoded) except Exception as e: tb = traceback.format_exc() self.log.error(f"Error while handling message: {tb}") @@ -228,28 +236,27 @@ class WebSocketServer: if len(sockets_to_use) > 0: for socket in sockets_to_use: try: - await socket.send(response) + await socket.send_str(response) except Exception as e: tb = traceback.format_exc() self.log.error(f"Unexpected exception trying to send to websocket: {e} {tb}") self.remove_connection(socket) await socket.close() - except Exception as e: - tb = traceback.format_exc() - service_name = "Unknown" - if websocket in self.remote_address_map: - service_name = self.remote_address_map[websocket] - if isinstance(e, ConnectionClosedOK): - self.log.info(f"ConnectionClosedOk. Closing websocket with {service_name} {e}") - elif isinstance(e, WebSocketException): - self.log.info(f"Websocket exception. Closing websocket with {service_name} {e} {tb}") + break else: - self.log.error(f"Unexpected exception in websocket: {e} {tb}") - finally: - self.remove_connection(websocket) - await websocket.close() + service_name = "Unknown" + if ws in self.remote_address_map: + service_name = self.remote_address_map[ws] + if msg.type == WSMsgType.CLOSE: + self.log.info(f"ConnectionClosed. Closing websocket with {service_name}") + elif msg.type == WSMsgType.ERROR: + self.log.info(f"Websocket exception. Closing websocket with {service_name}. {ws.exception()}") - def remove_connection(self, websocket: WebSocketServerProtocol): + self.remove_connection(ws) + await ws.close() + break + + def remove_connection(self, websocket: WebSocketResponse): service_name = None if websocket in self.remote_address_map: service_name = self.remote_address_map[websocket] @@ -263,31 +270,8 @@ class WebSocketServer: after_removal.append(connection) self.connections[service_name] = after_removal - async def ping_task(self) -> None: - restart = True - await asyncio.sleep(30) - for remote_address, service_name in self.remote_address_map.items(): - if service_name in self.connections: - sockets = self.connections[service_name] - for socket in sockets: - if socket.remote_address[1] == remote_address: - try: - self.log.info(f"About to ping: {service_name}") - await socket.ping() - except asyncio.CancelledError: - self.log.info("Ping task received Cancel") - restart = False - break - except Exception as e: - self.log.info(f"Ping error: {e}") - self.log.warning("Ping failed, connection closed.") - self.remove_connection(socket) - await socket.close() - if restart is True: - self.ping_job = asyncio.create_task(self.ping_task()) - async def handle_message( - self, websocket: WebSocketServerProtocol, message: WsRpcMessage + self, websocket: WebSocketResponse, message: WsRpcMessage ) -> Tuple[Optional[str], List[Any]]: """ This function gets called when new message is received via websocket. @@ -631,7 +615,7 @@ class WebSocketServer: for websocket in websockets: try: - await websocket.send(response) + await websocket.send_str(response) except Exception as e: tb = traceback.format_exc() self.log.error(f"Unexpected exception trying to send to websocket: {e} {tb}") @@ -690,7 +674,7 @@ class WebSocketServer: for websocket in websockets: try: - await websocket.send(response) + await websocket.send_str(response) except Exception as e: tb = traceback.format_exc() self.log.error(f"Unexpected exception trying to send to websocket: {e} {tb}") @@ -1150,20 +1134,13 @@ class WebSocketServer: return response - async def exit(self) -> Dict[str, Any]: - jobs = [] - for k in self.services.keys(): - jobs.append(kill_service(self.root_path, self.services, k)) - if jobs: - await asyncio.wait(jobs) - self.services.clear() - + async def exit(self) -> None: + if self.websocket_runner is not None: + await self.websocket_runner.cleanup() + self.shutdown_event.set() log.info("chia daemon exiting") - response = {"success": True} - return response - - async def register_service(self, websocket: WebSocketServerProtocol, request: Dict[str, Any]) -> Dict[str, Any]: + async def register_service(self, websocket: WebSocketResponse, request: Dict[str, Any]) -> Dict[str, Any]: self.log.info(f"Register service {request}") service = request["service"] if service not in self.connections: @@ -1179,8 +1156,6 @@ class WebSocketServer: } else: self.remote_address_map[websocket] = service - if self.ping_job is None: - self.ping_job = asyncio.create_task(self.ping_task()) self.log.info(f"registered for service {service}") log.info(f"{response}") return response @@ -1351,7 +1326,6 @@ async def kill_service( if process is None: return False del services[service_name] - result = await kill_process(process, root_path, service_name, "", delay_before_kill) return result @@ -1361,68 +1335,6 @@ def is_running(services: Dict[str, subprocess.Popen], service_name: str) -> bool return process is not None and process.poll() is None -def create_server_for_daemon(root_path: Path): - routes = web.RouteTableDef() - - services: Dict = dict() - - @routes.get("/daemon/ping/") - async def ping(request: web.Request) -> web.Response: - return web.Response(text="pong") - - @routes.get("/daemon/service/start/") - async def start_service(request: web.Request) -> web.Response: - service_name = request.query.get("service") - if service_name is None or not validate_service(service_name): - r = f"{service_name} unknown service" - return web.Response(text=str(r)) - - if is_running(services, service_name): - r = f"{service_name} already running" - return web.Response(text=str(r)) - - try: - process, pid_path = launch_service(root_path, service_name) - services[service_name] = process - r = f"{service_name} started" - except (subprocess.SubprocessError, IOError): - log.exception(f"problem starting {service_name}") - r = f"{service_name} start failed" - - return web.Response(text=str(r)) - - @routes.get("/daemon/service/stop/") - async def stop_service(request: web.Request) -> web.Response: - service_name = request.query.get("service") - if service_name is None: - r = f"{service_name} unknown service" - return web.Response(text=str(r)) - r = str(await kill_service(root_path, services, service_name)) - return web.Response(text=str(r)) - - @routes.get("/daemon/service/is_running/") - async def is_running_handler(request: web.Request) -> web.Response: - service_name = request.query.get("service") - if service_name is None: - r = f"{service_name} unknown service" - return web.Response(text=str(r)) - - r = str(is_running(services, service_name)) - return web.Response(text=str(r)) - - @routes.get("/daemon/exit/") - async def exit(request: web.Request): - jobs = [] - for k in services.keys(): - jobs.append(kill_service(root_path, services, k)) - if jobs: - await asyncio.wait(jobs) - services.clear() - - # we can't await `site.stop()` here because that will cause a deadlock, waiting for this - # request to exit - - def singleton(lockfile: Path, text: str = "semaphore") -> Optional[TextIO]: """ Open a lockfile exclusively. @@ -1474,14 +1386,20 @@ async def async_run_daemon(root_path: Path, wait_for_unlock: bool = False) -> in print("daemon: already launching") return 2 + shutdown_event = asyncio.Event() + # TODO: clean this up, ensuring lockfile isn't removed until the listen port is open - create_server_for_daemon(root_path) ws_server = WebSocketServer( - root_path, ca_crt_path, ca_key_path, crt_path, key_path, run_check_keys_on_unlock=wait_for_unlock + root_path, + ca_crt_path, + ca_key_path, + crt_path, + key_path, + shutdown_event, + run_check_keys_on_unlock=wait_for_unlock, ) await ws_server.start() - assert ws_server.websocket_server is not None - await ws_server.websocket_server.wait_closed() + await shutdown_event.wait() log.info("Daemon WebSocketServer closed") # sys.stdout.close() return 0 diff --git a/chia/farmer/farmer.py b/chia/farmer/farmer.py index 3bb68a6fa5..9979925859 100644 --- a/chia/farmer/farmer.py +++ b/chia/farmer/farmer.py @@ -217,6 +217,9 @@ class Farmer: await self.cache_clear_task if self.update_pool_state_task is not None: await self.update_pool_state_task + if self.keychain_proxy is not None: + await self.keychain_proxy.close() + await asyncio.sleep(0.5) # https://docs.aiohttp.org/en/stable/client_advanced.html#graceful-shutdown self.started = False def _set_state_changed_callback(self, callback: Callable): diff --git a/chia/rpc/rpc_server.py b/chia/rpc/rpc_server.py index dbabcea5fa..4abf169692 100644 --- a/chia/rpc/rpc_server.py +++ b/chia/rpc/rpc_server.py @@ -5,7 +5,7 @@ import traceback from pathlib import Path from typing import Any, Callable, Dict, List, Optional -import aiohttp +from aiohttp import ClientConnectorError, ClientSession, ClientWebSocketResponse, WSMsgType, web from chia.rpc.util import wrap_http_handler from chia.server.outbound_message import NodeType @@ -17,6 +17,7 @@ from chia.util.json_util import dict_to_json_str from chia.util.ws_message import create_payload, create_payload_dict, format_response, pong log = logging.getLogger(__name__) +max_message_size = 50 * 1024 * 1024 # 50MB class RpcServer: @@ -29,7 +30,8 @@ class RpcServer: self.stop_cb: Callable = stop_cb self.log = log self.shut_down = False - self.websocket: Optional[aiohttp.ClientWebSocketResponse] = None + self.websocket: Optional[ClientWebSocketResponse] = None + self.client_session: Optional[ClientSession] = None self.service_name = service_name self.root_path = root_path self.net_config = net_config @@ -45,6 +47,8 @@ class RpcServer: self.shut_down = True if self.websocket is not None: await self.websocket.close() + if self.client_session is not None: + await self.client_session.close() async def _state_changed(self, *args): if self.websocket is None: @@ -168,7 +172,7 @@ class RpcServer: async def close_connection(self, request: Dict): node_id = hexstr_to_bytes(request["node_id"]) if self.rpc_api.service.server is None: - raise aiohttp.web.HTTPInternalServerError() + raise web.HTTPInternalServerError() connections_to_close = [c for c in self.rpc_api.service.server.get_connections() if c.peer_node_id == node_id] if len(connections_to_close) == 0: raise ValueError(f"Connection with node_id {node_id.hex()} does not exist") @@ -243,52 +247,52 @@ class RpcServer: while True: msg = await ws.receive() - if msg.type == aiohttp.WSMsgType.TEXT: + if msg.type == WSMsgType.TEXT: message = msg.data.strip() # self.log.info(f"received message: {message}") await self.safe_handle(ws, message) - elif msg.type == aiohttp.WSMsgType.BINARY: + elif msg.type == WSMsgType.BINARY: self.log.debug("Received binary data") - elif msg.type == aiohttp.WSMsgType.PING: + elif msg.type == WSMsgType.PING: self.log.debug("Ping received") await ws.pong() - elif msg.type == aiohttp.WSMsgType.PONG: + elif msg.type == WSMsgType.PONG: self.log.debug("Pong received") else: - if msg.type == aiohttp.WSMsgType.CLOSE: + if msg.type == WSMsgType.CLOSE: self.log.debug("Closing RPC websocket") await ws.close() - elif msg.type == aiohttp.WSMsgType.ERROR: + elif msg.type == WSMsgType.ERROR: self.log.error("Error during receive %s" % ws.exception()) - elif msg.type == aiohttp.WSMsgType.CLOSED: + elif msg.type == WSMsgType.CLOSED: pass break - await ws.close() - async def connect_to_daemon(self, self_hostname: str, daemon_port: uint16): - while True: + while not self.shut_down: try: - if self.shut_down: - break - async with aiohttp.ClientSession() as session: - async with session.ws_connect( - f"wss://{self_hostname}:{daemon_port}", - autoclose=True, - autoping=True, - heartbeat=60, - ssl_context=self.ssl_context, - max_msg_size=100 * 1024 * 1024, - ) as ws: - self.websocket = ws - await self.connection(ws) - self.websocket = None - except aiohttp.ClientConnectorError: + self.client_session = ClientSession() + self.websocket = await self.client_session.ws_connect( + f"wss://{self_hostname}:{daemon_port}", + autoclose=True, + autoping=True, + heartbeat=60, + ssl_context=self.ssl_context, + max_msg_size=max_message_size, + ) + await self.connection(self.websocket) + except ClientConnectorError: self.log.warning(f"Cannot connect to daemon at ws://{self_hostname}:{daemon_port}") except Exception as e: tb = traceback.format_exc() self.log.warning(f"Exception: {tb} {type(e)}") + if self.websocket is not None: + await self.websocket.close() + if self.client_session is not None: + await self.client_session.close() + self.websocket = None + self.client_session = None await asyncio.sleep(2) @@ -306,17 +310,15 @@ async def start_rpc_server( Starts an HTTP server with the following RPC methods, to be used by local clients to query the node. """ - app = aiohttp.web.Application() + app = web.Application() rpc_server = RpcServer(rpc_api, rpc_api.service_name, stop_cb, root_path, net_config) rpc_server.rpc_api.service._set_state_changed_callback(rpc_server.state_changed) - app.add_routes( - [aiohttp.web.post(route, wrap_http_handler(func)) for (route, func) in rpc_server.get_routes().items()] - ) + app.add_routes([web.post(route, wrap_http_handler(func)) for (route, func) in rpc_server.get_routes().items()]) if connect_to_daemon: daemon_connection = asyncio.create_task(rpc_server.connect_to_daemon(self_hostname, daemon_port)) - runner = aiohttp.web.AppRunner(app, access_log=None) + runner = web.AppRunner(app, access_log=None) await runner.setup() - site = aiohttp.web.TCPSite(runner, self_hostname, int(rpc_port), ssl_context=rpc_server.ssl_context) + site = web.TCPSite(runner, self_hostname, int(rpc_port), ssl_context=rpc_server.ssl_context) await site.start() async def cleanup(): diff --git a/chia/server/server.py b/chia/server/server.py index 948985165c..9edac98fc1 100644 --- a/chia/server/server.py +++ b/chia/server/server.py @@ -33,6 +33,8 @@ from chia.util.ints import uint16 from chia.util.network import is_in_network, is_localhost from chia.util.ssl_check import verify_ssl_certs_and_keys +max_message_size = 50 * 1024 * 1024 # 50MB + def ssl_context_for_server( ca_cert: Path, @@ -277,7 +279,7 @@ class ChiaServer: if request.remote in self.banned_peers and time.time() < self.banned_peers[request.remote]: self.log.warning(f"Peer {request.remote} is banned, refusing connection") return None - ws = web.WebSocketResponse(max_msg_size=50 * 1024 * 1024) + ws = web.WebSocketResponse(max_msg_size=max_message_size) await ws.prepare(request) close_event = asyncio.Event() cert_bytes = request.transport._ssl_protocol._extra["ssl_object"].getpeercert(True) @@ -422,7 +424,7 @@ class ChiaServer: self.log.debug(f"Connecting: {url}, Peer info: {target_node}") try: ws = await session.ws_connect( - url, autoclose=True, autoping=True, heartbeat=60, ssl=ssl_context, max_msg_size=50 * 1024 * 1024 + url, autoclose=True, autoping=True, heartbeat=60, ssl=ssl_context, max_msg_size=max_message_size ) except ServerDisconnectedError: self.log.debug(f"Server disconnected error connecting to {url}. Perhaps we are banned by the peer.") diff --git a/chia/wallet/wallet_node.py b/chia/wallet/wallet_node.py index 0ad91aa383..6b942ee5f1 100644 --- a/chia/wallet/wallet_node.py +++ b/chia/wallet/wallet_node.py @@ -6,33 +6,32 @@ import time import traceback from asyncio import CancelledError from pathlib import Path -from typing import Callable, Dict, List, Optional, Set, Tuple, Any, Iterator +from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Tuple -from blspy import PrivateKey, AugSchemeMPL +from blspy import AugSchemeMPL, PrivateKey from packaging.version import Version from chia.consensus.block_record import BlockRecord from chia.consensus.blockchain import ReceiveBlockResult from chia.consensus.constants import ConsensusConstants from chia.daemon.keychain_proxy import ( + KeychainProxy, KeychainProxyConnectionFailure, + KeyringIsEmpty, connect_to_keychain_and_validate, wrap_local_keychain, - KeychainProxy, - KeyringIsEmpty, ) -from chia.util.chunks import chunks from chia.protocols import wallet_protocol from chia.protocols.full_node_protocol import RequestProofOfWeight, RespondProofOfWeight from chia.protocols.protocol_message_types import ProtocolMessageTypes from chia.protocols.wallet_protocol import ( - RespondToCoinUpdates, CoinState, - RespondToPhUpdates, - RespondBlockHeader, - RequestSESInfo, - RespondSESInfo, RequestHeaderBlocks, + RequestSESInfo, + RespondBlockHeader, + RespondSESInfo, + RespondToCoinUpdates, + RespondToPhUpdates, ) from chia.server.node_discovery import WalletPeers from chia.server.outbound_message import Message, NodeType, make_msg @@ -46,29 +45,30 @@ from chia.types.coin_spend import CoinSpend from chia.types.header_block import HeaderBlock from chia.types.mempool_inclusion_status import MempoolInclusionStatus from chia.types.peer_info import PeerInfo -from chia.types.weight_proof import WeightProof, SubEpochData +from chia.types.weight_proof import SubEpochData, WeightProof from chia.util.byte_types import hexstr_to_bytes +from chia.util.chunks import chunks from chia.util.config import WALLET_PEERS_PATH_KEY_DEPRECATED from chia.util.default_root import STANDALONE_ROOT_PATH from chia.util.ints import uint32, uint64 -from chia.util.keychain import KeyringIsLocked, Keychain +from chia.util.keychain import Keychain, KeyringIsLocked from chia.util.path import mkdir, path_from_root -from chia.wallet.util.new_peak_queue import NewPeakQueue, NewPeakQueueTypes, NewPeakItem +from chia.util.profiler import profile_task +from chia.wallet.transaction_record import TransactionRecord +from chia.wallet.util.new_peak_queue import NewPeakItem, NewPeakQueue, NewPeakQueueTypes from chia.wallet.util.peer_request_cache import PeerRequestCache, can_use_peer_request_cache from chia.wallet.util.wallet_sync_utils import ( - request_and_validate_removals, - request_and_validate_additions, - fetch_last_tx_from_peer, - subscribe_to_phs, - subscribe_to_coin_updates, - last_change_height_cs, fetch_header_blocks_in_range, + fetch_last_tx_from_peer, + last_change_height_cs, + request_and_validate_additions, + request_and_validate_removals, + subscribe_to_coin_updates, + subscribe_to_phs, ) +from chia.wallet.wallet_action import WalletAction from chia.wallet.wallet_coin_record import WalletCoinRecord from chia.wallet.wallet_state_manager import WalletStateManager -from chia.wallet.transaction_record import TransactionRecord -from chia.wallet.wallet_action import WalletAction -from chia.util.profiler import profile_task class WalletNode: @@ -269,6 +269,9 @@ class WalletNode: if self.wallet_state_manager is not None: await self.wallet_state_manager._await_closed() self.wallet_state_manager = None + if self.keychain_proxy is not None: + await self.keychain_proxy.close() + await asyncio.sleep(0.5) # https://docs.aiohttp.org/en/stable/client_advanced.html#graceful-shutdown self.logged_in = False self.wallet_peers = None diff --git a/setup.py b/setup.py index c746a4cf26..df9ef8b411 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,6 @@ dependencies = [ "PyYAML==5.4.1", # Used for config file format "setproctitle==1.2.2", # Gives the chia processes readable names "sortedcontainers==2.4.0", # For maintaining sorted mempools - "websockets==8.1.0", # For use in wallet RPC and electron UI # TODO: when moving to click 8 remove the pinning of black noted below "click==7.1.2", # For the CLI "dnspythonchia==2.2.0", # Query DNS seeds diff --git a/tests/core/test_daemon_rpc.py b/tests/core/test_daemon_rpc.py index 5cf5780eae..cec32ef33e 100644 --- a/tests/core/test_daemon_rpc.py +++ b/tests/core/test_daemon_rpc.py @@ -9,7 +9,9 @@ class TestDaemonRpc: async def test_get_version_rpc(self, get_daemon, bt): ws_server = get_daemon config = bt.config - client = await connect_to_daemon(config["self_hostname"], config["daemon_port"], bt.get_daemon_ssl_context()) + client = await connect_to_daemon( + config["self_hostname"], config["daemon_port"], 50 * 1000 * 1000, bt.get_daemon_ssl_context() + ) response = await client.get_version() assert response["data"]["success"] diff --git a/tests/setup_nodes.py b/tests/setup_nodes.py index abcdc04706..4cfb701d9c 100644 --- a/tests/setup_nodes.py +++ b/tests/setup_nodes.py @@ -1,6 +1,5 @@ -import logging import asyncio - +import logging from secrets import token_bytes from typing import Dict, List @@ -8,23 +7,23 @@ from chia.consensus.constants import ConsensusConstants from chia.full_node.full_node_api import FullNodeAPI from chia.server.start_service import Service from chia.server.start_wallet import service_kwargs_for_wallet -from tests.block_tools import create_block_tools_async, test_constants, BlockTools -from tests.setup_services import ( - setup_full_node, - setup_harvester, - setup_farmer, - setup_introducer, - setup_vdf_clients, - setup_timelord, - setup_vdf_client, - setup_daemon, -) -from tests.util.keyring import TempKeyring -from tests.util.socket import find_available_listen_port from chia.util.hash import std_hash from chia.util.ints import uint16, uint32 from chia.util.keychain import bytes_to_mnemonic +from tests.block_tools import BlockTools, create_block_tools_async, test_constants +from tests.setup_services import ( + setup_daemon, + setup_farmer, + setup_full_node, + setup_harvester, + setup_introducer, + setup_timelord, + setup_vdf_client, + setup_vdf_clients, +) from tests.time_out_assert import time_out_assert_custom_interval +from tests.util.keyring import TempKeyring +from tests.util.socket import find_available_listen_port def cleanup_keyring(keyring: TempKeyring): diff --git a/tests/setup_services.py b/tests/setup_services.py index ebd1db149c..848506fe7f 100644 --- a/tests/setup_services.py +++ b/tests/setup_services.py @@ -6,7 +6,7 @@ from secrets import token_bytes from typing import AsyncGenerator, Optional from chia.consensus.constants import ConsensusConstants -from chia.daemon.server import WebSocketServer, create_server_for_daemon, daemon_launch_lock_path, singleton +from chia.daemon.server import WebSocketServer, daemon_launch_lock_path, singleton from chia.server.start_farmer import service_kwargs_for_farmer from chia.server.start_full_node import service_kwargs_for_full_node from chia.server.start_harvester import service_kwargs_for_harvester @@ -36,8 +36,8 @@ async def setup_daemon(btools: BlockTools) -> AsyncGenerator[WebSocketServer, No ca_crt_path = root_path / config["private_ssl_ca"]["crt"] ca_key_path = root_path / config["private_ssl_ca"]["key"] assert lockfile is not None - create_server_for_daemon(btools.root_path) - ws_server = WebSocketServer(root_path, ca_crt_path, ca_key_path, crt_path, key_path) + shutdown_event = asyncio.Event() + ws_server = WebSocketServer(root_path, ca_crt_path, ca_key_path, crt_path, key_path, shutdown_event) await ws_server.start() yield ws_server From a2fa8dda01ef9c45fb144c27b30edc80c37a713e Mon Sep 17 00:00:00 2001 From: Amine Khaldi Date: Tue, 5 Apr 2022 18:34:57 +0100 Subject: [PATCH 32/63] Prepare test blocks and plots only for tests that need them. This saves us a couple more hours of CI running time. (#10975) --- .github/workflows/build-test-macos-core-cmds.yml | 8 +------- .github/workflows/build-test-macos-core-consensus.yml | 8 +------- .github/workflows/build-test-macos-core-custom_types.yml | 8 +------- .github/workflows/build-test-macos-generator.yml | 8 +------- .github/workflows/build-test-macos-tools.yml | 8 +------- .github/workflows/build-test-macos-util.yml | 8 +------- .github/workflows/build-test-macos-wallet-did_wallet.yml | 8 +------- .github/workflows/build-test-macos-wallet-rl_wallet.yml | 8 +------- .github/workflows/build-test-ubuntu-core-cmds.yml | 8 +------- .github/workflows/build-test-ubuntu-core-consensus.yml | 8 +------- .github/workflows/build-test-ubuntu-core-custom_types.yml | 8 +------- .github/workflows/build-test-ubuntu-generator.yml | 8 +------- .github/workflows/build-test-ubuntu-tools.yml | 8 +------- .github/workflows/build-test-ubuntu-util.yml | 8 +------- .github/workflows/build-test-ubuntu-wallet-did_wallet.yml | 8 +------- .github/workflows/build-test-ubuntu-wallet-rl_wallet.yml | 8 +------- tests/blockchain/config.py | 1 + tests/core/config.py | 1 + tests/core/daemon/config.py | 1 + tests/core/full_node/config.py | 1 + tests/core/full_node/full_sync/config.py | 1 + tests/core/full_node/stores/config.py | 1 + tests/core/server/config.py | 1 + tests/core/ssl/config.py | 1 + tests/core/util/config.py | 1 + tests/farmer_harvester/config.py | 1 + tests/plotting/config.py | 1 + tests/pools/config.py | 1 + tests/simulation/config.py | 1 + tests/testconfig.py | 2 +- tests/wallet/cat_wallet/config.py | 1 + tests/wallet/config.py | 1 + tests/wallet/rpc/config.py | 1 + tests/wallet/simple_sync/config.py | 1 + tests/wallet/sync/config.py | 1 + tests/weight_proof/config.py | 1 + 36 files changed, 36 insertions(+), 113 deletions(-) create mode 100644 tests/core/server/config.py create mode 100644 tests/farmer_harvester/config.py create mode 100644 tests/wallet/rpc/config.py create mode 100644 tests/wallet/simple_sync/config.py diff --git a/.github/workflows/build-test-macos-core-cmds.yml b/.github/workflows/build-test-macos-core-cmds.yml index 6d64dfaa4f..687ae6c949 100644 --- a/.github/workflows/build-test-macos-core-cmds.yml +++ b/.github/workflows/build-test-macos-core-cmds.yml @@ -65,13 +65,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 - with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 +# Omitted checking out blocks and plots repo Chia-Network/test-cache - name: Run install script env: diff --git a/.github/workflows/build-test-macos-core-consensus.yml b/.github/workflows/build-test-macos-core-consensus.yml index d85ee8a281..f5cd798982 100644 --- a/.github/workflows/build-test-macos-core-consensus.yml +++ b/.github/workflows/build-test-macos-core-consensus.yml @@ -65,13 +65,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 - with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 +# Omitted checking out blocks and plots repo Chia-Network/test-cache - name: Run install script env: diff --git a/.github/workflows/build-test-macos-core-custom_types.yml b/.github/workflows/build-test-macos-core-custom_types.yml index b6f1aa512e..130d15420c 100644 --- a/.github/workflows/build-test-macos-core-custom_types.yml +++ b/.github/workflows/build-test-macos-core-custom_types.yml @@ -65,13 +65,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 - with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 +# Omitted checking out blocks and plots repo Chia-Network/test-cache - name: Run install script env: diff --git a/.github/workflows/build-test-macos-generator.yml b/.github/workflows/build-test-macos-generator.yml index 77c89e44d9..e695c65e96 100644 --- a/.github/workflows/build-test-macos-generator.yml +++ b/.github/workflows/build-test-macos-generator.yml @@ -65,13 +65,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 - with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 +# Omitted checking out blocks and plots repo Chia-Network/test-cache - name: Run install script env: diff --git a/.github/workflows/build-test-macos-tools.yml b/.github/workflows/build-test-macos-tools.yml index d304aca53d..e0a4396b6e 100644 --- a/.github/workflows/build-test-macos-tools.yml +++ b/.github/workflows/build-test-macos-tools.yml @@ -65,13 +65,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 - with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 +# Omitted checking out blocks and plots repo Chia-Network/test-cache - name: Run install script env: diff --git a/.github/workflows/build-test-macos-util.yml b/.github/workflows/build-test-macos-util.yml index 8661fbff15..f0f2334a0d 100644 --- a/.github/workflows/build-test-macos-util.yml +++ b/.github/workflows/build-test-macos-util.yml @@ -65,13 +65,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 - with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 +# Omitted checking out blocks and plots repo Chia-Network/test-cache - name: Run install script env: diff --git a/.github/workflows/build-test-macos-wallet-did_wallet.yml b/.github/workflows/build-test-macos-wallet-did_wallet.yml index fbb4fcb432..4f97d0ea2f 100644 --- a/.github/workflows/build-test-macos-wallet-did_wallet.yml +++ b/.github/workflows/build-test-macos-wallet-did_wallet.yml @@ -65,13 +65,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 - with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 +# Omitted checking out blocks and plots repo Chia-Network/test-cache - name: Run install script env: diff --git a/.github/workflows/build-test-macos-wallet-rl_wallet.yml b/.github/workflows/build-test-macos-wallet-rl_wallet.yml index 5d74c4e6bc..1e19516260 100644 --- a/.github/workflows/build-test-macos-wallet-rl_wallet.yml +++ b/.github/workflows/build-test-macos-wallet-rl_wallet.yml @@ -65,13 +65,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 - with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 +# Omitted checking out blocks and plots repo Chia-Network/test-cache - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-core-cmds.yml b/.github/workflows/build-test-ubuntu-core-cmds.yml index 88ee7ab3cb..f5f8ed214d 100644 --- a/.github/workflows/build-test-ubuntu-core-cmds.yml +++ b/.github/workflows/build-test-ubuntu-core-cmds.yml @@ -65,13 +65,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 - with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 +# Omitted checking out blocks and plots repo Chia-Network/test-cache - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-core-consensus.yml b/.github/workflows/build-test-ubuntu-core-consensus.yml index 307f85716f..f9045c31ba 100644 --- a/.github/workflows/build-test-ubuntu-core-consensus.yml +++ b/.github/workflows/build-test-ubuntu-core-consensus.yml @@ -65,13 +65,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 - with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 +# Omitted checking out blocks and plots repo Chia-Network/test-cache - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-core-custom_types.yml b/.github/workflows/build-test-ubuntu-core-custom_types.yml index 94f493f641..447a286273 100644 --- a/.github/workflows/build-test-ubuntu-core-custom_types.yml +++ b/.github/workflows/build-test-ubuntu-core-custom_types.yml @@ -65,13 +65,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 - with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 +# Omitted checking out blocks and plots repo Chia-Network/test-cache - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-generator.yml b/.github/workflows/build-test-ubuntu-generator.yml index 1aa74d5daf..3f928e2c04 100644 --- a/.github/workflows/build-test-ubuntu-generator.yml +++ b/.github/workflows/build-test-ubuntu-generator.yml @@ -65,13 +65,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 - with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 +# Omitted checking out blocks and plots repo Chia-Network/test-cache - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-tools.yml b/.github/workflows/build-test-ubuntu-tools.yml index 8b8ff6cfbe..97877660be 100644 --- a/.github/workflows/build-test-ubuntu-tools.yml +++ b/.github/workflows/build-test-ubuntu-tools.yml @@ -65,13 +65,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 - with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 +# Omitted checking out blocks and plots repo Chia-Network/test-cache - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-util.yml b/.github/workflows/build-test-ubuntu-util.yml index 6f4768ccee..1d0567e628 100644 --- a/.github/workflows/build-test-ubuntu-util.yml +++ b/.github/workflows/build-test-ubuntu-util.yml @@ -65,13 +65,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 - with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 +# Omitted checking out blocks and plots repo Chia-Network/test-cache - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-wallet-did_wallet.yml b/.github/workflows/build-test-ubuntu-wallet-did_wallet.yml index f7e9f1012f..06bf1ad15d 100644 --- a/.github/workflows/build-test-ubuntu-wallet-did_wallet.yml +++ b/.github/workflows/build-test-ubuntu-wallet-did_wallet.yml @@ -65,13 +65,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 - with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 +# Omitted checking out blocks and plots repo Chia-Network/test-cache - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-wallet-rl_wallet.yml b/.github/workflows/build-test-ubuntu-wallet-rl_wallet.yml index 3d241ad1bb..c791e3f32c 100644 --- a/.github/workflows/build-test-ubuntu-wallet-rl_wallet.yml +++ b/.github/workflows/build-test-ubuntu-wallet-rl_wallet.yml @@ -65,13 +65,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 - with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 +# Omitted checking out blocks and plots repo Chia-Network/test-cache - name: Run install script env: diff --git a/tests/blockchain/config.py b/tests/blockchain/config.py index 93f77cd99a..6a5974362b 100644 --- a/tests/blockchain/config.py +++ b/tests/blockchain/config.py @@ -1,2 +1,3 @@ parallel = True job_timeout = 60 +checkout_blocks_and_plots = True diff --git a/tests/core/config.py b/tests/core/config.py index 7f9e1e1a76..235efb181c 100644 --- a/tests/core/config.py +++ b/tests/core/config.py @@ -1 +1,2 @@ parallel = True +checkout_blocks_and_plots = True diff --git a/tests/core/daemon/config.py b/tests/core/daemon/config.py index 685e5b3a49..0fbdc68914 100644 --- a/tests/core/daemon/config.py +++ b/tests/core/daemon/config.py @@ -1 +1,2 @@ install_timelord = True +checkout_blocks_and_plots = True diff --git a/tests/core/full_node/config.py b/tests/core/full_node/config.py index 2cbb1c0883..58f7edfd7a 100644 --- a/tests/core/full_node/config.py +++ b/tests/core/full_node/config.py @@ -2,3 +2,4 @@ parallel = True job_timeout = 50 check_resource_usage = True +checkout_blocks_and_plots = True diff --git a/tests/core/full_node/full_sync/config.py b/tests/core/full_node/full_sync/config.py index 251dd4220d..49234ca992 100644 --- a/tests/core/full_node/full_sync/config.py +++ b/tests/core/full_node/full_sync/config.py @@ -1,2 +1,3 @@ job_timeout = 60 parallel = True +checkout_blocks_and_plots = True diff --git a/tests/core/full_node/stores/config.py b/tests/core/full_node/stores/config.py index a36e5dd8a3..97cd63fb80 100644 --- a/tests/core/full_node/stores/config.py +++ b/tests/core/full_node/stores/config.py @@ -2,3 +2,4 @@ parallel = True job_timeout = 40 check_resource_usage = True +checkout_blocks_and_plots = True diff --git a/tests/core/server/config.py b/tests/core/server/config.py new file mode 100644 index 0000000000..0257db4372 --- /dev/null +++ b/tests/core/server/config.py @@ -0,0 +1 @@ +checkout_blocks_and_plots = True diff --git a/tests/core/ssl/config.py b/tests/core/ssl/config.py index 7f9e1e1a76..235efb181c 100644 --- a/tests/core/ssl/config.py +++ b/tests/core/ssl/config.py @@ -1 +1,2 @@ parallel = True +checkout_blocks_and_plots = True diff --git a/tests/core/util/config.py b/tests/core/util/config.py index 7f9e1e1a76..235efb181c 100644 --- a/tests/core/util/config.py +++ b/tests/core/util/config.py @@ -1 +1,2 @@ parallel = True +checkout_blocks_and_plots = True diff --git a/tests/farmer_harvester/config.py b/tests/farmer_harvester/config.py new file mode 100644 index 0000000000..0257db4372 --- /dev/null +++ b/tests/farmer_harvester/config.py @@ -0,0 +1 @@ +checkout_blocks_and_plots = True diff --git a/tests/plotting/config.py b/tests/plotting/config.py index c5495db277..b0a0afced8 100644 --- a/tests/plotting/config.py +++ b/tests/plotting/config.py @@ -1,2 +1,3 @@ parallel = True install_timelord = False +checkout_blocks_and_plots = True diff --git a/tests/pools/config.py b/tests/pools/config.py index be5a59232c..244e435f0e 100644 --- a/tests/pools/config.py +++ b/tests/pools/config.py @@ -1,2 +1,3 @@ parallel = 2 job_timeout = 60 +checkout_blocks_and_plots = True diff --git a/tests/simulation/config.py b/tests/simulation/config.py index 4ef7da0588..c983484401 100644 --- a/tests/simulation/config.py +++ b/tests/simulation/config.py @@ -1,2 +1,3 @@ job_timeout = 60 install_timelord = True +checkout_blocks_and_plots = True diff --git a/tests/testconfig.py b/tests/testconfig.py index 3aae8f4cea..72a3cedac7 100644 --- a/tests/testconfig.py +++ b/tests/testconfig.py @@ -10,7 +10,7 @@ oses = ["ubuntu", "macos"] # Defaults are conservative. parallel: Union[bool, int, Literal["auto"]] = False -checkout_blocks_and_plots = True +checkout_blocks_and_plots = False install_timelord = False check_resource_usage = False job_timeout = 30 diff --git a/tests/wallet/cat_wallet/config.py b/tests/wallet/cat_wallet/config.py index eb21fe13cd..671807f010 100644 --- a/tests/wallet/cat_wallet/config.py +++ b/tests/wallet/cat_wallet/config.py @@ -1,2 +1,3 @@ # flake8: noqa: E501 job_timeout = 50 +checkout_blocks_and_plots = True diff --git a/tests/wallet/config.py b/tests/wallet/config.py index e90fb48eaa..dd660d39ef 100644 --- a/tests/wallet/config.py +++ b/tests/wallet/config.py @@ -1,3 +1,4 @@ # flake8: noqa: E501 job_timeout = 40 parallel = True +checkout_blocks_and_plots = True diff --git a/tests/wallet/rpc/config.py b/tests/wallet/rpc/config.py new file mode 100644 index 0000000000..0257db4372 --- /dev/null +++ b/tests/wallet/rpc/config.py @@ -0,0 +1 @@ +checkout_blocks_and_plots = True diff --git a/tests/wallet/simple_sync/config.py b/tests/wallet/simple_sync/config.py new file mode 100644 index 0000000000..0257db4372 --- /dev/null +++ b/tests/wallet/simple_sync/config.py @@ -0,0 +1 @@ +checkout_blocks_and_plots = True diff --git a/tests/wallet/sync/config.py b/tests/wallet/sync/config.py index d9b815b24c..c478c63e00 100644 --- a/tests/wallet/sync/config.py +++ b/tests/wallet/sync/config.py @@ -1 +1,2 @@ job_timeout = 60 +checkout_blocks_and_plots = True diff --git a/tests/weight_proof/config.py b/tests/weight_proof/config.py index 7f9e1e1a76..235efb181c 100644 --- a/tests/weight_proof/config.py +++ b/tests/weight_proof/config.py @@ -1 +1,2 @@ parallel = True +checkout_blocks_and_plots = True From 6ed61bfbe50fb6ab5a612a4ff5ae039c2b80dc29 Mon Sep 17 00:00:00 2001 From: William Allen Date: Tue, 5 Apr 2022 15:28:33 -0500 Subject: [PATCH 33/63] Adding clean-workspace step to benchmarks (#11063) --- .github/workflows/benchmarks.yml | 77 +++++++++++++++++--------------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 0dc6ff3bbe..527fc9b71a 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -5,7 +5,7 @@ on: branches: - main tags: - - '**' + - '**' pull_request: branches: - '**' @@ -27,45 +27,48 @@ jobs: python-version: [ 3.9 ] steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - name: Clean workspace + uses: Chia-Network/actions/clean-workspace@main - - name: Setup Python environment - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 - - name: Get pip cache dir - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" + - name: Setup Python environment + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} - - name: Cache pip - uses: actions/cache@v3 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" - - name: Install ubuntu dependencies - env: - DEBIAN_FRONTEND: noninteractive - run: | - sudo apt-get install -y software-properties-common - sudo add-apt-repository ppa:deadsnakes/ppa - sudo apt-get update - sudo apt-get install -y python${{ matrix.python-version }}-venv python${{ matrix.python-version }}-distutils git + - name: Cache pip + uses: actions/cache@v3 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- - - name: Run install script - env: - INSTALL_PYTHON_VERSION: ${{ matrix.python-version }} - run: | - sh install.sh -d + - name: Install ubuntu dependencies + env: + DEBIAN_FRONTEND: noninteractive + run: | + sudo apt-get install -y software-properties-common + sudo add-apt-repository ppa:deadsnakes/ppa + sudo apt-get update + sudo apt-get install -y python${{ matrix.python-version }}-venv python${{ matrix.python-version }}-distutils git - - name: pytest - run: | - . ./activate - ./venv/bin/py.test -n 0 -m benchmark tests + - name: Run install script + env: + INSTALL_PYTHON_VERSION: ${{ matrix.python-version }} + run: | + sh install.sh -d + + - name: pytest + run: | + . ./activate + ./venv/bin/py.test -n 0 -m benchmark tests From 45e4c7c1568b48c5f42b7bf2337f03d8128ff37e Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 5 Apr 2022 18:16:38 -0400 Subject: [PATCH 34/63] Checkout test blocks and plots for benchmarks workflow (#11068) --- .github/workflows/benchmarks.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 527fc9b71a..d01ed34c8d 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -25,6 +25,8 @@ jobs: max-parallel: 4 matrix: python-version: [ 3.9 ] + env: + CHIA_ROOT: ${{ github.workspace }}/.chia/mainnet steps: - name: Clean workspace @@ -62,6 +64,14 @@ jobs: sudo apt-get update sudo apt-get install -y python${{ matrix.python-version }}-venv python${{ matrix.python-version }}-distutils git + - name: Checkout test blocks and plots + uses: actions/checkout@v3 + with: + repository: 'Chia-Network/test-cache' + path: '.chia' + ref: '0.28.0' + fetch-depth: 1 + - name: Run install script env: INSTALL_PYTHON_VERSION: ${{ matrix.python-version }} From bb57ccffa942afc654a0874bad3bef6613045756 Mon Sep 17 00:00:00 2001 From: Jeff Date: Tue, 5 Apr 2022 16:09:00 -0700 Subject: [PATCH 35/63] =?UTF-8?q?Improve=20handling=20of=20unknown=20pendi?= =?UTF-8?q?ng=20balances=20(likely=20change=20from=20addi=E2=80=A6=20(#109?= =?UTF-8?q?84)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Improve handling of unknown pending balances (likely change from adding a maker fee). Minor improvement for fingerprint selection -- enter/return selects the logged-in fingerprint. * Minor output formatting improvements when showing offer summaries. Minor wallet key selection improvements. Added tests for print_offer_summary * Linter fixes * isort fix * Coroutine -> Awaitable * Removed problematic fee calculation from get_pending_amounts per feedback. --- chia/cmds/wallet_funcs.py | 80 ++++++++++++++------- chia/wallet/trading/offer.py | 6 -- tests/core/cmds/test_wallet.py | 124 +++++++++++++++++++++++++++++++++ 3 files changed, 178 insertions(+), 32 deletions(-) create mode 100644 tests/core/cmds/test_wallet.py diff --git a/chia/cmds/wallet_funcs.py b/chia/cmds/wallet_funcs.py index 7a8b42baab..5f5effbadf 100644 --- a/chia/cmds/wallet_funcs.py +++ b/chia/cmds/wallet_funcs.py @@ -4,7 +4,7 @@ import sys import time from datetime import datetime from decimal import Decimal -from typing import Any, Callable, List, Optional, Tuple, Dict +from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple import aiohttp @@ -24,6 +24,8 @@ from chia.wallet.trading.trade_status import TradeStatus from chia.wallet.transaction_record import TransactionRecord from chia.wallet.util.wallet_types import WalletType +CATNameResolver = Callable[[bytes32], Awaitable[Optional[Tuple[Optional[uint32], str]]]] + def print_transaction(tx: TransactionRecord, verbose: bool, name, address_prefix: str, mojo_per_unit: int) -> None: if verbose: @@ -308,21 +310,37 @@ def timestamp_to_time(timestamp): return datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d %H:%M:%S") -async def print_offer_summary(wallet_client: WalletRpcClient, sum_dict: dict): +async def print_offer_summary(cat_name_resolver: CATNameResolver, sum_dict: Dict[str, int], has_fee: bool = False): for asset_id, amount in sum_dict.items(): - if asset_id == "xch": - wid: str = "1" - name: str = "XCH" - unit: int = units["chia"] - else: - result = await wallet_client.cat_asset_id_to_name(bytes32.from_hexstr(asset_id)) - wid = "Unknown" + description: str = "" + unit: int = units["chia"] + wid: str = "1" if asset_id == "xch" else "" + mojo_amount: int = int(Decimal(amount)) + name: str = "XCH" + if asset_id != "xch": name = asset_id - unit = units["cat"] - if result is not None: - wid = str(result[0]) - name = result[1] - print(f" - {name} (Wallet ID: {wid}): {Decimal(int(amount)) / unit} ({int(Decimal(amount))} mojos)") + if asset_id == "unknown": + name = "Unknown" + unit = units["mojo"] + if has_fee: + description = " [Typically represents change returned from the included fee]" + else: + unit = units["cat"] + result = await cat_name_resolver(bytes32.from_hexstr(asset_id)) + if result is not None: + wid = str(result[0]) + name = result[1] + output: str = f" - {name}" + mojo_str: str = f"{mojo_amount} {'mojo' if mojo_amount == 1 else 'mojos'}" + if len(wid) > 0: + output += f" (Wallet ID: {wid})" + if unit == units["mojo"]: + output += f": {mojo_str}" + else: + output += f": {mojo_amount / unit} ({mojo_str})" + if len(description) > 0: + output += f" {description}" + print(output) async def print_trade_record(record, wallet_client: WalletRpcClient, summaries: bool = False) -> None: @@ -337,13 +355,16 @@ async def print_trade_record(record, wallet_client: WalletRpcClient, summaries: print("Summary:") offer = Offer.from_bytes(record.offer) offered, requested = offer.summary() + outbound_balances: Dict[str, int] = offer.get_pending_amounts() + fees: Decimal = Decimal(offer.bundle.fees()) + cat_name_resolver = wallet_client.cat_asset_id_to_name print(" OFFERED:") - await print_offer_summary(wallet_client, offered) + await print_offer_summary(cat_name_resolver, offered) print(" REQUESTED:") - await print_offer_summary(wallet_client, requested) - print("Pending Balances:") - await print_offer_summary(wallet_client, offer.get_pending_amounts()) - print(f"Fees: {Decimal(offer.bundle.fees()) / units['chia']}") + await print_offer_summary(cat_name_resolver, requested) + print("Pending Outbound Balances:") + await print_offer_summary(cat_name_resolver, outbound_balances, has_fee=(fees > 0)) + print(f"Included Fees: {fees / units['chia']}") print("---------------") @@ -411,12 +432,13 @@ async def take_offer(args: dict, wallet_client: WalletRpcClient, fingerprint: in return offered, requested = offer.summary() + cat_name_resolver = wallet_client.cat_asset_id_to_name print("Summary:") print(" OFFERED:") - await print_offer_summary(wallet_client, offered) + await print_offer_summary(cat_name_resolver, offered) print(" REQUESTED:") - await print_offer_summary(wallet_client, requested) - print(f"Fees: {Decimal(offer.bundle.fees()) / units['chia']}") + await print_offer_summary(cat_name_resolver, requested) + print(f"Included Fees: {Decimal(offer.bundle.fees()) / units['chia']}") if not examine_only: confirmation = input("Would you like to take this offer? (y/n): ") @@ -534,7 +556,7 @@ async def get_wallet(wallet_client: WalletRpcClient, fingerprint: int = None) -> current_sync_status = "Syncing" else: current_sync_status = "Not Synced" - print("Choose wallet key:") + print("Wallet keys:") for i, fp in enumerate(fingerprints): row: str = f"{i+1}) " row += "* " if fp == logged_in_fingerprint else spacing @@ -543,15 +565,21 @@ async def get_wallet(wallet_client: WalletRpcClient, fingerprint: int = None) -> row += f" ({current_sync_status})" print(row) val = None + prompt: str = ( + f"Choose a wallet key [1-{len(fingerprints)}] ('q' to quit, or Enter to use {logged_in_fingerprint}): " + ) while val is None: - val = input("Enter a number to pick or q to quit: ") + val = input(prompt) if val == "q": return None - if not val.isdigit(): + elif val == "" and logged_in_fingerprint is not None: + fingerprint = logged_in_fingerprint + break + elif not val.isdigit(): val = None else: index = int(val) - 1 - if index >= len(fingerprints): + if index < 0 or index >= len(fingerprints): print("Invalid value") val = None continue diff --git a/chia/wallet/trading/offer.py b/chia/wallet/trading/offer.py index 05d742f25c..4140a1e67a 100644 --- a/chia/wallet/trading/offer.py +++ b/chia/wallet/trading/offer.py @@ -193,12 +193,6 @@ class Offer: for addition in filter(lambda c: c.parent_coin_info == root_removal.name(), all_additions): pending_dict[name] += addition.amount - # Then we add a potential fee as pending XCH - fee: int = sum(c.amount for c in all_removals) - sum(c.amount for c in all_additions) - if fee > 0: - pending_dict.setdefault("xch", 0) - pending_dict["xch"] += fee - # Then we gather anything else as unknown sum_of_additions_so_far: int = sum(pending_dict.values()) unknown: int = sum([c.amount for c in non_ephemeral_removals]) - sum_of_additions_so_far diff --git a/tests/core/cmds/test_wallet.py b/tests/core/cmds/test_wallet.py new file mode 100644 index 0000000000..f211c5b50a --- /dev/null +++ b/tests/core/cmds/test_wallet.py @@ -0,0 +1,124 @@ +from typing import Any, Dict, Optional, Tuple + +import pytest + +from chia.cmds.wallet_funcs import print_offer_summary +from chia.types.blockchain_format.sized_bytes import bytes32 +from chia.util.ints import uint32 + +TEST_DUCKSAUCE_ASSET_ID = "1000000000000000000000000000000000000000000000000000000000000001" +TEST_CRUNCHBERRIES_ASSET_ID = "1000000000000000000000000000000000000000000000000000000000000002" +TEST_UNICORNTEARS_ASSET_ID = "1000000000000000000000000000000000000000000000000000000000000003" + +TEST_ASSET_ID_NAME_MAPPING: Dict[bytes32, Tuple[uint32, str]] = { + bytes32.from_hexstr(TEST_DUCKSAUCE_ASSET_ID): (uint32(2), "DuckSauce"), + bytes32.from_hexstr(TEST_CRUNCHBERRIES_ASSET_ID): (uint32(3), "CrunchBerries"), + bytes32.from_hexstr(TEST_UNICORNTEARS_ASSET_ID): (uint32(4), "UnicornTears"), +} + + +async def cat_name_resolver(asset_id: bytes32) -> Optional[Tuple[Optional[uint32], str]]: + return TEST_ASSET_ID_NAME_MAPPING.get(asset_id) + + +@pytest.mark.asyncio +async def test_print_offer_summary_xch(capsys: Any) -> None: + summary_dict = {"xch": 1_000_000_000_000} + + await print_offer_summary(cat_name_resolver, summary_dict) + + captured = capsys.readouterr() + + assert "XCH (Wallet ID: 1): 1.0 (1000000000000 mojos)" in captured.out + + +@pytest.mark.asyncio +async def test_print_offer_summary_cat(capsys: Any) -> None: + summary_dict = { + TEST_DUCKSAUCE_ASSET_ID: 1_000, + } + + await print_offer_summary(cat_name_resolver, summary_dict) + + captured = capsys.readouterr() + + assert "DuckSauce (Wallet ID: 2): 1.0 (1000 mojos)" in captured.out + + +@pytest.mark.asyncio +async def test_print_offer_summary_multiple_cats(capsys: Any) -> None: + summary_dict = { + TEST_DUCKSAUCE_ASSET_ID: 1_000, + TEST_CRUNCHBERRIES_ASSET_ID: 2_000, + } + + await print_offer_summary(cat_name_resolver, summary_dict) + + captured = capsys.readouterr() + + assert "DuckSauce (Wallet ID: 2): 1.0 (1000 mojos)" in captured.out + assert "CrunchBerries (Wallet ID: 3): 2.0 (2000 mojos)" in captured.out + + +@pytest.mark.asyncio +async def test_print_offer_summary_xch_and_cats(capsys: Any) -> None: + summary_dict = { + "xch": 2_500_000_000_000, + TEST_DUCKSAUCE_ASSET_ID: 1_111, + TEST_CRUNCHBERRIES_ASSET_ID: 2_222, + TEST_UNICORNTEARS_ASSET_ID: 3_333, + } + + await print_offer_summary(cat_name_resolver, summary_dict) + + captured = capsys.readouterr() + + assert "XCH (Wallet ID: 1): 2.5 (2500000000000 mojos)" in captured.out + assert "DuckSauce (Wallet ID: 2): 1.111 (1111 mojos)" in captured.out + assert "CrunchBerries (Wallet ID: 3): 2.222 (2222 mojos)" in captured.out + assert "UnicornTears (Wallet ID: 4): 3.333 (3333 mojos)" in captured.out + + +@pytest.mark.asyncio +async def test_print_offer_summary_xch_and_cats_with_zero_values(capsys: Any) -> None: + summary_dict = { + "xch": 0, + TEST_DUCKSAUCE_ASSET_ID: 0, + TEST_CRUNCHBERRIES_ASSET_ID: 0, + TEST_UNICORNTEARS_ASSET_ID: 0, + } + + await print_offer_summary(cat_name_resolver, summary_dict) + + captured = capsys.readouterr() + + assert "XCH (Wallet ID: 1): 0.0 (0 mojos)" in captured.out + assert "DuckSauce (Wallet ID: 2): 0.0 (0 mojos)" in captured.out + assert "CrunchBerries (Wallet ID: 3): 0.0 (0 mojos)" in captured.out + assert "UnicornTears (Wallet ID: 4): 0.0 (0 mojos)" in captured.out + + +@pytest.mark.asyncio +async def test_print_offer_summary_cat_with_fee_and_change(capsys: Any) -> None: + summary_dict = { + TEST_DUCKSAUCE_ASSET_ID: 1_000, + "unknown": 3_456, + } + + await print_offer_summary(cat_name_resolver, summary_dict, has_fee=True) + + captured = capsys.readouterr() + + assert "DuckSauce (Wallet ID: 2): 1.0 (1000 mojos)" in captured.out + assert "Unknown: 3456 mojos [Typically represents change returned from the included fee]" in captured.out + + +@pytest.mark.asyncio +async def test_print_offer_summary_xch_with_one_mojo(capsys: Any) -> None: + summary_dict = {"xch": 1} + + await print_offer_summary(cat_name_resolver, summary_dict) + + captured = capsys.readouterr() + + assert "XCH (Wallet ID: 1): 1e-12 (1 mojo)" in captured.out From 492503f10975e79635e97ba699f05484f0c167b4 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Wed, 6 Apr 2022 01:11:03 +0200 Subject: [PATCH 36/63] print average block rate at different block height windows (#11064) --- tools/test_full_sync.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/test_full_sync.py b/tools/test_full_sync.py index f895647479..d5e1a07413 100755 --- a/tools/test_full_sync.py +++ b/tools/test_full_sync.py @@ -83,6 +83,7 @@ async def run_sync_test(file: Path, db_version, profile: bool, single_thread: bo print() counter = 0 + height = 0 async with aiosqlite.connect(file) as in_db: rows = await in_db.execute( @@ -109,10 +110,16 @@ async def run_sync_test(file: Path, db_version, profile: bool, single_thread: bo assert success assert advanced_peak counter += len(block_batch) - print(f"\rheight {counter} {counter/(time.monotonic() - start_time):0.2f} blocks/s ", end="") + height += len(block_batch) + print(f"\rheight {height} {counter/(time.monotonic() - start_time):0.2f} blocks/s ", end="") block_batch = [] if check_log.exit_with_failure: raise RuntimeError("error printed to log. exiting") + + if counter >= 100000: + start_time = time.monotonic() + counter = 0 + print() finally: print("closing full node") full_node._close() From c238ce17ba324f67dda4bcad4e2bb7ca2d2086ca Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 5 Apr 2022 20:00:10 -0400 Subject: [PATCH 37/63] add -d for Install.ps1 (#11062) --- Install.ps1 | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/Install.ps1 b/Install.ps1 index c97d6d737e..ad6b260861 100644 --- a/Install.ps1 +++ b/Install.ps1 @@ -1,5 +1,16 @@ +param( + [Parameter(HelpMessage="install development dependencies")] + [switch]$d = $False +) + $ErrorActionPreference = "Stop" +$extras = @() +if ($d) +{ + $extras += "dev" +} + if ([Environment]::Is64BitOperatingSystem -eq $false) { Write-Output "Chia requires a 64-bit Windows installation" @@ -49,11 +60,21 @@ if ($openSSLVersion -lt 269488367) Write-Output "Anything before 1.1.1n is vulnerable to CVE-2022-0778." } +if ($extras.length -gt 0) +{ + $extras_cli = $extras -join "," + $extras_cli = "[$extras_cli]" +} +else +{ + $extras_cli = "" +} + py -m venv venv venv\scripts\python -m pip install --upgrade pip setuptools wheel venv\scripts\pip install --extra-index-url https://pypi.chia.net/simple/ miniupnpc==2.2.2 -venv\scripts\pip install --editable . --extra-index-url https://pypi.chia.net/simple/ +venv\scripts\pip install --editable ".$extras_cli" --extra-index-url https://pypi.chia.net/simple/ Write-Output "" Write-Output "Chia blockchain .\Install.ps1 complete." From d35c414c09d5befd33fa378738a49db7e3f3db36 Mon Sep 17 00:00:00 2001 From: Jeff Date: Wed, 6 Apr 2022 20:14:49 -0700 Subject: [PATCH 38/63] Set keychain_proxy to None in await_closed() to support reinitialization (#11075) * Set keychain_proxy to None in await_closed() to support reinitialization. * Added `shutting_down` param to _await_closed() to control whether the keychain_proxy is closed. --- chia/farmer/farmer.py | 10 ++++++---- chia/rpc/wallet_rpc_api.py | 2 +- chia/wallet/wallet_node.py | 10 ++++++---- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/chia/farmer/farmer.py b/chia/farmer/farmer.py index 9979925859..33f3b973a7 100644 --- a/chia/farmer/farmer.py +++ b/chia/farmer/farmer.py @@ -140,7 +140,7 @@ class Farmer: self.harvester_cache: Dict[str, Dict[str, HarvesterCacheEntry]] = {} async def ensure_keychain_proxy(self) -> KeychainProxy: - if not self.keychain_proxy: + if self.keychain_proxy is None: if self.local_keychain: self.keychain_proxy = wrap_local_keychain(self.local_keychain, log=self.log) else: @@ -212,13 +212,15 @@ class Farmer: def _close(self): self._shut_down = True - async def _await_closed(self): + async def _await_closed(self, shutting_down: bool = True): if self.cache_clear_task is not None: await self.cache_clear_task if self.update_pool_state_task is not None: await self.update_pool_state_task - if self.keychain_proxy is not None: - await self.keychain_proxy.close() + if shutting_down and self.keychain_proxy is not None: + proxy = self.keychain_proxy + self.keychain_proxy = None + await proxy.close() await asyncio.sleep(0.5) # https://docs.aiohttp.org/en/stable/client_advanced.html#graceful-shutdown self.started = False diff --git a/chia/rpc/wallet_rpc_api.py b/chia/rpc/wallet_rpc_api.py index 9ea76e3d24..e4439ea328 100644 --- a/chia/rpc/wallet_rpc_api.py +++ b/chia/rpc/wallet_rpc_api.py @@ -157,7 +157,7 @@ class WalletRpcApi: """ if self.service is not None: self.service._close() - peers_close_task: Optional[asyncio.Task] = await self.service._await_closed() + peers_close_task: Optional[asyncio.Task] = await self.service._await_closed(shutting_down=False) if peers_close_task is not None: await peers_close_task diff --git a/chia/wallet/wallet_node.py b/chia/wallet/wallet_node.py index 6b942ee5f1..b64f38cf1a 100644 --- a/chia/wallet/wallet_node.py +++ b/chia/wallet/wallet_node.py @@ -143,7 +143,7 @@ class WalletNode: self.LONG_SYNC_THRESHOLD = 200 async def ensure_keychain_proxy(self) -> KeychainProxy: - if not self.keychain_proxy: + if self.keychain_proxy is None: if self.local_keychain: self.keychain_proxy = wrap_local_keychain(self.local_keychain, log=self.log) else: @@ -259,7 +259,7 @@ class WalletNode: if self._secondary_peer_sync_task is not None: self._secondary_peer_sync_task.cancel() - async def _await_closed(self): + async def _await_closed(self, shutting_down: bool = True): self.log.info("self._await_closed") if self.server is not None: @@ -269,8 +269,10 @@ class WalletNode: if self.wallet_state_manager is not None: await self.wallet_state_manager._await_closed() self.wallet_state_manager = None - if self.keychain_proxy is not None: - await self.keychain_proxy.close() + if shutting_down and self.keychain_proxy is not None: + proxy = self.keychain_proxy + self.keychain_proxy = None + await proxy.close() await asyncio.sleep(0.5) # https://docs.aiohttp.org/en/stable/client_advanced.html#graceful-shutdown self.logged_in = False self.wallet_peers = None From 6026d734cf4eedf47fdc7250865186c7e09bcd46 Mon Sep 17 00:00:00 2001 From: Amine Khaldi Date: Thu, 7 Apr 2022 04:15:10 +0100 Subject: [PATCH 39/63] Significantly speedup preparing test blocks and plots by opting for a release download instead of a shallow git clone, and also by putting a caching layer on top of that. (#11065) --- .../workflows/build-test-macos-blockchain.yml | 24 ++++++++++++++----- .../build-test-macos-core-daemon.yml | 24 ++++++++++++++----- ...ld-test-macos-core-full_node-full_sync.yml | 24 ++++++++++++++----- ...build-test-macos-core-full_node-stores.yml | 24 ++++++++++++++----- .../build-test-macos-core-full_node.yml | 24 ++++++++++++++----- .../build-test-macos-core-server.yml | 24 ++++++++++++++----- .../workflows/build-test-macos-core-ssl.yml | 24 ++++++++++++++----- .../workflows/build-test-macos-core-util.yml | 24 ++++++++++++++----- .github/workflows/build-test-macos-core.yml | 24 ++++++++++++++----- .../build-test-macos-farmer_harvester.yml | 24 ++++++++++++++----- .../workflows/build-test-macos-plotting.yml | 24 ++++++++++++++----- .github/workflows/build-test-macos-pools.yml | 24 ++++++++++++++----- .../workflows/build-test-macos-simulation.yml | 24 ++++++++++++++----- .../build-test-macos-wallet-cat_wallet.yml | 24 ++++++++++++++----- .../workflows/build-test-macos-wallet-rpc.yml | 24 ++++++++++++++----- .../build-test-macos-wallet-simple_sync.yml | 24 ++++++++++++++----- .../build-test-macos-wallet-sync.yml | 24 ++++++++++++++----- .github/workflows/build-test-macos-wallet.yml | 24 ++++++++++++++----- .../build-test-macos-weight_proof.yml | 24 ++++++++++++++----- .../build-test-ubuntu-blockchain.yml | 24 ++++++++++++++----- .../build-test-ubuntu-core-daemon.yml | 24 ++++++++++++++----- ...d-test-ubuntu-core-full_node-full_sync.yml | 24 ++++++++++++++----- ...uild-test-ubuntu-core-full_node-stores.yml | 24 ++++++++++++++----- .../build-test-ubuntu-core-full_node.yml | 24 ++++++++++++++----- .../build-test-ubuntu-core-server.yml | 24 ++++++++++++++----- .../workflows/build-test-ubuntu-core-ssl.yml | 24 ++++++++++++++----- .../workflows/build-test-ubuntu-core-util.yml | 24 ++++++++++++++----- .github/workflows/build-test-ubuntu-core.yml | 24 ++++++++++++++----- .../build-test-ubuntu-farmer_harvester.yml | 24 ++++++++++++++----- .../workflows/build-test-ubuntu-plotting.yml | 24 ++++++++++++++----- .github/workflows/build-test-ubuntu-pools.yml | 24 ++++++++++++++----- .../build-test-ubuntu-simulation.yml | 24 ++++++++++++++----- .../build-test-ubuntu-wallet-cat_wallet.yml | 24 ++++++++++++++----- .../build-test-ubuntu-wallet-rpc.yml | 24 ++++++++++++++----- .../build-test-ubuntu-wallet-simple_sync.yml | 24 ++++++++++++++----- .../build-test-ubuntu-wallet-sync.yml | 24 ++++++++++++++----- .../workflows/build-test-ubuntu-wallet.yml | 24 ++++++++++++++----- .../build-test-ubuntu-weight_proof.yml | 24 ++++++++++++++----- .../checkout-test-plots.include.yml | 24 ++++++++++++++----- 39 files changed, 702 insertions(+), 234 deletions(-) diff --git a/.github/workflows/build-test-macos-blockchain.yml b/.github/workflows/build-test-macos-blockchain.yml index b39022abb9..8fcb0b1c10 100644 --- a/.github/workflows/build-test-macos-blockchain.yml +++ b/.github/workflows/build-test-macos-blockchain.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-core-daemon.yml b/.github/workflows/build-test-macos-core-daemon.yml index 043fb03b91..d017da0e3e 100644 --- a/.github/workflows/build-test-macos-core-daemon.yml +++ b/.github/workflows/build-test-macos-core-daemon.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-core-full_node-full_sync.yml b/.github/workflows/build-test-macos-core-full_node-full_sync.yml index bd45bb6dc2..61825197ea 100644 --- a/.github/workflows/build-test-macos-core-full_node-full_sync.yml +++ b/.github/workflows/build-test-macos-core-full_node-full_sync.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-core-full_node-stores.yml b/.github/workflows/build-test-macos-core-full_node-stores.yml index 65a81bb923..b25f81ea34 100644 --- a/.github/workflows/build-test-macos-core-full_node-stores.yml +++ b/.github/workflows/build-test-macos-core-full_node-stores.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-core-full_node.yml b/.github/workflows/build-test-macos-core-full_node.yml index 5e833ac6df..03b9cb834e 100644 --- a/.github/workflows/build-test-macos-core-full_node.yml +++ b/.github/workflows/build-test-macos-core-full_node.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-core-server.yml b/.github/workflows/build-test-macos-core-server.yml index f81fc0de33..03a771e6b6 100644 --- a/.github/workflows/build-test-macos-core-server.yml +++ b/.github/workflows/build-test-macos-core-server.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-core-ssl.yml b/.github/workflows/build-test-macos-core-ssl.yml index 95f52c10ec..51c348f8fe 100644 --- a/.github/workflows/build-test-macos-core-ssl.yml +++ b/.github/workflows/build-test-macos-core-ssl.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-core-util.yml b/.github/workflows/build-test-macos-core-util.yml index 984c43b7f0..95c8603deb 100644 --- a/.github/workflows/build-test-macos-core-util.yml +++ b/.github/workflows/build-test-macos-core-util.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-core.yml b/.github/workflows/build-test-macos-core.yml index 52a592a174..df94b8d09d 100644 --- a/.github/workflows/build-test-macos-core.yml +++ b/.github/workflows/build-test-macos-core.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-farmer_harvester.yml b/.github/workflows/build-test-macos-farmer_harvester.yml index 60bcf838f1..3017a35790 100644 --- a/.github/workflows/build-test-macos-farmer_harvester.yml +++ b/.github/workflows/build-test-macos-farmer_harvester.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-plotting.yml b/.github/workflows/build-test-macos-plotting.yml index 83ef742b82..80a5c9ceba 100644 --- a/.github/workflows/build-test-macos-plotting.yml +++ b/.github/workflows/build-test-macos-plotting.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-pools.yml b/.github/workflows/build-test-macos-pools.yml index e83532a8ff..39969bddaf 100644 --- a/.github/workflows/build-test-macos-pools.yml +++ b/.github/workflows/build-test-macos-pools.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-simulation.yml b/.github/workflows/build-test-macos-simulation.yml index 5a1f7f7c3e..44a920ac31 100644 --- a/.github/workflows/build-test-macos-simulation.yml +++ b/.github/workflows/build-test-macos-simulation.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-wallet-cat_wallet.yml b/.github/workflows/build-test-macos-wallet-cat_wallet.yml index 32bc83d1ba..d38bb5a5a5 100644 --- a/.github/workflows/build-test-macos-wallet-cat_wallet.yml +++ b/.github/workflows/build-test-macos-wallet-cat_wallet.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-wallet-rpc.yml b/.github/workflows/build-test-macos-wallet-rpc.yml index 63de5e94b0..674e246e6e 100644 --- a/.github/workflows/build-test-macos-wallet-rpc.yml +++ b/.github/workflows/build-test-macos-wallet-rpc.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-wallet-simple_sync.yml b/.github/workflows/build-test-macos-wallet-simple_sync.yml index 0b5ee26a52..c80bd4d72b 100644 --- a/.github/workflows/build-test-macos-wallet-simple_sync.yml +++ b/.github/workflows/build-test-macos-wallet-simple_sync.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-wallet-sync.yml b/.github/workflows/build-test-macos-wallet-sync.yml index af8635d600..bedeaa6016 100644 --- a/.github/workflows/build-test-macos-wallet-sync.yml +++ b/.github/workflows/build-test-macos-wallet-sync.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-wallet.yml b/.github/workflows/build-test-macos-wallet.yml index a756ecb9fa..100585f786 100644 --- a/.github/workflows/build-test-macos-wallet.yml +++ b/.github/workflows/build-test-macos-wallet.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-weight_proof.yml b/.github/workflows/build-test-macos-weight_proof.yml index 019df26033..0870d1f954 100644 --- a/.github/workflows/build-test-macos-weight_proof.yml +++ b/.github/workflows/build-test-macos-weight_proof.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-blockchain.yml b/.github/workflows/build-test-ubuntu-blockchain.yml index fda7296d91..e6a5aa8723 100644 --- a/.github/workflows/build-test-ubuntu-blockchain.yml +++ b/.github/workflows/build-test-ubuntu-blockchain.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-core-daemon.yml b/.github/workflows/build-test-ubuntu-core-daemon.yml index 9b69c941e9..19f60ca1e7 100644 --- a/.github/workflows/build-test-ubuntu-core-daemon.yml +++ b/.github/workflows/build-test-ubuntu-core-daemon.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-core-full_node-full_sync.yml b/.github/workflows/build-test-ubuntu-core-full_node-full_sync.yml index 8e5769be2d..546c3d21bd 100644 --- a/.github/workflows/build-test-ubuntu-core-full_node-full_sync.yml +++ b/.github/workflows/build-test-ubuntu-core-full_node-full_sync.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-core-full_node-stores.yml b/.github/workflows/build-test-ubuntu-core-full_node-stores.yml index d9cc30278c..be9cef49ad 100644 --- a/.github/workflows/build-test-ubuntu-core-full_node-stores.yml +++ b/.github/workflows/build-test-ubuntu-core-full_node-stores.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-core-full_node.yml b/.github/workflows/build-test-ubuntu-core-full_node.yml index 9e703a06b3..e59db07ded 100644 --- a/.github/workflows/build-test-ubuntu-core-full_node.yml +++ b/.github/workflows/build-test-ubuntu-core-full_node.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-core-server.yml b/.github/workflows/build-test-ubuntu-core-server.yml index 47bb7e7f4f..a770ae9a40 100644 --- a/.github/workflows/build-test-ubuntu-core-server.yml +++ b/.github/workflows/build-test-ubuntu-core-server.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-core-ssl.yml b/.github/workflows/build-test-ubuntu-core-ssl.yml index ba3c7945f5..050996b890 100644 --- a/.github/workflows/build-test-ubuntu-core-ssl.yml +++ b/.github/workflows/build-test-ubuntu-core-ssl.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-core-util.yml b/.github/workflows/build-test-ubuntu-core-util.yml index 938fd0f403..68b23bd9c9 100644 --- a/.github/workflows/build-test-ubuntu-core-util.yml +++ b/.github/workflows/build-test-ubuntu-core-util.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-core.yml b/.github/workflows/build-test-ubuntu-core.yml index 01be4c7181..bd32713578 100644 --- a/.github/workflows/build-test-ubuntu-core.yml +++ b/.github/workflows/build-test-ubuntu-core.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-farmer_harvester.yml b/.github/workflows/build-test-ubuntu-farmer_harvester.yml index 6cd3a81af0..127ead45c0 100644 --- a/.github/workflows/build-test-ubuntu-farmer_harvester.yml +++ b/.github/workflows/build-test-ubuntu-farmer_harvester.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-plotting.yml b/.github/workflows/build-test-ubuntu-plotting.yml index 26c186f954..099f50f6d7 100644 --- a/.github/workflows/build-test-ubuntu-plotting.yml +++ b/.github/workflows/build-test-ubuntu-plotting.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-pools.yml b/.github/workflows/build-test-ubuntu-pools.yml index 9ae59262f9..a4284b00e1 100644 --- a/.github/workflows/build-test-ubuntu-pools.yml +++ b/.github/workflows/build-test-ubuntu-pools.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-simulation.yml b/.github/workflows/build-test-ubuntu-simulation.yml index be0a0f55c3..805f646b14 100644 --- a/.github/workflows/build-test-ubuntu-simulation.yml +++ b/.github/workflows/build-test-ubuntu-simulation.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-wallet-cat_wallet.yml b/.github/workflows/build-test-ubuntu-wallet-cat_wallet.yml index b52dfde0c5..25835db6d9 100644 --- a/.github/workflows/build-test-ubuntu-wallet-cat_wallet.yml +++ b/.github/workflows/build-test-ubuntu-wallet-cat_wallet.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-wallet-rpc.yml b/.github/workflows/build-test-ubuntu-wallet-rpc.yml index 05b4cef7a9..4ceb70b32b 100644 --- a/.github/workflows/build-test-ubuntu-wallet-rpc.yml +++ b/.github/workflows/build-test-ubuntu-wallet-rpc.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-wallet-simple_sync.yml b/.github/workflows/build-test-ubuntu-wallet-simple_sync.yml index fdaa81b6dd..899501734b 100644 --- a/.github/workflows/build-test-ubuntu-wallet-simple_sync.yml +++ b/.github/workflows/build-test-ubuntu-wallet-simple_sync.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-wallet-sync.yml b/.github/workflows/build-test-ubuntu-wallet-sync.yml index 797731d3f3..5da886be88 100644 --- a/.github/workflows/build-test-ubuntu-wallet-sync.yml +++ b/.github/workflows/build-test-ubuntu-wallet-sync.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-wallet.yml b/.github/workflows/build-test-ubuntu-wallet.yml index 28f2b1da0a..a6f734c9ae 100644 --- a/.github/workflows/build-test-ubuntu-wallet.yml +++ b/.github/workflows/build-test-ubuntu-wallet.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-weight_proof.yml b/.github/workflows/build-test-ubuntu-weight_proof.yml index 34f33582e2..abccbf8229 100644 --- a/.github/workflows/build-test-ubuntu-weight_proof.yml +++ b/.github/workflows/build-test-ubuntu-weight_proof.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/tests/runner_templates/checkout-test-plots.include.yml b/tests/runner_templates/checkout-test-plots.include.yml index 1118ef00b0..6d3239bbe6 100644 --- a/tests/runner_templates/checkout-test-plots.include.yml +++ b/tests/runner_templates/checkout-test-plots.include.yml @@ -1,7 +1,19 @@ - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia From 42245d74ebe73e819e1bad4614fa6be41f51e143 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Apr 2022 09:17:53 -0700 Subject: [PATCH 40/63] Bump github/super-linter from 4.9.1 to 4.9.2 (#11067) Bumps [github/super-linter](https://github.com/github/super-linter) from 4.9.1 to 4.9.2. - [Release notes](https://github.com/github/super-linter/releases) - [Changelog](https://github.com/github/super-linter/blob/main/docs/release-process.md) - [Commits](https://github.com/github/super-linter/compare/v4.9.1...v4.9.2) --- updated-dependencies: - dependency-name: github/super-linter dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/super-linter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/super-linter.yml b/.github/workflows/super-linter.yml index 8059e90352..2fa6bf3863 100644 --- a/.github/workflows/super-linter.yml +++ b/.github/workflows/super-linter.yml @@ -55,7 +55,7 @@ jobs: # Run Linter against code base # ################################ - name: Lint Code Base - uses: github/super-linter@v4.9.1 + uses: github/super-linter@v4.9.2 # uses: docker://github/super-linter:v3.10.2 env: VALIDATE_ALL_CODEBASE: true From c0d346d428325e6cc4a670b265eabee93dd7f6d4 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 7 Apr 2022 12:18:15 -0400 Subject: [PATCH 41/63] Remove dead snakes usage from benchmark tests (#11053) --- .github/workflows/benchmarks.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index d01ed34c8d..b7589f89c4 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -55,15 +55,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Install ubuntu dependencies - env: - DEBIAN_FRONTEND: noninteractive - run: | - sudo apt-get install -y software-properties-common - sudo add-apt-repository ppa:deadsnakes/ppa - sudo apt-get update - sudo apt-get install -y python${{ matrix.python-version }}-venv python${{ matrix.python-version }}-distutils git - - name: Checkout test blocks and plots uses: actions/checkout@v3 with: From d892e14c646511227b4f3dec1676b93ef8314810 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 7 Apr 2022 12:18:54 -0400 Subject: [PATCH 42/63] Handle INSTALL_PYTHON_VERSION in Install.ps1, otherwise search 3.9/3.8/3.7 (#11034) * Handle INSTALL_PYTHON_VERSION in Install.ps1, otherwise search 3.9/3.8/3.7 * fix python availability check in Install.ps1 * when Install.ps1 does not find an acceptable python, list supported versions in order * Update Install.ps1 Co-authored-by: Matt Hauff Co-authored-by: Matt Hauff --- Install.ps1 | 48 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/Install.ps1 b/Install.ps1 index ad6b260861..8350d2082c 100644 --- a/Install.ps1 +++ b/Install.ps1 @@ -43,17 +43,47 @@ if ($null -eq (Get-Command py -ErrorAction SilentlyContinue)) Exit 1 } -$pythonVersion = (py --version).split(" ")[1] -if ([version]$pythonVersion -lt [version]"3.7.0") +$supportedPythonVersions = "3.9", "3.8", "3.7" +if (Test-Path env:INSTALL_PYTHON_VERSION) { - Write-Output "Found Python version:" $pythonVersion - Write-Output "Installation requires Python 3.7 or later" - Exit 1 + $pythonVersion = $env:INSTALL_PYTHON_VERSION } -Write-Output "Python version is:" $pythonVersion +else +{ + foreach ($version in $supportedPythonVersions) + { + try + { + py -$version --version 2>&1 >$null + $result = $? + } + catch + { + $result = $false + } + if ($result) + { + $pythonVersion = $version + break + } + } -$openSSLVersionStr = (py -c 'import ssl; print(ssl.OPENSSL_VERSION)') -$openSSLVersion = (py -c 'import ssl; print(ssl.OPENSSL_VERSION_NUMBER)') + if (-not $pythonVersion) + { + $reversedPythonVersions = $supportedPythonVersions.clone() + [array]::Reverse($reversedPythonVersions) + $reversedPythonVersions = $reversedPythonVersions -join ", " + Write-Output "No usable Python version found, supported versions are: $reversedPythonVersions" + Exit 1 + } +} + +$fullPythonVersion = (py -$pythonVersion --version).split(" ")[1] + +Write-Output "Python version is: $fullPythonVersion" + +$openSSLVersionStr = (py -$pythonVersion -c 'import ssl; print(ssl.OPENSSL_VERSION)') +$openSSLVersion = (py -$pythonVersion -c 'import ssl; print(ssl.OPENSSL_VERSION_NUMBER)') if ($openSSLVersion -lt 269488367) { Write-Output "Found Python with OpenSSL version:" $openSSLVersionStr @@ -70,7 +100,7 @@ else $extras_cli = "" } -py -m venv venv +py -$pythonVersion -m venv venv venv\scripts\python -m pip install --upgrade pip setuptools wheel venv\scripts\pip install --extra-index-url https://pypi.chia.net/simple/ miniupnpc==2.2.2 From 0917d0ae781dfbd3db543e9f4d9fa794895804d2 Mon Sep 17 00:00:00 2001 From: dustinface <35775977+xdustinface@users.noreply.github.com> Date: Thu, 7 Apr 2022 18:19:37 +0200 Subject: [PATCH 43/63] wallet: Drop `puzzles/genesis_checkers.py` and related puzzles (#10790) Its all duplicated code and puzzles as far as i can tell, see `chia/wallet/puzzles/tails.py`. --- .isort.cfg | 1 - chia/wallet/cat_wallet/cat_wallet.py | 2 +- .../puzzles/delegated_genesis_checker.clvm | 25 --- .../delegated_genesis_checker.clvm.hex | 1 - ...egated_genesis_checker.clvm.hex.sha256tree | 1 - .../puzzles/genesis-by-coin-id-with-0.clvm | 26 --- .../genesis-by-coin-id-with-0.clvm.hex | 1 - ...esis-by-coin-id-with-0.clvm.hex.sha256tree | 1 - .../genesis-by-puzzle-hash-with-0.clvm | 24 -- .../genesis-by-puzzle-hash-with-0.clvm.hex | 1 - ...-by-puzzle-hash-with-0.clvm.hex.sha256tree | 1 - chia/wallet/puzzles/genesis_checkers.py | 208 ------------------ mypy.ini | 2 +- tests/clvm/test_clvm_compilation.py | 3 - 14 files changed, 2 insertions(+), 295 deletions(-) delete mode 100644 chia/wallet/puzzles/delegated_genesis_checker.clvm delete mode 100644 chia/wallet/puzzles/delegated_genesis_checker.clvm.hex delete mode 100644 chia/wallet/puzzles/delegated_genesis_checker.clvm.hex.sha256tree delete mode 100644 chia/wallet/puzzles/genesis-by-coin-id-with-0.clvm delete mode 100644 chia/wallet/puzzles/genesis-by-coin-id-with-0.clvm.hex delete mode 100644 chia/wallet/puzzles/genesis-by-coin-id-with-0.clvm.hex.sha256tree delete mode 100644 chia/wallet/puzzles/genesis-by-puzzle-hash-with-0.clvm delete mode 100644 chia/wallet/puzzles/genesis-by-puzzle-hash-with-0.clvm.hex delete mode 100644 chia/wallet/puzzles/genesis-by-puzzle-hash-with-0.clvm.hex.sha256tree delete mode 100644 chia/wallet/puzzles/genesis_checkers.py diff --git a/.isort.cfg b/.isort.cfg index 291c8862f2..9ed754a63a 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -120,7 +120,6 @@ extend_skip= chia/wallet/did_wallet/did_wallet.py chia/wallet/lineage_proof.py chia/wallet/payment.py - chia/wallet/puzzles/genesis_checkers.py chia/wallet/puzzles/load_clvm.py chia/wallet/puzzles/prefarm/make_prefarm_ph.py chia/wallet/puzzles/prefarm/spend_prefarm.py diff --git a/chia/wallet/cat_wallet/cat_wallet.py b/chia/wallet/cat_wallet/cat_wallet.py index 4b9f2c19a1..b58c9b9ce9 100644 --- a/chia/wallet/cat_wallet/cat_wallet.py +++ b/chia/wallet/cat_wallet/cat_wallet.py @@ -36,7 +36,7 @@ from chia.wallet.derivation_record import DerivationRecord from chia.wallet.cat_wallet.lineage_store import CATLineageStore from chia.wallet.lineage_proof import LineageProof from chia.wallet.payment import Payment -from chia.wallet.puzzles.genesis_checkers import ALL_LIMITATIONS_PROGRAMS +from chia.wallet.puzzles.tails import ALL_LIMITATIONS_PROGRAMS from chia.wallet.puzzles.p2_delegated_puzzle_or_hidden_puzzle import ( DEFAULT_HIDDEN_PUZZLE_HASH, calculate_synthetic_secret_key, diff --git a/chia/wallet/puzzles/delegated_genesis_checker.clvm b/chia/wallet/puzzles/delegated_genesis_checker.clvm deleted file mode 100644 index 57cc677bd4..0000000000 --- a/chia/wallet/puzzles/delegated_genesis_checker.clvm +++ /dev/null @@ -1,25 +0,0 @@ -; This is a "limitations_program" for use with cat.clvm. -(mod ( - PUBKEY - Truths - parent_is_cat - lineage_proof - delta - inner_conditions - ( - delegated_puzzle - delegated_solution - ) - ) - - (include condition_codes.clvm) - - (defun sha256tree1 (TREE) - (if (l TREE) - (sha256 2 (sha256tree1 (f TREE)) (sha256tree1 (r TREE))) - (sha256 1 TREE))) - - (c (list AGG_SIG_UNSAFE PUBKEY (sha256tree1 delegated_puzzle)) - (a delegated_puzzle (c Truths (c parent_is_cat (c lineage_proof (c delta (c inner_conditions delegated_solution)))))) - ) -) \ No newline at end of file diff --git a/chia/wallet/puzzles/delegated_genesis_checker.clvm.hex b/chia/wallet/puzzles/delegated_genesis_checker.clvm.hex deleted file mode 100644 index d131b36b4d..0000000000 --- a/chia/wallet/puzzles/delegated_genesis_checker.clvm.hex +++ /dev/null @@ -1 +0,0 @@ -ff02ffff01ff04ffff04ff04ffff04ff05ffff04ffff02ff06ffff04ff02ffff04ff82027fff80808080ff80808080ffff02ff82027fffff04ff0bffff04ff17ffff04ff2fffff04ff5fffff04ff81bfff82057f80808080808080ffff04ffff01ff31ff02ffff03ffff07ff0580ffff01ff0bffff0102ffff02ff06ffff04ff02ffff04ff09ff80808080ffff02ff06ffff04ff02ffff04ff0dff8080808080ffff01ff0bffff0101ff058080ff0180ff018080 \ No newline at end of file diff --git a/chia/wallet/puzzles/delegated_genesis_checker.clvm.hex.sha256tree b/chia/wallet/puzzles/delegated_genesis_checker.clvm.hex.sha256tree deleted file mode 100644 index f1d6d7408d..0000000000 --- a/chia/wallet/puzzles/delegated_genesis_checker.clvm.hex.sha256tree +++ /dev/null @@ -1 +0,0 @@ -999c3696e167f8a79d938adc11feba3a3dcb39ccff69a426d570706e7b8ec399 diff --git a/chia/wallet/puzzles/genesis-by-coin-id-with-0.clvm b/chia/wallet/puzzles/genesis-by-coin-id-with-0.clvm deleted file mode 100644 index c136bb0703..0000000000 --- a/chia/wallet/puzzles/genesis-by-coin-id-with-0.clvm +++ /dev/null @@ -1,26 +0,0 @@ -; This is a "genesis checker" for use with cc.clvm. -; -; This checker allows new CATs to be created if they have a particular coin id as parent -; -; The genesis_id is curried in, making this lineage_check program unique and giving the CAT it's uniqueness -(mod ( - GENESIS_ID - Truths - parent_is_cat - lineage_proof - delta - inner_conditions - _ - ) - - (include cat_truths.clib) - - (if delta - (x) - (if (= (my_parent_cat_truth Truths) GENESIS_ID) - () - (x) - ) - ) - -) diff --git a/chia/wallet/puzzles/genesis-by-coin-id-with-0.clvm.hex b/chia/wallet/puzzles/genesis-by-coin-id-with-0.clvm.hex deleted file mode 100644 index 3f287e4482..0000000000 --- a/chia/wallet/puzzles/genesis-by-coin-id-with-0.clvm.hex +++ /dev/null @@ -1 +0,0 @@ -ff02ffff03ff2fffff01ff0880ffff01ff02ffff03ffff09ff2dff0280ff80ffff01ff088080ff018080ff0180 \ No newline at end of file diff --git a/chia/wallet/puzzles/genesis-by-coin-id-with-0.clvm.hex.sha256tree b/chia/wallet/puzzles/genesis-by-coin-id-with-0.clvm.hex.sha256tree deleted file mode 100644 index f240ff9417..0000000000 --- a/chia/wallet/puzzles/genesis-by-coin-id-with-0.clvm.hex.sha256tree +++ /dev/null @@ -1 +0,0 @@ -493afb89eed93ab86741b2aa61b8f5de495d33ff9b781dfc8919e602b2afa150 \ No newline at end of file diff --git a/chia/wallet/puzzles/genesis-by-puzzle-hash-with-0.clvm b/chia/wallet/puzzles/genesis-by-puzzle-hash-with-0.clvm deleted file mode 100644 index 720465075d..0000000000 --- a/chia/wallet/puzzles/genesis-by-puzzle-hash-with-0.clvm +++ /dev/null @@ -1,24 +0,0 @@ -; This is a "limitations_program" for use with cat.clvm. -; -; This checker allows new CATs to be created if their parent has a particular puzzle hash -(mod ( - GENESIS_PUZZLE_HASH - Truths - parent_is_cat - lineage_proof - delta - inner_conditions - (parent_parent_id parent_amount) - ) - - (include cat_truths.clib) - - ; Returns nil since we don't need to add any conditions - (if delta - (x) - (if (= (sha256 parent_parent_id GENESIS_PUZZLE_HASH parent_amount) (my_parent_cat_truth Truths)) - () - (x) - ) - ) -) diff --git a/chia/wallet/puzzles/genesis-by-puzzle-hash-with-0.clvm.hex b/chia/wallet/puzzles/genesis-by-puzzle-hash-with-0.clvm.hex deleted file mode 100644 index 2d367721d1..0000000000 --- a/chia/wallet/puzzles/genesis-by-puzzle-hash-with-0.clvm.hex +++ /dev/null @@ -1 +0,0 @@ -ff02ffff03ff2fffff01ff0880ffff01ff02ffff03ffff09ffff0bff82013fff02ff8202bf80ff2d80ff80ffff01ff088080ff018080ff0180 \ No newline at end of file diff --git a/chia/wallet/puzzles/genesis-by-puzzle-hash-with-0.clvm.hex.sha256tree b/chia/wallet/puzzles/genesis-by-puzzle-hash-with-0.clvm.hex.sha256tree deleted file mode 100644 index 69cdc4bce6..0000000000 --- a/chia/wallet/puzzles/genesis-by-puzzle-hash-with-0.clvm.hex.sha256tree +++ /dev/null @@ -1 +0,0 @@ -de5a6e06d41518be97ff6365694f4f89475dda773dede267caa33da63b434e36 \ No newline at end of file diff --git a/chia/wallet/puzzles/genesis_checkers.py b/chia/wallet/puzzles/genesis_checkers.py deleted file mode 100644 index b21ac5a7f9..0000000000 --- a/chia/wallet/puzzles/genesis_checkers.py +++ /dev/null @@ -1,208 +0,0 @@ -from typing import Tuple, Dict, List, Optional, Any - -from chia.types.blockchain_format.program import Program -from chia.types.blockchain_format.sized_bytes import bytes32 -from chia.types.spend_bundle import SpendBundle -from chia.util.ints import uint64 -from chia.util.byte_types import hexstr_to_bytes -from chia.wallet.lineage_proof import LineageProof -from chia.wallet.puzzles.load_clvm import load_clvm -from chia.wallet.cat_wallet.cat_utils import ( - CAT_MOD, - construct_cat_puzzle, - unsigned_spend_bundle_for_spendable_cats, - SpendableCAT, -) -from chia.wallet.cat_wallet.cat_info import CATInfo -from chia.wallet.transaction_record import TransactionRecord - -GENESIS_BY_ID_MOD = load_clvm("genesis-by-coin-id-with-0.clvm") -GENESIS_BY_PUZHASH_MOD = load_clvm("genesis-by-puzzle-hash-with-0.clvm") -EVERYTHING_WITH_SIG_MOD = load_clvm("everything_with_signature.clvm") -DELEGATED_LIMITATIONS_MOD = load_clvm("delegated_genesis_checker.clvm") - - -class LimitationsProgram: - @staticmethod - def match(uncurried_mod: Program, curried_args: Program) -> Tuple[bool, List[Program]]: - raise NotImplementedError("Need to implement 'match' on limitations programs") - - @staticmethod - def construct(args: List[Program]) -> Program: - raise NotImplementedError("Need to implement 'construct' on limitations programs") - - @staticmethod - def solve(args: List[Program], solution_dict: Dict) -> Program: - raise NotImplementedError("Need to implement 'solve' on limitations programs") - - @classmethod - async def generate_issuance_bundle( - cls, wallet, cat_tail_info: Dict, amount: uint64 - ) -> Tuple[TransactionRecord, SpendBundle]: - raise NotImplementedError("Need to implement 'generate_issuance_bundle' on limitations programs") - - -class GenesisById(LimitationsProgram): - """ - This TAIL allows for coins to be issued only by a specific "genesis" coin ID. - There can therefore only be one issuance. There is no minting or melting allowed. - """ - - @staticmethod - def match(uncurried_mod: Program, curried_args: Program) -> Tuple[bool, List[Program]]: - if uncurried_mod == GENESIS_BY_ID_MOD: - genesis_id = curried_args.first() - return True, [genesis_id] - else: - return False, [] - - @staticmethod - def construct(args: List[Program]) -> Program: - return GENESIS_BY_ID_MOD.curry(args[0]) - - @staticmethod - def solve(args: List[Program], solution_dict: Dict) -> Program: - return Program.to([]) - - @classmethod - async def generate_issuance_bundle(cls, wallet, _: Dict, amount: uint64) -> Tuple[TransactionRecord, SpendBundle]: - coins = await wallet.standard_wallet.select_coins(amount) - - origin = coins.copy().pop() - origin_id = origin.name() - - cat_inner: Program = await wallet.get_new_inner_puzzle() - await wallet.add_lineage(origin_id, LineageProof(), False) - genesis_coin_checker: Program = cls.construct([Program.to(origin_id)]) - - minted_cat_puzzle_hash: bytes32 = construct_cat_puzzle( - CAT_MOD, genesis_coin_checker.get_tree_hash(), cat_inner - ).get_tree_hash() - - tx_record: TransactionRecord = await wallet.standard_wallet.generate_signed_transaction( - amount, minted_cat_puzzle_hash, uint64(0), origin_id, coins - ) - assert tx_record.spend_bundle is not None - - inner_solution = wallet.standard_wallet.add_condition_to_solution( - Program.to([51, 0, -113, genesis_coin_checker, []]), - wallet.standard_wallet.make_solution( - primaries=[{"puzzlehash": cat_inner.get_tree_hash(), "amount": amount}], - ), - ) - eve_spend = unsigned_spend_bundle_for_spendable_cats( - CAT_MOD, - [ - SpendableCAT( - list(filter(lambda a: a.amount == amount, tx_record.additions))[0], - genesis_coin_checker.get_tree_hash(), - cat_inner, - inner_solution, - limitations_program_reveal=genesis_coin_checker, - ) - ], - ) - signed_eve_spend = await wallet.sign(eve_spend) - - if wallet.cat_info.my_tail is None: - await wallet.save_info( - CATInfo(genesis_coin_checker.get_tree_hash(), genesis_coin_checker), - False, - ) - - return tx_record, SpendBundle.aggregate([tx_record.spend_bundle, signed_eve_spend]) - - -class GenesisByPuzhash(LimitationsProgram): - """ - This TAIL allows for issuance of a certain coin only by a specific puzzle hash. - There is no minting or melting allowed. - """ - - @staticmethod - def match(uncurried_mod: Program, curried_args: Program) -> Tuple[bool, List[Program]]: - if uncurried_mod == GENESIS_BY_PUZHASH_MOD: - genesis_puzhash = curried_args.first() - return True, [genesis_puzhash] - else: - return False, [] - - @staticmethod - def construct(args: List[Program]) -> Program: - return GENESIS_BY_PUZHASH_MOD.curry(args[0]) - - @staticmethod - def solve(args: List[Program], solution_dict: Dict) -> Program: - pid = hexstr_to_bytes(solution_dict["parent_coin_info"]) - return Program.to([pid, solution_dict["amount"]]) - - -class EverythingWithSig(LimitationsProgram): - """ - This TAIL allows for issuance, minting, and melting as long as you provide a signature with the spend. - """ - - @staticmethod - def match(uncurried_mod: Program, curried_args: Program) -> Tuple[bool, List[Program]]: - if uncurried_mod == EVERYTHING_WITH_SIG_MOD: - pubkey = curried_args.first() - return True, [pubkey] - else: - return False, [] - - @staticmethod - def construct(args: List[Program]) -> Program: - return EVERYTHING_WITH_SIG_MOD.curry(args[0]) - - @staticmethod - def solve(args: List[Program], solution_dict: Dict) -> Program: - return Program.to([]) - - -class DelegatedLimitations(LimitationsProgram): - """ - This TAIL allows for another TAIL to be used, as long as a signature of that TAIL's puzzlehash is included. - """ - - @staticmethod - def match(uncurried_mod: Program, curried_args: Program) -> Tuple[bool, List[Program]]: - if uncurried_mod == DELEGATED_LIMITATIONS_MOD: - pubkey = curried_args.first() - return True, [pubkey] - else: - return False, [] - - @staticmethod - def construct(args: List[Program]) -> Program: - return DELEGATED_LIMITATIONS_MOD.curry(args[0]) - - @staticmethod - def solve(args: List[Program], solution_dict: Dict) -> Program: - signed_program = ALL_LIMITATIONS_PROGRAMS[solution_dict["signed_program"]["identifier"]] - inner_program_args = [Program.fromhex(item) for item in solution_dict["signed_program"]["args"]] - inner_solution_dict = solution_dict["program_arguments"] - return Program.to( - [ - signed_program.construct(inner_program_args), - signed_program.solve(inner_program_args, inner_solution_dict), - ] - ) - - -# This should probably be much more elegant than just a dictionary with strings as identifiers -# Right now this is small and experimental so it can stay like this -ALL_LIMITATIONS_PROGRAMS: Dict[str, Any] = { - "genesis_by_id": GenesisById, - "genesis_by_puzhash": GenesisByPuzhash, - "everything_with_signature": EverythingWithSig, - "delegated_limitations": DelegatedLimitations, -} - - -def match_limitations_program(limitations_program: Program) -> Tuple[Optional[LimitationsProgram], List[Program]]: - uncurried_mod, curried_args = limitations_program.uncurry() - for key, lp in ALL_LIMITATIONS_PROGRAMS.items(): - matched, args = lp.match(uncurried_mod, curried_args) - if matched: - return lp, args - return None, [] diff --git a/mypy.ini b/mypy.ini index ec9389612d..4c3283cc77 100644 --- a/mypy.ini +++ b/mypy.ini @@ -17,7 +17,7 @@ no_implicit_reexport = True strict_equality = True # list created by: venv/bin/mypy | sed -n 's/.py:.*//p' | sort | uniq | tr '/' '.' | tr '\n' ',' -[mypy-benchmarks.block_ref,benchmarks.block_store,benchmarks.coin_store,benchmarks.utils,build_scripts.installer-version,chia.clvm.spend_sim,chia.cmds.configure,chia.cmds.db,chia.cmds.db_upgrade_func,chia.cmds.farm_funcs,chia.cmds.init,chia.cmds.init_funcs,chia.cmds.keys,chia.cmds.keys_funcs,chia.cmds.passphrase,chia.cmds.passphrase_funcs,chia.cmds.plotnft,chia.cmds.plotnft_funcs,chia.cmds.plots,chia.cmds.plotters,chia.cmds.show,chia.cmds.start_funcs,chia.cmds.wallet,chia.cmds.wallet_funcs,chia.consensus.block_body_validation,chia.consensus.blockchain,chia.consensus.blockchain_interface,chia.consensus.block_creation,chia.consensus.block_header_validation,chia.consensus.block_record,chia.consensus.block_root_validation,chia.consensus.coinbase,chia.consensus.constants,chia.consensus.difficulty_adjustment,chia.consensus.get_block_challenge,chia.consensus.multiprocess_validation,chia.consensus.pos_quality,chia.consensus.vdf_info_computation,chia.daemon.client,chia.daemon.keychain_proxy,chia.daemon.keychain_server,chia.daemon.server,chia.farmer.farmer,chia.farmer.farmer_api,chia.full_node.block_height_map,chia.full_node.block_store,chia.full_node.bundle_tools,chia.full_node.coin_store,chia.full_node.full_node,chia.full_node.full_node_api,chia.full_node.full_node_store,chia.full_node.generator,chia.full_node.hint_store,chia.full_node.lock_queue,chia.full_node.mempool,chia.full_node.mempool_check_conditions,chia.full_node.mempool_manager,chia.full_node.pending_tx_cache,chia.full_node.sync_store,chia.full_node.weight_proof,chia.harvester.harvester,chia.harvester.harvester_api,chia.introducer.introducer,chia.introducer.introducer_api,chia.plotters.bladebit,chia.plotters.chiapos,chia.plotters.install_plotter,chia.plotters.madmax,chia.plotters.plotters,chia.plotters.plotters_util,chia.plotting.check_plots,chia.plotting.create_plots,chia.plotting.manager,chia.plotting.util,chia.pools.pool_config,chia.pools.pool_puzzles,chia.pools.pool_wallet,chia.pools.pool_wallet_info,chia.protocols.pool_protocol,chia.rpc.crawler_rpc_api,chia.rpc.farmer_rpc_api,chia.rpc.farmer_rpc_client,chia.rpc.full_node_rpc_api,chia.rpc.full_node_rpc_client,chia.rpc.harvester_rpc_api,chia.rpc.harvester_rpc_client,chia.rpc.rpc_client,chia.rpc.rpc_server,chia.rpc.timelord_rpc_api,chia.rpc.util,chia.rpc.wallet_rpc_api,chia.rpc.wallet_rpc_client,chia.seeder.crawler,chia.seeder.crawler_api,chia.seeder.crawl_store,chia.seeder.dns_server,chia.seeder.peer_record,chia.seeder.start_crawler,chia.server.address_manager,chia.server.address_manager_store,chia.server.connection_utils,chia.server.introducer_peers,chia.server.node_discovery,chia.server.peer_store_resolver,chia.server.rate_limits,chia.server.reconnect_task,chia.server.server,chia.server.ssl_context,chia.server.start_farmer,chia.server.start_full_node,chia.server.start_harvester,chia.server.start_introducer,chia.server.start_service,chia.server.start_timelord,chia.server.start_wallet,chia.server.upnp,chia.server.ws_connection,chia.simulator.full_node_simulator,chia.simulator.start_simulator,chia.ssl.create_ssl,chia.timelord.iters_from_block,chia.timelord.timelord,chia.timelord.timelord_api,chia.timelord.timelord_launcher,chia.timelord.timelord_state,chia.types.announcement,chia.types.blockchain_format.classgroup,chia.types.blockchain_format.coin,chia.types.blockchain_format.program,chia.types.blockchain_format.proof_of_space,chia.types.blockchain_format.tree_hash,chia.types.blockchain_format.vdf,chia.types.full_block,chia.types.header_block,chia.types.mempool_item,chia.types.name_puzzle_condition,chia.types.peer_info,chia.types.spend_bundle,chia.types.transaction_queue_entry,chia.types.unfinished_block,chia.types.unfinished_header_block,chia.util.api_decorators,chia.util.block_cache,chia.util.byte_types,chia.util.cached_bls,chia.util.check_fork_next_block,chia.util.chia_logging,chia.util.config,chia.util.db_wrapper,chia.util.dump_keyring,chia.util.file_keyring,chia.util.files,chia.util.hash,chia.util.ints,chia.util.json_util,chia.util.keychain,chia.util.keyring_wrapper,chia.util.log_exceptions,chia.util.lru_cache,chia.util.make_test_constants,chia.util.merkle_set,chia.util.network,chia.util.partial_func,chia.util.pip_import,chia.util.profiler,chia.util.safe_cancel_task,chia.util.service_groups,chia.util.ssl_check,chia.util.streamable,chia.util.struct_stream,chia.util.type_checking,chia.util.validate_alert,chia.wallet.block_record,chia.wallet.cat_wallet.cat_utils,chia.wallet.cat_wallet.cat_wallet,chia.wallet.cat_wallet.lineage_store,chia.wallet.chialisp,chia.wallet.did_wallet.did_wallet,chia.wallet.did_wallet.did_wallet_puzzles,chia.wallet.key_val_store,chia.wallet.lineage_proof,chia.wallet.payment,chia.wallet.puzzles.genesis_checkers,chia.wallet.puzzles.load_clvm,chia.wallet.puzzles.p2_conditions,chia.wallet.puzzles.p2_delegated_conditions,chia.wallet.puzzles.p2_delegated_puzzle,chia.wallet.puzzles.p2_delegated_puzzle_or_hidden_puzzle,chia.wallet.puzzles.p2_m_of_n_delegate_direct,chia.wallet.puzzles.p2_puzzle_hash,chia.wallet.puzzles.prefarm.spend_prefarm,chia.wallet.puzzles.puzzle_utils,chia.wallet.puzzles.rom_bootstrap_generator,chia.wallet.puzzles.singleton_top_layer,chia.wallet.puzzles.tails,chia.wallet.rl_wallet.rl_wallet,chia.wallet.rl_wallet.rl_wallet_puzzles,chia.wallet.secret_key_store,chia.wallet.settings.user_settings,chia.wallet.trade_manager,chia.wallet.trade_record,chia.wallet.trading.offer,chia.wallet.trading.trade_store,chia.wallet.transaction_record,chia.wallet.util.debug_spend_bundle,chia.wallet.util.new_peak_queue,chia.wallet.util.peer_request_cache,chia.wallet.util.wallet_sync_utils,chia.wallet.wallet,chia.wallet.wallet_action_store,chia.wallet.wallet_blockchain,chia.wallet.wallet_coin_store,chia.wallet.wallet_interested_store,chia.wallet.wallet_node,chia.wallet.wallet_node_api,chia.wallet.wallet_pool_store,chia.wallet.wallet_puzzle_store,chia.wallet.wallet_state_manager,chia.wallet.wallet_sync_store,chia.wallet.wallet_transaction_store,chia.wallet.wallet_user_store,chia.wallet.wallet_weight_proof_handler,installhelper,tests.blockchain.blockchain_test_utils,tests.blockchain.test_blockchain,tests.blockchain.test_blockchain_transactions,tests.block_tools,tests.build-init-files,tests.build-workflows,tests.clvm.coin_store,tests.clvm.test_chialisp_deserialization,tests.clvm.test_clvm_compilation,tests.clvm.test_program,tests.clvm.test_puzzle_compression,tests.clvm.test_puzzles,tests.clvm.test_serialized_program,tests.clvm.test_singletons,tests.clvm.test_spend_sim,tests.conftest,tests.connection_utils,tests.core.cmds.test_keys,tests.core.consensus.test_pot_iterations,tests.core.custom_types.test_coin,tests.core.custom_types.test_proof_of_space,tests.core.custom_types.test_spend_bundle,tests.core.daemon.test_daemon,tests.core.full_node.full_sync.test_full_sync,tests.core.full_node.stores.test_block_store,tests.core.full_node.stores.test_coin_store,tests.core.full_node.stores.test_full_node_store,tests.core.full_node.stores.test_hint_store,tests.core.full_node.stores.test_sync_store,tests.core.full_node.test_address_manager,tests.core.full_node.test_block_height_map,tests.core.full_node.test_conditions,tests.core.full_node.test_full_node,tests.core.full_node.test_mempool,tests.core.full_node.test_mempool_performance,tests.core.full_node.test_node_load,tests.core.full_node.test_peer_store_resolver,tests.core.full_node.test_performance,tests.core.full_node.test_transactions,tests.core.make_block_generator,tests.core.node_height,tests.core.server.test_dos,tests.core.server.test_rate_limits,tests.core.ssl.test_ssl,tests.core.test_cost_calculation,tests.core.test_crawler_rpc,tests.core.test_daemon_rpc,tests.core.test_db_conversion,tests.core.test_farmer_harvester_rpc,tests.core.test_filter,tests.core.test_full_node_rpc,tests.core.test_merkle_set,tests.core.test_setproctitle,tests.core.util.test_cached_bls,tests.core.util.test_config,tests.core.util.test_file_keyring_synchronization,tests.core.util.test_files,tests.core.util.test_keychain,tests.core.util.test_keyring_wrapper,tests.core.util.test_lru_cache,tests.core.util.test_significant_bits,tests.core.util.test_streamable,tests.core.util.test_type_checking,tests.farmer_harvester.test_farmer_harvester,tests.generator.test_compression,tests.generator.test_generator_types,tests.generator.test_list_to_batches,tests.generator.test_rom,tests.generator.test_scan,tests.plotting.test_plot_manager,tests.pools.test_pool_cmdline,tests.pools.test_pool_config,tests.pools.test_pool_puzzles_lifecycle,tests.pools.test_pool_rpc,tests.pools.test_wallet_pool_store,tests.setup_nodes,tests.setup_services,tests.simulation.test_simulation,tests.time_out_assert,tests.tools.test_full_sync,tests.tools.test_run_block,tests.util.alert_server,tests.util.benchmark_cost,tests.util.blockchain,tests.util.build_network_protocol_files,tests.util.db_connection,tests.util.generator_tools_testing,tests.util.keyring,tests.util.key_tool,tests.util.misc,tests.util.network,tests.util.rpc,tests.util.test_full_block_utils,tests.util.test_lock_queue,tests.util.test_network_protocol_files,tests.util.test_struct_stream,tests.wallet.cat_wallet.test_cat_lifecycle,tests.wallet.cat_wallet.test_cat_wallet,tests.wallet.cat_wallet.test_offer_lifecycle,tests.wallet.cat_wallet.test_trades,tests.wallet.did_wallet.test_did,tests.wallet.did_wallet.test_did_rpc,tests.wallet.rl_wallet.test_rl_rpc,tests.wallet.rl_wallet.test_rl_wallet,tests.wallet.rpc.test_wallet_rpc,tests.wallet.simple_sync.test_simple_sync_protocol,tests.wallet.sync.test_wallet_sync,tests.wallet.test_bech32m,tests.wallet.test_chialisp,tests.wallet.test_puzzle_store,tests.wallet.test_singleton,tests.wallet.test_singleton_lifecycle,tests.wallet.test_singleton_lifecycle_fast,tests.wallet.test_taproot,tests.wallet.test_wallet,tests.wallet.test_wallet_blockchain,tests.wallet.test_wallet_interested_store,tests.wallet.test_wallet_key_val_store,tests.wallet.test_wallet_user_store,tests.wallet_tools,tests.weight_proof.test_weight_proof,tools.analyze-chain,tools.run_block,tools.test_full_sync] +[mypy-benchmarks.block_ref,benchmarks.block_store,benchmarks.coin_store,benchmarks.utils,build_scripts.installer-version,chia.clvm.spend_sim,chia.cmds.configure,chia.cmds.db,chia.cmds.db_upgrade_func,chia.cmds.farm_funcs,chia.cmds.init,chia.cmds.init_funcs,chia.cmds.keys,chia.cmds.keys_funcs,chia.cmds.passphrase,chia.cmds.passphrase_funcs,chia.cmds.plotnft,chia.cmds.plotnft_funcs,chia.cmds.plots,chia.cmds.plotters,chia.cmds.show,chia.cmds.start_funcs,chia.cmds.wallet,chia.cmds.wallet_funcs,chia.consensus.block_body_validation,chia.consensus.blockchain,chia.consensus.blockchain_interface,chia.consensus.block_creation,chia.consensus.block_header_validation,chia.consensus.block_record,chia.consensus.block_root_validation,chia.consensus.coinbase,chia.consensus.constants,chia.consensus.difficulty_adjustment,chia.consensus.get_block_challenge,chia.consensus.multiprocess_validation,chia.consensus.pos_quality,chia.consensus.vdf_info_computation,chia.daemon.client,chia.daemon.keychain_proxy,chia.daemon.keychain_server,chia.daemon.server,chia.farmer.farmer,chia.farmer.farmer_api,chia.full_node.block_height_map,chia.full_node.block_store,chia.full_node.bundle_tools,chia.full_node.coin_store,chia.full_node.full_node,chia.full_node.full_node_api,chia.full_node.full_node_store,chia.full_node.generator,chia.full_node.hint_store,chia.full_node.lock_queue,chia.full_node.mempool,chia.full_node.mempool_check_conditions,chia.full_node.mempool_manager,chia.full_node.pending_tx_cache,chia.full_node.sync_store,chia.full_node.weight_proof,chia.harvester.harvester,chia.harvester.harvester_api,chia.introducer.introducer,chia.introducer.introducer_api,chia.plotters.bladebit,chia.plotters.chiapos,chia.plotters.install_plotter,chia.plotters.madmax,chia.plotters.plotters,chia.plotters.plotters_util,chia.plotting.check_plots,chia.plotting.create_plots,chia.plotting.manager,chia.plotting.util,chia.pools.pool_config,chia.pools.pool_puzzles,chia.pools.pool_wallet,chia.pools.pool_wallet_info,chia.protocols.pool_protocol,chia.rpc.crawler_rpc_api,chia.rpc.farmer_rpc_api,chia.rpc.farmer_rpc_client,chia.rpc.full_node_rpc_api,chia.rpc.full_node_rpc_client,chia.rpc.harvester_rpc_api,chia.rpc.harvester_rpc_client,chia.rpc.rpc_client,chia.rpc.rpc_server,chia.rpc.timelord_rpc_api,chia.rpc.util,chia.rpc.wallet_rpc_api,chia.rpc.wallet_rpc_client,chia.seeder.crawler,chia.seeder.crawler_api,chia.seeder.crawl_store,chia.seeder.dns_server,chia.seeder.peer_record,chia.seeder.start_crawler,chia.server.address_manager,chia.server.address_manager_store,chia.server.connection_utils,chia.server.introducer_peers,chia.server.node_discovery,chia.server.peer_store_resolver,chia.server.rate_limits,chia.server.reconnect_task,chia.server.server,chia.server.ssl_context,chia.server.start_farmer,chia.server.start_full_node,chia.server.start_harvester,chia.server.start_introducer,chia.server.start_service,chia.server.start_timelord,chia.server.start_wallet,chia.server.upnp,chia.server.ws_connection,chia.simulator.full_node_simulator,chia.simulator.start_simulator,chia.ssl.create_ssl,chia.timelord.iters_from_block,chia.timelord.timelord,chia.timelord.timelord_api,chia.timelord.timelord_launcher,chia.timelord.timelord_state,chia.types.announcement,chia.types.blockchain_format.classgroup,chia.types.blockchain_format.coin,chia.types.blockchain_format.program,chia.types.blockchain_format.proof_of_space,chia.types.blockchain_format.tree_hash,chia.types.blockchain_format.vdf,chia.types.full_block,chia.types.header_block,chia.types.mempool_item,chia.types.name_puzzle_condition,chia.types.peer_info,chia.types.spend_bundle,chia.types.transaction_queue_entry,chia.types.unfinished_block,chia.types.unfinished_header_block,chia.util.api_decorators,chia.util.block_cache,chia.util.byte_types,chia.util.cached_bls,chia.util.check_fork_next_block,chia.util.chia_logging,chia.util.config,chia.util.db_wrapper,chia.util.dump_keyring,chia.util.file_keyring,chia.util.files,chia.util.hash,chia.util.ints,chia.util.json_util,chia.util.keychain,chia.util.keyring_wrapper,chia.util.log_exceptions,chia.util.lru_cache,chia.util.make_test_constants,chia.util.merkle_set,chia.util.network,chia.util.partial_func,chia.util.pip_import,chia.util.profiler,chia.util.safe_cancel_task,chia.util.service_groups,chia.util.ssl_check,chia.util.streamable,chia.util.struct_stream,chia.util.type_checking,chia.util.validate_alert,chia.wallet.block_record,chia.wallet.cat_wallet.cat_utils,chia.wallet.cat_wallet.cat_wallet,chia.wallet.cat_wallet.lineage_store,chia.wallet.chialisp,chia.wallet.did_wallet.did_wallet,chia.wallet.did_wallet.did_wallet_puzzles,chia.wallet.key_val_store,chia.wallet.lineage_proof,chia.wallet.payment,chia.wallet.puzzles.load_clvm,chia.wallet.puzzles.p2_conditions,chia.wallet.puzzles.p2_delegated_conditions,chia.wallet.puzzles.p2_delegated_puzzle,chia.wallet.puzzles.p2_delegated_puzzle_or_hidden_puzzle,chia.wallet.puzzles.p2_m_of_n_delegate_direct,chia.wallet.puzzles.p2_puzzle_hash,chia.wallet.puzzles.prefarm.spend_prefarm,chia.wallet.puzzles.puzzle_utils,chia.wallet.puzzles.rom_bootstrap_generator,chia.wallet.puzzles.singleton_top_layer,chia.wallet.puzzles.tails,chia.wallet.rl_wallet.rl_wallet,chia.wallet.rl_wallet.rl_wallet_puzzles,chia.wallet.secret_key_store,chia.wallet.settings.user_settings,chia.wallet.trade_manager,chia.wallet.trade_record,chia.wallet.trading.offer,chia.wallet.trading.trade_store,chia.wallet.transaction_record,chia.wallet.util.debug_spend_bundle,chia.wallet.util.new_peak_queue,chia.wallet.util.peer_request_cache,chia.wallet.util.wallet_sync_utils,chia.wallet.wallet,chia.wallet.wallet_action_store,chia.wallet.wallet_blockchain,chia.wallet.wallet_coin_store,chia.wallet.wallet_interested_store,chia.wallet.wallet_node,chia.wallet.wallet_node_api,chia.wallet.wallet_pool_store,chia.wallet.wallet_puzzle_store,chia.wallet.wallet_state_manager,chia.wallet.wallet_sync_store,chia.wallet.wallet_transaction_store,chia.wallet.wallet_user_store,chia.wallet.wallet_weight_proof_handler,installhelper,tests.blockchain.blockchain_test_utils,tests.blockchain.test_blockchain,tests.blockchain.test_blockchain_transactions,tests.block_tools,tests.build-init-files,tests.build-workflows,tests.clvm.coin_store,tests.clvm.test_chialisp_deserialization,tests.clvm.test_clvm_compilation,tests.clvm.test_program,tests.clvm.test_puzzle_compression,tests.clvm.test_puzzles,tests.clvm.test_serialized_program,tests.clvm.test_singletons,tests.clvm.test_spend_sim,tests.conftest,tests.connection_utils,tests.core.cmds.test_keys,tests.core.consensus.test_pot_iterations,tests.core.custom_types.test_coin,tests.core.custom_types.test_proof_of_space,tests.core.custom_types.test_spend_bundle,tests.core.daemon.test_daemon,tests.core.full_node.full_sync.test_full_sync,tests.core.full_node.stores.test_block_store,tests.core.full_node.stores.test_coin_store,tests.core.full_node.stores.test_full_node_store,tests.core.full_node.stores.test_hint_store,tests.core.full_node.stores.test_sync_store,tests.core.full_node.test_address_manager,tests.core.full_node.test_block_height_map,tests.core.full_node.test_conditions,tests.core.full_node.test_full_node,tests.core.full_node.test_mempool,tests.core.full_node.test_mempool_performance,tests.core.full_node.test_node_load,tests.core.full_node.test_peer_store_resolver,tests.core.full_node.test_performance,tests.core.full_node.test_transactions,tests.core.make_block_generator,tests.core.node_height,tests.core.server.test_dos,tests.core.server.test_rate_limits,tests.core.ssl.test_ssl,tests.core.test_cost_calculation,tests.core.test_crawler_rpc,tests.core.test_daemon_rpc,tests.core.test_db_conversion,tests.core.test_farmer_harvester_rpc,tests.core.test_filter,tests.core.test_full_node_rpc,tests.core.test_merkle_set,tests.core.test_setproctitle,tests.core.util.test_cached_bls,tests.core.util.test_config,tests.core.util.test_file_keyring_synchronization,tests.core.util.test_files,tests.core.util.test_keychain,tests.core.util.test_keyring_wrapper,tests.core.util.test_lru_cache,tests.core.util.test_significant_bits,tests.core.util.test_streamable,tests.core.util.test_type_checking,tests.farmer_harvester.test_farmer_harvester,tests.generator.test_compression,tests.generator.test_generator_types,tests.generator.test_list_to_batches,tests.generator.test_rom,tests.generator.test_scan,tests.plotting.test_plot_manager,tests.pools.test_pool_cmdline,tests.pools.test_pool_config,tests.pools.test_pool_puzzles_lifecycle,tests.pools.test_pool_rpc,tests.pools.test_wallet_pool_store,tests.setup_nodes,tests.setup_services,tests.simulation.test_simulation,tests.time_out_assert,tests.tools.test_full_sync,tests.tools.test_run_block,tests.util.alert_server,tests.util.benchmark_cost,tests.util.blockchain,tests.util.build_network_protocol_files,tests.util.db_connection,tests.util.generator_tools_testing,tests.util.keyring,tests.util.key_tool,tests.util.misc,tests.util.network,tests.util.rpc,tests.util.test_full_block_utils,tests.util.test_lock_queue,tests.util.test_network_protocol_files,tests.util.test_struct_stream,tests.wallet.cat_wallet.test_cat_lifecycle,tests.wallet.cat_wallet.test_cat_wallet,tests.wallet.cat_wallet.test_offer_lifecycle,tests.wallet.cat_wallet.test_trades,tests.wallet.did_wallet.test_did,tests.wallet.did_wallet.test_did_rpc,tests.wallet.rl_wallet.test_rl_rpc,tests.wallet.rl_wallet.test_rl_wallet,tests.wallet.rpc.test_wallet_rpc,tests.wallet.simple_sync.test_simple_sync_protocol,tests.wallet.sync.test_wallet_sync,tests.wallet.test_bech32m,tests.wallet.test_chialisp,tests.wallet.test_puzzle_store,tests.wallet.test_singleton,tests.wallet.test_singleton_lifecycle,tests.wallet.test_singleton_lifecycle_fast,tests.wallet.test_taproot,tests.wallet.test_wallet,tests.wallet.test_wallet_blockchain,tests.wallet.test_wallet_interested_store,tests.wallet.test_wallet_key_val_store,tests.wallet.test_wallet_user_store,tests.wallet_tools,tests.weight_proof.test_weight_proof,tools.analyze-chain,tools.run_block,tools.test_full_sync] disallow_any_generics = False disallow_subclassing_any = False disallow_untyped_calls = False diff --git a/tests/clvm/test_clvm_compilation.py b/tests/clvm/test_clvm_compilation.py index 412d244da7..b3d0521767 100644 --- a/tests/clvm/test_clvm_compilation.py +++ b/tests/clvm/test_clvm_compilation.py @@ -40,9 +40,6 @@ wallet_program_files = set( "chia/wallet/puzzles/delegated_tail.clvm", "chia/wallet/puzzles/settlement_payments.clvm", "chia/wallet/puzzles/genesis_by_coin_id.clvm", - "chia/wallet/puzzles/genesis-by-puzzle-hash-with-0.clvm", - "chia/wallet/puzzles/delegated_genesis_checker.clvm", - "chia/wallet/puzzles/genesis-by-coin-id-with-0.clvm", ] ) From ab712362154672a48024dbf27fbe80e58e363bde Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Apr 2022 09:20:30 -0700 Subject: [PATCH 44/63] Bump cryptography from 3.4.7 to 36.0.2 (#10787) Bumps [cryptography](https://github.com/pyca/cryptography) from 3.4.7 to 36.0.2. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.4.7...36.0.2) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index df9ef8b411..8d4d20558b 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ dependencies = [ "colorama==0.4.4", # Colorizes terminal output "colorlog==6.6.0", # Adds color to logs "concurrent-log-handler==0.9.19", # Concurrently log and rotate logs - "cryptography==3.4.7", # Python cryptography library for TLS - keyring conflict + "cryptography==36.0.2", # Python cryptography library for TLS - keyring conflict "fasteners==0.16.3", # For interprocess file locking, expected to be replaced by filelock "filelock==3.4.2", # For reading and writing config multiprocess and multithread safely (non-reentrant locks) "keyring==23.0.1", # Store keys in MacOS Keychain, Windows Credential Locker From c8468bea0709eb7f71d0fc851511015cd6a448ce Mon Sep 17 00:00:00 2001 From: dustinface <35775977+xdustinface@users.noreply.github.com> Date: Thu, 7 Apr 2022 18:21:08 +0200 Subject: [PATCH 45/63] wallet: Improve logging in `create_more_puzzle_hashes` (#10761) * wallet: Improve logging in `create_more_puzzle_hashes` It's pretty spammy currently when scanning puzzle hashes. * scan -> create, Scanning -> Creating * `scanning_msg` -> `creating_msg` --- chia/wallet/wallet_state_manager.py | 76 ++++++++++++++++------------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/chia/wallet/wallet_state_manager.py b/chia/wallet/wallet_state_manager.py index 1a51de491c..02a9a4e913 100644 --- a/chia/wallet/wallet_state_manager.py +++ b/chia/wallet/wallet_state_manager.py @@ -266,44 +266,50 @@ class WalletStateManager: # If the key was replaced (from_zero=True), we should generate the puzzle hashes for the new key if from_zero: start_index = 0 + last_index = unused + to_generate + if start_index >= last_index: + self.log.debug(f"Nothing to create for for wallet_id: {wallet_id}, index: {start_index}") + else: + creating_msg = f"Creating puzzle hashes from {start_index} to {last_index} for wallet_id: {wallet_id}" + self.log.info(f"Start: {creating_msg}") + for index in range(start_index, last_index): + if WalletType(target_wallet.type()) == WalletType.POOLING_WALLET: + continue - for index in range(start_index, unused + to_generate): - if WalletType(target_wallet.type()) == WalletType.POOLING_WALLET: - continue - - # Hardened - pubkey: G1Element = self.get_public_key(uint32(index)) - puzzle: Program = target_wallet.puzzle_for_pk(bytes(pubkey)) - if puzzle is None: - self.log.error(f"Unable to create puzzles with wallet {target_wallet}") - break - puzzlehash: bytes32 = puzzle.get_tree_hash() - self.log.info(f"Puzzle at index {index} wallet ID {wallet_id} puzzle hash {puzzlehash.hex()}") - derivation_paths.append( - DerivationRecord( - uint32(index), puzzlehash, pubkey, target_wallet.type(), uint32(target_wallet.id()), True + # Hardened + pubkey: G1Element = self.get_public_key(uint32(index)) + puzzle: Program = target_wallet.puzzle_for_pk(bytes(pubkey)) + if puzzle is None: + self.log.error(f"Unable to create puzzles with wallet {target_wallet}") + break + puzzlehash: bytes32 = puzzle.get_tree_hash() + self.log.debug(f"Puzzle at index {index} wallet ID {wallet_id} puzzle hash {puzzlehash.hex()}") + derivation_paths.append( + DerivationRecord( + uint32(index), puzzlehash, pubkey, target_wallet.type(), uint32(target_wallet.id()), True + ) ) - ) - # Unhardened - pubkey_unhardened: G1Element = self.get_public_key_unhardened(uint32(index)) - puzzle_unhardened: Program = target_wallet.puzzle_for_pk(bytes(pubkey_unhardened)) - if puzzle_unhardened is None: - self.log.error(f"Unable to create puzzles with wallet {target_wallet}") - break - puzzlehash_unhardened: bytes32 = puzzle_unhardened.get_tree_hash() - self.log.info( - f"Puzzle at index {index} wallet ID {wallet_id} puzzle hash {puzzlehash_unhardened.hex()}" - ) - derivation_paths.append( - DerivationRecord( - uint32(index), - puzzlehash_unhardened, - pubkey_unhardened, - target_wallet.type(), - uint32(target_wallet.id()), - False, + # Unhardened + pubkey_unhardened: G1Element = self.get_public_key_unhardened(uint32(index)) + puzzle_unhardened: Program = target_wallet.puzzle_for_pk(bytes(pubkey_unhardened)) + if puzzle_unhardened is None: + self.log.error(f"Unable to create puzzles with wallet {target_wallet}") + break + puzzlehash_unhardened: bytes32 = puzzle_unhardened.get_tree_hash() + self.log.debug( + f"Puzzle at index {index} wallet ID {wallet_id} puzzle hash {puzzlehash_unhardened.hex()}" ) - ) + derivation_paths.append( + DerivationRecord( + uint32(index), + puzzlehash_unhardened, + pubkey_unhardened, + target_wallet.type(), + uint32(target_wallet.id()), + False, + ) + ) + self.log.info(f"Done: {creating_msg}") await self.puzzle_store.add_derivation_paths(derivation_paths, in_transaction) await self.add_interested_puzzle_hashes( [record.puzzle_hash for record in derivation_paths], From ffd3b19315571ee5a55f3e75e9678964dac5ce1b Mon Sep 17 00:00:00 2001 From: Kronus91 Date: Thu, 7 Apr 2022 09:22:59 -0700 Subject: [PATCH 46/63] Add /cat_get_unacknowledged API for accessing unknown CATs (#10382) * Add /cat_get_unacknowledged API for accessing unknown CATs * Reformat & fix cast issue * Integration tested & add unit test * Handle optional uint32 * Reformat * Reformat * Reformat * Merge PR 10308 * Reformat * Fix concurrent issue * Add state change notification * rename API * Fix failing tests * Updated state_change name Co-authored-by: Jeff Cruikshank --- chia/rpc/wallet_rpc_api.py | 11 +++++ chia/rpc/wallet_rpc_client.py | 4 ++ chia/wallet/wallet_interested_store.py | 59 +++++++++++++++++++++++++- chia/wallet/wallet_state_manager.py | 12 +++++- tests/wallet/rpc/test_wallet_rpc.py | 6 +++ 5 files changed, 90 insertions(+), 2 deletions(-) diff --git a/chia/rpc/wallet_rpc_api.py b/chia/rpc/wallet_rpc_api.py index e4439ea328..3f43af8095 100644 --- a/chia/rpc/wallet_rpc_api.py +++ b/chia/rpc/wallet_rpc_api.py @@ -91,6 +91,7 @@ class WalletRpcApi: "/cat_set_name": self.cat_set_name, "/cat_asset_id_to_name": self.cat_asset_id_to_name, "/cat_get_name": self.cat_get_name, + "/get_stray_cats": self.get_stray_cats, "/cat_spend": self.cat_spend, "/cat_get_asset_id": self.cat_get_asset_id, "/create_offer_for_ids": self.create_offer_for_ids, @@ -857,6 +858,16 @@ class WalletRpcApi: name: str = await wallet.get_name() return {"wallet_id": wallet_id, "name": name} + async def get_stray_cats(self, request): + """ + Get a list of all unacknowledged CATs + :param request: RPC request + :return: A list of unacknowledged CATs + """ + assert self.service.wallet_state_manager is not None + cats = await self.service.wallet_state_manager.interested_store.get_unacknowledged_tokens() + return {"stray_cats": cats} + async def cat_spend(self, request): assert self.service.wallet_state_manager is not None diff --git a/chia/rpc/wallet_rpc_client.py b/chia/rpc/wallet_rpc_client.py index 8d8afb6d4f..82058176ba 100644 --- a/chia/rpc/wallet_rpc_client.py +++ b/chia/rpc/wallet_rpc_client.py @@ -363,6 +363,10 @@ class WalletRpcClient(RpcClient): } return bytes.fromhex((await self.fetch("cat_get_asset_id", request))["asset_id"]) + async def get_stray_cats(self) -> Dict: + response = await self.fetch("get_stray_cats", {}) + return response["stray_cats"] + async def cat_asset_id_to_name(self, asset_id: bytes32) -> Optional[Tuple[Optional[uint32], str]]: request: Dict[str, Any] = { "asset_id": asset_id.hex(), diff --git a/chia/wallet/wallet_interested_store.py b/chia/wallet/wallet_interested_store.py index c46437b77f..1782887832 100644 --- a/chia/wallet/wallet_interested_store.py +++ b/chia/wallet/wallet_interested_store.py @@ -4,6 +4,7 @@ import aiosqlite from chia.types.blockchain_format.sized_bytes import bytes32 from chia.util.db_wrapper import DBWrapper +from chia.util.ints import uint32 class WalletInterestedStore: @@ -26,14 +27,21 @@ class WalletInterestedStore: await self.db_connection.execute( "CREATE TABLE IF NOT EXISTS interested_puzzle_hashes(puzzle_hash text PRIMARY KEY, wallet_id integer)" ) + + # Table for unknown CATs + fields = "asset_id text PRIMARY KEY, name text, first_seen_height integer, sender_puzzle_hash text" + await self.db_connection.execute(f"CREATE TABLE IF NOT EXISTS unacknowledged_asset_tokens({fields})") + await self.db_connection.commit() return self async def _clear_database(self): - cursor = await self.db_connection.execute("DELETE FROM puzzle_hashes") + cursor = await self.db_connection.execute("DELETE FROM interested_puzzle_hashes") await cursor.close() cursor = await self.db_connection.execute("DELETE FROM interested_coins") await cursor.close() + cursor = await self.db_connection.execute("DELETE FROM unacknowledged_asset_tokens") + await cursor.close() await self.db_connection.commit() async def get_interested_coin_ids(self) -> List[bytes32]: @@ -97,3 +105,52 @@ class WalletInterestedStore: if not in_transaction: await self.db_connection.commit() self.db_wrapper.lock.release() + + async def add_unacknowledged_token( + self, + asset_id: bytes32, + name: str, + first_seen_height: Optional[uint32], + sender_puzzle_hash: bytes32, + in_transaction: bool = True, + ) -> None: + """ + Add an unacknowledged CAT to the database. It will only be inserted once at the first time. + :param asset_id: CAT asset ID + :param name: Name of the CAT, for now it will be unknown until we integrate the CAT name service + :param first_seen_height: The block height of the wallet received this CAT in the first time + :param sender_puzzle_hash: The puzzle hash of the sender + :param in_transaction: In transaction or not + :return: None + """ + if not in_transaction: + await self.db_wrapper.lock.acquire() + try: + cursor = await self.db_connection.execute( + "INSERT OR IGNORE INTO unacknowledged_asset_tokens VALUES (?, ?, ?, ?)", + ( + asset_id.hex(), + name, + first_seen_height if first_seen_height is not None else 0, + sender_puzzle_hash.hex(), + ), + ) + await cursor.close() + finally: + if not in_transaction: + await self.db_connection.commit() + self.db_wrapper.lock.release() + + async def get_unacknowledged_tokens(self) -> List: + """ + Get a list of all unacknowledged CATs + :return: A json style list of unacknowledged CATs + """ + cursor = await self.db_connection.execute( + "SELECT asset_id, name, first_seen_height, sender_puzzle_hash FROM unacknowledged_asset_tokens" + ) + cats = await cursor.fetchall() + return [ + {"asset_id": cat[0], "name": cat[1], "first_seen_height": cat[2], "sender_puzzle_hash": cat[3]} + for cat in cats + ] diff --git a/chia/wallet/wallet_state_manager.py b/chia/wallet/wallet_state_manager.py index 02a9a4e913..e7044beec9 100644 --- a/chia/wallet/wallet_state_manager.py +++ b/chia/wallet/wallet_state_manager.py @@ -572,7 +572,8 @@ class WalletStateManager: self.log.info(f"Received state for the coin that doesn't belong to us {coin_state}") else: our_inner_puzzle: Program = self.main_wallet.puzzle_for_pk(bytes(derivation_record.pubkey)) - cat_puzzle = construct_cat_puzzle(CAT_MOD, bytes32(bytes(tail_hash)[1:]), our_inner_puzzle) + asset_id: bytes32 = bytes32(bytes(tail_hash)[1:]) + cat_puzzle = construct_cat_puzzle(CAT_MOD, asset_id, our_inner_puzzle) if cat_puzzle.get_tree_hash() != coin_state.coin.puzzle_hash: return None, None if bytes(tail_hash).hex()[2:] in self.default_cats or self.config.get( @@ -584,6 +585,15 @@ class WalletStateManager: wallet_id = cat_wallet.id() wallet_type = WalletType(cat_wallet.type()) self.state_changed("wallet_created") + else: + # Found unacknowledged CAT, save it in the database. + await self.interested_store.add_unacknowledged_token( + asset_id, + CATWallet.default_wallet_name_for_unknown_cat(asset_id.hex()), + parent_coin_state.spent_height, + parent_coin_state.coin.puzzle_hash, + ) + self.state_changed("added_stray_cat") return wallet_id, wallet_type diff --git a/tests/wallet/rpc/test_wallet_rpc.py b/tests/wallet/rpc/test_wallet_rpc.py index f92570b31a..87eb782839 100644 --- a/tests/wallet/rpc/test_wallet_rpc.py +++ b/tests/wallet/rpc/test_wallet_rpc.py @@ -477,6 +477,12 @@ class TestWalletRpc: for i in range(0, 5): await client.farm_block(encode_puzzle_hash(ph_2, "txch")) await asyncio.sleep(0.5) + # Test unacknowledged CAT + await wallet_node.wallet_state_manager.interested_store.add_unacknowledged_token( + asset_id, "Unknown", uint32(10000), bytes.fromhex("ABCD") + ) + cats = await client.get_stray_cats() + assert len(cats) == 1 await time_out_assert(10, eventual_balance_det, 16, client, cat_0_id) await time_out_assert(10, eventual_balance_det, 4, client_2, cat_1_id) From 4c9a0edc64be00cd6ad2d31fe1628ebfa1fd7948 Mon Sep 17 00:00:00 2001 From: Mariano Sorgente <3069354+mariano54@users.noreply.github.com> Date: Thu, 7 Apr 2022 12:24:59 -0400 Subject: [PATCH 47/63] Increases the probability of connecting to local trusted node (#10633) --- chia/wallet/util/wallet_sync_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/chia/wallet/util/wallet_sync_utils.py b/chia/wallet/util/wallet_sync_utils.py index 44675ee4d6..01e1a66ef6 100644 --- a/chia/wallet/util/wallet_sync_utils.py +++ b/chia/wallet/util/wallet_sync_utils.py @@ -41,7 +41,8 @@ async def fetch_last_tx_from_peer(height: uint32, peer: WSChiaConnection) -> Opt if response is not None and isinstance(response, RespondBlockHeader): if response.header_block.is_transaction_block: return response.header_block - else: + elif request_height < height: + # The peer might be slightly behind others but still synced, so we should allow fetching one more TX block break request_height = request_height - 1 return None From 8c0cdda880777bad86c6eaed45a1e061c5c5ee37 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Thu, 7 Apr 2022 21:04:44 +0200 Subject: [PATCH 48/63] extend tests in test_blockchain to include more conditions, as well as ensuring consensus rules allow unknown condition parameters (#11079) --- tests/blockchain/test_blockchain.py | 127 +++++++++++++++++++++++----- 1 file changed, 106 insertions(+), 21 deletions(-) diff --git a/tests/blockchain/test_blockchain.py b/tests/blockchain/test_blockchain.py index 59631b7614..346822d44e 100644 --- a/tests/blockchain/test_blockchain.py +++ b/tests/blockchain/test_blockchain.py @@ -1711,6 +1711,86 @@ class TestPreValidation: class TestBodyValidation: + + # TODO: add test for + # ASSERT_COIN_ANNOUNCEMENT, + # CREATE_COIN_ANNOUNCEMENT, + # CREATE_PUZZLE_ANNOUNCEMENT, + # ASSERT_PUZZLE_ANNOUNCEMENT, + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "opcode", + [ + ConditionOpcode.ASSERT_MY_AMOUNT, + ConditionOpcode.ASSERT_MY_PUZZLEHASH, + ConditionOpcode.ASSERT_MY_COIN_ID, + ConditionOpcode.ASSERT_MY_PARENT_ID, + ], + ) + @pytest.mark.parametrize("with_garbage", [True, False]) + async def test_conditions(self, empty_blockchain, opcode, with_garbage, bt): + b = empty_blockchain + blocks = bt.get_consecutive_blocks( + 3, + guarantee_transaction_block=True, + farmer_reward_puzzle_hash=bt.pool_ph, + pool_reward_puzzle_hash=bt.pool_ph, + genesis_timestamp=10000, + time_per_block=10, + ) + await _validate_and_add_block(empty_blockchain, blocks[0]) + await _validate_and_add_block(empty_blockchain, blocks[1]) + await _validate_and_add_block(empty_blockchain, blocks[2]) + + wt: WalletTool = bt.get_pool_wallet_tool() + + tx1: SpendBundle = wt.generate_signed_transaction( + 10, wt.get_new_puzzlehash(), list(blocks[-1].get_included_reward_coins())[0] + ) + coin1: Coin = tx1.additions()[0] + secret_key = wt.get_private_key_for_puzzle_hash(coin1.puzzle_hash) + synthetic_secret_key = calculate_synthetic_secret_key(secret_key, DEFAULT_HIDDEN_PUZZLE_HASH) + public_key = synthetic_secret_key.get_g1() + + if opcode == ConditionOpcode.ASSERT_MY_AMOUNT: + args = [int_to_bytes(coin1.amount)] + elif opcode == ConditionOpcode.ASSERT_MY_PUZZLEHASH: + args = [coin1.puzzle_hash] + elif opcode == ConditionOpcode.ASSERT_MY_COIN_ID: + args = [coin1.name()] + elif opcode == ConditionOpcode.ASSERT_MY_PARENT_ID: + args = [coin1.parent_coin_info] + # elif opcode == ConditionOpcode.RESERVE_FEE: + # args = [int_to_bytes(5)] + # TODO: since we use the production wallet code, we can't (easily) + # create a transaction with fee without also including a valid + # RESERVE_FEE condition + else: + assert False + + conditions = {opcode: [ConditionWithArgs(opcode, args + ([b"garbage"] if with_garbage else []))]} + + tx2: SpendBundle = wt.generate_signed_transaction(10, wt.get_new_puzzlehash(), coin1, condition_dic=conditions) + assert coin1 in tx2.removals() + coin2: Coin = tx2.additions()[0] + + bundles = SpendBundle.aggregate([tx1, tx2]) + blocks = bt.get_consecutive_blocks( + 1, + block_list_input=blocks, + guarantee_transaction_block=True, + transaction_data=bundles, + time_per_block=10, + ) + + pre_validation_results: List[PreValidationResult] = await b.pre_validate_blocks_multiprocessing( + [blocks[-1]], {}, validate_signatures=False + ) + # Ignore errors from pre-validation, we are testing block_body_validation + repl_preval_results = dataclasses.replace(pre_validation_results[0], error=None, required_iters=uint64(1)) + assert (await b.receive_block(blocks[-1], repl_preval_results))[0:-1] == (ReceiveBlockResult.NEW_PEAK, None, 2) + @pytest.mark.asyncio @pytest.mark.parametrize("opcode", [ConditionOpcode.AGG_SIG_ME, ConditionOpcode.AGG_SIG_UNSAFE]) @pytest.mark.parametrize( @@ -1744,9 +1824,7 @@ class TestBodyValidation: synthetic_secret_key = calculate_synthetic_secret_key(secret_key, DEFAULT_HIDDEN_PUZZLE_HASH) public_key = synthetic_secret_key.get_g1() - args = [public_key, b"msg"] - if with_garbage: - args.append(b"garbage") + args = [public_key, b"msg"] + ([b"garbage"] if with_garbage else []) conditions = {opcode: [ConditionWithArgs(opcode, args)]} tx2: SpendBundle = wt.generate_signed_transaction(10, wt.get_new_puzzlehash(), coin1, condition_dic=conditions) @@ -1771,27 +1849,32 @@ class TestBodyValidation: @pytest.mark.asyncio @pytest.mark.parametrize( - "opcode,lock_value,expected", + "opcode,lock_value,expected,with_garbage", [ - (ConditionOpcode.ASSERT_SECONDS_RELATIVE, -2, ReceiveBlockResult.NEW_PEAK), - (ConditionOpcode.ASSERT_SECONDS_RELATIVE, -1, ReceiveBlockResult.NEW_PEAK), - (ConditionOpcode.ASSERT_SECONDS_RELATIVE, 0, ReceiveBlockResult.NEW_PEAK), - (ConditionOpcode.ASSERT_SECONDS_RELATIVE, 1, ReceiveBlockResult.INVALID_BLOCK), - (ConditionOpcode.ASSERT_HEIGHT_RELATIVE, -2, ReceiveBlockResult.NEW_PEAK), - (ConditionOpcode.ASSERT_HEIGHT_RELATIVE, -1, ReceiveBlockResult.NEW_PEAK), - (ConditionOpcode.ASSERT_HEIGHT_RELATIVE, 0, ReceiveBlockResult.INVALID_BLOCK), - (ConditionOpcode.ASSERT_HEIGHT_RELATIVE, 1, ReceiveBlockResult.INVALID_BLOCK), - (ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE, 2, ReceiveBlockResult.NEW_PEAK), - (ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE, 3, ReceiveBlockResult.INVALID_BLOCK), - (ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE, 4, ReceiveBlockResult.INVALID_BLOCK), + (ConditionOpcode.ASSERT_SECONDS_RELATIVE, -2, ReceiveBlockResult.NEW_PEAK, False), + (ConditionOpcode.ASSERT_SECONDS_RELATIVE, -1, ReceiveBlockResult.NEW_PEAK, False), + (ConditionOpcode.ASSERT_SECONDS_RELATIVE, 0, ReceiveBlockResult.NEW_PEAK, False), + (ConditionOpcode.ASSERT_SECONDS_RELATIVE, 1, ReceiveBlockResult.INVALID_BLOCK, False), + (ConditionOpcode.ASSERT_HEIGHT_RELATIVE, -2, ReceiveBlockResult.NEW_PEAK, False), + (ConditionOpcode.ASSERT_HEIGHT_RELATIVE, -1, ReceiveBlockResult.NEW_PEAK, False), + (ConditionOpcode.ASSERT_HEIGHT_RELATIVE, 0, ReceiveBlockResult.INVALID_BLOCK, False), + (ConditionOpcode.ASSERT_HEIGHT_RELATIVE, 1, ReceiveBlockResult.INVALID_BLOCK, False), + (ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE, 2, ReceiveBlockResult.NEW_PEAK, False), + (ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE, 3, ReceiveBlockResult.INVALID_BLOCK, False), + (ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE, 4, ReceiveBlockResult.INVALID_BLOCK, False), # genesis timestamp is 10000 and each block is 10 seconds - (ConditionOpcode.ASSERT_SECONDS_ABSOLUTE, 10029, ReceiveBlockResult.NEW_PEAK), - (ConditionOpcode.ASSERT_SECONDS_ABSOLUTE, 10030, ReceiveBlockResult.NEW_PEAK), - (ConditionOpcode.ASSERT_SECONDS_ABSOLUTE, 10031, ReceiveBlockResult.INVALID_BLOCK), - (ConditionOpcode.ASSERT_SECONDS_ABSOLUTE, 10032, ReceiveBlockResult.INVALID_BLOCK), + (ConditionOpcode.ASSERT_SECONDS_ABSOLUTE, 10029, ReceiveBlockResult.NEW_PEAK, False), + (ConditionOpcode.ASSERT_SECONDS_ABSOLUTE, 10030, ReceiveBlockResult.NEW_PEAK, False), + (ConditionOpcode.ASSERT_SECONDS_ABSOLUTE, 10031, ReceiveBlockResult.INVALID_BLOCK, False), + (ConditionOpcode.ASSERT_SECONDS_ABSOLUTE, 10032, ReceiveBlockResult.INVALID_BLOCK, False), + # additional garbage at the end of parameters + (ConditionOpcode.ASSERT_SECONDS_RELATIVE, 0, ReceiveBlockResult.NEW_PEAK, True), + (ConditionOpcode.ASSERT_HEIGHT_RELATIVE, -1, ReceiveBlockResult.NEW_PEAK, True), + (ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE, 2, ReceiveBlockResult.NEW_PEAK, True), + (ConditionOpcode.ASSERT_SECONDS_ABSOLUTE, 10029, ReceiveBlockResult.NEW_PEAK, True), ], ) - async def test_ephemeral_timelock(self, empty_blockchain, opcode, lock_value, expected, bt): + async def test_ephemeral_timelock(self, empty_blockchain, opcode, lock_value, expected, with_garbage, bt): b = empty_blockchain blocks = bt.get_consecutive_blocks( 3, @@ -1807,7 +1890,9 @@ class TestBodyValidation: wt: WalletTool = bt.get_pool_wallet_tool() - conditions = {opcode: [ConditionWithArgs(opcode, [int_to_bytes(lock_value)])]} + conditions = { + opcode: [ConditionWithArgs(opcode, [int_to_bytes(lock_value)] + ([b"garbage"] if with_garbage else []))] + } tx1: SpendBundle = wt.generate_signed_transaction( 10, wt.get_new_puzzlehash(), list(blocks[-1].get_included_reward_coins())[0] From ded9f68583f13553ad6723e3c9424f5da1e978d0 Mon Sep 17 00:00:00 2001 From: dustinface <35775977+xdustinface@users.noreply.github.com> Date: Fri, 8 Apr 2022 02:10:44 +0200 Subject: [PATCH 49/63] chia|tests|github: Implement, integrate and test plot sync protocol (#9695) * protocols|server: Define new harvester plot refreshing protocol messages * protocols: Bump `protocol_version` to `0.0.34` * tests: Introduce `setup_farmer_multi_harvester` Allows to run a test setup with 1 farmer and mutiple harvesters. * plotting: Add an initial plot loading indication to `PlotManager` * plotting|tests: Don't add removed duplicates to `total_result.removed` `PlotRefreshResult.removed` should only contain plots that were loaded properly before they were removed. It shouldn't contain e.g. removed duplicates or invalid plots since those are synced in an extra sync step and not as diff but as whole list every time. * harvester: Reset `PlotManager` on shutdown * plot_sync: Implement plot sync protocol * farmer|harvester: Integrate and enable plot sync * tests: Implement tests for the plot sync protocol * farmer|tests: Drop obsolete harvester caching code * setup: Add `chia.plot_sync` to packages * plot_sync: Type hints in `DeltaType` * plot_sync: Drop parameters in `super()` calls * plot_sync: Introduce `send_response` helper in `Receiver._process` * plot_sync: Add some parentheses Co-authored-by: Kyle Altendorf * plot_sync: Additional hint for a `Receiver.process_path_list` parameter * plot_sync: Force named parameters in `Receiver.process_path_list` * test: Fix fixtures after rebase * tests: Fix sorting after rebase * tests: Return type hint for `plot_sync_setup` * tests: Rename `WSChiaConnection` and move it in the outer scope * tests|plot_sync: More type hints * tests: Rework some delta tests * tests: Drop a `range` and iterate over the list directly * tests: Use the proper flags to overwrite * test: More missing duplicates tests * tests: Drop `ExpectedResult.reset` * tests: Reduce some asserts * tests: Add messages to some `assert False` statements * tests: Introduce `ErrorSimulation` enum in `test_sync_simulated.py` * tests: Use `secrects` instead of `Crypto.Random` * Fixes after rebase * Import from `typing_extensions` to support python 3.7 * Drop task name to support python 3.7 * Introduce `Sender.syncing`, `Sender.connected` and a log about the task * Add `tests/plot_sync/config.py` * Align the multi harvester fixture with what we do in other places * Update the workflows Co-authored-by: Kyle Altendorf --- .../workflows/build-test-macos-plot_sync.yml | 107 ++++ .../workflows/build-test-ubuntu-plot_sync.yml | 109 ++++ chia/farmer/farmer.py | 113 +--- chia/farmer/farmer_api.py | 43 +- chia/harvester/harvester.py | 17 +- chia/harvester/harvester_api.py | 14 +- chia/plot_sync/__init__.py | 0 chia/plot_sync/delta.py | 59 ++ chia/plot_sync/exceptions.py | 54 ++ chia/plot_sync/receiver.py | 304 ++++++++++ chia/plot_sync/sender.py | 327 +++++++++++ chia/plot_sync/util.py | 27 + chia/plotting/manager.py | 10 +- chia/protocols/harvester_protocol.py | 79 ++- chia/protocols/protocol_message_types.py | 8 + chia/protocols/shared_protocol.py | 2 +- chia/server/rate_limits.py | 8 + setup.py | 1 + tests/conftest.py | 22 + tests/core/test_farmer_harvester_rpc.py | 10 - tests/plot_sync/__init__.py | 0 tests/plot_sync/config.py | 2 + tests/plot_sync/test_delta.py | 90 +++ tests/plot_sync/test_plot_sync.py | 537 ++++++++++++++++++ tests/plot_sync/test_receiver.py | 376 ++++++++++++ tests/plot_sync/test_sender.py | 102 ++++ tests/plot_sync/test_sync_simulated.py | 433 ++++++++++++++ tests/plot_sync/util.py | 53 ++ tests/plotting/test_plot_manager.py | 17 +- tests/setup_nodes.py | 65 ++- tests/setup_services.py | 20 +- 31 files changed, 2877 insertions(+), 132 deletions(-) create mode 100644 .github/workflows/build-test-macos-plot_sync.yml create mode 100644 .github/workflows/build-test-ubuntu-plot_sync.yml create mode 100644 chia/plot_sync/__init__.py create mode 100644 chia/plot_sync/delta.py create mode 100644 chia/plot_sync/exceptions.py create mode 100644 chia/plot_sync/receiver.py create mode 100644 chia/plot_sync/sender.py create mode 100644 chia/plot_sync/util.py create mode 100644 tests/plot_sync/__init__.py create mode 100644 tests/plot_sync/config.py create mode 100644 tests/plot_sync/test_delta.py create mode 100644 tests/plot_sync/test_plot_sync.py create mode 100644 tests/plot_sync/test_receiver.py create mode 100644 tests/plot_sync/test_sender.py create mode 100644 tests/plot_sync/test_sync_simulated.py create mode 100644 tests/plot_sync/util.py diff --git a/.github/workflows/build-test-macos-plot_sync.yml b/.github/workflows/build-test-macos-plot_sync.yml new file mode 100644 index 0000000000..d18e76cf7d --- /dev/null +++ b/.github/workflows/build-test-macos-plot_sync.yml @@ -0,0 +1,107 @@ +# +# THIS FILE IS GENERATED. SEE https://github.com/Chia-Network/chia-blockchain/tree/main/tests#readme +# +name: MacOS plot_sync Tests + +on: + push: + branches: + - main + tags: + - '**' + pull_request: + branches: + - '**' + +concurrency: + # SHA is added to the end if on `main` to let all main workflows run + group: ${{ github.ref }}-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref == 'refs/heads/main' && github.sha || '' }} + cancel-in-progress: true + +jobs: + build: + name: MacOS plot_sync Tests + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + max-parallel: 4 + matrix: + python-version: [3.8, 3.9] + os: [macOS-latest] + env: + CHIA_ROOT: ${{ github.workspace }}/.chia/mainnet + JOB_FILE_NAME: tests_${{ matrix.os }}_python-${{ matrix.python-version }}_plot_sync + + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup Python environment + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Create keychain for CI use + run: | + security create-keychain -p foo chiachain + security default-keychain -s chiachain + security unlock-keychain -p foo chiachain + security set-keychain-settings -t 7200 -u chiachain + + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + + - name: Cache pip + uses: actions/cache@v3 + with: + # Note that new runners may break this https://github.com/actions/cache/issues/292 + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Checkout test blocks and plots + uses: actions/checkout@v3 + with: + repository: 'Chia-Network/test-cache' + path: '.chia' + ref: '0.28.0' + fetch-depth: 1 + + - name: Run install script + env: + INSTALL_PYTHON_VERSION: ${{ matrix.python-version }} + run: | + brew install boost + sh install.sh -d + +# Omitted installing Timelord + + - name: Test plot_sync code with pytest + run: | + . ./activate + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/plot_sync/test_*.py --durations=10 -n 4 -m "not benchmark" + + - name: Process coverage data + run: | + venv/bin/coverage combine --rcfile=.coveragerc .coverage.* + venv/bin/coverage xml --rcfile=.coveragerc -o coverage.xml + mkdir coverage_reports + cp .coverage "coverage_reports/.coverage.${{ env.JOB_FILE_NAME }}" + cp coverage.xml "coverage_reports/coverage.${{ env.JOB_FILE_NAME }}.xml" + venv/bin/coverage report --rcfile=.coveragerc --show-missing + + - name: Publish coverage + uses: actions/upload-artifact@v2 + with: + name: coverage + path: coverage_reports/* + if-no-files-found: error +# +# THIS FILE IS GENERATED. SEE https://github.com/Chia-Network/chia-blockchain/tree/main/tests#readme +# diff --git a/.github/workflows/build-test-ubuntu-plot_sync.yml b/.github/workflows/build-test-ubuntu-plot_sync.yml new file mode 100644 index 0000000000..886573b574 --- /dev/null +++ b/.github/workflows/build-test-ubuntu-plot_sync.yml @@ -0,0 +1,109 @@ +# +# THIS FILE IS GENERATED. SEE https://github.com/Chia-Network/chia-blockchain/tree/main/tests#readme +# +name: Ubuntu plot_sync Test + +on: + push: + branches: + - main + tags: + - '**' + pull_request: + branches: + - '**' + +concurrency: + # SHA is added to the end if on `main` to let all main workflows run + group: ${{ github.ref }}-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref == 'refs/heads/main' && github.sha || '' }} + cancel-in-progress: true + +jobs: + build: + name: Ubuntu plot_sync Test + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + max-parallel: 4 + matrix: + python-version: [3.7, 3.8, 3.9] + os: [ubuntu-latest] + env: + CHIA_ROOT: ${{ github.workspace }}/.chia/mainnet + JOB_FILE_NAME: tests_${{ matrix.os }}_python-${{ matrix.python-version }}_plot_sync + + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup Python environment + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache npm + uses: actions/cache@v3 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + + - name: Cache pip + uses: actions/cache@v3 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Checkout test blocks and plots + uses: actions/checkout@v3 + with: + repository: 'Chia-Network/test-cache' + path: '.chia' + ref: '0.28.0' + fetch-depth: 1 + + - name: Run install script + env: + INSTALL_PYTHON_VERSION: ${{ matrix.python-version }} + run: | + sh install.sh -d + +# Omitted installing Timelord + + - name: Test plot_sync code with pytest + run: | + . ./activate + venv/bin/coverage run --rcfile=.coveragerc ./venv/bin/py.test tests/plot_sync/test_*.py --durations=10 -n 4 -m "not benchmark" -p no:monitor + + - name: Process coverage data + run: | + venv/bin/coverage combine --rcfile=.coveragerc .coverage.* + venv/bin/coverage xml --rcfile=.coveragerc -o coverage.xml + mkdir coverage_reports + cp .coverage "coverage_reports/.coverage.${{ env.JOB_FILE_NAME }}" + cp coverage.xml "coverage_reports/coverage.${{ env.JOB_FILE_NAME }}.xml" + venv/bin/coverage report --rcfile=.coveragerc --show-missing + + - name: Publish coverage + uses: actions/upload-artifact@v2 + with: + name: coverage + path: coverage_reports/* + if-no-files-found: error + +# Omitted resource usage check + +# +# THIS FILE IS GENERATED. SEE https://github.com/Chia-Network/chia-blockchain/tree/main/tests#readme +# diff --git a/chia/farmer/farmer.py b/chia/farmer/farmer.py index 33f3b973a7..f7e50e9ed8 100644 --- a/chia/farmer/farmer.py +++ b/chia/farmer/farmer.py @@ -18,6 +18,8 @@ from chia.daemon.keychain_proxy import ( connect_to_keychain_and_validate, wrap_local_keychain, ) +from chia.plot_sync.receiver import Receiver +from chia.plot_sync.delta import Delta from chia.pools.pool_config import PoolWalletConfig, load_pool_config, add_auth_key from chia.protocols import farmer_protocol, harvester_protocol from chia.protocols.pool_protocol import ( @@ -59,32 +61,12 @@ log = logging.getLogger(__name__) UPDATE_POOL_INFO_INTERVAL: int = 3600 UPDATE_POOL_FARMER_INFO_INTERVAL: int = 300 -UPDATE_HARVESTER_CACHE_INTERVAL: int = 90 """ HARVESTER PROTOCOL (FARMER <-> HARVESTER) """ -class HarvesterCacheEntry: - def __init__(self): - self.data: Optional[dict] = None - self.last_update: float = 0 - - def bump_last_update(self): - self.last_update = time.time() - - def set_data(self, data): - self.data = data - self.bump_last_update() - - def needs_update(self, update_interval: int): - return time.time() - self.last_update > update_interval - - def expired(self, update_interval: int): - return time.time() - self.last_update > update_interval * 10 - - class Farmer: def __init__( self, @@ -115,8 +97,7 @@ class Farmer: # to periodically clear the memory self.cache_add_time: Dict[bytes32, uint64] = {} - # Interval to request plots from connected harvesters - self.update_harvester_cache_interval = UPDATE_HARVESTER_CACHE_INTERVAL + self.plot_sync_receivers: Dict[bytes32, Receiver] = {} self.cache_clear_task: Optional[asyncio.Task] = None self.update_pool_state_task: Optional[asyncio.Task] = None @@ -137,8 +118,6 @@ class Farmer: # Last time we updated pool_state based on the config file self.last_config_access_time: uint64 = uint64(0) - self.harvester_cache: Dict[str, Dict[str, HarvesterCacheEntry]] = {} - async def ensure_keychain_proxy(self) -> KeychainProxy: if self.keychain_proxy is None: if self.local_keychain: @@ -256,6 +235,7 @@ class Farmer: self.harvester_handshake_task = None if peer.connection_type is NodeType.HARVESTER: + self.plot_sync_receivers[peer.peer_node_id] = Receiver(peer, self.plot_sync_callback) self.harvester_handshake_task = asyncio.create_task(handshake_task()) def set_server(self, server): @@ -274,6 +254,13 @@ class Farmer: def on_disconnect(self, connection: ws.WSChiaConnection): self.log.info(f"peer disconnected {connection.get_peer_logging()}") self.state_changed("close_connection", {}) + if connection.connection_type is NodeType.HARVESTER: + del self.plot_sync_receivers[connection.peer_node_id] + + async def plot_sync_callback(self, peer_id: bytes32, delta: Delta) -> None: + log.info(f"plot_sync_callback: peer_id {peer_id}, delta {delta}") + if not delta.empty(): + self.state_changed("new_plots", await self.get_harvesters()) async def _pool_get_pool_info(self, pool_config: PoolWalletConfig) -> Optional[Dict]: try: @@ -642,80 +629,17 @@ class Farmer: return None - async def update_cached_harvesters(self) -> bool: - # First remove outdated cache entries - self.log.debug(f"update_cached_harvesters cache entries: {len(self.harvester_cache)}") - remove_hosts = [] - for host, host_cache in self.harvester_cache.items(): - remove_peers = [] - for peer_id, peer_cache in host_cache.items(): - # If the peer cache is expired it means the harvester didn't respond for too long - if peer_cache.expired(self.update_harvester_cache_interval): - remove_peers.append(peer_id) - for key in remove_peers: - del host_cache[key] - if len(host_cache) == 0: - self.log.debug(f"update_cached_harvesters remove host: {host}") - remove_hosts.append(host) - for key in remove_hosts: - del self.harvester_cache[key] - # Now query each harvester and update caches - updated = False - for connection in self.server.get_connections(NodeType.HARVESTER): - cache_entry = await self.get_cached_harvesters(connection) - if cache_entry.needs_update(self.update_harvester_cache_interval): - self.log.debug(f"update_cached_harvesters update harvester: {connection.peer_node_id}") - cache_entry.bump_last_update() - response = await connection.request_plots( - harvester_protocol.RequestPlots(), timeout=self.update_harvester_cache_interval - ) - if response is not None: - if isinstance(response, harvester_protocol.RespondPlots): - new_data: Dict = response.to_json_dict() - if cache_entry.data != new_data: - updated = True - self.log.debug(f"update_cached_harvesters cache updated: {connection.peer_node_id}") - else: - self.log.debug(f"update_cached_harvesters no changes for: {connection.peer_node_id}") - cache_entry.set_data(new_data) - else: - self.log.error( - f"Invalid response from harvester:" - f"peer_host {connection.peer_host}, peer_node_id {connection.peer_node_id}" - ) - else: - self.log.error( - f"Harvester '{connection.peer_host}/{connection.peer_node_id}' did not respond: " - f"(version mismatch or time out {UPDATE_HARVESTER_CACHE_INTERVAL}s)" - ) - return updated - - async def get_cached_harvesters(self, connection: WSChiaConnection) -> HarvesterCacheEntry: - host_cache = self.harvester_cache.get(connection.peer_host) - if host_cache is None: - host_cache = {} - self.harvester_cache[connection.peer_host] = host_cache - node_cache = host_cache.get(connection.peer_node_id.hex()) - if node_cache is None: - node_cache = HarvesterCacheEntry() - host_cache[connection.peer_node_id.hex()] = node_cache - return node_cache - async def get_harvesters(self) -> Dict: harvesters: List = [] for connection in self.server.get_connections(NodeType.HARVESTER): self.log.debug(f"get_harvesters host: {connection.peer_host}, node_id: {connection.peer_node_id}") - cache_entry = await self.get_cached_harvesters(connection) - if cache_entry.data is not None: - harvester_object: dict = dict(cache_entry.data) - harvester_object["connection"] = { - "node_id": connection.peer_node_id.hex(), - "host": connection.peer_host, - "port": connection.peer_port, - } - harvesters.append(harvester_object) + receiver = self.plot_sync_receivers.get(connection.peer_node_id) + if receiver is not None: + harvesters.append(receiver.to_dict()) else: - self.log.debug(f"get_harvesters no cache: {connection.peer_host}, node_id: {connection.peer_node_id}") + self.log.debug( + f"get_harvesters invalid peer: {connection.peer_host}, node_id: {connection.peer_node_id}" + ) return {"harvesters": harvesters} @@ -766,9 +690,6 @@ class Farmer: self.state_changed("add_connection", {}) refresh_slept = 0 - # Handles harvester plots cache cleanup and updates - if await self.update_cached_harvesters(): - self.state_changed("new_plots", await self.get_harvesters()) except Exception: log.error(f"_periodically_clear_cache_and_refresh_task failed: {traceback.format_exc()}") diff --git a/chia/farmer/farmer_api.py b/chia/farmer/farmer_api.py index d0cb11577e..9e94a50523 100644 --- a/chia/farmer/farmer_api.py +++ b/chia/farmer/farmer_api.py @@ -11,7 +11,13 @@ from chia.consensus.network_type import NetworkType from chia.consensus.pot_iterations import calculate_iterations_quality, calculate_sp_interval_iters from chia.farmer.farmer import Farmer from chia.protocols import farmer_protocol, harvester_protocol -from chia.protocols.harvester_protocol import PoolDifficulty +from chia.protocols.harvester_protocol import ( + PoolDifficulty, + PlotSyncStart, + PlotSyncPlotList, + PlotSyncPathList, + PlotSyncDone, +) from chia.protocols.pool_protocol import ( get_current_authentication_token, PoolErrorCode, @@ -518,3 +524,38 @@ class FarmerAPI: @peer_required async def respond_plots(self, _: harvester_protocol.RespondPlots, peer: ws.WSChiaConnection): self.farmer.log.warning(f"Respond plots came too late from: {peer.get_peer_logging()}") + + @api_request + @peer_required + async def plot_sync_start(self, message: PlotSyncStart, peer: ws.WSChiaConnection): + await self.farmer.plot_sync_receivers[peer.peer_node_id].sync_started(message) + + @api_request + @peer_required + async def plot_sync_loaded(self, message: PlotSyncPlotList, peer: ws.WSChiaConnection): + await self.farmer.plot_sync_receivers[peer.peer_node_id].process_loaded(message) + + @api_request + @peer_required + async def plot_sync_removed(self, message: PlotSyncPathList, peer: ws.WSChiaConnection): + await self.farmer.plot_sync_receivers[peer.peer_node_id].process_removed(message) + + @api_request + @peer_required + async def plot_sync_invalid(self, message: PlotSyncPathList, peer: ws.WSChiaConnection): + await self.farmer.plot_sync_receivers[peer.peer_node_id].process_invalid(message) + + @api_request + @peer_required + async def plot_sync_keys_missing(self, message: PlotSyncPathList, peer: ws.WSChiaConnection): + await self.farmer.plot_sync_receivers[peer.peer_node_id].process_keys_missing(message) + + @api_request + @peer_required + async def plot_sync_duplicates(self, message: PlotSyncPathList, peer: ws.WSChiaConnection): + await self.farmer.plot_sync_receivers[peer.peer_node_id].process_duplicates(message) + + @api_request + @peer_required + async def plot_sync_done(self, message: PlotSyncDone, peer: ws.WSChiaConnection): + await self.farmer.plot_sync_receivers[peer.peer_node_id].sync_done(message) diff --git a/chia/harvester/harvester.py b/chia/harvester/harvester.py index b63b0c7a16..fab6a77da9 100644 --- a/chia/harvester/harvester.py +++ b/chia/harvester/harvester.py @@ -8,6 +8,7 @@ from typing import Callable, Dict, List, Optional, Tuple import chia.server.ws_connection as ws # lgtm [py/import-and-import-from] from chia.consensus.constants import ConsensusConstants +from chia.plot_sync.sender import Sender from chia.plotting.manager import PlotManager from chia.plotting.util import ( add_plot_directory, @@ -25,6 +26,7 @@ log = logging.getLogger(__name__) class Harvester: plot_manager: PlotManager + plot_sync_sender: Sender root_path: Path _is_shutdown: bool executor: ThreadPoolExecutor @@ -53,6 +55,7 @@ class Harvester: self.plot_manager = PlotManager( root_path, refresh_parameter=refresh_parameter, refresh_callback=self._plot_refresh_callback ) + self.plot_sync_sender = Sender(self.plot_manager) self._is_shutdown = False self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=config["num_threads"]) self.state_changed_callback = None @@ -70,9 +73,11 @@ class Harvester: self._is_shutdown = True self.executor.shutdown(wait=True) self.plot_manager.stop_refreshing() + self.plot_manager.reset() + self.plot_sync_sender.stop() async def _await_closed(self): - pass + await self.plot_sync_sender.await_closed() def _set_state_changed_callback(self, callback: Callable): self.state_changed_callback = callback @@ -90,12 +95,18 @@ class Harvester: f"duration: {update_result.duration:.2f} seconds, " f"total plots: {len(self.plot_manager.plots)}" ) - if len(update_result.loaded) > 0: - self.event_loop.call_soon_threadsafe(self._state_changed, "plots") + if event == PlotRefreshEvents.started: + self.plot_sync_sender.sync_start(update_result.remaining, self.plot_manager.initial_refresh()) + if event == PlotRefreshEvents.batch_processed: + self.plot_sync_sender.process_batch(update_result.loaded, update_result.remaining) + if event == PlotRefreshEvents.done: + self.plot_sync_sender.sync_done(update_result.removed, update_result.duration) def on_disconnect(self, connection: ws.WSChiaConnection): self.log.info(f"peer disconnected {connection.get_peer_logging()}") self._state_changed("close_connection") + self.plot_manager.stop_refreshing() + self.plot_sync_sender.stop() def get_plots(self) -> Tuple[List[Dict], List[str], List[str]]: self.log.debug(f"get_plots prover items: {self.plot_manager.plot_count()}") diff --git a/chia/harvester/harvester_api.py b/chia/harvester/harvester_api.py index 760a57cb8c..8852867859 100644 --- a/chia/harvester/harvester_api.py +++ b/chia/harvester/harvester_api.py @@ -10,7 +10,7 @@ from chia.harvester.harvester import Harvester from chia.plotting.util import PlotInfo, parse_plot_info from chia.protocols import harvester_protocol from chia.protocols.farmer_protocol import FarmingInfo -from chia.protocols.harvester_protocol import Plot +from chia.protocols.harvester_protocol import Plot, PlotSyncResponse from chia.protocols.protocol_message_types import ProtocolMessageTypes from chia.server.outbound_message import make_msg from chia.server.ws_connection import WSChiaConnection @@ -30,8 +30,11 @@ class HarvesterAPI: def _set_state_changed_callback(self, callback: Callable): self.harvester.state_changed_callback = callback + @peer_required @api_request - async def harvester_handshake(self, harvester_handshake: harvester_protocol.HarvesterHandshake): + async def harvester_handshake( + self, harvester_handshake: harvester_protocol.HarvesterHandshake, peer: WSChiaConnection + ): """ Handshake between the harvester and farmer. The harvester receives the pool public keys, as well as the farmer pks, which must be put into the plots, before the plotting process begins. @@ -40,7 +43,8 @@ class HarvesterAPI: self.harvester.plot_manager.set_public_keys( harvester_handshake.farmer_public_keys, harvester_handshake.pool_public_keys ) - + self.harvester.plot_sync_sender.set_connection(peer) + await self.harvester.plot_sync_sender.start() self.harvester.plot_manager.start_refreshing() @peer_required @@ -289,3 +293,7 @@ class HarvesterAPI: response = harvester_protocol.RespondPlots(plots_response, failed_to_open_filenames, no_key_filenames) return make_msg(ProtocolMessageTypes.respond_plots, response) + + @api_request + async def plot_sync_response(self, response: PlotSyncResponse): + self.harvester.plot_sync_sender.set_response(response) diff --git a/chia/plot_sync/__init__.py b/chia/plot_sync/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/chia/plot_sync/delta.py b/chia/plot_sync/delta.py new file mode 100644 index 0000000000..6a797ffc33 --- /dev/null +++ b/chia/plot_sync/delta.py @@ -0,0 +1,59 @@ +from dataclasses import dataclass, field +from typing import Dict, List, Union + +from chia.protocols.harvester_protocol import Plot + + +@dataclass +class DeltaType: + additions: Union[Dict[str, Plot], List[str]] + removals: List[str] + + def __str__(self) -> str: + return f"+{len(self.additions)}/-{len(self.removals)}" + + def clear(self) -> None: + self.additions.clear() + self.removals.clear() + + def empty(self) -> bool: + return len(self.additions) == 0 and len(self.removals) == 0 + + +@dataclass +class PlotListDelta(DeltaType): + additions: Dict[str, Plot] = field(default_factory=dict) + removals: List[str] = field(default_factory=list) + + +@dataclass +class PathListDelta(DeltaType): + additions: List[str] = field(default_factory=list) + removals: List[str] = field(default_factory=list) + + @staticmethod + def from_lists(old: List[str], new: List[str]) -> "PathListDelta": + return PathListDelta([x for x in new if x not in old], [x for x in old if x not in new]) + + +@dataclass +class Delta: + valid: PlotListDelta = field(default_factory=PlotListDelta) + invalid: PathListDelta = field(default_factory=PathListDelta) + keys_missing: PathListDelta = field(default_factory=PathListDelta) + duplicates: PathListDelta = field(default_factory=PathListDelta) + + def empty(self) -> bool: + return self.valid.empty() and self.invalid.empty() and self.keys_missing.empty() and self.duplicates.empty() + + def __str__(self) -> str: + return ( + f"valid {self.valid}, invalid {self.invalid}, keys missing: {self.keys_missing}, " + f"duplicates: {self.duplicates}" + ) + + def clear(self) -> None: + self.valid.clear() + self.invalid.clear() + self.keys_missing.clear() + self.duplicates.clear() diff --git a/chia/plot_sync/exceptions.py b/chia/plot_sync/exceptions.py new file mode 100644 index 0000000000..e972a2a293 --- /dev/null +++ b/chia/plot_sync/exceptions.py @@ -0,0 +1,54 @@ +from typing import Any + +from chia.plot_sync.util import ErrorCodes, State +from chia.protocols.harvester_protocol import PlotSyncIdentifier +from chia.server.ws_connection import NodeType +from chia.util.ints import uint64 + + +class PlotSyncException(Exception): + def __init__(self, message: str, error_code: ErrorCodes) -> None: + super().__init__(message) + self.error_code = error_code + + +class AlreadyStartedError(Exception): + def __init__(self) -> None: + super().__init__("Already started!") + + +class InvalidValueError(PlotSyncException): + def __init__(self, message: str, actual: Any, expected: Any, error_code: ErrorCodes) -> None: + super().__init__(f"{message}: Actual {actual}, Expected {expected}", error_code) + + +class InvalidIdentifierError(InvalidValueError): + def __init__(self, actual_identifier: PlotSyncIdentifier, expected_identifier: PlotSyncIdentifier) -> None: + super().__init__("Invalid identifier", actual_identifier, expected_identifier, ErrorCodes.invalid_identifier) + self.actual_identifier: PlotSyncIdentifier = actual_identifier + self.expected_identifier: PlotSyncIdentifier = expected_identifier + + +class InvalidLastSyncIdError(InvalidValueError): + def __init__(self, actual: uint64, expected: uint64) -> None: + super().__init__("Invalid last-sync-id", actual, expected, ErrorCodes.invalid_last_sync_id) + + +class InvalidConnectionTypeError(InvalidValueError): + def __init__(self, actual: NodeType, expected: NodeType) -> None: + super().__init__("Unexpected connection type", actual, expected, ErrorCodes.invalid_connection_type) + + +class PlotAlreadyAvailableError(PlotSyncException): + def __init__(self, state: State, path: str) -> None: + super().__init__(f"{state.name}: Plot already available - {path}", ErrorCodes.plot_already_available) + + +class PlotNotAvailableError(PlotSyncException): + def __init__(self, state: State, path: str) -> None: + super().__init__(f"{state.name}: Plot not available - {path}", ErrorCodes.plot_not_available) + + +class SyncIdsMatchError(PlotSyncException): + def __init__(self, state: State, sync_id: uint64) -> None: + super().__init__(f"{state.name}: Sync ids are equal - {sync_id}", ErrorCodes.sync_ids_match) diff --git a/chia/plot_sync/receiver.py b/chia/plot_sync/receiver.py new file mode 100644 index 0000000000..4df791b531 --- /dev/null +++ b/chia/plot_sync/receiver.py @@ -0,0 +1,304 @@ +import logging +import time +from typing import Any, Callable, Collection, Coroutine, Dict, List, Optional + +from chia.plot_sync.delta import Delta, PathListDelta, PlotListDelta +from chia.plot_sync.exceptions import ( + InvalidIdentifierError, + InvalidLastSyncIdError, + PlotAlreadyAvailableError, + PlotNotAvailableError, + PlotSyncException, + SyncIdsMatchError, +) +from chia.plot_sync.util import ErrorCodes, State +from chia.protocols.harvester_protocol import ( + Plot, + PlotSyncDone, + PlotSyncError, + PlotSyncIdentifier, + PlotSyncPathList, + PlotSyncPlotList, + PlotSyncResponse, + PlotSyncStart, +) +from chia.server.ws_connection import ProtocolMessageTypes, WSChiaConnection, make_msg +from chia.types.blockchain_format.sized_bytes import bytes32 +from chia.util.ints import int16, uint64 +from chia.util.streamable import _T_Streamable + +log = logging.getLogger(__name__) + + +class Receiver: + _connection: WSChiaConnection + _sync_state: State + _delta: Delta + _expected_sync_id: uint64 + _expected_message_id: uint64 + _last_sync_id: uint64 + _last_sync_time: float + _plots: Dict[str, Plot] + _invalid: List[str] + _keys_missing: List[str] + _duplicates: List[str] + _update_callback: Callable[[bytes32, Delta], Coroutine[Any, Any, None]] + + def __init__( + self, connection: WSChiaConnection, update_callback: Callable[[bytes32, Delta], Coroutine[Any, Any, None]] + ) -> None: + self._connection = connection + self._sync_state = State.idle + self._delta = Delta() + self._expected_sync_id = uint64(0) + self._expected_message_id = uint64(0) + self._last_sync_id = uint64(0) + self._last_sync_time = 0 + self._plots = {} + self._invalid = [] + self._keys_missing = [] + self._duplicates = [] + self._update_callback = update_callback # type: ignore[assignment, misc] + + def reset(self) -> None: + self._sync_state = State.idle + self._expected_sync_id = uint64(0) + self._expected_message_id = uint64(0) + self._last_sync_id = uint64(0) + self._last_sync_time = 0 + self._plots.clear() + self._invalid.clear() + self._keys_missing.clear() + self._duplicates.clear() + self._delta.clear() + + def bump_expected_message_id(self) -> None: + self._expected_message_id = uint64(self._expected_message_id + 1) + + def connection(self) -> WSChiaConnection: + return self._connection + + def state(self) -> State: + return self._sync_state + + def expected_sync_id(self) -> uint64: + return self._expected_sync_id + + def expected_message_id(self) -> uint64: + return self._expected_message_id + + def last_sync_id(self) -> uint64: + return self._last_sync_id + + def last_sync_time(self) -> float: + return self._last_sync_time + + def plots(self) -> Dict[str, Plot]: + return self._plots + + def invalid(self) -> List[str]: + return self._invalid + + def keys_missing(self) -> List[str]: + return self._keys_missing + + def duplicates(self) -> List[str]: + return self._duplicates + + async def _process( + self, method: Callable[[_T_Streamable], Any], message_type: ProtocolMessageTypes, message: Any + ) -> None: + async def send_response(plot_sync_error: Optional[PlotSyncError] = None) -> None: + if self._connection is not None: + await self._connection.send_message( + make_msg( + ProtocolMessageTypes.plot_sync_response, + PlotSyncResponse(message.identifier, int16(message_type.value), plot_sync_error), + ) + ) + + try: + await method(message) + await send_response() + except InvalidIdentifierError as e: + log.warning(f"_process: InvalidIdentifierError {e}") + await send_response(PlotSyncError(int16(e.error_code), f"{e}", e.expected_identifier)) + except PlotSyncException as e: + log.warning(f"_process: Error {e}") + await send_response(PlotSyncError(int16(e.error_code), f"{e}", None)) + except Exception as e: + log.warning(f"_process: Exception {e}") + await send_response(PlotSyncError(int16(ErrorCodes.unknown), f"{e}", None)) + + def _validate_identifier(self, identifier: PlotSyncIdentifier, start: bool = False) -> None: + sync_id_match = identifier.sync_id == self._expected_sync_id + message_id_match = identifier.message_id == self._expected_message_id + identifier_match = sync_id_match and message_id_match + if (start and not message_id_match) or (not start and not identifier_match): + expected: PlotSyncIdentifier = PlotSyncIdentifier( + identifier.timestamp, self._expected_sync_id, self._expected_message_id + ) + raise InvalidIdentifierError( + identifier, + expected, + ) + + async def _sync_started(self, data: PlotSyncStart) -> None: + if data.initial: + self.reset() + self._validate_identifier(data.identifier, True) + if data.last_sync_id != self.last_sync_id(): + raise InvalidLastSyncIdError(data.last_sync_id, self.last_sync_id()) + if data.last_sync_id == data.identifier.sync_id: + raise SyncIdsMatchError(State.idle, data.last_sync_id) + self._expected_sync_id = data.identifier.sync_id + self._delta.clear() + self._sync_state = State.loaded + self.bump_expected_message_id() + + async def sync_started(self, data: PlotSyncStart) -> None: + await self._process(self._sync_started, ProtocolMessageTypes.plot_sync_start, data) + + async def _process_loaded(self, plot_infos: PlotSyncPlotList) -> None: + self._validate_identifier(plot_infos.identifier) + + for plot_info in plot_infos.data: + if plot_info.filename in self._plots or plot_info.filename in self._delta.valid.additions: + raise PlotAlreadyAvailableError(State.loaded, plot_info.filename) + self._delta.valid.additions[plot_info.filename] = plot_info + + if plot_infos.final: + self._sync_state = State.removed + + self.bump_expected_message_id() + + async def process_loaded(self, plot_infos: PlotSyncPlotList) -> None: + await self._process(self._process_loaded, ProtocolMessageTypes.plot_sync_loaded, plot_infos) + + async def process_path_list( + self, + *, + state: State, + next_state: State, + target: Collection[str], + delta: List[str], + paths: PlotSyncPathList, + is_removal: bool = False, + ) -> None: + self._validate_identifier(paths.identifier) + + for path in paths.data: + if is_removal and (path not in target or path in delta): + raise PlotNotAvailableError(state, path) + if not is_removal and path in delta: + raise PlotAlreadyAvailableError(state, path) + delta.append(path) + + if paths.final: + self._sync_state = next_state + + self.bump_expected_message_id() + + async def _process_removed(self, paths: PlotSyncPathList) -> None: + await self.process_path_list( + state=State.removed, + next_state=State.invalid, + target=self._plots, + delta=self._delta.valid.removals, + paths=paths, + is_removal=True, + ) + + async def process_removed(self, paths: PlotSyncPathList) -> None: + await self._process(self._process_removed, ProtocolMessageTypes.plot_sync_removed, paths) + + async def _process_invalid(self, paths: PlotSyncPathList) -> None: + await self.process_path_list( + state=State.invalid, + next_state=State.keys_missing, + target=self._invalid, + delta=self._delta.invalid.additions, + paths=paths, + ) + + async def process_invalid(self, paths: PlotSyncPathList) -> None: + await self._process(self._process_invalid, ProtocolMessageTypes.plot_sync_invalid, paths) + + async def _process_keys_missing(self, paths: PlotSyncPathList) -> None: + await self.process_path_list( + state=State.keys_missing, + next_state=State.duplicates, + target=self._keys_missing, + delta=self._delta.keys_missing.additions, + paths=paths, + ) + + async def process_keys_missing(self, paths: PlotSyncPathList) -> None: + await self._process(self._process_keys_missing, ProtocolMessageTypes.plot_sync_keys_missing, paths) + + async def _process_duplicates(self, paths: PlotSyncPathList) -> None: + await self.process_path_list( + state=State.duplicates, + next_state=State.done, + target=self._duplicates, + delta=self._delta.duplicates.additions, + paths=paths, + ) + + async def process_duplicates(self, paths: PlotSyncPathList) -> None: + await self._process(self._process_duplicates, ProtocolMessageTypes.plot_sync_duplicates, paths) + + async def _sync_done(self, data: PlotSyncDone) -> None: + self._validate_identifier(data.identifier) + # Update ids + self._last_sync_id = self._expected_sync_id + self._expected_sync_id = uint64(0) + self._expected_message_id = uint64(0) + # First create the update delta (i.e. transform invalid/keys_missing into additions/removals) which we will + # send to the callback receiver below + delta_invalid: PathListDelta = PathListDelta.from_lists(self._invalid, self._delta.invalid.additions) + delta_keys_missing: PathListDelta = PathListDelta.from_lists( + self._keys_missing, self._delta.keys_missing.additions + ) + delta_duplicates: PathListDelta = PathListDelta.from_lists(self._duplicates, self._delta.duplicates.additions) + update = Delta( + PlotListDelta(self._delta.valid.additions.copy(), self._delta.valid.removals.copy()), + delta_invalid, + delta_keys_missing, + delta_duplicates, + ) + # Apply delta + self._plots.update(self._delta.valid.additions) + for removal in self._delta.valid.removals: + del self._plots[removal] + self._invalid = self._delta.invalid.additions.copy() + self._keys_missing = self._delta.keys_missing.additions.copy() + self._duplicates = self._delta.duplicates.additions.copy() + # Update state and bump last sync time + self._sync_state = State.idle + self._last_sync_time = time.time() + # Let the callback receiver know if this sync cycle caused any update + try: + await self._update_callback(self._connection.peer_node_id, update) # type: ignore[misc,call-arg] + except Exception as e: + log.error(f"_update_callback raised: {e}") + self._delta.clear() + + async def sync_done(self, data: PlotSyncDone) -> None: + await self._process(self._sync_done, ProtocolMessageTypes.plot_sync_done, data) + + def to_dict(self) -> Dict[str, Any]: + result: Dict[str, Any] = { + "connection": { + "node_id": self._connection.peer_node_id, + "host": self._connection.peer_host, + "port": self._connection.peer_port, + }, + "plots": list(self._plots.values()), + "failed_to_open_filenames": self._invalid, + "no_key_filenames": self._keys_missing, + "duplicates": self._duplicates, + } + if self._last_sync_time != 0: + result["last_sync_time"] = self._last_sync_time + return result diff --git a/chia/plot_sync/sender.py b/chia/plot_sync/sender.py new file mode 100644 index 0000000000..257051a4c2 --- /dev/null +++ b/chia/plot_sync/sender.py @@ -0,0 +1,327 @@ +import asyncio +import logging +import threading +import time +import traceback +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Generic, Iterable, List, Optional, Tuple, Type, TypeVar + +from typing_extensions import Protocol + +from chia.plot_sync.exceptions import AlreadyStartedError, InvalidConnectionTypeError +from chia.plot_sync.util import Constants +from chia.plotting.manager import PlotManager +from chia.plotting.util import PlotInfo +from chia.protocols.harvester_protocol import ( + Plot, + PlotSyncDone, + PlotSyncIdentifier, + PlotSyncPathList, + PlotSyncPlotList, + PlotSyncResponse, + PlotSyncStart, +) +from chia.server.ws_connection import NodeType, ProtocolMessageTypes, WSChiaConnection, make_msg +from chia.util.generator_tools import list_to_batches +from chia.util.ints import int16, uint32, uint64 + +log = logging.getLogger(__name__) + + +def _convert_plot_info_list(plot_infos: List[PlotInfo]) -> List[Plot]: + converted: List[Plot] = [] + for plot_info in plot_infos: + converted.append( + Plot( + filename=plot_info.prover.get_filename(), + size=plot_info.prover.get_size(), + plot_id=plot_info.prover.get_id(), + pool_public_key=plot_info.pool_public_key, + pool_contract_puzzle_hash=plot_info.pool_contract_puzzle_hash, + plot_public_key=plot_info.plot_public_key, + file_size=uint64(plot_info.file_size), + time_modified=uint64(int(plot_info.time_modified)), + ) + ) + return converted + + +class PayloadType(Protocol): + def __init__(self, identifier: PlotSyncIdentifier, *args: object) -> None: + ... + + +T = TypeVar("T", bound=PayloadType) + + +@dataclass +class MessageGenerator(Generic[T]): + sync_id: uint64 + message_type: ProtocolMessageTypes + message_id: uint64 + payload_type: Type[T] + args: Iterable[object] + + def generate(self) -> Tuple[PlotSyncIdentifier, T]: + identifier = PlotSyncIdentifier(uint64(int(time.time())), self.sync_id, self.message_id) + payload = self.payload_type(identifier, *self.args) + return identifier, payload + + +@dataclass +class ExpectedResponse: + message_type: ProtocolMessageTypes + identifier: PlotSyncIdentifier + message: Optional[PlotSyncResponse] = None + + def __str__(self) -> str: + return ( + f"expected_message_type: {self.message_type.name}, " + f"expected_identifier: {self.identifier}, message {self.message}" + ) + + +class Sender: + _plot_manager: PlotManager + _connection: Optional[WSChiaConnection] + _sync_id: uint64 + _next_message_id: uint64 + _messages: List[MessageGenerator[PayloadType]] + _last_sync_id: uint64 + _stop_requested = False + _task: Optional[asyncio.Task] # type: ignore[type-arg] # Asks for Task parameter which doesn't work + _lock: threading.Lock + _response: Optional[ExpectedResponse] + + def __init__(self, plot_manager: PlotManager) -> None: + self._plot_manager = plot_manager + self._connection = None + self._sync_id = uint64(0) + self._next_message_id = uint64(0) + self._messages = [] + self._last_sync_id = uint64(0) + self._stop_requested = False + self._task = None + self._lock = threading.Lock() + self._response = None + + def __str__(self) -> str: + return f"sync_id {self._sync_id}, next_message_id {self._next_message_id}, messages {len(self._messages)}" + + async def start(self) -> None: + if self._task is not None and self._stop_requested: + await self.await_closed() + if self._task is None: + self._task = asyncio.create_task(self._run()) + # TODO, Add typing in PlotManager + if not self._plot_manager.initial_refresh() or self._sync_id != 0: # type:ignore[no-untyped-call] + self._reset() + else: + raise AlreadyStartedError() + + def stop(self) -> None: + self._stop_requested = True + + async def await_closed(self) -> None: + if self._task is not None: + await self._task + self._task = None + self._reset() + self._stop_requested = False + + def set_connection(self, connection: WSChiaConnection) -> None: + assert connection.connection_type is not None + if connection.connection_type != NodeType.FARMER: + raise InvalidConnectionTypeError(connection.connection_type, NodeType.HARVESTER) + self._connection = connection + + def bump_next_message_id(self) -> None: + self._next_message_id = uint64(self._next_message_id + 1) + + def _reset(self) -> None: + log.debug(f"_reset {self}") + self._last_sync_id = uint64(0) + self._sync_id = uint64(0) + self._next_message_id = uint64(0) + self._messages.clear() + if self._lock.locked(): + self._lock.release() + if self._task is not None: + # TODO, Add typing in PlotManager + self.sync_start(self._plot_manager.plot_count(), True) # type:ignore[no-untyped-call] + for remaining, batch in list_to_batches( + list(self._plot_manager.plots.values()), self._plot_manager.refresh_parameter.batch_size + ): + self.process_batch(batch, remaining) + self.sync_done([], 0) + + async def _wait_for_response(self) -> bool: + start = time.time() + assert self._response is not None + while time.time() - start < Constants.message_timeout and self._response.message is None: + await asyncio.sleep(0.1) + return self._response.message is not None + + def set_response(self, response: PlotSyncResponse) -> bool: + if self._response is None or self._response.message is not None: + log.warning(f"set_response skip unexpected response: {response}") + return False + if time.time() - float(response.identifier.timestamp) > Constants.message_timeout: + log.warning(f"set_response skip expired response: {response}") + return False + if response.identifier.sync_id != self._response.identifier.sync_id: + log.warning( + "set_response unexpected sync-id: " f"{response.identifier.sync_id}/{self._response.identifier.sync_id}" + ) + return False + if response.identifier.message_id != self._response.identifier.message_id: + log.warning( + "set_response unexpected message-id: " + f"{response.identifier.message_id}/{self._response.identifier.message_id}" + ) + return False + if response.message_type != int16(self._response.message_type.value): + log.warning( + "set_response unexpected message-type: " f"{response.message_type}/{self._response.message_type.value}" + ) + return False + log.debug(f"set_response valid {response}") + self._response.message = response + return True + + def _add_message(self, message_type: ProtocolMessageTypes, payload_type: Any, *args: Any) -> None: + assert self._sync_id != 0 + message_id = uint64(len(self._messages)) + self._messages.append(MessageGenerator(self._sync_id, message_type, message_id, payload_type, args)) + + async def _send_next_message(self) -> bool: + def failed(message: str) -> bool: + # By forcing a reset we try to get back into a normal state if some not recoverable failure came up. + log.warning(message) + self._reset() + return False + + assert len(self._messages) >= self._next_message_id + message_generator = self._messages[self._next_message_id] + identifier, payload = message_generator.generate() + if self._sync_id == 0 or identifier.sync_id != self._sync_id or identifier.message_id != self._next_message_id: + return failed(f"Invalid message generator {message_generator} for {self}") + + self._response = ExpectedResponse(message_generator.message_type, identifier) + log.debug(f"_send_next_message send {message_generator.message_type.name}: {payload}") + if self._connection is None or not await self._connection.send_message( + make_msg(message_generator.message_type, payload) + ): + return failed(f"Send failed {self._connection}") + if not await self._wait_for_response(): + log.info(f"_send_next_message didn't receive response {self._response}") + return False + + assert self._response.message is not None + if self._response.message.error is not None: + recovered = False + expected = self._response.message.error.expected_identifier + # If we have a recoverable error there is a `expected_identifier` included + if expected is not None: + # If the receiver has a zero sync/message id and we already sent all messages from the current event + # we most likely missed the response to the done message. We can finalize the sync and move on here. + all_sent = ( + self._messages[-1].message_type == ProtocolMessageTypes.plot_sync_done + and self._next_message_id == len(self._messages) - 1 + ) + if expected.sync_id == expected.message_id == 0 and all_sent: + self._finalize_sync() + recovered = True + elif self._sync_id == expected.sync_id and expected.message_id < len(self._messages): + self._next_message_id = expected.message_id + recovered = True + if not recovered: + return failed(f"Not recoverable error {self._response.message}") + return True + + if self._response.message_type == ProtocolMessageTypes.plot_sync_done: + self._finalize_sync() + else: + self.bump_next_message_id() + + return True + + def _add_list_batched(self, message_type: ProtocolMessageTypes, payload_type: Any, data: List[Any]) -> None: + if len(data) == 0: + self._add_message(message_type, payload_type, [], True) + return + for remaining, batch in list_to_batches(data, self._plot_manager.refresh_parameter.batch_size): + self._add_message(message_type, payload_type, batch, remaining == 0) + + def sync_start(self, count: float, initial: bool) -> None: + log.debug(f"sync_start {self}: count {count}, initial {initial}") + self._lock.acquire() + sync_id = int(time.time()) + # Make sure we have unique sync-id's even if we restart refreshing within a second (i.e. in tests) + if sync_id == self._last_sync_id: + sync_id = sync_id + 1 + log.debug(f"sync_start {sync_id}") + self._sync_id = uint64(sync_id) + self._add_message( + ProtocolMessageTypes.plot_sync_start, PlotSyncStart, initial, self._last_sync_id, uint32(int(count)) + ) + + def process_batch(self, loaded: List[PlotInfo], remaining: int) -> None: + log.debug(f"process_batch {self}: loaded {len(loaded)}, remaining {remaining}") + if len(loaded) > 0 or remaining == 0: + converted = _convert_plot_info_list(loaded) + self._add_message(ProtocolMessageTypes.plot_sync_loaded, PlotSyncPlotList, converted, remaining == 0) + + def sync_done(self, removed: List[Path], duration: float) -> None: + log.debug(f"sync_done {self}: removed {len(removed)}, duration {duration}") + removed_list = [str(x) for x in removed] + self._add_list_batched( + ProtocolMessageTypes.plot_sync_removed, + PlotSyncPathList, + removed_list, + ) + failed_to_open_list = [str(x) for x in list(self._plot_manager.failed_to_open_filenames)] + self._add_list_batched(ProtocolMessageTypes.plot_sync_invalid, PlotSyncPathList, failed_to_open_list) + no_key_list = [str(x) for x in self._plot_manager.no_key_filenames] + self._add_list_batched(ProtocolMessageTypes.plot_sync_keys_missing, PlotSyncPathList, no_key_list) + # TODO, Add typing in PlotManager + duplicates_list: List[str] = self._plot_manager.get_duplicates().copy() # type:ignore[no-untyped-call] + self._add_list_batched(ProtocolMessageTypes.plot_sync_duplicates, PlotSyncPathList, duplicates_list) + self._add_message(ProtocolMessageTypes.plot_sync_done, PlotSyncDone, uint64(int(duration))) + + def _finalize_sync(self) -> None: + log.debug(f"_finalize_sync {self}") + assert self._sync_id != 0 + self._last_sync_id = self._sync_id + self._sync_id = uint64(0) + self._next_message_id = uint64(0) + self._messages.clear() + self._lock.release() + + def sync_active(self) -> bool: + return self._lock.locked() and self._sync_id != 0 + + def connected(self) -> bool: + return self._connection is not None + + async def _run(self) -> None: + """ + This is the sender task responsible to send new messages during sync as they come into Sender._messages + triggered by the plot manager callback. + """ + while not self._stop_requested: + try: + while not self.connected() or not self.sync_active(): + if self._stop_requested: + return + await asyncio.sleep(0.1) + while not self._stop_requested and self.sync_active(): + if self._next_message_id >= len(self._messages): + await asyncio.sleep(0.1) + continue + if not await self._send_next_message(): + await asyncio.sleep(Constants.message_timeout) + except Exception as e: + log.error(f"Exception: {e} {traceback.format_exc()}") + self._reset() diff --git a/chia/plot_sync/util.py b/chia/plot_sync/util.py new file mode 100644 index 0000000000..5776631a73 --- /dev/null +++ b/chia/plot_sync/util.py @@ -0,0 +1,27 @@ +from enum import IntEnum + + +class Constants: + message_timeout: int = 10 + + +class State(IntEnum): + idle = 0 + loaded = 1 + removed = 2 + invalid = 3 + keys_missing = 4 + duplicates = 5 + done = 6 + + +class ErrorCodes(IntEnum): + unknown = -1 + invalid_state = 0 + invalid_peer_id = 1 + invalid_identifier = 2 + invalid_last_sync_id = 3 + invalid_connection_type = 4 + plot_already_available = 5 + plot_not_available = 6 + sync_ids_match = 7 diff --git a/chia/plotting/manager.py b/chia/plotting/manager.py index 6fcd84982e..7f48cbfc30 100644 --- a/chia/plotting/manager.py +++ b/chia/plotting/manager.py @@ -131,6 +131,7 @@ class PlotManager: _refresh_thread: Optional[threading.Thread] _refreshing_enabled: bool _refresh_callback: Callable + _initial: bool def __init__( self, @@ -158,6 +159,7 @@ class PlotManager: self._refresh_thread = None self._refreshing_enabled = False self._refresh_callback = refresh_callback # type: ignore + self._initial = True def __enter__(self): self._lock.acquire() @@ -172,6 +174,7 @@ class PlotManager: self.plot_filename_paths.clear() self.failed_to_open_filenames.clear() self.no_key_filenames.clear() + self._initial = True def set_refresh_callback(self, callback: Callable): self._refresh_callback = callback # type: ignore @@ -180,6 +183,9 @@ class PlotManager: self.farmer_public_keys = farmer_public_keys self.pool_public_keys = pool_public_keys + def initial_refresh(self): + return self._initial + def public_keys_available(self): return len(self.farmer_public_keys) and len(self.pool_public_keys) @@ -262,7 +268,6 @@ class PlotManager: loaded_plot = Path(path) / Path(plot_filename) if loaded_plot not in plot_paths: paths_to_remove.append(path) - total_result.removed.append(loaded_plot) for path in paths_to_remove: duplicated_paths.remove(path) @@ -290,6 +295,9 @@ class PlotManager: if self._refreshing_enabled: self._refresh_callback(PlotRefreshEvents.done, total_result) + # Reset the initial refresh indication + self._initial = False + # Cleanup unused cache available_ids = set([plot_info.prover.get_id() for plot_info in self.plots.values()]) invalid_cache_keys = [plot_id for plot_id in self.cache.keys() if plot_id not in available_ids] diff --git a/chia/protocols/harvester_protocol.py b/chia/protocols/harvester_protocol.py index b48165773f..4c5cddc145 100644 --- a/chia/protocols/harvester_protocol.py +++ b/chia/protocols/harvester_protocol.py @@ -5,7 +5,7 @@ from blspy import G1Element, G2Element from chia.types.blockchain_format.proof_of_space import ProofOfSpace from chia.types.blockchain_format.sized_bytes import bytes32 -from chia.util.ints import uint8, uint64 +from chia.util.ints import int16, uint8, uint32, uint64 from chia.util.streamable import Streamable, streamable """ @@ -95,3 +95,80 @@ class RespondPlots(Streamable): plots: List[Plot] failed_to_open_filenames: List[str] no_key_filenames: List[str] + + +@dataclass(frozen=True) +@streamable +class PlotSyncIdentifier(Streamable): + timestamp: uint64 + sync_id: uint64 + message_id: uint64 + + +@dataclass(frozen=True) +@streamable +class PlotSyncStart(Streamable): + identifier: PlotSyncIdentifier + initial: bool + last_sync_id: uint64 + plot_file_count: uint32 + + def __str__(self) -> str: + return ( + f"PlotSyncStart: identifier {self.identifier}, initial {self.initial}, " + f"last_sync_id {self.last_sync_id}, plot_file_count {self.plot_file_count}" + ) + + +@dataclass(frozen=True) +@streamable +class PlotSyncPathList(Streamable): + identifier: PlotSyncIdentifier + data: List[str] + final: bool + + def __str__(self) -> str: + return f"PlotSyncPathList: identifier {self.identifier}, count {len(self.data)}, final {self.final}" + + +@dataclass(frozen=True) +@streamable +class PlotSyncPlotList(Streamable): + identifier: PlotSyncIdentifier + data: List[Plot] + final: bool + + def __str__(self) -> str: + return f"PlotSyncPlotList: identifier {self.identifier}, count {len(self.data)}, final {self.final}" + + +@dataclass(frozen=True) +@streamable +class PlotSyncDone(Streamable): + identifier: PlotSyncIdentifier + duration: uint64 + + def __str__(self) -> str: + return f"PlotSyncDone: identifier {self.identifier}, duration {self.duration}" + + +@dataclass(frozen=True) +@streamable +class PlotSyncError(Streamable): + code: int16 + message: str + expected_identifier: Optional[PlotSyncIdentifier] + + def __str__(self) -> str: + return f"PlotSyncError: code {self.code}, count {self.message}, expected_identifier {self.expected_identifier}" + + +@dataclass(frozen=True) +@streamable +class PlotSyncResponse(Streamable): + identifier: PlotSyncIdentifier + message_type: int16 + error: Optional[PlotSyncError] + + def __str__(self) -> str: + return f"PlotSyncResponse: identifier {self.identifier}, message_type {self.message_type}, error {self.error}" diff --git a/chia/protocols/protocol_message_types.py b/chia/protocols/protocol_message_types.py index 7596f45547..b54e2717b4 100644 --- a/chia/protocols/protocol_message_types.py +++ b/chia/protocols/protocol_message_types.py @@ -86,6 +86,14 @@ class ProtocolMessageTypes(Enum): new_signage_point_harvester = 66 request_plots = 67 respond_plots = 68 + plot_sync_start = 78 + plot_sync_loaded = 79 + plot_sync_removed = 80 + plot_sync_invalid = 81 + plot_sync_keys_missing = 82 + plot_sync_duplicates = 83 + plot_sync_done = 84 + plot_sync_response = 85 # More wallet protocol coin_state_update = 69 diff --git a/chia/protocols/shared_protocol.py b/chia/protocols/shared_protocol.py index d7c0e6cd94..ed1cc9e7d6 100644 --- a/chia/protocols/shared_protocol.py +++ b/chia/protocols/shared_protocol.py @@ -5,7 +5,7 @@ from typing import List, Tuple from chia.util.ints import uint8, uint16 from chia.util.streamable import Streamable, streamable -protocol_version = "0.0.33" +protocol_version = "0.0.34" """ Handshake when establishing a connection between two servers. diff --git a/chia/server/rate_limits.py b/chia/server/rate_limits.py index 78f4d69340..b70c04b0f8 100644 --- a/chia/server/rate_limits.py +++ b/chia/server/rate_limits.py @@ -97,6 +97,14 @@ rate_limits_other = { ProtocolMessageTypes.farm_new_block: RLSettings(200, 200), ProtocolMessageTypes.request_plots: RLSettings(10, 10 * 1024 * 1024), ProtocolMessageTypes.respond_plots: RLSettings(10, 100 * 1024 * 1024), + ProtocolMessageTypes.plot_sync_start: RLSettings(1000, 100 * 1024 * 1024), + ProtocolMessageTypes.plot_sync_loaded: RLSettings(1000, 100 * 1024 * 1024), + ProtocolMessageTypes.plot_sync_removed: RLSettings(1000, 100 * 1024 * 1024), + ProtocolMessageTypes.plot_sync_invalid: RLSettings(1000, 100 * 1024 * 1024), + ProtocolMessageTypes.plot_sync_keys_missing: RLSettings(1000, 100 * 1024 * 1024), + ProtocolMessageTypes.plot_sync_duplicates: RLSettings(1000, 100 * 1024 * 1024), + ProtocolMessageTypes.plot_sync_done: RLSettings(1000, 100 * 1024 * 1024), + ProtocolMessageTypes.plot_sync_response: RLSettings(3000, 100 * 1024 * 1024), ProtocolMessageTypes.coin_state_update: RLSettings(1000, 100 * 1024 * 1024), ProtocolMessageTypes.register_interest_in_puzzle_hash: RLSettings(1000, 100 * 1024 * 1024), ProtocolMessageTypes.respond_to_ph_update: RLSettings(1000, 100 * 1024 * 1024), diff --git a/setup.py b/setup.py index 8d4d20558b..9c6e98238d 100644 --- a/setup.py +++ b/setup.py @@ -92,6 +92,7 @@ kwargs = dict( "chia.farmer", "chia.harvester", "chia.introducer", + "chia.plot_sync", "chia.plotters", "chia.plotting", "chia.pools", diff --git a/tests/conftest.py b/tests/conftest.py index 9dd58b313a..f5377b5761 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,9 @@ import pytest_asyncio import tempfile from tests.setup_nodes import setup_node_and_wallet, setup_n_nodes, setup_two_nodes +from pathlib import Path +from typing import Any, AsyncIterator, Dict, List, Tuple +from chia.server.start_service import Service # Set spawn after stdlib imports, but before other imports from chia.clvm.spend_sim import SimClient, SpendSim @@ -39,6 +42,7 @@ from pathlib import Path from chia.util.keyring_wrapper import KeyringWrapper from tests.block_tools import BlockTools, test_constants, create_block_tools, create_block_tools_async from tests.util.keyring import TempKeyring +from tests.setup_nodes import setup_farmer_multi_harvester @pytest.fixture(scope="session") @@ -403,6 +407,24 @@ async def two_nodes_one_block(bt, wallet_a): yield _ +@pytest_asyncio.fixture(scope="function") +async def farmer_one_harvester(tmp_path: Path, bt: BlockTools) -> AsyncIterator[Tuple[List[Service], Service]]: + async for _ in setup_farmer_multi_harvester(bt, 1, tmp_path, test_constants): + yield _ + + +@pytest_asyncio.fixture(scope="function") +async def farmer_two_harvester(tmp_path: Path, bt: BlockTools) -> AsyncIterator[Tuple[List[Service], Service]]: + async for _ in setup_farmer_multi_harvester(bt, 2, tmp_path, test_constants): + yield _ + + +@pytest_asyncio.fixture(scope="function") +async def farmer_three_harvester(tmp_path: Path, bt: BlockTools) -> AsyncIterator[Tuple[List[Service], Service]]: + async for _ in setup_farmer_multi_harvester(bt, 3, tmp_path, test_constants): + yield _ + + # TODO: Ideally, the db_version should be the (parameterized) db_version # fixture, to test all versions of the database schema. This doesn't work # because of a hack in shutting down the full node, which means you cannot run diff --git a/tests/core/test_farmer_harvester_rpc.py b/tests/core/test_farmer_harvester_rpc.py index 72a16ba9e4..e312698fe0 100644 --- a/tests/core/test_farmer_harvester_rpc.py +++ b/tests/core/test_farmer_harvester_rpc.py @@ -112,7 +112,6 @@ async def test_farmer_get_harvesters(harvester_farmer_environment): harvester_rpc_api, harvester_rpc_client, ) = harvester_farmer_environment - farmer_api = farmer_service._api harvester = harvester_service._node num_plots = 0 @@ -125,11 +124,6 @@ async def test_farmer_get_harvesters(harvester_farmer_environment): await time_out_assert(10, non_zero_plots) - # Reset cache and force updates cache every second to make sure the farmer gets the most recent data - update_interval_before = farmer_api.farmer.update_harvester_cache_interval - farmer_api.farmer.update_harvester_cache_interval = 1 - farmer_api.farmer.harvester_cache = {} - async def test_get_harvesters(): harvester.plot_manager.trigger_refresh() await time_out_assert(5, harvester.plot_manager.needs_refresh, value=False) @@ -144,10 +138,6 @@ async def test_farmer_get_harvesters(harvester_farmer_environment): await time_out_assert_custom_interval(30, 1, test_get_harvesters) - # Reset cache and reset update interval to avoid hitting the rate limit - farmer_api.farmer.update_harvester_cache_interval = update_interval_before - farmer_api.farmer.harvester_cache = {} - @pytest.mark.asyncio async def test_farmer_signage_point_endpoints(harvester_farmer_environment): diff --git a/tests/plot_sync/__init__.py b/tests/plot_sync/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/plot_sync/config.py b/tests/plot_sync/config.py new file mode 100644 index 0000000000..235efb181c --- /dev/null +++ b/tests/plot_sync/config.py @@ -0,0 +1,2 @@ +parallel = True +checkout_blocks_and_plots = True diff --git a/tests/plot_sync/test_delta.py b/tests/plot_sync/test_delta.py new file mode 100644 index 0000000000..057e449bda --- /dev/null +++ b/tests/plot_sync/test_delta.py @@ -0,0 +1,90 @@ +import logging +from typing import List + +import pytest +from blspy import G1Element + +from chia.plot_sync.delta import Delta, DeltaType, PathListDelta, PlotListDelta +from chia.protocols.harvester_protocol import Plot +from chia.types.blockchain_format.sized_bytes import bytes32 +from chia.util.ints import uint8, uint64 + +log = logging.getLogger(__name__) + + +def dummy_plot(path: str) -> Plot: + return Plot(path, uint8(32), bytes32(b"\00" * 32), G1Element(), None, G1Element(), uint64(0), uint64(0)) + + +@pytest.mark.parametrize( + ["delta"], + [ + pytest.param(PathListDelta(), id="path list"), + pytest.param(PlotListDelta(), id="plot list"), + ], +) +def test_list_delta(delta: DeltaType) -> None: + assert delta.empty() + if type(delta) == PathListDelta: + assert delta.additions == [] + elif type(delta) == PlotListDelta: + assert delta.additions == {} + else: + assert False + assert delta.removals == [] + assert delta.empty() + if type(delta) == PathListDelta: + delta.additions.append("0") + elif type(delta) == PlotListDelta: + delta.additions["0"] = dummy_plot("0") + else: + assert False, "Invalid delta type" + assert not delta.empty() + delta.removals.append("0") + assert not delta.empty() + delta.additions.clear() + assert not delta.empty() + delta.clear() + assert delta.empty() + + +@pytest.mark.parametrize( + ["old", "new", "result"], + [ + [[], [], PathListDelta()], + [["1"], ["0"], PathListDelta(["0"], ["1"])], + [["1", "2", "3"], ["1", "2", "3"], PathListDelta([], [])], + [["2", "1", "3"], ["2", "3", "1"], PathListDelta([], [])], + [["2"], ["2", "3", "1"], PathListDelta(["3", "1"], [])], + [["2"], ["1", "3"], PathListDelta(["1", "3"], ["2"])], + [["1"], ["1", "2", "3"], PathListDelta(["2", "3"], [])], + [[], ["1", "2", "3"], PathListDelta(["1", "2", "3"], [])], + [["-1"], ["1", "2", "3"], PathListDelta(["1", "2", "3"], ["-1"])], + [["-1", "1"], ["2", "3"], PathListDelta(["2", "3"], ["-1", "1"])], + [["-1", "1", "2"], ["2", "3"], PathListDelta(["3"], ["-1", "1"])], + [["-1", "2", "3"], ["2", "3"], PathListDelta([], ["-1"])], + [["-1", "2", "3", "-2"], ["2", "3"], PathListDelta([], ["-1", "-2"])], + [["-2", "2", "3", "-1"], ["2", "3"], PathListDelta([], ["-2", "-1"])], + ], +) +def test_path_list_delta_from_lists(old: List[str], new: List[str], result: PathListDelta) -> None: + assert PathListDelta.from_lists(old, new) == result + + +def test_delta_empty() -> None: + delta: Delta = Delta() + all_deltas: List[DeltaType] = [delta.valid, delta.invalid, delta.keys_missing, delta.duplicates] + assert delta.empty() + for d1 in all_deltas: + delta.valid.additions["0"] = dummy_plot("0") + delta.invalid.additions.append("0") + delta.keys_missing.additions.append("0") + delta.duplicates.additions.append("0") + assert not delta.empty() + for d2 in all_deltas: + if d2 is not d1: + d2.clear() + assert not delta.empty() + assert not delta.empty() + d1.clear() + assert delta.empty() diff --git a/tests/plot_sync/test_plot_sync.py b/tests/plot_sync/test_plot_sync.py new file mode 100644 index 0000000000..96e1fcadd8 --- /dev/null +++ b/tests/plot_sync/test_plot_sync.py @@ -0,0 +1,537 @@ +from dataclasses import dataclass, field +from pathlib import Path +from shutil import copy +from typing import List, Optional, Tuple + +import pytest +import pytest_asyncio +from blspy import G1Element + +from chia.farmer.farmer_api import Farmer +from chia.harvester.harvester_api import Harvester +from chia.plot_sync.delta import Delta, PathListDelta, PlotListDelta +from chia.plot_sync.receiver import Receiver +from chia.plot_sync.sender import Sender +from chia.plot_sync.util import State +from chia.plotting.manager import PlotManager +from chia.plotting.util import add_plot_directory, remove_plot_directory +from chia.protocols.harvester_protocol import Plot +from chia.server.start_service import Service +from chia.types.blockchain_format.sized_bytes import bytes32 +from chia.util.config import create_default_chia_config +from chia.util.ints import uint8, uint64 +from tests.block_tools import BlockTools +from tests.plot_sync.util import start_harvester_service +from tests.plotting.test_plot_manager import MockPlotInfo, TestDirectory +from tests.plotting.util import get_test_plots +from tests.time_out_assert import time_out_assert + + +def synced(sender: Sender, receiver: Receiver, previous_last_sync_id: int) -> bool: + return ( + sender._last_sync_id != previous_last_sync_id + and sender._last_sync_id == receiver._last_sync_id != 0 + and receiver.state() == State.idle + and not sender._lock.locked() + ) + + +def assert_path_list_matches(expected_list: List[str], actual_list: List[str]) -> None: + assert len(expected_list) == len(actual_list) + for item in expected_list: + assert str(item) in actual_list + + +@dataclass +class ExpectedResult: + valid_count: int = 0 + valid_delta: PlotListDelta = field(default_factory=PlotListDelta) + invalid_count: int = 0 + invalid_delta: PathListDelta = field(default_factory=PathListDelta) + keys_missing_count: int = 0 + keys_missing_delta: PathListDelta = field(default_factory=PathListDelta) + duplicates_count: int = 0 + duplicates_delta: PathListDelta = field(default_factory=PathListDelta) + callback_passed: bool = False + + def add_valid(self, list_plots: List[MockPlotInfo]) -> None: + def create_mock_plot(info: MockPlotInfo) -> Plot: + return Plot( + info.prover.get_filename(), + uint8(0), + bytes32(b"\x00" * 32), + None, + None, + G1Element(), + uint64(0), + uint64(0), + ) + + self.valid_count += len(list_plots) + self.valid_delta.additions.update({x.prover.get_filename(): create_mock_plot(x) for x in list_plots}) + + def remove_valid(self, list_paths: List[Path]) -> None: + self.valid_count -= len(list_paths) + self.valid_delta.removals += [str(x) for x in list_paths] + + def add_invalid(self, list_paths: List[Path]) -> None: + self.invalid_count += len(list_paths) + self.invalid_delta.additions += [str(x) for x in list_paths] + + def remove_invalid(self, list_paths: List[Path]) -> None: + self.invalid_count -= len(list_paths) + self.invalid_delta.removals += [str(x) for x in list_paths] + + def add_keys_missing(self, list_paths: List[Path]) -> None: + self.keys_missing_count += len(list_paths) + self.keys_missing_delta.additions += [str(x) for x in list_paths] + + def remove_keys_missing(self, list_paths: List[Path]) -> None: + self.keys_missing_count -= len(list_paths) + self.keys_missing_delta.removals += [str(x) for x in list_paths] + + def add_duplicates(self, list_paths: List[Path]) -> None: + self.duplicates_count += len(list_paths) + self.duplicates_delta.additions += [str(x) for x in list_paths] + + def remove_duplicates(self, list_paths: List[Path]) -> None: + self.duplicates_count -= len(list_paths) + self.duplicates_delta.removals += [str(x) for x in list_paths] + + +@dataclass +class Environment: + root_path: Path + harvester_services: List[Service] + farmer_service: Service + harvesters: List[Harvester] + farmer: Farmer + dir_1: TestDirectory + dir_2: TestDirectory + dir_3: TestDirectory + dir_4: TestDirectory + dir_invalid: TestDirectory + dir_keys_missing: TestDirectory + dir_duplicates: TestDirectory + expected: List[ExpectedResult] + + def get_harvester(self, peer_id: bytes32) -> Optional[Harvester]: + for harvester in self.harvesters: + assert harvester.server is not None + if harvester.server.node_id == peer_id: + return harvester + return None + + def add_directory(self, harvester_index: int, directory: TestDirectory, state: State = State.loaded) -> None: + add_plot_directory(self.harvesters[harvester_index].root_path, str(directory.path)) + if state == State.loaded: + self.expected[harvester_index].add_valid(directory.plot_info_list()) + elif state == State.invalid: + self.expected[harvester_index].add_invalid(directory.path_list()) + elif state == State.keys_missing: + self.expected[harvester_index].add_keys_missing(directory.path_list()) + elif state == State.duplicates: + self.expected[harvester_index].add_duplicates(directory.path_list()) + else: + assert False, "Invalid state" + + def remove_directory(self, harvester_index: int, directory: TestDirectory, state: State = State.removed) -> None: + remove_plot_directory(self.harvesters[harvester_index].root_path, str(directory.path)) + if state == State.removed: + self.expected[harvester_index].remove_valid(directory.path_list()) + elif state == State.invalid: + self.expected[harvester_index].remove_invalid(directory.path_list()) + elif state == State.keys_missing: + self.expected[harvester_index].remove_keys_missing(directory.path_list()) + elif state == State.duplicates: + self.expected[harvester_index].remove_duplicates(directory.path_list()) + else: + assert False, "Invalid state" + + def add_all_directories(self, harvester_index: int) -> None: + self.add_directory(harvester_index, self.dir_1) + self.add_directory(harvester_index, self.dir_2) + self.add_directory(harvester_index, self.dir_3) + self.add_directory(harvester_index, self.dir_4) + self.add_directory(harvester_index, self.dir_keys_missing, State.keys_missing) + self.add_directory(harvester_index, self.dir_invalid, State.invalid) + # Note: This does not add dir_duplicates since its important that the duplicated plots are loaded after the + # the original ones. + # self.add_directory(harvester_index, self.dir_duplicates, State.duplicates) + + def remove_all_directories(self, harvester_index: int) -> None: + self.remove_directory(harvester_index, self.dir_1) + self.remove_directory(harvester_index, self.dir_2) + self.remove_directory(harvester_index, self.dir_3) + self.remove_directory(harvester_index, self.dir_4) + self.remove_directory(harvester_index, self.dir_keys_missing, State.keys_missing) + self.remove_directory(harvester_index, self.dir_invalid, State.invalid) + self.remove_directory(harvester_index, self.dir_duplicates, State.duplicates) + + async def plot_sync_callback(self, peer_id: bytes32, delta: Delta) -> None: + harvester: Optional[Harvester] = self.get_harvester(peer_id) + assert harvester is not None + expected = self.expected[self.harvesters.index(harvester)] + assert len(expected.valid_delta.additions) == len(delta.valid.additions) + for path, plot_info in expected.valid_delta.additions.items(): + assert path in delta.valid.additions + plot = harvester.plot_manager.plots.get(Path(path), None) + assert plot is not None + assert plot.prover.get_filename() == delta.valid.additions[path].filename + assert plot.prover.get_size() == delta.valid.additions[path].size + assert plot.prover.get_id() == delta.valid.additions[path].plot_id + assert plot.pool_public_key == delta.valid.additions[path].pool_public_key + assert plot.pool_contract_puzzle_hash == delta.valid.additions[path].pool_contract_puzzle_hash + assert plot.plot_public_key == delta.valid.additions[path].plot_public_key + assert plot.file_size == delta.valid.additions[path].file_size + assert int(plot.time_modified) == delta.valid.additions[path].time_modified + + assert_path_list_matches(expected.valid_delta.removals, delta.valid.removals) + assert_path_list_matches(expected.invalid_delta.additions, delta.invalid.additions) + assert_path_list_matches(expected.invalid_delta.removals, delta.invalid.removals) + assert_path_list_matches(expected.keys_missing_delta.additions, delta.keys_missing.additions) + assert_path_list_matches(expected.keys_missing_delta.removals, delta.keys_missing.removals) + assert_path_list_matches(expected.duplicates_delta.additions, delta.duplicates.additions) + assert_path_list_matches(expected.duplicates_delta.removals, delta.duplicates.removals) + expected.valid_delta.clear() + expected.invalid_delta.clear() + expected.keys_missing_delta.clear() + expected.duplicates_delta.clear() + expected.callback_passed = True + + async def run_sync_test(self) -> None: + plot_manager: PlotManager + assert len(self.harvesters) == len(self.expected) + last_sync_ids: List[uint64] = [] + # Run the test in two steps, first trigger the refresh on both harvesters + for harvester in self.harvesters: + plot_manager = harvester.plot_manager + assert harvester.server is not None + receiver = self.farmer.plot_sync_receivers[harvester.server.node_id] + # Make sure to reset the passed flag always before a new run + self.expected[self.harvesters.index(harvester)].callback_passed = False + receiver._update_callback = self.plot_sync_callback + assert harvester.plot_sync_sender._last_sync_id == receiver._last_sync_id + last_sync_ids.append(harvester.plot_sync_sender._last_sync_id) + plot_manager.start_refreshing() + plot_manager.trigger_refresh() + # Then wait for them to be synced with the farmer and validate them + for harvester in self.harvesters: + plot_manager = harvester.plot_manager + assert harvester.server is not None + receiver = self.farmer.plot_sync_receivers[harvester.server.node_id] + await time_out_assert(10, plot_manager.needs_refresh, value=False) + harvester_index = self.harvesters.index(harvester) + await time_out_assert( + 10, synced, True, harvester.plot_sync_sender, receiver, last_sync_ids[harvester_index] + ) + expected = self.expected[harvester_index] + assert plot_manager.plot_count() == len(receiver.plots()) == expected.valid_count + assert len(plot_manager.failed_to_open_filenames) == len(receiver.invalid()) == expected.invalid_count + assert len(plot_manager.no_key_filenames) == len(receiver.keys_missing()) == expected.keys_missing_count + assert len(plot_manager.get_duplicates()) == len(receiver.duplicates()) == expected.duplicates_count + assert expected.callback_passed + assert expected.valid_delta.empty() + assert expected.invalid_delta.empty() + assert expected.keys_missing_delta.empty() + assert expected.duplicates_delta.empty() + for path, plot_info in plot_manager.plots.items(): + assert str(path) in receiver.plots() + assert plot_info.prover.get_filename() == receiver.plots()[str(path)].filename + assert plot_info.prover.get_size() == receiver.plots()[str(path)].size + assert plot_info.prover.get_id() == receiver.plots()[str(path)].plot_id + assert plot_info.pool_public_key == receiver.plots()[str(path)].pool_public_key + assert plot_info.pool_contract_puzzle_hash == receiver.plots()[str(path)].pool_contract_puzzle_hash + assert plot_info.plot_public_key == receiver.plots()[str(path)].plot_public_key + assert plot_info.file_size == receiver.plots()[str(path)].file_size + assert int(plot_info.time_modified) == receiver.plots()[str(path)].time_modified + for path in plot_manager.failed_to_open_filenames: + assert str(path) in receiver.invalid() + for path in plot_manager.no_key_filenames: + assert str(path) in receiver.keys_missing() + for path in plot_manager.get_duplicates(): + assert str(path) in receiver.duplicates() + + async def handshake_done(self, index: int) -> bool: + return ( + self.harvesters[index].plot_manager._refresh_thread is not None + and len(self.harvesters[index].plot_manager.farmer_public_keys) > 0 + ) + + +@pytest_asyncio.fixture(scope="function") +async def environment( + bt: BlockTools, tmp_path: Path, farmer_two_harvester: Tuple[List[Service], Service] +) -> Environment: + def new_test_dir(name: str, plot_list: List[Path]) -> TestDirectory: + return TestDirectory(tmp_path / "plots" / name, plot_list) + + plots: List[Path] = get_test_plots() + plots_invalid: List[Path] = get_test_plots()[0:3] + plots_keys_missing: List[Path] = get_test_plots("not_in_keychain") + # Create 4 directories where: dir_n contains n plots + directories: List[TestDirectory] = [] + offset: int = 0 + while len(directories) < 4: + dir_number = len(directories) + 1 + directories.append(new_test_dir(f"{dir_number}", plots[offset : offset + dir_number])) + offset += dir_number + + dir_invalid: TestDirectory = new_test_dir("invalid", plots_invalid) + dir_keys_missing: TestDirectory = new_test_dir("keys_missing", plots_keys_missing) + dir_duplicates: TestDirectory = new_test_dir("duplicates", directories[3].plots) + create_default_chia_config(tmp_path) + + # Invalidate the plots in `dir_invalid` + for path in dir_invalid.path_list(): + with open(path, "wb") as file: + file.write(bytes(100)) + + harvester_services: List[Service] + farmer_service: Service + harvester_services, farmer_service = farmer_two_harvester + farmer: Farmer = farmer_service._node + harvesters: List[Harvester] = [await start_harvester_service(service) for service in harvester_services] + for harvester in harvesters: + harvester.plot_manager.set_public_keys( + bt.plot_manager.farmer_public_keys.copy(), bt.plot_manager.pool_public_keys.copy() + ) + + assert len(farmer.plot_sync_receivers) == 2 + + return Environment( + tmp_path, + harvester_services, + farmer_service, + harvesters, + farmer, + directories[0], + directories[1], + directories[2], + directories[3], + dir_invalid, + dir_keys_missing, + dir_duplicates, + [ExpectedResult() for _ in harvesters], + ) + + +@pytest.mark.asyncio +async def test_sync_valid(environment: Environment) -> None: + env: Environment = environment + env.add_directory(0, env.dir_1) + env.add_directory(1, env.dir_2) + await env.run_sync_test() + # Run again two times to make sure we still get the same results in repeated refresh intervals + env.expected[0].valid_delta.clear() + env.expected[1].valid_delta.clear() + await env.run_sync_test() + await env.run_sync_test() + env.add_directory(0, env.dir_3) + env.add_directory(1, env.dir_4) + await env.run_sync_test() + while len(env.dir_3.path_list()): + drop_plot = env.dir_3.path_list()[0] + drop_plot.unlink() + env.dir_3.drop(drop_plot) + env.expected[0].remove_valid([drop_plot]) + await env.run_sync_test() + env.remove_directory(0, env.dir_3) + await env.run_sync_test() + env.remove_directory(1, env.dir_4) + await env.run_sync_test() + env.remove_directory(0, env.dir_1) + env.remove_directory(1, env.dir_2) + await env.run_sync_test() + + +@pytest.mark.asyncio +async def test_sync_invalid(environment: Environment) -> None: + env: Environment = environment + assert len(env.farmer.plot_sync_receivers) == 2 + # Use dir_3 and dir_4 in this test because the invalid plots are copies from dir_1 + dir_2 + env.add_directory(0, env.dir_3) + env.add_directory(0, env.dir_invalid, State.invalid) + env.add_directory(1, env.dir_4) + await env.run_sync_test() + # Run again two times to make sure we still get the same results in repeated refresh intervals + await env.run_sync_test() + await env.run_sync_test() + # Drop all but two of the invalid plots + assert len(env.dir_invalid) > 2 + for _ in range(len(env.dir_invalid) - 2): + drop_plot = env.dir_invalid.path_list()[0] + drop_plot.unlink() + env.dir_invalid.drop(drop_plot) + env.expected[0].remove_invalid([drop_plot]) + await env.run_sync_test() + assert len(env.dir_invalid) == 2 + # Add the directory to the first harvester too + env.add_directory(1, env.dir_invalid, State.invalid) + await env.run_sync_test() + # Recover one the remaining invalid plot + for path in get_test_plots(): + if path.name == env.dir_invalid.path_list()[0].name: + copy(path, env.dir_invalid.path) + for i in range(len(env.harvesters)): + env.expected[i].add_valid([env.dir_invalid.plot_info_list()[0]]) + env.expected[i].remove_invalid([env.dir_invalid.path_list()[0]]) + env.harvesters[i].plot_manager.refresh_parameter.retry_invalid_seconds = 0 + await env.run_sync_test() + for i in [0, 1]: + remove_plot_directory(env.harvesters[i].root_path, str(env.dir_invalid.path)) + env.expected[i].remove_valid([env.dir_invalid.path_list()[0]]) + env.expected[i].remove_invalid([env.dir_invalid.path_list()[1]]) + await env.run_sync_test() + + +@pytest.mark.asyncio +async def test_sync_keys_missing(environment: Environment) -> None: + env: Environment = environment + env.add_directory(0, env.dir_1) + env.add_directory(0, env.dir_keys_missing, State.keys_missing) + env.add_directory(1, env.dir_2) + await env.run_sync_test() + # Run again two times to make sure we still get the same results in repeated refresh intervals + await env.run_sync_test() + await env.run_sync_test() + # Drop all but 2 plots with missing keys and test sync inbetween + assert len(env.dir_keys_missing) > 2 + for _ in range(len(env.dir_keys_missing) - 2): + drop_plot = env.dir_keys_missing.path_list()[0] + drop_plot.unlink() + env.dir_keys_missing.drop(drop_plot) + env.expected[0].remove_keys_missing([drop_plot]) + await env.run_sync_test() + assert len(env.dir_keys_missing) == 2 + # Add the plots with missing keys to the other harvester + env.add_directory(0, env.dir_3) + env.add_directory(1, env.dir_keys_missing, State.keys_missing) + await env.run_sync_test() + # Add the missing keys to the first harvester's plot manager + env.harvesters[0].plot_manager.farmer_public_keys.append(G1Element()) + env.harvesters[0].plot_manager.pool_public_keys.append(G1Element()) + # And validate they become valid now + env.expected[0].add_valid(env.dir_keys_missing.plot_info_list()) + env.expected[0].remove_keys_missing(env.dir_keys_missing.path_list()) + await env.run_sync_test() + # Drop the valid plots from one harvester and the keys missing plots from the other harvester + env.remove_directory(0, env.dir_keys_missing) + env.remove_directory(1, env.dir_keys_missing, State.keys_missing) + await env.run_sync_test() + + +@pytest.mark.asyncio +async def test_sync_duplicates(environment: Environment) -> None: + env: Environment = environment + # dir_4 and then dir_duplicates contain the same plots. Load dir_4 first to make sure the plots seen as duplicates + # are from dir_duplicates. + env.add_directory(0, env.dir_4) + await env.run_sync_test() + env.add_directory(0, env.dir_duplicates, State.duplicates) + env.add_directory(1, env.dir_2) + await env.run_sync_test() + # Run again two times to make sure we still get the same results in repeated refresh intervals + await env.run_sync_test() + await env.run_sync_test() + # Drop all but 1 duplicates and test sync in-between + assert len(env.dir_duplicates) > 2 + for _ in range(len(env.dir_duplicates) - 2): + drop_plot = env.dir_duplicates.path_list()[0] + drop_plot.unlink() + env.dir_duplicates.drop(drop_plot) + env.expected[0].remove_duplicates([drop_plot]) + await env.run_sync_test() + assert len(env.dir_duplicates) == 2 + # Removing dir_4 now leads to the plots in dir_duplicates to become loaded instead + env.remove_directory(0, env.dir_4) + env.expected[0].remove_duplicates(env.dir_duplicates.path_list()) + env.expected[0].add_valid(env.dir_duplicates.plot_info_list()) + await env.run_sync_test() + + +async def add_and_validate_all_directories(env: Environment) -> None: + # Add all available directories to both harvesters and make sure they load and get synced + env.add_all_directories(0) + env.add_all_directories(1) + await env.run_sync_test() + env.add_directory(0, env.dir_duplicates, State.duplicates) + env.add_directory(1, env.dir_duplicates, State.duplicates) + await env.run_sync_test() + + +async def remove_and_validate_all_directories(env: Environment) -> None: + # Remove all available directories to both harvesters and make sure they are removed and get synced + env.remove_all_directories(0) + env.remove_all_directories(1) + await env.run_sync_test() + + +@pytest.mark.asyncio +async def test_add_and_remove_all_directories(environment: Environment) -> None: + await add_and_validate_all_directories(environment) + await remove_and_validate_all_directories(environment) + + +@pytest.mark.asyncio +async def test_harvester_restart(environment: Environment) -> None: + env: Environment = environment + # Load all directories for both harvesters + await add_and_validate_all_directories(env) + # Stop the harvester and make sure the receiver gets dropped on the farmer and refreshing gets stopped + env.harvester_services[0].stop() + await env.harvester_services[0].wait_closed() + assert len(env.farmer.plot_sync_receivers) == 1 + assert not env.harvesters[0].plot_manager._refreshing_enabled + assert not env.harvesters[0].plot_manager.needs_refresh() + # Start the harvester, wait for the handshake and make sure the receiver comes back + await env.harvester_services[0].start() + await time_out_assert(5, env.handshake_done, True, 0) + assert len(env.farmer.plot_sync_receivers) == 2 + # Remove the duplicates dir to avoid conflicts with the original plots + env.remove_directory(0, env.dir_duplicates) + # Reset the expected data for harvester 0 and re-add all directories because of the restart + env.expected[0] = ExpectedResult() + env.add_all_directories(0) + # Run the refresh two times and make sure everything recovers and stays recovered after harvester restart + await env.run_sync_test() + env.add_directory(0, env.dir_duplicates, State.duplicates) + await env.run_sync_test() + + +@pytest.mark.asyncio +async def test_farmer_restart(environment: Environment) -> None: + env: Environment = environment + # Load all directories for both harvesters + await add_and_validate_all_directories(env) + last_sync_ids: List[uint64] = [] + for i in range(0, len(env.harvesters)): + last_sync_ids.append(env.harvesters[i].plot_sync_sender._last_sync_id) + # Stop the farmer and make sure both receivers get dropped and refreshing gets stopped on the harvesters + env.farmer_service.stop() + await env.farmer_service.wait_closed() + assert len(env.farmer.plot_sync_receivers) == 0 + assert not env.harvesters[0].plot_manager._refreshing_enabled + assert not env.harvesters[1].plot_manager._refreshing_enabled + # Start the farmer, wait for the handshake and make sure the receivers come back + await env.farmer_service.start() + await time_out_assert(5, env.handshake_done, True, 0) + await time_out_assert(5, env.handshake_done, True, 1) + assert len(env.farmer.plot_sync_receivers) == 2 + # Do not use run_sync_test here, to have a more realistic test scenario just wait for the harvesters to be synced. + # The handshake should trigger re-sync. + for i in range(0, len(env.harvesters)): + harvester: Harvester = env.harvesters[i] + assert harvester.server is not None + receiver = env.farmer.plot_sync_receivers[harvester.server.node_id] + await time_out_assert(10, synced, True, harvester.plot_sync_sender, receiver, last_sync_ids[i]) + # Validate the sync + for harvester in env.harvesters: + plot_manager: PlotManager = harvester.plot_manager + assert harvester.server is not None + receiver = env.farmer.plot_sync_receivers[harvester.server.node_id] + expected = env.expected[env.harvesters.index(harvester)] + assert plot_manager.plot_count() == len(receiver.plots()) == expected.valid_count + assert len(plot_manager.failed_to_open_filenames) == len(receiver.invalid()) == expected.invalid_count + assert len(plot_manager.no_key_filenames) == len(receiver.keys_missing()) == expected.keys_missing_count + assert len(plot_manager.get_duplicates()) == len(receiver.duplicates()) == expected.duplicates_count diff --git a/tests/plot_sync/test_receiver.py b/tests/plot_sync/test_receiver.py new file mode 100644 index 0000000000..5c63ae4126 --- /dev/null +++ b/tests/plot_sync/test_receiver.py @@ -0,0 +1,376 @@ +import logging +import time +from secrets import token_bytes +from typing import Any, Callable, List, Tuple, Type, Union + +import pytest +from blspy import G1Element + +from chia.plot_sync.delta import Delta +from chia.plot_sync.receiver import Receiver +from chia.plot_sync.util import ErrorCodes, State +from chia.protocols.harvester_protocol import ( + Plot, + PlotSyncDone, + PlotSyncIdentifier, + PlotSyncPathList, + PlotSyncPlotList, + PlotSyncResponse, + PlotSyncStart, +) +from chia.server.ws_connection import NodeType +from chia.types.blockchain_format.sized_bytes import bytes32 +from chia.util.ints import uint8, uint32, uint64 +from chia.util.streamable import _T_Streamable +from tests.plot_sync.util import get_dummy_connection + +log = logging.getLogger(__name__) + +next_message_id = uint64(0) + + +def assert_default_values(receiver: Receiver) -> None: + assert receiver.state() == State.idle + assert receiver.expected_sync_id() == 0 + assert receiver.expected_message_id() == 0 + assert receiver.last_sync_id() == 0 + assert receiver.last_sync_time() == 0 + assert receiver.plots() == {} + assert receiver.invalid() == [] + assert receiver.keys_missing() == [] + assert receiver.duplicates() == [] + + +async def dummy_callback(_: bytes32, __: Delta) -> None: + pass + + +class SyncStepData: + state: State + function: Any + payload_type: Any + args: Any + + def __init__( + self, state: State, function: Callable[[_T_Streamable], Any], payload_type: Type[_T_Streamable], *args: Any + ) -> None: + self.state = state + self.function = function + self.payload_type = payload_type + self.args = args + + +def plot_sync_identifier(current_sync_id: uint64, message_id: uint64) -> PlotSyncIdentifier: + return PlotSyncIdentifier(uint64(0), current_sync_id, message_id) + + +def create_payload(payload_type: Any, start: bool, *args: Any) -> Any: + global next_message_id + if start: + next_message_id = uint64(0) + next_identifier = plot_sync_identifier(uint64(1), next_message_id) + next_message_id = uint64(next_message_id + 1) + return payload_type(next_identifier, *args) + + +def assert_error_response(plot_sync: Receiver, error_code: ErrorCodes) -> None: + connection = plot_sync.connection() + assert connection is not None + message = connection.last_sent_message + assert message is not None + response: PlotSyncResponse = PlotSyncResponse.from_bytes(message.data) + assert response.error is not None + assert response.error.code == error_code.value + + +def pre_function_validate(receiver: Receiver, data: Union[List[Plot], List[str]], expected_state: State) -> None: + if expected_state == State.loaded: + for plot_info in data: + assert type(plot_info) == Plot + assert plot_info.filename not in receiver.plots() + elif expected_state == State.removed: + for path in data: + assert path in receiver.plots() + elif expected_state == State.invalid: + for path in data: + assert path not in receiver.invalid() + elif expected_state == State.keys_missing: + for path in data: + assert path not in receiver.keys_missing() + elif expected_state == State.duplicates: + for path in data: + assert path not in receiver.duplicates() + + +def post_function_validate(receiver: Receiver, data: Union[List[Plot], List[str]], expected_state: State) -> None: + if expected_state == State.loaded: + for plot_info in data: + assert type(plot_info) == Plot + assert plot_info.filename in receiver._delta.valid.additions + elif expected_state == State.removed: + for path in data: + assert path in receiver._delta.valid.removals + elif expected_state == State.invalid: + for path in data: + assert path in receiver._delta.invalid.additions + elif expected_state == State.keys_missing: + for path in data: + assert path in receiver._delta.keys_missing.additions + elif expected_state == State.duplicates: + for path in data: + assert path in receiver._delta.duplicates.additions + + +@pytest.mark.asyncio +async def run_sync_step(receiver: Receiver, sync_step: SyncStepData, expected_state: State) -> None: + assert receiver.state() == expected_state + last_sync_time_before = receiver._last_sync_time + # For the the list types invoke the trigger function in batches + if sync_step.payload_type == PlotSyncPlotList or sync_step.payload_type == PlotSyncPathList: + step_data, _ = sync_step.args + assert len(step_data) == 10 + # Invoke batches of: 1, 2, 3, 4 items and validate the data against plot store before and after + indexes = [0, 1, 3, 6, 10] + for i in range(0, len(indexes) - 1): + invoke_data = step_data[indexes[i] : indexes[i + 1]] + pre_function_validate(receiver, invoke_data, expected_state) + await sync_step.function( + create_payload(sync_step.payload_type, False, invoke_data, i == (len(indexes) - 2)) + ) + post_function_validate(receiver, invoke_data, expected_state) + else: + # For Start/Done just invoke it.. + await sync_step.function(create_payload(sync_step.payload_type, sync_step.state == State.idle, *sync_step.args)) + # Make sure we moved to the next state + assert receiver.state() != expected_state + if sync_step.payload_type == PlotSyncDone: + assert receiver._last_sync_time != last_sync_time_before + else: + assert receiver._last_sync_time == last_sync_time_before + + +def plot_sync_setup() -> Tuple[Receiver, List[SyncStepData]]: + harvester_connection = get_dummy_connection(NodeType.HARVESTER) + receiver = Receiver(harvester_connection, dummy_callback) # type:ignore[arg-type] + + # Create example plot data + path_list = [str(x) for x in range(0, 40)] + plot_info_list = [ + Plot( + filename=str(x), + size=uint8(0), + plot_id=bytes32(token_bytes(32)), + pool_contract_puzzle_hash=None, + pool_public_key=None, + plot_public_key=G1Element(), + file_size=uint64(0), + time_modified=uint64(0), + ) + for x in path_list + ] + + # Manually add the plots we want to remove in tests + receiver._plots = {plot_info.filename: plot_info for plot_info in plot_info_list[0:10]} + + sync_steps: List[SyncStepData] = [ + SyncStepData(State.idle, receiver.sync_started, PlotSyncStart, False, uint64(0), uint32(len(plot_info_list))), + SyncStepData(State.loaded, receiver.process_loaded, PlotSyncPlotList, plot_info_list[10:20], True), + SyncStepData(State.removed, receiver.process_removed, PlotSyncPathList, path_list[0:10], True), + SyncStepData(State.invalid, receiver.process_invalid, PlotSyncPathList, path_list[20:30], True), + SyncStepData(State.keys_missing, receiver.process_keys_missing, PlotSyncPathList, path_list[30:40], True), + SyncStepData(State.duplicates, receiver.process_duplicates, PlotSyncPathList, path_list[10:20], True), + SyncStepData(State.done, receiver.sync_done, PlotSyncDone, uint64(0)), + ] + + return receiver, sync_steps + + +def test_default_values() -> None: + assert_default_values(Receiver(get_dummy_connection(NodeType.HARVESTER), dummy_callback)) # type:ignore[arg-type] + + +@pytest.mark.asyncio +async def test_reset() -> None: + receiver, sync_steps = plot_sync_setup() + connection_before = receiver.connection() + # Assign some dummy values + receiver._sync_state = State.done + receiver._expected_sync_id = uint64(1) + receiver._expected_message_id = uint64(1) + receiver._last_sync_id = uint64(1) + receiver._last_sync_time = time.time() + receiver._invalid = ["1"] + receiver._keys_missing = ["1"] + receiver._delta.valid.additions = receiver.plots().copy() + receiver._delta.valid.removals = ["1"] + receiver._delta.invalid.additions = ["1"] + receiver._delta.invalid.removals = ["1"] + receiver._delta.keys_missing.additions = ["1"] + receiver._delta.keys_missing.removals = ["1"] + receiver._delta.duplicates.additions = ["1"] + receiver._delta.duplicates.removals = ["1"] + # Call `reset` and make sure all expected values are set back to their defaults. + receiver.reset() + assert_default_values(receiver) + assert receiver._delta == Delta() + # Connection should remain + assert receiver.connection() == connection_before + + +@pytest.mark.asyncio +async def test_to_dict() -> None: + receiver, sync_steps = plot_sync_setup() + plot_sync_dict_1 = receiver.to_dict() + assert "plots" in plot_sync_dict_1 and len(plot_sync_dict_1["plots"]) == 10 + assert "failed_to_open_filenames" in plot_sync_dict_1 and len(plot_sync_dict_1["failed_to_open_filenames"]) == 0 + assert "no_key_filenames" in plot_sync_dict_1 and len(plot_sync_dict_1["no_key_filenames"]) == 0 + assert "last_sync_time" not in plot_sync_dict_1 + assert plot_sync_dict_1["connection"] == { + "node_id": receiver.connection().peer_node_id, + "host": receiver.connection().peer_host, + "port": receiver.connection().peer_port, + } + + # We should get equal dicts + plot_sync_dict_2 = receiver.to_dict() + assert plot_sync_dict_1 == plot_sync_dict_2 + + dict_2_paths = [x.filename for x in plot_sync_dict_2["plots"]] + for plot_info in sync_steps[State.loaded].args[0]: + assert plot_info.filename not in dict_2_paths + + # Walk through all states from idle to done and run them with the test data + for state in State: + await run_sync_step(receiver, sync_steps[state], state) + + plot_sync_dict_3 = receiver.to_dict() + dict_3_paths = [x.filename for x in plot_sync_dict_3["plots"]] + for plot_info in sync_steps[State.loaded].args[0]: + assert plot_info.filename in dict_3_paths + + for path in sync_steps[State.removed].args[0]: + assert path not in plot_sync_dict_3["plots"] + + for path in sync_steps[State.invalid].args[0]: + assert path in plot_sync_dict_3["failed_to_open_filenames"] + + for path in sync_steps[State.keys_missing].args[0]: + assert path in plot_sync_dict_3["no_key_filenames"] + + for path in sync_steps[State.duplicates].args[0]: + assert path in plot_sync_dict_3["duplicates"] + + assert plot_sync_dict_3["last_sync_time"] > 0 + + +@pytest.mark.asyncio +async def test_sync_flow() -> None: + receiver, sync_steps = plot_sync_setup() + + for plot_info in sync_steps[State.loaded].args[0]: + assert plot_info.filename not in receiver.plots() + + for path in sync_steps[State.removed].args[0]: + assert path in receiver.plots() + + for path in sync_steps[State.invalid].args[0]: + assert path not in receiver.invalid() + + for path in sync_steps[State.keys_missing].args[0]: + assert path not in receiver.keys_missing() + + for path in sync_steps[State.duplicates].args[0]: + assert path not in receiver.duplicates() + + # Walk through all states from idle to done and run them with the test data + for state in State: + await run_sync_step(receiver, sync_steps[state], state) + + for plot_info in sync_steps[State.loaded].args[0]: + assert plot_info.filename in receiver.plots() + + for path in sync_steps[State.removed].args[0]: + assert path not in receiver.plots() + + for path in sync_steps[State.invalid].args[0]: + assert path in receiver.invalid() + + for path in sync_steps[State.keys_missing].args[0]: + assert path in receiver.keys_missing() + + for path in sync_steps[State.duplicates].args[0]: + assert path in receiver.duplicates() + + # We should be in idle state again + assert receiver.state() == State.idle + + +@pytest.mark.asyncio +async def test_invalid_ids() -> None: + receiver, sync_steps = plot_sync_setup() + for state in State: + assert receiver.state() == state + current_step = sync_steps[state] + if receiver.state() == State.idle: + # Set last_sync_id for the tests below + receiver._last_sync_id = uint64(1) + # Test "sync_started last doesn't match" + invalid_last_sync_id_param = PlotSyncStart( + plot_sync_identifier(uint64(0), uint64(0)), False, uint64(2), uint32(0) + ) + await current_step.function(invalid_last_sync_id_param) + assert_error_response(receiver, ErrorCodes.invalid_last_sync_id) + # Test "last_sync_id == new_sync_id" + invalid_sync_id_match_param = PlotSyncStart( + plot_sync_identifier(uint64(1), uint64(0)), False, uint64(1), uint32(0) + ) + await current_step.function(invalid_sync_id_match_param) + assert_error_response(receiver, ErrorCodes.sync_ids_match) + # Reset the last_sync_id to the default + receiver._last_sync_id = uint64(0) + else: + # Test invalid sync_id + invalid_sync_id_param = current_step.payload_type( + plot_sync_identifier(uint64(10), uint64(receiver.expected_message_id())), *current_step.args + ) + await current_step.function(invalid_sync_id_param) + assert_error_response(receiver, ErrorCodes.invalid_identifier) + # Test invalid message_id + invalid_message_id_param = current_step.payload_type( + plot_sync_identifier(receiver.expected_sync_id(), uint64(receiver.expected_message_id() + 1)), + *current_step.args, + ) + await current_step.function(invalid_message_id_param) + assert_error_response(receiver, ErrorCodes.invalid_identifier) + payload = create_payload(current_step.payload_type, state == State.idle, *current_step.args) + await current_step.function(payload) + + +@pytest.mark.parametrize( + ["state_to_fail", "expected_error_code"], + [ + pytest.param(State.loaded, ErrorCodes.plot_already_available, id="already available plots"), + pytest.param(State.invalid, ErrorCodes.plot_already_available, id="already available paths"), + pytest.param(State.removed, ErrorCodes.plot_not_available, id="not available"), + ], +) +@pytest.mark.asyncio +async def test_plot_errors(state_to_fail: State, expected_error_code: ErrorCodes) -> None: + receiver, sync_steps = plot_sync_setup() + for state in State: + assert receiver.state() == state + current_step = sync_steps[state] + if state == state_to_fail: + plot_infos, _ = current_step.args + await current_step.function(create_payload(current_step.payload_type, False, plot_infos, False)) + identifier = plot_sync_identifier(receiver.expected_sync_id(), receiver.expected_message_id()) + invalid_payload = current_step.payload_type(identifier, plot_infos, True) + await current_step.function(invalid_payload) + if state == state_to_fail: + assert_error_response(receiver, expected_error_code) + return + else: + await current_step.function( + create_payload(current_step.payload_type, state == State.idle, *current_step.args) + ) + assert False, "Didn't fail in the expected state" diff --git a/tests/plot_sync/test_sender.py b/tests/plot_sync/test_sender.py new file mode 100644 index 0000000000..09747ec8bb --- /dev/null +++ b/tests/plot_sync/test_sender.py @@ -0,0 +1,102 @@ +import pytest + +from chia.plot_sync.exceptions import AlreadyStartedError, InvalidConnectionTypeError +from chia.plot_sync.sender import ExpectedResponse, Sender +from chia.plot_sync.util import Constants +from chia.protocols.harvester_protocol import PlotSyncIdentifier, PlotSyncResponse +from chia.server.ws_connection import NodeType, ProtocolMessageTypes +from chia.util.ints import int16, uint64 +from tests.block_tools import BlockTools +from tests.plot_sync.util import get_dummy_connection, plot_sync_identifier + + +def test_default_values(bt: BlockTools) -> None: + sender = Sender(bt.plot_manager) + assert sender._plot_manager == bt.plot_manager + assert sender._connection is None + assert sender._sync_id == uint64(0) + assert sender._next_message_id == uint64(0) + assert sender._messages == [] + assert sender._last_sync_id == uint64(0) + assert not sender._stop_requested + assert sender._task is None + assert not sender._lock.locked() + assert sender._response is None + + +def test_set_connection_values(bt: BlockTools) -> None: + farmer_connection = get_dummy_connection(NodeType.FARMER) + sender = Sender(bt.plot_manager) + # Test invalid NodeType values + for connection_type in NodeType: + if connection_type != NodeType.FARMER: + pytest.raises( + InvalidConnectionTypeError, + sender.set_connection, + get_dummy_connection(connection_type, farmer_connection.peer_node_id), + ) + # Test setting a valid connection works + sender.set_connection(farmer_connection) # type:ignore[arg-type] + assert sender._connection is not None + assert sender._connection == farmer_connection # type: ignore[comparison-overlap] + + +@pytest.mark.asyncio +async def test_start_stop_send_task(bt: BlockTools) -> None: + sender = Sender(bt.plot_manager) + # Make sure starting/restarting works + for _ in range(2): + assert sender._task is None + await sender.start() + assert sender._task is not None + with pytest.raises(AlreadyStartedError): + await sender.start() + assert not sender._stop_requested + sender.stop() + assert sender._stop_requested + await sender.await_closed() + assert not sender._stop_requested + assert sender._task is None + + +def test_set_response(bt: BlockTools) -> None: + sender = Sender(bt.plot_manager) + + def new_expected_response(sync_id: int, message_id: int, message_type: ProtocolMessageTypes) -> ExpectedResponse: + return ExpectedResponse(message_type, plot_sync_identifier(uint64(sync_id), uint64(message_id))) + + def new_response_message(sync_id: int, message_id: int, message_type: ProtocolMessageTypes) -> PlotSyncResponse: + return PlotSyncResponse( + plot_sync_identifier(uint64(sync_id), uint64(message_id)), int16(int(message_type.value)), None + ) + + response_message = new_response_message(0, 1, ProtocolMessageTypes.plot_sync_start) + assert sender._response is None + # Should trigger unexpected response because `Farmer._response` is `None` + assert not sender.set_response(response_message) + # Set `Farmer._response` and make sure the response gets assigned properly + sender._response = new_expected_response(0, 1, ProtocolMessageTypes.plot_sync_start) + assert sender._response.message is None + assert sender.set_response(response_message) + assert sender._response.message is not None + # Should trigger unexpected response because we already received the message for the currently expected response + assert not sender.set_response(response_message) + # Test expired message + expected_response = new_expected_response(1, 0, ProtocolMessageTypes.plot_sync_start) + sender._response = expected_response + expired_identifier = PlotSyncIdentifier( + uint64(expected_response.identifier.timestamp - Constants.message_timeout - 1), + expected_response.identifier.sync_id, + expected_response.identifier.message_id, + ) + expired_message = PlotSyncResponse(expired_identifier, int16(int(ProtocolMessageTypes.plot_sync_start.value)), None) + assert not sender.set_response(expired_message) + # Test invalid sync-id + sender._response = new_expected_response(2, 0, ProtocolMessageTypes.plot_sync_start) + assert not sender.set_response(new_response_message(3, 0, ProtocolMessageTypes.plot_sync_start)) + # Test invalid message-id + sender._response = new_expected_response(2, 1, ProtocolMessageTypes.plot_sync_start) + assert not sender.set_response(new_response_message(2, 2, ProtocolMessageTypes.plot_sync_start)) + # Test invalid message-type + sender._response = new_expected_response(3, 0, ProtocolMessageTypes.plot_sync_start) + assert not sender.set_response(new_response_message(3, 0, ProtocolMessageTypes.plot_sync_loaded)) diff --git a/tests/plot_sync/test_sync_simulated.py b/tests/plot_sync/test_sync_simulated.py new file mode 100644 index 0000000000..ae83dc7b64 --- /dev/null +++ b/tests/plot_sync/test_sync_simulated.py @@ -0,0 +1,433 @@ +import asyncio +import functools +import logging +import time +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from secrets import token_bytes +from typing import Any, Dict, List, Optional, Set, Tuple + +import pytest +from blspy import G1Element + +from chia.farmer.farmer_api import Farmer +from chia.harvester.harvester_api import Harvester +from chia.plot_sync.receiver import Receiver +from chia.plot_sync.sender import Sender +from chia.plot_sync.util import Constants +from chia.plotting.manager import PlotManager +from chia.plotting.util import PlotInfo +from chia.protocols.harvester_protocol import PlotSyncError, PlotSyncResponse +from chia.server.start_service import Service +from chia.server.ws_connection import ProtocolMessageTypes, WSChiaConnection, make_msg +from chia.types.blockchain_format.sized_bytes import bytes32 +from chia.util.generator_tools import list_to_batches +from chia.util.ints import int16, uint64 +from tests.plot_sync.util import start_harvester_service +from tests.time_out_assert import time_out_assert + +log = logging.getLogger(__name__) + + +class ErrorSimulation(Enum): + DropEveryFourthMessage = 1 + DropThreeMessages = 2 + RespondTooLateEveryFourthMessage = 3 + RespondTwice = 4 + NonRecoverableError = 5 + NotConnected = 6 + + +@dataclass +class TestData: + harvester: Harvester + plot_sync_sender: Sender + plot_sync_receiver: Receiver + event_loop: asyncio.AbstractEventLoop + plots: Dict[Path, PlotInfo] = field(default_factory=dict) + invalid: List[PlotInfo] = field(default_factory=list) + keys_missing: List[PlotInfo] = field(default_factory=list) + duplicates: List[PlotInfo] = field(default_factory=list) + + async def run( + self, + *, + loaded: List[PlotInfo], + removed: List[PlotInfo], + invalid: List[PlotInfo], + keys_missing: List[PlotInfo], + duplicates: List[PlotInfo], + initial: bool, + ) -> None: + for plot_info in loaded: + assert plot_info.prover.get_filename() not in self.plots + for plot_info in removed: + assert plot_info.prover.get_filename() in self.plots + + self.invalid = invalid + self.keys_missing = keys_missing + self.duplicates = duplicates + + removed_paths: List[Path] = [p.prover.get_filename() for p in removed] if removed is not None else [] + invalid_dict: Dict[Path, int] = {p.prover.get_filename(): 0 for p in self.invalid} + keys_missing_set: Set[Path] = set([p.prover.get_filename() for p in self.keys_missing]) + duplicates_set: Set[str] = set([p.prover.get_filename() for p in self.duplicates]) + + # Inject invalid plots into `PlotManager` of the harvester so that the callback calls below can use them + # to sync them to the farmer. + self.harvester.plot_manager.failed_to_open_filenames = invalid_dict + # Inject key missing plots into `PlotManager` of the harvester so that the callback calls below can use them + # to sync them to the farmer. + self.harvester.plot_manager.no_key_filenames = keys_missing_set + # Inject duplicated plots into `PlotManager` of the harvester so that the callback calls below can use them + # to sync them to the farmer. + for plot_info in loaded: + plot_path = Path(plot_info.prover.get_filename()) + self.harvester.plot_manager.plot_filename_paths[plot_path.name] = (str(plot_path.parent), set()) + for duplicate in duplicates_set: + plot_path = Path(duplicate) + assert plot_path.name in self.harvester.plot_manager.plot_filename_paths + self.harvester.plot_manager.plot_filename_paths[plot_path.name][1].add(str(plot_path.parent)) + + batch_size = self.harvester.plot_manager.refresh_parameter.batch_size + + # Used to capture the sync id in `run_internal` + sync_id: Optional[uint64] = None + + def run_internal() -> None: + nonlocal sync_id + # Simulate one plot manager refresh cycle by calling the methods directly. + self.harvester.plot_sync_sender.sync_start(len(loaded), initial) + sync_id = self.plot_sync_sender._sync_id + if len(loaded) == 0: + self.harvester.plot_sync_sender.process_batch([], 0) + for remaining, batch in list_to_batches(loaded, batch_size): + self.harvester.plot_sync_sender.process_batch(batch, remaining) + self.harvester.plot_sync_sender.sync_done(removed_paths, 0) + + await self.event_loop.run_in_executor(None, run_internal) + + async def sync_done() -> bool: + assert sync_id is not None + return self.plot_sync_receiver.last_sync_id() == self.plot_sync_sender._last_sync_id == sync_id + + await time_out_assert(60, sync_done) + + for plot_info in loaded: + self.plots[plot_info.prover.get_filename()] = plot_info + for plot_info in removed: + del self.plots[plot_info.prover.get_filename()] + + def validate_plot_sync(self) -> None: + assert len(self.plots) == len(self.plot_sync_receiver.plots()) + assert len(self.invalid) == len(self.plot_sync_receiver.invalid()) + assert len(self.keys_missing) == len(self.plot_sync_receiver.keys_missing()) + for _, plot_info in self.plots.items(): + assert plot_info.prover.get_filename() not in self.plot_sync_receiver.invalid() + assert plot_info.prover.get_filename() not in self.plot_sync_receiver.keys_missing() + assert plot_info.prover.get_filename() in self.plot_sync_receiver.plots() + synced_plot = self.plot_sync_receiver.plots()[plot_info.prover.get_filename()] + assert plot_info.prover.get_filename() == synced_plot.filename + assert plot_info.pool_public_key == synced_plot.pool_public_key + assert plot_info.pool_contract_puzzle_hash == synced_plot.pool_contract_puzzle_hash + assert plot_info.plot_public_key == synced_plot.plot_public_key + assert plot_info.file_size == synced_plot.file_size + assert uint64(int(plot_info.time_modified)) == synced_plot.time_modified + for plot_info in self.invalid: + assert plot_info.prover.get_filename() not in self.plot_sync_receiver.plots() + assert plot_info.prover.get_filename() in self.plot_sync_receiver.invalid() + assert plot_info.prover.get_filename() not in self.plot_sync_receiver.keys_missing() + assert plot_info.prover.get_filename() not in self.plot_sync_receiver.duplicates() + for plot_info in self.keys_missing: + assert plot_info.prover.get_filename() not in self.plot_sync_receiver.plots() + assert plot_info.prover.get_filename() not in self.plot_sync_receiver.invalid() + assert plot_info.prover.get_filename() in self.plot_sync_receiver.keys_missing() + assert plot_info.prover.get_filename() not in self.plot_sync_receiver.duplicates() + for plot_info in self.duplicates: + assert plot_info.prover.get_filename() not in self.plot_sync_receiver.invalid() + assert plot_info.prover.get_filename() not in self.plot_sync_receiver.keys_missing() + assert plot_info.prover.get_filename() in self.plot_sync_receiver.duplicates() + + +@dataclass +class TestRunner: + test_data: List[TestData] + + def __init__( + self, harvesters: List[Harvester], farmer: Farmer, event_loop: asyncio.events.AbstractEventLoop + ) -> None: + self.test_data = [] + for harvester in harvesters: + assert harvester.server is not None + self.test_data.append( + TestData( + harvester, + harvester.plot_sync_sender, + farmer.plot_sync_receivers[harvester.server.node_id], + event_loop, + ) + ) + + async def run( + self, + index: int, + *, + loaded: List[PlotInfo], + removed: List[PlotInfo], + invalid: List[PlotInfo], + keys_missing: List[PlotInfo], + duplicates: List[PlotInfo], + initial: bool, + ) -> None: + await self.test_data[index].run( + loaded=loaded, + removed=removed, + invalid=invalid, + keys_missing=keys_missing, + duplicates=duplicates, + initial=initial, + ) + for data in self.test_data: + data.validate_plot_sync() + + +async def skip_processing(self: Any, _: WSChiaConnection, message_type: ProtocolMessageTypes, message: Any) -> bool: + self.message_counter += 1 + if self.simulate_error == ErrorSimulation.DropEveryFourthMessage: + if self.message_counter % 4 == 0: + return True + if self.simulate_error == ErrorSimulation.DropThreeMessages: + if 2 < self.message_counter < 6: + return True + if self.simulate_error == ErrorSimulation.RespondTooLateEveryFourthMessage: + if self.message_counter % 4 == 0: + await asyncio.sleep(Constants.message_timeout + 1) + return False + if self.simulate_error == ErrorSimulation.RespondTwice: + await self.connection().send_message( + make_msg( + ProtocolMessageTypes.plot_sync_response, + PlotSyncResponse(message.identifier, int16(message_type.value), None), + ) + ) + if self.simulate_error == ErrorSimulation.NonRecoverableError and self.message_counter > 1: + await self.connection().send_message( + make_msg( + ProtocolMessageTypes.plot_sync_response, + PlotSyncResponse( + message.identifier, int16(message_type.value), PlotSyncError(int16(0), "non recoverable", None) + ), + ) + ) + self.simulate_error = 0 + return True + return False + + +async def _testable_process( + self: Any, peer: WSChiaConnection, message_type: ProtocolMessageTypes, message: Any +) -> None: + if await skip_processing(self, peer, message_type, message): + return + await self.original_process(peer, message_type, message) + + +async def create_test_runner( + harvester_services: List[Service], farmer: Farmer, event_loop: asyncio.events.AbstractEventLoop +) -> TestRunner: + assert len(farmer.plot_sync_receivers) == 0 + harvesters: List[Harvester] = [await start_harvester_service(service) for service in harvester_services] + for receiver in farmer.plot_sync_receivers.values(): + receiver.simulate_error = 0 # type: ignore[attr-defined] + receiver.message_counter = 0 # type: ignore[attr-defined] + receiver.original_process = receiver._process # type: ignore[attr-defined] + receiver._process = functools.partial(_testable_process, receiver) # type: ignore[assignment] + return TestRunner(harvesters, farmer, event_loop) + + +def create_example_plots(count: int) -> List[PlotInfo]: + @dataclass + class DiskProver: + file_name: str + plot_id: bytes32 + size: int + + def get_filename(self) -> str: + return self.file_name + + def get_id(self) -> bytes32: + return self.plot_id + + def get_size(self) -> int: + return self.size + + return [ + PlotInfo( + prover=DiskProver(f"{x}", bytes32(token_bytes(32)), x % 255), + pool_public_key=None, + pool_contract_puzzle_hash=None, + plot_public_key=G1Element(), + file_size=uint64(0), + time_modified=time.time(), + ) + for x in range(0, count) + ] + + +@pytest.mark.asyncio +async def test_sync_simulated( + farmer_three_harvester: Tuple[List[Service], Service], event_loop: asyncio.events.AbstractEventLoop +) -> None: + harvester_services: List[Service] + farmer_service: Service + harvester_services, farmer_service = farmer_three_harvester + farmer: Farmer = farmer_service._node + test_runner: TestRunner = await create_test_runner(harvester_services, farmer, event_loop) + plots = create_example_plots(31000) + + await test_runner.run( + 0, loaded=plots[0:10000], removed=[], invalid=[], keys_missing=[], duplicates=plots[0:1000], initial=True + ) + await test_runner.run( + 1, + loaded=plots[10000:20000], + removed=[], + invalid=plots[30000:30100], + keys_missing=[], + duplicates=[], + initial=True, + ) + await test_runner.run( + 2, + loaded=plots[20000:30000], + removed=[], + invalid=[], + keys_missing=plots[30100:30200], + duplicates=[], + initial=True, + ) + await test_runner.run( + 0, + loaded=[], + removed=[], + invalid=plots[30300:30400], + keys_missing=plots[30400:30453], + duplicates=[], + initial=False, + ) + await test_runner.run(0, loaded=[], removed=[], invalid=[], keys_missing=[], duplicates=[], initial=False) + await test_runner.run( + 0, loaded=[], removed=plots[5000:10000], invalid=[], keys_missing=[], duplicates=[], initial=False + ) + await test_runner.run( + 1, loaded=[], removed=plots[10000:20000], invalid=[], keys_missing=[], duplicates=[], initial=False + ) + await test_runner.run( + 2, loaded=[], removed=plots[20000:29000], invalid=[], keys_missing=[], duplicates=[], initial=False + ) + await test_runner.run( + 0, loaded=[], removed=plots[0:5000], invalid=[], keys_missing=[], duplicates=[], initial=False + ) + await test_runner.run( + 2, + loaded=plots[5000:10000], + removed=plots[29000:30000], + invalid=plots[30000:30500], + keys_missing=plots[30500:31000], + duplicates=plots[5000:6000], + initial=False, + ) + await test_runner.run( + 2, loaded=[], removed=plots[5000:10000], invalid=[], keys_missing=[], duplicates=[], initial=False + ) + assert len(farmer.plot_sync_receivers) == 3 + for plot_sync in farmer.plot_sync_receivers.values(): + assert len(plot_sync.plots()) == 0 + + +@pytest.mark.parametrize( + "simulate_error", + [ + ErrorSimulation.DropEveryFourthMessage, + ErrorSimulation.DropThreeMessages, + ErrorSimulation.RespondTooLateEveryFourthMessage, + ErrorSimulation.RespondTwice, + ], +) +@pytest.mark.asyncio +async def test_farmer_error_simulation( + farmer_one_harvester: Tuple[List[Service], Service], + event_loop: asyncio.events.AbstractEventLoop, + simulate_error: ErrorSimulation, +) -> None: + Constants.message_timeout = 5 + harvester_services: List[Service] + farmer_service: Service + harvester_services, farmer_service = farmer_one_harvester + test_runner: TestRunner = await create_test_runner(harvester_services, farmer_service._node, event_loop) + batch_size = test_runner.test_data[0].harvester.plot_manager.refresh_parameter.batch_size + plots = create_example_plots(batch_size + 3) + receiver = test_runner.test_data[0].plot_sync_receiver + receiver.simulate_error = simulate_error # type: ignore[attr-defined] + await test_runner.run( + 0, + loaded=plots[0 : batch_size + 1], + removed=[], + invalid=[plots[batch_size + 1]], + keys_missing=[plots[batch_size + 2]], + duplicates=[], + initial=True, + ) + + +@pytest.mark.parametrize("simulate_error", [ErrorSimulation.NonRecoverableError, ErrorSimulation.NotConnected]) +@pytest.mark.asyncio +async def test_sync_reset_cases( + farmer_one_harvester: Tuple[List[Service], Service], + event_loop: asyncio.events.AbstractEventLoop, + simulate_error: ErrorSimulation, +) -> None: + harvester_services: List[Service] + farmer_service: Service + harvester_services, farmer_service = farmer_one_harvester + test_runner: TestRunner = await create_test_runner(harvester_services, farmer_service._node, event_loop) + test_data: TestData = test_runner.test_data[0] + plot_manager: PlotManager = test_data.harvester.plot_manager + plots = create_example_plots(30) + # Inject some data into `PlotManager` of the harvester so that we can validate the reset worked and triggered a + # fresh sync of all available data of the plot manager + for plot_info in plots[0:10]: + test_data.plots[plot_info.prover.get_filename()] = plot_info + plot_manager.plots = test_data.plots + test_data.invalid = plots[10:20] + test_data.keys_missing = plots[20:30] + test_data.plot_sync_receiver.simulate_error = simulate_error # type: ignore[attr-defined] + sender: Sender = test_runner.test_data[0].plot_sync_sender + started_sync_id: uint64 = uint64(0) + + plot_manager.failed_to_open_filenames = {p.prover.get_filename(): 0 for p in test_data.invalid} + plot_manager.no_key_filenames = set([p.prover.get_filename() for p in test_data.keys_missing]) + + async def wait_for_reset() -> bool: + assert started_sync_id != 0 + return sender._sync_id != started_sync_id != 0 + + async def sync_done() -> bool: + assert started_sync_id != 0 + return test_data.plot_sync_receiver.last_sync_id() == sender._last_sync_id == started_sync_id + + # Send start and capture the sync_id + sender.sync_start(len(plots), True) + started_sync_id = sender._sync_id + # Sleep 2 seconds to make sure we have a different sync_id after the reset which gets triggered + await asyncio.sleep(2) + saved_connection = sender._connection + if simulate_error == ErrorSimulation.NotConnected: + sender._connection = None + sender.process_batch(plots, 0) + await time_out_assert(60, wait_for_reset) + started_sync_id = sender._sync_id + sender._connection = saved_connection + await time_out_assert(60, sync_done) + test_runner.test_data[0].validate_plot_sync() diff --git a/tests/plot_sync/util.py b/tests/plot_sync/util.py new file mode 100644 index 0000000000..823616f50b --- /dev/null +++ b/tests/plot_sync/util.py @@ -0,0 +1,53 @@ +import time +from dataclasses import dataclass +from secrets import token_bytes +from typing import Optional + +from chia.harvester.harvester_api import Harvester +from chia.plot_sync.sender import Sender +from chia.protocols.harvester_protocol import PlotSyncIdentifier +from chia.server.start_service import Service +from chia.server.ws_connection import Message, NodeType +from chia.types.blockchain_format.sized_bytes import bytes32 +from chia.util.ints import uint64 +from tests.time_out_assert import time_out_assert + + +@dataclass +class WSChiaConnectionDummy: + connection_type: NodeType + peer_node_id: bytes32 + peer_host: str = "localhost" + peer_port: int = 0 + last_sent_message: Optional[Message] = None + + async def send_message(self, message: Message) -> None: + self.last_sent_message = message + + +def get_dummy_connection(node_type: NodeType, peer_id: Optional[bytes32] = None) -> WSChiaConnectionDummy: + return WSChiaConnectionDummy(node_type, bytes32(token_bytes(32)) if peer_id is None else peer_id) + + +def plot_sync_identifier(current_sync_id: uint64, message_id: uint64) -> PlotSyncIdentifier: + return PlotSyncIdentifier(uint64(int(time.time())), current_sync_id, message_id) + + +async def start_harvester_service(harvester_service: Service) -> Harvester: + # Set the `last_refresh_time` of the plot manager to avoid initial plot loading + harvester: Harvester = harvester_service._node + harvester.plot_manager.last_refresh_time = time.time() + await harvester_service.start() + harvester.plot_manager.stop_refreshing() # type: ignore[no-untyped-call] # TODO, Add typing in PlotManager + + assert harvester.plot_sync_sender._sync_id == 0 + assert harvester.plot_sync_sender._next_message_id == 0 + assert harvester.plot_sync_sender._last_sync_id == 0 + assert harvester.plot_sync_sender._messages == [] + + def wait_for_farmer_connection(plot_sync_sender: Sender) -> bool: + return plot_sync_sender._connection is not None + + await time_out_assert(10, wait_for_farmer_connection, True, harvester.plot_sync_sender) + + return harvester diff --git a/tests/plotting/test_plot_manager.py b/tests/plotting/test_plot_manager.py index f2a0e843c2..fea50cc4e1 100644 --- a/tests/plotting/test_plot_manager.py +++ b/tests/plotting/test_plot_manager.py @@ -236,7 +236,7 @@ async def test_plot_refreshing(test_plot_environment): trigger=trigger_remove_plot, test_path=drop_path, expect_loaded=[], - expect_removed=[drop_path], + expect_removed=[], expect_processed=len(env.dir_1) + len(env.dir_2) + len(dir_duplicates), expect_duplicates=len(dir_duplicates), expected_directories=3, @@ -262,7 +262,7 @@ async def test_plot_refreshing(test_plot_environment): trigger=remove_plot_directory, test_path=dir_duplicates.path, expect_loaded=[], - expect_removed=dir_duplicates.path_list(), + expect_removed=[], expect_processed=len(env.dir_1) + len(env.dir_2), expect_duplicates=0, expected_directories=2, @@ -316,7 +316,7 @@ async def test_plot_refreshing(test_plot_environment): trigger=trigger_remove_plot, test_path=drop_path, expect_loaded=[], - expect_removed=[drop_path], + expect_removed=[], expect_processed=len(env.dir_1) + len(env.dir_2) + len(dir_duplicates), expect_duplicates=len(env.dir_1), expected_directories=3, @@ -357,6 +357,17 @@ async def test_plot_refreshing(test_plot_environment): ) +@pytest.mark.asyncio +async def test_initial_refresh_flag(test_plot_environment: TestEnvironment) -> None: + env: TestEnvironment = test_plot_environment + assert env.refresh_tester.plot_manager.initial_refresh() + for _ in range(2): + await env.refresh_tester.run(PlotRefreshResult()) + assert not env.refresh_tester.plot_manager.initial_refresh() + env.refresh_tester.plot_manager.reset() + assert env.refresh_tester.plot_manager.initial_refresh() + + @pytest.mark.asyncio async def test_invalid_plots(test_plot_environment): env: TestEnvironment = test_plot_environment diff --git a/tests/setup_nodes.py b/tests/setup_nodes.py index 4cfb701d9c..03a97c386d 100644 --- a/tests/setup_nodes.py +++ b/tests/setup_nodes.py @@ -1,12 +1,15 @@ import asyncio import logging from secrets import token_bytes -from typing import Dict, List +from typing import AsyncIterator, Dict, List, Tuple +from pathlib import Path from chia.consensus.constants import ConsensusConstants +from chia.cmds.init_funcs import init from chia.full_node.full_node_api import FullNodeAPI from chia.server.start_service import Service from chia.server.start_wallet import service_kwargs_for_wallet +from chia.util.config import load_config, save_config from chia.util.hash import std_hash from chia.util.ints import uint16, uint32 from chia.util.keychain import bytes_to_mnemonic @@ -294,7 +297,7 @@ async def setup_harvester_farmer(bt: BlockTools, consensus_constants: ConsensusC harvester_rpc_port = find_available_listen_port("harvester rpc") node_iters = [ setup_harvester( - bt, + bt.root_path, bt.config["self_hostname"], harvester_port, harvester_rpc_port, @@ -320,6 +323,62 @@ async def setup_harvester_farmer(bt: BlockTools, consensus_constants: ConsensusC await _teardown_nodes(node_iters) +async def setup_farmer_multi_harvester( + block_tools: BlockTools, + harvester_count: int, + temp_dir: Path, + consensus_constants: ConsensusConstants, +) -> AsyncIterator[Tuple[List[Service], Service]]: + farmer_port = find_available_listen_port("farmer") + farmer_rpc_port = find_available_listen_port("farmer rpc") + + node_iterators = [ + setup_farmer( + block_tools, block_tools.config["self_hostname"], farmer_port, farmer_rpc_port, consensus_constants + ) + ] + + for i in range(0, harvester_count): + root_path: Path = temp_dir / str(i) + init(None, root_path) + init(block_tools.root_path / "config" / "ssl" / "ca", root_path) + config = load_config(root_path, "config.yaml") + config["logging"]["log_stdout"] = True + config["selected_network"] = "testnet0" + config["harvester"]["selected_network"] = "testnet0" + harvester_port = find_available_listen_port("harvester") + harvester_rpc_port = find_available_listen_port("harvester rpc") + save_config(root_path, "config.yaml", config) + node_iterators.append( + setup_harvester( + root_path, + block_tools.config["self_hostname"], + harvester_port, + harvester_rpc_port, + farmer_port, + consensus_constants, + False, + ) + ) + + farmer_service = await node_iterators[0].__anext__() + harvester_services = [] + for node in node_iterators[1:]: + harvester_service = await node.__anext__() + harvester_services.append(harvester_service) + + yield harvester_services, farmer_service + + for harvester_service in harvester_services: + harvester_service.stop() + await harvester_service.wait_closed() + + farmer_service.stop() + await farmer_service.wait_closed() + + await _teardown_nodes(node_iterators) + + async def setup_full_system( consensus_constants: ConsensusConstants, shared_b_tools: BlockTools, @@ -353,7 +412,7 @@ async def setup_full_system( node_iters = [ setup_introducer(shared_b_tools, introducer_port), setup_harvester( - shared_b_tools, + shared_b_tools.root_path, shared_b_tools.config["self_hostname"], harvester_port, harvester_rpc_port, diff --git a/tests/setup_services.py b/tests/setup_services.py index 848506fe7f..27b6b42410 100644 --- a/tests/setup_services.py +++ b/tests/setup_services.py @@ -2,6 +2,7 @@ import asyncio import logging import signal import sqlite3 +from pathlib import Path from secrets import token_bytes from typing import AsyncGenerator, Optional @@ -16,8 +17,8 @@ from chia.server.start_timelord import service_kwargs_for_timelord from chia.server.start_wallet import service_kwargs_for_wallet from chia.simulator.start_simulator import service_kwargs_for_full_node_simulator from chia.timelord.timelord_launcher import kill_processes, spawn_process -from chia.types.peer_info import PeerInfo from chia.util.bech32m import encode_puzzle_hash +from chia.util.config import load_config, save_config from chia.util.ints import uint16 from chia.util.keychain import bytes_to_mnemonic from tests.block_tools import BlockTools @@ -184,7 +185,7 @@ async def setup_wallet_node( async def setup_harvester( - b_tools: BlockTools, + root_path: Path, self_hostname: str, port, rpc_port, @@ -192,15 +193,14 @@ async def setup_harvester( consensus_constants: ConsensusConstants, start_service: bool = True, ): - - config = b_tools.config["harvester"] - config["port"] = port - config["rpc_port"] = rpc_port - kwargs = service_kwargs_for_harvester(b_tools.root_path, config, consensus_constants) + config = load_config(root_path, "config.yaml") + config["harvester"]["port"] = port + config["harvester"]["rpc_port"] = rpc_port + config["harvester"]["farmer_peer"]["host"] = self_hostname + config["harvester"]["farmer_peer"]["port"] = farmer_port + save_config(root_path, "config.yaml", config) + kwargs = service_kwargs_for_harvester(root_path, config["harvester"], consensus_constants) kwargs.update( - server_listen_ports=[port], - advertised_port=port, - connect_peers=[PeerInfo(self_hostname, farmer_port)], parse_cli_args=False, connect_to_daemon=False, service_name_prefix="test_", From c7a89fc425f42f610a206160239279d96b935602 Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Thu, 7 Apr 2022 19:43:14 -0500 Subject: [PATCH 50/63] Add wallentx as additional assignee on mozilla CA update PRs (#11089) --- .github/workflows/mozilla-ca-cert.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/mozilla-ca-cert.yml b/.github/workflows/mozilla-ca-cert.yml index a637f430e1..c616df40f2 100644 --- a/.github/workflows/mozilla-ca-cert.yml +++ b/.github/workflows/mozilla-ca-cert.yml @@ -28,5 +28,6 @@ jobs: commit-message: "adding ca updates" delete-branch: true reviewers: "wjblanke,emlowe" + assignees: "wallentx" title: "CA Cert updates" token: "${{ secrets.GITHUB_TOKEN }}" From 02a880fbac6f0c304b84b6f485d10af2251f7dbc Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 7 Apr 2022 23:43:27 -0400 Subject: [PATCH 51/63] rebuild workflows (#11092) --- .../workflows/build-test-macos-plot_sync.yml | 24 ++++++++++++++----- .../workflows/build-test-ubuntu-plot_sync.yml | 24 ++++++++++++++----- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-test-macos-plot_sync.yml b/.github/workflows/build-test-macos-plot_sync.yml index d18e76cf7d..606b7d71a7 100644 --- a/.github/workflows/build-test-macos-plot_sync.yml +++ b/.github/workflows/build-test-macos-plot_sync.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-plot_sync.yml b/.github/workflows/build-test-ubuntu-plot_sync.yml index 886573b574..54fa8b19e6 100644 --- a/.github/workflows/build-test-ubuntu-plot_sync.yml +++ b/.github/workflows/build-test-ubuntu-plot_sync.yml @@ -65,13 +65,25 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Checkout test blocks and plots - uses: actions/checkout@v3 + - name: Daily (POC) cache key invalidation for test blocks and plots + id: today-date + run: date +%F > today.txt + + - name: Cache test blocks and plots + uses: actions/cache@v2 + id: test-blocks-plots with: - repository: 'Chia-Network/test-cache' - path: '.chia' - ref: '0.28.0' - fetch-depth: 1 + path: | + ${{ github.workspace }}/.chia/blocks + ${{ github.workspace }}/.chia/test-plots + key: ${{ hashFiles('today.txt') }} + + - name: Checkout test blocks and plots + if: steps.test-blocks-plots.outputs.cache-hit != 'true' + run: | + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + mkdir ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia - name: Run install script env: From 21fb6f260e59763295eb1eda747a17dc48744e2d Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Fri, 8 Apr 2022 18:37:10 +0200 Subject: [PATCH 52/63] transition to using chia_rs module (#11094) --- chia/full_node/mempool_check_conditions.py | 6 +++--- chia/types/blockchain_format/program.py | 6 +++--- chia/types/spend_bundle_conditions.py | 2 +- chia/util/full_block_utils.py | 2 +- setup.py | 2 +- tests/wallet/test_singleton.py | 4 ++-- tools/analyze-chain.py | 6 +++--- tools/run_block.py | 2 +- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/chia/full_node/mempool_check_conditions.py b/chia/full_node/mempool_check_conditions.py index 10c44cd1e3..1457a29f45 100644 --- a/chia/full_node/mempool_check_conditions.py +++ b/chia/full_node/mempool_check_conditions.py @@ -1,7 +1,6 @@ import logging - from typing import Dict, Optional -from clvm_rs import MEMPOOL_MODE, COND_CANON_INTS, NO_NEG_DIV +from chia_rs import MEMPOOL_MODE, COND_CANON_INTS, NO_NEG_DIV, STRICT_ARGS_COUNT from chia.consensus.default_constants import DEFAULT_CONSTANTS from chia.consensus.cost_calculator import NPCResult @@ -43,7 +42,8 @@ def get_name_puzzle_conditions( assert (MEMPOOL_MODE & NO_NEG_DIV) != 0 if mempool_mode: - flags = MEMPOOL_MODE + # Don't apply the strict args count rule yet + flags = MEMPOOL_MODE & (~STRICT_ARGS_COUNT) elif unwrap(height) >= DEFAULT_CONSTANTS.SOFT_FORK_HEIGHT: # conditions must use integers in canonical encoding (i.e. no redundant # leading zeros) diff --git a/chia/types/blockchain_format/program.py b/chia/types/blockchain_format/program.py index 36278ef7ec..096f72d25b 100644 --- a/chia/types/blockchain_format/program.py +++ b/chia/types/blockchain_format/program.py @@ -5,7 +5,7 @@ from clvm import SExp from clvm.casts import int_from_bytes from clvm.EvalError import EvalError from clvm.serialize import sexp_from_stream, sexp_to_stream -from clvm_rs import MEMPOOL_MODE, run_chia_program, serialized_length, run_generator2 +from chia_rs import MEMPOOL_MODE, run_chia_program, serialized_length, run_generator from clvm_tools.curry import curry, uncurry from chia.types.blockchain_format.sized_bytes import bytes32 @@ -222,7 +222,7 @@ class SerializedProgram: def run_with_cost(self, max_cost: int, *args) -> Tuple[int, Program]: return self._run(max_cost, 0, *args) - # returns an optional error code and an optional SpendBundleConditions + # returns an optional error code and an optional SpendBundleConditions (from chia_rs) # exactly one of those will hold a value def run_as_generator( self, max_cost: int, flags: int, *args @@ -238,7 +238,7 @@ class SerializedProgram: else: serialized_args += _serialize(args[0]) - err, conds = run_generator2( + err, conds = run_generator( self._buf, serialized_args, max_cost, diff --git a/chia/types/spend_bundle_conditions.py b/chia/types/spend_bundle_conditions.py index 0c59fb732e..3bae9b34d2 100644 --- a/chia/types/spend_bundle_conditions.py +++ b/chia/types/spend_bundle_conditions.py @@ -7,7 +7,7 @@ from chia.util.streamable import Streamable, streamable # the Spend and SpendBundleConditions classes are mirrors of native types, returned by -# run_generator2 +# run_generator @dataclass(frozen=True) @streamable class Spend(Streamable): diff --git a/chia/util/full_block_utils.py b/chia/util/full_block_utils.py index bc41e03f8f..a1f076b9ce 100644 --- a/chia/util/full_block_utils.py +++ b/chia/util/full_block_utils.py @@ -1,7 +1,7 @@ from typing import Callable, Optional from blspy import G1Element, G2Element -from clvm_rs import serialized_length +from chia_rs import serialized_length from chia.types.blockchain_format.program import SerializedProgram diff --git a/setup.py b/setup.py index 9c6e98238d..2ac24927f4 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ dependencies = [ "chiapos==1.0.9", # proof of space "clvm==0.9.7", "clvm_tools==0.4.4", # Currying, Program.to, other conveniences - "clvm_rs==0.1.19", + "chia_rs==0.1.1", "clvm-tools-rs==0.1.7", # Rust implementation of clvm_tools "aiohttp==3.7.4", # HTTP server for full node rpc "aiosqlite==0.17.0", # asyncio wrapper for sqlite, to store blocks diff --git a/tests/wallet/test_singleton.py b/tests/wallet/test_singleton.py index bf3eba6f71..8f806e71c7 100644 --- a/tests/wallet/test_singleton.py +++ b/tests/wallet/test_singleton.py @@ -51,7 +51,7 @@ def test_only_odd_coins(): try: cost, result = SINGLETON_MOD.run_with_cost(INFINITE_COST, solution) except Exception as e: - assert e.args == ("clvm raise",) + assert e.args == ("clvm raise", "80") else: assert False @@ -84,7 +84,7 @@ def test_only_one_odd_coin_created(): try: cost, result = SINGLETON_MOD.run_with_cost(INFINITE_COST, solution) except Exception as e: - assert e.args == ("clvm raise",) + assert e.args == ("clvm raise", "80") else: assert False solution = Program.to( diff --git a/tools/analyze-chain.py b/tools/analyze-chain.py index 6d8484d51e..ccf2de8287 100755 --- a/tools/analyze-chain.py +++ b/tools/analyze-chain.py @@ -10,7 +10,7 @@ from typing import List from time import time -from clvm_rs import run_generator2, MEMPOOL_MODE +from chia_rs import run_generator, MEMPOOL_MODE from chia.types.full_block import FullBlock from chia.types.blockchain_format.program import Program @@ -21,7 +21,7 @@ from chia.util.ints import uint32 GENERATOR_ROM = bytes(get_generator()) -# returns an optional error code and an optional PySpendBundleConditions (from clvm_rs) +# returns an optional error code and an optional PySpendBundleConditions (from chia_rs) # exactly one of those will hold a value and the number of seconds it took to # run def run_gen(env_data: bytes, block_program_args: bytes, flags: uint32): @@ -36,7 +36,7 @@ def run_gen(env_data: bytes, block_program_args: bytes, flags: uint32): try: start_time = time() - err, result = run_generator2( + err, result = run_generator( GENERATOR_ROM, env_data, max_cost, diff --git a/tools/run_block.py b/tools/run_block.py index c8fe416adf..6588b26e2c 100644 --- a/tools/run_block.py +++ b/tools/run_block.py @@ -42,7 +42,7 @@ from typing import List, Tuple, Dict import click -from clvm_rs import COND_CANON_INTS, NO_NEG_DIV +from chia_rs import COND_CANON_INTS, NO_NEG_DIV from chia.consensus.constants import ConsensusConstants from chia.consensus.default_constants import DEFAULT_CONSTANTS From 7cd23b007738999a33d68086288c4ed0b5020654 Mon Sep 17 00:00:00 2001 From: Adam Kelly <338792+aqk@users.noreply.github.com> Date: Fri, 8 Apr 2022 09:57:08 -0700 Subject: [PATCH 53/63] Fix the case of claiming a large number of coins (#11038) * Fix the case of claiming a large number of coins with a fee from a pool wallet * Revert change to unrelated test * Set PoolWallet.DEFAULT_MAX_CLAIM_SPENDS to 300 * A few review improvements --- chia/pools/pool_wallet.py | 31 ++++++-- chia/rpc/wallet_rpc_api.py | 3 +- chia/rpc/wallet_rpc_client.py | 8 +- chia/wallet/wallet_state_manager.py | 4 +- tests/pools/test_pool_rpc.py | 109 +++++++++++++++++++++++++--- 5 files changed, 133 insertions(+), 22 deletions(-) diff --git a/chia/pools/pool_wallet.py b/chia/pools/pool_wallet.py index 341cb758d9..b41a2585cb 100644 --- a/chia/pools/pool_wallet.py +++ b/chia/pools/pool_wallet.py @@ -61,6 +61,7 @@ class PoolWallet: MINIMUM_INITIAL_BALANCE = 1 MINIMUM_RELATIVE_LOCK_HEIGHT = 5 MAXIMUM_RELATIVE_LOCK_HEIGHT = 1000 + DEFAULT_MAX_CLAIM_SPENDS = 100 wallet_state_manager: Any log: logging.Logger @@ -326,6 +327,7 @@ class PoolWallet: block_spends: List[CoinSpend], block_height: uint32, in_transaction: bool, + *, name: str = None, ): """ @@ -778,13 +780,21 @@ class PoolWallet: travel_tx, fee_tx = await self.generate_travel_transactions(fee) return total_fee, travel_tx, fee_tx - async def claim_pool_rewards(self, fee: uint64) -> Tuple[TransactionRecord, Optional[TransactionRecord]]: + async def claim_pool_rewards( + self, fee: uint64, max_spends_in_tx: Optional[int] + ) -> Tuple[TransactionRecord, Optional[TransactionRecord]]: # Search for p2_puzzle_hash coins, and spend them with the singleton if await self.have_unconfirmed_transaction(): raise ValueError( "Cannot claim due to unconfirmed transaction. If this is stuck, delete the unconfirmed transaction." ) + if max_spends_in_tx is None: + max_spends_in_tx = self.DEFAULT_MAX_CLAIM_SPENDS + elif max_spends_in_tx <= 0: + self.log.info(f"Bad max_spends_in_tx value of {max_spends_in_tx}. Set to {self.DEFAULT_MAX_CLAIM_SPENDS}.") + max_spends_in_tx = self.DEFAULT_MAX_CLAIM_SPENDS + unspent_coin_records: List[CoinRecord] = list( await self.wallet_state_manager.coin_store.get_unspent_coins_for_wallet(self.wallet_id) ) @@ -807,13 +817,20 @@ class PoolWallet: all_spends: List[CoinSpend] = [] total_amount = 0 - current_coin_record = None + # The coins being claimed are gathered into the `SpendBundle`, :absorb_spend: + # We use an announcement in the fee spend to ensure that the claim spend is spent in the same block as the fee + # We only need to do this for one of the coins, because each `SpendBundle` can only be spent as a unit + + first_coin_record = None for coin_record in unspent_coin_records: if coin_record.coin not in coin_to_height_farmed: continue - current_coin_record = coin_record - if len(all_spends) >= 100: - # Limit the total number of spends, so it fits into the block + if first_coin_record is None: + first_coin_record = coin_record + if len(all_spends) >= max_spends_in_tx: + # Limit the total number of spends, so the SpendBundle fits into the block + self.log.info(f"pool wallet truncating absorb to {max_spends_in_tx} spends to fit into block") + print(f"pool wallet truncating absorb to {max_spends_in_tx} spends to fit into block") break absorb_spend: List[CoinSpend] = create_absorb_spend( last_solution, @@ -830,7 +847,7 @@ class PoolWallet: self.log.info( f"Farmer coin: {coin_record.coin} {coin_record.coin.name()} {coin_to_height_farmed[coin_record.coin]}" ) - if len(all_spends) == 0 or current_coin_record is None: + if len(all_spends) == 0 or first_coin_record is None: raise ValueError("Nothing to claim, no unspent coinbase rewards") claim_spend: SpendBundle = SpendBundle(all_spends, G2Element()) @@ -840,7 +857,7 @@ class PoolWallet: fee_tx = None if fee > 0: - absorb_announce = Announcement(current_coin_record.coin.name(), b"$") + absorb_announce = Announcement(first_coin_record.coin.name(), b"$") fee_tx = await self.generate_fee_transaction(fee, coin_announcements=[absorb_announce]) full_spend = SpendBundle.aggregate([fee_tx.spend_bundle, claim_spend]) diff --git a/chia/rpc/wallet_rpc_api.py b/chia/rpc/wallet_rpc_api.py index 3f43af8095..d4e7820411 100644 --- a/chia/rpc/wallet_rpc_api.py +++ b/chia/rpc/wallet_rpc_api.py @@ -1408,13 +1408,14 @@ class WalletRpcApi: if await self.service.wallet_state_manager.synced() is False: raise ValueError("Wallet needs to be fully synced before collecting rewards") fee = uint64(request.get("fee", 0)) + max_spends_in_tx = request.get("max_spends_in_tx", None) wallet_id = uint32(request["wallet_id"]) wallet: PoolWallet = self.service.wallet_state_manager.wallets[wallet_id] if wallet.type() != uint8(WalletType.POOLING_WALLET): raise ValueError(f"Wallet with wallet id: {wallet_id} is not a plotNFT wallet.") async with self.service.wallet_state_manager.lock: - transaction, fee_tx = await wallet.claim_pool_rewards(fee) + transaction, fee_tx = await wallet.claim_pool_rewards(fee, max_spends_in_tx) state: PoolWalletInfo = await wallet.get_current_state() return {"state": state.to_json_dict(), "transaction": transaction, "fee_transaction": fee_tx} diff --git a/chia/rpc/wallet_rpc_client.py b/chia/rpc/wallet_rpc_client.py index 82058176ba..135fb9f37e 100644 --- a/chia/rpc/wallet_rpc_client.py +++ b/chia/rpc/wallet_rpc_client.py @@ -327,8 +327,12 @@ class WalletRpcClient(RpcClient): reply = parse_result_transactions(reply) return reply - async def pw_absorb_rewards(self, wallet_id: str, fee: uint64 = uint64(0)) -> Dict: - reply = await self.fetch("pw_absorb_rewards", {"wallet_id": wallet_id, "fee": fee}) + async def pw_absorb_rewards( + self, wallet_id: str, fee: uint64 = uint64(0), max_spends_in_tx: Optional[int] = None + ) -> Dict: + reply = await self.fetch( + "pw_absorb_rewards", {"wallet_id": wallet_id, "fee": fee, "max_spends_in_tx": max_spends_in_tx} + ) reply["state"] = PoolWalletInfo.from_json_dict(reply["state"]) reply = parse_result_transactions(reply) return reply diff --git a/chia/wallet/wallet_state_manager.py b/chia/wallet/wallet_state_manager.py index e7044beec9..01282e3293 100644 --- a/chia/wallet/wallet_state_manager.py +++ b/chia/wallet/wallet_state_manager.py @@ -860,8 +860,8 @@ class WalletStateManager: child.coin.name(), [launcher_spend], child.spent_height, - True, - "pool_wallet", + in_transaction=True, + name="pool_wallet", ) launcher_spend_additions = launcher_spend.additions() assert len(launcher_spend_additions) == 1 diff --git a/tests/pools/test_pool_rpc.py b/tests/pools/test_pool_rpc.py index 504431f5e9..de66f42df2 100644 --- a/tests/pools/test_pool_rpc.py +++ b/tests/pools/test_pool_rpc.py @@ -4,13 +4,14 @@ import tempfile from dataclasses import dataclass from pathlib import Path from shutil import rmtree -from typing import Any, Optional, List, Dict +from typing import Any, Optional, List, Dict, Tuple, AsyncGenerator import pytest import pytest_asyncio from blspy import G1Element from chia.consensus.block_rewards import calculate_base_farmer_reward, calculate_pool_reward +from chia.full_node.full_node_api import FullNodeAPI from chia.pools.pool_puzzles import SINGLETON_LAUNCHER_HASH from chia.pools.pool_wallet_info import PoolWalletInfo, PoolSingletonState from chia.protocols import full_node_protocol @@ -38,6 +39,7 @@ from tests.util.socket import find_available_listen_port # TODO: Compare deducted fees in all tests against reported total_fee log = logging.getLogger(__name__) FEE_AMOUNT = 2000000000000 +MAX_WAIT_SECS = 20 # A high value for WAIT_SECS is useful when paused in the debugger def get_pool_plot_dir(): @@ -90,7 +92,7 @@ PREFARMED_BLOCKS = 4 @pytest_asyncio.fixture(scope="function") -async def one_wallet_node_and_rpc(bt, self_hostname): +async def one_wallet_node_and_rpc(bt, self_hostname) -> AsyncGenerator[Tuple[WalletRpcClient, Any, FullNodeAPI], None]: rmtree(get_pool_plot_dir(), ignore_errors=True) async for nodes in setup_simulators_and_wallets(1, 1, {}): full_nodes, wallets = nodes @@ -104,7 +106,7 @@ async def one_wallet_node_and_rpc(bt, self_hostname): api_user = WalletRpcApi(wallet_node_0) config = bt.config daemon_port = config["daemon_port"] - test_rpc_port = find_available_listen_port("rpc_port") + test_rpc_port = uint16(find_available_listen_port("rpc_port")) rpc_cleanup = await start_rpc_server( api_user, @@ -510,7 +512,7 @@ class TestPoolWalletRpc: absorb_tx1.spend_bundle.debug() await time_out_assert( - 10, + MAX_WAIT_SECS, full_node_api.full_node.mempool_manager.get_spendbundle, absorb_tx1.spend_bundle, absorb_tx1.name, @@ -545,6 +547,95 @@ class TestPoolWalletRpc: assert (250000000000 + fee) in [tx.additions[0].amount for tx in tx1] # await time_out_assert(10, wallet_0.get_confirmed_balance, total_block_rewards) + @pytest.mark.asyncio + @pytest.mark.parametrize("trusted_and_fee", [(True, FEE_AMOUNT * 2)]) + async def test_absorb_self_multiple_coins(self, one_wallet_node_and_rpc, trusted_and_fee, bt, self_hostname): + trusted, fee = trusted_and_fee + client, wallet_node_0, full_node_api = one_wallet_node_and_rpc + if trusted: + wallet_node_0.config["trusted_peers"] = { + full_node_api.full_node.server.node_id.hex(): full_node_api.full_node.server.node_id.hex() + } + else: + wallet_node_0.config["trusted_peers"] = {} + + await wallet_node_0.server.start_client( + PeerInfo(self_hostname, uint16(full_node_api.full_node.server._port)), None + ) + wallet_0 = wallet_node_0.wallet_state_manager.main_wallet + total_block_rewards = await get_total_block_rewards(PREFARMED_BLOCKS) + await time_out_assert(10, wallet_0.get_confirmed_balance, total_block_rewards) + await time_out_assert(10, wallet_node_0.wallet_state_manager.blockchain.get_peak_height, PREFARMED_BLOCKS) + + our_ph = await wallet_0.get_new_puzzlehash() + assert len(await client.get_wallets(WalletType.POOLING_WALLET)) == 0 + + await time_out_assert(10, wallet_is_synced, True, wallet_node_0, full_node_api) + creation_tx: TransactionRecord = await client.create_new_pool_wallet( + our_ph, "", 0, f"{self_hostname}:5000", "new", "SELF_POOLING", fee + ) + + await time_out_assert( + 10, + full_node_api.full_node.mempool_manager.get_spendbundle, + creation_tx.spend_bundle, + creation_tx.name, + ) + await farm_blocks(full_node_api, our_ph, 1) + # await asyncio.sleep(5) + + async def pool_wallet_created(): + try: + status: PoolWalletInfo = (await client.pw_status(2))[0] + return status.current.state == PoolSingletonState.SELF_POOLING.value + except ValueError: + return False + + await time_out_assert(10, pool_wallet_created) + + status: PoolWalletInfo = (await client.pw_status(2))[0] + async with TemporaryPoolPlot(bt, status.p2_singleton_puzzle_hash) as pool_plot: + all_blocks = await full_node_api.get_all_full_blocks() + blocks = bt.get_consecutive_blocks( + 3, + block_list_input=all_blocks, + force_plot_id=pool_plot.plot_id, + farmer_reward_puzzle_hash=our_ph, + guarantee_transaction_block=True, + ) + + await full_node_api.full_node.respond_block(full_node_protocol.RespondBlock(blocks[-3])) + await full_node_api.full_node.respond_block(full_node_protocol.RespondBlock(blocks[-2])) + await full_node_api.full_node.respond_block(full_node_protocol.RespondBlock(blocks[-1])) + await asyncio.sleep(2) + + bal = await client.get_wallet_balance(2) + assert bal["confirmed_wallet_balance"] == 2 * 1750000000000 + + await farm_blocks(full_node_api, our_ph, 6) + await asyncio.sleep(6) + + # Claim + absorb_tx: TransactionRecord = (await client.pw_absorb_rewards(2, fee, 1))["transaction"] + await time_out_assert( + 5, + full_node_api.full_node.mempool_manager.get_spendbundle, + absorb_tx.spend_bundle, + absorb_tx.name, + ) + await farm_blocks(full_node_api, our_ph, 2) + await asyncio.sleep(2) + new_status: PoolWalletInfo = (await client.pw_status(2))[0] + assert status.current == new_status.current + assert status.tip_singleton_coin_id != new_status.tip_singleton_coin_id + main_bal = await client.get_wallet_balance(1) + pool_bal = await client.get_wallet_balance(2) + assert pool_bal["confirmed_wallet_balance"] == 2 * 1750000000000 + assert main_bal["confirmed_wallet_balance"] == 26499999999999 + print(pool_bal) + print("---") + print(main_bal) + @pytest.mark.asyncio @pytest.mark.parametrize("trusted_and_fee", [(True, FEE_AMOUNT), (False, 0)]) async def test_absorb_pooling(self, one_wallet_node_and_rpc, trusted_and_fee, bt, self_hostname): @@ -843,8 +934,6 @@ class TestPoolWalletRpc: PeerInfo(self_hostname, uint16(full_node_api.full_node.server._port)), None ) - WAIT_SECS = 200 - try: assert len(await client.get_wallets(WalletType.POOLING_WALLET)) == 0 @@ -852,7 +941,7 @@ class TestPoolWalletRpc: await farm_blocks(full_node_api, our_ph, 1) return (await wallets[0].get_confirmed_balance()) > 0 - await time_out_assert(timeout=WAIT_SECS, function=have_chia) + await time_out_assert(timeout=MAX_WAIT_SECS, function=have_chia) await time_out_assert(10, wallet_is_synced, True, wallet_nodes[0], full_node_api) creation_tx: TransactionRecord = await client.create_new_pool_wallet( @@ -909,7 +998,7 @@ class TestPoolWalletRpc: pw_status: PoolWalletInfo = (await client.pw_status(wallet_id))[0] return pw_status.current.state == PoolSingletonState.FARMING_TO_POOL.value - await time_out_assert(timeout=WAIT_SECS, function=status_is_farming_to_pool) + await time_out_assert(timeout=MAX_WAIT_SECS, function=status_is_farming_to_pool) await time_out_assert(10, wallet_is_synced, True, wallet_nodes[0], full_node_api) @@ -924,7 +1013,7 @@ class TestPoolWalletRpc: pw_status: PoolWalletInfo = (await client.pw_status(wallet_id))[0] return pw_status.current.state == PoolSingletonState.LEAVING_POOL.value - await time_out_assert(timeout=WAIT_SECS, function=status_is_leaving) + await time_out_assert(timeout=MAX_WAIT_SECS, function=status_is_leaving) async def status_is_self_pooling(): # Farm enough blocks to wait for relative_lock_height @@ -932,7 +1021,7 @@ class TestPoolWalletRpc: pw_status: PoolWalletInfo = (await client.pw_status(wallet_id))[0] return pw_status.current.state == PoolSingletonState.SELF_POOLING.value - await time_out_assert(timeout=WAIT_SECS, function=status_is_self_pooling) + await time_out_assert(timeout=MAX_WAIT_SECS, function=status_is_self_pooling) assert len(await wallets[0].wallet_state_manager.tx_store.get_unconfirmed_for_wallet(2)) == 0 finally: From 2f9e718073d88758dcd7a5d9d15f97469045fdf5 Mon Sep 17 00:00:00 2001 From: Mariano Sorgente <3069354+mariano54@users.noreply.github.com> Date: Fri, 8 Apr 2022 12:57:59 -0400 Subject: [PATCH 54/63] Ms.fast test blockchain (#11051) * more work on test blockchain * Optimize test_blockchain.py * Fix weight proof bug * Rename variable * first rc_sub_slot hash bug * New plots * try with a new ID * Run without cache * Address test blocks and plots preparation. * Update constant in test_compact_protocol(). * Update this constant too. * Revert accidental altering of the gui submodule in ae7e3295f280a591e76c4dffdea75fb74ea5de6f. * Fix benchmark test * Revert mozilla-ca change * Rebase on main Co-authored-by: almog Co-authored-by: Amine Khaldi --- .github/workflows/benchmarks.yml | 2 +- .../workflows/build-test-macos-blockchain.yml | 10 +-- .../build-test-macos-core-daemon.yml | 10 +-- ...ld-test-macos-core-full_node-full_sync.yml | 10 +-- ...build-test-macos-core-full_node-stores.yml | 10 +-- .../build-test-macos-core-full_node.yml | 10 +-- .../build-test-macos-core-server.yml | 10 +-- .../workflows/build-test-macos-core-ssl.yml | 10 +-- .../workflows/build-test-macos-core-util.yml | 10 +-- .github/workflows/build-test-macos-core.yml | 10 +-- .../build-test-macos-farmer_harvester.yml | 10 +-- .../workflows/build-test-macos-plot_sync.yml | 10 +-- .../workflows/build-test-macos-plotting.yml | 10 +-- .github/workflows/build-test-macos-pools.yml | 10 +-- .../workflows/build-test-macos-simulation.yml | 10 +-- .../build-test-macos-wallet-cat_wallet.yml | 10 +-- .../workflows/build-test-macos-wallet-rpc.yml | 10 +-- .../build-test-macos-wallet-simple_sync.yml | 10 +-- .../build-test-macos-wallet-sync.yml | 10 +-- .github/workflows/build-test-macos-wallet.yml | 10 +-- .../build-test-macos-weight_proof.yml | 10 +-- .../build-test-ubuntu-blockchain.yml | 10 +-- .../build-test-ubuntu-core-daemon.yml | 10 +-- ...d-test-ubuntu-core-full_node-full_sync.yml | 10 +-- ...uild-test-ubuntu-core-full_node-stores.yml | 10 +-- .../build-test-ubuntu-core-full_node.yml | 10 +-- .../build-test-ubuntu-core-server.yml | 10 +-- .../workflows/build-test-ubuntu-core-ssl.yml | 10 +-- .../workflows/build-test-ubuntu-core-util.yml | 10 +-- .github/workflows/build-test-ubuntu-core.yml | 10 +-- .../build-test-ubuntu-farmer_harvester.yml | 10 +-- .../workflows/build-test-ubuntu-plot_sync.yml | 10 +-- .../workflows/build-test-ubuntu-plotting.yml | 10 +-- .github/workflows/build-test-ubuntu-pools.yml | 10 +-- .../build-test-ubuntu-simulation.yml | 10 +-- .../build-test-ubuntu-wallet-cat_wallet.yml | 10 +-- .../build-test-ubuntu-wallet-rpc.yml | 10 +-- .../build-test-ubuntu-wallet-simple_sync.yml | 10 +-- .../build-test-ubuntu-wallet-sync.yml | 10 +-- .../workflows/build-test-ubuntu-wallet.yml | 10 +-- .../build-test-ubuntu-weight_proof.yml | 10 +-- chia/full_node/weight_proof.py | 6 +- chia/plotting/create_plots.py | 6 +- tests/block_tools.py | 4 +- tests/blockchain/test_blockchain.py | 72 +++++++++++++------ tests/conftest.py | 56 +++++++++++++-- .../full_node/full_sync/test_full_sync.py | 6 +- tests/core/full_node/test_full_node.py | 4 +- tests/core/full_node/test_performance.py | 7 +- .../checkout-test-plots.include.yml | 10 +-- tests/util/blockchain.py | 14 +++- 51 files changed, 255 insertions(+), 332 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index b7589f89c4..ac025730c1 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -60,7 +60,7 @@ jobs: with: repository: 'Chia-Network/test-cache' path: '.chia' - ref: '0.28.0' + ref: '0.29.0' fetch-depth: 1 - name: Run install script diff --git a/.github/workflows/build-test-macos-blockchain.yml b/.github/workflows/build-test-macos-blockchain.yml index 8fcb0b1c10..a816459ff7 100644 --- a/.github/workflows/build-test-macos-blockchain.yml +++ b/.github/workflows/build-test-macos-blockchain.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-core-daemon.yml b/.github/workflows/build-test-macos-core-daemon.yml index d017da0e3e..950f08481d 100644 --- a/.github/workflows/build-test-macos-core-daemon.yml +++ b/.github/workflows/build-test-macos-core-daemon.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-core-full_node-full_sync.yml b/.github/workflows/build-test-macos-core-full_node-full_sync.yml index 61825197ea..fd8e5da8d2 100644 --- a/.github/workflows/build-test-macos-core-full_node-full_sync.yml +++ b/.github/workflows/build-test-macos-core-full_node-full_sync.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-core-full_node-stores.yml b/.github/workflows/build-test-macos-core-full_node-stores.yml index b25f81ea34..72a06b6ca0 100644 --- a/.github/workflows/build-test-macos-core-full_node-stores.yml +++ b/.github/workflows/build-test-macos-core-full_node-stores.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-core-full_node.yml b/.github/workflows/build-test-macos-core-full_node.yml index 03b9cb834e..83d6cb0156 100644 --- a/.github/workflows/build-test-macos-core-full_node.yml +++ b/.github/workflows/build-test-macos-core-full_node.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-core-server.yml b/.github/workflows/build-test-macos-core-server.yml index 03a771e6b6..e93f23b2f8 100644 --- a/.github/workflows/build-test-macos-core-server.yml +++ b/.github/workflows/build-test-macos-core-server.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-core-ssl.yml b/.github/workflows/build-test-macos-core-ssl.yml index 51c348f8fe..db9903634a 100644 --- a/.github/workflows/build-test-macos-core-ssl.yml +++ b/.github/workflows/build-test-macos-core-ssl.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-core-util.yml b/.github/workflows/build-test-macos-core-util.yml index 95c8603deb..0935d2d546 100644 --- a/.github/workflows/build-test-macos-core-util.yml +++ b/.github/workflows/build-test-macos-core-util.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-core.yml b/.github/workflows/build-test-macos-core.yml index df94b8d09d..4fe7cab249 100644 --- a/.github/workflows/build-test-macos-core.yml +++ b/.github/workflows/build-test-macos-core.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-farmer_harvester.yml b/.github/workflows/build-test-macos-farmer_harvester.yml index 3017a35790..9c9f2b4736 100644 --- a/.github/workflows/build-test-macos-farmer_harvester.yml +++ b/.github/workflows/build-test-macos-farmer_harvester.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-plot_sync.yml b/.github/workflows/build-test-macos-plot_sync.yml index 606b7d71a7..315c909bb4 100644 --- a/.github/workflows/build-test-macos-plot_sync.yml +++ b/.github/workflows/build-test-macos-plot_sync.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-plotting.yml b/.github/workflows/build-test-macos-plotting.yml index 80a5c9ceba..30d44782fd 100644 --- a/.github/workflows/build-test-macos-plotting.yml +++ b/.github/workflows/build-test-macos-plotting.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-pools.yml b/.github/workflows/build-test-macos-pools.yml index 39969bddaf..9279455daa 100644 --- a/.github/workflows/build-test-macos-pools.yml +++ b/.github/workflows/build-test-macos-pools.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-simulation.yml b/.github/workflows/build-test-macos-simulation.yml index 44a920ac31..a7bd5c6695 100644 --- a/.github/workflows/build-test-macos-simulation.yml +++ b/.github/workflows/build-test-macos-simulation.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-wallet-cat_wallet.yml b/.github/workflows/build-test-macos-wallet-cat_wallet.yml index d38bb5a5a5..70858e93b7 100644 --- a/.github/workflows/build-test-macos-wallet-cat_wallet.yml +++ b/.github/workflows/build-test-macos-wallet-cat_wallet.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-wallet-rpc.yml b/.github/workflows/build-test-macos-wallet-rpc.yml index 674e246e6e..68a95ea7a7 100644 --- a/.github/workflows/build-test-macos-wallet-rpc.yml +++ b/.github/workflows/build-test-macos-wallet-rpc.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-wallet-simple_sync.yml b/.github/workflows/build-test-macos-wallet-simple_sync.yml index c80bd4d72b..0a1a359b36 100644 --- a/.github/workflows/build-test-macos-wallet-simple_sync.yml +++ b/.github/workflows/build-test-macos-wallet-simple_sync.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-wallet-sync.yml b/.github/workflows/build-test-macos-wallet-sync.yml index bedeaa6016..c92ca8e21e 100644 --- a/.github/workflows/build-test-macos-wallet-sync.yml +++ b/.github/workflows/build-test-macos-wallet-sync.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-wallet.yml b/.github/workflows/build-test-macos-wallet.yml index 100585f786..0a71248808 100644 --- a/.github/workflows/build-test-macos-wallet.yml +++ b/.github/workflows/build-test-macos-wallet.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-macos-weight_proof.yml b/.github/workflows/build-test-macos-weight_proof.yml index 0870d1f954..9b737c31e4 100644 --- a/.github/workflows/build-test-macos-weight_proof.yml +++ b/.github/workflows/build-test-macos-weight_proof.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-blockchain.yml b/.github/workflows/build-test-ubuntu-blockchain.yml index e6a5aa8723..60dbb45169 100644 --- a/.github/workflows/build-test-ubuntu-blockchain.yml +++ b/.github/workflows/build-test-ubuntu-blockchain.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-core-daemon.yml b/.github/workflows/build-test-ubuntu-core-daemon.yml index 19f60ca1e7..3611af06d2 100644 --- a/.github/workflows/build-test-ubuntu-core-daemon.yml +++ b/.github/workflows/build-test-ubuntu-core-daemon.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-core-full_node-full_sync.yml b/.github/workflows/build-test-ubuntu-core-full_node-full_sync.yml index 546c3d21bd..a2295de7eb 100644 --- a/.github/workflows/build-test-ubuntu-core-full_node-full_sync.yml +++ b/.github/workflows/build-test-ubuntu-core-full_node-full_sync.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-core-full_node-stores.yml b/.github/workflows/build-test-ubuntu-core-full_node-stores.yml index be9cef49ad..fbcf347037 100644 --- a/.github/workflows/build-test-ubuntu-core-full_node-stores.yml +++ b/.github/workflows/build-test-ubuntu-core-full_node-stores.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-core-full_node.yml b/.github/workflows/build-test-ubuntu-core-full_node.yml index e59db07ded..97208996a1 100644 --- a/.github/workflows/build-test-ubuntu-core-full_node.yml +++ b/.github/workflows/build-test-ubuntu-core-full_node.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-core-server.yml b/.github/workflows/build-test-ubuntu-core-server.yml index a770ae9a40..c96f2e13f1 100644 --- a/.github/workflows/build-test-ubuntu-core-server.yml +++ b/.github/workflows/build-test-ubuntu-core-server.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-core-ssl.yml b/.github/workflows/build-test-ubuntu-core-ssl.yml index 050996b890..0edc22b104 100644 --- a/.github/workflows/build-test-ubuntu-core-ssl.yml +++ b/.github/workflows/build-test-ubuntu-core-ssl.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-core-util.yml b/.github/workflows/build-test-ubuntu-core-util.yml index 68b23bd9c9..2e5c699cb9 100644 --- a/.github/workflows/build-test-ubuntu-core-util.yml +++ b/.github/workflows/build-test-ubuntu-core-util.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-core.yml b/.github/workflows/build-test-ubuntu-core.yml index bd32713578..ed717b529b 100644 --- a/.github/workflows/build-test-ubuntu-core.yml +++ b/.github/workflows/build-test-ubuntu-core.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-farmer_harvester.yml b/.github/workflows/build-test-ubuntu-farmer_harvester.yml index 127ead45c0..03a3050232 100644 --- a/.github/workflows/build-test-ubuntu-farmer_harvester.yml +++ b/.github/workflows/build-test-ubuntu-farmer_harvester.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-plot_sync.yml b/.github/workflows/build-test-ubuntu-plot_sync.yml index 54fa8b19e6..fddf0cd9c2 100644 --- a/.github/workflows/build-test-ubuntu-plot_sync.yml +++ b/.github/workflows/build-test-ubuntu-plot_sync.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-plotting.yml b/.github/workflows/build-test-ubuntu-plotting.yml index 099f50f6d7..882d0b0c11 100644 --- a/.github/workflows/build-test-ubuntu-plotting.yml +++ b/.github/workflows/build-test-ubuntu-plotting.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-pools.yml b/.github/workflows/build-test-ubuntu-pools.yml index a4284b00e1..d2671e3a18 100644 --- a/.github/workflows/build-test-ubuntu-pools.yml +++ b/.github/workflows/build-test-ubuntu-pools.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-simulation.yml b/.github/workflows/build-test-ubuntu-simulation.yml index 805f646b14..d6fbec1fc1 100644 --- a/.github/workflows/build-test-ubuntu-simulation.yml +++ b/.github/workflows/build-test-ubuntu-simulation.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-wallet-cat_wallet.yml b/.github/workflows/build-test-ubuntu-wallet-cat_wallet.yml index 25835db6d9..af979fc0fd 100644 --- a/.github/workflows/build-test-ubuntu-wallet-cat_wallet.yml +++ b/.github/workflows/build-test-ubuntu-wallet-cat_wallet.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-wallet-rpc.yml b/.github/workflows/build-test-ubuntu-wallet-rpc.yml index 4ceb70b32b..bdeb794df4 100644 --- a/.github/workflows/build-test-ubuntu-wallet-rpc.yml +++ b/.github/workflows/build-test-ubuntu-wallet-rpc.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-wallet-simple_sync.yml b/.github/workflows/build-test-ubuntu-wallet-simple_sync.yml index 899501734b..7a9585c713 100644 --- a/.github/workflows/build-test-ubuntu-wallet-simple_sync.yml +++ b/.github/workflows/build-test-ubuntu-wallet-simple_sync.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-wallet-sync.yml b/.github/workflows/build-test-ubuntu-wallet-sync.yml index 5da886be88..ceb86d8b42 100644 --- a/.github/workflows/build-test-ubuntu-wallet-sync.yml +++ b/.github/workflows/build-test-ubuntu-wallet-sync.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-wallet.yml b/.github/workflows/build-test-ubuntu-wallet.yml index a6f734c9ae..0e88ee66d5 100644 --- a/.github/workflows/build-test-ubuntu-wallet.yml +++ b/.github/workflows/build-test-ubuntu-wallet.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/.github/workflows/build-test-ubuntu-weight_proof.yml b/.github/workflows/build-test-ubuntu-weight_proof.yml index abccbf8229..041aaef86b 100644 --- a/.github/workflows/build-test-ubuntu-weight_proof.yml +++ b/.github/workflows/build-test-ubuntu-weight_proof.yml @@ -65,10 +65,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -76,14 +72,14 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia - name: Run install script env: diff --git a/chia/full_node/weight_proof.py b/chia/full_node/weight_proof.py index 1fac853175..7756b8e65a 100644 --- a/chia/full_node/weight_proof.py +++ b/chia/full_node/weight_proof.py @@ -1278,7 +1278,7 @@ def validate_recent_blocks( ses = False height = block.height for sub_slot in block.finished_sub_slots: - prev_challenge = challenge + prev_challenge = sub_slot.challenge_chain.challenge_chain_end_of_slot_vdf.challenge challenge = sub_slot.challenge_chain.get_hash() deficit = sub_slot.reward_chain.deficit if sub_slot.challenge_chain.subepoch_summary_hash is not None: @@ -1504,6 +1504,10 @@ def __get_rc_sub_slot( assert segment.rc_slot_end_info is not None if idx != 0: + # this is not the first slot, ses details should not be included + ses_hash = None + new_ssi = None + new_diff = None cc_vdf_info = VDFInfo(sub_slot.cc_slot_end_info.challenge, curr_ssi, sub_slot.cc_slot_end_info.output) if sub_slot.icc_slot_end_info is not None: icc_slot_end_info = VDFInfo( diff --git a/chia/plotting/create_plots.py b/chia/plotting/create_plots.py index c72c686a6d..aef0ac773d 100644 --- a/chia/plotting/create_plots.py +++ b/chia/plotting/create_plots.py @@ -156,9 +156,9 @@ async def create_plots( if args.size < config["min_mainnet_k_size"] and test_private_keys is None: log.warning(f"Creating plots with size k={args.size}, which is less than the minimum required for mainnet") - if args.size < 22: - log.warning("k under 22 is not supported. Increasing k to 22") - args.size = 22 + if args.size < 20: + log.warning("k under 22 is not supported. Increasing k to 21") + args.size = 20 if keys.pool_public_key is not None: log.info( diff --git a/tests/block_tools.py b/tests/block_tools.py index 6e480fdd25..a000f494d4 100644 --- a/tests/block_tools.py +++ b/tests/block_tools.py @@ -98,7 +98,7 @@ test_constants = DEFAULT_CONSTANTS.replace( **{ "MIN_PLOT_SIZE": 18, "MIN_BLOCKS_PER_CHALLENGE_BLOCK": 12, - "DIFFICULTY_STARTING": 2 ** 12, + "DIFFICULTY_STARTING": 2 ** 10, "DISCRIMINANT_SIZE_BITS": 16, "SUB_EPOCH_BLOCKS": 170, "WEIGHT_PROOF_THRESHOLD": 2, @@ -277,7 +277,7 @@ class BlockTools: tmp_dir = self.temp_dir args = Namespace() # Can't go much lower than 20, since plots start having no solutions and more buggy - args.size = 22 + args.size = 20 # Uses many plots for testing, in order to guarantee proofs of space at every height args.num = 1 args.buffer = 100 diff --git a/tests/blockchain/test_blockchain.py b/tests/blockchain/test_blockchain.py index 346822d44e..f6d9852bc8 100644 --- a/tests/blockchain/test_blockchain.py +++ b/tests/blockchain/test_blockchain.py @@ -187,6 +187,7 @@ class TestBlockHeaderValidation: "reward_chain.challenge_chain_sub_slot_hash", new_finished_ss_3.challenge_chain.get_hash(), ) + log.warning(f"Number of slots: {len(block.finished_sub_slots)}") block_bad_3 = recursive_replace( block, "finished_sub_slots", [new_finished_ss_3] + block.finished_sub_slots[1:] ) @@ -741,11 +742,12 @@ class TestBlockHeaderValidation: await _validate_and_add_block(blockchain, block_bad, expected_result=ReceiveBlockResult.INVALID_BLOCK) @pytest.mark.asyncio - async def test_empty_sub_slots_epoch(self, empty_blockchain, bt): + async def test_empty_sub_slots_epoch(self, empty_blockchain, default_400_blocks, bt): # 2m # Tests adding an empty sub slot after the sub-epoch / epoch. # Also tests overflow block in epoch - blocks_base = bt.get_consecutive_blocks(test_constants.EPOCH_BLOCKS) + blocks_base = default_400_blocks[: test_constants.EPOCH_BLOCKS] + assert len(blocks_base) == test_constants.EPOCH_BLOCKS blocks_1 = bt.get_consecutive_blocks(1, block_list_input=blocks_base, force_overflow=True) blocks_2 = bt.get_consecutive_blocks(1, skip_slots=3, block_list_input=blocks_base, force_overflow=True) for block in blocks_base: @@ -779,10 +781,19 @@ class TestBlockHeaderValidation: @pytest.mark.asyncio async def test_invalid_cc_sub_slot_vdf(self, empty_blockchain, bt): # 2q - blocks = bt.get_consecutive_blocks(10) + blocks: List[FullBlock] = [] + found_overflow_slot: bool = False - for block in blocks: - if len(block.finished_sub_slots): + while not found_overflow_slot: + blocks = bt.get_consecutive_blocks(1, blocks) + block = blocks[-1] + if ( + len(block.finished_sub_slots) + and is_overflow_block(test_constants, block.reward_chain_block.signage_point_index) + and block.finished_sub_slots[-1].challenge_chain.challenge_chain_end_of_slot_vdf.output + != ClassgroupElement.get_default_element() + ): + found_overflow_slot = True # Bad iters new_finished_ss = recursive_replace( block.finished_sub_slots[-1], @@ -798,9 +809,11 @@ class TestBlockHeaderValidation: "reward_chain.challenge_chain_sub_slot_hash", new_finished_ss.challenge_chain.get_hash(), ) + log.warning(f"Num slots: {len(block.finished_sub_slots)}") block_bad = recursive_replace( block, "finished_sub_slots", block.finished_sub_slots[:-1] + [new_finished_ss] ) + log.warning(f"Signage point index: {block_bad.reward_chain_block.signage_point_index}") await _validate_and_add_block(empty_blockchain, block_bad, expected_error=Err.INVALID_CC_EOS_VDF) # Bad output @@ -845,7 +858,9 @@ class TestBlockHeaderValidation: ) await _validate_and_add_block_multi_error( - empty_blockchain, block_bad_3, [Err.INVALID_CC_EOS_VDF, Err.INVALID_PREV_CHALLENGE_SLOT_HASH] + empty_blockchain, + block_bad_3, + [Err.INVALID_CC_EOS_VDF, Err.INVALID_PREV_CHALLENGE_SLOT_HASH, Err.INVALID_POSPACE], ) # Bad proof @@ -864,9 +879,18 @@ class TestBlockHeaderValidation: @pytest.mark.asyncio async def test_invalid_rc_sub_slot_vdf(self, empty_blockchain, bt): # 2p - blocks = bt.get_consecutive_blocks(10) - for block in blocks: - if len(block.finished_sub_slots): + blocks: List[FullBlock] = [] + found_block: bool = False + + while not found_block: + blocks = bt.get_consecutive_blocks(1, blocks) + block = blocks[-1] + if ( + len(block.finished_sub_slots) + and block.finished_sub_slots[-1].reward_chain.end_of_slot_vdf.output + != ClassgroupElement.get_default_element() + ): + found_block = True # Bad iters new_finished_ss = recursive_replace( block.finished_sub_slots[-1], @@ -1011,7 +1035,9 @@ class TestBlockHeaderValidation: while True: blocks = bt.get_consecutive_blocks(1, block_list_input=blocks) - if len(blocks[-1].finished_sub_slots) > 0: + if len(blocks[-1].finished_sub_slots) > 0 and is_overflow_block( + test_constants, blocks[-1].reward_chain_block.signage_point_index + ): new_finished_ss: EndOfSubSlotBundle = recursive_replace( blocks[-1].finished_sub_slots[0], "challenge_chain", @@ -2860,7 +2886,7 @@ class TestReorgs: assert b.get_peak().height == 16 @pytest.mark.asyncio - async def test_long_reorg(self, empty_blockchain, default_10000_blocks, bt): + async def test_long_reorg(self, empty_blockchain, default_1500_blocks, test_long_reorg_blocks, bt): # Reorg longer than a difficulty adjustment # Also tests higher weight chain but lower height b = empty_blockchain @@ -2869,7 +2895,7 @@ class TestReorgs: num_blocks_chain_2 = 3 * test_constants.EPOCH_BLOCKS + test_constants.MAX_SUB_SLOT_BLOCKS + 8 assert num_blocks_chain_1 < 10000 - blocks = default_10000_blocks[:num_blocks_chain_1] + blocks = default_1500_blocks[:num_blocks_chain_1] for block in blocks: await _validate_and_add_block(b, block, skip_prevalidation=True) @@ -2877,15 +2903,15 @@ class TestReorgs: chain_1_weight = b.get_peak().weight assert chain_1_height == (num_blocks_chain_1 - 1) - # These blocks will have less time between them (timestamp) and therefore will make difficulty go up + # The reorg blocks will have less time between them (timestamp) and therefore will make difficulty go up # This means that the weight will grow faster, and we can get a heavier chain with lower height - blocks_reorg_chain = bt.get_consecutive_blocks( - num_blocks_chain_2 - num_blocks_chain_2_start, - blocks[:num_blocks_chain_2_start], - seed=b"2", - time_per_block=8, - ) - for reorg_block in blocks_reorg_chain: + + # If these assert fail, you probably need to change the fixture in test_long_reorg_blocks to create the + # right amount of blocks at the right time + assert test_long_reorg_blocks[num_blocks_chain_2_start - 1] == default_1500_blocks[num_blocks_chain_2_start - 1] + assert test_long_reorg_blocks[num_blocks_chain_2_start] != default_1500_blocks[num_blocks_chain_2_start] + + for reorg_block in test_long_reorg_blocks: if reorg_block.height < num_blocks_chain_2_start: await _validate_and_add_block( b, reorg_block, expected_result=ReceiveBlockResult.ALREADY_HAVE_BLOCK, skip_prevalidation=True @@ -2905,11 +2931,11 @@ class TestReorgs: assert b.get_peak().height < chain_1_height @pytest.mark.asyncio - async def test_long_compact_blockchain(self, empty_blockchain, default_10000_blocks_compact): + async def test_long_compact_blockchain(self, empty_blockchain, default_2000_blocks_compact): b = empty_blockchain - for block in default_10000_blocks_compact: + for block in default_2000_blocks_compact: await _validate_and_add_block(b, block, skip_prevalidation=True) - assert b.get_peak().height == len(default_10000_blocks_compact) - 1 + assert b.get_peak().height == len(default_2000_blocks_compact) - 1 @pytest.mark.asyncio async def test_reorg_from_genesis(self, empty_blockchain, bt): diff --git a/tests/conftest.py b/tests/conftest.py index f5377b5761..cad1186f7e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -101,21 +101,21 @@ def softfork_height(request): return request.param -block_format_version = "rc4" +saved_blocks_version = "rc5" @pytest.fixture(scope="session") def default_400_blocks(bt): from tests.util.blockchain import persistent_blocks - return persistent_blocks(400, f"test_blocks_400_{block_format_version}.db", bt, seed=b"alternate2") + return persistent_blocks(400, f"test_blocks_400_{saved_blocks_version}.db", bt, seed=b"400") @pytest.fixture(scope="session") def default_1000_blocks(bt): from tests.util.blockchain import persistent_blocks - return persistent_blocks(1000, f"test_blocks_1000_{block_format_version}.db", bt) + return persistent_blocks(1000, f"test_blocks_1000_{saved_blocks_version}.db", bt, seed=b"1000") @pytest.fixture(scope="session") @@ -123,22 +123,63 @@ def pre_genesis_empty_slots_1000_blocks(bt): from tests.util.blockchain import persistent_blocks return persistent_blocks( - 1000, f"pre_genesis_empty_slots_1000_blocks{block_format_version}.db", bt, seed=b"alternate2", empty_sub_slots=1 + 1000, + f"pre_genesis_empty_slots_1000_blocks{saved_blocks_version}.db", + bt, + seed=b"empty_slots", + empty_sub_slots=1, ) +@pytest.fixture(scope="session") +def default_1500_blocks(bt): + from tests.util.blockchain import persistent_blocks + + return persistent_blocks(1500, f"test_blocks_1500_{saved_blocks_version}.db", bt, seed=b"1500") + + @pytest.fixture(scope="session") def default_10000_blocks(bt): from tests.util.blockchain import persistent_blocks - return persistent_blocks(10000, f"test_blocks_10000_{block_format_version}.db", bt) + return persistent_blocks(10000, f"test_blocks_10000_{saved_blocks_version}.db", bt, seed=b"10000") @pytest.fixture(scope="session") def default_20000_blocks(bt): from tests.util.blockchain import persistent_blocks - return persistent_blocks(20000, f"test_blocks_20000_{block_format_version}.db", bt) + return persistent_blocks(20000, f"test_blocks_20000_{saved_blocks_version}.db", bt, seed=b"20000") + + +@pytest.fixture(scope="session") +def test_long_reorg_blocks(bt, default_1500_blocks): + from tests.util.blockchain import persistent_blocks + + return persistent_blocks( + 758, + f"test_blocks_long_reorg_{saved_blocks_version}.db", + bt, + block_list_input=default_1500_blocks[:320], + seed=b"reorg_blocks", + time_per_block=8, + ) + + +@pytest.fixture(scope="session") +def default_2000_blocks_compact(bt): + from tests.util.blockchain import persistent_blocks + + return persistent_blocks( + 2000, + f"test_blocks_2000_compact_{saved_blocks_version}.db", + bt, + normalized_to_identity_cc_eos=True, + normalized_to_identity_icc_eos=True, + normalized_to_identity_cc_ip=True, + normalized_to_identity_cc_sp=True, + seed=b"2000_compact", + ) @pytest.fixture(scope="session") @@ -147,12 +188,13 @@ def default_10000_blocks_compact(bt): return persistent_blocks( 10000, - f"test_blocks_10000_compact_{block_format_version}.db", + f"test_blocks_10000_compact_{saved_blocks_version}.db", bt, normalized_to_identity_cc_eos=True, normalized_to_identity_icc_eos=True, normalized_to_identity_cc_ip=True, normalized_to_identity_cc_sp=True, + seed=b"1000_compact", ) diff --git a/tests/core/full_node/full_sync/test_full_sync.py b/tests/core/full_node/full_sync/test_full_sync.py index df244e7bbe..5513f7bdf5 100644 --- a/tests/core/full_node/full_sync/test_full_sync.py +++ b/tests/core/full_node/full_sync/test_full_sync.py @@ -278,7 +278,7 @@ class TestFullSync: @pytest.mark.asyncio async def test_sync_bad_peak_while_synced( - self, three_nodes, default_1000_blocks, default_10000_blocks, self_hostname + self, three_nodes, default_1000_blocks, default_1500_blocks, self_hostname ): # Must be larger than "sync_block_behind_threshold" in the config num_blocks_initial = len(default_1000_blocks) - 250 @@ -292,7 +292,7 @@ class TestFullSync: await full_node_1.full_node.respond_block(full_node_protocol.RespondBlock(block)) # Node 3 syncs from a different blockchain - for block in default_10000_blocks[:1100]: + for block in default_1500_blocks[:1100]: await full_node_3.full_node.respond_block(full_node_protocol.RespondBlock(block)) await server_2.start_client(PeerInfo(self_hostname, uint16(server_1._port)), full_node_2.full_node.on_connect) @@ -304,7 +304,7 @@ class TestFullSync: # node 2 should keep being synced and receive blocks await server_3.start_client(PeerInfo(self_hostname, uint16(server_3._port)), full_node_3.full_node.on_connect) # trigger long sync in full node 2 - peak_block = default_10000_blocks[1050] + peak_block = default_1500_blocks[1050] await server_2.start_client(PeerInfo(self_hostname, uint16(server_3._port)), full_node_2.full_node.on_connect) con = server_2.all_connections[full_node_3.full_node.server.node_id] peak = full_node_protocol.NewPeak( diff --git a/tests/core/full_node/test_full_node.py b/tests/core/full_node/test_full_node.py index c71e8b0758..bb9c0e0596 100644 --- a/tests/core/full_node/test_full_node.py +++ b/tests/core/full_node/test_full_node.py @@ -1541,7 +1541,7 @@ class TestFullNodeProtocol: ) # Note: the below numbers depend on the block cache, so might need to be updated - assert cc_eos_count == 4 and icc_eos_count == 3 + assert cc_eos_count == 3 and icc_eos_count == 3 for compact_proof in timelord_protocol_finished: await full_node_1.full_node.respond_compact_proof_of_time(compact_proof) stored_blocks = await full_node_1.get_all_full_blocks() @@ -1563,7 +1563,7 @@ class TestFullNodeProtocol: if block.challenge_chain_ip_proof.normalized_to_identity: has_compact_cc_ip_vdf = True # Note: the below numbers depend on the block cache, so might need to be updated - assert cc_eos_compact_count == 4 + assert cc_eos_compact_count == 3 assert icc_eos_compact_count == 3 assert has_compact_cc_sp_vdf assert has_compact_cc_ip_vdf diff --git a/tests/core/full_node/test_performance.py b/tests/core/full_node/test_performance.py index 58e1edeac3..fa57244f65 100644 --- a/tests/core/full_node/test_performance.py +++ b/tests/core/full_node/test_performance.py @@ -10,6 +10,7 @@ import pytest from clvm.casts import int_to_bytes from chia.consensus.block_record import BlockRecord +from chia.consensus.pot_iterations import is_overflow_block from chia.full_node.full_node_api import FullNodeAPI from chia.protocols import full_node_protocol as fnp from chia.types.condition_opcodes import ConditionOpcode @@ -155,8 +156,12 @@ class TestPerformance: guarantee_transaction_block=True, ) block = blocks[-1] + if is_overflow_block(bt.constants, block.reward_chain_block.signage_point_index): + sub_slots = block.finished_sub_slots[:-1] + else: + sub_slots = block.finished_sub_slots unfinished = UnfinishedBlock( - block.finished_sub_slots, + sub_slots, block.reward_chain_block.get_unfinished(), block.challenge_chain_sp_proof, block.reward_chain_sp_proof, diff --git a/tests/runner_templates/checkout-test-plots.include.yml b/tests/runner_templates/checkout-test-plots.include.yml index 6d3239bbe6..af48e1a220 100644 --- a/tests/runner_templates/checkout-test-plots.include.yml +++ b/tests/runner_templates/checkout-test-plots.include.yml @@ -1,7 +1,3 @@ - - name: Daily (POC) cache key invalidation for test blocks and plots - id: today-date - run: date +%F > today.txt - - name: Cache test blocks and plots uses: actions/cache@v2 id: test-blocks-plots @@ -9,11 +5,11 @@ path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: ${{ hashFiles('today.txt') }} + key: 0.29.0 - name: Checkout test blocks and plots if: steps.test-blocks-plots.outputs.cache-hit != 'true' run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.28.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.28.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia diff --git a/tests/util/blockchain.py b/tests/util/blockchain.py index 95d6908b06..ed1a628aba 100644 --- a/tests/util/blockchain.py +++ b/tests/util/blockchain.py @@ -1,7 +1,7 @@ import os import pickle from pathlib import Path -from typing import List +from typing import List, Optional import aiosqlite import tempfile @@ -45,9 +45,13 @@ def persistent_blocks( normalized_to_identity_icc_eos: bool = False, normalized_to_identity_cc_sp: bool = False, normalized_to_identity_cc_ip: bool = False, + block_list_input: List[FullBlock] = None, + time_per_block: Optional[float] = None, ): # try loading from disc, if not create new blocks.db file # TODO hash fixtures.py and blocktool.py, add to path, delete if the files changed + if block_list_input is None: + block_list_input = [] block_path_dir = DEFAULT_ROOT_PATH.parent.joinpath("blocks") file_path = block_path_dir.joinpath(db_name) @@ -65,7 +69,7 @@ def persistent_blocks( blocks: List[FullBlock] = [] for block_bytes in block_bytes_list: blocks.append(FullBlock.from_bytes(block_bytes)) - if len(blocks) == num_of_blocks: + if len(blocks) == num_of_blocks + len(block_list_input): print(f"\n loaded {file_path} with {len(blocks)} blocks") return blocks except EOFError: @@ -80,6 +84,8 @@ def persistent_blocks( seed, empty_sub_slots, bt, + block_list_input, + time_per_block, normalized_to_identity_cc_eos, normalized_to_identity_icc_eos, normalized_to_identity_cc_sp, @@ -93,6 +99,8 @@ def new_test_db( seed: bytes, empty_sub_slots: int, bt: BlockTools, + block_list_input: List[FullBlock], + time_per_block: Optional[float], normalized_to_identity_cc_eos: bool = False, # CC_EOS, normalized_to_identity_icc_eos: bool = False, # ICC_EOS normalized_to_identity_cc_sp: bool = False, # CC_SP, @@ -101,6 +109,8 @@ def new_test_db( print(f"create {path} with {num_of_blocks} blocks with ") blocks: List[FullBlock] = bt.get_consecutive_blocks( num_of_blocks, + block_list_input=block_list_input, + time_per_block=time_per_block, seed=seed, skip_slots=empty_sub_slots, normalized_to_identity_cc_eos=normalized_to_identity_cc_eos, From 415236bf679b9f88e5047d22e1df6e09010a034a Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 8 Apr 2022 12:58:46 -0400 Subject: [PATCH 55/63] can we get by without dead snakes? (#11070) * can we get by without dead snakes? * Update install-timelord.sh * Revert "Update install-timelord.sh" This reverts commit cba3250b095efbac5459c430102a7914243bfa96. * do not install python dev package for timelords build in ci it is already there... * more quotes for sh --- .../build-test-macos-core-daemon.yml | 10 +---- .../workflows/build-test-macos-simulation.yml | 10 +---- .../build-test-ubuntu-core-daemon.yml | 10 +---- .../build-test-ubuntu-simulation.yml | 10 +---- install-timelord.sh | 37 +++++++++++++++++-- .../install-timelord.include.yml | 10 +---- 6 files changed, 38 insertions(+), 49 deletions(-) diff --git a/.github/workflows/build-test-macos-core-daemon.yml b/.github/workflows/build-test-macos-core-daemon.yml index 950f08481d..7ab9dc765a 100644 --- a/.github/workflows/build-test-macos-core-daemon.yml +++ b/.github/workflows/build-test-macos-core-daemon.yml @@ -88,18 +88,10 @@ jobs: brew install boost sh install.sh -d - - name: Install Ubuntu dependencies - if: startsWith(matrix.os, 'ubuntu') - run: | - sudo apt-get install software-properties-common - sudo add-apt-repository ppa:deadsnakes/ppa - sudo apt-get update - sudo apt-get install python${{ matrix.python-version }}-venv python${{ matrix.python-version }}-distutils git -y - - name: Install timelord run: | . ./activate - sh install-timelord.sh + sh install-timelord.sh -n ./vdf_bench square_asm 400000 - name: Test core-daemon code with pytest diff --git a/.github/workflows/build-test-macos-simulation.yml b/.github/workflows/build-test-macos-simulation.yml index a7bd5c6695..9d8e1f57eb 100644 --- a/.github/workflows/build-test-macos-simulation.yml +++ b/.github/workflows/build-test-macos-simulation.yml @@ -88,18 +88,10 @@ jobs: brew install boost sh install.sh -d - - name: Install Ubuntu dependencies - if: startsWith(matrix.os, 'ubuntu') - run: | - sudo apt-get install software-properties-common - sudo add-apt-repository ppa:deadsnakes/ppa - sudo apt-get update - sudo apt-get install python${{ matrix.python-version }}-venv python${{ matrix.python-version }}-distutils git -y - - name: Install timelord run: | . ./activate - sh install-timelord.sh + sh install-timelord.sh -n ./vdf_bench square_asm 400000 - name: Test simulation code with pytest diff --git a/.github/workflows/build-test-ubuntu-core-daemon.yml b/.github/workflows/build-test-ubuntu-core-daemon.yml index 3611af06d2..1992c859f1 100644 --- a/.github/workflows/build-test-ubuntu-core-daemon.yml +++ b/.github/workflows/build-test-ubuntu-core-daemon.yml @@ -87,18 +87,10 @@ jobs: run: | sh install.sh -d - - name: Install Ubuntu dependencies - if: startsWith(matrix.os, 'ubuntu') - run: | - sudo apt-get install software-properties-common - sudo add-apt-repository ppa:deadsnakes/ppa - sudo apt-get update - sudo apt-get install python${{ matrix.python-version }}-venv python${{ matrix.python-version }}-distutils git -y - - name: Install timelord run: | . ./activate - sh install-timelord.sh + sh install-timelord.sh -n ./vdf_bench square_asm 400000 - name: Test core-daemon code with pytest diff --git a/.github/workflows/build-test-ubuntu-simulation.yml b/.github/workflows/build-test-ubuntu-simulation.yml index d6fbec1fc1..06435f947e 100644 --- a/.github/workflows/build-test-ubuntu-simulation.yml +++ b/.github/workflows/build-test-ubuntu-simulation.yml @@ -87,18 +87,10 @@ jobs: run: | sh install.sh -d - - name: Install Ubuntu dependencies - if: startsWith(matrix.os, 'ubuntu') - run: | - sudo apt-get install software-properties-common - sudo add-apt-repository ppa:deadsnakes/ppa - sudo apt-get update - sudo apt-get install python${{ matrix.python-version }}-venv python${{ matrix.python-version }}-distutils git -y - - name: Install timelord run: | . ./activate - sh install-timelord.sh + sh install-timelord.sh -n ./vdf_bench square_asm 400000 - name: Test simulation code with pytest diff --git a/install-timelord.sh b/install-timelord.sh index 3c99263768..7368887492 100644 --- a/install-timelord.sh +++ b/install-timelord.sh @@ -2,6 +2,29 @@ set -o errexit +USAGE_TEXT="\ +Usage: $0 [-d] + + -n do not install Python development package, Python.h etc + -h display this help and exit +" + +usage() { + echo "${USAGE_TEXT}" +} + +INSTALL_PYTHON_DEV=1 + +while getopts nh flag +do + case "${flag}" in + # development + n) INSTALL_PYTHON_DEV=;; + h) usage; exit 0;; + *) echo; usage; exit 1;; + esac +done + if [ -z "$VIRTUAL_ENV" ]; then echo "This requires the chia python virtual environment." echo "Execute '. ./activate' before running." @@ -13,6 +36,12 @@ echo "Timelord requires CMake 3.14+ to compile vdf_client." PYTHON_VERSION=$(python -c 'import sys; print(f"python{sys.version_info.major}.{sys.version_info.minor}")') echo "Python version: $PYTHON_VERSION" +if [ "$INSTALL_PYTHON_DEV" ]; then + PYTHON_DEV_DEPENDENCY=lib"$PYTHON_VERSION"-dev +else + PYTHON_DEV_DEPENDENCY= +fi + export BUILD_VDF_BENCH=Y # Installs the useful vdf_bench test of CPU squaring speed THE_PATH=$(python -c 'import pkg_resources; print( pkg_resources.get_distribution("chiavdf").location)' 2>/dev/null)/vdf_client CHIAVDF_VERSION=$(python -c 'from setup import dependencies; t = [_ for _ in dependencies if _.startswith("chiavdf")][0]; print(t)') @@ -63,16 +92,16 @@ else # If Ubuntu version is older than 20.04LTS then upgrade CMake ubuntu_cmake_install # Install remaining needed development tools - assumes venv and prior run of install.sh - echo apt-get install libgmp-dev libboost-python-dev lib"$PYTHON_VERSION"-dev libboost-system-dev build-essential -y - sudo apt-get install libgmp-dev libboost-python-dev lib"$PYTHON_VERSION"-dev libboost-system-dev build-essential -y + echo apt-get install libgmp-dev libboost-python-dev "$PYTHON_DEV_DEPENDENCY" libboost-system-dev build-essential -y + sudo apt-get install libgmp-dev libboost-python-dev "$PYTHON_DEV_DEPENDENCY" libboost-system-dev build-essential -y echo venv/bin/python -m pip install --force --no-binary chiavdf "$CHIAVDF_VERSION" venv/bin/python -m pip install --force --no-binary chiavdf "$CHIAVDF_VERSION" symlink_vdf_bench "$PYTHON_VERSION" elif [ -e venv/bin/python ] && test $RHEL_BASED; then echo "Installing chiavdf from source on RedHat/CentOS/Fedora" # Install remaining needed development tools - assumes venv and prior run of install.sh - echo yum install gcc gcc-c++ gmp-devel python3-devel libtool make autoconf automake openssl-devel libevent-devel boost-devel python3 -y - sudo yum install gcc gcc-c++ gmp-devel python3-devel libtool make autoconf automake openssl-devel libevent-devel boost-devel python3 -y + echo yum install gcc gcc-c++ gmp-devel "$PYTHON_DEV_DEPENDENCY" libtool make autoconf automake openssl-devel libevent-devel boost-devel python3 -y + sudo yum install gcc gcc-c++ gmp-devel "$PYTHON_DEV_DEPENDENCY" libtool make autoconf automake openssl-devel libevent-devel boost-devel python3 -y echo venv/bin/python -m pip install --force --no-binary chiavdf "$CHIAVDF_VERSION" venv/bin/python -m pip install --force --no-binary chiavdf "$CHIAVDF_VERSION" symlink_vdf_bench "$PYTHON_VERSION" diff --git a/tests/runner_templates/install-timelord.include.yml b/tests/runner_templates/install-timelord.include.yml index 0093f48f69..0fbdcf20e8 100644 --- a/tests/runner_templates/install-timelord.include.yml +++ b/tests/runner_templates/install-timelord.include.yml @@ -1,13 +1,5 @@ - - name: Install Ubuntu dependencies - if: startsWith(matrix.os, 'ubuntu') - run: | - sudo apt-get install software-properties-common - sudo add-apt-repository ppa:deadsnakes/ppa - sudo apt-get update - sudo apt-get install python${{ matrix.python-version }}-venv python${{ matrix.python-version }}-distutils git -y - - name: Install timelord run: | . ./activate - sh install-timelord.sh + sh install-timelord.sh -n ./vdf_bench square_asm 400000 From 34d48a108b57cea93309c2a4adff606ebd1e7ca3 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 8 Apr 2022 12:59:10 -0400 Subject: [PATCH 56/63] Changelog from 1.3.3 (#11081) * Updating changelog * Update appdmg to 0.6.4 to work with macos 12.3 (#10886) * restrict click to < 8.1 for black https://github.com/pallets/click/issues/2225 Doing this instead of updating since updating black will change several files due to some formatting change. I would like to take that on separately from unbreaking CI. * Check for vulnerable openssl (#10988) * Check for vulnerable openssl * Update OpenSSL on MacOS * First attempt - openssl Ubuntu 18.04 and 20.04 * place local/bin ahead in PATH * specify install openssl * correct path * run ldconfig * stop building and check for patched openssl * spell sudo right by removing it * Remove openssl building - 1st attempt RHs * Test Windows OpenSSL version HT @AmineKhaldi * Get updated openssl version (#10991) * Get updated openssl version * Update pyinstaller * Fix typo * lets try this * Let's try this * Try this Co-authored-by: Earle Lowe * Gh 1.3.3v2 (#11011) * Non Hobo patch the winstaller for CVE-2022-0778 (#10995) * install.sh is not upgrading OpenSSL on MacOS (#11003) * MacOS isn't updating OpenSSL in install.sh * Exit if no brew on MacOS * Code the if tree like a pro instead. Co-authored-by: Kyle Altendorf Co-authored-by: Kyle Altendorf * Remove hobo patch * apt show not needed (#10997) * install/upgrade openssl on Arch Linux also * Update CHANGELOG * revert Arch change backport Co-authored-by: Kyle Altendorf Co-authored-by: wallentx Co-authored-by: Chris Marslender Co-authored-by: Gene Hoffman <30377676+hoffmang9@users.noreply.github.com> Co-authored-by: William Allen Co-authored-by: Earle Lowe --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35248b96e1..343909b93c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,19 @@ for setuptools_scm/PEP 440 reasons. ## [Unreleased] +## 1.3.3 Chia blockchain 2022-4-02 + +### Fixed + +- In version 1.3.2 our patch for the OpenSSL vulnerability was not complete for the Windows installer. Thank you @xsmolasses of Core-Pool. +- MacOS would not update openssl when installing via `install.sh` +- Some debugging information remained in `install.sh` + +## 1.3.2 Chia blockchain 2022-4-01 + +### Fixed + +- Fixed OpenSSL vulnerability CVE-2022-0778 ## 1.3.1 Chia blockchain 2022-3-16 From 13512be2b28e91dfcb9ebff1cbef8bab97127307 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 8 Apr 2022 12:59:36 -0400 Subject: [PATCH 57/63] consistently name installer github actions artifact zips (#11096) --- .github/workflows/build-linux-arm64-installer.yml | 2 +- .github/workflows/build-linux-installer-deb.yml | 2 +- .github/workflows/build-linux-installer-rpm.yml | 2 +- .github/workflows/build-macos-installer.yml | 2 +- .github/workflows/build-macos-m1-installer.yml | 2 +- .github/workflows/build-windows-installer.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-linux-arm64-installer.yml b/.github/workflows/build-linux-arm64-installer.yml index 6b05a2560e..6c4a1f5ac5 100644 --- a/.github/workflows/build-linux-arm64-installer.yml +++ b/.github/workflows/build-linux-arm64-installer.yml @@ -120,7 +120,7 @@ jobs: - name: Upload Linux artifacts uses: actions/upload-artifact@v2 with: - name: Linux-ARM-64-Installer + name: chia-installers-linux-deb-arm64 path: ${{ github.workspace }}/build_scripts/final_installer/ - name: Configure AWS Credentials diff --git a/.github/workflows/build-linux-installer-deb.yml b/.github/workflows/build-linux-installer-deb.yml index e9690e8bfa..5538090962 100644 --- a/.github/workflows/build-linux-installer-deb.yml +++ b/.github/workflows/build-linux-installer-deb.yml @@ -163,7 +163,7 @@ jobs: - name: Upload Linux artifacts uses: actions/upload-artifact@v2 with: - name: Linux-Installers + name: chia-installers-linux-deb-intel path: ${{ github.workspace }}/build_scripts/final_installer/ - name: Configure AWS Credentials diff --git a/.github/workflows/build-linux-installer-rpm.yml b/.github/workflows/build-linux-installer-rpm.yml index aa9c85ed83..4d3f6ff45c 100644 --- a/.github/workflows/build-linux-installer-rpm.yml +++ b/.github/workflows/build-linux-installer-rpm.yml @@ -123,7 +123,7 @@ jobs: - name: Upload Linux artifacts uses: actions/upload-artifact@v2 with: - name: Linux-Installers + name: chia-installers-linux-rpm-intel path: ${{ github.workspace }}/build_scripts/final_installer/ - name: Configure AWS Credentials diff --git a/.github/workflows/build-macos-installer.yml b/.github/workflows/build-macos-installer.yml index da60f6f20a..cb568c9518 100644 --- a/.github/workflows/build-macos-installer.yml +++ b/.github/workflows/build-macos-installer.yml @@ -150,7 +150,7 @@ jobs: - name: Upload MacOS artifacts uses: actions/upload-artifact@v2 with: - name: Chia-Installer-MacOS-intel-dmg + name: chia-installers-macos-dmg-intel path: ${{ github.workspace }}/build_scripts/final_installer/ - name: Create Checksums diff --git a/.github/workflows/build-macos-m1-installer.yml b/.github/workflows/build-macos-m1-installer.yml index 18d08db16d..67082a946d 100644 --- a/.github/workflows/build-macos-m1-installer.yml +++ b/.github/workflows/build-macos-m1-installer.yml @@ -124,7 +124,7 @@ jobs: - name: Upload MacOS artifacts uses: actions/upload-artifact@v2 with: - name: Chia-Installer-MacOS-arm64-dmg + name: chia-installers-macos-dmg-arm64 path: ${{ github.workspace }}/build_scripts/final_installer/ - name: Install AWS CLI diff --git a/.github/workflows/build-windows-installer.yml b/.github/workflows/build-windows-installer.yml index 67ee617b5e..4830cfe913 100644 --- a/.github/workflows/build-windows-installer.yml +++ b/.github/workflows/build-windows-installer.yml @@ -155,7 +155,7 @@ jobs: - name: Upload Windows exe's to artifacts uses: actions/upload-artifact@v2.2.2 with: - name: Windows-Exe + name: chia-installers-windows-exe-intel path: ${{ github.workspace }}\chia-blockchain-gui\Chia-win32-x64\ - name: Upload Installer to artifacts From e6f60c572ebda42d99899128cbc41dd270e5f293 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 8 Apr 2022 12:59:59 -0400 Subject: [PATCH 58/63] git -C and consistent activation in installer builds (#11098) --- .github/workflows/build-linux-arm64-installer.yml | 7 +++---- .github/workflows/build-linux-installer-deb.yml | 7 +++---- .github/workflows/build-linux-installer-rpm.yml | 7 +++---- .github/workflows/build-macos-installer.yml | 5 ++--- .github/workflows/build-macos-m1-installer.yml | 5 ++--- .github/workflows/build-windows-installer.yml | 1 + 6 files changed, 14 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-linux-arm64-installer.yml b/.github/workflows/build-linux-arm64-installer.yml index 6c4a1f5ac5..f2a65b1b5b 100644 --- a/.github/workflows/build-linux-arm64-installer.yml +++ b/.github/workflows/build-linux-arm64-installer.yml @@ -110,11 +110,10 @@ jobs: - name: Build arm64 .deb package run: | - . ./activate ldd --version - cd ./chia-blockchain-gui - git status - cd ../build_scripts + git -C ./chia-blockchain-gui status + . ./activate + cd ./build_scripts sh build_linux_deb.sh arm64 - name: Upload Linux artifacts diff --git a/.github/workflows/build-linux-installer-deb.yml b/.github/workflows/build-linux-installer-deb.yml index 5538090962..3724482108 100644 --- a/.github/workflows/build-linux-installer-deb.yml +++ b/.github/workflows/build-linux-installer-deb.yml @@ -153,11 +153,10 @@ jobs: - name: Build .deb package run: | - . ./activate ldd --version - cd ./chia-blockchain-gui - git status - cd ../build_scripts + git -C ./chia-blockchain-gui status + . ./activate + cd ./build_scripts sh build_linux_deb.sh amd64 - name: Upload Linux artifacts diff --git a/.github/workflows/build-linux-installer-rpm.yml b/.github/workflows/build-linux-installer-rpm.yml index 4d3f6ff45c..43ef0ccdf8 100644 --- a/.github/workflows/build-linux-installer-rpm.yml +++ b/.github/workflows/build-linux-installer-rpm.yml @@ -113,11 +113,10 @@ jobs: - name: Build .rpm package run: | - . ./activate ldd --version - cd ./chia-blockchain-gui - git status - cd ../build_scripts + git -C ./chia-blockchain-gui status + . ./activate + cd ./build_scripts sh build_linux_rpm.sh amd64 - name: Upload Linux artifacts diff --git a/.github/workflows/build-macos-installer.yml b/.github/workflows/build-macos-installer.yml index cb568c9518..2bf43d1f5b 100644 --- a/.github/workflows/build-macos-installer.yml +++ b/.github/workflows/build-macos-installer.yml @@ -141,10 +141,9 @@ jobs: APPLE_NOTARIZE_USERNAME: "${{ secrets.APPLE_NOTARIZE_USERNAME }}" APPLE_NOTARIZE_PASSWORD: "${{ secrets.APPLE_NOTARIZE_PASSWORD }}" run: | + git -C ./chia-blockchain-gui status . ./activate - cd ./chia-blockchain-gui - git status - cd ../build_scripts + cd ./build_scripts sh build_macos.sh - name: Upload MacOS artifacts diff --git a/.github/workflows/build-macos-m1-installer.yml b/.github/workflows/build-macos-m1-installer.yml index 67082a946d..9c728c395c 100644 --- a/.github/workflows/build-macos-m1-installer.yml +++ b/.github/workflows/build-macos-m1-installer.yml @@ -115,10 +115,9 @@ jobs: APPLE_NOTARIZE_PASSWORD: "${{ secrets.APPLE_NOTARIZE_PASSWORD }}" run: | export PATH=$(brew --prefix node@16)/bin:$PATH + git -C ./chia-blockchain-gui status . ./activate - cd ./chia-blockchain-gui - arch -arm64 git status - cd ../build_scripts + cd ./build_scripts arch -arm64 sh build_macos_m1.sh - name: Upload MacOS artifacts diff --git a/.github/workflows/build-windows-installer.yml b/.github/workflows/build-windows-installer.yml index 4830cfe913..7fbd981c03 100644 --- a/.github/workflows/build-windows-installer.yml +++ b/.github/workflows/build-windows-installer.yml @@ -150,6 +150,7 @@ jobs: run: | $env:path="C:\Program` Files` (x86)\Microsoft` Visual` Studio\2019\Enterprise\SDK\ScopeCppSDK\vc15\VC\bin\;$env:path" $env:path="C:\Program` Files` (x86)\Windows` Kits\10\App` Certification` Kit;$env:path" + git -C .\chia-blockchain-gui status .\build_scripts\build_windows.ps1 - name: Upload Windows exe's to artifacts From f928283b3d0871f7e5e08000791f03121328e90d Mon Sep 17 00:00:00 2001 From: William Blanke Date: Fri, 8 Apr 2022 10:17:32 -0700 Subject: [PATCH 59/63] updated gui to d714c21b4ee3ebbc7d18b5f819772cd9868d0bf5 --- chia-blockchain-gui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chia-blockchain-gui b/chia-blockchain-gui index 054d7b342e..d714c21b4e 160000 --- a/chia-blockchain-gui +++ b/chia-blockchain-gui @@ -1 +1 @@ -Subproject commit 054d7b342e7c8284c9b58a775f87d393a1008bfe +Subproject commit d714c21b4ee3ebbc7d18b5f819772cd9868d0bf5 From 2d4045b9d222091bdd79881a81fb2c0795f46c18 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 8 Apr 2022 16:49:21 -0400 Subject: [PATCH 60/63] only check the version once in installer build workflows (#11099) --- .github/workflows/build-linux-arm64-installer.yml | 2 ++ .github/workflows/build-linux-installer-deb.yml | 2 ++ .github/workflows/build-linux-installer-rpm.yml | 2 ++ .github/workflows/build-macos-installer.yml | 1 + .github/workflows/build-macos-m1-installer.yml | 1 + .github/workflows/build-windows-installer.yml | 1 + build_scripts/build_linux_deb.sh | 3 --- build_scripts/build_linux_rpm.sh | 3 --- build_scripts/build_macos.sh | 3 --- build_scripts/build_macos_m1.sh | 3 --- build_scripts/build_windows.ps1 | 3 --- 11 files changed, 9 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-linux-arm64-installer.yml b/.github/workflows/build-linux-arm64-installer.yml index f2a65b1b5b..61ad71d550 100644 --- a/.github/workflows/build-linux-arm64-installer.yml +++ b/.github/workflows/build-linux-arm64-installer.yml @@ -109,6 +109,8 @@ jobs: sh install.sh - name: Build arm64 .deb package + env: + CHIA_INSTALLER_VERSION: ${{ steps.version_number.outputs.CHIA_INSTALLER_VERSION }} run: | ldd --version git -C ./chia-blockchain-gui status diff --git a/.github/workflows/build-linux-installer-deb.yml b/.github/workflows/build-linux-installer-deb.yml index 3724482108..a9feddfe14 100644 --- a/.github/workflows/build-linux-installer-deb.yml +++ b/.github/workflows/build-linux-installer-deb.yml @@ -152,6 +152,8 @@ jobs: sudo apt-get install -y jq - name: Build .deb package + env: + CHIA_INSTALLER_VERSION: ${{ steps.version_number.outputs.CHIA_INSTALLER_VERSION }} run: | ldd --version git -C ./chia-blockchain-gui status diff --git a/.github/workflows/build-linux-installer-rpm.yml b/.github/workflows/build-linux-installer-rpm.yml index 43ef0ccdf8..c97c855260 100644 --- a/.github/workflows/build-linux-installer-rpm.yml +++ b/.github/workflows/build-linux-installer-rpm.yml @@ -112,6 +112,8 @@ jobs: sh install.sh - name: Build .rpm package + env: + CHIA_INSTALLER_VERSION: ${{ steps.version_number.outputs.CHIA_INSTALLER_VERSION }} run: | ldd --version git -C ./chia-blockchain-gui status diff --git a/.github/workflows/build-macos-installer.yml b/.github/workflows/build-macos-installer.yml index 2bf43d1f5b..fa9f7d12dc 100644 --- a/.github/workflows/build-macos-installer.yml +++ b/.github/workflows/build-macos-installer.yml @@ -137,6 +137,7 @@ jobs: - name: Build MacOS DMG env: + CHIA_INSTALLER_VERSION: ${{ steps.version_number.outputs.CHIA_INSTALLER_VERSION }} NOTARIZE: ${{ steps.check_secrets.outputs.HAS_APPLE_SECRET }} APPLE_NOTARIZE_USERNAME: "${{ secrets.APPLE_NOTARIZE_USERNAME }}" APPLE_NOTARIZE_PASSWORD: "${{ secrets.APPLE_NOTARIZE_PASSWORD }}" diff --git a/.github/workflows/build-macos-m1-installer.yml b/.github/workflows/build-macos-m1-installer.yml index 9c728c395c..323e636311 100644 --- a/.github/workflows/build-macos-m1-installer.yml +++ b/.github/workflows/build-macos-m1-installer.yml @@ -110,6 +110,7 @@ jobs: - name: Build MacOS DMG env: + CHIA_INSTALLER_VERSION: ${{ steps.version_number.outputs.CHIA_INSTALLER_VERSION }} NOTARIZE: ${{ steps.check_secrets.outputs.HAS_APPLE_SECRET }} APPLE_NOTARIZE_USERNAME: "${{ secrets.APPLE_NOTARIZE_USERNAME }}" APPLE_NOTARIZE_PASSWORD: "${{ secrets.APPLE_NOTARIZE_PASSWORD }}" diff --git a/.github/workflows/build-windows-installer.yml b/.github/workflows/build-windows-installer.yml index 7fbd981c03..ae2161f0b3 100644 --- a/.github/workflows/build-windows-installer.yml +++ b/.github/workflows/build-windows-installer.yml @@ -145,6 +145,7 @@ jobs: - name: Build Windows installer with build_scripts\build_windows.ps1 env: + CHIA_INSTALLER_VERSION: ${{ steps.version_number.outputs.CHIA_INSTALLER_VERSION }} WIN_CODE_SIGN_PASS: ${{ secrets.WIN_CODE_SIGN_PASS }} HAS_SECRET: ${{ steps.check_secrets.outputs.HAS_SIGNING_SECRET }} run: | diff --git a/build_scripts/build_linux_deb.sh b/build_scripts/build_linux_deb.sh index b60b6744ac..ea493bafe5 100644 --- a/build_scripts/build_linux_deb.sh +++ b/build_scripts/build_linux_deb.sh @@ -13,11 +13,8 @@ else DIR_NAME="chia-blockchain-linux-arm64" fi -pip install setuptools_scm -# The environment variable CHIA_INSTALLER_VERSION needs to be defined # If the env variable NOTARIZE and the username and password variables are # set, this will attempt to Notarize the signed DMG -CHIA_INSTALLER_VERSION=$(python installer-version.py) if [ ! "$CHIA_INSTALLER_VERSION" ]; then echo "WARNING: No environment variable CHIA_INSTALLER_VERSION set. Using 0.0.0." diff --git a/build_scripts/build_linux_rpm.sh b/build_scripts/build_linux_rpm.sh index e8fa595fc0..7ec656eeef 100644 --- a/build_scripts/build_linux_rpm.sh +++ b/build_scripts/build_linux_rpm.sh @@ -14,11 +14,8 @@ else DIR_NAME="chia-blockchain-linux-arm64" fi -pip install setuptools_scm -# The environment variable CHIA_INSTALLER_VERSION needs to be defined # If the env variable NOTARIZE and the username and password variables are # set, this will attempt to Notarize the signed DMG -CHIA_INSTALLER_VERSION=$(python installer-version.py) if [ ! "$CHIA_INSTALLER_VERSION" ]; then echo "WARNING: No environment variable CHIA_INSTALLER_VERSION set. Using 0.0.0." diff --git a/build_scripts/build_macos.sh b/build_scripts/build_macos.sh index d1831e285f..2cacc4d9fb 100644 --- a/build_scripts/build_macos.sh +++ b/build_scripts/build_macos.sh @@ -2,11 +2,8 @@ set -o errexit -o nounset -pip install setuptools_scm -# The environment variable CHIA_INSTALLER_VERSION needs to be defined. # If the env variable NOTARIZE and the username and password variables are # set, this will attempt to Notarize the signed DMG. -CHIA_INSTALLER_VERSION=$(python installer-version.py) if [ ! "$CHIA_INSTALLER_VERSION" ]; then echo "WARNING: No environment variable CHIA_INSTALLER_VERSION set. Using 0.0.0." diff --git a/build_scripts/build_macos_m1.sh b/build_scripts/build_macos_m1.sh index 8cb006e7b4..45f063373b 100644 --- a/build_scripts/build_macos_m1.sh +++ b/build_scripts/build_macos_m1.sh @@ -2,11 +2,8 @@ set -o errexit -o nounset -pip install setuptools_scm -# The environment variable CHIA_INSTALLER_VERSION needs to be defined. # If the env variable NOTARIZE and the username and password variables are # set, this will attempt to Notarize the signed DMG. -CHIA_INSTALLER_VERSION=$(python installer-version.py) if [ ! "$CHIA_INSTALLER_VERSION" ]; then echo "WARNING: No environment variable CHIA_INSTALLER_VERSION set. Using 0.0.0." diff --git a/build_scripts/build_windows.ps1 b/build_scripts/build_windows.ps1 index 52a2a14b63..38498d42b5 100644 --- a/build_scripts/build_windows.ps1 +++ b/build_scripts/build_windows.ps1 @@ -32,12 +32,9 @@ python -m pip install --upgrade pip pip install wheel pep517 pip install pywin32 pip install pyinstaller==4.9 -pip install setuptools_scm Write-Output " ---" -Write-Output "Get CHIA_INSTALLER_VERSION" # The environment variable CHIA_INSTALLER_VERSION needs to be defined -$env:CHIA_INSTALLER_VERSION = python .\build_scripts\installer-version.py -win if (-not (Test-Path env:CHIA_INSTALLER_VERSION)) { $env:CHIA_INSTALLER_VERSION = '0.0.0' From 5dea4a238e1912f4ea298de54257611f089bf546 Mon Sep 17 00:00:00 2001 From: William Blanke Date: Fri, 8 Apr 2022 14:11:04 -0700 Subject: [PATCH 61/63] updated gui to 5f8b23fc7deb0b07f665c075ed491059f4d8b95c --- chia-blockchain-gui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chia-blockchain-gui b/chia-blockchain-gui index d714c21b4e..5f8b23fc7d 160000 --- a/chia-blockchain-gui +++ b/chia-blockchain-gui @@ -1 +1 @@ -Subproject commit d714c21b4ee3ebbc7d18b5f819772cd9868d0bf5 +Subproject commit 5f8b23fc7deb0b07f665c075ed491059f4d8b95c From 930a4b682558e934a2ed41b48b515b8a5a08087b Mon Sep 17 00:00:00 2001 From: William Blanke Date: Fri, 8 Apr 2022 15:03:13 -0700 Subject: [PATCH 62/63] updated gui to fccbd3e10d27673e39c01f0f89e47b5455b8331a --- chia-blockchain-gui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chia-blockchain-gui b/chia-blockchain-gui index 5f8b23fc7d..fccbd3e10d 160000 --- a/chia-blockchain-gui +++ b/chia-blockchain-gui @@ -1 +1 @@ -Subproject commit 5f8b23fc7deb0b07f665c075ed491059f4d8b95c +Subproject commit fccbd3e10d27673e39c01f0f89e47b5455b8331a From a48fd431009ff0cd4874e4da99bc3071f45564da Mon Sep 17 00:00:00 2001 From: dustinface <35775977+xdustinface@users.noreply.github.com> Date: Sat, 9 Apr 2022 03:29:32 +0200 Subject: [PATCH 63/63] streamable: Simplify and force correct usage (#10509) * streamable: Merge `strictdataclass` into `Streamable` class * tests: Test not supported streamable types * streamable: Reorder decorators * streamable: Simplify streamable decorator and force correct usage/syntax * streamable: Just move some stuff around in the file * streamable: Improve syntax error messages * mypy: Drop `type_checking.py` and `test_type_checking.py` from exclusion * streamable: Use cached fields instead of `__annotations__` This is now possible after merging `__post_init__` into `Streamable` * Introduce `DefinitionError` as `StreamableError` * `/t` -> ` ` --- benchmarks/streamable.py | 6 +- chia/clvm/spend_sim.py | 6 +- chia/consensus/block_record.py | 2 +- chia/consensus/cost_calculator.py | 2 +- chia/consensus/multiprocess_validation.py | 2 +- chia/full_node/block_height_map.py | 2 +- chia/full_node/full_node_store.py | 2 +- chia/full_node/signage_point.py | 2 +- chia/plotting/manager.py | 4 +- chia/pools/pool_config.py | 2 +- chia/pools/pool_wallet_info.py | 4 +- chia/protocols/farmer_protocol.py | 10 +- chia/protocols/full_node_protocol.py | 50 ++-- chia/protocols/harvester_protocol.py | 32 +-- chia/protocols/introducer_protocol.py | 4 +- chia/protocols/pool_protocol.py | 26 +- chia/protocols/shared_protocol.py | 2 +- chia/protocols/timelord_protocol.py | 14 +- chia/protocols/wallet_protocol.py | 58 ++-- chia/seeder/peer_record.py | 2 +- chia/server/address_manager_store.py | 2 +- chia/server/outbound_message.py | 2 +- chia/simulator/simulator_protocol.py | 4 +- chia/timelord/timelord.py | 2 +- chia/types/blockchain_format/classgroup.py | 2 +- chia/types/blockchain_format/coin.py | 2 +- chia/types/blockchain_format/foliage.py | 8 +- chia/types/blockchain_format/pool_target.py | 2 +- .../types/blockchain_format/proof_of_space.py | 2 +- .../blockchain_format/reward_chain_block.py | 4 +- chia/types/blockchain_format/slots.py | 10 +- .../blockchain_format/sub_epoch_summary.py | 2 +- chia/types/blockchain_format/vdf.py | 4 +- chia/types/coin_record.py | 2 +- chia/types/coin_spend.py | 2 +- chia/types/condition_with_args.py | 2 +- chia/types/end_of_slot_bundle.py | 2 +- chia/types/full_block.py | 2 +- chia/types/generator_types.py | 2 +- chia/types/header_block.py | 2 +- chia/types/mempool_item.py | 2 +- chia/types/peer_info.py | 4 +- chia/types/spend_bundle.py | 2 +- chia/types/spend_bundle_conditions.py | 4 +- chia/types/unfinished_block.py | 2 +- chia/types/unfinished_header_block.py | 2 +- chia/types/weight_proof.py | 14 +- chia/util/streamable.py | 264 ++++++++++++------ chia/util/type_checking.py | 103 ------- chia/wallet/block_record.py | 2 +- chia/wallet/cat_wallet/cat_info.py | 4 +- chia/wallet/did_wallet/did_info.py | 2 +- chia/wallet/lineage_proof.py | 2 +- chia/wallet/rl_wallet/rl_wallet.py | 2 +- chia/wallet/settings/settings_objects.py | 2 +- chia/wallet/trade_record.py | 2 +- chia/wallet/transaction_record.py | 2 +- chia/wallet/wallet_info.py | 4 +- mypy.ini | 2 +- tests/core/util/test_streamable.py | 206 ++++++++++++-- tests/core/util/test_type_checking.py | 91 ------ 61 files changed, 549 insertions(+), 461 deletions(-) delete mode 100644 chia/util/type_checking.py delete mode 100644 tests/core/util/test_type_checking.py diff --git a/benchmarks/streamable.py b/benchmarks/streamable.py index 47e77d224a..d940a421a6 100644 --- a/benchmarks/streamable.py +++ b/benchmarks/streamable.py @@ -17,14 +17,14 @@ from chia.util.streamable import Streamable, streamable _version = 1 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class BenchmarkInner(Streamable): a: str -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class BenchmarkMiddle(Streamable): a: uint64 b: List[bytes32] @@ -33,8 +33,8 @@ class BenchmarkMiddle(Streamable): e: BenchmarkInner -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class BenchmarkClass(Streamable): a: Optional[BenchmarkMiddle] b: Optional[BenchmarkMiddle] diff --git a/chia/clvm/spend_sim.py b/chia/clvm/spend_sim.py index b448966e4a..d143b6ed4f 100644 --- a/chia/clvm/spend_sim.py +++ b/chia/clvm/spend_sim.py @@ -38,15 +38,15 @@ and is designed so that you could test with it and then swap in a real rpc clien """ -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SimFullBlock(Streamable): transactions_generator: Optional[BlockGenerator] height: uint32 # Note that height is not on a regular FullBlock -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SimBlockRecord(Streamable): reward_claims_incorporated: List[Coin] height: uint32 @@ -69,8 +69,8 @@ class SimBlockRecord(Streamable): ) -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SimStore(Streamable): timestamp: uint64 block_height: uint32 diff --git a/chia/consensus/block_record.py b/chia/consensus/block_record.py index 520a9497b1..a6ccfc4f7c 100644 --- a/chia/consensus/block_record.py +++ b/chia/consensus/block_record.py @@ -11,8 +11,8 @@ from chia.util.ints import uint8, uint32, uint64, uint128 from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class BlockRecord(Streamable): """ This class is not included or hashed into the blockchain, but it is kept in memory as a more diff --git a/chia/consensus/cost_calculator.py b/chia/consensus/cost_calculator.py index c207ccf561..34577a82cb 100644 --- a/chia/consensus/cost_calculator.py +++ b/chia/consensus/cost_calculator.py @@ -6,8 +6,8 @@ from chia.util.ints import uint16, uint64 from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NPCResult(Streamable): error: Optional[uint16] conds: Optional[SpendBundleConditions] diff --git a/chia/consensus/multiprocess_validation.py b/chia/consensus/multiprocess_validation.py index d908ab5808..6cf3d30281 100644 --- a/chia/consensus/multiprocess_validation.py +++ b/chia/consensus/multiprocess_validation.py @@ -35,8 +35,8 @@ from chia.util.streamable import Streamable, dataclass_from_dict, streamable log = logging.getLogger(__name__) -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PreValidationResult(Streamable): error: Optional[uint16] required_iters: Optional[uint64] # Iff error is None diff --git a/chia/full_node/block_height_map.py b/chia/full_node/block_height_map.py index 49e60df164..321225d7a4 100644 --- a/chia/full_node/block_height_map.py +++ b/chia/full_node/block_height_map.py @@ -13,8 +13,8 @@ from chia.util.db_wrapper import DBWrapper2 log = logging.getLogger(__name__) -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SesCache(Streamable): content: List[Tuple[uint32, bytes]] diff --git a/chia/full_node/full_node_store.py b/chia/full_node/full_node_store.py index 7282669987..0626e8e1a1 100644 --- a/chia/full_node/full_node_store.py +++ b/chia/full_node/full_node_store.py @@ -29,8 +29,8 @@ from chia.util.streamable import Streamable, streamable log = logging.getLogger(__name__) -@dataclasses.dataclass(frozen=True) @streamable +@dataclasses.dataclass(frozen=True) class FullNodeStorePeakResult(Streamable): added_eos: Optional[EndOfSubSlotBundle] new_signage_points: List[Tuple[uint8, SignagePoint]] diff --git a/chia/full_node/signage_point.py b/chia/full_node/signage_point.py index be79026fd5..9230d6f3f3 100644 --- a/chia/full_node/signage_point.py +++ b/chia/full_node/signage_point.py @@ -5,8 +5,8 @@ from chia.types.blockchain_format.vdf import VDFInfo, VDFProof from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SignagePoint(Streamable): cc_vdf: Optional[VDFInfo] cc_proof: Optional[VDFProof] diff --git a/chia/plotting/manager.py b/chia/plotting/manager.py index 7f48cbfc30..24ec531e92 100644 --- a/chia/plotting/manager.py +++ b/chia/plotting/manager.py @@ -32,16 +32,16 @@ log = logging.getLogger(__name__) CURRENT_VERSION: uint16 = uint16(0) -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class CacheEntry(Streamable): pool_public_key: Optional[G1Element] pool_contract_puzzle_hash: Optional[bytes32] plot_public_key: G1Element -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class DiskCache(Streamable): version: uint16 data: List[Tuple[bytes32, CacheEntry]] diff --git a/chia/pools/pool_config.py b/chia/pools/pool_config.py index ffa07e962e..cc4519169a 100644 --- a/chia/pools/pool_config.py +++ b/chia/pools/pool_config.py @@ -25,8 +25,8 @@ pool_list: log = logging.getLogger(__name__) -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PoolWalletConfig(Streamable): launcher_id: bytes32 pool_url: str diff --git a/chia/pools/pool_wallet_info.py b/chia/pools/pool_wallet_info.py index 42c5e4aebc..3fef250952 100644 --- a/chia/pools/pool_wallet_info.py +++ b/chia/pools/pool_wallet_info.py @@ -38,8 +38,8 @@ LEAVING_POOL = PoolSingletonState.LEAVING_POOL FARMING_TO_POOL = PoolSingletonState.FARMING_TO_POOL -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PoolState(Streamable): """ `PoolState` is a type that is serialized to the blockchain to track the state of the user's pool singleton @@ -97,8 +97,8 @@ def create_pool_state( return ps -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PoolWalletInfo(Streamable): """ Internal Pool Wallet state, not destined for the blockchain. This can be completely derived with diff --git a/chia/protocols/farmer_protocol.py b/chia/protocols/farmer_protocol.py index 1d2c4f062e..e60a421d16 100644 --- a/chia/protocols/farmer_protocol.py +++ b/chia/protocols/farmer_protocol.py @@ -15,8 +15,8 @@ Note: When changing this file, also change protocol_message_types.py, and the pr """ -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewSignagePoint(Streamable): challenge_hash: bytes32 challenge_chain_sp: bytes32 @@ -26,8 +26,8 @@ class NewSignagePoint(Streamable): signage_point_index: uint8 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class DeclareProofOfSpace(Streamable): challenge_hash: bytes32 challenge_chain_sp: bytes32 @@ -41,16 +41,16 @@ class DeclareProofOfSpace(Streamable): pool_signature: Optional[G2Element] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestSignedValues(Streamable): quality_string: bytes32 foliage_block_data_hash: bytes32 foliage_transaction_block_hash: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class FarmingInfo(Streamable): challenge_hash: bytes32 sp_hash: bytes32 @@ -60,8 +60,8 @@ class FarmingInfo(Streamable): total_plots: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SignedValues(Streamable): quality_string: bytes32 foliage_block_data_signature: G2Element diff --git a/chia/protocols/full_node_protocol.py b/chia/protocols/full_node_protocol.py index cbafdad00c..793dabbe94 100644 --- a/chia/protocols/full_node_protocol.py +++ b/chia/protocols/full_node_protocol.py @@ -18,8 +18,8 @@ Note: When changing this file, also change protocol_message_types.py, and the pr """ -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewPeak(Streamable): header_hash: bytes32 height: uint32 @@ -28,102 +28,102 @@ class NewPeak(Streamable): unfinished_reward_block_hash: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewTransaction(Streamable): transaction_id: bytes32 cost: uint64 fees: uint64 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestTransaction(Streamable): transaction_id: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondTransaction(Streamable): transaction: SpendBundle -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestProofOfWeight(Streamable): total_number_of_blocks: uint32 tip: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondProofOfWeight(Streamable): wp: WeightProof tip: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestBlock(Streamable): height: uint32 include_transaction_block: bool -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RejectBlock(Streamable): height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestBlocks(Streamable): start_height: uint32 end_height: uint32 include_transaction_block: bool -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondBlocks(Streamable): start_height: uint32 end_height: uint32 blocks: List[FullBlock] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RejectBlocks(Streamable): start_height: uint32 end_height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondBlock(Streamable): block: FullBlock -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewUnfinishedBlock(Streamable): unfinished_reward_hash: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestUnfinishedBlock(Streamable): unfinished_reward_hash: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondUnfinishedBlock(Streamable): unfinished_block: UnfinishedBlock -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewSignagePointOrEndOfSubSlot(Streamable): prev_challenge_hash: Optional[bytes32] challenge_hash: bytes32 @@ -131,16 +131,16 @@ class NewSignagePointOrEndOfSubSlot(Streamable): last_rc_infusion: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestSignagePointOrEndOfSubSlot(Streamable): challenge_hash: bytes32 index_from_challenge: uint8 last_rc_infusion: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondSignagePoint(Streamable): index_from_challenge: uint8 challenge_chain_vdf: VDFInfo @@ -149,20 +149,20 @@ class RespondSignagePoint(Streamable): reward_chain_proof: VDFProof -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondEndOfSubSlot(Streamable): end_of_slot_bundle: EndOfSubSlotBundle -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestMempoolTransactions(Streamable): filter: bytes -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewCompactVDF(Streamable): height: uint32 header_hash: bytes32 @@ -170,8 +170,8 @@ class NewCompactVDF(Streamable): vdf_info: VDFInfo -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestCompactVDF(Streamable): height: uint32 header_hash: bytes32 @@ -179,8 +179,8 @@ class RequestCompactVDF(Streamable): vdf_info: VDFInfo -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondCompactVDF(Streamable): height: uint32 header_hash: bytes32 @@ -189,15 +189,15 @@ class RespondCompactVDF(Streamable): vdf_proof: VDFProof -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestPeers(Streamable): """ Return full list of peers """ -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondPeers(Streamable): peer_list: List[TimestampedPeerInfo] diff --git a/chia/protocols/harvester_protocol.py b/chia/protocols/harvester_protocol.py index 4c5cddc145..d3323a48a0 100644 --- a/chia/protocols/harvester_protocol.py +++ b/chia/protocols/harvester_protocol.py @@ -14,23 +14,23 @@ Note: When changing this file, also change protocol_message_types.py, and the pr """ -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PoolDifficulty(Streamable): difficulty: uint64 sub_slot_iters: uint64 pool_contract_puzzle_hash: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class HarvesterHandshake(Streamable): farmer_public_keys: List[G1Element] pool_public_keys: List[G1Element] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewSignagePointHarvester(Streamable): challenge_hash: bytes32 difficulty: uint64 @@ -40,8 +40,8 @@ class NewSignagePointHarvester(Streamable): pool_difficulties: List[PoolDifficulty] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewProofOfSpace(Streamable): challenge_hash: bytes32 sp_hash: bytes32 @@ -50,8 +50,8 @@ class NewProofOfSpace(Streamable): signage_point_index: uint8 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestSignatures(Streamable): plot_identifier: str challenge_hash: bytes32 @@ -59,8 +59,8 @@ class RequestSignatures(Streamable): messages: List[bytes32] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondSignatures(Streamable): plot_identifier: str challenge_hash: bytes32 @@ -70,8 +70,8 @@ class RespondSignatures(Streamable): message_signatures: List[Tuple[bytes32, G2Element]] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class Plot(Streamable): filename: str size: uint8 @@ -83,30 +83,30 @@ class Plot(Streamable): time_modified: uint64 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestPlots(Streamable): pass -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondPlots(Streamable): plots: List[Plot] failed_to_open_filenames: List[str] no_key_filenames: List[str] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PlotSyncIdentifier(Streamable): timestamp: uint64 sync_id: uint64 message_id: uint64 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PlotSyncStart(Streamable): identifier: PlotSyncIdentifier initial: bool @@ -120,8 +120,8 @@ class PlotSyncStart(Streamable): ) -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PlotSyncPathList(Streamable): identifier: PlotSyncIdentifier data: List[str] @@ -131,8 +131,8 @@ class PlotSyncPathList(Streamable): return f"PlotSyncPathList: identifier {self.identifier}, count {len(self.data)}, final {self.final}" -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PlotSyncPlotList(Streamable): identifier: PlotSyncIdentifier data: List[Plot] @@ -142,8 +142,8 @@ class PlotSyncPlotList(Streamable): return f"PlotSyncPlotList: identifier {self.identifier}, count {len(self.data)}, final {self.final}" -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PlotSyncDone(Streamable): identifier: PlotSyncIdentifier duration: uint64 @@ -152,8 +152,8 @@ class PlotSyncDone(Streamable): return f"PlotSyncDone: identifier {self.identifier}, duration {self.duration}" -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PlotSyncError(Streamable): code: int16 message: str @@ -163,8 +163,8 @@ class PlotSyncError(Streamable): return f"PlotSyncError: code {self.code}, count {self.message}, expected_identifier {self.expected_identifier}" -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PlotSyncResponse(Streamable): identifier: PlotSyncIdentifier message_type: int16 diff --git a/chia/protocols/introducer_protocol.py b/chia/protocols/introducer_protocol.py index 7eadacb214..e9eeab9978 100644 --- a/chia/protocols/introducer_protocol.py +++ b/chia/protocols/introducer_protocol.py @@ -10,15 +10,15 @@ Note: When changing this file, also change protocol_message_types.py, and the pr """ -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestPeersIntroducer(Streamable): """ Return full list of peers """ -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondPeersIntroducer(Streamable): peer_list: List[TimestampedPeerInfo] diff --git a/chia/protocols/pool_protocol.py b/chia/protocols/pool_protocol.py index e6a1f7f8ff..3b8678ec90 100644 --- a/chia/protocols/pool_protocol.py +++ b/chia/protocols/pool_protocol.py @@ -33,8 +33,8 @@ class PoolErrorCode(Enum): # Used to verify GET /farmer and GET /login -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class AuthenticationPayload(Streamable): method_name: str launcher_id: bytes32 @@ -43,8 +43,8 @@ class AuthenticationPayload(Streamable): # GET /pool_info -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class GetPoolInfoResponse(Streamable): name: str logo_url: str @@ -60,8 +60,8 @@ class GetPoolInfoResponse(Streamable): # POST /partial -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PostPartialPayload(Streamable): launcher_id: bytes32 authentication_token: uint64 @@ -71,16 +71,16 @@ class PostPartialPayload(Streamable): harvester_id: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PostPartialRequest(Streamable): payload: PostPartialPayload aggregate_signature: G2Element # Response in success case -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PostPartialResponse(Streamable): new_difficulty: uint64 @@ -89,8 +89,8 @@ class PostPartialResponse(Streamable): # Response in success case -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class GetFarmerResponse(Streamable): authentication_public_key: G1Element payout_instructions: str @@ -101,8 +101,8 @@ class GetFarmerResponse(Streamable): # POST /farmer -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PostFarmerPayload(Streamable): launcher_id: bytes32 authentication_token: uint64 @@ -111,16 +111,16 @@ class PostFarmerPayload(Streamable): suggested_difficulty: Optional[uint64] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PostFarmerRequest(Streamable): payload: PostFarmerPayload signature: G2Element # Response in success case -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PostFarmerResponse(Streamable): welcome_message: str @@ -128,8 +128,8 @@ class PostFarmerResponse(Streamable): # PUT /farmer -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PutFarmerPayload(Streamable): launcher_id: bytes32 authentication_token: uint64 @@ -138,16 +138,16 @@ class PutFarmerPayload(Streamable): suggested_difficulty: Optional[uint64] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PutFarmerRequest(Streamable): payload: PutFarmerPayload signature: G2Element # Response in success case -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PutFarmerResponse(Streamable): authentication_public_key: Optional[bool] payout_instructions: Optional[bool] @@ -158,8 +158,8 @@ class PutFarmerResponse(Streamable): # Response in error case for all endpoints of the pool protocol -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class ErrorResponse(Streamable): error_code: uint16 error_message: Optional[str] diff --git a/chia/protocols/shared_protocol.py b/chia/protocols/shared_protocol.py index ed1cc9e7d6..5b0f608f54 100644 --- a/chia/protocols/shared_protocol.py +++ b/chia/protocols/shared_protocol.py @@ -19,8 +19,8 @@ class Capability(IntEnum): BASE = 1 # Base capability just means it supports the chia protocol at mainnet -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class Handshake(Streamable): network_id: str protocol_version: str diff --git a/chia/protocols/timelord_protocol.py b/chia/protocols/timelord_protocol.py index 282bb09bfb..6db1c32e8d 100644 --- a/chia/protocols/timelord_protocol.py +++ b/chia/protocols/timelord_protocol.py @@ -16,8 +16,8 @@ Note: When changing this file, also change protocol_message_types.py, and the pr """ -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewPeakTimelord(Streamable): reward_chain_block: RewardChainBlock difficulty: uint64 @@ -31,8 +31,8 @@ class NewPeakTimelord(Streamable): passes_ses_height_but_not_yet_included: bool -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewUnfinishedBlockTimelord(Streamable): reward_chain_block: RewardChainBlockUnfinished # Reward chain trunk data difficulty: uint64 @@ -44,8 +44,8 @@ class NewUnfinishedBlockTimelord(Streamable): rc_prev: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewInfusionPointVDF(Streamable): unfinished_reward_hash: bytes32 challenge_chain_ip_vdf: VDFInfo @@ -56,8 +56,8 @@ class NewInfusionPointVDF(Streamable): infused_challenge_chain_ip_proof: Optional[VDFProof] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewSignagePointVDF(Streamable): index_from_challenge: uint8 challenge_chain_sp_vdf: VDFInfo @@ -66,14 +66,14 @@ class NewSignagePointVDF(Streamable): reward_chain_sp_proof: VDFProof -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewEndOfSubSlotVDF(Streamable): end_of_sub_slot_bundle: EndOfSubSlotBundle -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestCompactProofOfTime(Streamable): new_proof_of_time: VDFInfo header_hash: bytes32 @@ -81,8 +81,8 @@ class RequestCompactProofOfTime(Streamable): field_vdf: uint8 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondCompactProofOfTime(Streamable): vdf_info: VDFInfo vdf_proof: VDFProof diff --git a/chia/protocols/wallet_protocol.py b/chia/protocols/wallet_protocol.py index efb0650aa7..a96edca4be 100644 --- a/chia/protocols/wallet_protocol.py +++ b/chia/protocols/wallet_protocol.py @@ -15,15 +15,15 @@ Note: When changing this file, also change protocol_message_types.py, and the pr """ -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestPuzzleSolution(Streamable): coin_name: bytes32 height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PuzzleSolutionResponse(Streamable): coin_name: bytes32 height: uint32 @@ -31,35 +31,35 @@ class PuzzleSolutionResponse(Streamable): solution: Program -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondPuzzleSolution(Streamable): response: PuzzleSolutionResponse -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RejectPuzzleSolution(Streamable): coin_name: bytes32 height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SendTransaction(Streamable): transaction: SpendBundle -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class TransactionAck(Streamable): txid: bytes32 status: uint8 # MempoolInclusionStatus error: Optional[str] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class NewPeakWallet(Streamable): header_hash: bytes32 height: uint32 @@ -67,34 +67,34 @@ class NewPeakWallet(Streamable): fork_point_with_previous_peak: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestBlockHeader(Streamable): height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondBlockHeader(Streamable): header_block: HeaderBlock -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RejectHeaderRequest(Streamable): height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestRemovals(Streamable): height: uint32 header_hash: bytes32 coin_names: Optional[List[bytes32]] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondRemovals(Streamable): height: uint32 header_hash: bytes32 @@ -102,23 +102,23 @@ class RespondRemovals(Streamable): proofs: Optional[List[Tuple[bytes32, bytes]]] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RejectRemovalsRequest(Streamable): height: uint32 header_hash: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestAdditions(Streamable): height: uint32 header_hash: Optional[bytes32] puzzle_hashes: Optional[List[bytes32]] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondAdditions(Streamable): height: uint32 header_hash: bytes32 @@ -126,75 +126,75 @@ class RespondAdditions(Streamable): proofs: Optional[List[Tuple[bytes32, bytes, Optional[bytes]]]] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RejectAdditionsRequest(Streamable): height: uint32 header_hash: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestHeaderBlocks(Streamable): start_height: uint32 end_height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RejectHeaderBlocks(Streamable): start_height: uint32 end_height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondHeaderBlocks(Streamable): start_height: uint32 end_height: uint32 header_blocks: List[HeaderBlock] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class CoinState(Streamable): coin: Coin spent_height: Optional[uint32] created_height: Optional[uint32] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RegisterForPhUpdates(Streamable): puzzle_hashes: List[bytes32] min_height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondToPhUpdates(Streamable): puzzle_hashes: List[bytes32] min_height: uint32 coin_states: List[CoinState] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RegisterForCoinUpdates(Streamable): coin_ids: List[bytes32] min_height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondToCoinUpdates(Streamable): coin_ids: List[bytes32] min_height: uint32 coin_states: List[CoinState] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class CoinStateUpdate(Streamable): height: uint32 fork_height: uint32 @@ -202,27 +202,27 @@ class CoinStateUpdate(Streamable): items: List[CoinState] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestChildren(Streamable): coin_name: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondChildren(Streamable): coin_states: List[CoinState] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RequestSESInfo(Streamable): start_height: uint32 end_height: uint32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RespondSESInfo(Streamable): reward_chain_hash: List[bytes32] heights: List[List[uint32]] diff --git a/chia/seeder/peer_record.py b/chia/seeder/peer_record.py index d307c4065a..2a6f8dde48 100644 --- a/chia/seeder/peer_record.py +++ b/chia/seeder/peer_record.py @@ -6,8 +6,8 @@ from chia.util.ints import uint32, uint64 from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PeerRecord(Streamable): peer_id: str ip_address: str diff --git a/chia/server/address_manager_store.py b/chia/server/address_manager_store.py index 3f725cb3dd..25069133c3 100644 --- a/chia/server/address_manager_store.py +++ b/chia/server/address_manager_store.py @@ -21,8 +21,8 @@ from typing import Any, Dict, List, Optional, Tuple log = logging.getLogger(__name__) -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PeerDataSerialization(Streamable): """ Serializable property bag for the peer data that was previously stored in sqlite. diff --git a/chia/server/outbound_message.py b/chia/server/outbound_message.py index 66861b413a..ba55ca7b8b 100644 --- a/chia/server/outbound_message.py +++ b/chia/server/outbound_message.py @@ -31,8 +31,8 @@ class Delivery(IntEnum): SPECIFIC = 6 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class Message(Streamable): type: uint8 # one of ProtocolMessageTypes # message id diff --git a/chia/simulator/simulator_protocol.py b/chia/simulator/simulator_protocol.py index f512ab04c3..29d8492d17 100644 --- a/chia/simulator/simulator_protocol.py +++ b/chia/simulator/simulator_protocol.py @@ -5,14 +5,14 @@ from chia.util.ints import uint32 from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class FarmNewBlockProtocol(Streamable): puzzle_hash: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class ReorgProtocol(Streamable): old_index: uint32 new_index: uint32 diff --git a/chia/timelord/timelord.py b/chia/timelord/timelord.py index e3fd5ae58a..456b6641d1 100644 --- a/chia/timelord/timelord.py +++ b/chia/timelord/timelord.py @@ -41,8 +41,8 @@ from chia.util.streamable import Streamable, streamable log = logging.getLogger(__name__) -@dataclasses.dataclass(frozen=True) @streamable +@dataclasses.dataclass(frozen=True) class BlueboxProcessData(Streamable): challenge: bytes32 size_bits: uint16 diff --git a/chia/types/blockchain_format/classgroup.py b/chia/types/blockchain_format/classgroup.py index a3bf22ad93..10a83e93e9 100644 --- a/chia/types/blockchain_format/classgroup.py +++ b/chia/types/blockchain_format/classgroup.py @@ -5,8 +5,8 @@ from chia.types.blockchain_format.sized_bytes import bytes100 from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class ClassgroupElement(Streamable): """ Represents a classgroup element (a,b,c) where a, b, and c are 512 bit signed integers. However this is using diff --git a/chia/types/blockchain_format/coin.py b/chia/types/blockchain_format/coin.py index 1ba4e0a04b..0709f62c62 100644 --- a/chia/types/blockchain_format/coin.py +++ b/chia/types/blockchain_format/coin.py @@ -9,8 +9,8 @@ from chia.util.ints import uint64 from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class Coin(Streamable): """ This structure is used in the body for the reward and fees genesis coins. diff --git a/chia/types/blockchain_format/foliage.py b/chia/types/blockchain_format/foliage.py index 043f361b35..412e40ba39 100644 --- a/chia/types/blockchain_format/foliage.py +++ b/chia/types/blockchain_format/foliage.py @@ -10,8 +10,8 @@ from chia.util.ints import uint64 from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class TransactionsInfo(Streamable): # Information that goes along with each transaction block generator_root: bytes32 # sha256 of the block generator in this block @@ -22,8 +22,8 @@ class TransactionsInfo(Streamable): reward_claims_incorporated: List[Coin] # These can be in any order -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class FoliageTransactionBlock(Streamable): # Information that goes along with each transaction block that is relevant for light clients prev_transaction_block_hash: bytes32 @@ -34,8 +34,8 @@ class FoliageTransactionBlock(Streamable): transactions_info_hash: bytes32 -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class FoliageBlockData(Streamable): # Part of the block that is signed by the plot key unfinished_reward_block_hash: bytes32 @@ -45,8 +45,8 @@ class FoliageBlockData(Streamable): extension_data: bytes32 # Used for future updates. Can be any 32 byte value initially -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class Foliage(Streamable): # The entire foliage block, containing signature and the unsigned back pointer # The hash of this is the "header hash". Note that for unfinished blocks, the prev_block_hash diff --git a/chia/types/blockchain_format/pool_target.py b/chia/types/blockchain_format/pool_target.py index 57659c9f63..6d9b126d76 100644 --- a/chia/types/blockchain_format/pool_target.py +++ b/chia/types/blockchain_format/pool_target.py @@ -5,8 +5,8 @@ from chia.util.ints import uint32 from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PoolTarget(Streamable): puzzle_hash: bytes32 max_height: uint32 # A max height of 0 means it is valid forever diff --git a/chia/types/blockchain_format/proof_of_space.py b/chia/types/blockchain_format/proof_of_space.py index 72d90a7a9f..f9d1fc8bdb 100644 --- a/chia/types/blockchain_format/proof_of_space.py +++ b/chia/types/blockchain_format/proof_of_space.py @@ -15,8 +15,8 @@ from chia.util.streamable import Streamable, streamable log = logging.getLogger(__name__) -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class ProofOfSpace(Streamable): challenge: bytes32 pool_public_key: Optional[G1Element] # Only one of these two should be present diff --git a/chia/types/blockchain_format/reward_chain_block.py b/chia/types/blockchain_format/reward_chain_block.py index 9bcb4bf597..515d032c09 100644 --- a/chia/types/blockchain_format/reward_chain_block.py +++ b/chia/types/blockchain_format/reward_chain_block.py @@ -10,8 +10,8 @@ from chia.util.ints import uint8, uint32, uint128 from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RewardChainBlockUnfinished(Streamable): total_iters: uint128 signage_point_index: uint8 @@ -23,8 +23,8 @@ class RewardChainBlockUnfinished(Streamable): reward_chain_sp_signature: G2Element -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RewardChainBlock(Streamable): weight: uint128 height: uint32 diff --git a/chia/types/blockchain_format/slots.py b/chia/types/blockchain_format/slots.py index 0f55073f5d..a230dcf043 100644 --- a/chia/types/blockchain_format/slots.py +++ b/chia/types/blockchain_format/slots.py @@ -10,8 +10,8 @@ from chia.util.ints import uint8, uint64 from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class ChallengeBlockInfo(Streamable): # The hash of this is used as the challenge_hash for the ICC VDF proof_of_space: ProofOfSpace challenge_chain_sp_vdf: Optional[VDFInfo] # Only present if not the first sp @@ -19,8 +19,8 @@ class ChallengeBlockInfo(Streamable): # The hash of this is used as the challen challenge_chain_ip_vdf: VDFInfo -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class ChallengeChainSubSlot(Streamable): challenge_chain_end_of_slot_vdf: VDFInfo infused_challenge_chain_sub_slot_hash: Optional[bytes32] # Only at the end of a slot @@ -29,14 +29,14 @@ class ChallengeChainSubSlot(Streamable): new_difficulty: Optional[uint64] # Only at the end of epoch, sub-epoch, and slot -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class InfusedChallengeChainSubSlot(Streamable): infused_challenge_chain_end_of_slot_vdf: VDFInfo -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RewardChainSubSlot(Streamable): end_of_slot_vdf: VDFInfo challenge_chain_sub_slot_hash: bytes32 @@ -44,8 +44,8 @@ class RewardChainSubSlot(Streamable): deficit: uint8 # 16 or less. usually zero -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SubSlotProofs(Streamable): challenge_chain_slot_proof: VDFProof infused_challenge_chain_slot_proof: Optional[VDFProof] diff --git a/chia/types/blockchain_format/sub_epoch_summary.py b/chia/types/blockchain_format/sub_epoch_summary.py index 4191823743..6a1fc89fa6 100644 --- a/chia/types/blockchain_format/sub_epoch_summary.py +++ b/chia/types/blockchain_format/sub_epoch_summary.py @@ -6,8 +6,8 @@ from chia.util.ints import uint8, uint64 from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SubEpochSummary(Streamable): prev_subepoch_summary_hash: bytes32 reward_chain_hash: bytes32 # hash of reward chain at end of last segment diff --git a/chia/types/blockchain_format/vdf.py b/chia/types/blockchain_format/vdf.py index 4d417502cc..55b47595b9 100644 --- a/chia/types/blockchain_format/vdf.py +++ b/chia/types/blockchain_format/vdf.py @@ -44,16 +44,16 @@ def verify_vdf( ) -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class VDFInfo(Streamable): challenge: bytes32 # Used to generate the discriminant (VDF group) number_of_iterations: uint64 output: ClassgroupElement -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class VDFProof(Streamable): witness_type: uint8 witness: bytes diff --git a/chia/types/coin_record.py b/chia/types/coin_record.py index 85579e687d..b195ad3de0 100644 --- a/chia/types/coin_record.py +++ b/chia/types/coin_record.py @@ -8,8 +8,8 @@ from chia.util.ints import uint32, uint64 from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class CoinRecord(Streamable): """ These are values that correspond to a CoinName that are used diff --git a/chia/types/coin_spend.py b/chia/types/coin_spend.py index b37a61ac08..895fd761c3 100644 --- a/chia/types/coin_spend.py +++ b/chia/types/coin_spend.py @@ -7,8 +7,8 @@ from chia.util.chain_utils import additions_for_solution, fee_for_solution from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class CoinSpend(Streamable): """ This is a rather disparate data structure that validates coin transfers. It's generally populated diff --git a/chia/types/condition_with_args.py b/chia/types/condition_with_args.py index 2222baa3e3..5f0dbce11f 100644 --- a/chia/types/condition_with_args.py +++ b/chia/types/condition_with_args.py @@ -5,8 +5,8 @@ from chia.types.condition_opcodes import ConditionOpcode from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class ConditionWithArgs(Streamable): """ This structure is used to store parsed CLVM conditions diff --git a/chia/types/end_of_slot_bundle.py b/chia/types/end_of_slot_bundle.py index 0e7292b61b..72d708f5e7 100644 --- a/chia/types/end_of_slot_bundle.py +++ b/chia/types/end_of_slot_bundle.py @@ -10,8 +10,8 @@ from chia.types.blockchain_format.slots import ( from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class EndOfSubSlotBundle(Streamable): challenge_chain: ChallengeChainSubSlot infused_challenge_chain: Optional[InfusedChallengeChainSubSlot] diff --git a/chia/types/full_block.py b/chia/types/full_block.py index 25a0772d62..086fb6ea76 100644 --- a/chia/types/full_block.py +++ b/chia/types/full_block.py @@ -11,8 +11,8 @@ from chia.util.ints import uint32 from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class FullBlock(Streamable): # All the information required to validate a block finished_sub_slots: List[EndOfSubSlotBundle] # If first sb diff --git a/chia/types/generator_types.py b/chia/types/generator_types.py index 060f3d85d9..7b86520bf3 100644 --- a/chia/types/generator_types.py +++ b/chia/types/generator_types.py @@ -21,8 +21,8 @@ class CompressorArg: end: int -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class BlockGenerator(Streamable): program: SerializedProgram generator_refs: List[SerializedProgram] diff --git a/chia/types/header_block.py b/chia/types/header_block.py index afc5c5a504..e0b44abd24 100644 --- a/chia/types/header_block.py +++ b/chia/types/header_block.py @@ -8,8 +8,8 @@ from chia.types.end_of_slot_bundle import EndOfSubSlotBundle from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class HeaderBlock(Streamable): # Same as a FullBlock but without TransactionInfo and Generator (but with filter), used by light clients finished_sub_slots: List[EndOfSubSlotBundle] # If first sb diff --git a/chia/types/mempool_item.py b/chia/types/mempool_item.py index 9e89b86cc2..dfd588a7ed 100644 --- a/chia/types/mempool_item.py +++ b/chia/types/mempool_item.py @@ -10,8 +10,8 @@ from chia.util.ints import uint64 from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class MempoolItem(Streamable): spend_bundle: SpendBundle fee: uint64 diff --git a/chia/types/peer_info.py b/chia/types/peer_info.py index 4404b1fc85..7435cb1809 100644 --- a/chia/types/peer_info.py +++ b/chia/types/peer_info.py @@ -6,8 +6,8 @@ from chia.util.ints import uint16, uint64 from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class PeerInfo(Streamable): host: str port: uint16 @@ -59,8 +59,8 @@ class PeerInfo(Streamable): return group -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class TimestampedPeerInfo(Streamable): host: str port: uint16 diff --git a/chia/types/spend_bundle.py b/chia/types/spend_bundle.py index f8e9977cce..5c003838a0 100644 --- a/chia/types/spend_bundle.py +++ b/chia/types/spend_bundle.py @@ -15,8 +15,8 @@ from chia.wallet.util.debug_spend_bundle import debug_spend_bundle from .coin_spend import CoinSpend -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SpendBundle(Streamable): """ This is a list of coins being spent along with their solution programs, and a single diff --git a/chia/types/spend_bundle_conditions.py b/chia/types/spend_bundle_conditions.py index 3bae9b34d2..0175850305 100644 --- a/chia/types/spend_bundle_conditions.py +++ b/chia/types/spend_bundle_conditions.py @@ -8,8 +8,8 @@ from chia.util.streamable import Streamable, streamable # the Spend and SpendBundleConditions classes are mirrors of native types, returned by # run_generator -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class Spend(Streamable): coin_id: bytes32 puzzle_hash: bytes32 @@ -19,8 +19,8 @@ class Spend(Streamable): agg_sig_me: List[Tuple[bytes48, bytes]] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SpendBundleConditions(Streamable): spends: List[Spend] reserve_fee: uint64 diff --git a/chia/types/unfinished_block.py b/chia/types/unfinished_block.py index 491ab364f2..da110a66c1 100644 --- a/chia/types/unfinished_block.py +++ b/chia/types/unfinished_block.py @@ -10,8 +10,8 @@ from chia.util.ints import uint32 from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class UnfinishedBlock(Streamable): # Full block, without the final VDFs finished_sub_slots: List[EndOfSubSlotBundle] # If first sb diff --git a/chia/types/unfinished_header_block.py b/chia/types/unfinished_header_block.py index 39db047c1e..da6f3b436a 100644 --- a/chia/types/unfinished_header_block.py +++ b/chia/types/unfinished_header_block.py @@ -8,8 +8,8 @@ from chia.types.end_of_slot_bundle import EndOfSubSlotBundle from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class UnfinishedHeaderBlock(Streamable): # Same as a FullBlock but without TransactionInfo and Generator, used by light clients finished_sub_slots: List[EndOfSubSlotBundle] # If first sb diff --git a/chia/types/weight_proof.py b/chia/types/weight_proof.py index bb958040ea..47f5b91187 100644 --- a/chia/types/weight_proof.py +++ b/chia/types/weight_proof.py @@ -11,8 +11,8 @@ from chia.util.ints import uint8, uint32, uint64, uint128 from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SubEpochData(Streamable): reward_chain_hash: bytes32 num_blocks_overflow: uint8 @@ -31,8 +31,8 @@ class SubEpochData(Streamable): # total number of challenge blocks == total number of reward chain blocks -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SubSlotData(Streamable): # if infused proof_of_space: Optional[ProofOfSpace] @@ -65,37 +65,37 @@ class SubSlotData(Streamable): return False -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class SubEpochChallengeSegment(Streamable): sub_epoch_n: uint32 sub_slots: List[SubSlotData] rc_slot_end_info: Optional[VDFInfo] # in first segment of each sub_epoch -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) # this is used only for serialization to database class SubEpochSegments(Streamable): challenge_segments: List[SubEpochChallengeSegment] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) # this is used only for serialization to database class RecentChainData(Streamable): recent_chain_data: List[HeaderBlock] -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class ProofBlockHeader(Streamable): finished_sub_slots: List[EndOfSubSlotBundle] reward_chain_block: RewardChainBlock -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class WeightProof(Streamable): sub_epochs: List[SubEpochData] sub_epoch_segments: List[SubEpochChallengeSegment] # sampled sub epoch diff --git a/chia/util/streamable.py b/chia/util/streamable.py index 2602709600..cf545fd483 100644 --- a/chia/util/streamable.py +++ b/chia/util/streamable.py @@ -5,7 +5,7 @@ import io import pprint import sys from enum import Enum -from typing import Any, BinaryIO, Dict, get_type_hints, List, Tuple, Type, TypeVar, Callable, Optional, Iterator +from typing import Any, BinaryIO, Dict, get_type_hints, List, Tuple, Type, TypeVar, Union, Callable, Optional, Iterator from blspy import G1Element, G2Element, PrivateKey from typing_extensions import Literal @@ -14,20 +14,31 @@ from chia.types.blockchain_format.sized_bytes import bytes32 from chia.util.byte_types import hexstr_to_bytes from chia.util.hash import std_hash from chia.util.ints import int64, int512, uint32, uint64, uint128 -from chia.util.type_checking import is_type_List, is_type_SpecificOptional, is_type_Tuple, strictdataclass if sys.version_info < (3, 8): def get_args(t: Type[Any]) -> Tuple[Any, ...]: return getattr(t, "__args__", ()) + def get_origin(t: Type[Any]) -> Optional[Type[Any]]: + return getattr(t, "__origin__", None) + else: - from typing import get_args + from typing import get_args, get_origin pp = pprint.PrettyPrinter(indent=1, width=120, compact=True) + +class StreamableError(Exception): + pass + + +class DefinitionError(StreamableError): + pass + + # TODO: Remove hack, this allows streaming these objects from binary size_hints = { "PrivateKey": PrivateKey.PRIVATE_KEY_SIZE, @@ -48,6 +59,27 @@ big_ints = [uint64, int64, uint128, int512] _T_Streamable = TypeVar("_T_Streamable", bound="Streamable") +# Caches to store the fields and (de)serialization methods for all available streamable classes. +FIELDS_FOR_STREAMABLE_CLASS = {} +STREAM_FUNCTIONS_FOR_STREAMABLE_CLASS = {} +PARSE_FUNCTIONS_FOR_STREAMABLE_CLASS = {} + + +def is_type_List(f_type: Type) -> bool: + return get_origin(f_type) == list or f_type == list + + +def is_type_SpecificOptional(f_type) -> bool: + """ + Returns true for types such as Optional[T], but not Optional, or T. + """ + return get_origin(f_type) == Union and get_args(f_type)[1]() is None + + +def is_type_Tuple(f_type: Type) -> bool: + return get_origin(f_type) == tuple or f_type == tuple + + def dataclass_from_dict(klass, d): """ Converts a dictionary based on a dataclass, into an instance of that dataclass. @@ -124,81 +156,6 @@ def recurse_jsonify(d): return d -STREAM_FUNCTIONS_FOR_STREAMABLE_CLASS = {} -PARSE_FUNCTIONS_FOR_STREAMABLE_CLASS = {} -FIELDS_FOR_STREAMABLE_CLASS = {} - - -def streamable(cls: Any): - """ - This is a decorator for class definitions. It applies the strictdataclass decorator, - which checks all types at construction. It also defines a simple serialization format, - and adds parse, from bytes, stream, and __bytes__ methods. - - The primitives are: - * Sized ints serialized in big endian format, e.g. uint64 - * Sized bytes serialized in big endian format, e.g. bytes32 - * BLS public keys serialized in bls format (48 bytes) - * BLS signatures serialized in bls format (96 bytes) - * bool serialized into 1 byte (0x01 or 0x00) - * bytes serialized as a 4 byte size prefix and then the bytes. - * ConditionOpcode is serialized as a 1 byte value. - * str serialized as a 4 byte size prefix and then the utf-8 representation in bytes. - - An item is one of: - * primitive - * Tuple[item1, .. itemx] - * List[item1, .. itemx] - * Optional[item] - * Custom item - - A streamable must be a Tuple at the root level (although a dataclass is used here instead). - Iters are serialized in the following way: - - 1. A tuple of x items is serialized by appending the serialization of each item. - 2. A List is serialized into a 4 byte size prefix (number of items) and the serialization of each item. - 3. An Optional is serialized into a 1 byte prefix of 0x00 or 0x01, and if it's one, it's followed by the - serialization of the item. - 4. A Custom item is serialized by calling the .parse method, passing in the stream of bytes into it. An example is - a CLVM program. - - All of the constituents must have parse/from_bytes, and stream/__bytes__ and therefore - be of fixed size. For example, int cannot be a constituent since it is not a fixed size, - whereas uint32 can be. - - Furthermore, a get_hash() member is added, which performs a serialization and a sha256. - - This class is used for deterministic serialization and hashing, for consensus critical - objects such as the block header. - - Make sure to use the Streamable class as a parent class when using the streamable decorator, - as it will allow linters to recognize the methods that are added by the decorator. Also, - use the @dataclass(frozen=True) decorator as well, for linters to recognize constructor - arguments. - """ - - cls1 = strictdataclass(cls) - t = type(cls.__name__, (cls1, Streamable), {}) - - stream_functions = [] - parse_functions = [] - try: - hints = get_type_hints(t) - fields = {field.name: hints.get(field.name, field.type) for field in dataclasses.fields(t)} - except Exception: - fields = {} - - FIELDS_FOR_STREAMABLE_CLASS[t] = fields - - for _, f_type in fields.items(): - stream_functions.append(cls.function_to_stream_one_item(f_type)) - parse_functions.append(cls.function_to_parse_one_item(f_type)) - - STREAM_FUNCTIONS_FOR_STREAMABLE_CLASS[t] = stream_functions - PARSE_FUNCTIONS_FOR_STREAMABLE_CLASS[t] = parse_functions - return t - - def parse_bool(f: BinaryIO) -> bool: bool_byte = f.read(1) assert bool_byte is not None and len(bool_byte) == 1 # Checks for EOF @@ -298,7 +255,158 @@ def stream_str(item: Any, f: BinaryIO) -> None: f.write(str_bytes) +def streamable(cls: Any): + """ + This decorator forces correct streamable protocol syntax/usage and populates the caches for types hints and + (de)serialization methods for all members of the class. The correct usage is: + + @streamable + @dataclass(frozen=True) + class Example(Streamable): + ... + + The order how the decorator are applied and the inheritance from Streamable are forced. The explicit inheritance is + required because mypy doesn't analyse the type returned by decorators, so we can't just inherit from inside the + decorator. The dataclass decorator is required to fetch type hints, let mypy validate constructor calls and restrict + direct modification of objects by `frozen=True`. + """ + + correct_usage_string: str = ( + "Correct usage is:\n\n@streamable\n@dataclass(frozen=True)\nclass Example(Streamable):\n ..." + ) + + if not dataclasses.is_dataclass(cls): + raise DefinitionError(f"@dataclass(frozen=True) required first. {correct_usage_string}") + + try: + object.__new__(cls)._streamable_test_if_dataclass_frozen_ = None + except dataclasses.FrozenInstanceError: + pass + else: + raise DefinitionError(f"dataclass needs to be frozen. {correct_usage_string}") + + if not issubclass(cls, Streamable): + raise DefinitionError(f"Streamable inheritance required. {correct_usage_string}") + + stream_functions = [] + parse_functions = [] + try: + hints = get_type_hints(cls) + fields = {field.name: hints.get(field.name, field.type) for field in dataclasses.fields(cls)} + except Exception: + fields = {} + + FIELDS_FOR_STREAMABLE_CLASS[cls] = fields + + for _, f_type in fields.items(): + stream_functions.append(cls.function_to_stream_one_item(f_type)) + parse_functions.append(cls.function_to_parse_one_item(f_type)) + + STREAM_FUNCTIONS_FOR_STREAMABLE_CLASS[cls] = stream_functions + PARSE_FUNCTIONS_FOR_STREAMABLE_CLASS[cls] = parse_functions + return cls + + class Streamable: + """ + This class defines a simple serialization format, and adds methods to parse from/to bytes and json. It also + validates and parses all fields at construction in ยด__post_init__` to make sure all fields have the correct type + and can be streamed/parsed properly. + + The available primitives are: + * Sized ints serialized in big endian format, e.g. uint64 + * Sized bytes serialized in big endian format, e.g. bytes32 + * BLS public keys serialized in bls format (48 bytes) + * BLS signatures serialized in bls format (96 bytes) + * bool serialized into 1 byte (0x01 or 0x00) + * bytes serialized as a 4 byte size prefix and then the bytes. + * ConditionOpcode is serialized as a 1 byte value. + * str serialized as a 4 byte size prefix and then the utf-8 representation in bytes. + + An item is one of: + * primitive + * Tuple[item1, .. itemx] + * List[item1, .. itemx] + * Optional[item] + * Custom item + + A streamable must be a Tuple at the root level (although a dataclass is used here instead). + Iters are serialized in the following way: + + 1. A tuple of x items is serialized by appending the serialization of each item. + 2. A List is serialized into a 4 byte size prefix (number of items) and the serialization of each item. + 3. An Optional is serialized into a 1 byte prefix of 0x00 or 0x01, and if it's one, it's followed by the + serialization of the item. + 4. A Custom item is serialized by calling the .parse method, passing in the stream of bytes into it. An example is + a CLVM program. + + All of the constituents must have parse/from_bytes, and stream/__bytes__ and therefore + be of fixed size. For example, int cannot be a constituent since it is not a fixed size, + whereas uint32 can be. + + Furthermore, a get_hash() member is added, which performs a serialization and a sha256. + + This class is used for deterministic serialization and hashing, for consensus critical + objects such as the block header. + + Make sure to use the streamable decorator when inheriting from the Streamable class to prepare the streaming caches. + """ + + def post_init_parse(self, item: Any, f_name: str, f_type: Type) -> Any: + if is_type_List(f_type): + collected_list: List = [] + inner_type: Type = get_args(f_type)[0] + # wjb assert inner_type != get_args(List)[0] # type: ignore + if not is_type_List(type(item)): + raise ValueError(f"Wrong type for {f_name}, need a list.") + for el in item: + collected_list.append(self.post_init_parse(el, f_name, inner_type)) + return collected_list + if is_type_SpecificOptional(f_type): + if item is None: + return None + else: + inner_type: Type = get_args(f_type)[0] # type: ignore + return self.post_init_parse(item, f_name, inner_type) + if is_type_Tuple(f_type): + collected_list = [] + if not is_type_Tuple(type(item)) and not is_type_List(type(item)): + raise ValueError(f"Wrong type for {f_name}, need a tuple.") + if len(item) != len(get_args(f_type)): + raise ValueError(f"Wrong number of elements in tuple {f_name}.") + for i in range(len(item)): + inner_type = get_args(f_type)[i] + tuple_item = item[i] + collected_list.append(self.post_init_parse(tuple_item, f_name, inner_type)) + return tuple(collected_list) + if not isinstance(item, f_type): + try: + item = f_type(item) + except (TypeError, AttributeError, ValueError): + try: + item = f_type.from_bytes(item) + except Exception: + item = f_type.from_bytes(bytes(item)) + if not isinstance(item, f_type): + raise ValueError(f"Wrong type for {f_name}") + return item + + def __post_init__(self): + try: + fields = FIELDS_FOR_STREAMABLE_CLASS[type(self)] + except Exception: + fields = {} + data = self.__dict__ + for (f_name, f_type) in fields.items(): + if f_name not in data: + raise ValueError(f"Field {f_name} not present") + try: + if not isinstance(data[f_name], f_type): + object.__setattr__(self, f_name, self.post_init_parse(data[f_name], f_name, f_type)) + except TypeError: + # Throws a TypeError because we cannot call isinstance for subscripted generics like Optional[int] + object.__setattr__(self, f_name, self.post_init_parse(data[f_name], f_name, f_type)) + @classmethod def function_to_parse_one_item(cls, f_type: Type) -> Callable[[BinaryIO], Any]: """ diff --git a/chia/util/type_checking.py b/chia/util/type_checking.py deleted file mode 100644 index 8f9adf38e5..0000000000 --- a/chia/util/type_checking.py +++ /dev/null @@ -1,103 +0,0 @@ -import dataclasses -import sys -from typing import Any, List, Optional, Tuple, Type, Union - -if sys.version_info < (3, 8): - - def get_args(t: Type[Any]) -> Tuple[Any, ...]: - return getattr(t, "__args__", ()) - - def get_origin(t: Type[Any]) -> Optional[Type[Any]]: - return getattr(t, "__origin__", None) - -else: - - from typing import get_args, get_origin - - -def is_type_List(f_type: Type) -> bool: - return get_origin(f_type) == list or f_type == list - - -def is_type_SpecificOptional(f_type) -> bool: - """ - Returns true for types such as Optional[T], but not Optional, or T. - """ - return get_origin(f_type) == Union and get_args(f_type)[1]() is None - - -def is_type_Tuple(f_type: Type) -> bool: - return get_origin(f_type) == tuple or f_type == tuple - - -def strictdataclass(cls: Any): - class _Local: - """ - Dataclass where all fields must be type annotated, and type checking is performed - at initialization, even recursively through Lists. Non-annotated fields are ignored. - Also, for any fields which have a type with .from_bytes(bytes) or constructor(bytes), - bytes can be passed in and the type can be constructed. - """ - - def parse_item(self, item: Any, f_name: str, f_type: Type) -> Any: - if is_type_List(f_type): - collected_list: List = [] - inner_type: Type = get_args(f_type)[0] - # wjb assert inner_type != get_args(List)[0] # type: ignore - if not is_type_List(type(item)): - raise ValueError(f"Wrong type for {f_name}, need a list.") - for el in item: - collected_list.append(self.parse_item(el, f_name, inner_type)) - return collected_list - if is_type_SpecificOptional(f_type): - if item is None: - return None - else: - inner_type: Type = get_args(f_type)[0] # type: ignore - return self.parse_item(item, f_name, inner_type) - if is_type_Tuple(f_type): - collected_list = [] - if not is_type_Tuple(type(item)) and not is_type_List(type(item)): - raise ValueError(f"Wrong type for {f_name}, need a tuple.") - if len(item) != len(get_args(f_type)): - raise ValueError(f"Wrong number of elements in tuple {f_name}.") - for i in range(len(item)): - inner_type = get_args(f_type)[i] - tuple_item = item[i] - collected_list.append(self.parse_item(tuple_item, f_name, inner_type)) - return tuple(collected_list) - if not isinstance(item, f_type): - try: - item = f_type(item) - except (TypeError, AttributeError, ValueError): - try: - item = f_type.from_bytes(item) - except Exception: - item = f_type.from_bytes(bytes(item)) - if not isinstance(item, f_type): - raise ValueError(f"Wrong type for {f_name}") - return item - - def __post_init__(self): - try: - fields = self.__annotations__ # pylint: disable=no-member - except Exception: - fields = {} - data = self.__dict__ - for (f_name, f_type) in fields.items(): - if f_name not in data: - raise ValueError(f"Field {f_name} not present") - try: - if not isinstance(data[f_name], f_type): - object.__setattr__(self, f_name, self.parse_item(data[f_name], f_name, f_type)) - except TypeError: - # Throws a TypeError because we cannot call isinstance for subscripted generics like Optional[int] - object.__setattr__(self, f_name, self.parse_item(data[f_name], f_name, f_type)) - - class NoTypeChecking: - __no_type_check__ = True - - cls1 = dataclasses.dataclass(cls, init=False, frozen=True) # type: ignore - if dataclasses.fields(cls1) == (): - return type(cls.__name__, (cls1, _Local, NoTypeChecking), {}) - return type(cls.__name__, (cls1, _Local), {}) diff --git a/chia/wallet/block_record.py b/chia/wallet/block_record.py index 81cc7c57c0..8324592523 100644 --- a/chia/wallet/block_record.py +++ b/chia/wallet/block_record.py @@ -6,8 +6,8 @@ from chia.types.header_block import HeaderBlock from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class HeaderBlockRecord(Streamable): """ These are values that are stored in the wallet database, corresponding to information diff --git a/chia/wallet/cat_wallet/cat_info.py b/chia/wallet/cat_wallet/cat_info.py index 9e2d6eadf5..78c9fec5ed 100644 --- a/chia/wallet/cat_wallet/cat_info.py +++ b/chia/wallet/cat_wallet/cat_info.py @@ -7,8 +7,8 @@ from chia.wallet.lineage_proof import LineageProof from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class CATInfo(Streamable): limitations_program_hash: bytes32 my_tail: Optional[Program] # this is the program @@ -16,8 +16,8 @@ class CATInfo(Streamable): # We used to store all of the lineage proofs here but it was very slow to serialize for a lot of transactions # so we moved it to CATLineageStore. We keep this around for migration purposes. -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class LegacyCATInfo(Streamable): limitations_program_hash: bytes32 my_tail: Optional[Program] # this is the program diff --git a/chia/wallet/did_wallet/did_info.py b/chia/wallet/did_wallet/did_info.py index 5b57402afa..d4a83a77e4 100644 --- a/chia/wallet/did_wallet/did_info.py +++ b/chia/wallet/did_wallet/did_info.py @@ -9,8 +9,8 @@ from chia.types.blockchain_format.program import Program from chia.types.blockchain_format.coin import Coin -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class DIDInfo(Streamable): origin_coin: Optional[Coin] # Coin ID of this coin is our DID backup_ids: List[bytes] diff --git a/chia/wallet/lineage_proof.py b/chia/wallet/lineage_proof.py index 177f2d1274..ec5d3119d3 100644 --- a/chia/wallet/lineage_proof.py +++ b/chia/wallet/lineage_proof.py @@ -7,8 +7,8 @@ from chia.util.ints import uint64 from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class LineageProof(Streamable): parent_name: Optional[bytes32] = None inner_puzzle_hash: Optional[bytes32] = None diff --git a/chia/wallet/rl_wallet/rl_wallet.py b/chia/wallet/rl_wallet/rl_wallet.py index e51fe841c7..3db219e1b1 100644 --- a/chia/wallet/rl_wallet/rl_wallet.py +++ b/chia/wallet/rl_wallet/rl_wallet.py @@ -34,8 +34,8 @@ from chia.wallet.wallet_coin_record import WalletCoinRecord from chia.wallet.wallet_info import WalletInfo -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class RLInfo(Streamable): type: str admin_pubkey: Optional[bytes] diff --git a/chia/wallet/settings/settings_objects.py b/chia/wallet/settings/settings_objects.py index 9878a2c4ee..27f6f92c60 100644 --- a/chia/wallet/settings/settings_objects.py +++ b/chia/wallet/settings/settings_objects.py @@ -3,8 +3,8 @@ from dataclasses import dataclass from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class BackupInitialized(Streamable): """ Stores user decision regarding import of backup info diff --git a/chia/wallet/trade_record.py b/chia/wallet/trade_record.py index 278181b81d..08c56bb4bb 100644 --- a/chia/wallet/trade_record.py +++ b/chia/wallet/trade_record.py @@ -9,8 +9,8 @@ from chia.wallet.trading.offer import Offer from chia.wallet.trading.trade_status import TradeStatus -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class TradeRecord(Streamable): """ Used for storing transaction data and status in wallets. diff --git a/chia/wallet/transaction_record.py b/chia/wallet/transaction_record.py index f43907f1ba..b13b6d3660 100644 --- a/chia/wallet/transaction_record.py +++ b/chia/wallet/transaction_record.py @@ -12,8 +12,8 @@ from chia.util.streamable import Streamable, streamable from chia.wallet.util.transaction_type import TransactionType -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class TransactionRecord(Streamable): """ Used for storing transaction data and status in wallets. diff --git a/chia/wallet/wallet_info.py b/chia/wallet/wallet_info.py index 4567540f6d..1d02bb6e72 100644 --- a/chia/wallet/wallet_info.py +++ b/chia/wallet/wallet_info.py @@ -5,8 +5,8 @@ from chia.util.ints import uint8, uint32 from chia.util.streamable import Streamable, streamable -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class WalletInfo(Streamable): """ This object represents the wallet data as it is stored in DB. @@ -24,8 +24,8 @@ class WalletInfo(Streamable): data: str -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class WalletInfoBackup(Streamable): """ Used for transforming list of WalletInfo objects into bytes. diff --git a/mypy.ini b/mypy.ini index 4c3283cc77..4c3f96cbfd 100644 --- a/mypy.ini +++ b/mypy.ini @@ -17,7 +17,7 @@ no_implicit_reexport = True strict_equality = True # list created by: venv/bin/mypy | sed -n 's/.py:.*//p' | sort | uniq | tr '/' '.' | tr '\n' ',' -[mypy-benchmarks.block_ref,benchmarks.block_store,benchmarks.coin_store,benchmarks.utils,build_scripts.installer-version,chia.clvm.spend_sim,chia.cmds.configure,chia.cmds.db,chia.cmds.db_upgrade_func,chia.cmds.farm_funcs,chia.cmds.init,chia.cmds.init_funcs,chia.cmds.keys,chia.cmds.keys_funcs,chia.cmds.passphrase,chia.cmds.passphrase_funcs,chia.cmds.plotnft,chia.cmds.plotnft_funcs,chia.cmds.plots,chia.cmds.plotters,chia.cmds.show,chia.cmds.start_funcs,chia.cmds.wallet,chia.cmds.wallet_funcs,chia.consensus.block_body_validation,chia.consensus.blockchain,chia.consensus.blockchain_interface,chia.consensus.block_creation,chia.consensus.block_header_validation,chia.consensus.block_record,chia.consensus.block_root_validation,chia.consensus.coinbase,chia.consensus.constants,chia.consensus.difficulty_adjustment,chia.consensus.get_block_challenge,chia.consensus.multiprocess_validation,chia.consensus.pos_quality,chia.consensus.vdf_info_computation,chia.daemon.client,chia.daemon.keychain_proxy,chia.daemon.keychain_server,chia.daemon.server,chia.farmer.farmer,chia.farmer.farmer_api,chia.full_node.block_height_map,chia.full_node.block_store,chia.full_node.bundle_tools,chia.full_node.coin_store,chia.full_node.full_node,chia.full_node.full_node_api,chia.full_node.full_node_store,chia.full_node.generator,chia.full_node.hint_store,chia.full_node.lock_queue,chia.full_node.mempool,chia.full_node.mempool_check_conditions,chia.full_node.mempool_manager,chia.full_node.pending_tx_cache,chia.full_node.sync_store,chia.full_node.weight_proof,chia.harvester.harvester,chia.harvester.harvester_api,chia.introducer.introducer,chia.introducer.introducer_api,chia.plotters.bladebit,chia.plotters.chiapos,chia.plotters.install_plotter,chia.plotters.madmax,chia.plotters.plotters,chia.plotters.plotters_util,chia.plotting.check_plots,chia.plotting.create_plots,chia.plotting.manager,chia.plotting.util,chia.pools.pool_config,chia.pools.pool_puzzles,chia.pools.pool_wallet,chia.pools.pool_wallet_info,chia.protocols.pool_protocol,chia.rpc.crawler_rpc_api,chia.rpc.farmer_rpc_api,chia.rpc.farmer_rpc_client,chia.rpc.full_node_rpc_api,chia.rpc.full_node_rpc_client,chia.rpc.harvester_rpc_api,chia.rpc.harvester_rpc_client,chia.rpc.rpc_client,chia.rpc.rpc_server,chia.rpc.timelord_rpc_api,chia.rpc.util,chia.rpc.wallet_rpc_api,chia.rpc.wallet_rpc_client,chia.seeder.crawler,chia.seeder.crawler_api,chia.seeder.crawl_store,chia.seeder.dns_server,chia.seeder.peer_record,chia.seeder.start_crawler,chia.server.address_manager,chia.server.address_manager_store,chia.server.connection_utils,chia.server.introducer_peers,chia.server.node_discovery,chia.server.peer_store_resolver,chia.server.rate_limits,chia.server.reconnect_task,chia.server.server,chia.server.ssl_context,chia.server.start_farmer,chia.server.start_full_node,chia.server.start_harvester,chia.server.start_introducer,chia.server.start_service,chia.server.start_timelord,chia.server.start_wallet,chia.server.upnp,chia.server.ws_connection,chia.simulator.full_node_simulator,chia.simulator.start_simulator,chia.ssl.create_ssl,chia.timelord.iters_from_block,chia.timelord.timelord,chia.timelord.timelord_api,chia.timelord.timelord_launcher,chia.timelord.timelord_state,chia.types.announcement,chia.types.blockchain_format.classgroup,chia.types.blockchain_format.coin,chia.types.blockchain_format.program,chia.types.blockchain_format.proof_of_space,chia.types.blockchain_format.tree_hash,chia.types.blockchain_format.vdf,chia.types.full_block,chia.types.header_block,chia.types.mempool_item,chia.types.name_puzzle_condition,chia.types.peer_info,chia.types.spend_bundle,chia.types.transaction_queue_entry,chia.types.unfinished_block,chia.types.unfinished_header_block,chia.util.api_decorators,chia.util.block_cache,chia.util.byte_types,chia.util.cached_bls,chia.util.check_fork_next_block,chia.util.chia_logging,chia.util.config,chia.util.db_wrapper,chia.util.dump_keyring,chia.util.file_keyring,chia.util.files,chia.util.hash,chia.util.ints,chia.util.json_util,chia.util.keychain,chia.util.keyring_wrapper,chia.util.log_exceptions,chia.util.lru_cache,chia.util.make_test_constants,chia.util.merkle_set,chia.util.network,chia.util.partial_func,chia.util.pip_import,chia.util.profiler,chia.util.safe_cancel_task,chia.util.service_groups,chia.util.ssl_check,chia.util.streamable,chia.util.struct_stream,chia.util.type_checking,chia.util.validate_alert,chia.wallet.block_record,chia.wallet.cat_wallet.cat_utils,chia.wallet.cat_wallet.cat_wallet,chia.wallet.cat_wallet.lineage_store,chia.wallet.chialisp,chia.wallet.did_wallet.did_wallet,chia.wallet.did_wallet.did_wallet_puzzles,chia.wallet.key_val_store,chia.wallet.lineage_proof,chia.wallet.payment,chia.wallet.puzzles.load_clvm,chia.wallet.puzzles.p2_conditions,chia.wallet.puzzles.p2_delegated_conditions,chia.wallet.puzzles.p2_delegated_puzzle,chia.wallet.puzzles.p2_delegated_puzzle_or_hidden_puzzle,chia.wallet.puzzles.p2_m_of_n_delegate_direct,chia.wallet.puzzles.p2_puzzle_hash,chia.wallet.puzzles.prefarm.spend_prefarm,chia.wallet.puzzles.puzzle_utils,chia.wallet.puzzles.rom_bootstrap_generator,chia.wallet.puzzles.singleton_top_layer,chia.wallet.puzzles.tails,chia.wallet.rl_wallet.rl_wallet,chia.wallet.rl_wallet.rl_wallet_puzzles,chia.wallet.secret_key_store,chia.wallet.settings.user_settings,chia.wallet.trade_manager,chia.wallet.trade_record,chia.wallet.trading.offer,chia.wallet.trading.trade_store,chia.wallet.transaction_record,chia.wallet.util.debug_spend_bundle,chia.wallet.util.new_peak_queue,chia.wallet.util.peer_request_cache,chia.wallet.util.wallet_sync_utils,chia.wallet.wallet,chia.wallet.wallet_action_store,chia.wallet.wallet_blockchain,chia.wallet.wallet_coin_store,chia.wallet.wallet_interested_store,chia.wallet.wallet_node,chia.wallet.wallet_node_api,chia.wallet.wallet_pool_store,chia.wallet.wallet_puzzle_store,chia.wallet.wallet_state_manager,chia.wallet.wallet_sync_store,chia.wallet.wallet_transaction_store,chia.wallet.wallet_user_store,chia.wallet.wallet_weight_proof_handler,installhelper,tests.blockchain.blockchain_test_utils,tests.blockchain.test_blockchain,tests.blockchain.test_blockchain_transactions,tests.block_tools,tests.build-init-files,tests.build-workflows,tests.clvm.coin_store,tests.clvm.test_chialisp_deserialization,tests.clvm.test_clvm_compilation,tests.clvm.test_program,tests.clvm.test_puzzle_compression,tests.clvm.test_puzzles,tests.clvm.test_serialized_program,tests.clvm.test_singletons,tests.clvm.test_spend_sim,tests.conftest,tests.connection_utils,tests.core.cmds.test_keys,tests.core.consensus.test_pot_iterations,tests.core.custom_types.test_coin,tests.core.custom_types.test_proof_of_space,tests.core.custom_types.test_spend_bundle,tests.core.daemon.test_daemon,tests.core.full_node.full_sync.test_full_sync,tests.core.full_node.stores.test_block_store,tests.core.full_node.stores.test_coin_store,tests.core.full_node.stores.test_full_node_store,tests.core.full_node.stores.test_hint_store,tests.core.full_node.stores.test_sync_store,tests.core.full_node.test_address_manager,tests.core.full_node.test_block_height_map,tests.core.full_node.test_conditions,tests.core.full_node.test_full_node,tests.core.full_node.test_mempool,tests.core.full_node.test_mempool_performance,tests.core.full_node.test_node_load,tests.core.full_node.test_peer_store_resolver,tests.core.full_node.test_performance,tests.core.full_node.test_transactions,tests.core.make_block_generator,tests.core.node_height,tests.core.server.test_dos,tests.core.server.test_rate_limits,tests.core.ssl.test_ssl,tests.core.test_cost_calculation,tests.core.test_crawler_rpc,tests.core.test_daemon_rpc,tests.core.test_db_conversion,tests.core.test_farmer_harvester_rpc,tests.core.test_filter,tests.core.test_full_node_rpc,tests.core.test_merkle_set,tests.core.test_setproctitle,tests.core.util.test_cached_bls,tests.core.util.test_config,tests.core.util.test_file_keyring_synchronization,tests.core.util.test_files,tests.core.util.test_keychain,tests.core.util.test_keyring_wrapper,tests.core.util.test_lru_cache,tests.core.util.test_significant_bits,tests.core.util.test_streamable,tests.core.util.test_type_checking,tests.farmer_harvester.test_farmer_harvester,tests.generator.test_compression,tests.generator.test_generator_types,tests.generator.test_list_to_batches,tests.generator.test_rom,tests.generator.test_scan,tests.plotting.test_plot_manager,tests.pools.test_pool_cmdline,tests.pools.test_pool_config,tests.pools.test_pool_puzzles_lifecycle,tests.pools.test_pool_rpc,tests.pools.test_wallet_pool_store,tests.setup_nodes,tests.setup_services,tests.simulation.test_simulation,tests.time_out_assert,tests.tools.test_full_sync,tests.tools.test_run_block,tests.util.alert_server,tests.util.benchmark_cost,tests.util.blockchain,tests.util.build_network_protocol_files,tests.util.db_connection,tests.util.generator_tools_testing,tests.util.keyring,tests.util.key_tool,tests.util.misc,tests.util.network,tests.util.rpc,tests.util.test_full_block_utils,tests.util.test_lock_queue,tests.util.test_network_protocol_files,tests.util.test_struct_stream,tests.wallet.cat_wallet.test_cat_lifecycle,tests.wallet.cat_wallet.test_cat_wallet,tests.wallet.cat_wallet.test_offer_lifecycle,tests.wallet.cat_wallet.test_trades,tests.wallet.did_wallet.test_did,tests.wallet.did_wallet.test_did_rpc,tests.wallet.rl_wallet.test_rl_rpc,tests.wallet.rl_wallet.test_rl_wallet,tests.wallet.rpc.test_wallet_rpc,tests.wallet.simple_sync.test_simple_sync_protocol,tests.wallet.sync.test_wallet_sync,tests.wallet.test_bech32m,tests.wallet.test_chialisp,tests.wallet.test_puzzle_store,tests.wallet.test_singleton,tests.wallet.test_singleton_lifecycle,tests.wallet.test_singleton_lifecycle_fast,tests.wallet.test_taproot,tests.wallet.test_wallet,tests.wallet.test_wallet_blockchain,tests.wallet.test_wallet_interested_store,tests.wallet.test_wallet_key_val_store,tests.wallet.test_wallet_user_store,tests.wallet_tools,tests.weight_proof.test_weight_proof,tools.analyze-chain,tools.run_block,tools.test_full_sync] +[mypy-benchmarks.block_ref,benchmarks.block_store,benchmarks.coin_store,benchmarks.utils,build_scripts.installer-version,chia.clvm.spend_sim,chia.cmds.configure,chia.cmds.db,chia.cmds.db_upgrade_func,chia.cmds.farm_funcs,chia.cmds.init,chia.cmds.init_funcs,chia.cmds.keys,chia.cmds.keys_funcs,chia.cmds.passphrase,chia.cmds.passphrase_funcs,chia.cmds.plotnft,chia.cmds.plotnft_funcs,chia.cmds.plots,chia.cmds.plotters,chia.cmds.show,chia.cmds.start_funcs,chia.cmds.wallet,chia.cmds.wallet_funcs,chia.consensus.block_body_validation,chia.consensus.blockchain,chia.consensus.blockchain_interface,chia.consensus.block_creation,chia.consensus.block_header_validation,chia.consensus.block_record,chia.consensus.block_root_validation,chia.consensus.coinbase,chia.consensus.constants,chia.consensus.difficulty_adjustment,chia.consensus.get_block_challenge,chia.consensus.multiprocess_validation,chia.consensus.pos_quality,chia.consensus.vdf_info_computation,chia.daemon.client,chia.daemon.keychain_proxy,chia.daemon.keychain_server,chia.daemon.server,chia.farmer.farmer,chia.farmer.farmer_api,chia.full_node.block_height_map,chia.full_node.block_store,chia.full_node.bundle_tools,chia.full_node.coin_store,chia.full_node.full_node,chia.full_node.full_node_api,chia.full_node.full_node_store,chia.full_node.generator,chia.full_node.hint_store,chia.full_node.lock_queue,chia.full_node.mempool,chia.full_node.mempool_check_conditions,chia.full_node.mempool_manager,chia.full_node.pending_tx_cache,chia.full_node.sync_store,chia.full_node.weight_proof,chia.harvester.harvester,chia.harvester.harvester_api,chia.introducer.introducer,chia.introducer.introducer_api,chia.plotters.bladebit,chia.plotters.chiapos,chia.plotters.install_plotter,chia.plotters.madmax,chia.plotters.plotters,chia.plotters.plotters_util,chia.plotting.check_plots,chia.plotting.create_plots,chia.plotting.manager,chia.plotting.util,chia.pools.pool_config,chia.pools.pool_puzzles,chia.pools.pool_wallet,chia.pools.pool_wallet_info,chia.protocols.pool_protocol,chia.rpc.crawler_rpc_api,chia.rpc.farmer_rpc_api,chia.rpc.farmer_rpc_client,chia.rpc.full_node_rpc_api,chia.rpc.full_node_rpc_client,chia.rpc.harvester_rpc_api,chia.rpc.harvester_rpc_client,chia.rpc.rpc_client,chia.rpc.rpc_server,chia.rpc.timelord_rpc_api,chia.rpc.util,chia.rpc.wallet_rpc_api,chia.rpc.wallet_rpc_client,chia.seeder.crawler,chia.seeder.crawler_api,chia.seeder.crawl_store,chia.seeder.dns_server,chia.seeder.peer_record,chia.seeder.start_crawler,chia.server.address_manager,chia.server.address_manager_store,chia.server.connection_utils,chia.server.introducer_peers,chia.server.node_discovery,chia.server.peer_store_resolver,chia.server.rate_limits,chia.server.reconnect_task,chia.server.server,chia.server.ssl_context,chia.server.start_farmer,chia.server.start_full_node,chia.server.start_harvester,chia.server.start_introducer,chia.server.start_service,chia.server.start_timelord,chia.server.start_wallet,chia.server.upnp,chia.server.ws_connection,chia.simulator.full_node_simulator,chia.simulator.start_simulator,chia.ssl.create_ssl,chia.timelord.iters_from_block,chia.timelord.timelord,chia.timelord.timelord_api,chia.timelord.timelord_launcher,chia.timelord.timelord_state,chia.types.announcement,chia.types.blockchain_format.classgroup,chia.types.blockchain_format.coin,chia.types.blockchain_format.program,chia.types.blockchain_format.proof_of_space,chia.types.blockchain_format.tree_hash,chia.types.blockchain_format.vdf,chia.types.full_block,chia.types.header_block,chia.types.mempool_item,chia.types.name_puzzle_condition,chia.types.peer_info,chia.types.spend_bundle,chia.types.transaction_queue_entry,chia.types.unfinished_block,chia.types.unfinished_header_block,chia.util.api_decorators,chia.util.block_cache,chia.util.byte_types,chia.util.cached_bls,chia.util.check_fork_next_block,chia.util.chia_logging,chia.util.config,chia.util.db_wrapper,chia.util.dump_keyring,chia.util.file_keyring,chia.util.files,chia.util.hash,chia.util.ints,chia.util.json_util,chia.util.keychain,chia.util.keyring_wrapper,chia.util.log_exceptions,chia.util.lru_cache,chia.util.make_test_constants,chia.util.merkle_set,chia.util.network,chia.util.partial_func,chia.util.pip_import,chia.util.profiler,chia.util.safe_cancel_task,chia.util.service_groups,chia.util.ssl_check,chia.util.streamable,chia.util.struct_stream,chia.util.validate_alert,chia.wallet.block_record,chia.wallet.cat_wallet.cat_utils,chia.wallet.cat_wallet.cat_wallet,chia.wallet.cat_wallet.lineage_store,chia.wallet.chialisp,chia.wallet.did_wallet.did_wallet,chia.wallet.did_wallet.did_wallet_puzzles,chia.wallet.key_val_store,chia.wallet.lineage_proof,chia.wallet.payment,chia.wallet.puzzles.load_clvm,chia.wallet.puzzles.p2_conditions,chia.wallet.puzzles.p2_delegated_conditions,chia.wallet.puzzles.p2_delegated_puzzle,chia.wallet.puzzles.p2_delegated_puzzle_or_hidden_puzzle,chia.wallet.puzzles.p2_m_of_n_delegate_direct,chia.wallet.puzzles.p2_puzzle_hash,chia.wallet.puzzles.prefarm.spend_prefarm,chia.wallet.puzzles.puzzle_utils,chia.wallet.puzzles.rom_bootstrap_generator,chia.wallet.puzzles.singleton_top_layer,chia.wallet.puzzles.tails,chia.wallet.rl_wallet.rl_wallet,chia.wallet.rl_wallet.rl_wallet_puzzles,chia.wallet.secret_key_store,chia.wallet.settings.user_settings,chia.wallet.trade_manager,chia.wallet.trade_record,chia.wallet.trading.offer,chia.wallet.trading.trade_store,chia.wallet.transaction_record,chia.wallet.util.debug_spend_bundle,chia.wallet.util.new_peak_queue,chia.wallet.util.peer_request_cache,chia.wallet.util.wallet_sync_utils,chia.wallet.wallet,chia.wallet.wallet_action_store,chia.wallet.wallet_blockchain,chia.wallet.wallet_coin_store,chia.wallet.wallet_interested_store,chia.wallet.wallet_node,chia.wallet.wallet_node_api,chia.wallet.wallet_pool_store,chia.wallet.wallet_puzzle_store,chia.wallet.wallet_state_manager,chia.wallet.wallet_sync_store,chia.wallet.wallet_transaction_store,chia.wallet.wallet_user_store,chia.wallet.wallet_weight_proof_handler,installhelper,tests.blockchain.blockchain_test_utils,tests.blockchain.test_blockchain,tests.blockchain.test_blockchain_transactions,tests.block_tools,tests.build-init-files,tests.build-workflows,tests.clvm.coin_store,tests.clvm.test_chialisp_deserialization,tests.clvm.test_clvm_compilation,tests.clvm.test_program,tests.clvm.test_puzzle_compression,tests.clvm.test_puzzles,tests.clvm.test_serialized_program,tests.clvm.test_singletons,tests.clvm.test_spend_sim,tests.conftest,tests.connection_utils,tests.core.cmds.test_keys,tests.core.consensus.test_pot_iterations,tests.core.custom_types.test_coin,tests.core.custom_types.test_proof_of_space,tests.core.custom_types.test_spend_bundle,tests.core.daemon.test_daemon,tests.core.full_node.full_sync.test_full_sync,tests.core.full_node.stores.test_block_store,tests.core.full_node.stores.test_coin_store,tests.core.full_node.stores.test_full_node_store,tests.core.full_node.stores.test_hint_store,tests.core.full_node.stores.test_sync_store,tests.core.full_node.test_address_manager,tests.core.full_node.test_block_height_map,tests.core.full_node.test_conditions,tests.core.full_node.test_full_node,tests.core.full_node.test_mempool,tests.core.full_node.test_mempool_performance,tests.core.full_node.test_node_load,tests.core.full_node.test_peer_store_resolver,tests.core.full_node.test_performance,tests.core.full_node.test_transactions,tests.core.make_block_generator,tests.core.node_height,tests.core.server.test_dos,tests.core.server.test_rate_limits,tests.core.ssl.test_ssl,tests.core.test_cost_calculation,tests.core.test_crawler_rpc,tests.core.test_daemon_rpc,tests.core.test_db_conversion,tests.core.test_farmer_harvester_rpc,tests.core.test_filter,tests.core.test_full_node_rpc,tests.core.test_merkle_set,tests.core.test_setproctitle,tests.core.util.test_cached_bls,tests.core.util.test_config,tests.core.util.test_file_keyring_synchronization,tests.core.util.test_files,tests.core.util.test_keychain,tests.core.util.test_keyring_wrapper,tests.core.util.test_lru_cache,tests.core.util.test_significant_bits,tests.core.util.test_streamable,tests.farmer_harvester.test_farmer_harvester,tests.generator.test_compression,tests.generator.test_generator_types,tests.generator.test_list_to_batches,tests.generator.test_rom,tests.generator.test_scan,tests.plotting.test_plot_manager,tests.pools.test_pool_cmdline,tests.pools.test_pool_config,tests.pools.test_pool_puzzles_lifecycle,tests.pools.test_pool_rpc,tests.pools.test_wallet_pool_store,tests.setup_nodes,tests.setup_services,tests.simulation.test_simulation,tests.time_out_assert,tests.tools.test_full_sync,tests.tools.test_run_block,tests.util.alert_server,tests.util.benchmark_cost,tests.util.blockchain,tests.util.build_network_protocol_files,tests.util.db_connection,tests.util.generator_tools_testing,tests.util.keyring,tests.util.key_tool,tests.util.misc,tests.util.network,tests.util.rpc,tests.util.test_full_block_utils,tests.util.test_lock_queue,tests.util.test_network_protocol_files,tests.util.test_struct_stream,tests.wallet.cat_wallet.test_cat_lifecycle,tests.wallet.cat_wallet.test_cat_wallet,tests.wallet.cat_wallet.test_offer_lifecycle,tests.wallet.cat_wallet.test_trades,tests.wallet.did_wallet.test_did,tests.wallet.did_wallet.test_did_rpc,tests.wallet.rl_wallet.test_rl_rpc,tests.wallet.rl_wallet.test_rl_wallet,tests.wallet.rpc.test_wallet_rpc,tests.wallet.simple_sync.test_simple_sync_protocol,tests.wallet.sync.test_wallet_sync,tests.wallet.test_bech32m,tests.wallet.test_chialisp,tests.wallet.test_puzzle_store,tests.wallet.test_singleton,tests.wallet.test_singleton_lifecycle,tests.wallet.test_singleton_lifecycle_fast,tests.wallet.test_taproot,tests.wallet.test_wallet,tests.wallet.test_wallet_blockchain,tests.wallet.test_wallet_interested_store,tests.wallet.test_wallet_key_val_store,tests.wallet.test_wallet_user_store,tests.wallet_tools,tests.weight_proof.test_weight_proof,tools.analyze-chain,tools.run_block,tools.test_full_sync] disallow_any_generics = False disallow_subclassing_any = False disallow_untyped_calls = False diff --git a/tests/core/util/test_streamable.py b/tests/core/util/test_streamable.py index 5562d03d15..65b3255212 100644 --- a/tests/core/util/test_streamable.py +++ b/tests/core/util/test_streamable.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import List, Optional, Tuple +from typing import Dict, List, Optional, Tuple import io import pytest @@ -14,6 +14,7 @@ from chia.types.full_block import FullBlock from chia.types.weight_proof import SubEpochChallengeSegment from chia.util.ints import uint8, uint32, uint64 from chia.util.streamable import ( + DefinitionError, Streamable, streamable, parse_bool, @@ -25,13 +26,147 @@ from chia.util.streamable import ( parse_tuple, parse_size_hints, parse_str, + is_type_List, + is_type_SpecificOptional, ) from tests.setup_nodes import test_constants -def test_basic(): +def test_int_not_supported() -> None: + with raises(NotImplementedError): + + @streamable + @dataclass(frozen=True) + class TestClassInt(Streamable): + a: int + + +def test_float_not_supported() -> None: + with raises(NotImplementedError): + + @streamable + @dataclass(frozen=True) + class TestClassFloat(Streamable): + a: float + + +def test_dict_not_suppported() -> None: + with raises(NotImplementedError): + + @streamable + @dataclass(frozen=True) + class TestClassDict(Streamable): + a: Dict[str, str] + + +def test_pure_dataclass_not_supported() -> None: @dataclass(frozen=True) + class DataClassOnly: + a: uint8 + + with raises(NotImplementedError): + + @streamable + @dataclass(frozen=True) + class TestClassDataclass(Streamable): + a: DataClassOnly + + +def test_plain_class_not_supported() -> None: + class PlainClass: + a: uint8 + + with raises(NotImplementedError): + + @streamable + @dataclass(frozen=True) + class TestClassPlain(Streamable): + a: PlainClass + + +def test_basic_list(): + a = [1, 2, 3] + assert is_type_List(type(a)) + assert is_type_List(List) + assert is_type_List(List[int]) + assert is_type_List(List[uint8]) + assert is_type_List(list) + assert not is_type_List(Tuple) + assert not is_type_List(tuple) + assert not is_type_List(dict) + + +def test_not_lists(): + assert not is_type_List(Dict) + + +def test_basic_optional(): + assert is_type_SpecificOptional(Optional[int]) + assert is_type_SpecificOptional(Optional[Optional[int]]) + assert not is_type_SpecificOptional(List[int]) + + +def test_StrictDataClass(): @streamable + @dataclass(frozen=True) + class TestClass1(Streamable): + a: uint8 + b: str + + good: TestClass1 = TestClass1(24, "!@12") + assert TestClass1.__name__ == "TestClass1" + assert good + assert good.a == 24 + assert good.b == "!@12" + good2 = TestClass1(52, bytes([1, 2, 3])) + assert good2.b == str(bytes([1, 2, 3])) + + +def test_StrictDataClassBad(): + @streamable + @dataclass(frozen=True) + class TestClass2(Streamable): + a: uint8 + b = 0 + + assert TestClass2(25) + + with raises(TypeError): + TestClass2(1, 2) # pylint: disable=too-many-function-args + + +def test_StrictDataClassLists(): + @streamable + @dataclass(frozen=True) + class TestClass(Streamable): + a: List[uint8] + b: List[List[uint8]] + + assert TestClass([1, 2, 3], [[uint8(200), uint8(25)], [uint8(25)]]) + + with raises(ValueError): + TestClass({"1": 1}, [[uint8(200), uint8(25)], [uint8(25)]]) + + with raises(ValueError): + TestClass([1, 2, 3], [uint8(200), uint8(25)]) + + +def test_StrictDataClassOptional(): + @streamable + @dataclass(frozen=True) + class TestClass(Streamable): + a: Optional[uint8] + b: Optional[uint8] + c: Optional[Optional[uint8]] + d: Optional[Optional[uint8]] + + good = TestClass(12, None, 13, None) + assert good + + +def test_basic(): + @streamable + @dataclass(frozen=True) class TestClass(Streamable): a: uint32 b: uint32 @@ -48,8 +183,8 @@ def test_basic(): def test_variable_size(): - @dataclass(frozen=True) @streamable + @dataclass(frozen=True) class TestClass2(Streamable): a: uint32 b: uint32 @@ -60,8 +195,8 @@ def test_variable_size(): with raises(NotImplementedError): - @dataclass(frozen=True) @streamable + @dataclass(frozen=True) class TestClass3(Streamable): a: int @@ -72,8 +207,8 @@ def test_json(bt): assert FullBlock.from_json_dict(dict_block) == block -@dataclass(frozen=True) @streamable +@dataclass(frozen=True) class OptionalTestClass(Streamable): a: Optional[str] b: Optional[bool] @@ -99,13 +234,13 @@ def test_optional_json(a: Optional[str], b: Optional[bool], c: Optional[List[Opt def test_recursive_json(): - @dataclass(frozen=True) @streamable + @dataclass(frozen=True) class TestClass1(Streamable): a: List[uint32] - @dataclass(frozen=True) @streamable + @dataclass(frozen=True) class TestClass2(Streamable): a: uint32 b: List[Optional[List[TestClass1]]] @@ -130,8 +265,8 @@ def test_ambiguous_deserialization_optionals(): with raises(AssertionError): SubEpochChallengeSegment.from_bytes(b"\x00\x00\x00\x03\xff\xff\xff\xff") - @dataclass(frozen=True) @streamable + @dataclass(frozen=True) class TestClassOptional(Streamable): a: Optional[uint8] @@ -144,8 +279,8 @@ def test_ambiguous_deserialization_optionals(): def test_ambiguous_deserialization_int(): - @dataclass(frozen=True) @streamable + @dataclass(frozen=True) class TestClassUint(Streamable): a: uint32 @@ -155,8 +290,8 @@ def test_ambiguous_deserialization_int(): def test_ambiguous_deserialization_list(): - @dataclass(frozen=True) @streamable + @dataclass(frozen=True) class TestClassList(Streamable): a: List[uint8] @@ -166,8 +301,8 @@ def test_ambiguous_deserialization_list(): def test_ambiguous_deserialization_tuple(): - @dataclass(frozen=True) @streamable + @dataclass(frozen=True) class TestClassTuple(Streamable): a: Tuple[uint8, str] @@ -177,8 +312,8 @@ def test_ambiguous_deserialization_tuple(): def test_ambiguous_deserialization_str(): - @dataclass(frozen=True) @streamable + @dataclass(frozen=True) class TestClassStr(Streamable): a: str @@ -188,8 +323,8 @@ def test_ambiguous_deserialization_str(): def test_ambiguous_deserialization_bytes(): - @dataclass(frozen=True) @streamable + @dataclass(frozen=True) class TestClassBytes(Streamable): a: bytes @@ -205,8 +340,8 @@ def test_ambiguous_deserialization_bytes(): def test_ambiguous_deserialization_bool(): - @dataclass(frozen=True) @streamable + @dataclass(frozen=True) class TestClassBool(Streamable): a: bool @@ -219,8 +354,8 @@ def test_ambiguous_deserialization_bool(): def test_ambiguous_deserialization_program(): - @dataclass(frozen=True) @streamable + @dataclass(frozen=True) class TestClassProgram(Streamable): a: Program @@ -233,8 +368,8 @@ def test_ambiguous_deserialization_program(): def test_streamable_empty(): - @dataclass(frozen=True) @streamable + @dataclass(frozen=True) class A(Streamable): pass @@ -414,3 +549,42 @@ def test_parse_str(): # EOF off by one with raises(AssertionError): parse_str(io.BytesIO(b"\x00\x00\x02\x01" + b"a" * 512)) + + +def test_wrong_decorator_order(): + + with raises(DefinitionError): + + @dataclass(frozen=True) + @streamable + class WrongDecoratorOrder(Streamable): + pass + + +def test_dataclass_not_frozen(): + + with raises(DefinitionError): + + @streamable + @dataclass(frozen=False) + class DataclassNotFrozen(Streamable): + pass + + +def test_dataclass_missing(): + + with raises(DefinitionError): + + @streamable + class DataclassMissing(Streamable): + pass + + +def test_streamable_inheritance_missing(): + + with raises(DefinitionError): + + @streamable + @dataclass(frozen=True) + class StreamableInheritanceMissing: + pass diff --git a/tests/core/util/test_type_checking.py b/tests/core/util/test_type_checking.py deleted file mode 100644 index 8e90f8ad4c..0000000000 --- a/tests/core/util/test_type_checking.py +++ /dev/null @@ -1,91 +0,0 @@ -import unittest -from dataclasses import dataclass -from typing import Dict, List, Optional, Tuple - -from pytest import raises - -from chia.util.ints import uint8 -from chia.util.type_checking import is_type_List, is_type_SpecificOptional, strictdataclass - - -class TestIsTypeList(unittest.TestCase): - def test_basic_list(self): - a = [1, 2, 3] - assert is_type_List(type(a)) - assert is_type_List(List) - assert is_type_List(List[int]) - assert is_type_List(List[uint8]) - assert is_type_List(list) - assert not is_type_List(Tuple) - assert not is_type_List(tuple) - assert not is_type_List(dict) - - def test_not_lists(self): - assert not is_type_List(Dict) - - -class TestIsTypeSpecificOptional(unittest.TestCase): - def test_basic_optional(self): - assert is_type_SpecificOptional(Optional[int]) - assert is_type_SpecificOptional(Optional[Optional[int]]) - assert not is_type_SpecificOptional(List[int]) - - -class TestStrictClass(unittest.TestCase): - def test_StrictDataClass(self): - @dataclass(frozen=True) - @strictdataclass - class TestClass1: - a: int - b: str - - good: TestClass1 = TestClass1(24, "!@12") - assert TestClass1.__name__ == "TestClass1" - assert good - assert good.a == 24 - assert good.b == "!@12" - good2 = TestClass1(52, bytes([1, 2, 3])) - assert good2.b == str(bytes([1, 2, 3])) - - def test_StrictDataClassBad(self): - @dataclass(frozen=True) - @strictdataclass - class TestClass2: - a: int - b = 0 - - assert TestClass2(25) - - with raises(TypeError): - TestClass2(1, 2) # pylint: disable=too-many-function-args - - def test_StrictDataClassLists(self): - @dataclass(frozen=True) - @strictdataclass - class TestClass: - a: List[int] - b: List[List[uint8]] - - assert TestClass([1, 2, 3], [[uint8(200), uint8(25)], [uint8(25)]]) - - with raises(ValueError): - TestClass({"1": 1}, [[uint8(200), uint8(25)], [uint8(25)]]) - - with raises(ValueError): - TestClass([1, 2, 3], [uint8(200), uint8(25)]) - - def test_StrictDataClassOptional(self): - @dataclass(frozen=True) - @strictdataclass - class TestClass: - a: Optional[int] - b: Optional[int] - c: Optional[Optional[int]] - d: Optional[Optional[int]] - - good = TestClass(12, None, 13, None) - assert good - - -if __name__ == "__main__": - unittest.main()