From e7a8f021d3280e0795ea722d9f75679bc1f9c970 Mon Sep 17 00:00:00 2001 From: Jack Nelson Date: Tue, 28 Jun 2022 14:15:52 -0400 Subject: [PATCH 01/62] Fix sql error when only config file is deleted (#12087) * fix table already exists error * catch when db exists without possible race conditon avoid race condition hazard --- chia/cmds/init_funcs.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/chia/cmds/init_funcs.py b/chia/cmds/init_funcs.py index faa2aa1968..0dfd6ee970 100644 --- a/chia/cmds/init_funcs.py +++ b/chia/cmds/init_funcs.py @@ -510,9 +510,13 @@ def chia_init( db_path_replaced = config["database_path"].replace("CHALLENGE", config["selected_network"]) db_path = path_from_root(root_path, db_path_replaced) db_path.parent.mkdir(parents=True, exist_ok=True) - - with sqlite3.connect(db_path) as connection: - set_db_version(connection, 2) + try: + # create new v2 db file + with sqlite3.connect(db_path) as connection: + set_db_version(connection, 2) + except sqlite3.OperationalError: + # db already exists, so we're good + pass print("") print("To see your keys, run 'chia keys show --show-mnemonic-seed'") From 2d812c50024a2c0147427d4feb1341a66ac56362 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Wed, 29 Jun 2022 11:34:14 +0200 Subject: [PATCH 02/62] resurrect WalletCoinStore unittest (#12116) --- chia/wallet/wallet_coin_store.py | 3 +- tests/util/db_connection.py | 16 + tests/wallet/test_wallet_coin_store.py | 387 +++++++++++++++++++++++++ tests/wallet/test_wallet_store.py | 230 --------------- 4 files changed, 404 insertions(+), 232 deletions(-) create mode 100644 tests/wallet/test_wallet_coin_store.py delete mode 100644 tests/wallet/test_wallet_store.py diff --git a/chia/wallet/wallet_coin_store.py b/chia/wallet/wallet_coin_store.py index 852fde3a96..95ee501bc6 100644 --- a/chia/wallet/wallet_coin_store.py +++ b/chia/wallet/wallet_coin_store.py @@ -243,8 +243,7 @@ class WalletCoinStore: async def rollback_to_block(self, height: int): """ - Rolls back the blockchain to block_index. All blocks confirmed after this point - are removed from the LCA. All coins confirmed after this point are removed. + Rolls back the blockchain to block_index. All coins confirmed after this point are removed. All coins spent after this point are set to unspent. Can be -1 (rollback all) """ # Delete from storage diff --git a/tests/util/db_connection.py b/tests/util/db_connection.py index 110ba27493..3b2dbd8aad 100644 --- a/tests/util/db_connection.py +++ b/tests/util/db_connection.py @@ -1,5 +1,6 @@ from pathlib import Path from chia.util.db_wrapper import DBWrapper2 +from chia.util.db_wrapper import DBWrapper import tempfile import aiosqlite @@ -33,3 +34,18 @@ class DBConnection: async def __aexit__(self, exc_t, exc_v, exc_tb) -> None: await self._db_wrapper.close() self.db_path.unlink() + + +# This is just here until all DBWrappers have been upgraded to DBWrapper2 +class DBConnection1: + async def __aenter__(self) -> DBWrapper: + self.db_path = Path(tempfile.NamedTemporaryFile().name) + if self.db_path.exists(): + self.db_path.unlink() + self._db_connection = await aiosqlite.connect(self.db_path) + self._db_wrapper = DBWrapper(self._db_connection) + return self._db_wrapper + + async def __aexit__(self, exc_t, exc_v, exc_tb) -> None: + await self._db_connection.close() + self.db_path.unlink() diff --git a/tests/wallet/test_wallet_coin_store.py b/tests/wallet/test_wallet_coin_store.py new file mode 100644 index 0000000000..00b7e28bd6 --- /dev/null +++ b/tests/wallet/test_wallet_coin_store.py @@ -0,0 +1,387 @@ +from secrets import token_bytes + +import pytest + +from chia.types.blockchain_format.coin import Coin +from chia.util.ints import uint32, uint64 +from chia.wallet.util.wallet_types import WalletType +from chia.wallet.wallet_coin_record import WalletCoinRecord +from chia.wallet.wallet_coin_store import WalletCoinStore +from tests.util.db_connection import DBConnection1 + +coin_1 = Coin(token_bytes(32), token_bytes(32), uint64(12312)) +coin_2 = Coin(coin_1.parent_coin_info, token_bytes(32), uint64(12311)) +coin_3 = Coin(token_bytes(32), token_bytes(32), uint64(12312)) +coin_4 = Coin(token_bytes(32), token_bytes(32), uint64(12312)) +coin_5 = Coin(token_bytes(32), token_bytes(32), uint64(12312)) +coin_6 = Coin(token_bytes(32), coin_4.puzzle_hash, uint64(12312)) +coin_7 = Coin(token_bytes(32), token_bytes(32), uint64(12312)) +record_replaced = WalletCoinRecord(coin_1, uint32(8), uint32(0), False, True, WalletType.STANDARD_WALLET, 0) +record_1 = WalletCoinRecord(coin_1, uint32(4), uint32(0), False, True, WalletType.STANDARD_WALLET, 0) +record_2 = WalletCoinRecord(coin_2, uint32(5), uint32(0), False, True, WalletType.STANDARD_WALLET, 0) +record_3 = WalletCoinRecord( + coin_3, + uint32(5), + uint32(10), + True, + False, + WalletType.STANDARD_WALLET, + 0, +) +record_4 = WalletCoinRecord( + coin_4, + uint32(5), + uint32(15), + True, + False, + WalletType.STANDARD_WALLET, + 0, +) +record_5 = WalletCoinRecord( + coin_5, + uint32(5), + uint32(15), + False, + False, + WalletType.STANDARD_WALLET, + 1, +) +record_6 = WalletCoinRecord( + coin_6, + uint32(5), + uint32(15), + True, + False, + WalletType.STANDARD_WALLET, + 2, +) +record_7 = WalletCoinRecord( + coin_7, + uint32(5), + uint32(15), + False, + False, + WalletType.POOLING_WALLET, + 2, +) + + +@pytest.mark.asyncio +async def test_add_replace_get() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletCoinStore.create(db_wrapper) + + assert await store.get_coin_record(coin_1.name()) is None + await store.add_coin_record(record_replaced) + await store.add_coin_record(record_1) + await store.add_coin_record(record_2) + await store.add_coin_record(record_3) + await store.add_coin_record(record_4) + assert await store.get_coin_record(coin_1.name()) == record_1 + + +@pytest.mark.asyncio +async def test_persistance() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletCoinStore.create(db_wrapper) + await store.add_coin_record(record_1) + + store = await WalletCoinStore.create(db_wrapper) + assert await store.get_coin_record(coin_1.name()) == record_1 + + +@pytest.mark.asyncio +async def test_set_spent() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletCoinStore.create(db_wrapper) + await store.add_coin_record(record_1) + + assert not (await store.get_coin_record(coin_1.name())).spent + await store.set_spent(coin_1.name(), uint32(12)) + assert (await store.get_coin_record(coin_1.name())).spent + assert (await store.get_coin_record(coin_1.name())).spent_block_height == 12 + + +@pytest.mark.asyncio +async def test_get_records_by_puzzle_hash() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletCoinStore.create(db_wrapper) + + await store.add_coin_record(record_4) + await store.add_coin_record(record_5) + await store.add_coin_record(record_5) + await store.add_coin_record(record_6) + assert len(await store.get_coin_records_by_puzzle_hash(record_6.coin.puzzle_hash)) == 2 # 4 and 6 + assert len(await store.get_coin_records_by_puzzle_hash(token_bytes(32))) == 0 + + assert await store.get_coin_record(coin_6.name()) == record_6 + assert await store.get_coin_record(token_bytes(32)) is None + + +@pytest.mark.asyncio +async def test_get_unspent_coins_for_wallet() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletCoinStore.create(db_wrapper) + + assert await store.get_unspent_coins_for_wallet(1) == set() + + await store.add_coin_record(record_4) # this is spent + await store.add_coin_record(record_5) + await store.add_coin_record(record_6) # this is spent + await store.add_coin_record(record_7) + + assert await store.get_unspent_coins_for_wallet(1) == set([record_5]) + assert await store.get_unspent_coins_for_wallet(2) == set([record_7]) + assert await store.get_unspent_coins_for_wallet(3) == set() + + await store.set_spent(coin_4.name(), uint32(12)) + + assert await store.get_unspent_coins_for_wallet(1) == set([record_5]) + assert await store.get_unspent_coins_for_wallet(2) == set([record_7]) + assert await store.get_unspent_coins_for_wallet(3) == set() + + await store.set_spent(coin_7.name(), uint32(12)) + + assert await store.get_unspent_coins_for_wallet(1) == set([record_5]) + assert await store.get_unspent_coins_for_wallet(2) == set() + assert await store.get_unspent_coins_for_wallet(3) == set() + + await store.set_spent(coin_5.name(), uint32(12)) + + assert await store.get_unspent_coins_for_wallet(1) == set() + assert await store.get_unspent_coins_for_wallet(2) == set() + assert await store.get_unspent_coins_for_wallet(3) == set() + + +@pytest.mark.asyncio +async def test_get_records_by_parent_id() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletCoinStore.create(db_wrapper) + + await store.add_coin_record(record_1) + await store.add_coin_record(record_2) + await store.add_coin_record(record_3) + await store.add_coin_record(record_4) + await store.add_coin_record(record_5) + await store.add_coin_record(record_6) + await store.add_coin_record(record_7) + + assert set(await store.get_coin_records_by_parent_id(coin_1.parent_coin_info)) == set([record_1, record_2]) + assert set(await store.get_coin_records_by_parent_id(coin_2.parent_coin_info)) == set([record_1, record_2]) + assert await store.get_coin_records_by_parent_id(coin_3.parent_coin_info) == [record_3] + assert await store.get_coin_records_by_parent_id(coin_4.parent_coin_info) == [record_4] + assert await store.get_coin_records_by_parent_id(coin_5.parent_coin_info) == [record_5] + assert await store.get_coin_records_by_parent_id(coin_6.parent_coin_info) == [record_6] + assert await store.get_coin_records_by_parent_id(coin_7.parent_coin_info) == [record_7] + + +@pytest.mark.asyncio +async def test_get_multiple_coin_records() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletCoinStore.create(db_wrapper) + + await store.add_coin_record(record_1) + await store.add_coin_record(record_2) + await store.add_coin_record(record_3) + await store.add_coin_record(record_4) + await store.add_coin_record(record_5) + await store.add_coin_record(record_6) + await store.add_coin_record(record_7) + + assert set(await store.get_multiple_coin_records([coin_1.name(), coin_2.name(), coin_3.name()])) == set( + [record_1, record_2, record_3] + ) + + assert set(await store.get_multiple_coin_records([coin_5.name(), coin_6.name(), coin_7.name()])) == set( + [record_5, record_6, record_7] + ) + + assert ( + set( + await store.get_multiple_coin_records( + [ + coin_1.name(), + coin_2.name(), + coin_3.name(), + coin_4.name(), + coin_5.name(), + coin_6.name(), + coin_7.name(), + ] + ) + ) + == set([record_1, record_2, record_3, record_4, record_5, record_6, record_7]) + ) + + +@pytest.mark.asyncio +async def test_delete_coin_record() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletCoinStore.create(db_wrapper) + + await store.add_coin_record(record_1) + await store.add_coin_record(record_2) + await store.add_coin_record(record_3) + await store.add_coin_record(record_4) + await store.add_coin_record(record_5) + await store.add_coin_record(record_6) + await store.add_coin_record(record_7) + + assert ( + set( + await store.get_multiple_coin_records( + [ + coin_1.name(), + coin_2.name(), + coin_3.name(), + coin_4.name(), + coin_5.name(), + coin_6.name(), + coin_7.name(), + ] + ) + ) + == set([record_1, record_2, record_3, record_4, record_5, record_6, record_7]) + ) + + assert await store.get_coin_record(coin_1.name()) == record_1 + + await store.delete_coin_record(coin_1.name()) + + assert await store.get_coin_record(coin_1.name()) is None + assert set( + await store.get_multiple_coin_records( + [coin_2.name(), coin_3.name(), coin_4.name(), coin_5.name(), coin_6.name(), coin_7.name()] + ) + ) == set([record_2, record_3, record_4, record_5, record_6, record_7]) + + +def record(c: Coin, *, confirmed: int, spent: int) -> WalletCoinRecord: + return WalletCoinRecord(c, uint32(confirmed), uint32(spent), spent != 0, False, WalletType.STANDARD_WALLET, 0) + + +@pytest.mark.asyncio +async def test_get_coins_to_check() -> None: + + r1 = record(coin_1, confirmed=1, spent=0) + r2 = record(coin_2, confirmed=2, spent=4) + r3 = record(coin_3, confirmed=3, spent=5) + r4 = record(coin_4, confirmed=4, spent=6) + r5 = record(coin_5, confirmed=5, spent=7) + # these spent heights violate the invariant + r6 = record(coin_6, confirmed=6, spent=1) + r7 = record(coin_7, confirmed=7, spent=2) + + async with DBConnection1() as db_wrapper: + store = await WalletCoinStore.create(db_wrapper) + + await store.add_coin_record(r1) + await store.add_coin_record(r2) + await store.add_coin_record(r3) + await store.add_coin_record(r4) + await store.add_coin_record(r5) + await store.add_coin_record(r6) + await store.add_coin_record(r7) + + for i in range(10): + + coins = await store.get_coins_to_check(i) + + # r1 is unspent and should always be included, regardless of height + assert r1 in coins + # r2 was spent at height 4 + assert (r2 in coins) == (i < 4) + # r3 was spent at height 5 + assert (r3 in coins) == (i < 5) + # r4 was spent at height 6 + assert (r4 in coins) == (i < 6) + # r5 was spent at height 7 + assert (r5 in coins) == (i < 7) + # r6 was confirmed at height 6 + assert (r6 in coins) == (i < 6) + # r7 was confirmed at height 7 + assert (r7 in coins) == (i < 7) + + +@pytest.mark.asyncio +async def test_get_first_coin_height() -> None: + + r1 = record(coin_1, confirmed=1, spent=0) + r2 = record(coin_2, confirmed=2, spent=4) + r3 = record(coin_3, confirmed=3, spent=5) + r4 = record(coin_4, confirmed=4, spent=6) + r5 = record(coin_5, confirmed=5, spent=7) + + async with DBConnection1() as db_wrapper: + store = await WalletCoinStore.create(db_wrapper) + + assert await store.get_first_coin_height() is None + + await store.add_coin_record(r5) + assert await store.get_first_coin_height() == 5 + await store.add_coin_record(r4) + assert await store.get_first_coin_height() == 4 + await store.add_coin_record(r3) + assert await store.get_first_coin_height() == 3 + await store.add_coin_record(r2) + assert await store.get_first_coin_height() == 2 + await store.add_coin_record(r1) + assert await store.get_first_coin_height() == 1 + + +@pytest.mark.asyncio +async def test_rollback_to_block() -> None: + + r1 = record(coin_1, confirmed=1, spent=0) + r2 = record(coin_2, confirmed=2, spent=4) + r3 = record(coin_3, confirmed=3, spent=5) + r4 = record(coin_4, confirmed=4, spent=6) + r5 = record(coin_5, confirmed=5, spent=7) + + async with DBConnection1() as db_wrapper: + store = await WalletCoinStore.create(db_wrapper) + + await store.add_coin_record(r1) + await store.add_coin_record(r2) + await store.add_coin_record(r3) + await store.add_coin_record(r4) + await store.add_coin_record(r5) + + assert set( + await store.get_multiple_coin_records( + [ + coin_1.name(), + coin_2.name(), + coin_3.name(), + coin_4.name(), + coin_5.name(), + ] + ) + ) == set( + [ + r1, + r2, + r3, + r4, + r5, + ] + ) + + assert await store.get_coin_record(coin_5.name()) == r5 + + await store.rollback_to_block(6) + + new_r5 = await store.get_coin_record(coin_5.name()) + assert not new_r5.spent + assert new_r5.spent_block_height == 0 + assert new_r5 != r5 + + assert await store.get_coin_record(coin_4.name()) == r4 + + await store.rollback_to_block(4) + + assert await store.get_coin_record(coin_5.name()) is None + new_r4 = await store.get_coin_record(coin_4.name()) + assert not new_r4.spent + assert new_r4.spent_block_height == 0 + assert new_r4 != r4 diff --git a/tests/wallet/test_wallet_store.py b/tests/wallet/test_wallet_store.py deleted file mode 100644 index dddb3c3ff9..0000000000 --- a/tests/wallet/test_wallet_store.py +++ /dev/null @@ -1,230 +0,0 @@ -# TODO: write tests for other stores -# import asyncio -# from pathlib import Path -# from secrets import token_bytes -# import aiosqlite -# import pytest -# from chia.util.ints import uint32, uint64, uint128 -# from chia.wallet.wallet_coin_record import WalletCoinRecord -# from chia.wallet.util.wallet_types import WalletType -# from chia.types.coin import Coin -# -# -# @pytest.fixture(scope="module") -# def event_loop(): -# loop = asyncio.get_event_loop() -# yield loop -# -# -# class TestWalletStore: -# @pytest.mark.asyncio -# async def test_store(self): -# db_filename = Path("blockchain_wallet_store_test.db") -# -# if db_filename.exists(): -# db_filename.unlink() -# -# db_connection = await aiosqlite.connect(db_filename) -# store = await WalletStore.create(db_connection) -# try: -# coin_1 = Coin(token_bytes(32), token_bytes(32), uint64(12312)) -# coin_2 = Coin(token_bytes(32), token_bytes(32), uint64(12312)) -# coin_3 = Coin(token_bytes(32), token_bytes(32), uint64(12312)) -# coin_4 = Coin(token_bytes(32), token_bytes(32), uint64(12312)) -# record_replaced = WalletCoinRecord(coin_1, uint32(8), uint32(0), -# False, True, WalletType.STANDARD_WALLET, 0) -# record_1 = WalletCoinRecord(coin_1, uint32(4), uint32(0), False, -# True, WalletType.STANDARD_WALLET, 0) -# record_2 = WalletCoinRecord(coin_2, uint32(5), uint32(0), -# False, True, WalletType.STANDARD_WALLET, 0) -# record_3 = WalletCoinRecord( -# coin_3, -# uint32(5), -# uint32(10), -# True, -# False, -# WalletType.STANDARD_WALLET, -# 0, -# ) -# record_4 = WalletCoinRecord( -# coin_4, -# uint32(5), -# uint32(15), -# True, -# False, -# WalletType.STANDARD_WALLET, -# 0, -# ) -# -# # Test add (replace) and get -# assert await store.get_coin_record(coin_1.name()) is None -# await store.add_coin_record(record_replaced) -# await store.add_coin_record(record_1) -# await store.add_coin_record(record_2) -# await store.add_coin_record(record_3) -# await store.add_coin_record(record_4) -# assert await store.get_coin_record(coin_1.name()) == record_1 -# -# # Test persistance -# await db_connection.close() -# db_connection = await aiosqlite.connect(db_filename) -# store = await WalletStore.create(db_connection) -# assert await store.get_coin_record(coin_1.name()) == record_1 -# -# # Test set spent -# await store.set_spent(coin_1.name(), uint32(12)) -# assert (await store.get_coin_record(coin_1.name())).spent -# assert (await store.get_coin_record(coin_1.name())).spent_block_index == 12 -# -# # No coins at height 3 -# assert len(await store.get_unspent_coins_at_height(3)) == 0 -# assert len(await store.get_unspent_coins_at_height(4)) == 1 -# assert len(await store.get_unspent_coins_at_height(5)) == 4 -# assert len(await store.get_unspent_coins_at_height(11)) == 3 -# assert len(await store.get_unspent_coins_at_height(12)) == 2 -# assert len(await store.get_unspent_coins_at_height(15)) == 1 -# assert len(await store.get_unspent_coins_at_height(16)) == 1 -# assert len(await store.get_unspent_coins_at_height()) == 1 -# -# assert len(await store.get_unspent_coins_for_wallet(0)) == 1 -# assert len(await store.get_unspent_coins_for_wallet(1)) == 0 -# -# coin_5 = Coin(token_bytes(32), token_bytes(32), uint64(12312)) -# record_5 = WalletCoinRecord( -# coin_5, -# uint32(5), -# uint32(15), -# False, -# False, -# WalletType.STANDARD_WALLET, -# 1, -# ) -# await store.add_coin_record(record_5) -# assert len(await store.get_unspent_coins_for_wallet(1)) == 1 -# -# assert len(await store.get_spendable_for_index(100, 1)) == 1 -# assert len(await store.get_spendable_for_index(100, 0)) == 1 -# assert len(await store.get_spendable_for_index(0, 0)) == 0 -# -# coin_6 = Coin(token_bytes(32), coin_4.puzzle_hash, uint64(12312)) -# await store.add_coin_record(record_5) -# record_6 = WalletCoinRecord( -# coin_6, -# uint32(5), -# uint32(15), -# True, -# False, -# WalletType.STANDARD_WALLET, -# 2, -# ) -# await store.add_coin_record(record_6) -# assert len(await store.get_coin_records_by_puzzle_hash(record_6.coin.puzzle_hash)) == 2 # 4 and 6 -# assert len(await store.get_coin_records_by_puzzle_hash(token_bytes(32))) == 0 -# -# assert await store.get_coin_record_by_coin_id(coin_6.name()) == record_6 -# assert await store.get_coin_record_by_coin_id(token_bytes(32)) is None -# -# # BLOCKS -# assert len(await store.get_lca_path()) == 0 -# -# # NOT lca block -# br_1 = BlockRecord( -# token_bytes(32), -# token_bytes(32), -# uint32(0), -# uint128(100), -# None, -# None, -# None, -# None, -# uint64(0), -# ) -# assert await store.get_block_record(br_1.header_hash) is None -# await store.add_block_record(br_1, False) -# assert len(await store.get_lca_path()) == 0 -# assert await store.get_block_record(br_1.header_hash) == br_1 -# -# # LCA genesis -# await store.add_block_record(br_1, True) -# assert await store.get_block_record(br_1.header_hash) == br_1 -# assert len(await store.get_lca_path()) == 1 -# assert (await store.get_lca_path())[br_1.header_hash] == br_1 -# -# br_2 = BlockRecord( -# token_bytes(32), -# token_bytes(32), -# uint32(1), -# uint128(100), -# None, -# None, -# None, -# None, -# uint64(0), -# ) -# await store.add_block_record(br_2, False) -# assert len(await store.get_lca_path()) == 1 -# await store.add_block_to_path(br_2.header_hash) -# assert len(await store.get_lca_path()) == 2 -# assert (await store.get_lca_path())[br_2.header_hash] == br_2 -# -# br_3 = BlockRecord( -# token_bytes(32), -# token_bytes(32), -# uint32(2), -# uint128(100), -# None, -# None, -# None, -# None, -# uint64(0), -# ) -# await store.add_block_record(br_3, True) -# assert len(await store.get_lca_path()) == 3 -# await store.remove_block_records_from_path(1) -# assert len(await store.get_lca_path()) == 2 -# -# await store.rollback_lca_to_block(0) -# assert len(await store.get_unspent_coins_at_height()) == 0 -# -# coin_7 = Coin(token_bytes(32), token_bytes(32), uint64(12312)) -# coin_8 = Coin(token_bytes(32), token_bytes(32), uint64(12312)) -# coin_9 = Coin(token_bytes(32), token_bytes(32), uint64(12312)) -# coin_10 = Coin(token_bytes(32), token_bytes(32), uint64(12312)) -# record_7 = WalletCoinRecord(coin_7, uint32(0), uint32(1), True, False, WalletType.STANDARD_WALLET, 1) -# record_8 = WalletCoinRecord(coin_8, uint32(1), uint32(2), True, False, WalletType.STANDARD_WALLET, 1) -# record_9 = WalletCoinRecord(coin_9, uint32(2), uint32(3), True, False, WalletType.STANDARD_WALLET, 1) -# record_10 = WalletCoinRecord( -# coin_10, -# uint32(3), -# uint32(4), -# True, -# False, -# WalletType.STANDARD_WALLET, -# 1, -# ) -# -# await store.add_coin_record(record_7) -# await store.add_coin_record(record_8) -# await store.add_coin_record(record_9) -# await store.add_coin_record(record_10) -# assert len(await store.get_unspent_coins_at_height(0)) == 1 -# assert len(await store.get_unspent_coins_at_height(1)) == 1 -# assert len(await store.get_unspent_coins_at_height(2)) == 1 -# assert len(await store.get_unspent_coins_at_height(3)) == 1 -# assert len(await store.get_unspent_coins_at_height(4)) == 0 -# -# await store.add_block_record(br_2, True) -# await store.add_block_record(br_3, True) -# -# await store.rollback_lca_to_block(1) -# -# assert len(await store.get_unspent_coins_at_height(0)) == 1 -# assert len(await store.get_unspent_coins_at_height(1)) == 1 -# assert len(await store.get_unspent_coins_at_height(2)) == 1 -# assert len(await store.get_unspent_coins_at_height(3)) == 1 -# assert len(await store.get_unspent_coins_at_height(4)) == 1 -# -# except AssertionError: -# await db_connection.close() -# raise -# await db_connection.close() From 749162d9fead35d2beb2d34bdc7d90df4d5ec6d5 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 29 Jun 2022 05:34:41 -0400 Subject: [PATCH 03/62] hint chia.clvm.spend_sim (#11819) * hint chia.clvm.spend_sim * adjust comments * undo * Created typo corrected to create --- chia/clvm/spend_sim.py | 57 ++++++++++++------- mypy.ini | 2 +- tests/wallet/nft_wallet/test_nft_lifecycle.py | 6 +- 3 files changed, 41 insertions(+), 24 deletions(-) diff --git a/chia/clvm/spend_sim.py b/chia/clvm/spend_sim.py index 784b18000f..bea956ad32 100644 --- a/chia/clvm/spend_sim.py +++ b/chia/clvm/spend_sim.py @@ -1,12 +1,14 @@ import aiosqlite import random +from pathlib import Path from dataclasses import dataclass -from typing import Optional, List, Dict, Tuple, Any +from typing import Optional, List, Dict, Tuple, Any, Type, TypeVar from chia.types.blockchain_format.sized_bytes import bytes32 from chia.types.blockchain_format.coin import Coin from chia.types.blockchain_format.program import Program, SerializedProgram +from chia.types.mempool_item import MempoolItem from chia.util.ints import uint64, uint32 from chia.util.hash import std_hash from chia.util.errors import Err, ValidationError @@ -45,6 +47,9 @@ class SimFullBlock(Streamable): height: uint32 # Note that height is not on a regular FullBlock +_T_SimBlockRecord = TypeVar("_T_SimBlockRecord", bound="SimBlockRecord") + + @streamable @dataclass(frozen=True) class SimBlockRecord(Streamable): @@ -57,7 +62,7 @@ class SimBlockRecord(Streamable): prev_transaction_block_hash: bytes32 @classmethod - def create(cls, rci: List[Coin], height: uint32, timestamp: uint64): + def create(cls: Type[_T_SimBlockRecord], rci: List[Coin], height: uint32, timestamp: uint64) -> _T_SimBlockRecord: return cls( rci, height, @@ -78,6 +83,9 @@ class SimStore(Streamable): blocks: List[SimFullBlock] +_T_SpendSim = TypeVar("_T_SpendSim", bound="SpendSim") + + class SpendSim: db_wrapper: DBWrapper2 @@ -89,7 +97,9 @@ class SpendSim: defaults: ConsensusConstants @classmethod - async def create(cls, db_path=None, defaults=DEFAULT_CONSTANTS): + async def create( + cls: Type[_T_SpendSim], db_path: Optional[Path] = None, defaults: ConsensusConstants = DEFAULT_CONSTANTS + ) -> _T_SpendSim: self = cls() if db_path is None: uri = f"file:db_{random.randint(0, 99999999)}?mode=memory&cache=shared" @@ -114,15 +124,16 @@ class SpendSim: self.block_height = store_data.block_height self.block_records = store_data.block_records self.blocks = store_data.blocks - self.mempool_manager.peak = self.block_records[-1] + # Create a protocol to make BlockRecord and SimBlockRecord interchangeable. + self.mempool_manager.peak = self.block_records[-1] # type: ignore[assignment] else: - self.timestamp = 1 - self.block_height = 0 + self.timestamp = uint64(1) + self.block_height = uint32(0) self.block_records = [] self.blocks = [] return self - async def close(self): + async def close(self) -> None: async with self.db_wrapper.write_db() as conn: c = await conn.execute("DELETE FROM block_data") await c.close() @@ -133,10 +144,11 @@ class SpendSim: await c.close() await self.db_wrapper.close() - async def new_peak(self): - await self.mempool_manager.new_peak(self.block_records[-1], None) + async def new_peak(self) -> None: + # Create a protocol to make BlockRecord and SimBlockRecord interchangeable. + await self.mempool_manager.new_peak(self.block_records[-1], None) # type: ignore[arg-type] - def new_coin_record(self, coin: Coin, coinbase=False) -> CoinRecord: + def new_coin_record(self, coin: Coin, coinbase: bool = False) -> CoinRecord: return CoinRecord( coin, uint32(self.block_height + 1), @@ -164,7 +176,7 @@ class SpendSim: return None return simple_solution_generator(bundle) - async def farm_block(self, puzzle_hash: bytes32 = bytes32(b"0" * 32)): + async def farm_block(self, puzzle_hash: bytes32 = bytes32(b"0" * 32)) -> Tuple[List[Coin], List[Coin]]: # Fees get calculated fees = uint64(0) if self.mempool_manager.mempool.spends: @@ -234,13 +246,13 @@ class SpendSim: def get_height(self) -> uint32: return self.block_height - def pass_time(self, time: uint64): + def pass_time(self, time: uint64) -> None: self.timestamp = uint64(self.timestamp + time) - def pass_blocks(self, blocks: uint32): + def pass_blocks(self, blocks: uint32) -> None: self.block_height = uint32(self.block_height + blocks) - async def rewind(self, block_height: uint32): + async def rewind(self, block_height: uint32) -> None: new_br_list = list(filter(lambda br: br.height <= block_height, self.block_records)) new_block_list = list(filter(lambda block: block.height <= block_height, self.blocks)) self.block_records = new_br_list @@ -255,7 +267,7 @@ class SpendSim: class SimClient: - def __init__(self, service): + def __init__(self, service: SpendSim) -> None: self.service = service async def push_tx(self, spend_bundle: SpendBundle) -> Tuple[MempoolInclusionStatus, Optional[Err]]: @@ -270,7 +282,7 @@ class SimClient: ) return status, error - async def get_coin_record_by_name(self, name: bytes32) -> CoinRecord: + async def get_coin_record_by_name(self, name: bytes32) -> Optional[CoinRecord]: return await self.service.mempool_manager.coin_store.get_coin_record(name) async def get_coin_records_by_names( @@ -363,8 +375,13 @@ class SimClient: return additions, removals async def get_puzzle_and_solution(self, coin_id: bytes32, height: uint32) -> Optional[CoinSpend]: - generator = list(filter(lambda block: block.height == height, self.service.blocks))[0].transactions_generator - coin_record = await self.service.mempool_manager.coin_store.get_coin_record(coin_id) + filtered_generators = list(filter(lambda block: block.height == height, self.service.blocks)) + # real consideration should be made for the None cases instead of just hint ignoring + generator: BlockGenerator = filtered_generators[0].transactions_generator # type: ignore[assignment] + coin_record: CoinRecord + coin_record = await self.service.mempool_manager.coin_store.get_coin_record( # type: ignore[assignment] + coin_id, + ) error, puzzle, solution = get_puzzle_and_solution_for_coin( generator, coin_id, @@ -380,13 +397,13 @@ class SimClient: async def get_all_mempool_tx_ids(self) -> List[bytes32]: return list(self.service.mempool_manager.mempool.spends.keys()) - async def get_all_mempool_items(self) -> Dict[bytes32, Dict]: + async def get_all_mempool_items(self) -> Dict[bytes32, MempoolItem]: spends = {} for tx_id, item in self.service.mempool_manager.mempool.spends.items(): spends[tx_id] = item return spends - async def get_mempool_item_by_tx_id(self, tx_id: bytes32) -> Optional[Dict]: + async def get_mempool_item_by_tx_id(self, tx_id: bytes32) -> Optional[Dict[str, Any]]: item = self.service.mempool_manager.get_mempool_item(tx_id) if item is None: return None diff --git a/mypy.ini b/mypy.ini index 579ad27fd2..dba101a390 100644 --- a/mypy.ini +++ b/mypy.ini @@ -17,7 +17,7 @@ no_implicit_reexport = True strict_equality = True # list created by: venv/bin/mypy | sed -n 's/.py:.*//p' | sort | uniq | tr '/' '.' | tr '\n' ',' -[mypy-benchmarks.block_ref,benchmarks.block_store,benchmarks.coin_store,benchmarks.utils,build_scripts.installer-version,chia.clvm.spend_sim,chia.cmds.configure,chia.cmds.db,chia.cmds.db_upgrade_func,chia.cmds.farm_funcs,chia.cmds.init,chia.cmds.init_funcs,chia.cmds.keys,chia.cmds.keys_funcs,chia.cmds.passphrase,chia.cmds.passphrase_funcs,chia.cmds.plotnft,chia.cmds.plotnft_funcs,chia.cmds.plots,chia.cmds.plotters,chia.cmds.show,chia.cmds.start_funcs,chia.cmds.wallet,chia.cmds.wallet_funcs,chia.daemon.client,chia.daemon.keychain_proxy,chia.daemon.keychain_server,chia.daemon.server,chia.farmer.farmer,chia.farmer.farmer_api,chia.full_node.block_height_map,chia.full_node.block_store,chia.full_node.bundle_tools,chia.full_node.coin_store,chia.full_node.full_node,chia.full_node.full_node_api,chia.full_node.full_node_store,chia.full_node.generator,chia.full_node.hint_store,chia.full_node.lock_queue,chia.full_node.mempool,chia.full_node.mempool_check_conditions,chia.full_node.mempool_manager,chia.full_node.pending_tx_cache,chia.full_node.sync_store,chia.full_node.weight_proof,chia.harvester.harvester,chia.harvester.harvester_api,chia.introducer.introducer,chia.introducer.introducer_api,chia.plotters.bladebit,chia.plotters.chiapos,chia.plotters.install_plotter,chia.plotters.madmax,chia.plotters.plotters,chia.plotters.plotters_util,chia.plotting.check_plots,chia.plotting.create_plots,chia.plotting.manager,chia.plotting.util,chia.pools.pool_config,chia.pools.pool_puzzles,chia.pools.pool_wallet,chia.pools.pool_wallet_info,chia.protocols.pool_protocol,chia.rpc.crawler_rpc_api,chia.rpc.farmer_rpc_api,chia.rpc.farmer_rpc_client,chia.rpc.full_node_rpc_api,chia.rpc.full_node_rpc_client,chia.rpc.harvester_rpc_api,chia.rpc.harvester_rpc_client,chia.rpc.rpc_client,chia.rpc.timelord_rpc_api,chia.rpc.util,chia.rpc.wallet_rpc_api,chia.rpc.wallet_rpc_client,chia.seeder.crawler,chia.seeder.crawler_api,chia.seeder.crawl_store,chia.seeder.dns_server,chia.seeder.peer_record,chia.seeder.start_crawler,chia.server.address_manager,chia.server.address_manager_store,chia.server.connection_utils,chia.server.introducer_peers,chia.server.node_discovery,chia.server.peer_store_resolver,chia.server.rate_limits,chia.server.reconnect_task,chia.server.server,chia.server.ssl_context,chia.server.start_farmer,chia.server.start_full_node,chia.server.start_harvester,chia.server.start_introducer,chia.server.start_service,chia.server.start_timelord,chia.server.start_wallet,chia.server.ws_connection,chia.simulator.full_node_simulator,chia.simulator.start_simulator,chia.ssl.create_ssl,chia.timelord.iters_from_block,chia.timelord.timelord,chia.timelord.timelord_api,chia.timelord.timelord_launcher,chia.timelord.timelord_state,chia.types.announcement,chia.types.blockchain_format.classgroup,chia.types.blockchain_format.coin,chia.types.blockchain_format.program,chia.types.blockchain_format.proof_of_space,chia.types.blockchain_format.tree_hash,chia.types.blockchain_format.vdf,chia.types.full_block,chia.types.header_block,chia.types.mempool_item,chia.types.name_puzzle_condition,chia.types.peer_info,chia.types.spend_bundle,chia.types.transaction_queue_entry,chia.types.unfinished_block,chia.types.unfinished_header_block,chia.util.api_decorators,chia.util.block_cache,chia.util.cached_bls,chia.util.check_fork_next_block,chia.util.chia_logging,chia.util.config,chia.util.db_wrapper,chia.util.dump_keyring,chia.util.file_keyring,chia.util.files,chia.util.hash,chia.util.json_util,chia.util.keychain,chia.util.keyring_wrapper,chia.util.log_exceptions,chia.util.lru_cache,chia.util.make_test_constants,chia.util.merkle_set,chia.util.network,chia.util.partial_func,chia.util.pip_import,chia.util.profiler,chia.util.safe_cancel_task,chia.util.service_groups,chia.util.ssl_check,chia.util.validate_alert,chia.wallet.block_record,chia.wallet.cat_wallet.cat_utils,chia.wallet.cat_wallet.cat_wallet,chia.wallet.cat_wallet.lineage_store,chia.wallet.chialisp,chia.wallet.did_wallet.did_wallet,chia.wallet.did_wallet.did_wallet_puzzles,chia.wallet.key_val_store,chia.wallet.lineage_proof,chia.wallet.nft_wallet.nft_wallet,chia.wallet.payment,chia.wallet.puzzles.load_clvm,chia.wallet.puzzles.p2_conditions,chia.wallet.puzzles.p2_delegated_conditions,chia.wallet.puzzles.p2_delegated_puzzle,chia.wallet.puzzles.p2_delegated_puzzle_or_hidden_puzzle,chia.wallet.puzzles.p2_m_of_n_delegate_direct,chia.wallet.puzzles.p2_puzzle_hash,chia.wallet.puzzles.prefarm.spend_prefarm,chia.wallet.puzzles.puzzle_utils,chia.wallet.puzzles.rom_bootstrap_generator,chia.wallet.puzzles.singleton_top_layer,chia.wallet.puzzles.tails,chia.wallet.rl_wallet.rl_wallet,chia.wallet.rl_wallet.rl_wallet_puzzles,chia.wallet.secret_key_store,chia.wallet.settings.user_settings,chia.wallet.trade_manager,chia.wallet.trade_record,chia.wallet.trading.offer,chia.wallet.trading.trade_store,chia.wallet.transaction_record,chia.wallet.util.debug_spend_bundle,chia.wallet.util.new_peak_queue,chia.wallet.util.peer_request_cache,chia.wallet.util.wallet_sync_utils,chia.wallet.wallet,chia.wallet.wallet_action_store,chia.wallet.wallet_blockchain,chia.wallet.wallet_coin_store,chia.wallet.wallet_interested_store,chia.wallet.wallet_node,chia.wallet.wallet_node_api,chia.wallet.wallet_pool_store,chia.wallet.wallet_puzzle_store,chia.wallet.wallet_state_manager,chia.wallet.wallet_sync_store,chia.wallet.wallet_transaction_store,chia.wallet.wallet_user_store,chia.wallet.wallet_weight_proof_handler,installhelper,tests.blockchain.blockchain_test_utils,tests.blockchain.test_blockchain,tests.blockchain.test_blockchain_transactions,tests.block_tools,tests.build-init-files,tests.build-workflows,tests.clvm.coin_store,tests.clvm.test_chialisp_deserialization,tests.clvm.test_clvm_compilation,tests.clvm.test_program,tests.clvm.test_puzzle_compression,tests.clvm.test_puzzles,tests.clvm.test_serialized_program,tests.clvm.test_singletons,tests.clvm.test_spend_sim,tests.conftest,tests.connection_utils,tests.core.cmds.test_keys,tests.core.consensus.test_pot_iterations,tests.core.custom_types.test_coin,tests.core.custom_types.test_proof_of_space,tests.core.custom_types.test_spend_bundle,tests.core.daemon.test_daemon,tests.core.full_node.full_sync.test_full_sync,tests.core.full_node.stores.test_block_store,tests.core.full_node.stores.test_coin_store,tests.core.full_node.stores.test_full_node_store,tests.core.full_node.stores.test_hint_store,tests.core.full_node.stores.test_sync_store,tests.core.full_node.test_address_manager,tests.core.full_node.test_block_height_map,tests.core.full_node.test_conditions,tests.core.full_node.test_full_node,tests.core.full_node.test_mempool,tests.core.full_node.test_mempool_performance,tests.core.full_node.test_node_load,tests.core.full_node.test_peer_store_resolver,tests.core.full_node.test_performance,tests.core.full_node.test_transactions,tests.core.make_block_generator,tests.core.node_height,tests.core.server.test_dos,tests.core.server.test_rate_limits,tests.core.ssl.test_ssl,tests.core.test_cost_calculation,tests.core.test_crawler_rpc,tests.core.test_daemon_rpc,tests.core.test_db_conversion,tests.core.test_db_validation,tests.core.test_farmer_harvester_rpc,tests.core.test_filter,tests.core.test_full_node_rpc,tests.core.test_merkle_set,tests.core.test_setproctitle,tests.core.util.test_cached_bls,tests.core.util.test_config,tests.core.util.test_file_keyring_synchronization,tests.core.util.test_files,tests.core.util.test_keychain,tests.core.util.test_keyring_wrapper,tests.core.util.test_lru_cache,tests.core.util.test_significant_bits,tests.farmer_harvester.test_farmer_harvester,tests.generator.test_compression,tests.generator.test_generator_types,tests.generator.test_list_to_batches,tests.generator.test_rom,tests.generator.test_scan,tests.plotting.test_plot_manager,tests.pools.test_pool_cmdline,tests.pools.test_pool_config,tests.pools.test_pool_puzzles_lifecycle,tests.pools.test_pool_rpc,tests.pools.test_wallet_pool_store,tests.setup_nodes,tests.setup_services,tests.simulation.test_simulation,tests.time_out_assert,tests.tools.test_full_sync,tests.tools.test_run_block,tests.util.alert_server,tests.util.benchmark_cost,tests.util.blockchain,tests.util.build_network_protocol_files,tests.util.db_connection,tests.util.generator_tools_testing,tests.util.keyring,tests.util.key_tool,tests.util.rpc,tests.util.test_full_block_utils,tests.util.test_lock_queue,tests.util.test_misc,tests.util.test_network,tests.util.test_network_protocol_files,tests.wallet.cat_wallet.test_cat_lifecycle,tests.wallet.cat_wallet.test_cat_wallet,tests.wallet.cat_wallet.test_offer_lifecycle,tests.wallet.cat_wallet.test_trades,tests.wallet.did_wallet.test_did,tests.wallet.did_wallet.test_did_rpc,tests.wallet.did_wallet.test_nft_rpc,tests.wallet.did_wallet.test_nft_wallet,tests.wallet.rl_wallet.test_rl_rpc,tests.wallet.rl_wallet.test_rl_wallet,tests.wallet.rpc.test_wallet_rpc,tests.wallet.simple_sync.test_simple_sync_protocol,tests.wallet.sync.test_wallet_sync,tests.wallet.test_bech32m,tests.wallet.test_chialisp,tests.wallet.test_puzzle_store,tests.wallet.test_singleton,tests.wallet.test_singleton_lifecycle,tests.wallet.test_singleton_lifecycle_fast,tests.wallet.test_taproot,tests.wallet.test_wallet_blockchain,tests.wallet.test_wallet_interested_store,tests.wallet.test_wallet_key_val_store,tests.wallet.test_wallet_user_store,tests.wallet_tools,tests.weight_proof.test_weight_proof,tools.analyze-chain,tools.run_block,tools.test_full_sync,tests.wallet.nft_wallet.test_nft_wallet,chia.wallet.nft_wallet.nft_puzzles,tests.wallet.nft_wallet.test_nft_puzzles] +[mypy-benchmarks.block_ref,benchmarks.block_store,benchmarks.coin_store,benchmarks.utils,build_scripts.installer-version,chia.cmds.configure,chia.cmds.db,chia.cmds.db_upgrade_func,chia.cmds.farm_funcs,chia.cmds.init,chia.cmds.init_funcs,chia.cmds.keys,chia.cmds.keys_funcs,chia.cmds.passphrase,chia.cmds.passphrase_funcs,chia.cmds.plotnft,chia.cmds.plotnft_funcs,chia.cmds.plots,chia.cmds.plotters,chia.cmds.show,chia.cmds.start_funcs,chia.cmds.wallet,chia.cmds.wallet_funcs,chia.daemon.client,chia.daemon.keychain_proxy,chia.daemon.keychain_server,chia.daemon.server,chia.farmer.farmer,chia.farmer.farmer_api,chia.full_node.block_height_map,chia.full_node.block_store,chia.full_node.bundle_tools,chia.full_node.coin_store,chia.full_node.full_node,chia.full_node.full_node_api,chia.full_node.full_node_store,chia.full_node.generator,chia.full_node.hint_store,chia.full_node.lock_queue,chia.full_node.mempool,chia.full_node.mempool_check_conditions,chia.full_node.mempool_manager,chia.full_node.pending_tx_cache,chia.full_node.sync_store,chia.full_node.weight_proof,chia.harvester.harvester,chia.harvester.harvester_api,chia.introducer.introducer,chia.introducer.introducer_api,chia.plotters.bladebit,chia.plotters.chiapos,chia.plotters.install_plotter,chia.plotters.madmax,chia.plotters.plotters,chia.plotters.plotters_util,chia.plotting.check_plots,chia.plotting.create_plots,chia.plotting.manager,chia.plotting.util,chia.pools.pool_config,chia.pools.pool_puzzles,chia.pools.pool_wallet,chia.pools.pool_wallet_info,chia.protocols.pool_protocol,chia.rpc.crawler_rpc_api,chia.rpc.farmer_rpc_api,chia.rpc.farmer_rpc_client,chia.rpc.full_node_rpc_api,chia.rpc.full_node_rpc_client,chia.rpc.harvester_rpc_api,chia.rpc.harvester_rpc_client,chia.rpc.rpc_client,chia.rpc.timelord_rpc_api,chia.rpc.util,chia.rpc.wallet_rpc_api,chia.rpc.wallet_rpc_client,chia.seeder.crawler,chia.seeder.crawler_api,chia.seeder.crawl_store,chia.seeder.dns_server,chia.seeder.peer_record,chia.seeder.start_crawler,chia.server.address_manager,chia.server.address_manager_store,chia.server.connection_utils,chia.server.introducer_peers,chia.server.node_discovery,chia.server.peer_store_resolver,chia.server.rate_limits,chia.server.reconnect_task,chia.server.server,chia.server.ssl_context,chia.server.start_farmer,chia.server.start_full_node,chia.server.start_harvester,chia.server.start_introducer,chia.server.start_service,chia.server.start_timelord,chia.server.start_wallet,chia.server.ws_connection,chia.simulator.full_node_simulator,chia.simulator.start_simulator,chia.ssl.create_ssl,chia.timelord.iters_from_block,chia.timelord.timelord,chia.timelord.timelord_api,chia.timelord.timelord_launcher,chia.timelord.timelord_state,chia.types.announcement,chia.types.blockchain_format.classgroup,chia.types.blockchain_format.coin,chia.types.blockchain_format.program,chia.types.blockchain_format.proof_of_space,chia.types.blockchain_format.tree_hash,chia.types.blockchain_format.vdf,chia.types.full_block,chia.types.header_block,chia.types.mempool_item,chia.types.name_puzzle_condition,chia.types.peer_info,chia.types.spend_bundle,chia.types.transaction_queue_entry,chia.types.unfinished_block,chia.types.unfinished_header_block,chia.util.api_decorators,chia.util.block_cache,chia.util.cached_bls,chia.util.check_fork_next_block,chia.util.chia_logging,chia.util.config,chia.util.db_wrapper,chia.util.dump_keyring,chia.util.file_keyring,chia.util.files,chia.util.hash,chia.util.json_util,chia.util.keychain,chia.util.keyring_wrapper,chia.util.log_exceptions,chia.util.lru_cache,chia.util.make_test_constants,chia.util.merkle_set,chia.util.network,chia.util.partial_func,chia.util.pip_import,chia.util.profiler,chia.util.safe_cancel_task,chia.util.service_groups,chia.util.ssl_check,chia.util.validate_alert,chia.wallet.block_record,chia.wallet.cat_wallet.cat_utils,chia.wallet.cat_wallet.cat_wallet,chia.wallet.cat_wallet.lineage_store,chia.wallet.chialisp,chia.wallet.did_wallet.did_wallet,chia.wallet.did_wallet.did_wallet_puzzles,chia.wallet.key_val_store,chia.wallet.lineage_proof,chia.wallet.nft_wallet.nft_wallet,chia.wallet.payment,chia.wallet.puzzles.load_clvm,chia.wallet.puzzles.p2_conditions,chia.wallet.puzzles.p2_delegated_conditions,chia.wallet.puzzles.p2_delegated_puzzle,chia.wallet.puzzles.p2_delegated_puzzle_or_hidden_puzzle,chia.wallet.puzzles.p2_m_of_n_delegate_direct,chia.wallet.puzzles.p2_puzzle_hash,chia.wallet.puzzles.prefarm.spend_prefarm,chia.wallet.puzzles.puzzle_utils,chia.wallet.puzzles.rom_bootstrap_generator,chia.wallet.puzzles.singleton_top_layer,chia.wallet.puzzles.tails,chia.wallet.rl_wallet.rl_wallet,chia.wallet.rl_wallet.rl_wallet_puzzles,chia.wallet.secret_key_store,chia.wallet.settings.user_settings,chia.wallet.trade_manager,chia.wallet.trade_record,chia.wallet.trading.offer,chia.wallet.trading.trade_store,chia.wallet.transaction_record,chia.wallet.util.debug_spend_bundle,chia.wallet.util.new_peak_queue,chia.wallet.util.peer_request_cache,chia.wallet.util.wallet_sync_utils,chia.wallet.wallet,chia.wallet.wallet_action_store,chia.wallet.wallet_blockchain,chia.wallet.wallet_coin_store,chia.wallet.wallet_interested_store,chia.wallet.wallet_node,chia.wallet.wallet_node_api,chia.wallet.wallet_pool_store,chia.wallet.wallet_puzzle_store,chia.wallet.wallet_state_manager,chia.wallet.wallet_sync_store,chia.wallet.wallet_transaction_store,chia.wallet.wallet_user_store,chia.wallet.wallet_weight_proof_handler,installhelper,tests.blockchain.blockchain_test_utils,tests.blockchain.test_blockchain,tests.blockchain.test_blockchain_transactions,tests.block_tools,tests.build-init-files,tests.build-workflows,tests.clvm.coin_store,tests.clvm.test_chialisp_deserialization,tests.clvm.test_clvm_compilation,tests.clvm.test_program,tests.clvm.test_puzzle_compression,tests.clvm.test_puzzles,tests.clvm.test_serialized_program,tests.clvm.test_singletons,tests.clvm.test_spend_sim,tests.conftest,tests.connection_utils,tests.core.cmds.test_keys,tests.core.consensus.test_pot_iterations,tests.core.custom_types.test_coin,tests.core.custom_types.test_proof_of_space,tests.core.custom_types.test_spend_bundle,tests.core.daemon.test_daemon,tests.core.full_node.full_sync.test_full_sync,tests.core.full_node.stores.test_block_store,tests.core.full_node.stores.test_coin_store,tests.core.full_node.stores.test_full_node_store,tests.core.full_node.stores.test_hint_store,tests.core.full_node.stores.test_sync_store,tests.core.full_node.test_address_manager,tests.core.full_node.test_block_height_map,tests.core.full_node.test_conditions,tests.core.full_node.test_full_node,tests.core.full_node.test_mempool,tests.core.full_node.test_mempool_performance,tests.core.full_node.test_node_load,tests.core.full_node.test_peer_store_resolver,tests.core.full_node.test_performance,tests.core.full_node.test_transactions,tests.core.make_block_generator,tests.core.node_height,tests.core.server.test_dos,tests.core.server.test_rate_limits,tests.core.ssl.test_ssl,tests.core.test_cost_calculation,tests.core.test_crawler_rpc,tests.core.test_daemon_rpc,tests.core.test_db_conversion,tests.core.test_db_validation,tests.core.test_farmer_harvester_rpc,tests.core.test_filter,tests.core.test_full_node_rpc,tests.core.test_merkle_set,tests.core.test_setproctitle,tests.core.util.test_cached_bls,tests.core.util.test_config,tests.core.util.test_file_keyring_synchronization,tests.core.util.test_files,tests.core.util.test_keychain,tests.core.util.test_keyring_wrapper,tests.core.util.test_lru_cache,tests.core.util.test_significant_bits,tests.farmer_harvester.test_farmer_harvester,tests.generator.test_compression,tests.generator.test_generator_types,tests.generator.test_list_to_batches,tests.generator.test_rom,tests.generator.test_scan,tests.plotting.test_plot_manager,tests.pools.test_pool_cmdline,tests.pools.test_pool_config,tests.pools.test_pool_puzzles_lifecycle,tests.pools.test_pool_rpc,tests.pools.test_wallet_pool_store,tests.setup_nodes,tests.setup_services,tests.simulation.test_simulation,tests.time_out_assert,tests.tools.test_full_sync,tests.tools.test_run_block,tests.util.alert_server,tests.util.benchmark_cost,tests.util.blockchain,tests.util.build_network_protocol_files,tests.util.db_connection,tests.util.generator_tools_testing,tests.util.keyring,tests.util.key_tool,tests.util.rpc,tests.util.test_full_block_utils,tests.util.test_lock_queue,tests.util.test_misc,tests.util.test_network,tests.util.test_network_protocol_files,tests.wallet.cat_wallet.test_cat_lifecycle,tests.wallet.cat_wallet.test_cat_wallet,tests.wallet.cat_wallet.test_offer_lifecycle,tests.wallet.cat_wallet.test_trades,tests.wallet.did_wallet.test_did,tests.wallet.did_wallet.test_did_rpc,tests.wallet.did_wallet.test_nft_rpc,tests.wallet.did_wallet.test_nft_wallet,tests.wallet.rl_wallet.test_rl_rpc,tests.wallet.rl_wallet.test_rl_wallet,tests.wallet.rpc.test_wallet_rpc,tests.wallet.simple_sync.test_simple_sync_protocol,tests.wallet.sync.test_wallet_sync,tests.wallet.test_bech32m,tests.wallet.test_chialisp,tests.wallet.test_puzzle_store,tests.wallet.test_singleton,tests.wallet.test_singleton_lifecycle,tests.wallet.test_singleton_lifecycle_fast,tests.wallet.test_taproot,tests.wallet.test_wallet_blockchain,tests.wallet.test_wallet_interested_store,tests.wallet.test_wallet_key_val_store,tests.wallet.test_wallet_user_store,tests.wallet_tools,tests.weight_proof.test_weight_proof,tools.analyze-chain,tools.run_block,tools.test_full_sync,tests.wallet.nft_wallet.test_nft_wallet,chia.wallet.nft_wallet.nft_puzzles,tests.wallet.nft_wallet.test_nft_puzzles] disallow_any_generics = False disallow_subclassing_any = False disallow_untyped_calls = False diff --git a/tests/wallet/nft_wallet/test_nft_lifecycle.py b/tests/wallet/nft_wallet/test_nft_lifecycle.py index 5ac132152e..556ff4f698 100644 --- a/tests/wallet/nft_wallet/test_nft_lifecycle.py +++ b/tests/wallet/nft_wallet/test_nft_lifecycle.py @@ -131,7 +131,7 @@ async def test_state_layer(setup_sim: Tuple[SpendSim, SimClient], metadata_updat await sim.farm_block() state_layer_puzzle = create_nft_layer_puzzle_with_curry_params(metadata, METADATA_UPDATER_PUZZLE_HASH, ACS) finally: - await sim.close() # type: ignore + await sim.close() @pytest.mark.asyncio() @@ -238,7 +238,7 @@ async def test_ownership_layer(setup_sim: Tuple[SpendSim, SimClient]) -> None: ACS, ).get_tree_hash() finally: - await sim.close() # type: ignore + await sim.close() @pytest.mark.asyncio() @@ -362,4 +362,4 @@ async def test_default_transfer_program(setup_sim: Tuple[SpendSim, SimClient]) - assert result == (MempoolInclusionStatus.SUCCESS, None) await sim.farm_block() finally: - await sim.close() # type: ignore + await sim.close() From 4a69bc16135dcb36cf153f6c13924de6c681afec Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Wed, 29 Jun 2022 23:50:29 +0200 Subject: [PATCH 04/62] Wallet coin store cache (#12130) * remove wallet coin store 'cache', where all coins are kept in RAM. Rely on the sqlite DB * some DB optimizations to WalletCoinStore --- chia/wallet/wallet_coin_store.py | 134 +++++-------------------- chia/wallet/wallet_node.py | 3 - tests/wallet/test_wallet_coin_store.py | 12 +-- 3 files changed, 32 insertions(+), 117 deletions(-) diff --git a/chia/wallet/wallet_coin_store.py b/chia/wallet/wallet_coin_store.py index 95ee501bc6..6d07fb8aa3 100644 --- a/chia/wallet/wallet_coin_store.py +++ b/chia/wallet/wallet_coin_store.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Optional, Set +from typing import List, Optional, Set import aiosqlite import sqlite3 @@ -17,10 +17,6 @@ class WalletCoinStore: """ db_connection: aiosqlite.Connection - # coin_record_cache keeps ALL coin records in memory. [record_name: record] - coin_record_cache: Dict[bytes32, WalletCoinRecord] - # unspent_coin_wallet_cache keeps ALL unspent coin records for wallet in memory [wallet_id: [record_name: record]] - unspent_coin_wallet_cache: Dict[int, Dict[bytes32, WalletCoinRecord]] db_wrapper: DBWrapper @classmethod @@ -61,9 +57,6 @@ class WalletCoinStore: await self.db_connection.execute("CREATE INDEX IF NOT EXISTS wallet_id on coin_record(wallet_id)") await self.db_connection.commit() - self.coin_record_cache = {} - self.unspent_coin_wallet_cache = {} - await self.rebuild_wallet_cache() return self async def _clear_database(self): @@ -71,49 +64,23 @@ class WalletCoinStore: await cursor.close() await self.db_connection.commit() - async def rebuild_wallet_cache(self): - # First update all coins that were reorged, then re-add coin_records - all_coins = await self.get_all_coins() - self.unspent_coin_wallet_cache = {} - self.coin_record_cache = {} - for coin_record in all_coins: - name = coin_record.name() - self.coin_record_cache[name] = coin_record - if coin_record.spent is False: - if coin_record.wallet_id not in self.unspent_coin_wallet_cache: - self.unspent_coin_wallet_cache[coin_record.wallet_id] = {} - self.unspent_coin_wallet_cache[coin_record.wallet_id][name] = coin_record - async def get_multiple_coin_records(self, coin_names: List[bytes32]) -> List[WalletCoinRecord]: """Return WalletCoinRecord(s) that have a coin name in the specified list""" - if set(coin_names).issubset(set(self.coin_record_cache.keys())): - return list(filter(lambda cr: cr.coin.name() in coin_names, self.coin_record_cache.values())) - else: - as_hexes = [cn.hex() for cn in coin_names] - cursor = await self.db_connection.execute( - f'SELECT * from coin_record WHERE coin_name in ({"?," * (len(as_hexes) - 1)}?)', tuple(as_hexes) - ) - rows = await cursor.fetchall() - await cursor.close() + if len(coin_names) == 0: + return [] - return [self.coin_record_from_row(row) for row in rows] + as_hexes = [cn.hex() for cn in coin_names] + rows = await self.db_connection.execute_fetchall( + f'SELECT * from coin_record WHERE coin_name in ({"?," * (len(as_hexes) - 1)}?)', tuple(as_hexes) + ) + + return [self.coin_record_from_row(row) for row in rows] # Store CoinRecord in DB and ram cache async def add_coin_record(self, record: WalletCoinRecord, name: Optional[bytes32] = None) -> None: - # update wallet cache if name is None: name = record.name() - self.coin_record_cache[name] = record - if record.wallet_id in self.unspent_coin_wallet_cache: - if record.spent and name in self.unspent_coin_wallet_cache[record.wallet_id]: - self.unspent_coin_wallet_cache[record.wallet_id].pop(name) - if not record.spent: - self.unspent_coin_wallet_cache[record.wallet_id][name] = record - else: - if not record.spent: - self.unspent_coin_wallet_cache[record.wallet_id] = {} - self.unspent_coin_wallet_cache[record.wallet_id][name] = record - + assert record.spent == (record.spent_block_height != 0) cursor = await self.db_connection.execute( "INSERT OR REPLACE INTO coin_record VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( @@ -133,13 +100,6 @@ class WalletCoinStore: # Sometimes we realize that a coin is actually not interesting to us so we need to delete it async def delete_coin_record(self, coin_name: bytes32) -> None: - if coin_name in self.coin_record_cache: - coin_record = self.coin_record_cache.pop(coin_name) - if coin_record.wallet_id in self.unspent_coin_wallet_cache: - coin_cache = self.unspent_coin_wallet_cache[coin_record.wallet_id] - if coin_name in coin_cache: - coin_cache.pop(coin_record.coin.name()) - c = await self.db_connection.execute("DELETE FROM coin_record WHERE coin_name=?", (coin_name.hex(),)) await c.close() @@ -170,74 +130,57 @@ class WalletCoinStore: async def get_coin_record(self, coin_name: bytes32) -> Optional[WalletCoinRecord]: """Returns CoinRecord with specified coin id.""" - if coin_name in self.coin_record_cache: - return self.coin_record_cache[coin_name] - cursor = await self.db_connection.execute("SELECT * from coin_record WHERE coin_name=?", (coin_name.hex(),)) - row = await cursor.fetchone() - await cursor.close() + rows = list( + await self.db_connection.execute_fetchall("SELECT * from coin_record WHERE coin_name=?", (coin_name.hex(),)) + ) - if row is None: + if len(rows) == 0: return None - return self.coin_record_from_row(row) + return self.coin_record_from_row(rows[0]) async def get_first_coin_height(self) -> Optional[uint32]: """Returns height of first confirmed coin""" - cursor = await self.db_connection.execute("SELECT MIN(confirmed_height) FROM coin_record;") - row = await cursor.fetchone() - await cursor.close() + rows = list(await self.db_connection.execute_fetchall("SELECT MIN(confirmed_height) FROM coin_record")) - if row is not None and row[0] is not None: - return uint32(row[0]) + if len(rows) != 0 and rows[0][0] is not None: + return uint32(rows[0][0]) return None async def get_unspent_coins_for_wallet(self, wallet_id: int) -> Set[WalletCoinRecord]: """Returns set of CoinRecords that have not been spent yet for a wallet.""" - if wallet_id in self.unspent_coin_wallet_cache: - wallet_coins: Dict[bytes32, WalletCoinRecord] = self.unspent_coin_wallet_cache[wallet_id] - return set(wallet_coins.values()) - else: - return set() - - async def get_all_coins(self) -> Set[WalletCoinRecord]: - """Returns set of all CoinRecords.""" - cursor = await self.db_connection.execute("SELECT * from coin_record") - rows = await cursor.fetchall() - await cursor.close() - + rows = await self.db_connection.execute_fetchall( + "SELECT * FROM coin_record WHERE wallet_id=? AND spent_height=0", (wallet_id,) + ) return set(self.coin_record_from_row(row) for row in rows) async def get_coins_to_check(self, check_height) -> Set[WalletCoinRecord]: """Returns set of all CoinRecords.""" - cursor = await self.db_connection.execute( + rows = await self.db_connection.execute_fetchall( "SELECT * from coin_record where spent_height=0 or spent_height>? or confirmed_height>?", ( check_height, check_height, ), ) - rows = await cursor.fetchall() - await cursor.close() return set(self.coin_record_from_row(row) for row in rows) # Checks DB and DiffStores for CoinRecords with puzzle_hash and returns them async def get_coin_records_by_puzzle_hash(self, puzzle_hash: bytes32) -> List[WalletCoinRecord]: """Returns a list of all coin records with the given puzzle hash""" - cursor = await self.db_connection.execute("SELECT * from coin_record WHERE puzzle_hash=?", (puzzle_hash.hex(),)) - rows = await cursor.fetchall() - await cursor.close() + rows = await self.db_connection.execute_fetchall( + "SELECT * from coin_record WHERE puzzle_hash=?", (puzzle_hash.hex(),) + ) return [self.coin_record_from_row(row) for row in rows] # Checks DB and DiffStores for CoinRecords with parent_coin_info and returns them async def get_coin_records_by_parent_id(self, parent_coin_info: bytes32) -> List[WalletCoinRecord]: """Returns a list of all coin records with the given parent id""" - cursor = await self.db_connection.execute( + rows = await self.db_connection.execute_fetchall( "SELECT * from coin_record WHERE coin_parent=?", (parent_coin_info.hex(),) ) - rows = await cursor.fetchall() - await cursor.close() return [self.coin_record_from_row(row) for row in rows] @@ -246,31 +189,6 @@ class WalletCoinStore: Rolls back the blockchain to block_index. All coins confirmed after this point are removed. All coins spent after this point are set to unspent. Can be -1 (rollback all) """ - # Delete from storage - delete_queue: List[WalletCoinRecord] = [] - for coin_name, coin_record in self.coin_record_cache.items(): - if coin_record.spent_block_height > height: - new_record = WalletCoinRecord( - coin_record.coin, - coin_record.confirmed_block_height, - uint32(0), - False, - coin_record.coinbase, - coin_record.wallet_type, - coin_record.wallet_id, - ) - self.coin_record_cache[coin_record.coin.name()] = new_record - if coin_record.wallet_id in self.unspent_coin_wallet_cache: - self.unspent_coin_wallet_cache[coin_record.wallet_id][coin_record.coin.name()] = new_record - if coin_record.confirmed_block_height > height: - delete_queue.append(coin_record) - - for coin_record in delete_queue: - self.coin_record_cache.pop(coin_record.coin.name()) - if coin_record.wallet_id in self.unspent_coin_wallet_cache: - coin_cache = self.unspent_coin_wallet_cache[coin_record.wallet_id] - if coin_record.coin.name() in coin_cache: - coin_cache.pop(coin_record.coin.name()) c1 = await self.db_connection.execute("DELETE FROM coin_record WHERE confirmed_height>?", (height,)) await c1.close() diff --git a/chia/wallet/wallet_node.py b/chia/wallet/wallet_node.py index 833ad0f89f..5f6653b65b 100644 --- a/chia/wallet/wallet_node.py +++ b/chia/wallet/wallet_node.py @@ -514,7 +514,6 @@ class WalletNode: tb = traceback.format_exc() self.log.error(f"Exception while perform_atomic_rollback: {e} {tb}") await self.wallet_state_manager.db_wrapper.rollback_transaction() - await self.wallet_state_manager.coin_store.rebuild_wallet_cache() await self.wallet_state_manager.tx_store.rebuild_tx_cache() await self.wallet_state_manager.pool_store.rebuild_cache() raise @@ -712,7 +711,6 @@ class WalletNode: tb = traceback.format_exc() self.log.error(f"Exception while adding state: {e} {tb}") await self.wallet_state_manager.db_wrapper.rollback_transaction() - await self.wallet_state_manager.coin_store.rebuild_wallet_cache() await self.wallet_state_manager.tx_store.rebuild_tx_cache() await self.wallet_state_manager.pool_store.rebuild_cache() else: @@ -749,7 +747,6 @@ class WalletNode: await self.wallet_state_manager.db_wrapper.commit_transaction() except Exception as e: await self.wallet_state_manager.db_wrapper.rollback_transaction() - await self.wallet_state_manager.coin_store.rebuild_wallet_cache() await self.wallet_state_manager.tx_store.rebuild_tx_cache() await self.wallet_state_manager.pool_store.rebuild_cache() tb = traceback.format_exc() diff --git a/tests/wallet/test_wallet_coin_store.py b/tests/wallet/test_wallet_coin_store.py index 00b7e28bd6..cc7b21730a 100644 --- a/tests/wallet/test_wallet_coin_store.py +++ b/tests/wallet/test_wallet_coin_store.py @@ -40,7 +40,7 @@ record_4 = WalletCoinRecord( record_5 = WalletCoinRecord( coin_5, uint32(5), - uint32(15), + uint32(0), False, False, WalletType.STANDARD_WALLET, @@ -58,7 +58,7 @@ record_6 = WalletCoinRecord( record_7 = WalletCoinRecord( coin_7, uint32(5), - uint32(15), + uint32(0), False, False, WalletType.POOLING_WALLET, @@ -125,10 +125,10 @@ async def test_get_unspent_coins_for_wallet() -> None: assert await store.get_unspent_coins_for_wallet(1) == set() - await store.add_coin_record(record_4) # this is spent - await store.add_coin_record(record_5) - await store.add_coin_record(record_6) # this is spent - await store.add_coin_record(record_7) + await store.add_coin_record(record_4) # this is spent and wallet 0 + await store.add_coin_record(record_5) # wallet 1 + await store.add_coin_record(record_6) # this is spent and wallet 2 + await store.add_coin_record(record_7) # wallet 2 assert await store.get_unspent_coins_for_wallet(1) == set([record_5]) assert await store.get_unspent_coins_for_wallet(2) == set([record_7]) From 0e669f7a3855d3c334ba609207092347854252de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Jun 2022 02:47:55 -0500 Subject: [PATCH 05/62] Bump plist from 3.0.4 to 3.0.5 in /build_scripts/npm_macos_m1 (#10899) Bumps [plist](https://github.com/TooTallNate/node-plist) from 3.0.4 to 3.0.5. - [Release notes](https://github.com/TooTallNate/node-plist/releases) - [Changelog](https://github.com/TooTallNate/plist.js/blob/master/History.md) - [Commits](https://github.com/TooTallNate/node-plist/commits) --- updated-dependencies: - dependency-name: plist dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build_scripts/npm_macos_m1/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build_scripts/npm_macos_m1/package-lock.json b/build_scripts/npm_macos_m1/package-lock.json index 66df6b7672..f9ce1ec621 100644 --- a/build_scripts/npm_macos_m1/package-lock.json +++ b/build_scripts/npm_macos_m1/package-lock.json @@ -7279,9 +7279,9 @@ } }, "node_modules/plist": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.4.tgz", - "integrity": "sha512-ksrr8y9+nXOxQB2osVNqrgvX/XQPOXaU4BQMKjYq8PvaY1U18mo+fKgBSwzK+luSyinOuPae956lSVcBwxlAMg==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.5.tgz", + "integrity": "sha512-83vX4eYdQp3vP9SxuYgEM/G/pJQqLUz/V/xzPrzruLs7fz7jxGQ1msZ/mg1nwZxUSuOp4sb+/bEIbRrbzZRxDA==", "dependencies": { "base64-js": "^1.5.1", "xmlbuilder": "^9.0.7" @@ -14859,9 +14859,9 @@ } }, "plist": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.4.tgz", - "integrity": "sha512-ksrr8y9+nXOxQB2osVNqrgvX/XQPOXaU4BQMKjYq8PvaY1U18mo+fKgBSwzK+luSyinOuPae956lSVcBwxlAMg==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.5.tgz", + "integrity": "sha512-83vX4eYdQp3vP9SxuYgEM/G/pJQqLUz/V/xzPrzruLs7fz7jxGQ1msZ/mg1nwZxUSuOp4sb+/bEIbRrbzZRxDA==", "requires": { "base64-js": "^1.5.1", "xmlbuilder": "^9.0.7" From 94405c5737ba1dba3e759609cc7126be55256159 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Jun 2022 02:48:23 -0500 Subject: [PATCH 06/62] Bump plist from 3.0.4 to 3.0.5 in /build_scripts/npm_macos (#10898) Bumps [plist](https://github.com/TooTallNate/node-plist) from 3.0.4 to 3.0.5. - [Release notes](https://github.com/TooTallNate/node-plist/releases) - [Changelog](https://github.com/TooTallNate/plist.js/blob/master/History.md) - [Commits](https://github.com/TooTallNate/node-plist/commits) --- updated-dependencies: - dependency-name: plist dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build_scripts/npm_macos/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build_scripts/npm_macos/package-lock.json b/build_scripts/npm_macos/package-lock.json index acfd2b6d24..f8cf4d61e0 100644 --- a/build_scripts/npm_macos/package-lock.json +++ b/build_scripts/npm_macos/package-lock.json @@ -7311,9 +7311,9 @@ } }, "node_modules/plist": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.4.tgz", - "integrity": "sha512-ksrr8y9+nXOxQB2osVNqrgvX/XQPOXaU4BQMKjYq8PvaY1U18mo+fKgBSwzK+luSyinOuPae956lSVcBwxlAMg==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.5.tgz", + "integrity": "sha512-83vX4eYdQp3vP9SxuYgEM/G/pJQqLUz/V/xzPrzruLs7fz7jxGQ1msZ/mg1nwZxUSuOp4sb+/bEIbRrbzZRxDA==", "dependencies": { "base64-js": "^1.5.1", "xmlbuilder": "^9.0.7" @@ -14931,9 +14931,9 @@ } }, "plist": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.4.tgz", - "integrity": "sha512-ksrr8y9+nXOxQB2osVNqrgvX/XQPOXaU4BQMKjYq8PvaY1U18mo+fKgBSwzK+luSyinOuPae956lSVcBwxlAMg==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.5.tgz", + "integrity": "sha512-83vX4eYdQp3vP9SxuYgEM/G/pJQqLUz/V/xzPrzruLs7fz7jxGQ1msZ/mg1nwZxUSuOp4sb+/bEIbRrbzZRxDA==", "requires": { "base64-js": "^1.5.1", "xmlbuilder": "^9.0.7" From a54901fedb816329f42a9ab3b583389076b86001 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Jun 2022 11:56:22 -0500 Subject: [PATCH 07/62] Bump plist from 3.0.4 to 3.0.5 in /build_scripts/npm_linux_deb (#10895) Bumps [plist](https://github.com/TooTallNate/node-plist) from 3.0.4 to 3.0.5. - [Release notes](https://github.com/TooTallNate/node-plist/releases) - [Changelog](https://github.com/TooTallNate/plist.js/blob/master/History.md) - [Commits](https://github.com/TooTallNate/node-plist/commits) --- updated-dependencies: - dependency-name: plist dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build_scripts/npm_linux_deb/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build_scripts/npm_linux_deb/package-lock.json b/build_scripts/npm_linux_deb/package-lock.json index 125349dd7f..f463a96ada 100644 --- a/build_scripts/npm_linux_deb/package-lock.json +++ b/build_scripts/npm_linux_deb/package-lock.json @@ -6216,9 +6216,9 @@ } }, "node_modules/plist": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.4.tgz", - "integrity": "sha512-ksrr8y9+nXOxQB2osVNqrgvX/XQPOXaU4BQMKjYq8PvaY1U18mo+fKgBSwzK+luSyinOuPae956lSVcBwxlAMg==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.5.tgz", + "integrity": "sha512-83vX4eYdQp3vP9SxuYgEM/G/pJQqLUz/V/xzPrzruLs7fz7jxGQ1msZ/mg1nwZxUSuOp4sb+/bEIbRrbzZRxDA==", "dependencies": { "base64-js": "^1.5.1", "xmlbuilder": "^9.0.7" @@ -12928,9 +12928,9 @@ } }, "plist": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.4.tgz", - "integrity": "sha512-ksrr8y9+nXOxQB2osVNqrgvX/XQPOXaU4BQMKjYq8PvaY1U18mo+fKgBSwzK+luSyinOuPae956lSVcBwxlAMg==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.5.tgz", + "integrity": "sha512-83vX4eYdQp3vP9SxuYgEM/G/pJQqLUz/V/xzPrzruLs7fz7jxGQ1msZ/mg1nwZxUSuOp4sb+/bEIbRrbzZRxDA==", "requires": { "base64-js": "^1.5.1", "xmlbuilder": "^9.0.7" From 426d35b8d447332729963f3ba8e4232165859c95 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Thu, 30 Jun 2022 19:09:55 +0200 Subject: [PATCH 08/62] remove wallet transaction store cache (#12134) * remove transaction cache from WalletTransactionStore. rely on the sqlite DB * optimze aiosqlite use in wallet_transaction_store --- chia/rpc/wallet_rpc_api.py | 2 - chia/wallet/wallet_node.py | 3 - chia/wallet/wallet_transaction_store.py | 160 +++++------------------- 3 files changed, 31 insertions(+), 134 deletions(-) diff --git a/chia/rpc/wallet_rpc_api.py b/chia/rpc/wallet_rpc_api.py index c953f81b48..6298ff4843 100644 --- a/chia/rpc/wallet_rpc_api.py +++ b/chia/rpc/wallet_rpc_api.py @@ -841,8 +841,6 @@ class WalletRpcApi: if self.service.wallet_state_manager.wallets[wallet_id].type() == WalletType.POOLING_WALLET.value: self.service.wallet_state_manager.wallets[wallet_id].target_state = None await self.service.wallet_state_manager.tx_store.db_wrapper.commit_transaction() - # Update the cache - await self.service.wallet_state_manager.tx_store.rebuild_tx_cache() return {} async def select_coins(self, request) -> Dict[str, object]: diff --git a/chia/wallet/wallet_node.py b/chia/wallet/wallet_node.py index 5f6653b65b..b0c7abbda8 100644 --- a/chia/wallet/wallet_node.py +++ b/chia/wallet/wallet_node.py @@ -514,7 +514,6 @@ class WalletNode: tb = traceback.format_exc() self.log.error(f"Exception while perform_atomic_rollback: {e} {tb}") await self.wallet_state_manager.db_wrapper.rollback_transaction() - await self.wallet_state_manager.tx_store.rebuild_tx_cache() await self.wallet_state_manager.pool_store.rebuild_cache() raise else: @@ -711,7 +710,6 @@ class WalletNode: tb = traceback.format_exc() self.log.error(f"Exception while adding state: {e} {tb}") await self.wallet_state_manager.db_wrapper.rollback_transaction() - await self.wallet_state_manager.tx_store.rebuild_tx_cache() await self.wallet_state_manager.pool_store.rebuild_cache() else: await self.wallet_state_manager.blockchain.clean_block_records() @@ -747,7 +745,6 @@ class WalletNode: await self.wallet_state_manager.db_wrapper.commit_transaction() except Exception as e: await self.wallet_state_manager.db_wrapper.rollback_transaction() - await self.wallet_state_manager.tx_store.rebuild_tx_cache() await self.wallet_state_manager.pool_store.rebuild_cache() tb = traceback.format_exc() self.log.error(f"Error adding states.. {e} {tb}") diff --git a/chia/wallet/wallet_transaction_store.py b/chia/wallet/wallet_transaction_store.py index 02b330f311..c121b94b14 100644 --- a/chia/wallet/wallet_transaction_store.py +++ b/chia/wallet/wallet_transaction_store.py @@ -30,9 +30,7 @@ class WalletTransactionStore: db_connection: aiosqlite.Connection db_wrapper: DBWrapper - tx_record_cache: Dict[bytes32, TransactionRecord] tx_submitted: Dict[bytes32, Tuple[int, int]] # tx_id: [time submitted: count] - unconfirmed_for_wallet: Dict[int, Dict[bytes32, TransactionRecord]] last_wallet_tx_resend_time: int # Epoch time in seconds @classmethod @@ -87,26 +85,10 @@ class WalletTransactionStore: ) await self.db_connection.commit() - self.tx_record_cache = {} self.tx_submitted = {} - self.unconfirmed_for_wallet = {} self.last_wallet_tx_resend_time = int(time.time()) - await self.rebuild_tx_cache() return self - async def rebuild_tx_cache(self): - # init cache here - all_records = await self.get_all_transactions() - self.tx_record_cache = {} - self.unconfirmed_for_wallet = {} - - for record in all_records: - self.tx_record_cache[record.name] = record - if record.wallet_id not in self.unconfirmed_for_wallet: - self.unconfirmed_for_wallet[record.wallet_id] = {} - if not record.confirmed: - self.unconfirmed_for_wallet[record.wallet_id][record.name] = record - async def _clear_database(self): cursor = await self.db_connection.execute("DELETE FROM transaction_record") await cursor.close() @@ -116,19 +98,10 @@ class WalletTransactionStore: """ Store TransactionRecord in DB and Cache. """ - self.tx_record_cache[record.name] = record - if record.wallet_id not in self.unconfirmed_for_wallet: - self.unconfirmed_for_wallet[record.wallet_id] = {} - unconfirmed_dict = self.unconfirmed_for_wallet[record.wallet_id] - if record.confirmed and record.name in unconfirmed_dict: - unconfirmed_dict.pop(record.name) - if not record.confirmed: - unconfirmed_dict[record.name] = record - if not in_transaction: await self.db_wrapper.lock.acquire() try: - cursor = await self.db_connection.execute( + await self.db_connection.execute_insert( "INSERT OR REPLACE INTO transaction_record VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( bytes(record), @@ -145,25 +118,13 @@ class WalletTransactionStore: record.type, ), ) - await cursor.close() if not in_transaction: await self.db_connection.commit() - except BaseException: - if not in_transaction: - await self.rebuild_tx_cache() - raise finally: if not in_transaction: self.db_wrapper.lock.release() async def delete_transaction_record(self, tx_id: bytes32) -> None: - if tx_id in self.tx_record_cache: - tx_record = self.tx_record_cache.pop(tx_id) - if tx_record.wallet_id in self.unconfirmed_for_wallet: - tx_cache = self.unconfirmed_for_wallet[tx_record.wallet_id] - if tx_id in tx_cache: - tx_cache.pop(tx_id) - c = await self.db_connection.execute("DELETE FROM transaction_record WHERE bundle_id=?", (tx_id,)) await c.close() @@ -277,16 +238,12 @@ class WalletTransactionStore: """ Checks DB and cache for TransactionRecord with id: id and returns it. """ - if tx_id in self.tx_record_cache: - return self.tx_record_cache[tx_id] - # NOTE: bundle_id is being stored as bytes, not hex - cursor = await self.db_connection.execute("SELECT * from transaction_record WHERE bundle_id=?", (tx_id,)) - row = await cursor.fetchone() - await cursor.close() - if row is not None: - record = TransactionRecord.from_bytes(row[0]) - return record + rows = list( + await self.db_connection.execute_fetchall("SELECT * from transaction_record WHERE bundle_id=?", (tx_id,)) + ) + if len(rows) > 0: + return TransactionRecord.from_bytes(rows[0][0]) return None async def get_not_sent(self, *, include_accepted_txs=False) -> List[TransactionRecord]: @@ -294,12 +251,10 @@ class WalletTransactionStore: Returns the list of transactions that have not been received by full node yet. """ current_time = int(time.time()) - cursor = await self.db_connection.execute( + rows = await self.db_connection.execute_fetchall( "SELECT * from transaction_record WHERE confirmed=?", (0,), ) - rows = await cursor.fetchall() - await cursor.close() records = [] for row in rows: @@ -331,11 +286,9 @@ class WalletTransactionStore: """ fee_int = TransactionType.FEE_REWARD.value pool_int = TransactionType.COINBASE_REWARD.value - cursor = await self.db_connection.execute( + rows = await self.db_connection.execute_fetchall( "SELECT * from transaction_record WHERE confirmed=? and (type=? or type=?)", (1, fee_int, pool_int) ) - rows = await cursor.fetchall() - await cursor.close() records = [] for row in rows: @@ -349,9 +302,7 @@ class WalletTransactionStore: Returns the list of all transaction that have not yet been confirmed. """ - cursor = await self.db_connection.execute("SELECT * from transaction_record WHERE confirmed=?", (0,)) - rows = await cursor.fetchall() - await cursor.close() + rows = await self.db_connection.execute_fetchall("SELECT * from transaction_record WHERE confirmed=?", (0,)) records = [] for row in rows: @@ -364,10 +315,10 @@ class WalletTransactionStore: """ Returns the list of transaction that have not yet been confirmed. """ - if wallet_id in self.unconfirmed_for_wallet: - return list(self.unconfirmed_for_wallet[wallet_id].values()) - else: - return [] + rows = await self.db_connection.execute_fetchall( + "SELECT transaction_record from transaction_record WHERE confirmed=0 AND wallet_id=?", (wallet_id,) + ) + return [TransactionRecord.from_bytes(row[0]) for row in rows] async def get_transactions_between( self, wallet_id: int, start, end, sort_key=None, reverse=False, to_puzzle_hash: Optional[bytes32] = None @@ -392,113 +343,64 @@ class WalletTransactionStore: else: query_str = SortKey[sort_key].ascending() - cursor = await self.db_connection.execute( + rows = await self.db_connection.execute_fetchall( f"SELECT * from transaction_record where wallet_id=?{puzz_hash_where}" f" {query_str}, rowid" f" LIMIT {start}, {limit}", (wallet_id,), ) - rows = await cursor.fetchall() - await cursor.close() - records = [] - for row in rows: - record = TransactionRecord.from_bytes(row[0]) - records.append(record) - - return records + return [TransactionRecord.from_bytes(row[0]) for row in rows] async def get_transaction_count_for_wallet(self, wallet_id) -> int: - cursor = await self.db_connection.execute( - "SELECT COUNT(*) FROM transaction_record where wallet_id=?", (wallet_id,) + rows = list( + await self.db_connection.execute_fetchall( + "SELECT COUNT(*) FROM transaction_record where wallet_id=?", (wallet_id,) + ) ) - count_result = await cursor.fetchone() - if count_result is not None: - count = count_result[0] - else: - count = 0 - await cursor.close() - return count + return 0 if len(rows) == 0 else rows[0][0] async def get_all_transactions_for_wallet(self, wallet_id: int, type: int = None) -> List[TransactionRecord]: """ Returns all stored transactions. """ if type is None: - cursor = await self.db_connection.execute( + rows = await self.db_connection.execute_fetchall( "SELECT * from transaction_record where wallet_id=?", (wallet_id,) ) else: - cursor = await self.db_connection.execute( + rows = await self.db_connection.execute_fetchall( "SELECT * from transaction_record where wallet_id=? and type=?", ( wallet_id, type, ), ) - rows = await cursor.fetchall() - await cursor.close() - records = [] - - cache_set = set() - for row in rows: - record = TransactionRecord.from_bytes(row[0]) - records.append(record) - cache_set.add(record.name) - - return records + return [TransactionRecord.from_bytes(row[0]) for row in rows] async def get_all_transactions(self) -> List[TransactionRecord]: """ Returns all stored transactions. """ - cursor = await self.db_connection.execute("SELECT * from transaction_record") - rows = await cursor.fetchall() - await cursor.close() - records = [] - - for row in rows: - record = TransactionRecord.from_bytes(row[0]) - records.append(record) - - return records + rows = await self.db_connection.execute_fetchall("SELECT * from transaction_record") + return [TransactionRecord.from_bytes(row[0]) for row in rows] async def get_transaction_above(self, height: int) -> List[TransactionRecord]: # Can be -1 (get all tx) - cursor = await self.db_connection.execute( + rows = await self.db_connection.execute_fetchall( "SELECT * from transaction_record WHERE confirmed_at_height>?", (height,) ) - rows = await cursor.fetchall() - await cursor.close() - records = [] - - for row in rows: - record = TransactionRecord.from_bytes(row[0]) - records.append(record) - - return records + return [TransactionRecord.from_bytes(row[0]) for row in rows] async def get_transactions_by_trade_id(self, trade_id: bytes32) -> List[TransactionRecord]: - cursor = await self.db_connection.execute("SELECT * from transaction_record WHERE trade_id=?", (trade_id,)) - rows = await cursor.fetchall() - await cursor.close() - records = [] - - for row in rows: - record = TransactionRecord.from_bytes(row[0]) - records.append(record) - - return records + rows = await self.db_connection.execute_fetchall( + "SELECT * from transaction_record WHERE trade_id=?", (trade_id,) + ) + return [TransactionRecord.from_bytes(row[0]) for row in rows] async def rollback_to_block(self, height: int): # Delete from storage - to_delete = [] - for tx in self.tx_record_cache.values(): - if tx.confirmed_at_height > height: - to_delete.append(tx) - for tx in to_delete: - self.tx_record_cache.pop(tx.name) self.tx_submitted = {} c1 = await self.db_connection.execute("DELETE FROM transaction_record WHERE confirmed_at_height>?", (height,)) await c1.close() From c78a560fd9296b900d58741346bd249bf83889a4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Jun 2022 14:41:25 -0500 Subject: [PATCH 09/62] Bump plist from 3.0.4 to 3.0.5 in /build_scripts/npm_windows (#10897) Bumps [plist](https://github.com/TooTallNate/node-plist) from 3.0.4 to 3.0.5. - [Release notes](https://github.com/TooTallNate/node-plist/releases) - [Changelog](https://github.com/TooTallNate/plist.js/blob/master/History.md) - [Commits](https://github.com/TooTallNate/node-plist/commits) --- updated-dependencies: - dependency-name: plist dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build_scripts/npm_windows/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build_scripts/npm_windows/package-lock.json b/build_scripts/npm_windows/package-lock.json index ac5d260a90..9a50ef540f 100644 --- a/build_scripts/npm_windows/package-lock.json +++ b/build_scripts/npm_windows/package-lock.json @@ -5996,9 +5996,9 @@ } }, "node_modules/plist": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.4.tgz", - "integrity": "sha512-ksrr8y9+nXOxQB2osVNqrgvX/XQPOXaU4BQMKjYq8PvaY1U18mo+fKgBSwzK+luSyinOuPae956lSVcBwxlAMg==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.5.tgz", + "integrity": "sha512-83vX4eYdQp3vP9SxuYgEM/G/pJQqLUz/V/xzPrzruLs7fz7jxGQ1msZ/mg1nwZxUSuOp4sb+/bEIbRrbzZRxDA==", "dependencies": { "base64-js": "^1.5.1", "xmlbuilder": "^9.0.7" @@ -12495,9 +12495,9 @@ } }, "plist": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.4.tgz", - "integrity": "sha512-ksrr8y9+nXOxQB2osVNqrgvX/XQPOXaU4BQMKjYq8PvaY1U18mo+fKgBSwzK+luSyinOuPae956lSVcBwxlAMg==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.5.tgz", + "integrity": "sha512-83vX4eYdQp3vP9SxuYgEM/G/pJQqLUz/V/xzPrzruLs7fz7jxGQ1msZ/mg1nwZxUSuOp4sb+/bEIbRrbzZRxDA==", "requires": { "base64-js": "^1.5.1", "xmlbuilder": "^9.0.7" From 298d3c034cf93109411bcbf7e43a09f5d5e420b4 Mon Sep 17 00:00:00 2001 From: Mariano Sorgente <3069354+mariano54@users.noreply.github.com> Date: Fri, 1 Jul 2022 04:52:55 +0900 Subject: [PATCH 10/62] Fix issue with unsynced local node (#12159) --- chia/wallet/util/wallet_sync_utils.py | 4 +++- chia/wallet/wallet_node_api.py | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/chia/wallet/util/wallet_sync_utils.py b/chia/wallet/util/wallet_sync_utils.py index 4491f5342d..94968a702a 100644 --- a/chia/wallet/util/wallet_sync_utils.py +++ b/chia/wallet/util/wallet_sync_utils.py @@ -23,6 +23,8 @@ from chia.protocols.wallet_protocol import ( RespondToCoinUpdates, RespondHeaderBlocks, RequestHeaderBlocks, + RejectHeaderBlocks, + RejectBlockHeaders, ) from chia.server.ws_connection import WSChiaConnection from chia.types.blockchain_format.coin import hash_coin_ids, Coin @@ -325,7 +327,7 @@ async def request_header_blocks( response = await peer.request_block_headers(RequestBlockHeaders(start_height, end_height, False)) else: response = await peer.request_header_blocks(RequestHeaderBlocks(start_height, end_height)) - if response is None: + if response is None or isinstance(response, RejectBlockHeaders) or isinstance(response, RejectHeaderBlocks): return None return response.header_blocks diff --git a/chia/wallet/wallet_node_api.py b/chia/wallet/wallet_node_api.py index a576d2d2a5..38b6ec7d7b 100644 --- a/chia/wallet/wallet_node_api.py +++ b/chia/wallet/wallet_node_api.py @@ -148,6 +148,10 @@ class WalletNodeAPI: async def reject_header_blocks(self, request: wallet_protocol.RejectHeaderBlocks): self.log.warning(f"Reject header blocks: {request}") + @api_request + async def reject_block_headers(self, request: wallet_protocol.RejectBlockHeaders): + pass + @execute_task @peer_required @api_request From 7b84c8e26102799174bcc3ab499a48e4f6927981 Mon Sep 17 00:00:00 2001 From: Mariano Sorgente <3069354+mariano54@users.noreply.github.com> Date: Fri, 1 Jul 2022 04:53:29 +0900 Subject: [PATCH 11/62] Remove double bytes32 conversion (#12161) --- chia/util/streamable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chia/util/streamable.py b/chia/util/streamable.py index 5b948816e6..3e93306a43 100644 --- a/chia/util/streamable.py +++ b/chia/util/streamable.py @@ -645,7 +645,7 @@ class Streamable: stream_func(getattr(self, field.name), f) def get_hash(self) -> bytes32: - return bytes32(std_hash(bytes(self), skip_bytes_conversion=True)) + return std_hash(bytes(self), skip_bytes_conversion=True) @classmethod def from_bytes(cls: Any, blob: bytes) -> Any: From 16aafffb71d297f27ca4e99cba69a7efc3033a3e Mon Sep 17 00:00:00 2001 From: Sebastjan Trepca Date: Thu, 30 Jun 2022 21:53:53 +0200 Subject: [PATCH 12/62] fix for transfer nft with did (#12163) --- tests/wallet/nft_wallet/test_nft_wallet.py | 36 ++++++++-------------- 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/tests/wallet/nft_wallet/test_nft_wallet.py b/tests/wallet/nft_wallet/test_nft_wallet.py index 15f9c6c1e9..06911f72ed 100644 --- a/tests/wallet/nft_wallet/test_nft_wallet.py +++ b/tests/wallet/nft_wallet/test_nft_wallet.py @@ -13,6 +13,7 @@ from chia.simulator.simulator_protocol import FarmNewBlockProtocol from chia.types.blockchain_format.program import Program from chia.types.blockchain_format.sized_bytes import bytes32 from chia.types.peer_info import PeerInfo +from chia.types.spend_bundle import SpendBundle from chia.util.bech32m import encode_puzzle_hash from chia.util.byte_types import hexstr_to_bytes from chia.util.ints import uint16, uint32, uint64 @@ -50,9 +51,10 @@ async def wait_rpc_state_condition( return {} -async def make_new_block_with(resp, full_node_api, ph): +async def make_new_block_with(resp: Dict, full_node_api: FullNodeSimulator, ph: bytes32) -> SpendBundle: assert resp.get("success") sb = resp["spend_bundle"] + assert isinstance(sb, SpendBundle) await time_out_assert_not_none(5, full_node_api.full_node.mempool_manager.get_spendbundle, sb.name()) await full_node_api.farm_new_transaction_block(FarmNewBlockProtocol(ph)) return sb @@ -863,8 +865,7 @@ async def test_nft_transfer_nft_with_did(two_wallet_nodes: Any, trusted: Any) -> spend_bundle = spend_bundle_list[0].spend_bundle await time_out_assert_not_none(5, full_node_api.full_node.mempool_manager.get_spendbundle, spend_bundle.name()) - for _ in range(1, num_blocks): - await full_node_api.farm_new_transaction_block(FarmNewBlockProtocol(ph)) + await full_node_api.farm_new_transaction_block(FarmNewBlockProtocol(ph)) await time_out_assert(15, wallet_0.get_pending_change_balance, 0) hex_did_id = did_wallet.get_my_DID() hmr_did_id = encode_puzzle_hash(bytes32.from_hexstr(hex_did_id), DID_HRP) @@ -886,22 +887,14 @@ async def test_nft_transfer_nft_with_did(two_wallet_nodes: Any, trusted: Any) -> "fee": fee, } ) - assert resp.get("success") - sb = resp["spend_bundle"] - - # ensure hints are generated - assert compute_memos(sb) - await time_out_assert_not_none(5, full_node_api.full_node.mempool_manager.get_spendbundle, sb.name()) - - for i in range(1, num_blocks): - await full_node_api.farm_new_transaction_block(FarmNewBlockProtocol(ph1)) + await make_new_block_with(resp, full_node_api, ph1) # Check DID NFT coins_response = await wait_rpc_state_condition( 5, api_0.nft_get_nfts, [dict(wallet_id=nft_wallet_0_id)], lambda x: x["nft_list"] ) - await time_out_assert(10, wallet_0.get_unconfirmed_balance, 7999999999898) - await time_out_assert(10, wallet_0.get_confirmed_balance, 7999999999898) + await time_out_assert(10, wallet_0.get_unconfirmed_balance, 5999999999898) + await time_out_assert(10, wallet_0.get_confirmed_balance, 5999999999898) coins = coins_response["nft_list"] assert len(coins) == 1 assert coins[0].owner_did.hex() == hex_did_id @@ -929,8 +922,8 @@ async def test_nft_transfer_nft_with_did(two_wallet_nodes: Any, trusted: Any) -> await wait_rpc_state_condition( 5, api_0.nft_get_nfts, [dict(wallet_id=nft_wallet_0_id)], lambda x: not x["nft_list"] ) - await time_out_assert(10, wallet_0.get_unconfirmed_balance, 7999999999798) - await time_out_assert(10, wallet_0.get_confirmed_balance, 7999999999798) + await time_out_assert(10, wallet_0.get_unconfirmed_balance, 5999999999798) + await time_out_assert(10, wallet_0.get_confirmed_balance, 5999999999798) # wait for all wallets to be created await time_out_assert(10, len, 3, wallet_1.wallet_state_manager.wallets) did_wallet_1 = wallet_1.wallet_state_manager.wallets[3] @@ -946,22 +939,19 @@ async def test_nft_transfer_nft_with_did(two_wallet_nodes: Any, trusted: Any) -> assert coins_response["nft_list"][0].owner_did is None nft_coin_id = coins_response["nft_list"][0].nft_coin_id - await time_out_assert(10, did_wallet_1.get_spendable_balance, 1) + await time_out_assert(20, did_wallet_1.get_spendable_balance, 1) # Set DID resp = await api_1.nft_set_nft_did( dict(wallet_id=nft_wallet_id_1, did_id=hmr_did_id, nft_coin_id=nft_coin_id.hex(), fee=fee) ) - assert resp.get("success") - - for i in range(1, num_blocks): - await full_node_api.farm_new_transaction_block(FarmNewBlockProtocol(ph)) + await make_new_block_with(resp, full_node_api, ph) coins_response = await wait_rpc_state_condition( 5, api_1.nft_get_by_did, [dict(did_id=hmr_did_id)], lambda x: x.get("wallet_id", 0) > 0 ) - await time_out_assert(10, wallet_1.get_unconfirmed_balance, 12000000000100) - await time_out_assert(10, wallet_1.get_confirmed_balance, 12000000000100) + await time_out_assert(10, wallet_1.get_unconfirmed_balance, 10000000000100) + await time_out_assert(10, wallet_1.get_confirmed_balance, 10000000000100) nft_wallet_1_id = coins_response.get("wallet_id") assert nft_wallet_1_id # Check NFT DID is set now From 7550a13eba4a84dc66fb6c1e02347fdb7ad5c0fd Mon Sep 17 00:00:00 2001 From: Mariano Sorgente <3069354+mariano54@users.noreply.github.com> Date: Fri, 1 Jul 2022 10:08:13 +0900 Subject: [PATCH 13/62] Fix some more flakiness (#12160) * Fix some more flakiness * Fixed few other flakes * Revert debug change * Fix NFT flaky test * Try again to fix the wallet RPC test * More increased timeouts --- tests/core/server/test_rate_limits.py | 18 ++++---- tests/wallet/rpc/test_wallet_rpc.py | 8 ++++ tests/wallet/sync/test_wallet_sync.py | 5 +- tests/wallet/test_wallet.py | 66 +++++++++++++-------------- 4 files changed, 52 insertions(+), 45 deletions(-) diff --git a/tests/core/server/test_rate_limits.py b/tests/core/server/test_rate_limits.py index f29c2bbcb1..67b5862522 100644 --- a/tests/core/server/test_rate_limits.py +++ b/tests/core/server/test_rate_limits.py @@ -35,11 +35,11 @@ class TestRateLimits: # Too many messages r = RateLimiter(incoming=True) new_tx_message = make_msg(ProtocolMessageTypes.new_transaction, bytes([1] * 40)) - for i in range(4900): + for i in range(4999): assert r.process_msg_and_check(new_tx_message, rl_v2, rl_v2) saw_disconnect = False - for i in range(4900): + for i in range(4999): response = r.process_msg_and_check(new_tx_message, rl_v2, rl_v2) if not response: saw_disconnect = True @@ -48,7 +48,7 @@ class TestRateLimits: # Non-tx message r = RateLimiter(incoming=True) new_peak_message = make_msg(ProtocolMessageTypes.new_peak, bytes([1] * 40)) - for i in range(20): + for i in range(200): assert r.process_msg_and_check(new_peak_message, rl_v2, rl_v2) saw_disconnect = False @@ -80,7 +80,7 @@ class TestRateLimits: # Too much data r = RateLimiter(incoming=True) tx_message = make_msg(ProtocolMessageTypes.respond_transaction, bytes([1] * 500 * 1024)) - for i in range(10): + for i in range(40): assert r.process_msg_and_check(tx_message, rl_v2, rl_v2) saw_disconnect = False @@ -110,14 +110,14 @@ class TestRateLimits: message_2 = make_msg(ProtocolMessageTypes.request_blocks, bytes([1] * 64)) message_3 = make_msg(ProtocolMessageTypes.plot_sync_start, bytes([1] * 64)) - for i in range(450): + for i in range(500): assert r.process_msg_and_check(message_1, rl_v2, rl_v2) - for i in range(450): + for i in range(500): assert r.process_msg_and_check(message_2, rl_v2, rl_v2) saw_disconnect = False - for i in range(450): + for i in range(500): response = r.process_msg_and_check(message_3, rl_v2, rl_v2) if not response: saw_disconnect = True @@ -158,11 +158,11 @@ class TestRateLimits: # Counts reset also r = RateLimiter(True, 5) new_tx_message = make_msg(ProtocolMessageTypes.new_transaction, bytes([1] * 40)) - for i in range(4900): + for i in range(4999): assert r.process_msg_and_check(new_tx_message, rl_v2, rl_v2) saw_disconnect = False - for i in range(4900): + for i in range(4999): response = r.process_msg_and_check(new_tx_message, rl_v2, rl_v2) if not response: saw_disconnect = True diff --git a/tests/wallet/rpc/test_wallet_rpc.py b/tests/wallet/rpc/test_wallet_rpc.py index a34c5da063..42df3d1d61 100644 --- a/tests/wallet/rpc/test_wallet_rpc.py +++ b/tests/wallet/rpc/test_wallet_rpc.py @@ -854,7 +854,14 @@ async def test_nft_endpoints(wallet_rpc_environment: WalletRpcTestEnvironment): for _ in range(3): await farm_transaction_block(full_node_api, wallet_1_node) + await time_out_assert(15, wallet_is_synced, True, wallet_1_node, full_node_api) nft_wallet: NFTWallet = wallet_1_node.wallet_state_manager.wallets[nft_wallet_id] + + def have_nfts(): + return len(nft_wallet.get_current_nfts()) > 0 + + await time_out_assert(15, have_nfts, True) + # Test with the hex version of nft_id nft_id = nft_wallet.get_current_nfts()[0].coin.name().hex() nft_info = (await wallet_1_rpc.get_nft_info(nft_id))["nft_info"] @@ -870,6 +877,7 @@ async def test_nft_endpoints(wallet_rpc_environment: WalletRpcTestEnvironment): for _ in range(3): await farm_transaction_block(full_node_api, wallet_1_node) + await time_out_assert(15, wallet_is_synced, True, wallet_1_node, full_node_api) nft_wallet_id_1 = ( await wallet_2_node.wallet_state_manager.get_all_wallet_info_entries(wallet_type=WalletType.NFT) diff --git a/tests/wallet/sync/test_wallet_sync.py b/tests/wallet/sync/test_wallet_sync.py index 370dc34082..469d3646f2 100644 --- a/tests/wallet/sync/test_wallet_sync.py +++ b/tests/wallet/sync/test_wallet_sync.py @@ -160,9 +160,8 @@ class TestWalletSync: # Tests a reorg with the wallet num_blocks = 30 - blocks_reorg = bt.get_consecutive_blocks( - num_blocks, block_list_input=default_400_blocks[:-5], current_time=True - ) + blocks_reorg = bt.get_consecutive_blocks(num_blocks - 1, block_list_input=default_400_blocks[:-5]) + blocks_reorg = bt.get_consecutive_blocks(1, blocks_reorg, guarantee_transaction_block=True, current_time=True) for i in range(1, len(blocks_reorg)): await full_node_api.full_node.respond_block(full_node_protocol.RespondBlock(blocks_reorg[i])) diff --git a/tests/wallet/test_wallet.py b/tests/wallet/test_wallet.py index 78dad8ef77..4fbfa4e9d3 100644 --- a/tests/wallet/test_wallet.py +++ b/tests/wallet/test_wallet.py @@ -85,7 +85,7 @@ class TestWalletSimulator: return True await time_out_assert(20, check_tx_are_pool_farm_rewards, True) - await time_out_assert(5, wallet.get_confirmed_balance, funds) + await time_out_assert(10, wallet.get_confirmed_balance, funds) @pytest.mark.parametrize( "trusted", @@ -123,7 +123,7 @@ class TestWalletSimulator: ) await time_out_assert(10, wallet.get_confirmed_balance, funds) - await time_out_assert(5, wallet.get_unconfirmed_balance, funds) + await time_out_assert(10, wallet.get_unconfirmed_balance, funds) tx = await wallet.generate_signed_transaction( uint64(10), @@ -132,9 +132,9 @@ class TestWalletSimulator: ) await wallet.push_transaction(tx) - await time_out_assert(5, wallet.get_confirmed_balance, funds) - await time_out_assert(5, wallet.get_unconfirmed_balance, funds - 10) - await time_out_assert(5, full_node_api.full_node.mempool_manager.get_spendbundle, tx.spend_bundle, tx.name) + await time_out_assert(10, wallet.get_confirmed_balance, funds) + await time_out_assert(10, wallet.get_unconfirmed_balance, funds - 10) + await time_out_assert(10, full_node_api.full_node.mempool_manager.get_spendbundle, tx.spend_bundle, tx.name) for i in range(0, num_blocks): await full_node_api.farm_new_transaction_block(FarmNewBlockProtocol(ph)) @@ -146,8 +146,8 @@ class TestWalletSimulator: ] ) - await time_out_assert(5, wallet.get_confirmed_balance, new_funds - 10) - await time_out_assert(5, wallet.get_unconfirmed_balance, new_funds - 10) + await time_out_assert(10, wallet.get_confirmed_balance, new_funds - 10) + await time_out_assert(10, wallet.get_unconfirmed_balance, new_funds - 10) @pytest.mark.parametrize( "trusted", @@ -193,7 +193,7 @@ class TestWalletSimulator: ] ) - await time_out_assert(5, wallet.get_confirmed_balance, funds) + await time_out_assert(10, wallet.get_confirmed_balance, funds) @pytest.mark.parametrize( "trusted", @@ -250,7 +250,7 @@ class TestWalletSimulator: [calculate_pool_reward(uint32(i)) + calculate_base_farmer_reward(uint32(i)) for i in range(1, num_blocks)] ) - await time_out_assert(5, wallet_0.wallet_state_manager.main_wallet.get_confirmed_balance, funds) + await time_out_assert(10, wallet_0.wallet_state_manager.main_wallet.get_confirmed_balance, funds) tx = await wallet_0.wallet_state_manager.main_wallet.generate_signed_transaction( uint64(10), bytes32(32 * b"0"), uint64(0) @@ -435,8 +435,8 @@ class TestWalletSimulator: [calculate_pool_reward(uint32(i)) + calculate_base_farmer_reward(uint32(i)) for i in range(1, num_blocks)] ) - await time_out_assert(5, wallet.get_confirmed_balance, funds) - await time_out_assert(5, wallet.get_unconfirmed_balance, funds) + await time_out_assert(10, wallet.get_confirmed_balance, funds) + await time_out_assert(10, wallet.get_unconfirmed_balance, funds) assert await wallet.get_confirmed_balance() == funds assert await wallet.get_unconfirmed_balance() == funds @@ -453,10 +453,10 @@ class TestWalletSimulator: assert fees == tx_fee await wallet.push_transaction(tx) - await time_out_assert(5, full_node_1.full_node.mempool_manager.get_spendbundle, tx.spend_bundle, tx.name) + await time_out_assert(20, full_node_1.full_node.mempool_manager.get_spendbundle, tx.spend_bundle, tx.name) - await time_out_assert(5, wallet.get_confirmed_balance, funds) - await time_out_assert(5, wallet.get_unconfirmed_balance, funds - tx_amount - tx_fee) + await time_out_assert(20, wallet.get_confirmed_balance, funds) + await time_out_assert(20, wallet.get_unconfirmed_balance, funds - tx_amount - tx_fee) for i in range(0, num_blocks): await full_node_1.farm_new_transaction_block(FarmNewBlockProtocol(bytes32(32 * b"0"))) @@ -468,8 +468,8 @@ class TestWalletSimulator: ] ) - await time_out_assert(5, wallet.get_confirmed_balance, new_funds - tx_amount - tx_fee) - await time_out_assert(5, wallet.get_unconfirmed_balance, new_funds - tx_amount - tx_fee) + await time_out_assert(10, wallet.get_confirmed_balance, new_funds - tx_amount - tx_fee) + await time_out_assert(10, wallet.get_unconfirmed_balance, new_funds - tx_amount - tx_fee) @pytest.mark.parametrize( "trusted", @@ -510,7 +510,7 @@ class TestWalletSimulator: [calculate_pool_reward(uint32(i)) + calculate_base_farmer_reward(uint32(i)) for i in range(1, num_blocks)] ) - await time_out_assert(5, wallet.get_confirmed_balance, funds) + await time_out_assert(20, wallet.get_confirmed_balance, funds) primaries: List[AmountWithPuzzlehash] = [] for i in range(0, 60): @@ -615,7 +615,7 @@ class TestWalletSimulator: ) await time_out_assert(10, wallet.get_confirmed_balance, funds) - await time_out_assert(5, wallet.get_unconfirmed_balance, funds) + await time_out_assert(10, wallet.get_unconfirmed_balance, funds) assert await wallet.get_confirmed_balance() == funds assert await wallet.get_unconfirmed_balance() == funds @@ -658,8 +658,8 @@ class TestWalletSimulator: ) await wallet.push_transaction(stolen_tx) - await time_out_assert(5, wallet.get_confirmed_balance, funds) - await time_out_assert(5, wallet.get_unconfirmed_balance, funds - stolen_cs.coin.amount) + await time_out_assert(10, wallet.get_confirmed_balance, funds) + await time_out_assert(10, wallet.get_unconfirmed_balance, funds - stolen_cs.coin.amount) for i in range(0, num_blocks): await full_node_1.farm_new_transaction_block(FarmNewBlockProtocol(bytes32(32 * b"0"))) @@ -718,14 +718,14 @@ class TestWalletSimulator: assert tx.spend_bundle is not None await wallet.push_transaction(tx) await full_node_api.full_node.respond_transaction(tx.spend_bundle, tx.name) - await time_out_assert(5, full_node_api.full_node.mempool_manager.get_spendbundle, tx.spend_bundle, tx.name) - await time_out_assert(5, wallet.get_confirmed_balance, funds) + await time_out_assert(10, full_node_api.full_node.mempool_manager.get_spendbundle, tx.spend_bundle, tx.name) + await time_out_assert(10, wallet.get_confirmed_balance, funds) for i in range(0, 2): await full_node_api.farm_new_transaction_block(FarmNewBlockProtocol(bytes32(32 * b"0"))) - await time_out_assert(5, wallet_2.get_confirmed_balance, 1000) + await time_out_assert(10, wallet_2.get_confirmed_balance, 1000) funds -= 1000 - await time_out_assert(5, wallet_node.wallet_state_manager.blockchain.get_peak_height, 7) + await time_out_assert(10, wallet_node.wallet_state_manager.blockchain.get_peak_height, 7) peak = full_node_api.full_node.blockchain.get_peak() assert peak is not None peak_height = peak.height @@ -743,8 +743,8 @@ class TestWalletSimulator: ] ) - await time_out_assert(7, full_node_api.full_node.blockchain.get_peak_height, peak_height + 3) - await time_out_assert(7, wallet_node.wallet_state_manager.blockchain.get_peak_height, peak_height + 3) + await time_out_assert(20, full_node_api.full_node.blockchain.get_peak_height, peak_height + 3) + await time_out_assert(20, wallet_node.wallet_state_manager.blockchain.get_peak_height, peak_height + 3) # Farm a few blocks so we can confirm the resubmitted transaction for i in range(0, num_blocks): @@ -864,8 +864,8 @@ class TestWalletSimulator: ] ) - await time_out_assert(5, wallet.get_confirmed_balance, funds) - await time_out_assert(5, wallet.get_unconfirmed_balance, funds) + await time_out_assert(10, wallet.get_confirmed_balance, funds) + await time_out_assert(10, wallet.get_unconfirmed_balance, funds) AMOUNT_TO_SEND = 4000000000000 coins = await wallet.select_coins(uint64(AMOUNT_TO_SEND)) @@ -883,12 +883,12 @@ class TestWalletSimulator: assert paid_coin.parent_coin_info == coin_list[2].name() await wallet.push_transaction(tx) - await time_out_assert(5, wallet.get_confirmed_balance, funds) - await time_out_assert(5, wallet.get_unconfirmed_balance, funds - AMOUNT_TO_SEND) - await time_out_assert(5, full_node_api.full_node.mempool_manager.get_spendbundle, tx.spend_bundle, tx.name) + await time_out_assert(10, wallet.get_confirmed_balance, funds) + await time_out_assert(10, wallet.get_unconfirmed_balance, funds - AMOUNT_TO_SEND) + await time_out_assert(10, full_node_api.full_node.mempool_manager.get_spendbundle, tx.spend_bundle, tx.name) for i in range(0, num_blocks): await full_node_api.farm_new_transaction_block(FarmNewBlockProtocol(bytes32([0] * 32))) - await time_out_assert(5, wallet.get_confirmed_balance, funds - AMOUNT_TO_SEND) - await time_out_assert(5, wallet.get_unconfirmed_balance, funds - AMOUNT_TO_SEND) + await time_out_assert(10, wallet.get_confirmed_balance, funds - AMOUNT_TO_SEND) + await time_out_assert(10, wallet.get_unconfirmed_balance, funds - AMOUNT_TO_SEND) From bddafa42f5f95af10c2147b6b08b9de110b639c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Jun 2022 20:34:38 -0500 Subject: [PATCH 14/62] Bump node-fetch from 2.6.6 to 2.6.7 in /build_scripts/npm_windows (#10422) Bumps [node-fetch](https://github.com/node-fetch/node-fetch) from 2.6.6 to 2.6.7. - [Release notes](https://github.com/node-fetch/node-fetch/releases) - [Commits](https://github.com/node-fetch/node-fetch/compare/v2.6.6...v2.6.7) --- updated-dependencies: - dependency-name: node-fetch dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build_scripts/npm_windows/package-lock.json | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/build_scripts/npm_windows/package-lock.json b/build_scripts/npm_windows/package-lock.json index 9a50ef540f..a968c736d4 100644 --- a/build_scripts/npm_windows/package-lock.json +++ b/build_scripts/npm_windows/package-lock.json @@ -5165,14 +5165,22 @@ "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==" }, "node_modules/node-fetch": { - "version": "2.6.6", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", - "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", "dependencies": { "whatwg-url": "^5.0.0" }, "engines": { "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } } }, "node_modules/node-fetch/node_modules/tr46": { @@ -11865,9 +11873,9 @@ "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==" }, "node-fetch": { - "version": "2.6.6", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", - "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", "requires": { "whatwg-url": "^5.0.0" }, From 46bcaec2e87ac9f335cdedbe20ec4bb136552669 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Jun 2022 22:23:58 -0500 Subject: [PATCH 15/62] Bump plist from 3.0.4 to 3.0.5 in /build_scripts/npm_linux_rpm (#10896) Bumps [plist](https://github.com/TooTallNate/node-plist) from 3.0.4 to 3.0.5. - [Release notes](https://github.com/TooTallNate/node-plist/releases) - [Changelog](https://github.com/TooTallNate/plist.js/blob/master/History.md) - [Commits](https://github.com/TooTallNate/node-plist/commits) --- updated-dependencies: - dependency-name: plist dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build_scripts/npm_linux_rpm/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build_scripts/npm_linux_rpm/package-lock.json b/build_scripts/npm_linux_rpm/package-lock.json index 615f2f2329..128c024ba9 100644 --- a/build_scripts/npm_linux_rpm/package-lock.json +++ b/build_scripts/npm_linux_rpm/package-lock.json @@ -6058,9 +6058,9 @@ } }, "node_modules/plist": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.4.tgz", - "integrity": "sha512-ksrr8y9+nXOxQB2osVNqrgvX/XQPOXaU4BQMKjYq8PvaY1U18mo+fKgBSwzK+luSyinOuPae956lSVcBwxlAMg==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.5.tgz", + "integrity": "sha512-83vX4eYdQp3vP9SxuYgEM/G/pJQqLUz/V/xzPrzruLs7fz7jxGQ1msZ/mg1nwZxUSuOp4sb+/bEIbRrbzZRxDA==", "dependencies": { "base64-js": "^1.5.1", "xmlbuilder": "^9.0.7" @@ -12624,9 +12624,9 @@ } }, "plist": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.4.tgz", - "integrity": "sha512-ksrr8y9+nXOxQB2osVNqrgvX/XQPOXaU4BQMKjYq8PvaY1U18mo+fKgBSwzK+luSyinOuPae956lSVcBwxlAMg==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/plist/-/plist-3.0.5.tgz", + "integrity": "sha512-83vX4eYdQp3vP9SxuYgEM/G/pJQqLUz/V/xzPrzruLs7fz7jxGQ1msZ/mg1nwZxUSuOp4sb+/bEIbRrbzZRxDA==", "requires": { "base64-js": "^1.5.1", "xmlbuilder": "^9.0.7" From 39c55994fbfae7b4c0938459fe5b3fda82de6912 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Fri, 1 Jul 2022 17:45:52 +0200 Subject: [PATCH 16/62] WalletTransactionStore unit test (#12171) * add unit test for WalletTransactionStore * minor simplification of WalletTransactionStore --- chia/wallet/transaction_record.py | 2 + chia/wallet/transaction_sorting.py | 4 +- chia/wallet/wallet_transaction_store.py | 37 +- tests/wallet/test_transaction_store.py | 652 ++++++++++++++++++++++++ 4 files changed, 670 insertions(+), 25 deletions(-) create mode 100644 tests/wallet/test_transaction_store.py diff --git a/chia/wallet/transaction_record.py b/chia/wallet/transaction_record.py index b13b6d3660..87797c1578 100644 --- a/chia/wallet/transaction_record.py +++ b/chia/wallet/transaction_record.py @@ -36,6 +36,8 @@ class TransactionRecord(Streamable): sent_to: List[Tuple[str, uint8, Optional[str]]] trade_id: Optional[bytes32] type: uint32 # TransactionType + + # name is also called bundle_id and tx_id name: bytes32 memos: List[Tuple[bytes32, List[bytes]]] diff --git a/chia/wallet/transaction_sorting.py b/chia/wallet/transaction_sorting.py index ec56934845..09d778cf92 100644 --- a/chia/wallet/transaction_sorting.py +++ b/chia/wallet/transaction_sorting.py @@ -2,8 +2,8 @@ import enum class SortKey(enum.Enum): - CONFIRMED_AT_HEIGHT = "order by confirmed_at_height {ASC}" - RELEVANCE = "order by confirmed {ASC}, confirmed_at_height {DESC}, created_at_time {DESC}" + CONFIRMED_AT_HEIGHT = "ORDER BY confirmed_at_height {ASC}" + RELEVANCE = "ORDER BY confirmed {ASC}, confirmed_at_height {DESC}, created_at_time {DESC}" def ascending(self) -> str: return self.value.format(ASC="ASC", DESC="DESC") diff --git a/chia/wallet/wallet_transaction_store.py b/chia/wallet/wallet_transaction_store.py index c121b94b14..0f986a59a4 100644 --- a/chia/wallet/wallet_transaction_store.py +++ b/chia/wallet/wallet_transaction_store.py @@ -210,7 +210,7 @@ class WalletTransactionStore: await self.add_transaction_record(tx, False) return True - async def tx_reorged(self, record: TransactionRecord, in_transaction: bool): + async def tx_reorged(self, record: TransactionRecord, in_transaction: bool) -> None: """ Updates transaction sent count to 0 and resets confirmation data """ @@ -246,14 +246,17 @@ class WalletTransactionStore: return TransactionRecord.from_bytes(rows[0][0]) return None + # TODO: This should probably be split into separate function, one that + # queries the state and one that updates it. Also, include_accepted_txs=True + # might be a separate function too. + # also, the current time should be passed in as a paramter async def get_not_sent(self, *, include_accepted_txs=False) -> List[TransactionRecord]: """ Returns the list of transactions that have not been received by full node yet. """ current_time = int(time.time()) rows = await self.db_connection.execute_fetchall( - "SELECT * from transaction_record WHERE confirmed=?", - (0,), + "SELECT * from transaction_record WHERE confirmed=0", ) records = [] @@ -287,29 +290,17 @@ class WalletTransactionStore: fee_int = TransactionType.FEE_REWARD.value pool_int = TransactionType.COINBASE_REWARD.value rows = await self.db_connection.execute_fetchall( - "SELECT * from transaction_record WHERE confirmed=? and (type=? or type=?)", (1, fee_int, pool_int) + "SELECT * from transaction_record WHERE confirmed=1 and (type=? or type=?)", (fee_int, pool_int) ) - records = [] - - for row in rows: - record = TransactionRecord.from_bytes(row[0]) - records.append(record) - - return records + return [TransactionRecord.from_bytes(row[0]) for row in rows] async def get_all_unconfirmed(self) -> List[TransactionRecord]: """ Returns the list of all transaction that have not yet been confirmed. """ - rows = await self.db_connection.execute_fetchall("SELECT * from transaction_record WHERE confirmed=?", (0,)) - records = [] - - for row in rows: - record = TransactionRecord.from_bytes(row[0]) - records.append(record) - - return records + rows = await self.db_connection.execute_fetchall("SELECT * from transaction_record WHERE confirmed=0") + return [TransactionRecord.from_bytes(row[0]) for row in rows] async def get_unconfirmed_for_wallet(self, wallet_id: int) -> List[TransactionRecord]: """ @@ -331,7 +322,7 @@ class WalletTransactionStore: if to_puzzle_hash is None: puzz_hash_where = "" else: - puzz_hash_where = f' and to_puzzle_hash="{to_puzzle_hash.hex()}"' + puzz_hash_where = f' AND to_puzzle_hash="{to_puzzle_hash.hex()}"' if sort_key is None: sort_key = "CONFIRMED_AT_HEIGHT" @@ -344,7 +335,7 @@ class WalletTransactionStore: query_str = SortKey[sort_key].ascending() rows = await self.db_connection.execute_fetchall( - f"SELECT * from transaction_record where wallet_id=?{puzz_hash_where}" + f"SELECT * from transaction_record WHERE wallet_id=?{puzz_hash_where}" f" {query_str}, rowid" f" LIMIT {start}, {limit}", (wallet_id,), @@ -366,11 +357,11 @@ class WalletTransactionStore: """ if type is None: rows = await self.db_connection.execute_fetchall( - "SELECT * from transaction_record where wallet_id=?", (wallet_id,) + "SELECT * FROM transaction_record WHERE wallet_id=?", (wallet_id,) ) else: rows = await self.db_connection.execute_fetchall( - "SELECT * from transaction_record where wallet_id=? and type=?", + "SELECT * FROM transaction_record WHERE wallet_id=? AND type=?", ( wallet_id, type, diff --git a/tests/wallet/test_transaction_store.py b/tests/wallet/test_transaction_store.py new file mode 100644 index 0000000000..3ed382aca5 --- /dev/null +++ b/tests/wallet/test_transaction_store.py @@ -0,0 +1,652 @@ +import dataclasses +from secrets import token_bytes +from typing import Any, List + +import pytest + +from chia.types.blockchain_format.coin import Coin +from chia.types.blockchain_format.sized_bytes import bytes32 +from chia.types.mempool_inclusion_status import MempoolInclusionStatus +from chia.util.errors import Err +from chia.util.ints import uint8, uint32, uint64 +from chia.wallet.transaction_record import TransactionRecord +from chia.wallet.util.transaction_type import TransactionType +from chia.wallet.wallet_transaction_store import WalletTransactionStore, filter_ok_mempool_status +from tests.util.db_connection import DBConnection1 + +coin_1 = Coin(token_bytes(32), token_bytes(32), uint64(12312)) +coin_2 = Coin(token_bytes(32), token_bytes(32), uint64(1234)) +coin_3 = Coin(token_bytes(32), token_bytes(32), uint64(12312 - 1234)) + +tr1 = TransactionRecord( + uint32(0), # confirmed height + uint64(1000), # created_at_time + bytes32(token_bytes(32)), # to_puzzle_hash + uint64(1234), # amount + uint64(12), # fee_amount + False, # confirmed + uint32(0), # sent + None, # Optional[SpendBundle] spend_bundle + [coin_2, coin_3], # additions + [coin_1], # removals + uint32(1), # wallet_id + [], # List[Tuple[str, uint8, Optional[str]]] sent_to + bytes32(token_bytes(32)), # trade_id + uint32(TransactionType.OUTGOING_TX), # type + bytes32(token_bytes(32)), # name + [], # List[Tuple[bytes32, List[bytes]]] memos +) + + +@pytest.mark.asyncio +async def test_add() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + assert await store.get_transaction_record(tr1.name) is None + await store.add_transaction_record(tr1, False) + assert await store.get_transaction_record(tr1.name) == tr1 + + +@pytest.mark.asyncio +async def test_delete() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + await store.add_transaction_record(tr1, False) + assert await store.get_transaction_record(tr1.name) == tr1 + await store.delete_transaction_record(tr1.name) + assert await store.get_transaction_record(tr1.name) is None + + +@pytest.mark.asyncio +async def test_set_confirmed() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + await store.add_transaction_record(tr1, False) + await store.set_confirmed(tr1.name, uint32(100)) + + assert await store.get_transaction_record(tr1.name) == dataclasses.replace( + tr1, confirmed=True, confirmed_at_height=uint32(100) + ) + + +@pytest.mark.asyncio +async def test_increment_sent_noop() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + assert ( + await store.increment_sent(bytes32(token_bytes(32)), "peer1", MempoolInclusionStatus.PENDING, None) is False + ) + + +@pytest.mark.asyncio +async def test_increment_sent() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + await store.add_transaction_record(tr1, False) + tr = await store.get_transaction_record(tr1.name) + assert tr.sent == 0 + assert tr.sent_to == [] + + assert await store.increment_sent(tr1.name, "peer1", MempoolInclusionStatus.PENDING, None) is True + tr = await store.get_transaction_record(tr1.name) + assert tr.sent == 1 + assert tr.sent_to == [("peer1", uint8(2), None)] + + assert await store.increment_sent(tr1.name, "peer1", MempoolInclusionStatus.SUCCESS, None) is True + tr = await store.get_transaction_record(tr1.name) + assert tr.sent == 1 + assert tr.sent_to == [("peer1", uint8(2), None), ("peer1", uint8(1), None)] + + assert await store.increment_sent(tr1.name, "peer2", MempoolInclusionStatus.SUCCESS, None) is True + tr = await store.get_transaction_record(tr1.name) + assert tr.sent == 2 + assert tr.sent_to == [("peer1", uint8(2), None), ("peer1", uint8(1), None), ("peer2", uint8(1), None)] + + +@pytest.mark.asyncio +async def test_increment_sent_error() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + await store.add_transaction_record(tr1, False) + tr = await store.get_transaction_record(tr1.name) + assert tr.sent == 0 + assert tr.sent_to == [] + + await store.increment_sent(tr1.name, "peer1", MempoolInclusionStatus.FAILED, Err.MEMPOOL_NOT_INITIALIZED) + tr = await store.get_transaction_record(tr1.name) + assert tr.sent == 1 + assert tr.sent_to == [("peer1", uint8(3), "MEMPOOL_NOT_INITIALIZED")] + + +def test_filter_ok_mempool_status() -> None: + assert filter_ok_mempool_status([("peer1", uint8(1), None)]) == [] + assert filter_ok_mempool_status([("peer1", uint8(2), None)]) == [] + assert filter_ok_mempool_status([("peer1", uint8(3), None)]) == [("peer1", uint8(3), None)] + assert filter_ok_mempool_status( + [("peer1", uint8(2), None), ("peer1", uint8(1), None), ("peer1", uint8(3), None)] + ) == [("peer1", uint8(3), None)] + + assert filter_ok_mempool_status([("peer1", uint8(3), "message does not matter")]) == [ + ("peer1", uint8(3), "message does not matter") + ] + assert filter_ok_mempool_status([("peer1", uint8(2), "message does not matter")]) == [] + + +@pytest.mark.asyncio +async def test_tx_reorged_update() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + tr = dataclasses.replace(tr1, sent=2, sent_to=[("peer1", uint8(1), None), ("peer2", uint8(1), None)]) + await store.add_transaction_record(tr, False) + tr = await store.get_transaction_record(tr.name) + assert tr.sent == 2 + assert tr.sent_to == [("peer1", uint8(1), None), ("peer2", uint8(1), None)] + + await store.tx_reorged(tr, False) + tr = await store.get_transaction_record(tr1.name) + assert tr.sent == 0 + assert tr.sent_to == [] + + +@pytest.mark.asyncio +async def test_tx_reorged_add() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + tr = dataclasses.replace(tr1, sent=2, sent_to=[("peer1", uint8(1), None), ("peer2", uint8(1), None)]) + + await store.get_transaction_record(tr.name) is None + await store.tx_reorged(tr, False) + tr = await store.get_transaction_record(tr.name) + assert tr.sent == 0 + assert tr.sent_to == [] + + +@pytest.mark.asyncio +async def test_get_tx_record() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + tr2 = dataclasses.replace(tr1, name=token_bytes(32)) + tr3 = dataclasses.replace(tr1, name=token_bytes(32)) + + assert await store.get_transaction_record(tr1.name) is None + await store.add_transaction_record(tr1, False) + assert await store.get_transaction_record(tr1.name) == tr1 + + assert await store.get_transaction_record(tr2.name) is None + await store.add_transaction_record(tr2, False) + assert await store.get_transaction_record(tr2.name) == tr2 + + assert await store.get_transaction_record(tr3.name) is None + await store.add_transaction_record(tr3, False) + assert await store.get_transaction_record(tr3.name) == tr3 + + assert await store.get_transaction_record(tr1.name) == tr1 + assert await store.get_transaction_record(tr2.name) == tr2 + assert await store.get_transaction_record(tr3.name) == tr3 + + +@pytest.mark.asyncio +async def test_get_farming_rewards() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + test_trs: List[TransactionRecord] = [] + # tr1 is type OUTGOING_TX + + for conf in [True, False]: + for type in [ + TransactionType.INCOMING_TX, + TransactionType.OUTGOING_TX, + TransactionType.COINBASE_REWARD, + TransactionType.FEE_REWARD, + TransactionType.INCOMING_TRADE, + TransactionType.OUTGOING_TRADE, + ]: + test_trs.append( + dataclasses.replace( + tr1, + name=token_bytes(32), + confirmed=conf, + confirmed_at_height=uint32(100 if conf else 0), + type=type, + ) + ) + + for tr in test_trs: + await store.add_transaction_record(tr, False) + assert await store.get_transaction_record(tr.name) == tr + + rewards = await store.get_farming_rewards() + assert len(rewards) == 2 + assert test_trs[2] in rewards + assert test_trs[3] in rewards + + +@pytest.mark.asyncio +async def test_get_all_unconfirmed() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + tr2 = dataclasses.replace(tr1, name=token_bytes(32), confirmed=True, confirmed_at_height=uint32(100)) + await store.add_transaction_record(tr1, False) + await store.add_transaction_record(tr2, False) + + assert await store.get_all_unconfirmed() == [tr1] + + +@pytest.mark.asyncio +async def test_get_unconfirmed_for_wallet() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + tr2 = dataclasses.replace(tr1, name=token_bytes(32), confirmed=True, confirmed_at_height=uint32(100)) + tr3 = dataclasses.replace(tr1, name=token_bytes(32), wallet_id=2) + tr4 = dataclasses.replace(tr2, name=token_bytes(32), wallet_id=2) + await store.add_transaction_record(tr1, False) + await store.add_transaction_record(tr2, False) + await store.add_transaction_record(tr3, False) + await store.add_transaction_record(tr4, False) + + assert await store.get_unconfirmed_for_wallet(1) == [tr1] + assert await store.get_unconfirmed_for_wallet(2) == [tr3] + + +@pytest.mark.asyncio +async def test_transaction_count_for_wallet() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + tr2 = dataclasses.replace(tr1, name=token_bytes(32), wallet_id=2) + + # 5 transactions in wallet_id 1 + await store.add_transaction_record(tr1, False) + await store.add_transaction_record(dataclasses.replace(tr1, name=token_bytes(32)), False) + await store.add_transaction_record(dataclasses.replace(tr1, name=token_bytes(32)), False) + await store.add_transaction_record(dataclasses.replace(tr1, name=token_bytes(32)), False) + await store.add_transaction_record(dataclasses.replace(tr1, name=token_bytes(32)), False) + + # 2 transactions in wallet_id 2 + await store.add_transaction_record(tr2, False) + await store.add_transaction_record(dataclasses.replace(tr2, name=token_bytes(32)), False) + + assert await store.get_transaction_count_for_wallet(1) == 5 + assert await store.get_transaction_count_for_wallet(2) == 2 + + +@pytest.mark.asyncio +async def test_all_transactions_for_wallet() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + test_trs: List[TransactionRecord] = [] + for wallet_id in [1, 2]: + for type in [ + TransactionType.INCOMING_TX, + TransactionType.OUTGOING_TX, + TransactionType.COINBASE_REWARD, + TransactionType.FEE_REWARD, + TransactionType.INCOMING_TRADE, + TransactionType.OUTGOING_TRADE, + ]: + test_trs.append(dataclasses.replace(tr1, name=token_bytes(32), wallet_id=wallet_id, type=type)) + + for tr in test_trs: + await store.add_transaction_record(tr, False) + + assert await store.get_all_transactions_for_wallet(1) == test_trs[:6] + assert await store.get_all_transactions_for_wallet(2) == test_trs[6:] + + assert await store.get_all_transactions_for_wallet(1, TransactionType.INCOMING_TX) == [test_trs[0]] + assert await store.get_all_transactions_for_wallet(1, TransactionType.OUTGOING_TX) == [test_trs[1]] + assert await store.get_all_transactions_for_wallet(1, TransactionType.INCOMING_TRADE) == [test_trs[4]] + assert await store.get_all_transactions_for_wallet(1, TransactionType.OUTGOING_TRADE) == [test_trs[5]] + + assert await store.get_all_transactions_for_wallet(2, TransactionType.INCOMING_TX) == [test_trs[6]] + assert await store.get_all_transactions_for_wallet(2, TransactionType.OUTGOING_TX) == [test_trs[7]] + assert await store.get_all_transactions_for_wallet(2, TransactionType.INCOMING_TRADE) == [test_trs[10]] + assert await store.get_all_transactions_for_wallet(2, TransactionType.OUTGOING_TRADE) == [test_trs[11]] + + +def cmp(lhs: List[Any], rhs: List[Any]) -> bool: + if len(rhs) != len(lhs): + return False + + for e in lhs: + if e not in rhs: + return False + return True + + +@pytest.mark.asyncio +async def test_get_all_transactions() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + test_trs: List[TransactionRecord] = [] + assert await store.get_all_transactions() == [] + for wallet_id in [1, 2, 3, 4]: + test_trs.append(dataclasses.replace(tr1, name=token_bytes(32), wallet_id=wallet_id)) + + for tr in test_trs: + await store.add_transaction_record(tr, False) + + all_trs = await store.get_all_transactions() + assert cmp(all_trs, test_trs) + + +@pytest.mark.asyncio +async def test_get_transaction_above() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + test_trs: List[TransactionRecord] = [] + assert await store.get_transaction_above(uint32(0)) == [] + for height in range(10): + test_trs.append(dataclasses.replace(tr1, name=token_bytes(32), confirmed_at_height=uint32(height))) + + for tr in test_trs: + await store.add_transaction_record(tr, False) + + for height in range(10): + trs = await store.get_transaction_above(uint32(height)) + assert cmp(trs, test_trs[height + 1 :]) + + +@pytest.mark.asyncio +async def test_get_tx_by_trade_id() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + tr2 = dataclasses.replace(tr1, name=token_bytes(32), trade_id=token_bytes(32)) + tr3 = dataclasses.replace(tr1, name=token_bytes(32), trade_id=token_bytes(32)) + tr4 = dataclasses.replace(tr1, name=token_bytes(32)) + + assert await store.get_transactions_by_trade_id(tr1.trade_id) == [] + await store.add_transaction_record(tr1, False) + assert await store.get_transactions_by_trade_id(tr1.trade_id) == [tr1] + + assert await store.get_transactions_by_trade_id(tr2.trade_id) == [] + await store.add_transaction_record(tr2, False) + assert await store.get_transactions_by_trade_id(tr2.trade_id) == [tr2] + + assert await store.get_transactions_by_trade_id(tr3.trade_id) == [] + await store.add_transaction_record(tr3, False) + assert await store.get_transactions_by_trade_id(tr3.trade_id) == [tr3] + + # tr1 and tr4 have the same trade_id + assert await store.get_transactions_by_trade_id(tr4.trade_id) == [tr1] + await store.add_transaction_record(tr4, False) + assert cmp(await store.get_transactions_by_trade_id(tr4.trade_id), [tr1, tr4]) + + assert cmp(await store.get_transactions_by_trade_id(tr1.trade_id), [tr1, tr4]) + assert await store.get_transactions_by_trade_id(tr2.trade_id) == [tr2] + assert await store.get_transactions_by_trade_id(tr3.trade_id) == [tr3] + assert cmp(await store.get_transactions_by_trade_id(tr4.trade_id), [tr1, tr4]) + + +@pytest.mark.asyncio +async def test_rollback_to_block() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + test_trs: List[TransactionRecord] = [] + for height in range(10): + test_trs.append(dataclasses.replace(tr1, name=token_bytes(32), confirmed_at_height=uint32(height))) + + for tr in test_trs: + await store.add_transaction_record(tr, False) + + await store.rollback_to_block(uint32(6)) + all_trs = await store.get_all_transactions() + assert cmp(all_trs, test_trs[:7]) + + await store.rollback_to_block(uint32(5)) + all_trs = await store.get_all_transactions() + assert cmp(all_trs, test_trs[:6]) + + +@pytest.mark.asyncio +async def test_delete_unconfirmed() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + tr2 = dataclasses.replace(tr1, name=token_bytes(32), confirmed=True) + tr3 = dataclasses.replace(tr1, name=token_bytes(32), confirmed=True, wallet_id=2) + tr4 = dataclasses.replace(tr1, name=token_bytes(32), wallet_id=2) + + await store.add_transaction_record(tr1, False) + await store.add_transaction_record(tr2, False) + await store.add_transaction_record(tr3, False) + await store.add_transaction_record(tr4, False) + + assert cmp(await store.get_all_transactions(), [tr1, tr2, tr3, tr4]) + await store.delete_unconfirmed_transactions(1) + assert cmp(await store.get_all_transactions(), [tr2, tr3, tr4]) + await store.delete_unconfirmed_transactions(2) + assert cmp(await store.get_all_transactions(), [tr2, tr3]) + + +@pytest.mark.asyncio +async def test_get_transactions_between_confirmed() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + tr2 = dataclasses.replace(tr1, name=token_bytes(32), confirmed_at_height=uint32(1)) + tr3 = dataclasses.replace(tr1, name=token_bytes(32), confirmed_at_height=uint32(2)) + tr4 = dataclasses.replace(tr1, name=token_bytes(32), confirmed_at_height=uint32(3)) + tr5 = dataclasses.replace(tr1, name=token_bytes(32), confirmed_at_height=uint32(4)) + + await store.add_transaction_record(tr1, False) + await store.add_transaction_record(tr2, False) + await store.add_transaction_record(tr3, False) + await store.add_transaction_record(tr4, False) + await store.add_transaction_record(tr5, False) + + # test different limits + assert await store.get_transactions_between(1, 0, 1) == [tr1] + assert await store.get_transactions_between(1, 0, 2) == [tr1, tr2] + assert await store.get_transactions_between(1, 0, 3) == [tr1, tr2, tr3] + assert await store.get_transactions_between(1, 0, 100) == [tr1, tr2, tr3, tr4, tr5] + + # test different start offsets + assert await store.get_transactions_between(1, 1, 100) == [tr2, tr3, tr4, tr5] + assert await store.get_transactions_between(1, 2, 100) == [tr3, tr4, tr5] + assert await store.get_transactions_between(1, 3, 100) == [tr4, tr5] + + # wallet 2 is empty + assert await store.get_transactions_between(2, 0, 100) == [] + + # reverse + + # test different limits + assert await store.get_transactions_between(1, 0, 1, reverse=True) == [tr5] + assert await store.get_transactions_between(1, 0, 2, reverse=True) == [tr5, tr4] + assert await store.get_transactions_between(1, 0, 3, reverse=True) == [tr5, tr4, tr3] + assert await store.get_transactions_between(1, 0, 100, reverse=True) == [tr5, tr4, tr3, tr2, tr1] + + # test different start offsets + assert await store.get_transactions_between(1, 1, 100, reverse=True) == [tr4, tr3, tr2, tr1] + assert await store.get_transactions_between(1, 2, 100, reverse=True) == [tr3, tr2, tr1] + assert await store.get_transactions_between(1, 3, 100, reverse=True) == [tr2, tr1] + + +@pytest.mark.asyncio +async def test_get_transactions_between_relevance() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + t1 = dataclasses.replace( + tr1, name=token_bytes(32), confirmed=False, confirmed_at_height=uint32(2), created_at_time=1000 + ) + t2 = dataclasses.replace( + tr1, name=token_bytes(32), confirmed=False, confirmed_at_height=uint32(2), created_at_time=999 + ) + t3 = dataclasses.replace( + tr1, name=token_bytes(32), confirmed=False, confirmed_at_height=uint32(1), created_at_time=1000 + ) + t4 = dataclasses.replace( + tr1, name=token_bytes(32), confirmed=False, confirmed_at_height=uint32(1), created_at_time=999 + ) + + t5 = dataclasses.replace( + tr1, name=token_bytes(32), confirmed=True, confirmed_at_height=uint32(2), created_at_time=1000 + ) + t6 = dataclasses.replace( + tr1, name=token_bytes(32), confirmed=True, confirmed_at_height=uint32(2), created_at_time=999 + ) + t7 = dataclasses.replace( + tr1, name=token_bytes(32), confirmed=True, confirmed_at_height=uint32(1), created_at_time=1000 + ) + t8 = dataclasses.replace( + tr1, name=token_bytes(32), confirmed=True, confirmed_at_height=uint32(1), created_at_time=999 + ) + + await store.add_transaction_record(t1, False) + await store.add_transaction_record(t2, False) + await store.add_transaction_record(t3, False) + await store.add_transaction_record(t4, False) + await store.add_transaction_record(t5, False) + await store.add_transaction_record(t6, False) + await store.add_transaction_record(t7, False) + await store.add_transaction_record(t8, False) + + # test different limits + assert await store.get_transactions_between(1, 0, 1, sort_key="RELEVANCE") == [t1] + assert await store.get_transactions_between(1, 0, 2, sort_key="RELEVANCE") == [t1, t2] + assert await store.get_transactions_between(1, 0, 3, sort_key="RELEVANCE") == [t1, t2, t3] + assert await store.get_transactions_between(1, 0, 100, sort_key="RELEVANCE") == [t1, t2, t3, t4, t5, t6, t7, t8] + + # test different start offsets + assert await store.get_transactions_between(1, 1, 100, sort_key="RELEVANCE") == [t2, t3, t4, t5, t6, t7, t8] + assert await store.get_transactions_between(1, 2, 100, sort_key="RELEVANCE") == [t3, t4, t5, t6, t7, t8] + assert await store.get_transactions_between(1, 3, 100, sort_key="RELEVANCE") == [t4, t5, t6, t7, t8] + assert await store.get_transactions_between(1, 4, 100, sort_key="RELEVANCE") == [t5, t6, t7, t8] + + # wallet 2 is empty + assert await store.get_transactions_between(2, 0, 100, sort_key="RELEVANCE") == [] + + # reverse + + # test different limits + assert await store.get_transactions_between(1, 0, 1, sort_key="RELEVANCE", reverse=True) == [t8] + assert await store.get_transactions_between(1, 0, 2, sort_key="RELEVANCE", reverse=True) == [t8, t7] + assert await store.get_transactions_between(1, 0, 3, sort_key="RELEVANCE", reverse=True) == [t8, t7, t6] + assert await store.get_transactions_between(1, 0, 100, sort_key="RELEVANCE", reverse=True) == [ + t8, + t7, + t6, + t5, + t4, + t3, + t2, + t1, + ] + + # test different start offsets + assert await store.get_transactions_between(1, 1, 100, sort_key="RELEVANCE", reverse=True) == [ + t7, + t6, + t5, + t4, + t3, + t2, + t1, + ] + assert await store.get_transactions_between(1, 2, 100, sort_key="RELEVANCE", reverse=True) == [ + t6, + t5, + t4, + t3, + t2, + t1, + ] + assert await store.get_transactions_between(1, 3, 100, sort_key="RELEVANCE", reverse=True) == [ + t5, + t4, + t3, + t2, + t1, + ] + + +@pytest.mark.asyncio +async def test_get_transactions_between_to_puzzle_hash() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + ph1 = token_bytes(32) + ph2 = token_bytes(32) + + tr2 = dataclasses.replace(tr1, name=token_bytes(32), confirmed_at_height=uint32(1), to_puzzle_hash=ph1) + tr3 = dataclasses.replace(tr1, name=token_bytes(32), confirmed_at_height=uint32(2), to_puzzle_hash=ph1) + tr4 = dataclasses.replace(tr1, name=token_bytes(32), confirmed_at_height=uint32(3), to_puzzle_hash=ph2) + tr5 = dataclasses.replace(tr1, name=token_bytes(32), confirmed_at_height=uint32(4), to_puzzle_hash=ph2) + + await store.add_transaction_record(tr1, False) + await store.add_transaction_record(tr2, False) + await store.add_transaction_record(tr3, False) + await store.add_transaction_record(tr4, False) + await store.add_transaction_record(tr5, False) + + # test different limits + assert await store.get_transactions_between(1, 0, 100, to_puzzle_hash=ph1) == [tr2, tr3] + assert await store.get_transactions_between(1, 0, 100, to_puzzle_hash=ph2) == [tr4, tr5] + + # test different start offsets + assert await store.get_transactions_between(1, 1, 100, to_puzzle_hash=ph1) == [tr3] + assert await store.get_transactions_between(1, 1, 100, to_puzzle_hash=ph2) == [tr5] + + # reverse + + # test different limits + assert await store.get_transactions_between(1, 0, 100, to_puzzle_hash=ph1, reverse=True) == [tr3, tr2] + assert await store.get_transactions_between(1, 0, 100, to_puzzle_hash=ph2, reverse=True) == [tr5, tr4] + + # test different start offsets + assert await store.get_transactions_between(1, 1, 100, to_puzzle_hash=ph1, reverse=True) == [tr2] + assert await store.get_transactions_between(1, 1, 100, to_puzzle_hash=ph2, reverse=True) == [tr4] + + +@pytest.mark.asyncio +async def test_get_not_sent() -> None: + async with DBConnection1() as db_wrapper: + store = await WalletTransactionStore.create(db_wrapper) + + tr2 = dataclasses.replace(tr1, name=token_bytes(32), confirmed=True, confirmed_at_height=uint32(1)) + tr3 = dataclasses.replace(tr1, name=token_bytes(32)) + tr4 = dataclasses.replace(tr1, name=token_bytes(32)) + + await store.add_transaction_record(tr1, False) + await store.add_transaction_record(tr2, False) + await store.add_transaction_record(tr3, False) + await store.add_transaction_record(tr4, False) + + not_sent = await store.get_not_sent() + assert cmp(not_sent, [tr1, tr3, tr4]) + + not_sent = await store.get_not_sent() + assert cmp(not_sent, [tr1, tr3, tr4]) + + not_sent = await store.get_not_sent() + assert cmp(not_sent, [tr1, tr3, tr4]) + + not_sent = await store.get_not_sent() + assert cmp(not_sent, [tr1, tr3, tr4]) + + not_sent = await store.get_not_sent() + assert cmp(not_sent, [tr1, tr3, tr4]) + + # the 6th time we call this function, we don't get any unsent txs + not_sent = await store.get_not_sent() + assert cmp(not_sent, []) + + # TODO: also cover include_accepted_txs=True From 8055468579514e78f6eb162b0b516f77812d8607 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Jul 2022 10:47:46 -0500 Subject: [PATCH 17/62] Bump minimist from 1.2.5 to 1.2.6 in /build_scripts/npm_macos (#10870) Bumps [minimist](https://github.com/substack/minimist) from 1.2.5 to 1.2.6. - [Release notes](https://github.com/substack/minimist/releases) - [Commits](https://github.com/substack/minimist/compare/1.2.5...1.2.6) --- updated-dependencies: - dependency-name: minimist dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build_scripts/npm_macos/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build_scripts/npm_macos/package-lock.json b/build_scripts/npm_macos/package-lock.json index f8cf4d61e0..cc7e6c8d17 100644 --- a/build_scripts/npm_macos/package-lock.json +++ b/build_scripts/npm_macos/package-lock.json @@ -6158,9 +6158,9 @@ } }, "node_modules/minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, "node_modules/minimist-options": { "version": "4.1.0", @@ -14054,9 +14054,9 @@ } }, "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, "minimist-options": { "version": "4.1.0", From 299718aac9bad9f31308d6ead14c484e159587ec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Jul 2022 10:48:31 -0500 Subject: [PATCH 18/62] Bump node-fetch from 2.6.6 to 2.6.7 in /build_scripts/npm_linux_rpm (#10421) Bumps [node-fetch](https://github.com/node-fetch/node-fetch) from 2.6.6 to 2.6.7. - [Release notes](https://github.com/node-fetch/node-fetch/releases) - [Commits](https://github.com/node-fetch/node-fetch/compare/v2.6.6...v2.6.7) --- updated-dependencies: - dependency-name: node-fetch dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build_scripts/npm_linux_rpm/package-lock.json | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/build_scripts/npm_linux_rpm/package-lock.json b/build_scripts/npm_linux_rpm/package-lock.json index 128c024ba9..77adc008c0 100644 --- a/build_scripts/npm_linux_rpm/package-lock.json +++ b/build_scripts/npm_linux_rpm/package-lock.json @@ -5227,14 +5227,22 @@ "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==" }, "node_modules/node-fetch": { - "version": "2.6.6", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", - "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", "dependencies": { "whatwg-url": "^5.0.0" }, "engines": { "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } } }, "node_modules/node-fetch/node_modules/tr46": { @@ -11994,9 +12002,9 @@ "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==" }, "node-fetch": { - "version": "2.6.6", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", - "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", "requires": { "whatwg-url": "^5.0.0" }, From 60a7581df8c32a48ea57e71ed9cac73dac6d8c12 Mon Sep 17 00:00:00 2001 From: dustinface <35775977+xdustinface@users.noreply.github.com> Date: Fri, 1 Jul 2022 20:18:57 +0200 Subject: [PATCH 19/62] streamable: Turn `dataclass_from_dict` into `streamable_from_dict` (#11763) * Restrict `dataclass_from_dict` to valid streamable classes They need to have the decorator applied for the required cache being populated. * `dataclass_from_dict` -> `streamable_from_dict` * `TestDataclassFromDict` -> `StreamableFromDict` --- chia/types/spend_bundle.py | 4 +- chia/util/streamable.py | 23 ++----- tests/core/util/test_streamable.py | 106 +++++++++++++---------------- 3 files changed, 55 insertions(+), 78 deletions(-) diff --git a/chia/types/spend_bundle.py b/chia/types/spend_bundle.py index 9c70aa918f..0667375f4f 100644 --- a/chia/types/spend_bundle.py +++ b/chia/types/spend_bundle.py @@ -8,7 +8,7 @@ from blspy import AugSchemeMPL, G2Element from chia.consensus.default_constants import DEFAULT_CONSTANTS from chia.types.blockchain_format.coin import Coin from chia.types.blockchain_format.sized_bytes import bytes32 -from chia.util.streamable import Streamable, dataclass_from_dict, recurse_jsonify, streamable +from chia.util.streamable import Streamable, streamable_from_dict, recurse_jsonify, streamable from chia.wallet.util.debug_spend_bundle import debug_spend_bundle from .coin_spend import CoinSpend @@ -95,7 +95,7 @@ class SpendBundle(Streamable): warnings.warn("`coin_solutions` is now `coin_spends` in `SpendBundle.from_json_dict`") else: raise ValueError("JSON contains both `coin_solutions` and `coin_spends`, just use `coin_spends`") - return dataclass_from_dict(cls, json_dict) + return streamable_from_dict(cls, json_dict) def to_json_dict(self, include_legacy_keys: bool = True, exclude_modern_keys: bool = True): if include_legacy_keys is False and exclude_modern_keys is True: diff --git a/chia/util/streamable.py b/chia/util/streamable.py index 3e93306a43..10e8c5b34e 100644 --- a/chia/util/streamable.py +++ b/chia/util/streamable.py @@ -169,7 +169,7 @@ def convert_primitive(f_type: Type[Any], item: Any) -> Any: raise TypeError(f"Can't convert type {type(item).__name__} to {f_type.__name__}: {e}") from e -def dataclass_from_dict(klass: Type[Any], item: Any) -> Any: +def streamable_from_dict(klass: Type[_T_Streamable], item: Any) -> _T_Streamable: """ Converts a dictionary based on a dataclass, into an instance of that dataclass. Recursively goes through lists, optionals, and dictionaries. @@ -179,22 +179,12 @@ def dataclass_from_dict(klass: Type[Any], item: Any) -> Any: if not isinstance(item, dict): raise TypeError(f"expected: dict, actual: {type(item).__name__}") - if klass not in CONVERT_FUNCTIONS_FOR_STREAMABLE_CLASS: - # For non-streamable dataclasses we can't populate the cache on startup, so we do it here for convert - # functions only. - fields = create_fields_cache(klass) - convert_funcs = [function_to_convert_one_item(field.type) for field in fields] - FIELDS_FOR_STREAMABLE_CLASS[klass] = fields - CONVERT_FUNCTIONS_FOR_STREAMABLE_CLASS[klass] = convert_funcs - else: - fields = FIELDS_FOR_STREAMABLE_CLASS[klass] - convert_funcs = CONVERT_FUNCTIONS_FOR_STREAMABLE_CLASS[klass] - + fields = FIELDS_FOR_STREAMABLE_CLASS[klass] try: return klass( **{ field.name: convert_func(item[field.name]) - for field, convert_func in zip(fields, convert_funcs) + for field, convert_func in zip(fields, CONVERT_FUNCTIONS_FOR_STREAMABLE_CLASS[klass]) if field.name in item } ) @@ -224,9 +214,6 @@ def function_to_convert_one_item(f_type: Type[Any]) -> ConvertFunctionType: convert_inner_func = function_to_convert_one_item(inner_type) # Ignoring for now as the proper solution isn't obvious return lambda items: convert_list(convert_inner_func, items) # type: ignore[arg-type] - elif dataclasses.is_dataclass(f_type): - # Type is a dataclass, data is a dictionary - return lambda item: dataclass_from_dict(f_type, item) elif hasattr(f_type, "from_json_dict"): return lambda item: f_type.from_json_dict(item) elif issubclass(f_type, bytes): @@ -670,5 +657,5 @@ class Streamable: return ret @classmethod - def from_json_dict(cls: Any, json_dict: Dict[str, Any]) -> Any: - return dataclass_from_dict(cls, json_dict) + def from_json_dict(cls: Type[_T_Streamable], json_dict: Dict[str, Any]) -> _T_Streamable: + return streamable_from_dict(cls, json_dict) diff --git a/tests/core/util/test_streamable.py b/tests/core/util/test_streamable.py index 9413c1f6e4..9c418f056f 100644 --- a/tests/core/util/test_streamable.py +++ b/tests/core/util/test_streamable.py @@ -19,7 +19,6 @@ from chia.util.ints import uint8, uint32, uint64 from chia.util.streamable import ( DefinitionError, Streamable, - dataclass_from_dict, is_type_List, is_type_SpecificOptional, is_type_Tuple, @@ -32,6 +31,7 @@ from chia.util.streamable import ( parse_tuple, parse_uint32, streamable, + streamable_from_dict, write_uint32, ) from tests.block_tools import BlockTools @@ -94,41 +94,27 @@ def test_plain_class_not_supported() -> None: a: PlainClass -@dataclass -class TestDataclassFromDict1: - a: int +@streamable +@dataclass(frozen=True) +class StreamableFromDict1(Streamable): + a: uint8 b: str c: G1Element -@dataclass -class TestDataclassFromDict2: - a: TestDataclassFromDict1 - b: TestDataclassFromDict1 - c: float +@streamable +@dataclass(frozen=True) +class StreamableFromDict2(Streamable): + a: StreamableFromDict1 + b: StreamableFromDict1 + c: uint64 -def test_pure_dataclasses_in_dataclass_from_dict() -> None: - - d1_dict = {"a": 1, "b": "2", "c": str(G1Element())} - - d1: TestDataclassFromDict1 = dataclass_from_dict(TestDataclassFromDict1, d1_dict) - assert d1.a == 1 - assert d1.b == "2" - assert d1.c == G1Element() - - d2_dict = {"a": d1, "b": d1_dict, "c": 1.2345} - - d2: TestDataclassFromDict2 = dataclass_from_dict(TestDataclassFromDict2, d2_dict) - assert d2.a == d1 - assert d2.b == d1 - assert d2.c == 1.2345 - - -@dataclass -class ConvertTupleFailures: - a: Tuple[int, int] - b: Tuple[int, Tuple[int, int]] +@streamable +@dataclass(frozen=True) +class ConvertTupleFailures(Streamable): + a: Tuple[uint8, uint8] + b: Tuple[uint8, Tuple[uint8, uint8]] @pytest.mark.parametrize( @@ -149,13 +135,14 @@ class ConvertTupleFailures: def test_convert_tuple_failures(input_dict: Dict[str, Any], error: Any) -> None: with pytest.raises(error): - dataclass_from_dict(ConvertTupleFailures, input_dict) + streamable_from_dict(ConvertTupleFailures, input_dict) -@dataclass -class ConvertListFailures: - a: List[int] - b: List[List[int]] +@streamable +@dataclass(frozen=True) +class ConvertListFailures(Streamable): + a: List[uint8] + b: List[List[uint8]] @pytest.mark.parametrize( @@ -172,11 +159,12 @@ class ConvertListFailures: def test_convert_list_failures(input_dict: Dict[str, Any], error: Any) -> None: with pytest.raises(error): - dataclass_from_dict(ConvertListFailures, input_dict) + streamable_from_dict(ConvertListFailures, input_dict) -@dataclass -class ConvertByteTypeFailures: +@streamable +@dataclass(frozen=True) +class ConvertByteTypeFailures(Streamable): a: bytes4 b: bytes @@ -201,11 +189,12 @@ class ConvertByteTypeFailures: def test_convert_byte_type_failures(input_dict: Dict[str, Any], error: Any) -> None: with pytest.raises(error): - dataclass_from_dict(ConvertByteTypeFailures, input_dict) + streamable_from_dict(ConvertByteTypeFailures, input_dict) -@dataclass -class ConvertUnhashableTypeFailures: +@streamable +@dataclass(frozen=True) +class ConvertUnhashableTypeFailures(Streamable): a: G1Element @@ -226,7 +215,7 @@ class ConvertUnhashableTypeFailures: def test_convert_unhashable_type_failures(input_dict: Dict[str, Any], error: Any) -> None: with pytest.raises(error): - dataclass_from_dict(ConvertUnhashableTypeFailures, input_dict) + streamable_from_dict(ConvertUnhashableTypeFailures, input_dict) class NoStrClass: @@ -234,9 +223,10 @@ class NoStrClass: raise RuntimeError("No string") -@dataclass -class ConvertPrimitiveFailures: - a: int +@streamable +@dataclass(frozen=True) +class ConvertPrimitiveFailures(Streamable): + a: uint8 b: uint8 c: str @@ -252,28 +242,28 @@ class ConvertPrimitiveFailures: def test_convert_primitive_failures(input_dict: Dict[str, Any], error: Any) -> None: with pytest.raises(error): - dataclass_from_dict(ConvertPrimitiveFailures, input_dict) + streamable_from_dict(ConvertPrimitiveFailures, input_dict) @pytest.mark.parametrize( "test_class, input_dict, error", [ - [TestDataclassFromDict1, {"a": "asdf", "b": "2", "c": G1Element()}, TypeError], - [TestDataclassFromDict1, {"a": 1, "b": "2"}, KeyError], - [TestDataclassFromDict1, {"a": 1, "b": "2", "c": "asd"}, TypeError], - [TestDataclassFromDict1, {"a": 1, "b": "2", "c": "00" * G1Element.SIZE}, TypeError], - [TestDataclassFromDict1, {"a": [], "b": "2", "c": G1Element()}, TypeError], - [TestDataclassFromDict1, {"a": {}, "b": "2", "c": G1Element()}, TypeError], - [TestDataclassFromDict2, {"a": "asdf", "b": 1.2345, "c": 1.2345}, TypeError], - [TestDataclassFromDict2, {"a": 1.2345, "b": {"a": 1, "b": "2"}, "c": 1.2345}, TypeError], - [TestDataclassFromDict2, {"a": {"a": 1, "b": "2", "c": G1Element()}, "b": {"a": 1, "b": "2"}}, KeyError], - [TestDataclassFromDict2, {"a": {"a": 1, "b": "2"}, "b": {"a": 1, "b": "2"}, "c": 1.2345}, KeyError], + [StreamableFromDict1, {"a": "asdf", "b": "2", "c": G1Element()}, TypeError], + [StreamableFromDict1, {"a": 1, "b": "2"}, KeyError], + [StreamableFromDict1, {"a": 1, "b": "2", "c": "asd"}, TypeError], + [StreamableFromDict1, {"a": 1, "b": "2", "c": "00" * G1Element.SIZE}, TypeError], + [StreamableFromDict1, {"a": [], "b": "2", "c": G1Element()}, TypeError], + [StreamableFromDict1, {"a": {}, "b": "2", "c": G1Element()}, TypeError], + [StreamableFromDict2, {"a": "asdf", "b": 12345, "c": 12345}, TypeError], + [StreamableFromDict2, {"a": 12345, "b": {"a": 1, "b": "2"}, "c": 12345}, TypeError], + [StreamableFromDict2, {"a": {"a": 1, "b": "2", "c": G1Element()}, "b": {"a": 1, "b": "2"}}, KeyError], + [StreamableFromDict2, {"a": {"a": 1, "b": "2"}, "b": {"a": 1, "b": "2"}, "c": 12345}, KeyError], ], ) -def test_dataclass_from_dict_failures(test_class: Type[Any], input_dict: Dict[str, Any], error: Any) -> None: +def test_streamable_from_dict_failures(test_class: Type[Streamable], input_dict: Dict[str, Any], error: Any) -> None: with pytest.raises(error): - dataclass_from_dict(test_class, input_dict) + streamable_from_dict(test_class, input_dict) @streamable From e569817ac750d6271ed0c03e4c817156a92372a0 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 1 Jul 2022 14:47:01 -0400 Subject: [PATCH 20/62] replace service `running_new_process=` parameter by `.setup_process_global_state()` method (#12172) --- chia/server/start_service.py | 32 +++++++++++--------------------- tests/setup_services.py | 18 ++++++------------ 2 files changed, 17 insertions(+), 33 deletions(-) diff --git a/chia/server/start_service.py b/chia/server/start_service.py index b1508ed5af..392bff247f 100644 --- a/chia/server/start_service.py +++ b/chia/server/start_service.py @@ -53,8 +53,6 @@ class Service: rpc_info: Optional[Tuple[type, int]] = None, parse_cli_args=True, connect_to_daemon=True, - running_new_process=True, - service_name_prefix="", max_request_body_size: Optional[int] = None, override_capabilities: Optional[List[Tuple[uint16, str]]] = None, ) -> None: @@ -71,24 +69,13 @@ class Service: self._rpc_close_task: Optional[asyncio.Task] = None self._network_id: str = network_id self.max_request_body_size = max_request_body_size - self._running_new_process = running_new_process - - # when we start this service as a component of an existing process, - # don't change its proctitle - if running_new_process: - proctitle_name = f"chia_{service_name_prefix}{service_name}" - setproctitle(proctitle_name) self._log = logging.getLogger(service_name) if parse_cli_args: - service_config = load_config_cli(root_path, "config.yaml", service_name) + self.service_config = load_config_cli(root_path, "config.yaml", service_name) else: - service_config = load_config(root_path, "config.yaml", service_name) - - # only initialize logging once per process - if running_new_process: - initialize_logging(service_name, service_config["logging"], root_path) + self.service_config = load_config(root_path, "config.yaml", service_name) self._rpc_info = rpc_info private_ca_crt, private_ca_key = private_ssl_ca_paths(root_path, self.config) @@ -96,7 +83,7 @@ class Service: inbound_rlp = self.config.get("inbound_rate_limit_percent") outbound_rlp = self.config.get("outbound_rate_limit_percent") if node_type == NodeType.WALLET: - inbound_rlp = service_config.get("inbound_rate_limit_percent", inbound_rlp) + inbound_rlp = self.service_config.get("inbound_rate_limit_percent", inbound_rlp) outbound_rlp = 60 capabilities_to_use: List[Tuple[uint16, str]] = capabilities if override_capabilities is not None: @@ -114,7 +101,7 @@ class Service: outbound_rlp, capabilities_to_use, root_path, - service_config, + self.service_config, (private_ca_crt, private_ca_key), (chia_ca_crt, chia_ca_key), name=f"{service_name}_server", @@ -154,9 +141,6 @@ class Service: self._did_start = True - if self._running_new_process: - self._enable_signals() - await self._node._start(**kwargs) self._node._shut_down = False @@ -201,7 +185,12 @@ class Service: await self.start() await self.wait_closed() - def _enable_signals(self) -> None: + async def setup_process_global_state(self) -> None: + # Being async forces this to be run from within an active event loop as is + # needed for the signal handler setup. + proctitle_name = f"chia_{self._service_name}" + setproctitle(proctitle_name) + initialize_logging(self._service_name, self.service_config["logging"], self.root_path) global main_pid main_pid = os.getpid() @@ -288,6 +277,7 @@ class Service: async def async_run_service(*args, **kwargs) -> None: service = Service(*args, **kwargs) + await service.setup_process_global_state() return await service.run() diff --git a/tests/setup_services.py b/tests/setup_services.py index 5f1cbf9f15..29b730cacc 100644 --- a/tests/setup_services.py +++ b/tests/setup_services.py @@ -118,12 +118,11 @@ async def setup_full_node( kwargs.update( parse_cli_args=False, connect_to_daemon=connect_to_daemon, - service_name_prefix="test_", ) if disable_capabilities is not None: kwargs.update(override_capabilities=get_capabilities(disable_capabilities)) - service = Service(**kwargs, running_new_process=False) + service = Service(**kwargs) await service.start() @@ -189,10 +188,9 @@ async def setup_wallet_node( kwargs.update( parse_cli_args=False, connect_to_daemon=False, - service_name_prefix="test_", ) - service = Service(**kwargs, running_new_process=False) + service = Service(**kwargs) await service.start() @@ -229,10 +227,9 @@ async def setup_harvester( kwargs.update( parse_cli_args=False, connect_to_daemon=False, - service_name_prefix="test_", ) - service = Service(**kwargs, running_new_process=False) + service = Service(**kwargs) if start_service: await service.start() @@ -278,10 +275,9 @@ async def setup_farmer( kwargs.update( parse_cli_args=False, connect_to_daemon=False, - service_name_prefix="test_", ) - service = Service(**kwargs, running_new_process=False) + service = Service(**kwargs) if start_service: await service.start() @@ -301,10 +297,9 @@ async def setup_introducer(bt: BlockTools, port): advertised_port=port, parse_cli_args=False, connect_to_daemon=False, - service_name_prefix="test_", ) - service = Service(**kwargs, running_new_process=False) + service = Service(**kwargs) await service.start() @@ -362,10 +357,9 @@ async def setup_timelord( kwargs.update( parse_cli_args=False, connect_to_daemon=False, - service_name_prefix="test_", ) - service = Service(**kwargs, running_new_process=False) + service = Service(**kwargs) await service.start() From 9cc5b729d566a57aadabdb8225ac89c13671af47 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Jul 2022 13:48:16 -0500 Subject: [PATCH 21/62] Bump filelock from 3.4.2 to 3.7.1 (#11728) Bumps [filelock](https://github.com/tox-dev/py-filelock) from 3.4.2 to 3.7.1. - [Release notes](https://github.com/tox-dev/py-filelock/releases) - [Changelog](https://github.com/tox-dev/py-filelock/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/py-filelock/compare/3.4.2...3.7.1) --- updated-dependencies: - dependency-name: filelock dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4cb169882f..533dce99d7 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ dependencies = [ "concurrent-log-handler==0.9.19", # Concurrently log and rotate logs "cryptography==36.0.2", # Python cryptography library for TLS - keyring conflict "fasteners==0.16.3", # For interprocess file locking, expected to be replaced by filelock - "filelock==3.4.2", # For reading and writing config multiprocess and multithread safely (non-reentrant locks) + "filelock==3.7.1", # For reading and writing config multiprocess and multithread safely (non-reentrant locks) "keyring==23.0.1", # Store keys in MacOS Keychain, Windows Credential Locker "keyrings.cryptfile==1.3.4", # Secure storage for keys on Linux (Will be replaced) # "keyrings.cryptfile==1.3.8", # Secure storage for keys on Linux (Will be replaced) From b07c6635221052a867d4d5717d2ca0479727c0f1 Mon Sep 17 00:00:00 2001 From: Earle Lowe <30607889+emlowe@users.noreply.github.com> Date: Fri, 1 Jul 2022 12:42:36 -0700 Subject: [PATCH 22/62] El.dependabot/node fetch combined (#12182) * Bump node-fetch from 2.6.6 to 2.6.7 in /build_scripts/npm_linux_deb Bumps [node-fetch](https://github.com/node-fetch/node-fetch) from 2.6.6 to 2.6.7. - [Release notes](https://github.com/node-fetch/node-fetch/releases) - [Commits](https://github.com/node-fetch/node-fetch/compare/v2.6.6...v2.6.7) --- updated-dependencies: - dependency-name: node-fetch dependency-type: indirect ... Signed-off-by: dependabot[bot] * Bump node-fetch from 2.6.6 to 2.6.7 in /build_scripts/npm_macos Bumps [node-fetch](https://github.com/node-fetch/node-fetch) from 2.6.6 to 2.6.7. - [Release notes](https://github.com/node-fetch/node-fetch/releases) - [Commits](https://github.com/node-fetch/node-fetch/compare/v2.6.6...v2.6.7) --- updated-dependencies: - dependency-name: node-fetch dependency-type: indirect ... Signed-off-by: dependabot[bot] * Bump node-fetch from 2.6.6 to 2.6.7 in /build_scripts/npm_macos_m1 Bumps [node-fetch](https://github.com/node-fetch/node-fetch) from 2.6.6 to 2.6.7. - [Release notes](https://github.com/node-fetch/node-fetch/releases) - [Commits](https://github.com/node-fetch/node-fetch/compare/v2.6.6...v2.6.7) --- updated-dependencies: - dependency-name: node-fetch dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build_scripts/npm_linux_deb/package-lock.json | 20 +++++++++++++------ build_scripts/npm_macos/package-lock.json | 20 +++++++++++++------ build_scripts/npm_macos_m1/package-lock.json | 20 +++++++++++++------ 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/build_scripts/npm_linux_deb/package-lock.json b/build_scripts/npm_linux_deb/package-lock.json index f463a96ada..033e1f8674 100644 --- a/build_scripts/npm_linux_deb/package-lock.json +++ b/build_scripts/npm_linux_deb/package-lock.json @@ -5385,14 +5385,22 @@ "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==" }, "node_modules/node-fetch": { - "version": "2.6.6", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", - "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", "dependencies": { "whatwg-url": "^5.0.0" }, "engines": { "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } } }, "node_modules/node-fetch/node_modules/tr46": { @@ -12298,9 +12306,9 @@ "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==" }, "node-fetch": { - "version": "2.6.6", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", - "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", "requires": { "whatwg-url": "^5.0.0" }, diff --git a/build_scripts/npm_macos/package-lock.json b/build_scripts/npm_macos/package-lock.json index cc7e6c8d17..f212ce8bb3 100644 --- a/build_scripts/npm_macos/package-lock.json +++ b/build_scripts/npm_macos/package-lock.json @@ -6372,14 +6372,22 @@ "optional": true }, "node_modules/node-fetch": { - "version": "2.6.6", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", - "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", "dependencies": { "whatwg-url": "^5.0.0" }, "engines": { "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } } }, "node_modules/node-fetch/node_modules/tr46": { @@ -14220,9 +14228,9 @@ "optional": true }, "node-fetch": { - "version": "2.6.6", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", - "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", "requires": { "whatwg-url": "^5.0.0" }, diff --git a/build_scripts/npm_macos_m1/package-lock.json b/build_scripts/npm_macos_m1/package-lock.json index f9ce1ec621..5831c20ad4 100644 --- a/build_scripts/npm_macos_m1/package-lock.json +++ b/build_scripts/npm_macos_m1/package-lock.json @@ -6344,14 +6344,22 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==" }, "node_modules/node-fetch": { - "version": "2.6.6", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", - "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", "dependencies": { "whatwg-url": "^5.0.0" }, "engines": { "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } } }, "node_modules/node-fetch/node_modules/tr46": { @@ -14152,9 +14160,9 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==" }, "node-fetch": { - "version": "2.6.6", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.6.tgz", - "integrity": "sha512-Z8/6vRlTUChSdIgMa51jxQ4lrw/Jy5SOW10ObaA47/RElsAN2c5Pn8bTgFGWn/ibwzXTE8qwr1Yzx28vsecXEA==", + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", "requires": { "whatwg-url": "^5.0.0" }, From ef1d2af651ae71f6c189e549639becf49a9b4f93 Mon Sep 17 00:00:00 2001 From: Earle Lowe <30607889+emlowe@users.noreply.github.com> Date: Fri, 1 Jul 2022 12:43:13 -0700 Subject: [PATCH 23/62] El.dependabot/minimist 1.2.6 (#12183) * Bump minimist from 1.2.5 to 1.2.6 in /build_scripts/npm_linux_rpm Bumps [minimist](https://github.com/substack/minimist) from 1.2.5 to 1.2.6. - [Release notes](https://github.com/substack/minimist/releases) - [Commits](https://github.com/substack/minimist/compare/1.2.5...1.2.6) --- updated-dependencies: - dependency-name: minimist dependency-type: indirect ... Signed-off-by: dependabot[bot] * Bump minimist from 1.2.5 to 1.2.6 in /build_scripts/npm_linux_deb Bumps [minimist](https://github.com/substack/minimist) from 1.2.5 to 1.2.6. - [Release notes](https://github.com/substack/minimist/releases) - [Commits](https://github.com/substack/minimist/compare/1.2.5...1.2.6) --- updated-dependencies: - dependency-name: minimist dependency-type: indirect ... Signed-off-by: dependabot[bot] * Bump minimist from 1.2.5 to 1.2.6 in /build_scripts/npm_macos_m1 Bumps [minimist](https://github.com/substack/minimist) from 1.2.5 to 1.2.6. - [Release notes](https://github.com/substack/minimist/releases) - [Commits](https://github.com/substack/minimist/compare/1.2.5...1.2.6) --- updated-dependencies: - dependency-name: minimist dependency-type: indirect ... Signed-off-by: dependabot[bot] * Bump minimist from 1.2.5 to 1.2.6 in /build_scripts/npm_windows Bumps [minimist](https://github.com/substack/minimist) from 1.2.5 to 1.2.6. - [Release notes](https://github.com/substack/minimist/releases) - [Commits](https://github.com/substack/minimist/compare/1.2.5...1.2.6) --- updated-dependencies: - dependency-name: minimist dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build_scripts/npm_linux_deb/package-lock.json | 12 ++++++------ build_scripts/npm_linux_rpm/package-lock.json | 12 ++++++------ build_scripts/npm_macos_m1/package-lock.json | 12 ++++++------ build_scripts/npm_windows/package-lock.json | 12 ++++++------ 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/build_scripts/npm_linux_deb/package-lock.json b/build_scripts/npm_linux_deb/package-lock.json index 033e1f8674..afb92e5aa2 100644 --- a/build_scripts/npm_linux_deb/package-lock.json +++ b/build_scripts/npm_linux_deb/package-lock.json @@ -5194,9 +5194,9 @@ } }, "node_modules/minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, "node_modules/minimist-options": { "version": "4.1.0", @@ -12163,9 +12163,9 @@ } }, "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, "minimist-options": { "version": "4.1.0", diff --git a/build_scripts/npm_linux_rpm/package-lock.json b/build_scripts/npm_linux_rpm/package-lock.json index 77adc008c0..42d76a74aa 100644 --- a/build_scripts/npm_linux_rpm/package-lock.json +++ b/build_scripts/npm_linux_rpm/package-lock.json @@ -5036,9 +5036,9 @@ } }, "node_modules/minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, "node_modules/minimist-options": { "version": "4.1.0", @@ -11859,9 +11859,9 @@ } }, "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, "minimist-options": { "version": "4.1.0", diff --git a/build_scripts/npm_macos_m1/package-lock.json b/build_scripts/npm_macos_m1/package-lock.json index 5831c20ad4..c604d19e17 100644 --- a/build_scripts/npm_macos_m1/package-lock.json +++ b/build_scripts/npm_macos_m1/package-lock.json @@ -6133,9 +6133,9 @@ } }, "node_modules/minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, "node_modules/minimist-options": { "version": "4.1.0", @@ -13997,9 +13997,9 @@ } }, "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, "minimist-options": { "version": "4.1.0", diff --git a/build_scripts/npm_windows/package-lock.json b/build_scripts/npm_windows/package-lock.json index a968c736d4..f752cda177 100644 --- a/build_scripts/npm_windows/package-lock.json +++ b/build_scripts/npm_windows/package-lock.json @@ -4974,9 +4974,9 @@ } }, "node_modules/minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, "node_modules/minimist-options": { "version": "4.1.0", @@ -11730,9 +11730,9 @@ } }, "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, "minimist-options": { "version": "4.1.0", From 52df955a1ad3d0333a53a4dc4b4b4cda578b69b1 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 2 Jul 2022 02:08:13 -0400 Subject: [PATCH 24/62] Make start_crawler.py consistent with start_*.py (#12196) --- chia/seeder/start_crawler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chia/seeder/start_crawler.py b/chia/seeder/start_crawler.py index bd1062cf8b..4fc1180298 100644 --- a/chia/seeder/start_crawler.py +++ b/chia/seeder/start_crawler.py @@ -16,7 +16,7 @@ from chia.util.default_root import DEFAULT_ROOT_PATH # See: https://bugs.python.org/issue29288 "".encode("idna") -SERVICE_NAME = "full_node" +SERVICE_NAME = "seeder" log = logging.getLogger(__name__) @@ -51,7 +51,7 @@ def service_kwargs_for_full_node_crawler( def main(): - config = load_config_cli(DEFAULT_ROOT_PATH, "config.yaml", "seeder") + config = load_config_cli(DEFAULT_ROOT_PATH, "config.yaml", SERVICE_NAME) overrides = config["network_overrides"]["constants"][config["selected_network"]] updated_constants = DEFAULT_CONSTANTS.replace_str_to_bytes(**overrides) kwargs = service_kwargs_for_full_node_crawler(DEFAULT_ROOT_PATH, config, updated_constants) From e57025a83e43077473da67b1281e230a1676f1de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Jul 2022 01:10:04 -0500 Subject: [PATCH 25/62] Bump keyring from 23.0.1 to 23.6.0 (#11829) * Bump keyring from 23.0.1 to 23.6.0 Bumps [keyring](https://github.com/jaraco/keyring) from 23.0.1 to 23.6.0. - [Release notes](https://github.com/jaraco/keyring/releases) - [Changelog](https://github.com/jaraco/keyring/blob/main/CHANGES.rst) - [Commits](https://github.com/jaraco/keyring/compare/v23.0.1...v23.6.0) --- updated-dependencies: - dependency-name: keyring dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Small type change for new keyring version Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Earle Lowe Co-authored-by: Earle Lowe <30607889+emlowe@users.noreply.github.com> --- chia/util/keyring_wrapper.py | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/chia/util/keyring_wrapper.py b/chia/util/keyring_wrapper.py index f830de1fc8..c1540e6470 100644 --- a/chia/util/keyring_wrapper.py +++ b/chia/util/keyring_wrapper.py @@ -50,7 +50,7 @@ def get_os_passphrase_store() -> Optional[OSPassphraseStore]: def check_legacy_keyring_keys_present(keyring: LegacyKeyring) -> bool: - from keyring.credentials import SimpleCredential + from keyring.credentials import Credential from chia.util.keychain import default_keychain_user, default_keychain_service, get_private_key_user, MAX_KEYS keychain_user: str = default_keychain_user() @@ -58,7 +58,7 @@ def check_legacy_keyring_keys_present(keyring: LegacyKeyring) -> bool: for index in range(0, MAX_KEYS): current_user: str = get_private_key_user(keychain_user, index) - credential: Optional[SimpleCredential] = keyring.get_credential(keychain_service, current_user) + credential: Optional[Credential] = keyring.get_credential(keychain_service, current_user) if credential is not None: return True return False diff --git a/setup.py b/setup.py index 533dce99d7..f6b614abfa 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ dependencies = [ "cryptography==36.0.2", # Python cryptography library for TLS - keyring conflict "fasteners==0.16.3", # For interprocess file locking, expected to be replaced by filelock "filelock==3.7.1", # For reading and writing config multiprocess and multithread safely (non-reentrant locks) - "keyring==23.0.1", # Store keys in MacOS Keychain, Windows Credential Locker + "keyring==23.6.0", # Store keys in MacOS Keychain, Windows Credential Locker "keyrings.cryptfile==1.3.4", # Secure storage for keys on Linux (Will be replaced) # "keyrings.cryptfile==1.3.8", # Secure storage for keys on Linux (Will be replaced) # See https://github.com/frispete/keyrings.cryptfile/issues/15 From 3ada87671884a8fb1caf16f5adcc9c21633d2f5c Mon Sep 17 00:00:00 2001 From: Matt Hauff Date: Fri, 1 Jul 2022 23:11:16 -0700 Subject: [PATCH 26/62] Refine wallet full node peer selection (#12118) * Refine wallet full node peer selection * forgot await * rework DID wallet loop * lint? * Wasn't properly returning untrusted/unsynced --- chia/rpc/wallet_rpc_api.py | 15 +++------ chia/wallet/cat_wallet/cat_wallet.py | 25 +++++--------- chia/wallet/did_wallet/did_wallet.py | 37 +++++++++----------- chia/wallet/nft_wallet/nft_wallet.py | 14 ++------ chia/wallet/wallet_node.py | 50 +++++++++++++++++----------- chia/wallet/wallet_state_manager.py | 10 +++--- 6 files changed, 68 insertions(+), 83 deletions(-) diff --git a/chia/rpc/wallet_rpc_api.py b/chia/rpc/wallet_rpc_api.py index 6298ff4843..23d963781d 100644 --- a/chia/rpc/wallet_rpc_api.py +++ b/chia/rpc/wallet_rpc_api.py @@ -1472,13 +1472,8 @@ class WalletRpcApi: coin_id = decode_puzzle_hash(coin_id) else: coin_id = bytes32.from_hexstr(coin_id) - peer = self.service.wallet_state_manager.wallet_node.get_full_node_peer() - if peer is None: - return {"success": False, "error": "Cannot find a full node peer."} # Get coin state - coin_state_list: List[CoinState] = await self.service.wallet_state_manager.wallet_node.get_coin_state( - [coin_id], peer=peer - ) + coin_state_list: List[CoinState] = await self.service.wallet_state_manager.wallet_node.get_coin_state([coin_id]) if coin_state_list is None or len(coin_state_list) < 1: return {"success": False, "error": f"Coin record 0x{coin_id.hex()} not found"} coin_state: CoinState = coin_state_list[0] @@ -1486,7 +1481,7 @@ class WalletRpcApi: # Find the unspent coin while coin_state.spent_height is not None: coin_state_list = await self.service.wallet_state_manager.wallet_node.fetch_children( - peer, coin_state.coin.name() + coin_state.coin.name() ) odd_coin = 0 for coin in coin_state_list: @@ -1499,7 +1494,7 @@ class WalletRpcApi: coin_state = coin_state_list[0] # Get parent coin parent_coin_state_list: List[CoinState] = await self.service.wallet_state_manager.wallet_node.get_coin_state( - [coin_state.coin.parent_coin_info], peer=peer + [coin_state.coin.parent_coin_info] ) if parent_coin_state_list is None or len(parent_coin_state_list) < 1: return { @@ -1508,7 +1503,7 @@ class WalletRpcApi: } parent_coin_state: CoinState = parent_coin_state_list[0] coin_spend: CoinSpend = await self.service.wallet_state_manager.wallet_node.fetch_puzzle_solution( - peer, parent_coin_state.spent_height, parent_coin_state.coin + parent_coin_state.spent_height, parent_coin_state.coin ) # convert to NFTInfo try: @@ -1535,7 +1530,7 @@ class WalletRpcApi: # Get launcher coin launcher_coin: List[CoinState] = await self.service.wallet_state_manager.wallet_node.get_coin_state( - [uncurried_nft.singleton_launcher_id], peer=peer + [uncurried_nft.singleton_launcher_id] ) if launcher_coin is None or len(launcher_coin) < 1 or launcher_coin[0].spent_height is None: return { diff --git a/chia/wallet/cat_wallet/cat_wallet.py b/chia/wallet/cat_wallet/cat_wallet.py index d60816e91b..6b2fe440ba 100644 --- a/chia/wallet/cat_wallet/cat_wallet.py +++ b/chia/wallet/cat_wallet/cat_wallet.py @@ -335,22 +335,15 @@ class CATWallet: lineage = await self.get_lineage_proof_for_coin(coin) if lineage is None: - for node_id, node in self.wallet_state_manager.wallet_node.server.all_connections.items(): - try: - coin_state = await self.wallet_state_manager.wallet_node.get_coin_state( - [coin.parent_coin_info], None, node - ) - # check for empty list and continue on to next node - if not coin_state: - continue - assert coin_state[0].coin.name() == coin.parent_coin_info - coin_spend = await self.wallet_state_manager.wallet_node.fetch_puzzle_solution( - node, coin_state[0].spent_height, coin_state[0].coin - ) - await self.puzzle_solution_received(coin_spend, parent_coin=coin_state[0].coin) - break - except Exception as e: - self.log.debug(f"Exception: {e}, traceback: {traceback.format_exc()}") + try: + coin_state = await self.wallet_state_manager.wallet_node.get_coin_state([coin.parent_coin_info]) + assert coin_state[0].coin.name() == coin.parent_coin_info + coin_spend = await self.wallet_state_manager.wallet_node.fetch_puzzle_solution( + coin_state[0].spent_height, coin_state[0].coin + ) + await self.puzzle_solution_received(coin_spend, parent_coin=coin_state[0].coin) + except Exception as e: + self.log.debug(f"Exception: {e}, traceback: {traceback.format_exc()}") async def puzzle_solution_received(self, coin_spend: CoinSpend, parent_coin: Coin): coin_name = coin_spend.coin.name() diff --git a/chia/wallet/did_wallet/did_wallet.py b/chia/wallet/did_wallet/did_wallet.py index 805a227304..ab67ed4ecf 100644 --- a/chia/wallet/did_wallet/did_wallet.py +++ b/chia/wallet/did_wallet/did_wallet.py @@ -434,22 +434,21 @@ class DIDWallet: did_info.origin_coin.name(), did_wallet_puzzles.metadata_to_program(json.loads(self.did_info.metadata)), ) - node = self.wallet_state_manager.wallet_node.get_full_node_peer() - children = await self.wallet_state_manager.wallet_node.fetch_children(node, did_info.origin_coin.name()) + wallet_node = self.wallet_state_manager.wallet_node + parent_coin: Coin = did_info.origin_coin while True: + children = await wallet_node.fetch_children(parent_coin.name()) if len(children) == 0: break children_state: CoinState = children[0] - coin = children_state.coin - name = coin.name() - children = await self.wallet_state_manager.wallet_node.fetch_children(node, name) + child_coin = children_state.coin future_parent = LineageProof( - coin.parent_coin_info, + child_coin.parent_coin_info, did_info.current_inner.get_tree_hash(), - uint64(coin.amount), + uint64(child_coin.amount), ) - await self.add_parent(coin.name(), future_parent, True) + await self.add_parent(child_coin.name(), future_parent, True) if children_state.spent_height != children_state.created_height: did_info = DIDInfo( did_info.origin_coin, @@ -457,7 +456,7 @@ class DIDWallet: did_info.num_of_backup_ids_needed, self.did_info.parent_info, did_info.current_inner, - coin, + child_coin, new_did_inner_puzhash, new_pubkey, False, @@ -466,23 +465,19 @@ class DIDWallet: await self.save_info(did_info, True) assert children_state.created_height - puzzle_solution_request = wallet_protocol.RequestPuzzleSolution( - coin.parent_coin_info, children_state.created_height + parent_spend = await wallet_node.fetch_puzzle_solution(children_state.created_height, parent_coin) + assert parent_spend is not None + parent_innerpuz = did_wallet_puzzles.get_innerpuzzle_from_puzzle( + parent_spend.puzzle_reveal.to_program() ) - parent_state: CoinState = ( - await self.wallet_state_manager.wallet_node.get_coin_state([coin.parent_coin_info]) - )[0] - response = await node.request_puzzle_solution(puzzle_solution_request) - req_puz_sol = response.response - assert req_puz_sol.puzzle is not None - parent_innerpuz = did_wallet_puzzles.get_innerpuzzle_from_puzzle(req_puz_sol.puzzle.to_program()) assert parent_innerpuz is not None parent_info = LineageProof( - parent_state.coin.parent_coin_info, + parent_coin.parent_coin_info, parent_innerpuz.get_tree_hash(), - uint64(parent_state.coin.amount), + uint64(parent_coin.amount), ) - await self.add_parent(coin.parent_coin_info, parent_info, True) + await self.add_parent(child_coin.parent_coin_info, parent_info, True) + parent_coin = child_coin assert parent_info is not None async def create_tandem_xch_tx( diff --git a/chia/wallet/nft_wallet/nft_wallet.py b/chia/wallet/nft_wallet/nft_wallet.py index 62a43031c1..fee5c73181 100644 --- a/chia/wallet/nft_wallet/nft_wallet.py +++ b/chia/wallet/nft_wallet/nft_wallet.py @@ -7,8 +7,6 @@ from typing import Any, Dict, List, Optional, Set, Tuple, Type, TypeVar from blspy import AugSchemeMPL, G2Element from chia.protocols.wallet_protocol import CoinState -from chia.server.outbound_message import NodeType -from chia.server.ws_connection import WSChiaConnection from chia.types.announcement import Announcement from chia.types.blockchain_format.coin import Coin from chia.types.blockchain_format.program import Program @@ -158,22 +156,14 @@ class NFTWallet: if coin_info.coin == coin: return wallet_node = self.wallet_state_manager.wallet_node - server = wallet_node.server - full_nodes: Dict[bytes32, WSChiaConnection] = server.connection_by_type.get(NodeType.FULL_NODE, {}) cs: Optional[CoinSpend] = None - coin_states: Optional[List[CoinState]] = await self.wallet_state_manager.wallet_node.get_coin_state( - [coin.parent_coin_info] - ) + coin_states: Optional[List[CoinState]] = await wallet_node.get_coin_state([coin.parent_coin_info]) if not coin_states: # farm coin return assert coin_states parent_coin = coin_states[0].coin - for node_id in full_nodes: - node = server.all_connections[node_id] - cs = await wallet_node.fetch_puzzle_solution(node, height, parent_coin) - if cs is not None: - break + cs = await wallet_node.fetch_puzzle_solution(height, parent_coin) assert cs is not None await self.puzzle_solution_received(cs, in_transaction=in_transaction) diff --git a/chia/wallet/wallet_node.py b/chia/wallet/wallet_node.py index b0c7abbda8..fbf2793086 100644 --- a/chia/wallet/wallet_node.py +++ b/chia/wallet/wallet_node.py @@ -827,13 +827,27 @@ class WalletNode: request.peak_hash, ) - def get_full_node_peer(self) -> Optional[WSChiaConnection]: + def get_full_node_peer(self, synced_only: bool = False) -> Optional[WSChiaConnection]: + """ + Get a full node, preferring synced & trusted > synced & untrusted > unsynced & trusted > unsynced & untrusted + """ if self._server is None: return None nodes = self.server.get_full_node_connections() if len(nodes) > 0: - return random.choice(nodes) + synced_peers = set(node for node in nodes if node.peer_node_id in self.synced_peers) + trusted_peers = set(node for node in nodes if self.is_trusted(node)) + if len(synced_peers & trusted_peers) > 0: + return random.choice(list(synced_peers & trusted_peers)) + elif len(synced_peers) > 0: + return random.choice(list(synced_peers)) + elif synced_only: + return None + elif len(trusted_peers) > 0: + return random.choice(list(trusted_peers)) + else: + return random.choice(list(nodes)) else: return None @@ -1462,7 +1476,13 @@ class WalletNode: peer_request_cache.add_to_blocks_validated(reward_chain_hash, height) return True - async def fetch_puzzle_solution(self, peer: WSChiaConnection, height: uint32, coin: Coin) -> CoinSpend: + async def fetch_puzzle_solution( + self, height: uint32, coin: Coin, peer: Optional[WSChiaConnection] = None + ) -> CoinSpend: + if peer is None: + peer = self.get_full_node_peer() + if peer is None: + raise ValueError("Could not find any peers to request puzzle and solution from") solution_response = await peer.request_puzzle_solution( wallet_protocol.RequestPuzzleSolution(coin.name(), height) ) @@ -1480,23 +1500,11 @@ class WalletNode: async def get_coin_state( self, coin_names: List[bytes32], fork_height: Optional[uint32] = None, peer: Optional[WSChiaConnection] = None ) -> List[CoinState]: - all_nodes = self.server.connection_by_type[NodeType.FULL_NODE] - if len(all_nodes.keys()) == 0: - raise ValueError("Not connected to the full node") - # Use supplied if provided, prioritize trusted otherwise - synced_peers = [node for node in all_nodes.values() if node.peer_node_id in self.synced_peers] if peer is None: - for node in synced_peers: - if self.is_trusted(node): - peer = node - break - if peer is None: - if len(synced_peers) > 0: - peer = synced_peers[0] - else: - peer = list(all_nodes.values())[0] + peer = self.get_full_node_peer() + if peer is None: + raise ValueError("Could not find any peers to request puzzle and solution from") - assert peer is not None msg = wallet_protocol.RegisterForCoinUpdates(coin_names, uint32(0)) coin_state: Optional[RespondToCoinUpdates] = await peer.register_interest_in_coin(msg) assert coin_state is not None @@ -1514,8 +1522,12 @@ class WalletNode: return coin_state.coin_states async def fetch_children( - self, peer: WSChiaConnection, coin_name: bytes32, fork_height: Optional[uint32] = None + self, coin_name: bytes32, fork_height: Optional[uint32] = None, peer: Optional[WSChiaConnection] = None ) -> List[CoinState]: + if peer is None: + peer = self.get_full_node_peer() + if peer is None: + raise ValueError("Could not find any peers to request puzzle and solution from") response: Optional[wallet_protocol.RespondChildren] = await peer.request_children( wallet_protocol.RequestChildren(coin_name) ) diff --git a/chia/wallet/wallet_state_manager.py b/chia/wallet/wallet_state_manager.py index ac71f12fb9..20d512a914 100644 --- a/chia/wallet/wallet_state_manager.py +++ b/chia/wallet/wallet_state_manager.py @@ -593,7 +593,7 @@ class WalletStateManager: assert parent_coin_state.spent_height == coin_state.created_height coin_spend: Optional[CoinSpend] = await self.wallet_node.fetch_puzzle_solution( - peer, parent_coin_state.spent_height, parent_coin_state.coin + parent_coin_state.spent_height, parent_coin_state.coin, peer ) if coin_spend is None: return None, None @@ -971,7 +971,7 @@ class WalletStateManager: ) await self.tx_store.add_transaction_record(tx_record, True) - children = await self.wallet_node.fetch_children(peer, coin_name, fork_height) + children = await self.wallet_node.fetch_children(coin_name, fork_height, peer) assert children is not None additions = [state.coin for state in children] if len(children) > 0: @@ -1048,7 +1048,7 @@ class WalletStateManager: while curr_coin_state.spent_height is not None: cs: CoinSpend = await self.wallet_node.fetch_puzzle_solution( - peer, curr_coin_state.spent_height, curr_coin_state.coin + curr_coin_state.spent_height, curr_coin_state.coin, peer ) success = await wallet.apply_state_transition(cs, curr_coin_state.spent_height) if not success: @@ -1079,7 +1079,7 @@ class WalletStateManager: # Check if a child is a singleton launcher if children is None: - children = await self.wallet_node.fetch_children(peer, coin_name, fork_height) + children = await self.wallet_node.fetch_children(coin_name, fork_height, peer) assert children is not None for child in children: if child.coin.puzzle_hash != SINGLETON_LAUNCHER_HASH: @@ -1090,7 +1090,7 @@ class WalletStateManager: # TODO handle spending launcher later block continue launcher_spend: Optional[CoinSpend] = await self.wallet_node.fetch_puzzle_solution( - peer, coin_state.spent_height, child.coin + coin_state.spent_height, child.coin, peer ) if launcher_spend is None: continue From e57a8e41f4c7767534565326e567a8150533983a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Jul 2022 01:13:01 -0500 Subject: [PATCH 27/62] Bump colorama from 0.4.4 to 0.4.5 (#12189) Bumps [colorama](https://github.com/tartley/colorama) from 0.4.4 to 0.4.5. - [Release notes](https://github.com/tartley/colorama/releases) - [Changelog](https://github.com/tartley/colorama/blob/master/CHANGELOG.rst) - [Commits](https://github.com/tartley/colorama/compare/0.4.4...0.4.5) --- updated-dependencies: - dependency-name: colorama dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f6b614abfa..9551235df1 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ dependencies = [ "aiohttp==3.8.1", # HTTP server for full node rpc "aiosqlite==0.17.0", # asyncio wrapper for sqlite, to store blocks "bitstring==3.1.9", # Binary data management library - "colorama==0.4.4", # Colorizes terminal output + "colorama==0.4.5", # Colorizes terminal output "colorlog==6.6.0", # Adds color to logs "concurrent-log-handler==0.9.19", # Concurrently log and rotate logs "cryptography==36.0.2", # Python cryptography library for TLS - keyring conflict From 95803f09feafd8e71ff5c7dc3f9563593dde519b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Jul 2022 01:13:30 -0500 Subject: [PATCH 28/62] Bump packaging from 21.0 to 21.3 (#12190) Bumps [packaging](https://github.com/pypa/packaging) from 21.0 to 21.3. - [Release notes](https://github.com/pypa/packaging/releases) - [Changelog](https://github.com/pypa/packaging/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pypa/packaging/compare/21.0...21.3) --- updated-dependencies: - dependency-name: packaging dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9551235df1..af61066b2c 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ dependencies = [ "dnslib==0.9.17", # dns lib "typing-extensions==4.0.1", # typing backports like Protocol and TypedDict "zstd==1.5.0.4", - "packaging==21.0", + "packaging==21.3", ] upnp_dependencies = [ From b54e3c315c60d3650f317ba0c3f1407063c6e8b1 Mon Sep 17 00:00:00 2001 From: Almog De Paz Date: Wed, 6 Jul 2022 00:27:01 +0300 Subject: [PATCH 29/62] dont request ses (#11733) * dont request ses, us wp instead * lint * fix inserted index * merge fixes * unused imports --- chia/wallet/wallet_node.py | 233 +++++++++++++++++++------------------ 1 file changed, 117 insertions(+), 116 deletions(-) diff --git a/chia/wallet/wallet_node.py b/chia/wallet/wallet_node.py index fbf2793086..fafd77a68a 100644 --- a/chia/wallet/wallet_node.py +++ b/chia/wallet/wallet_node.py @@ -26,9 +26,7 @@ from chia.protocols.full_node_protocol import RequestProofOfWeight, RespondProof from chia.protocols.protocol_message_types import ProtocolMessageTypes from chia.protocols.wallet_protocol import ( CoinState, - RequestSESInfo, RespondBlockHeader, - RespondSESInfo, RespondToCoinUpdates, RespondToPhUpdates, ) @@ -44,7 +42,7 @@ from chia.types.coin_spend import CoinSpend from chia.types.header_block import HeaderBlock from chia.types.mempool_inclusion_status import MempoolInclusionStatus from chia.types.peer_info import PeerInfo -from chia.types.weight_proof import SubEpochData, WeightProof +from chia.types.weight_proof import WeightProof from chia.util.byte_types import hexstr_to_bytes from chia.util.chunks import chunks from chia.util.config import WALLET_PEERS_PATH_KEY_DEPRECATED @@ -1355,126 +1353,129 @@ class WalletNode: self.log.error("Failed validation 1") return False return True + + # block is not included in wp recent chain + start = block.height + 1 + compare_to_recent = False + inserted: int = 0 + first_height_recent = weight_proof.recent_chain_data[0].height + if start > first_height_recent - 1000: + # compare up to weight_proof.recent_chain_data[0].height + compare_to_recent = True + end = first_height_recent else: - start = block.height + 1 - compare_to_recent = False - current_ses: Optional[SubEpochData] = None - inserted: Optional[SubEpochData] = None - first_height_recent = weight_proof.recent_chain_data[0].height - if start > first_height_recent - 1000: - compare_to_recent = True - end = first_height_recent - else: - if block.height < self.constants.SUB_EPOCH_BLOCKS: - inserted = weight_proof.sub_epochs[1] - end = self.constants.SUB_EPOCH_BLOCKS + inserted.num_blocks_overflow - else: - request = RequestSESInfo(block.height, block.height + 32) - res_ses: Optional[RespondSESInfo] = peer_request_cache.get_ses_request(block.height) - if res_ses is None: - res_ses = await peer.request_ses_hashes(request) - peer_request_cache.add_to_ses_requests(block.height, res_ses) - assert res_ses is not None - - ses_0 = res_ses.reward_chain_hash[0] - last_height = res_ses.heights[0][-1] # Last height in sub epoch - end = last_height - num_sub_epochs = len(weight_proof.sub_epochs) - for idx, ses in enumerate(weight_proof.sub_epochs): - if idx > num_sub_epochs - 3: - break - if ses.reward_chain_hash == ses_0: - current_ses = ses - inserted = weight_proof.sub_epochs[idx + 2] - break - if current_ses is None: - self.log.error("Failed validation 2") - return False - - all_peers = self.server.get_full_node_connections() - blocks: Optional[List[HeaderBlock]] = await fetch_header_blocks_in_range( - start, end, peer_request_cache, all_peers - ) - - if blocks is None: - self.log.error(f"Error fetching blocks {start} {end}") - return False - - if compare_to_recent and weight_proof.recent_chain_data[0].header_hash != blocks[-1].header_hash: - self.log.error("Failed validation 3") - return False - - if not compare_to_recent: - last = blocks[-1].finished_sub_slots[-1].reward_chain.get_hash() - if inserted is None or last != inserted.reward_chain_hash: - self.log.error("Failed validation 4") - return False - pk_m_sig: List[Tuple[G1Element, bytes32, G2Element]] = [] - sigs_to_cache: List[HeaderBlock] = [] - blocks_to_cache: List[Tuple[bytes32, uint32]] = [] - - signatures_to_validate: int = 30 - for idx in range(len(blocks)): - en_block = blocks[idx] - if idx < signatures_to_validate and not peer_request_cache.in_block_signatures_validated(en_block): - # Validate that the block is buried in the foliage by checking the signatures - pk_m_sig.append( - ( - en_block.reward_chain_block.proof_of_space.plot_public_key, - en_block.foliage.foliage_block_data.get_hash(), - en_block.foliage.foliage_block_data_signature, - ) - ) - sigs_to_cache.append(en_block) - - # This is the reward chain challenge. If this is in the cache, it means the prev block - # has been validated. We must at least check the first block to ensure they are connected - reward_chain_hash: bytes32 = en_block.reward_chain_block.reward_chain_ip_vdf.challenge - if idx != 0 and peer_request_cache.in_blocks_validated(reward_chain_hash): - # As soon as we see a block we have already concluded is in the chain, we can quit. - if idx > signatures_to_validate: + # get ses from wp + start_height = block.height + end_height = block.height + 32 + ses_start_height = 0 + end = 0 + for idx, ses in enumerate(weight_proof.sub_epochs): + if idx == len(weight_proof.sub_epochs) - 1: + break + next_ses_height = (idx + 1) * self.constants.SUB_EPOCH_BLOCKS + weight_proof.sub_epochs[ + idx + 1 + ].num_blocks_overflow + # start_ses_hash + if ses_start_height <= start_height < next_ses_height: + inserted = idx + 1 + if ses_start_height < end_height < next_ses_height: + end = next_ses_height break - else: - # Validate that the block is committed to by the weight proof - if idx == 0: - prev_block_rc_hash: bytes32 = block.reward_chain_block.get_hash() - prev_hash = block.header_hash else: - prev_block_rc_hash = blocks[idx - 1].reward_chain_block.get_hash() - prev_hash = blocks[idx - 1].header_hash + if idx > len(weight_proof.sub_epochs) - 3: + break + # else add extra ses as request start <-> end spans two ses + end = (idx + 2) * self.constants.SUB_EPOCH_BLOCKS + weight_proof.sub_epochs[ + idx + 2 + ].num_blocks_overflow + inserted += 1 + break + ses_start_height = next_ses_height - if not en_block.prev_header_hash == prev_hash: - self.log.error("Failed validation 5") - return False + if end == 0: + self.log.error("Error finding sub epoch") + return False + all_peers = self.server.get_full_node_connections() + blocks: Optional[List[HeaderBlock]] = await fetch_header_blocks_in_range( + start, end, peer_request_cache, all_peers + ) + if blocks is None: + self.log.error(f"Error fetching blocks {start} {end}") + return False - if len(en_block.finished_sub_slots) > 0: - reversed_slots = en_block.finished_sub_slots.copy() - reversed_slots.reverse() - for slot_idx, slot in enumerate(reversed_slots[:-1]): - hash_val = reversed_slots[slot_idx + 1].reward_chain.get_hash() - if not hash_val == slot.reward_chain.end_of_slot_vdf.challenge: - self.log.error("Failed validation 6") - return False - if not prev_block_rc_hash == reversed_slots[-1].reward_chain.end_of_slot_vdf.challenge: - self.log.error("Failed validation 7") - return False - else: - if not prev_block_rc_hash == reward_chain_hash: - self.log.error("Failed validation 8") - return False - blocks_to_cache.append((reward_chain_hash, en_block.height)) + if compare_to_recent and weight_proof.recent_chain_data[0].header_hash != blocks[-1].header_hash: + self.log.error("Failed validation 3") + return False - agg_sig: G2Element = AugSchemeMPL.aggregate([sig for (_, _, sig) in pk_m_sig]) - if not AugSchemeMPL.aggregate_verify( - [pk for (pk, _, _) in pk_m_sig], [m for (_, m, _) in pk_m_sig], agg_sig - ): - self.log.error("Failed signature validation") + if not compare_to_recent: + last = blocks[-1].finished_sub_slots[-1].reward_chain.get_hash() + if last != weight_proof.sub_epochs[inserted].reward_chain_hash: + self.log.error("Failed validation 4") return False - for header_block in sigs_to_cache: - peer_request_cache.add_to_block_signatures_validated(header_block) - for reward_chain_hash, height in blocks_to_cache: - peer_request_cache.add_to_blocks_validated(reward_chain_hash, height) - return True + pk_m_sig: List[Tuple[G1Element, bytes32, G2Element]] = [] + sigs_to_cache: List[HeaderBlock] = [] + blocks_to_cache: List[Tuple[bytes32, uint32]] = [] + + signatures_to_validate: int = 30 + for idx in range(len(blocks)): + en_block = blocks[idx] + if idx < signatures_to_validate and not peer_request_cache.in_block_signatures_validated(en_block): + # Validate that the block is buried in the foliage by checking the signatures + pk_m_sig.append( + ( + en_block.reward_chain_block.proof_of_space.plot_public_key, + en_block.foliage.foliage_block_data.get_hash(), + en_block.foliage.foliage_block_data_signature, + ) + ) + sigs_to_cache.append(en_block) + + # This is the reward chain challenge. If this is in the cache, it means the prev block + # has been validated. We must at least check the first block to ensure they are connected + reward_chain_hash: bytes32 = en_block.reward_chain_block.reward_chain_ip_vdf.challenge + if idx != 0 and peer_request_cache.in_blocks_validated(reward_chain_hash): + # As soon as we see a block we have already concluded is in the chain, we can quit. + if idx > signatures_to_validate: + break + else: + # Validate that the block is committed to by the weight proof + if idx == 0: + prev_block_rc_hash: bytes32 = block.reward_chain_block.get_hash() + prev_hash = block.header_hash + else: + prev_block_rc_hash = blocks[idx - 1].reward_chain_block.get_hash() + prev_hash = blocks[idx - 1].header_hash + + if not en_block.prev_header_hash == prev_hash: + self.log.error("Failed validation 5") + return False + + if len(en_block.finished_sub_slots) > 0: + reversed_slots = en_block.finished_sub_slots.copy() + reversed_slots.reverse() + for slot_idx, slot in enumerate(reversed_slots[:-1]): + hash_val = reversed_slots[slot_idx + 1].reward_chain.get_hash() + if not hash_val == slot.reward_chain.end_of_slot_vdf.challenge: + self.log.error("Failed validation 6") + return False + if not prev_block_rc_hash == reversed_slots[-1].reward_chain.end_of_slot_vdf.challenge: + self.log.error("Failed validation 7") + return False + else: + if not prev_block_rc_hash == reward_chain_hash: + self.log.error("Failed validation 8") + return False + blocks_to_cache.append((reward_chain_hash, en_block.height)) + + agg_sig: G2Element = AugSchemeMPL.aggregate([sig for (_, _, sig) in pk_m_sig]) + if not AugSchemeMPL.aggregate_verify([pk for (pk, _, _) in pk_m_sig], [m for (_, m, _) in pk_m_sig], agg_sig): + self.log.error("Failed signature validation") + return False + for header_block in sigs_to_cache: + peer_request_cache.add_to_block_signatures_validated(header_block) + for reward_chain_hash, height in blocks_to_cache: + peer_request_cache.add_to_blocks_validated(reward_chain_hash, height) + return True async def fetch_puzzle_solution( self, height: uint32, coin: Coin, peer: Optional[WSChiaConnection] = None From 61de7bdb393fd8c11737f2d34d15560bfb9d632b Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 5 Jul 2022 17:27:47 -0400 Subject: [PATCH 30/62] EndpointResult (#12026) * just create and apply EndpointResult * int -> uint32 * hexstr_to_bytes -> bytes32.from_hexstr * deal with returning tuples with success and error data * uint64 * assert height is not None * more uint64() for nft coin amount (cherry picked from commit 544d3d58922218e119d9307b0c2d62b3236f9b19) * TransactionRecord -> List[TransactionRecord] * return generic error from create_new_wallet() * Dict[Union[int, bytes32], int] * just ignore the optional coin issues --- chia/rpc/crawler_rpc_api.py | 6 +- chia/rpc/farmer_rpc_api.py | 28 +-- chia/rpc/full_node_rpc_api.py | 50 +++--- chia/rpc/harvester_rpc_api.py | 14 +- chia/rpc/rpc_server.py | 15 +- chia/rpc/wallet_rpc_api.py | 206 +++++++++++----------- chia/simulator/SimulatorFullNodeRpcApi.py | 4 +- chia/wallet/trade_manager.py | 28 ++- 8 files changed, 182 insertions(+), 169 deletions(-) diff --git a/chia/rpc/crawler_rpc_api.py b/chia/rpc/crawler_rpc_api.py index 9d9873aaf8..c1c9a8c772 100644 --- a/chia/rpc/crawler_rpc_api.py +++ b/chia/rpc/crawler_rpc_api.py @@ -1,7 +1,7 @@ import ipaddress from typing import Any, Dict, List, Optional -from chia.rpc.rpc_server import Endpoint +from chia.rpc.rpc_server import Endpoint, EndpointResult from chia.seeder.crawler import Crawler from chia.util.ws_message import WsRpcMessage, create_payload_dict @@ -28,7 +28,7 @@ class CrawlerRpcApi: return payloads - async def get_peer_counts(self, _request: Dict) -> Dict[str, Any]: + async def get_peer_counts(self, _request: Dict) -> EndpointResult: ipv6_addresses_count = 0 for host in self.service.best_timestamp_per_peer.keys(): try: @@ -52,7 +52,7 @@ class CrawlerRpcApi: } return data - async def get_ips_after_timestamp(self, _request: Dict) -> Dict[str, Any]: + async def get_ips_after_timestamp(self, _request: Dict) -> EndpointResult: after = _request.get("after", None) if after is None: raise ValueError("`after` is required and must be a unix timestamp") diff --git a/chia/rpc/farmer_rpc_api.py b/chia/rpc/farmer_rpc_api.py index 1c0915fb88..312327998f 100644 --- a/chia/rpc/farmer_rpc_api.py +++ b/chia/rpc/farmer_rpc_api.py @@ -7,7 +7,7 @@ from typing_extensions import Protocol from chia.farmer.farmer import Farmer from chia.plot_sync.receiver import Receiver from chia.protocols.harvester_protocol import Plot -from chia.rpc.rpc_server import Endpoint +from chia.rpc.rpc_server import Endpoint, EndpointResult from chia.types.blockchain_format.sized_bytes import bytes32 from chia.util.byte_types import hexstr_to_bytes from chia.util.ints import uint32 @@ -161,7 +161,7 @@ class FarmerRpcApi: return payloads - async def get_signage_point(self, request: Dict) -> Dict: + async def get_signage_point(self, request: Dict) -> EndpointResult: sp_hash = hexstr_to_bytes(request["sp_hash"]) for _, sps in self.service.sps.items(): for sp in sps: @@ -180,7 +180,7 @@ class FarmerRpcApi: } raise ValueError(f"Signage point {sp_hash.hex()} not found") - async def get_signage_points(self, _: Dict) -> Dict[str, Any]: + async def get_signage_points(self, _: Dict) -> EndpointResult: result: List[Dict[str, Any]] = [] for sps in self.service.sps.values(): for sp in sps: @@ -200,12 +200,12 @@ class FarmerRpcApi: ) return {"signage_points": result} - async def get_reward_targets(self, request: Dict) -> Dict: + async def get_reward_targets(self, request: Dict) -> EndpointResult: search_for_private_key = request["search_for_private_key"] max_ph_to_search = request.get("max_ph_to_search", 500) return await self.service.get_reward_targets(search_for_private_key, max_ph_to_search) - async def set_reward_targets(self, request: Dict) -> Dict: + async def set_reward_targets(self, request: Dict) -> EndpointResult: farmer_target, pool_target = None, None if "farmer_target" in request: farmer_target = request["farmer_target"] @@ -223,7 +223,7 @@ class FarmerRpcApi: ) return plot_count - async def get_pool_state(self, _: Dict) -> Dict: + async def get_pool_state(self, _: Dict) -> EndpointResult: pools_list = [] for p2_singleton_puzzle_hash, pool_dict in self.service.pool_state.items(): pool_state = pool_dict.copy() @@ -232,18 +232,18 @@ class FarmerRpcApi: pools_list.append(pool_state) return {"pool_state": pools_list} - async def set_payout_instructions(self, request: Dict) -> Dict: + async def set_payout_instructions(self, request: Dict) -> EndpointResult: launcher_id: bytes32 = bytes32.from_hexstr(request["launcher_id"]) await self.service.set_payout_instructions(launcher_id, request["payout_instructions"]) return {} - async def get_harvesters(self, _: Dict): + async def get_harvesters(self, _: Dict) -> EndpointResult: return await self.service.get_harvesters(False) - async def get_harvesters_summary(self, _: Dict[str, object]) -> Dict[str, object]: + async def get_harvesters_summary(self, _: Dict[str, object]) -> EndpointResult: return await self.service.get_harvesters(True) - async def get_harvester_plots_valid(self, request_dict: Dict[str, object]) -> Dict[str, object]: + async def get_harvester_plots_valid(self, request_dict: Dict[str, object]) -> EndpointResult: # TODO: Consider having a extra List[PlotInfo] in Receiver to avoid rebuilding the list for each call request = PlotInfoRequestData.from_json_dict(request_dict) plot_list = list(self.service.get_receiver(request.node_id).plots().values()) @@ -271,16 +271,16 @@ class FarmerRpcApi: source = sorted(source, reverse=request.reverse) return paginated_plot_request(source, request) - async def get_harvester_plots_invalid(self, request_dict: Dict[str, object]) -> Dict[str, object]: + async def get_harvester_plots_invalid(self, request_dict: Dict[str, object]) -> EndpointResult: return self.paginated_plot_path_request(Receiver.invalid, request_dict) - async def get_harvester_plots_keys_missing(self, request_dict: Dict[str, object]) -> Dict[str, object]: + async def get_harvester_plots_keys_missing(self, request_dict: Dict[str, object]) -> EndpointResult: return self.paginated_plot_path_request(Receiver.keys_missing, request_dict) - async def get_harvester_plots_duplicates(self, request_dict: Dict[str, object]) -> Dict[str, object]: + async def get_harvester_plots_duplicates(self, request_dict: Dict[str, object]) -> EndpointResult: return self.paginated_plot_path_request(Receiver.duplicates, request_dict) - async def get_pool_login_link(self, request: Dict) -> Dict: + async def get_pool_login_link(self, request: Dict) -> EndpointResult: launcher_id: bytes32 = bytes32(hexstr_to_bytes(request["launcher_id"])) login_link: Optional[str] = await self.service.generate_login_link(launcher_id) if login_link is None: diff --git a/chia/rpc/full_node_rpc_api.py b/chia/rpc/full_node_rpc_api.py index f9b7e4114a..c7dd6152c6 100644 --- a/chia/rpc/full_node_rpc_api.py +++ b/chia/rpc/full_node_rpc_api.py @@ -4,7 +4,7 @@ from chia.consensus.block_record import BlockRecord from chia.consensus.pos_quality import UI_ACTUAL_SPACE_CONSTANT_FACTOR from chia.full_node.full_node import FullNode from chia.full_node.mempool_check_conditions import get_puzzle_and_solution_for_coin -from chia.rpc.rpc_server import Endpoint +from chia.rpc.rpc_server import Endpoint, EndpointResult from chia.types.blockchain_format.program import Program, SerializedProgram from chia.types.blockchain_format.sized_bytes import bytes32 from chia.types.coin_record import CoinRecord @@ -96,11 +96,11 @@ class FullNodeRpcApi: # this function is just here for backwards-compatibility. It will probably # be removed in the future - async def get_initial_freeze_period(self, _: Dict): + async def get_initial_freeze_period(self, _: Dict) -> EndpointResult: # Mon May 03 2021 17:00:00 GMT+0000 return {"INITIAL_FREEZE_END_TIMESTAMP": 1620061200} - async def get_blockchain_state(self, _request: Dict): + async def get_blockchain_state(self, _request: Dict) -> EndpointResult: """ Returns a summary of the node's view of the blockchain. """ @@ -214,12 +214,12 @@ class FullNodeRpcApi: self.cached_blockchain_state = dict(response["blockchain_state"]) return response - async def get_network_info(self, request: Dict): + async def get_network_info(self, request: Dict) -> EndpointResult: network_name = self.service.config["selected_network"] address_prefix = self.service.config["network_overrides"]["config"][network_name]["address_prefix"] return {"network_name": network_name, "network_prefix": address_prefix} - async def get_recent_signage_point_or_eos(self, request: Dict): + async def get_recent_signage_point_or_eos(self, request: Dict) -> EndpointResult: if "sp_hash" not in request: challenge_hash: bytes32 = bytes32.from_hexstr(request["challenge_hash"]) # This is the case of getting an end of slot @@ -308,7 +308,7 @@ class FullNodeRpcApi: return {"signage_point": sp, "time_received": time_received, "reverted": True} - async def get_block(self, request: Dict) -> Dict[str, object]: + async def get_block(self, request: Dict) -> EndpointResult: if "header_hash" not in request: raise ValueError("No header_hash in request") header_hash = bytes32.from_hexstr(request["header_hash"]) @@ -319,7 +319,7 @@ class FullNodeRpcApi: return {"block": block} - async def get_blocks(self, request: Dict) -> Dict[str, object]: + async def get_blocks(self, request: Dict) -> EndpointResult: if "start" not in request: raise ValueError("No start in request") if "end" not in request: @@ -349,7 +349,7 @@ class FullNodeRpcApi: json_blocks.append(json) return {"blocks": json_blocks} - async def get_block_count_metrics(self, request: Dict): + async def get_block_count_metrics(self, request: Dict) -> EndpointResult: compact_blocks = 0 uncompact_blocks = 0 with log_exceptions(self.service.log, consume=True): @@ -369,7 +369,7 @@ class FullNodeRpcApi: } } - async def get_block_records(self, request: Dict) -> Dict[str, object]: + async def get_block_records(self, request: Dict) -> EndpointResult: if "start" not in request: raise ValueError("No start in request") if "end" not in request: @@ -399,7 +399,7 @@ class FullNodeRpcApi: records.append(record) return {"block_records": records} - async def get_block_record_by_height(self, request: Dict) -> Dict[str, object]: + async def get_block_record_by_height(self, request: Dict) -> EndpointResult: if "height" not in request: raise ValueError("No height in request") height = request["height"] @@ -418,7 +418,7 @@ class FullNodeRpcApi: raise ValueError(f"Block {header_hash} does not exist") return {"block_record": record} - async def get_block_record(self, request: Dict): + async def get_block_record(self, request: Dict) -> EndpointResult: if "header_hash" not in request: raise ValueError("header_hash not in request") header_hash_str = request["header_hash"] @@ -432,7 +432,7 @@ class FullNodeRpcApi: return {"block_record": record} - async def get_unfinished_block_headers(self, request: Dict) -> Dict[str, object]: + async def get_unfinished_block_headers(self, request: Dict) -> EndpointResult: peak: Optional[BlockRecord] = self.service.blockchain.get_peak() if peak is None: @@ -453,7 +453,7 @@ class FullNodeRpcApi: response_headers.append(unfinished_header_block) return {"headers": response_headers} - async def get_network_space(self, request: Dict) -> Dict[str, object]: + async def get_network_space(self, request: Dict) -> EndpointResult: """ Retrieves an estimate of total space validating the chain between two block header hashes. @@ -493,7 +493,7 @@ class FullNodeRpcApi: ) return {"space": uint128(int(network_space_bytes_estimate))} - async def get_coin_records_by_puzzle_hash(self, request: Dict) -> Dict[str, object]: + async def get_coin_records_by_puzzle_hash(self, request: Dict) -> EndpointResult: """ Retrieves the coins for a given puzzlehash, by default returns unspent coins. """ @@ -512,7 +512,7 @@ class FullNodeRpcApi: return {"coin_records": [coin_record_dict_backwards_compat(cr.to_json_dict()) for cr in coin_records]} - async def get_coin_records_by_puzzle_hashes(self, request: Dict) -> Dict[str, object]: + async def get_coin_records_by_puzzle_hashes(self, request: Dict) -> EndpointResult: """ Retrieves the coins for a given puzzlehash, by default returns unspent coins. """ @@ -534,7 +534,7 @@ class FullNodeRpcApi: return {"coin_records": [coin_record_dict_backwards_compat(cr.to_json_dict()) for cr in coin_records]} - async def get_coin_record_by_name(self, request: Dict) -> Dict[str, object]: + async def get_coin_record_by_name(self, request: Dict) -> EndpointResult: """ Retrieves a coin record by it's name. """ @@ -548,7 +548,7 @@ class FullNodeRpcApi: return {"coin_record": coin_record_dict_backwards_compat(coin_record.to_json_dict())} - async def get_coin_records_by_names(self, request: Dict) -> Dict[str, object]: + async def get_coin_records_by_names(self, request: Dict) -> EndpointResult: """ Retrieves the coins for given coin IDs, by default returns unspent coins. """ @@ -570,7 +570,7 @@ class FullNodeRpcApi: return {"coin_records": [coin_record_dict_backwards_compat(cr.to_json_dict()) for cr in coin_records]} - async def get_coin_records_by_parent_ids(self, request: Dict) -> Dict[str, object]: + async def get_coin_records_by_parent_ids(self, request: Dict) -> EndpointResult: """ Retrieves the coins for given parent coin IDs, by default returns unspent coins. """ @@ -592,7 +592,7 @@ class FullNodeRpcApi: return {"coin_records": [coin_record_dict_backwards_compat(cr.to_json_dict()) for cr in coin_records]} - async def get_coin_records_by_hint(self, request: Dict) -> Dict[str, object]: + async def get_coin_records_by_hint(self, request: Dict) -> EndpointResult: """ Retrieves coins by hint, by default returns unspent coins. """ @@ -621,7 +621,7 @@ class FullNodeRpcApi: return {"coin_records": [coin_record_dict_backwards_compat(cr.to_json_dict()) for cr in coin_records]} - async def push_tx(self, request: Dict) -> Dict[str, object]: + async def push_tx(self, request: Dict) -> EndpointResult: if "spend_bundle" not in request: raise ValueError("Spend bundle not in request") @@ -646,7 +646,7 @@ class FullNodeRpcApi: "status": status.name, } - async def get_puzzle_and_solution(self, request: Dict) -> Dict[str, object]: + async def get_puzzle_and_solution(self, request: Dict) -> EndpointResult: coin_name: bytes32 = bytes32.from_hexstr(request["coin_id"]) height = request["height"] coin_record = await self.service.coin_store.get_coin_record(coin_name) @@ -672,7 +672,7 @@ class FullNodeRpcApi: solution_ser: SerializedProgram = SerializedProgram.from_program(Program.to(solution)) return {"coin_solution": CoinSpend(coin_record.coin, puzzle_ser, solution_ser)} - async def get_additions_and_removals(self, request: Dict) -> Dict[str, object]: + async def get_additions_and_removals(self, request: Dict) -> EndpointResult: if "header_hash" not in request: raise ValueError("No header_hash in request") header_hash = bytes32.from_hexstr(request["header_hash"]) @@ -692,17 +692,17 @@ class FullNodeRpcApi: "removals": [coin_record_dict_backwards_compat(cr.to_json_dict()) for cr in removals], } - async def get_all_mempool_tx_ids(self, request: Dict) -> Dict[str, object]: + async def get_all_mempool_tx_ids(self, request: Dict) -> EndpointResult: ids = list(self.service.mempool_manager.mempool.spends.keys()) return {"tx_ids": ids} - async def get_all_mempool_items(self, request: Dict) -> Dict[str, object]: + async def get_all_mempool_items(self, request: Dict) -> EndpointResult: spends = {} for tx_id, item in self.service.mempool_manager.mempool.spends.items(): spends[tx_id.hex()] = item return {"mempool_items": spends} - async def get_mempool_item_by_tx_id(self, request: Dict) -> Dict[str, object]: + async def get_mempool_item_by_tx_id(self, request: Dict) -> EndpointResult: if "tx_id" not in request: raise ValueError("No tx_id in request") tx_id: bytes32 = bytes32.from_hexstr(request["tx_id"]) diff --git a/chia/rpc/harvester_rpc_api.py b/chia/rpc/harvester_rpc_api.py index 6abcbe16e0..0853fd3263 100644 --- a/chia/rpc/harvester_rpc_api.py +++ b/chia/rpc/harvester_rpc_api.py @@ -1,7 +1,7 @@ from typing import Any, Dict, List from chia.harvester.harvester import Harvester -from chia.rpc.rpc_server import Endpoint +from chia.rpc.rpc_server import Endpoint, EndpointResult from chia.util.ws_message import WsRpcMessage, create_payload_dict @@ -36,7 +36,7 @@ class HarvesterRpcApi: return payloads - async def get_plots(self, request: Dict) -> Dict: + async def get_plots(self, request: Dict) -> EndpointResult: plots, failed_to_open, not_found = self.service.get_plots() return { "plots": plots, @@ -44,27 +44,27 @@ class HarvesterRpcApi: "not_found_filenames": not_found, } - async def refresh_plots(self, request: Dict) -> Dict: + async def refresh_plots(self, request: Dict) -> EndpointResult: self.service.plot_manager.trigger_refresh() return {} - async def delete_plot(self, request: Dict) -> Dict: + async def delete_plot(self, request: Dict) -> EndpointResult: filename = request["filename"] if self.service.delete_plot(filename): return {} raise ValueError(f"Not able to delete file {filename}") - async def add_plot_directory(self, request: Dict) -> Dict: + async def add_plot_directory(self, request: Dict) -> EndpointResult: directory_name = request["dirname"] if await self.service.add_plot_directory(directory_name): return {} raise ValueError(f"Did not add plot directory {directory_name}") - async def get_plot_directories(self, request: Dict) -> Dict: + async def get_plot_directories(self, request: Dict) -> EndpointResult: plot_dirs = await self.service.get_plot_directories() return {"directories": plot_dirs} - async def remove_plot_directory(self, request: Dict) -> Dict: + async def remove_plot_directory(self, request: Dict) -> EndpointResult: directory_name = request["dirname"] if await self.service.remove_plot_directory(directory_name): return {} diff --git a/chia/rpc/rpc_server.py b/chia/rpc/rpc_server.py index bc187afe98..ed46b8f408 100644 --- a/chia/rpc/rpc_server.py +++ b/chia/rpc/rpc_server.py @@ -26,7 +26,8 @@ log = logging.getLogger(__name__) max_message_size = 50 * 1024 * 1024 # 50MB -Endpoint = Callable[[Dict[str, object]], Awaitable[Dict[str, object]]] +EndpointResult = Dict[str, Any] +Endpoint = Callable[[Dict[str, object]], Awaitable[EndpointResult]] @final @@ -107,13 +108,13 @@ class RpcServer: "/healthz": self.healthz, } - async def _get_routes(self, request: Dict[str, Any]) -> Dict[str, object]: + async def _get_routes(self, request: Dict[str, Any]) -> EndpointResult: return { "success": "true", "routes": list(self.get_routes().keys()), } - async def get_connections(self, request: Dict[str, Any]) -> Dict[str, object]: + async def get_connections(self, request: Dict[str, Any]) -> EndpointResult: request_node_type: Optional[NodeType] = None if "node_type" in request: request_node_type = NodeType(request["node_type"]) @@ -169,7 +170,7 @@ class RpcServer: ] return {"connections": con_info} - async def open_connection(self, request: Dict[str, Any]) -> Dict[str, object]: + async def open_connection(self, request: Dict[str, Any]) -> EndpointResult: host = request["host"] port = request["port"] target_node: PeerInfo = PeerInfo(host, uint16(int(port))) @@ -182,7 +183,7 @@ class RpcServer: raise ValueError("Start client failed, or server is not set") return {} - async def close_connection(self, request: Dict[str, Any]) -> Dict[str, object]: + async def close_connection(self, request: Dict[str, Any]) -> EndpointResult: node_id = hexstr_to_bytes(request["node_id"]) if self.rpc_api.service.server is None: raise web.HTTPInternalServerError() @@ -193,7 +194,7 @@ class RpcServer: await connection.close() return {} - async def stop_node(self, request: Dict[str, Any]) -> Dict[str, object]: + async def stop_node(self, request: Dict[str, Any]) -> EndpointResult: """ Shuts down the node. """ @@ -201,7 +202,7 @@ class RpcServer: self.stop_cb() return {} - async def healthz(self, request: Dict[str, Any]) -> Dict[str, object]: + async def healthz(self, request: Dict[str, Any]) -> EndpointResult: return { "success": "true", } diff --git a/chia/rpc/wallet_rpc_api.py b/chia/rpc/wallet_rpc_api.py index 23d963781d..5dcd04c70d 100644 --- a/chia/rpc/wallet_rpc_api.py +++ b/chia/rpc/wallet_rpc_api.py @@ -3,7 +3,7 @@ import dataclasses import json import logging from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple +from typing import Any, Dict, List, Optional, Set, Tuple, Union from blspy import G1Element, PrivateKey @@ -12,7 +12,7 @@ from chia.pools.pool_wallet import PoolWallet from chia.pools.pool_wallet_info import FARMING_TO_POOL, PoolState, PoolWalletInfo, create_pool_state from chia.protocols.protocol_message_types import ProtocolMessageTypes from chia.protocols.wallet_protocol import CoinState -from chia.rpc.rpc_server import Endpoint +from chia.rpc.rpc_server import Endpoint, EndpointResult from chia.server.outbound_message import NodeType, make_msg from chia.simulator.simulator_protocol import FarmNewBlockProtocol from chia.types.announcement import Announcement @@ -195,7 +195,7 @@ class WalletRpcApi: # Key management ########################################################################################## - async def log_in(self, request): + async def log_in(self, request) -> EndpointResult: """ Logs in the wallet with a specific key. """ @@ -212,10 +212,10 @@ class WalletRpcApi: return {"success": False, "error": "Unknown Error"} - async def get_logged_in_fingerprint(self, request: Dict): + async def get_logged_in_fingerprint(self, request: Dict) -> EndpointResult: return {"fingerprint": self.service.logged_in_fingerprint} - async def get_public_keys(self, request: Dict): + async def get_public_keys(self, request: Dict) -> EndpointResult: try: fingerprints = [ sk.get_g1().get_fingerprint() for (sk, seed) in await self.service.keychain_proxy.get_all_private_keys() @@ -237,7 +237,7 @@ class WalletRpcApi: log.error(f"Failed to get private key by fingerprint: {e}") return None, None - async def get_private_key(self, request): + async def get_private_key(self, request) -> EndpointResult: fingerprint = request["fingerprint"] sk, seed = await self._get_private_key(fingerprint) if sk is not None: @@ -254,10 +254,10 @@ class WalletRpcApi: } return {"success": False, "private_key": {"fingerprint": fingerprint}} - async def generate_mnemonic(self, request: Dict): + async def generate_mnemonic(self, request: Dict) -> EndpointResult: return {"mnemonic": generate_mnemonic().split(" ")} - async def add_key(self, request): + async def add_key(self, request) -> EndpointResult: if "mnemonic" not in request: raise ValueError("Mnemonic not in request") @@ -289,7 +289,7 @@ class WalletRpcApi: return {"fingerprint": fingerprint} raise ValueError("Failed to start") - async def delete_key(self, request): + async def delete_key(self, request) -> EndpointResult: await self._stop_wallet() fingerprint = request["fingerprint"] try: @@ -329,7 +329,7 @@ class WalletRpcApi: return found_farmer, found_pool - async def check_delete_key(self, request): + async def check_delete_key(self, request) -> EndpointResult: """Check the key use prior to possible deletion checks whether key is used for either farm or pool rewards checks if any wallets have a non-zero balance @@ -368,7 +368,7 @@ class WalletRpcApi: "wallet_balance": walletBalance, } - async def delete_all_keys(self, request: Dict): + async def delete_all_keys(self, request: Dict) -> EndpointResult: await self._stop_wallet() try: await self.service.keychain_proxy.delete_all_keys() @@ -384,28 +384,28 @@ class WalletRpcApi: # Wallet Node ########################################################################################## - async def get_sync_status(self, request: Dict): + async def get_sync_status(self, request: Dict) -> EndpointResult: syncing = self.service.wallet_state_manager.sync_mode synced = await self.service.wallet_state_manager.synced() return {"synced": synced, "syncing": syncing, "genesis_initialized": True} - async def get_height_info(self, request: Dict): + async def get_height_info(self, request: Dict) -> EndpointResult: height = await self.service.wallet_state_manager.blockchain.get_finished_sync_up_to() return {"height": height} - async def get_network_info(self, request: Dict): + async def get_network_info(self, request: Dict) -> EndpointResult: network_name = self.service.config["selected_network"] address_prefix = self.service.config["network_overrides"]["config"][network_name]["address_prefix"] return {"network_name": network_name, "network_prefix": address_prefix} - async def push_tx(self, request: Dict): + async def push_tx(self, request: Dict) -> EndpointResult: nodes = self.service.server.get_full_node_connections() if len(nodes) == 0: raise ValueError("Wallet is not currently connected to any full node peers") await self.service.push_tx(SpendBundle.from_bytes(hexstr_to_bytes(request["spend_bundle"]))) return {} - async def farm_block(self, request): + async def farm_block(self, request) -> EndpointResult: raw_puzzle_hash = decode_puzzle_hash(request["address"]) request = FarmNewBlockProtocol(raw_puzzle_hash) msg = make_msg(ProtocolMessageTypes.farm_new_block, request) @@ -417,7 +417,7 @@ class WalletRpcApi: # Wallet Management ########################################################################################## - async def get_wallets(self, request: Dict): + async def get_wallets(self, request: Dict) -> EndpointResult: include_data: bool = request.get("include_data", True) wallet_type: Optional[WalletType] = None if "type" in request: @@ -431,7 +431,7 @@ class WalletRpcApi: wallets = result return {"wallets": wallets} - async def create_new_wallet(self, request: Dict): + async def create_new_wallet(self, request: Dict) -> EndpointResult: wallet_state_manager = self.service.wallet_state_manager if await self.service.wallet_state_manager.synced() is False: @@ -641,13 +641,14 @@ class WalletRpcApi: else: # undefined wallet_type pass - return None + # TODO: rework this function to report detailed errors for each error case + return {"success": False, "error": "invalid request"} ########################################################################################## # Wallet ########################################################################################## - async def get_wallet_balance(self, request: Dict) -> Dict: + async def get_wallet_balance(self, request: Dict) -> EndpointResult: wallet_id = uint32(int(request["wallet_id"])) wallet = self.service.wallet_state_manager.wallets[wallet_id] @@ -699,7 +700,7 @@ class WalletRpcApi: return {"wallet_balance": wallet_balance} - async def get_transaction(self, request: Dict) -> Dict: + async def get_transaction(self, request: Dict) -> EndpointResult: transaction_id: bytes32 = bytes32(hexstr_to_bytes(request["transaction_id"])) tr: Optional[TransactionRecord] = await self.service.wallet_state_manager.get_transaction(transaction_id) if tr is None: @@ -710,7 +711,7 @@ class WalletRpcApi: "transaction_id": tr.name, } - async def get_transactions(self, request: Dict) -> Dict: + async def get_transactions(self, request: Dict) -> EndpointResult: wallet_id = int(request["wallet_id"]) start = request.get("start", 0) @@ -734,7 +735,7 @@ class WalletRpcApi: "wallet_id": wallet_id, } - async def get_transaction_count(self, request: Dict) -> Dict: + async def get_transaction_count(self, request: Dict) -> EndpointResult: wallet_id = int(request["wallet_id"]) count = await self.service.wallet_state_manager.tx_store.get_transaction_count_for_wallet(wallet_id) return { @@ -744,11 +745,11 @@ class WalletRpcApi: # this function is just here for backwards-compatibility. It will probably # be removed in the future - async def get_initial_freeze_period(self, _: Dict): + async def get_initial_freeze_period(self, _: Dict) -> EndpointResult: # Mon May 03 2021 17:00:00 GMT+0000 return {"INITIAL_FREEZE_END_TIMESTAMP": 1620061200} - async def get_next_address(self, request: Dict) -> Dict: + async def get_next_address(self, request: Dict) -> EndpointResult: """ Returns a new address """ @@ -774,11 +775,11 @@ class WalletRpcApi: "address": address, } - async def send_transaction(self, request): + async def send_transaction(self, request) -> EndpointResult: if await self.service.wallet_state_manager.synced() is False: raise ValueError("Wallet needs to be fully synced before sending transactions") - wallet_id = int(request["wallet_id"]) + wallet_id = uint32(request["wallet_id"]) wallet = self.service.wallet_state_manager.wallets[wallet_id] if wallet.type() == WalletType.CAT: @@ -812,7 +813,7 @@ class WalletRpcApi: "transaction_id": tx.name, } - async def send_transaction_multi(self, request) -> Dict: + async def send_transaction_multi(self, request) -> EndpointResult: if await self.service.wallet_state_manager.synced() is False: raise ValueError("Wallet needs to be fully synced before sending transactions") @@ -827,7 +828,7 @@ class WalletRpcApi: # Transaction may not have been included in the mempool yet. Use get_transaction to check. return {"transaction": transaction, "transaction_id": tr.name} - async def delete_unconfirmed_transactions(self, request): + async def delete_unconfirmed_transactions(self, request) -> EndpointResult: wallet_id = uint32(request["wallet_id"]) if wallet_id not in self.service.wallet_state_manager.wallets: raise ValueError(f"Wallet id {wallet_id} does not exist") @@ -843,7 +844,7 @@ class WalletRpcApi: await self.service.wallet_state_manager.tx_store.db_wrapper.commit_transaction() return {} - async def select_coins(self, request) -> Dict[str, object]: + async def select_coins(self, request) -> EndpointResult: if await self.service.wallet_state_manager.synced() is False: raise ValueError("Wallet needs to be fully synced before selecting coins") @@ -860,22 +861,22 @@ class WalletRpcApi: # CATs and Trading ########################################################################################## - async def get_cat_list(self, request): + async def get_cat_list(self, request) -> EndpointResult: return {"cat_list": list(DEFAULT_CATS.values())} - async def cat_set_name(self, request): - wallet_id = int(request["wallet_id"]) + async def cat_set_name(self, request) -> EndpointResult: + wallet_id = uint32(request["wallet_id"]) wallet: CATWallet = self.service.wallet_state_manager.wallets[wallet_id] await wallet.set_name(str(request["name"])) return {"wallet_id": wallet_id} - async def cat_get_name(self, request): - wallet_id = int(request["wallet_id"]) + async def cat_get_name(self, request) -> EndpointResult: + wallet_id = uint32(request["wallet_id"]) wallet: CATWallet = self.service.wallet_state_manager.wallets[wallet_id] name: str = await wallet.get_name() return {"wallet_id": wallet_id, "name": name} - async def get_stray_cats(self, request): + async def get_stray_cats(self, request) -> EndpointResult: """ Get a list of all unacknowledged CATs :param request: RPC request @@ -884,10 +885,10 @@ class WalletRpcApi: cats = await self.service.wallet_state_manager.interested_store.get_unacknowledged_tokens() return {"stray_cats": cats} - async def cat_spend(self, request): + async def cat_spend(self, request) -> EndpointResult: if await self.service.wallet_state_manager.synced() is False: raise ValueError("Wallet needs to be fully synced.") - wallet_id = int(request["wallet_id"]) + wallet_id = uint32(request["wallet_id"]) wallet: CATWallet = self.service.wallet_state_manager.wallets[wallet_id] puzzle_hash: bytes32 = decode_puzzle_hash(request["inner_address"]) @@ -903,7 +904,7 @@ class WalletRpcApi: else: fee = uint64(0) async with self.service.wallet_state_manager.lock: - txs: TransactionRecord = await wallet.generate_signed_transaction( + txs: List[TransactionRecord] = await wallet.generate_signed_transaction( [amount], [puzzle_hash], fee, memos=[memos] ) for tx in txs: @@ -914,13 +915,13 @@ class WalletRpcApi: "transaction_id": tx.name, } - async def cat_get_asset_id(self, request): - wallet_id = int(request["wallet_id"]) + async def cat_get_asset_id(self, request) -> EndpointResult: + wallet_id = uint32(request["wallet_id"]) wallet: CATWallet = self.service.wallet_state_manager.wallets[wallet_id] asset_id: str = wallet.get_asset_id() return {"asset_id": asset_id, "wallet_id": wallet_id} - async def cat_asset_id_to_name(self, request): + async def cat_asset_id_to_name(self, request) -> EndpointResult: wallet = await self.service.wallet_state_manager.get_wallet_for_asset_id(request["asset_id"]) if wallet is None: if request["asset_id"] in DEFAULT_CATS: @@ -930,7 +931,7 @@ class WalletRpcApi: else: return {"wallet_id": wallet.id(), "name": (await wallet.get_name())} - async def create_offer_for_ids(self, request): + async def create_offer_for_ids(self, request) -> EndpointResult: offer: Dict[str, int] = request["offer"] fee: uint64 = uint64(request.get("fee", 0)) validate_only: bool = request.get("validate_only", False) @@ -951,7 +952,7 @@ class WalletRpcApi: for key, value in driver_dict_str.items(): driver_dict[bytes32.from_hexstr(key)] = PuzzleInfo(value) - modified_offer = {} + modified_offer: Dict[Union[int, bytes32], int] = {} for key in offer: try: modified_offer[bytes32.from_hexstr(key)] = offer[key] @@ -959,49 +960,43 @@ class WalletRpcApi: modified_offer[int(key)] = offer[key] async with self.service.wallet_state_manager.lock: - ( - success, - trade_record, - error, - ) = await self.service.wallet_state_manager.trade_manager.create_offer_for_ids( + result = await self.service.wallet_state_manager.trade_manager.create_offer_for_ids( modified_offer, driver_dict, fee=fee, validate_only=validate_only ) - if success: + if result[0]: + success, trade_record, error = result return { "offer": Offer.from_bytes(trade_record.offer).to_bech32(), "trade_record": trade_record.to_json_dict_convenience(), } - raise ValueError(error) + raise ValueError(result[2]) - async def get_offer_summary(self, request): + async def get_offer_summary(self, request) -> EndpointResult: offer_hex: str = request["offer"] offer = Offer.from_bech32(offer_hex) offered, requested, infos = offer.summary() return {"summary": {"offered": offered, "requested": requested, "fees": offer.bundle.fees(), "infos": infos}} - async def check_offer_validity(self, request): + async def check_offer_validity(self, request) -> EndpointResult: offer_hex: str = request["offer"] offer = Offer.from_bech32(offer_hex) return {"valid": (await self.service.wallet_state_manager.trade_manager.check_offer_validity(offer))} - async def take_offer(self, request): + async def take_offer(self, request) -> EndpointResult: offer_hex: str = request["offer"] offer = Offer.from_bech32(offer_hex) fee: uint64 = uint64(request.get("fee", 0)) async with self.service.wallet_state_manager.lock: - ( - success, - trade_record, - error, - ) = await self.service.wallet_state_manager.trade_manager.respond_to_offer(offer, fee=fee) - if not success: - raise ValueError(error) + result = await self.service.wallet_state_manager.trade_manager.respond_to_offer(offer, fee=fee) + if not result[0]: + raise ValueError(result[2]) + success, trade_record, error = result return {"trade_record": trade_record.to_json_dict_convenience()} - async def get_offer(self, request: Dict): + async def get_offer(self, request: Dict) -> EndpointResult: trade_mgr = self.service.wallet_state_manager.trade_manager trade_id = bytes32.from_hexstr(request["trade_id"]) @@ -1014,7 +1009,7 @@ class WalletRpcApi: offer_value: Optional[str] = Offer.from_bytes(offer_to_return).to_bech32() if file_contents else None return {"trade_record": trade_record.to_json_dict_convenience(), "offer": offer_value} - async def get_all_offers(self, request: Dict): + async def get_all_offers(self, request: Dict) -> EndpointResult: trade_mgr = self.service.wallet_state_manager.trade_manager start: int = request.get("start", 0) @@ -1045,14 +1040,14 @@ class WalletRpcApi: return {"trade_records": result, "offers": offer_values} - async def get_offers_count(self, request: Dict): + async def get_offers_count(self, request: Dict) -> EndpointResult: trade_mgr = self.service.wallet_state_manager.trade_manager (total, my_offers_count, taken_offers_count) = await trade_mgr.trade_store.get_trades_count() return {"total": total, "my_offers_count": my_offers_count, "taken_offers_count": taken_offers_count} - async def cancel_offer(self, request: Dict): + async def cancel_offer(self, request: Dict) -> EndpointResult: wsm = self.service.wallet_state_manager secure = request["secure"] trade_id = bytes32.from_hexstr(request["trade_id"]) @@ -1069,7 +1064,7 @@ class WalletRpcApi: # Distributed Identities ########################################################################################## - async def did_set_wallet_name(self, request): + async def did_set_wallet_name(self, request) -> EndpointResult: wallet_id = uint32(request["wallet_id"]) wallet: DIDWallet = self.service.wallet_state_manager.wallets[wallet_id] if wallet.type() == WalletType.DECENTRALIZED_ID: @@ -1078,13 +1073,13 @@ class WalletRpcApi: else: return {"success": False, "error": f"Wallet id {wallet_id} is not a DID wallet"} - async def did_get_wallet_name(self, request): + async def did_get_wallet_name(self, request) -> EndpointResult: wallet_id = uint32(request["wallet_id"]) wallet: DIDWallet = self.service.wallet_state_manager.wallets[wallet_id] name: str = await wallet.get_name() return {"success": True, "wallet_id": wallet_id, "name": name} - async def did_update_recovery_ids(self, request): + async def did_update_recovery_ids(self, request) -> EndpointResult: wallet_id = uint32(request["wallet_id"]) wallet: DIDWallet = self.service.wallet_state_manager.wallets[wallet_id] recovery_list = [] @@ -1094,7 +1089,7 @@ class WalletRpcApi: if "num_verifications_required" in request: new_amount_verifications_required = uint64(request["num_verifications_required"]) else: - new_amount_verifications_required = len(recovery_list) + new_amount_verifications_required = uint64(len(recovery_list)) async with self.service.wallet_state_manager.lock: update_success = await wallet.update_recovery_list(recovery_list, new_amount_verifications_required) # Update coin with new ID info @@ -1104,7 +1099,7 @@ class WalletRpcApi: success = True return {"success": success} - async def did_update_metadata(self, request): + async def did_update_metadata(self, request) -> EndpointResult: wallet_id = uint32(request["wallet_id"]) wallet: DIDWallet = self.service.wallet_state_manager.wallets[wallet_id] if wallet.type() != WalletType.DECENTRALIZED_ID.value: @@ -1124,7 +1119,7 @@ class WalletRpcApi: else: return {"success": False, "error": f"Couldn't update metadata with input: {metadata}"} - async def did_get_did(self, request): + async def did_get_did(self, request) -> EndpointResult: wallet_id = uint32(request["wallet_id"]) wallet: DIDWallet = self.service.wallet_state_manager.wallets[wallet_id] my_did: str = encode_puzzle_hash(bytes32.fromhex(wallet.get_my_DID()), DID_HRP) @@ -1136,7 +1131,7 @@ class WalletRpcApi: coin = coins.pop() return {"success": True, "wallet_id": wallet_id, "my_did": my_did, "coin_id": coin.name()} - async def did_get_recovery_list(self, request): + async def did_get_recovery_list(self, request) -> EndpointResult: wallet_id = uint32(request["wallet_id"]) wallet: DIDWallet = self.service.wallet_state_manager.wallets[wallet_id] recovery_list = wallet.did_info.backup_ids @@ -1150,7 +1145,7 @@ class WalletRpcApi: "num_required": wallet.did_info.num_of_backup_ids_needed, } - async def did_get_metadata(self, request): + async def did_get_metadata(self, request) -> EndpointResult: wallet_id = uint32(request["wallet_id"]) wallet: DIDWallet = self.service.wallet_state_manager.wallets[wallet_id] metadata = json.loads(wallet.did_info.metadata) @@ -1160,7 +1155,7 @@ class WalletRpcApi: "metadata": metadata, } - async def did_recovery_spend(self, request): + async def did_recovery_spend(self, request) -> EndpointResult: wallet_id = uint32(request["wallet_id"]) wallet: DIDWallet = self.service.wallet_state_manager.wallets[wallet_id] if len(request["attest_data"]) < wallet.did_info.num_of_backup_ids_needed: @@ -1179,13 +1174,14 @@ class WalletRpcApi: pubkey = wallet.did_info.temp_pubkey if "puzhash" in request: - puzhash = hexstr_to_bytes(request["puzhash"]) + puzhash = bytes32.from_hexstr(request["puzhash"]) else: assert wallet.did_info.temp_puzhash is not None puzhash = wallet.did_info.temp_puzhash + # TODO: this ignore should be dealt with spend_bundle = await wallet.recovery_spend( - wallet.did_info.temp_coin, + wallet.did_info.temp_coin, # type: ignore[arg-type] puzhash, info_list, pubkey, @@ -1196,13 +1192,13 @@ class WalletRpcApi: else: return {"success": False} - async def did_get_pubkey(self, request): + async def did_get_pubkey(self, request) -> EndpointResult: wallet_id = uint32(request["wallet_id"]) wallet: DIDWallet = self.service.wallet_state_manager.wallets[wallet_id] pubkey = bytes((await wallet.wallet_state_manager.get_unused_derivation_record(wallet_id)).pubkey).hex() return {"success": True, "pubkey": pubkey} - async def did_create_attest(self, request): + async def did_create_attest(self, request) -> EndpointResult: wallet_id = uint32(request["wallet_id"]) wallet: DIDWallet = self.service.wallet_state_manager.wallets[wallet_id] async with self.service.wallet_state_manager.lock: @@ -1224,11 +1220,12 @@ class WalletRpcApi: else: return {"success": False} - async def did_get_information_needed_for_recovery(self, request): + async def did_get_information_needed_for_recovery(self, request) -> EndpointResult: wallet_id = uint32(request["wallet_id"]) did_wallet: DIDWallet = self.service.wallet_state_manager.wallets[wallet_id] my_did = encode_puzzle_hash(bytes32.from_hexstr(did_wallet.get_my_DID()), DID_HRP) - coin_name = did_wallet.did_info.temp_coin.name().hex() + # TODO: this ignore should be dealt with + coin_name = did_wallet.did_info.temp_coin.name().hex() # type: ignore[union-attr] return { "success": True, "wallet_id": wallet_id, @@ -1239,7 +1236,7 @@ class WalletRpcApi: "backup_dids": did_wallet.did_info.backup_ids, } - async def did_get_current_coin_info(self, request): + async def did_get_current_coin_info(self, request) -> EndpointResult: wallet_id = uint32(request["wallet_id"]) did_wallet: DIDWallet = self.service.wallet_state_manager.wallets[wallet_id] my_did = encode_puzzle_hash(bytes32.from_hexstr(did_wallet.get_my_DID()), DID_HRP) @@ -1255,12 +1252,12 @@ class WalletRpcApi: "did_amount": did_coin_threeple[2], } - async def did_create_backup_file(self, request): + async def did_create_backup_file(self, request) -> EndpointResult: wallet_id = uint32(request["wallet_id"]) did_wallet: DIDWallet = self.service.wallet_state_manager.wallets[wallet_id] return {"wallet_id": wallet_id, "success": True, "backup_data": did_wallet.create_backup()} - async def did_transfer_did(self, request): + async def did_transfer_did(self, request) -> EndpointResult: if await self.service.wallet_state_manager.synced() is False: raise ValueError("Wallet needs to be fully synced.") wallet_id = uint32(request["wallet_id"]) @@ -1281,7 +1278,7 @@ class WalletRpcApi: # NFT Wallet ########################################################################################## - async def nft_mint_nft(self, request) -> Dict: + async def nft_mint_nft(self, request) -> EndpointResult: log.debug("Got minting RPC request: %s", request) wallet_id = uint32(request["wallet_id"]) assert self.service.wallet_state_manager @@ -1340,7 +1337,7 @@ class WalletRpcApi: ) return {"wallet_id": wallet_id, "success": True, "spend_bundle": spend_bundle} - async def nft_get_nfts(self, request) -> Dict: + async def nft_get_nfts(self, request) -> EndpointResult: wallet_id = uint32(request["wallet_id"]) nft_wallet: NFTWallet = self.service.wallet_state_manager.wallets[wallet_id] nfts = nft_wallet.get_current_nfts() @@ -1368,7 +1365,7 @@ class WalletRpcApi: log.exception(f"Failed to set DID on NFT: {e}") return {"success": False, "error": f"Failed to set DID on NFT: {e}"} - async def nft_get_by_did(self, request) -> Dict: + async def nft_get_by_did(self, request) -> EndpointResult: did_id: Optional[bytes32] = None if "did_id" in request: did_id = decode_puzzle_hash(request["did_id"]) @@ -1377,7 +1374,7 @@ class WalletRpcApi: return {"wallet_id": wallet.wallet_id, "success": True} return {"error": f"Cannot find a NFT wallet DID = {did_id}", "success": False} - async def nft_get_wallet_did(self, request) -> Dict: + async def nft_get_wallet_did(self, request) -> EndpointResult: wallet_id = uint32(request["wallet_id"]) nft_wallet: NFTWallet = self.service.wallet_state_manager.wallets[wallet_id] if nft_wallet is not None: @@ -1390,7 +1387,7 @@ class WalletRpcApi: return {"success": True, "did_id": None if len(did_id) == 0 else did_id} return {"success": False, "error": f"Wallet {wallet_id} not found"} - async def nft_get_wallets_with_dids(self, request) -> Dict: + async def nft_get_wallets_with_dids(self, request) -> EndpointResult: all_wallets = self.service.wallet_state_manager.wallets.values() did_wallets_by_did_id: Dict[bytes32, uint32] = { wallet.did_info.origin_coin.name(): wallet.id() @@ -1415,7 +1412,7 @@ class WalletRpcApi: ) return {"success": True, "nft_wallets": did_nft_wallets} - async def nft_set_nft_status(self, request) -> Dict: + async def nft_set_nft_status(self, request) -> EndpointResult: try: wallet_id: uint32 = uint32(request["wallet_id"]) coin_id: bytes32 = bytes32.from_hexstr(request["coin_id"]) @@ -1429,7 +1426,7 @@ class WalletRpcApi: except Exception as e: return {"success": False, "error": f"Cannot change the status of the NFT.{e}"} - async def nft_transfer_nft(self, request): + async def nft_transfer_nft(self, request) -> EndpointResult: wallet_id = uint32(request["wallet_id"]) address = request["target_address"] if isinstance(address, str): @@ -1464,7 +1461,7 @@ class WalletRpcApi: log.exception(f"Failed to transfer NFT: {e}") return {"success": False, "error": str(e)} - async def nft_get_info(self, request: Dict) -> Dict[str, object]: + async def nft_get_info(self, request: Dict) -> EndpointResult: if "coin_id" not in request: return {"success": False, "error": "Coin ID is required."} coin_id = request["coin_id"] @@ -1551,7 +1548,7 @@ class WalletRpcApi: else: return {"success": True, "nft_info": nft_info} - async def nft_add_uri(self, request) -> Dict: + async def nft_add_uri(self, request) -> EndpointResult: wallet_id = uint32(request["wallet_id"]) # Note metadata updater can only add one uri for one field per spend. # If you want to add multiple uris for one field, you need to spend multiple times. @@ -1576,7 +1573,7 @@ class WalletRpcApi: # Rate Limited Wallet ########################################################################################## - async def rl_set_user_info(self, request): + async def rl_set_user_info(self, request) -> EndpointResult: wallet_id = uint32(int(request["wallet_id"])) rl_user = self.service.wallet_state_manager.wallets[wallet_id] origin = request["origin"] @@ -1591,8 +1588,8 @@ class WalletRpcApi: ) return {} - async def send_clawback_transaction(self, request): - wallet_id = int(request["wallet_id"]) + async def send_clawback_transaction(self, request) -> EndpointResult: + wallet_id = uint32(request["wallet_id"]) wallet: RLWallet = self.service.wallet_state_manager.wallets[wallet_id] fee = int(request["fee"]) @@ -1606,7 +1603,7 @@ class WalletRpcApi: "transaction_id": tx.name, } - async def add_rate_limited_funds(self, request): + async def add_rate_limited_funds(self, request) -> EndpointResult: wallet_id = uint32(request["wallet_id"]) wallet: RLWallet = self.service.wallet_state_manager.wallets[wallet_id] puzzle_hash = wallet.rl_get_aggregation_puzzlehash(wallet.rl_info.rl_puzzle_hash) @@ -1614,7 +1611,7 @@ class WalletRpcApi: await wallet.rl_add_funds(request["amount"], puzzle_hash, request["fee"]) return {"status": "SUCCESS"} - async def get_farmed_amount(self, request): + async def get_farmed_amount(self, request) -> EndpointResult: tx_records: List[TransactionRecord] = await self.service.wallet_state_manager.tx_store.get_farming_rewards() amount = 0 pool_reward_amount = 0 @@ -1630,6 +1627,11 @@ class WalletRpcApi: continue pool_reward_amount += record.amount height = record.height_farmed(self.service.constants.GENESIS_CHALLENGE) + # .get_farming_rewards() above queries for only confirmed records. This + # could be hinted by making TransactionRecord generic but streamable can't + # handle that presently. Existing code would have raised an exception + # anyways if this were to fail and we already have an assert below. + assert height is not None if record.type == TransactionType.FEE_REWARD: fee_amount += record.amount - calculate_base_farmer_reward(height) farmer_reward_amount += calculate_base_farmer_reward(height) @@ -1646,7 +1648,7 @@ class WalletRpcApi: "last_height_farmed": last_height_farmed, } - async def create_signed_transaction(self, request, hold_lock=True) -> Dict: + async def create_signed_transaction(self, request, hold_lock=True) -> EndpointResult: if "additions" not in request or len(request["additions"]) < 1: raise ValueError("Specify additions list") @@ -1742,7 +1744,7 @@ class WalletRpcApi: ########################################################################################## # Pool Wallet ########################################################################################## - async def pw_join_pool(self, request) -> Dict: + async def pw_join_pool(self, request) -> EndpointResult: fee = uint64(request.get("fee", 0)) wallet_id = uint32(request["wallet_id"]) wallet: PoolWallet = self.service.wallet_state_manager.wallets[wallet_id] @@ -1770,7 +1772,7 @@ class WalletRpcApi: total_fee, tx, fee_tx = await wallet.join_pool(new_target_state, fee) return {"total_fee": total_fee, "transaction": tx, "fee_transaction": fee_tx} - async def pw_self_pool(self, request) -> Dict: + async def pw_self_pool(self, request) -> EndpointResult: # Leaving a pool requires two state transitions. # First we transition to PoolSingletonState.LEAVING_POOL # Then we transition to FARMING_TO_POOL or SELF_POOLING @@ -1787,7 +1789,7 @@ class WalletRpcApi: total_fee, tx, fee_tx = await wallet.self_pool(fee) return {"total_fee": total_fee, "transaction": tx, "fee_transaction": fee_tx} - async def pw_absorb_rewards(self, request) -> Dict: + async def pw_absorb_rewards(self, request) -> EndpointResult: """Perform a sweep of the p2_singleton rewards controlled by the pool wallet singleton""" if await self.service.wallet_state_manager.synced() is False: raise ValueError("Wallet needs to be fully synced before collecting rewards") @@ -1803,7 +1805,7 @@ class WalletRpcApi: state: PoolWalletInfo = await wallet.get_current_state() return {"state": state.to_json_dict(), "transaction": transaction, "fee_transaction": fee_tx} - async def pw_status(self, request) -> Dict: + async def pw_status(self, request) -> EndpointResult: """Return the complete state of the Pool wallet with id `request["wallet_id"]`""" wallet_id = uint32(request["wallet_id"]) wallet: PoolWallet = self.service.wallet_state_manager.wallets[wallet_id] diff --git a/chia/simulator/SimulatorFullNodeRpcApi.py b/chia/simulator/SimulatorFullNodeRpcApi.py index 6dbc66b219..d97e18017b 100644 --- a/chia/simulator/SimulatorFullNodeRpcApi.py +++ b/chia/simulator/SimulatorFullNodeRpcApi.py @@ -1,7 +1,7 @@ from typing import Dict from chia.rpc.full_node_rpc_api import FullNodeRpcApi -from chia.rpc.rpc_server import Endpoint +from chia.rpc.rpc_server import Endpoint, EndpointResult from chia.simulator.simulator_protocol import FarmNewBlockProtocol from chia.util.bech32m import decode_puzzle_hash @@ -12,7 +12,7 @@ class SimulatorFullNodeRpcApi(FullNodeRpcApi): routes["/farm_tx_block"] = self.farm_tx_block return routes - async def farm_tx_block(self, _request: Dict[str, object]) -> Dict[str, object]: + async def farm_tx_block(self, _request: Dict[str, object]) -> EndpointResult: request_address = str(_request["address"]) ph = decode_puzzle_hash(request_address) req = FarmNewBlockProtocol(ph) diff --git a/chia/wallet/trade_manager.py b/chia/wallet/trade_manager.py index 0324e33c64..e27223938f 100644 --- a/chia/wallet/trade_manager.py +++ b/chia/wallet/trade_manager.py @@ -4,6 +4,8 @@ import time import traceback from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing_extensions import Literal + from chia.protocols.wallet_protocol import CoinState from chia.types.blockchain_format.coin import Coin, coin_as_list from chia.types.blockchain_format.program import Program @@ -297,12 +299,14 @@ class TradeManager: driver_dict: Optional[Dict[bytes32, PuzzleInfo]] = None, fee: uint64 = uint64(0), validate_only: bool = False, - ) -> Tuple[bool, Optional[TradeRecord], Optional[str]]: + ) -> Union[Tuple[Literal[True], TradeRecord, None], Tuple[Literal[False], None, str]]: if driver_dict is None: driver_dict = {} - success, created_offer, error = await self._create_offer_for_ids(offer, driver_dict, fee=fee) - if not success or created_offer is None: - raise Exception(f"Error creating offer: {error}") + result = await self._create_offer_for_ids(offer, driver_dict, fee=fee) + if not result[0] or result[1] is None: + raise Exception(f"Error creating offer: {result[2]}") + + success, created_offer, error = result now = uint64(int(time.time())) trade_offer: TradeRecord = TradeRecord( @@ -329,7 +333,7 @@ class TradeManager: offer_dict: Dict[Union[int, bytes32], int], driver_dict: Optional[Dict[bytes32, PuzzleInfo]] = None, fee: uint64 = uint64(0), - ) -> Tuple[bool, Optional[Offer], Optional[str]]: + ) -> Union[Tuple[Literal[True], Offer, None], Tuple[Literal[False], None, str]]: """ Offer is dictionary of wallet ids and amount """ @@ -582,7 +586,11 @@ class TradeManager: return txs - async def respond_to_offer(self, offer: Offer, fee=uint64(0)) -> Tuple[bool, Optional[TradeRecord], Optional[str]]: + async def respond_to_offer( + self, + offer: Offer, + fee=uint64(0), + ) -> Union[Tuple[Literal[True], TradeRecord, None], Tuple[Literal[False], None, str]]: take_offer_dict: Dict[Union[bytes32, int], int] = {} arbitrage: Dict[Optional[bytes32], int] = offer.arbitrage() @@ -605,9 +613,11 @@ class TradeManager: valid: bool = await self.check_offer_validity(offer) if not valid: return False, None, "This offer is no longer valid" - success, take_offer, error = await self._create_offer_for_ids(take_offer_dict, offer.driver_dict, fee=fee) - if not success or take_offer is None: - return False, None, error + result = await self._create_offer_for_ids(take_offer_dict, offer.driver_dict, fee=fee) + if not result[0] or result[1] is None: + return False, None, result[2] + + success, take_offer, error = result complete_offer = Offer.aggregate([offer, take_offer]) assert complete_offer.is_valid() From 7c5a4b2199cfb69b1974098bca0ff3f11c78bdf2 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Tue, 5 Jul 2022 23:29:12 +0200 Subject: [PATCH 31/62] remove pool transition store cache (#12146) --- chia/pools/pool_wallet.py | 12 ++-- chia/wallet/wallet_node.py | 3 - chia/wallet/wallet_pool_store.py | 94 ++++++++++++++------------- tests/pools/test_wallet_pool_store.py | 28 +++----- 4 files changed, 66 insertions(+), 71 deletions(-) diff --git a/chia/pools/pool_wallet.py b/chia/pools/pool_wallet.py index b41a2585cb..f2ca301036 100644 --- a/chia/pools/pool_wallet.py +++ b/chia/pools/pool_wallet.py @@ -180,7 +180,7 @@ class PoolWallet: raise ValueError(f"Invalid internal Pool State: {err}: {initial_target_state}") async def get_spend_history(self) -> List[Tuple[uint32, CoinSpend]]: - return self.wallet_state_manager.pool_store.get_spends_for_wallet(self.wallet_id) + return await self.wallet_state_manager.pool_store.get_spends_for_wallet(self.wallet_id) async def get_current_state(self) -> PoolWalletInfo: history: List[Tuple[uint32, CoinSpend]] = await self.get_spend_history() @@ -228,7 +228,7 @@ class PoolWallet: return await self.wallet_state_manager.tx_store.get_unconfirmed_for_wallet(self.wallet_id) async def get_tip(self) -> Tuple[uint32, CoinSpend]: - return self.wallet_state_manager.pool_store.get_spends_for_wallet(self.wallet_id)[-1] + return (await self.wallet_state_manager.pool_store.get_spends_for_wallet(self.wallet_id))[-1] async def update_pool_config(self) -> None: current_state: PoolWalletInfo = await self.get_current_state() @@ -286,7 +286,9 @@ class PoolWallet: self.log.info(f"New PoolWallet singleton tip_coin: {tip_spend} farmed at height {block_height}") # If we have reached the target state, resets it to None. Loops back to get current state - for _, added_spend in reversed(self.wallet_state_manager.pool_store.get_spends_for_wallet(self.wallet_id)): + for _, added_spend in reversed( + await self.wallet_state_manager.pool_store.get_spends_for_wallet(self.wallet_id) + ): latest_state: Optional[PoolState] = solution_to_pool_state(added_spend) if latest_state is not None: if self.target_state == latest_state: @@ -303,9 +305,9 @@ class PoolWallet: Returns True if the wallet should be removed. """ try: - history: List[Tuple[uint32, CoinSpend]] = self.wallet_state_manager.pool_store.get_spends_for_wallet( + history: List[Tuple[uint32, CoinSpend]] = await self.wallet_state_manager.pool_store.get_spends_for_wallet( self.wallet_id - ).copy() + ) prev_state: PoolWalletInfo = await self.get_current_state() await self.wallet_state_manager.pool_store.rollback(block_height, self.wallet_id, in_transaction) diff --git a/chia/wallet/wallet_node.py b/chia/wallet/wallet_node.py index fafd77a68a..4419a2a4c5 100644 --- a/chia/wallet/wallet_node.py +++ b/chia/wallet/wallet_node.py @@ -512,7 +512,6 @@ class WalletNode: tb = traceback.format_exc() self.log.error(f"Exception while perform_atomic_rollback: {e} {tb}") await self.wallet_state_manager.db_wrapper.rollback_transaction() - await self.wallet_state_manager.pool_store.rebuild_cache() raise else: await self.wallet_state_manager.blockchain.clean_block_records() @@ -708,7 +707,6 @@ class WalletNode: tb = traceback.format_exc() self.log.error(f"Exception while adding state: {e} {tb}") await self.wallet_state_manager.db_wrapper.rollback_transaction() - await self.wallet_state_manager.pool_store.rebuild_cache() else: await self.wallet_state_manager.blockchain.clean_block_records() @@ -743,7 +741,6 @@ class WalletNode: await self.wallet_state_manager.db_wrapper.commit_transaction() except Exception as e: await self.wallet_state_manager.db_wrapper.rollback_transaction() - await self.wallet_state_manager.pool_store.rebuild_cache() tb = traceback.format_exc() self.log.error(f"Error adding states.. {e} {tb}") return False diff --git a/chia/wallet/wallet_pool_store.py b/chia/wallet/wallet_pool_store.py index 2523dd61c0..2505c60e0c 100644 --- a/chia/wallet/wallet_pool_store.py +++ b/chia/wallet/wallet_pool_store.py @@ -1,5 +1,5 @@ import logging -from typing import List, Tuple, Dict, Optional +from typing import List, Tuple import aiosqlite @@ -13,7 +13,6 @@ log = logging.getLogger(__name__) class WalletPoolStore: db_connection: aiosqlite.Connection db_wrapper: DBWrapper - _state_transitions_cache: Dict[int, List[Tuple[uint32, CoinSpend]]] @classmethod async def create(cls, wrapper: DBWrapper): @@ -23,11 +22,15 @@ class WalletPoolStore: self.db_wrapper = wrapper await self.db_connection.execute( - "CREATE TABLE IF NOT EXISTS pool_state_transitions(transition_index integer, wallet_id integer, " - "height bigint, coin_spend blob, PRIMARY KEY(transition_index, wallet_id))" + "CREATE TABLE IF NOT EXISTS pool_state_transitions(" + " transition_index integer," + " wallet_id integer," + " height bigint," + " coin_spend blob," + " PRIMARY KEY(transition_index, wallet_id))" ) + await self.db_connection.commit() - await self.rebuild_cache() return self async def _clear_database(self): @@ -51,28 +54,48 @@ class WalletPoolStore: if not in_transaction: await self.db_wrapper.lock.acquire() try: - if wallet_id not in self._state_transitions_cache: - self._state_transitions_cache[wallet_id] = [] - all_state_transitions: List[Tuple[uint32, CoinSpend]] = self.get_spends_for_wallet(wallet_id) + # find the most recent transition in wallet_id + rows = list( + await self.db_connection.execute_fetchall( + "SELECT transition_index, height, coin_spend " + "FROM pool_state_transitions " + "WHERE wallet_id=? " + "ORDER BY transition_index DESC " + "LIMIT 1", + (wallet_id,), + ) + ) + serialized_spend = bytes(spend) + if len(rows) == 0: + transition_index = 0 + else: + existing = list( + await self.db_connection.execute_fetchall( + "SELECT COUNT(*) " + "FROM pool_state_transitions " + "WHERE wallet_id=? AND height=? AND coin_spend=?", + (wallet_id, height, serialized_spend), + ) + ) + if existing[0][0] != 0: + # we already have this transition in the DB + return - if (height, spend) in all_state_transitions: - return - - if len(all_state_transitions) > 0: - if height < all_state_transitions[-1][0]: + row = rows[0] + if height < row[1]: raise ValueError("Height cannot go down") - if spend.coin.parent_coin_info != all_state_transitions[-1][1].coin.name(): + prev = CoinSpend.from_bytes(row[2]) + if spend.coin.parent_coin_info != prev.coin.name(): raise ValueError("New spend does not extend") - - all_state_transitions.append((height, spend)) + transition_index = row[0] cursor = await self.db_connection.execute( - "INSERT OR REPLACE INTO pool_state_transitions VALUES (?, ?, ?, ?)", + "INSERT OR IGNORE INTO pool_state_transitions VALUES (?, ?, ?, ?)", ( - len(all_state_transitions) - 1, + transition_index + 1, wallet_id, height, - bytes(spend), + serialized_spend, ), ) await cursor.close() @@ -81,27 +104,16 @@ class WalletPoolStore: await self.db_connection.commit() self.db_wrapper.lock.release() - def get_spends_for_wallet(self, wallet_id: int) -> List[Tuple[uint32, CoinSpend]]: + async def get_spends_for_wallet(self, wallet_id: int) -> List[Tuple[uint32, CoinSpend]]: """ - Retrieves all entries for a wallet ID from the cache, works even if commit is not called yet. + Retrieves all entries for a wallet ID. """ - return self._state_transitions_cache.get(wallet_id, []) - async def rebuild_cache(self) -> None: - """ - This resets the cache, and loads all entries from the DB. Any entries in the cache that were not committed - are removed. This can happen if a state transition in wallet_blockchain fails. - """ - cursor = await self.db_connection.execute("SELECT * FROM pool_state_transitions ORDER BY transition_index") - rows = await cursor.fetchall() - await cursor.close() - self._state_transitions_cache = {} - for row in rows: - _, wallet_id, height, coin_spend_bytes = row - coin_spend: CoinSpend = CoinSpend.from_bytes(coin_spend_bytes) - if wallet_id not in self._state_transitions_cache: - self._state_transitions_cache[wallet_id] = [] - self._state_transitions_cache[wallet_id].append((height, coin_spend)) + rows = await self.db_connection.execute_fetchall( + "SELECT height, coin_spend FROM pool_state_transitions WHERE wallet_id=? ORDER BY transition_index", + (wallet_id,), + ) + return [(uint32(row[0]), CoinSpend.from_bytes(row[1])) for row in rows] async def rollback(self, height: int, wallet_id_arg: int, in_transaction: bool) -> None: """ @@ -113,14 +125,6 @@ class WalletPoolStore: if not in_transaction: await self.db_wrapper.lock.acquire() try: - for wallet_id, items in self._state_transitions_cache.items(): - remove_index_start: Optional[int] = None - for i, (item_block_height, _) in enumerate(items): - if item_block_height > height and wallet_id == wallet_id_arg: - remove_index_start = i - break - if remove_index_start is not None: - del items[remove_index_start:] cursor = await self.db_connection.execute( "DELETE FROM pool_state_transitions WHERE height>? AND wallet_id=?", (height, wallet_id_arg) ) diff --git a/tests/pools/test_wallet_pool_store.py b/tests/pools/test_wallet_pool_store.py index 27c3152a2c..f88397af2c 100644 --- a/tests/pools/test_wallet_pool_store.py +++ b/tests/pools/test_wallet_pool_store.py @@ -51,38 +51,35 @@ class TestWalletPoolStore: solution_0_alt: CoinSpend = make_child_solution(None, coin_0_alt) solution_1: CoinSpend = make_child_solution(solution_0) - assert store.get_spends_for_wallet(0) == [] - assert store.get_spends_for_wallet(1) == [] + assert await store.get_spends_for_wallet(0) == [] + assert await store.get_spends_for_wallet(1) == [] await store.add_spend(1, solution_1, 100, True) - assert store.get_spends_for_wallet(1) == [(100, solution_1)] + assert await store.get_spends_for_wallet(1) == [(100, solution_1)] # Idempotent await store.add_spend(1, solution_1, 100, True) - assert store.get_spends_for_wallet(1) == [(100, solution_1)] + assert await store.get_spends_for_wallet(1) == [(100, solution_1)] with pytest.raises(ValueError): await store.add_spend(1, solution_1, 101, True) # Rebuild cache, no longer present await db_wrapper.rollback_transaction() - await store.rebuild_cache() - assert store.get_spends_for_wallet(1) == [] + assert await store.get_spends_for_wallet(1) == [] - await store.rebuild_cache() await store.add_spend(1, solution_1, 100, False) - assert store.get_spends_for_wallet(1) == [(100, solution_1)] + assert await store.get_spends_for_wallet(1) == [(100, solution_1)] solution_1_alt: CoinSpend = make_child_solution(solution_0_alt) with pytest.raises(ValueError): await store.add_spend(1, solution_1_alt, 100, False) - assert store.get_spends_for_wallet(1) == [(100, solution_1)] + assert await store.get_spends_for_wallet(1) == [(100, solution_1)] solution_2: CoinSpend = make_child_solution(solution_1) await store.add_spend(1, solution_2, 100, False) - await store.rebuild_cache() solution_3: CoinSpend = make_child_solution(solution_2) await store.add_spend(1, solution_3, 100) solution_4: CoinSpend = make_child_solution(solution_3) @@ -90,21 +87,16 @@ class TestWalletPoolStore: with pytest.raises(ValueError): await store.add_spend(1, solution_4, 99) - await store.rebuild_cache() await store.add_spend(1, solution_4, 101) - await store.rebuild_cache() await store.rollback(101, 1, False) - await store.rebuild_cache() - assert store.get_spends_for_wallet(1) == [ + assert await store.get_spends_for_wallet(1) == [ (100, solution_1), (100, solution_2), (100, solution_3), (101, solution_4), ] - await store.rebuild_cache() await store.rollback(100, 1, False) - await store.rebuild_cache() - assert store.get_spends_for_wallet(1) == [ + assert await store.get_spends_for_wallet(1) == [ (100, solution_1), (100, solution_2), (100, solution_3), @@ -116,7 +108,7 @@ class TestWalletPoolStore: solution_5: CoinSpend = make_child_solution(solution_4) await store.add_spend(1, solution_5, 105) await store.rollback(99, 1, False) - assert store.get_spends_for_wallet(1) == [] + assert await store.get_spends_for_wallet(1) == [] finally: await db_connection.close() From 5504f36d4208036a177a2bd70c2eff7566581bf2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Jul 2022 16:29:44 -0500 Subject: [PATCH 32/62] Bump fasteners from 0.16.3 to 0.17.3 (#12206) Bumps [fasteners](https://github.com/harlowja/fasteners) from 0.16.3 to 0.17.3. - [Release notes](https://github.com/harlowja/fasteners/releases) - [Changelog](https://github.com/harlowja/fasteners/blob/main/CHANGELOG.md) - [Commits](https://github.com/harlowja/fasteners/compare/0.16.3...0.17.3) --- updated-dependencies: - dependency-name: fasteners dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index af61066b2c..325f2737b5 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ dependencies = [ "colorlog==6.6.0", # Adds color to logs "concurrent-log-handler==0.9.19", # Concurrently log and rotate logs "cryptography==36.0.2", # Python cryptography library for TLS - keyring conflict - "fasteners==0.16.3", # For interprocess file locking, expected to be replaced by filelock + "fasteners==0.17.3", # For interprocess file locking, expected to be replaced by filelock "filelock==3.7.1", # For reading and writing config multiprocess and multithread safely (non-reentrant locks) "keyring==23.6.0", # Store keys in MacOS Keychain, Windows Credential Locker "keyrings.cryptfile==1.3.4", # Secure storage for keys on Linux (Will be replaced) From d18fff0a8ae1c393f07155876dbb1c1d1ef3ff68 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Jul 2022 16:31:14 -0500 Subject: [PATCH 33/62] Bump typing-extensions from 4.0.1 to 4.3.0 (#12207) Bumps [typing-extensions](https://github.com/python/typing_extensions) from 4.0.1 to 4.3.0. - [Release notes](https://github.com/python/typing_extensions/releases) - [Changelog](https://github.com/python/typing_extensions/blob/main/CHANGELOG.md) - [Commits](https://github.com/python/typing_extensions/compare/4.0.1...4.3.0) --- updated-dependencies: - dependency-name: typing-extensions dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 325f2737b5..7f0b1835e0 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ dependencies = [ "dnspython==2.2.0", # Query DNS seeds "watchdog==2.1.9", # Filesystem event watching - watches keyring.yaml "dnslib==0.9.17", # dns lib - "typing-extensions==4.0.1", # typing backports like Protocol and TypedDict + "typing-extensions==4.3.0", # typing backports like Protocol and TypedDict "zstd==1.5.0.4", "packaging==21.3", ] From dea1182299f538ba031c2ab7720254bde6bb4440 Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Tue, 5 Jul 2022 16:45:43 -0500 Subject: [PATCH 34/62] Remove break (its preventing other sockets from getting data when earlier ones have an error) (#12241) --- chia/daemon/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/chia/daemon/server.py b/chia/daemon/server.py index 08b4dcca4c..cb7fac3580 100644 --- a/chia/daemon/server.py +++ b/chia/daemon/server.py @@ -243,7 +243,6 @@ class WebSocketServer: self.log.error(f"Unexpected exception trying to send to websocket: {e} {tb}") self.remove_connection(socket) await socket.close() - break else: service_name = "Unknown" if ws in self.remote_address_map: From 391c1a00777dc340c23e2c9642c8f5461e4875b2 Mon Sep 17 00:00:00 2001 From: Mariano Sorgente <3069354+mariano54@users.noreply.github.com> Date: Wed, 6 Jul 2022 07:30:16 +0900 Subject: [PATCH 35/62] Add reorg test (#12230) * Add reorg test * Add another reorg test --- tests/wallet/cat_wallet/test_cat_wallet.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/wallet/cat_wallet/test_cat_wallet.py b/tests/wallet/cat_wallet/test_cat_wallet.py index bfde40d42b..9d06bcc3ff 100644 --- a/tests/wallet/cat_wallet/test_cat_wallet.py +++ b/tests/wallet/cat_wallet/test_cat_wallet.py @@ -5,7 +5,7 @@ import pytest from chia.consensus.block_rewards import calculate_base_farmer_reward, calculate_pool_reward from chia.full_node.mempool_manager import MempoolManager -from chia.simulator.simulator_protocol import FarmNewBlockProtocol +from chia.simulator.simulator_protocol import FarmNewBlockProtocol, ReorgProtocol from chia.types.blockchain_format.coin import Coin from chia.types.blockchain_format.sized_bytes import bytes32 from chia.types.peer_info import PeerInfo @@ -98,6 +98,10 @@ class TestCATWallet: assert new_cat_wallet.cat_info.my_tail == cat_wallet.cat_info.my_tail assert await cat_wallet.lineage_store.get_all_lineage_proofs() == all_lineage + height = full_node_api.full_node.blockchain.get_peak_height() + await full_node_api.reorg_from_index_to_new_index(ReorgProtocol(height - num_blocks - 1, height + 1, 32 * b"1")) + await time_out_assert(15, cat_wallet.get_confirmed_balance, 0) + @pytest.mark.asyncio async def test_cat_creation_unique_lineage_store(self, self_hostname, two_wallet_nodes): num_blocks = 3 @@ -233,12 +237,15 @@ class TestCATWallet: 15, tx_in_pool, True, full_node_api.full_node.mempool_manager, tx_record.spend_bundle.name() ) - for i in range(1, num_blocks): - await full_node_api.farm_new_transaction_block(FarmNewBlockProtocol(ph)) + await full_node_api.farm_new_transaction_block(FarmNewBlockProtocol(ph)) await time_out_assert(15, cat_wallet.get_confirmed_balance, 55) await time_out_assert(15, cat_wallet.get_unconfirmed_balance, 55) + height = full_node_api.full_node.blockchain.get_peak_height() + await full_node_api.reorg_from_index_to_new_index(ReorgProtocol(height - 1, height + 1, 32 * b"1")) + await time_out_assert(15, cat_wallet.get_confirmed_balance, 40) + @pytest.mark.parametrize( "trusted", [True, False], From cb2528b78d418a3d9ac2ec61bfc65773d7790315 Mon Sep 17 00:00:00 2001 From: olivernyc Date: Wed, 6 Jul 2022 01:49:04 +0200 Subject: [PATCH 36/62] Remove unused finished_sync_up_to (#12224) --- chia/wallet/wallet_state_manager.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/chia/wallet/wallet_state_manager.py b/chia/wallet/wallet_state_manager.py index 20d512a914..4ae0847a8b 100644 --- a/chia/wallet/wallet_state_manager.py +++ b/chia/wallet/wallet_state_manager.py @@ -117,7 +117,6 @@ class WalletStateManager: blockchain: WalletBlockchain coin_store: WalletCoinStore sync_store: WalletSyncStore - finished_sync_up_to: uint32 interested_store: WalletInterestedStore multiprocessing_context: multiprocessing.context.BaseContext weight_proof_handler: WalletWeightProofHandler @@ -183,7 +182,6 @@ class WalletStateManager: self.wallet_node = wallet_node self.sync_mode = False self.sync_target = uint32(0) - self.finished_sync_up_to = uint32(0) multiprocessing_start_method = process_config_start_method(config=self.config, log=self.log) self.multiprocessing_context = multiprocessing.get_context(method=multiprocessing_start_method) self.weight_proof_handler = WalletWeightProofHandler( From c699a2a20ea149cb215fd125f5df6b3fdcd3cade Mon Sep 17 00:00:00 2001 From: olivernyc Date: Wed, 6 Jul 2022 01:49:27 +0200 Subject: [PATCH 37/62] Fix misleading argument name (#12180) --- chia/consensus/blockchain_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chia/consensus/blockchain_interface.py b/chia/consensus/blockchain_interface.py index dccc6d4265..bbb2524bca 100644 --- a/chia/consensus/blockchain_interface.py +++ b/chia/consensus/blockchain_interface.py @@ -71,7 +71,7 @@ class BlockchainInterface: return None async def persist_sub_epoch_challenge_segments( - self, sub_epoch_summary_height: bytes32, segments: List[SubEpochChallengeSegment] + self, sub_epoch_summary_hash: bytes32, segments: List[SubEpochChallengeSegment] ) -> None: pass From ef6359638ff04118fa055dced138a048dec28f36 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 6 Jul 2022 06:06:55 -0400 Subject: [PATCH 38/62] avoid responding to an ack (#12251) --- chia/rpc/rpc_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chia/rpc/rpc_server.py b/chia/rpc/rpc_server.py index ed46b8f408..5ddb6915ae 100644 --- a/chia/rpc/rpc_server.py +++ b/chia/rpc/rpc_server.py @@ -207,14 +207,14 @@ class RpcServer: "success": "true", } - async def ws_api(self, message: WsRpcMessage) -> Dict[str, object]: + async def ws_api(self, message: WsRpcMessage) -> Optional[Dict[str, object]]: """ This function gets called when new message is received via websocket. """ command = message["command"] if message["ack"]: - return {} + return None data: Dict[str, object] = {} if "data" in message: From f69bfb4033bcd58c552eab0bdf0c982681f77e9d Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 6 Jul 2022 08:07:15 -0400 Subject: [PATCH 39/62] catch up with types-aiofiles hinting --- chia/util/files.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/chia/util/files.py b/chia/util/files.py index b122132ca8..dde9d1fa45 100644 --- a/chia/util/files.py +++ b/chia/util/files.py @@ -3,10 +3,11 @@ import logging import os import shutil -from aiofiles import tempfile # type: ignore from pathlib import Path from typing import Union +from aiofiles import tempfile +from typing_extensions import Literal log = logging.getLogger(__name__) @@ -66,7 +67,7 @@ async def write_file_async(file_path: Path, data: Union[str, bytes], *, file_mod # Create the parent directory if necessary os.makedirs(file_path.parent, mode=dir_mode, exist_ok=True) - mode: str = "w+" if type(data) == str else "w+b" + mode: Literal["w+", "w+b"] = "w+" if type(data) == str else "w+b" temp_file_path: Path async with tempfile.NamedTemporaryFile(dir=file_path.parent, mode=mode, delete=False) as f: temp_file_path = f.name From 4197b39987312f5bd5a07782e90ee15a71490598 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 6 Jul 2022 18:24:47 -0400 Subject: [PATCH 40/62] use a variable in the workflow for blocks and plots version (#12209) * use a variable in the workflow for blocks and plots version * Update benchmarks.yml --- .github/workflows/benchmarks.yml | 3 ++- .github/workflows/test-single.yml | 11 ++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 43934d579b..0e51855b41 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -29,6 +29,7 @@ jobs: python-version: [ 3.9 ] env: CHIA_ROOT: ${{ github.workspace }}/.chia/mainnet + BLOCKS_AND_PLOTS_VERSION: 0.29.0 steps: - name: Clean workspace @@ -62,7 +63,7 @@ jobs: with: repository: 'Chia-Network/test-cache' path: '.chia' - ref: '0.29.0' + ref: ${{ env.BLOCKS_AND_PLOTS_VERSION }} fetch-depth: 1 - name: Run install script diff --git a/.github/workflows/test-single.yml b/.github/workflows/test-single.yml index 7b85c25b3b..4893157eeb 100644 --- a/.github/workflows/test-single.yml +++ b/.github/workflows/test-single.yml @@ -97,6 +97,7 @@ jobs: env: CHIA_ROOT: ${{ github.workspace }}/.chia/mainnet JOB_FILE_NAME: tests_${{ matrix.os.file_name }}_python-${{ matrix.python.file_name }}_${{ matrix.configuration.name }} + BLOCKS_AND_PLOTS_VERSION: 0.29.0 steps: - name: Configure git @@ -152,21 +153,21 @@ jobs: path: | ${{ github.workspace }}/.chia/blocks ${{ github.workspace }}/.chia/test-plots - key: 0.29.0 + key: ${{ env.BLOCKS_AND_PLOTS_VERSION }} - name: Checkout test blocks and plots (macOS, Ubuntu) if: steps.test-blocks-plots.outputs.cache-hit != 'true' && (matrix.os.matrix == 'ubuntu' || matrix.os.matrix == 'macos') run: | - wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.tar.gz | tar xzf - + wget -qO- https://github.com/Chia-Network/test-cache/archive/refs/tags/${{ env.BLOCKS_AND_PLOTS_VERSION }}.tar.gz | tar xzf - mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-${{ env.BLOCKS_AND_PLOTS_VERSION }}/* ${{ github.workspace }}/.chia - name: Checkout test blocks and plots (Windows) if: steps.test-blocks-plots.outputs.cache-hit != 'true' && matrix.os.matrix == 'windows' run: | - Invoke-WebRequest -OutFile blocks_and_plots.zip https://github.com/Chia-Network/test-cache/archive/refs/tags/0.29.0.zip; Expand-Archive blocks_and_plots.zip -DestinationPath . + Invoke-WebRequest -OutFile blocks_and_plots.zip https://github.com/Chia-Network/test-cache/archive/refs/tags/${{ env.BLOCKS_AND_PLOTS_VERSION }}.zip; Expand-Archive blocks_and_plots.zip -DestinationPath . mkdir ${{ github.workspace }}/.chia - mv ${{ github.workspace }}/test-cache-0.29.0/* ${{ github.workspace }}/.chia + mv ${{ github.workspace }}/test-cache-${{ env.BLOCKS_AND_PLOTS_VERSION }}/* ${{ github.workspace }}/.chia - name: Install boost (macOS) if: matrix.os.matrix == 'macos' From 526bc39efe6011c5e7944b8dd0943bc5b066b06d Mon Sep 17 00:00:00 2001 From: Matt Hauff Date: Mon, 11 Jul 2022 15:24:49 -0500 Subject: [PATCH 41/62] delete unused CLVM --- chia/wallet/puzzles/database_layer.clvm | 45 ----- chia/wallet/puzzles/database_layer.clvm.hex | 1 - .../database_layer.clvm.hex.sha256tree | 1 - chia/wallet/puzzles/database_offer.clvm | 70 ------- chia/wallet/puzzles/database_offer.clvm.hex | 1 - .../database_offer.clvm.hex.sha256tree | 1 - .../singleton_top_layer_atari_only.clvm | 173 ------------------ .../singleton_top_layer_atari_only.clvm.hex | 1 - ...n_top_layer_atari_only.clvm.hex.sha256tree | 1 - 9 files changed, 294 deletions(-) delete mode 100644 chia/wallet/puzzles/database_layer.clvm delete mode 100644 chia/wallet/puzzles/database_layer.clvm.hex delete mode 100644 chia/wallet/puzzles/database_layer.clvm.hex.sha256tree delete mode 100644 chia/wallet/puzzles/database_offer.clvm delete mode 100644 chia/wallet/puzzles/database_offer.clvm.hex delete mode 100644 chia/wallet/puzzles/database_offer.clvm.hex.sha256tree delete mode 100644 chia/wallet/puzzles/singleton_top_layer_atari_only.clvm delete mode 100644 chia/wallet/puzzles/singleton_top_layer_atari_only.clvm.hex delete mode 100644 chia/wallet/puzzles/singleton_top_layer_atari_only.clvm.hex.sha256tree diff --git a/chia/wallet/puzzles/database_layer.clvm b/chia/wallet/puzzles/database_layer.clvm deleted file mode 100644 index 3ccad7393b..0000000000 --- a/chia/wallet/puzzles/database_layer.clvm +++ /dev/null @@ -1,45 +0,0 @@ -(mod ( - DB_LAYER_MOD_HASH ; This mod file - used to update our state - CURRENT_STATE ; 32 byte merkle root - INNER_PUZZLE_HASH ; Our inner inner puzzle - spend_type - inner_solution ; if spend_type is 1 this is my_amount - inner_puzzle ; Useless in the report case, should hash to the curried in hash if doing update -) - - (include condition_codes.clvm) - (include curry-and-treehash.clinc) - - (defmacro assert items - (if (r items) - (list if (f items) (c assert (r items)) (q . (x))) - (f items) - ) - ) - - (defun sha256tree1 (TREE) - (if (l TREE) - (sha256 2 (sha256tree1 (f TREE)) (sha256tree1 (r TREE))) - (sha256 ONE TREE))) - - (defun-inline current_puzzle_hash (DB_LAYER_MOD_HASH CURRENT_STATE INNER_PUZZLE_HASH) - (puzzle-hash-of-curried-function DB_LAYER_MOD_HASH - (sha256 1 INNER_PUZZLE_HASH) - (sha256 1 CURRENT_STATE) - (sha256 1 DB_LAYER_MOD_HASH) - ) - ) - - ; spend_type 1 is REPORT current state - ; spend_type 0 is UPDATE the current state - (if spend_type - ; REPORT - (list - (list CREATE_PUZZLE_ANNOUNCEMENT CURRENT_STATE) - (list CREATE_COIN (current_puzzle_hash DB_LAYER_MOD_HASH CURRENT_STATE INNER_PUZZLE_HASH) inner_solution) - (list ASSERT_MY_AMOUNT inner_solution) - ) - ; UPDATE - (assert (= INNER_PUZZLE_HASH (sha256tree1 inner_puzzle)) (a inner_puzzle inner_solution)) - ) -) diff --git a/chia/wallet/puzzles/database_layer.clvm.hex b/chia/wallet/puzzles/database_layer.clvm.hex deleted file mode 100644 index 3390f4c355..0000000000 --- a/chia/wallet/puzzles/database_layer.clvm.hex +++ /dev/null @@ -1 +0,0 @@ -ff02ffff01ff02ffff03ff2fffff01ff04ffff04ff2cffff04ff0bff808080ffff04ffff04ff14ffff04ffff02ff2effff04ff02ffff04ff05ffff04ffff0bffff0101ff1780ffff04ffff0bffff0101ff0b80ffff04ffff0bffff0101ff0580ff80808080808080ffff04ff5fff80808080ffff04ffff04ff10ffff04ff5fff808080ff80808080ffff01ff02ffff03ffff09ff17ffff02ff3effff04ff02ffff04ff81bfff8080808080ffff01ff02ff81bfff5f80ffff01ff088080ff018080ff0180ffff04ffff01ffffff4902ff33ff3e04ffff01ff0102ffff02ffff03ff05ffff01ff02ff16ffff04ff02ffff04ff0dffff04ffff0bff3affff0bff12ff3c80ffff0bff3affff0bff3affff0bff12ff2a80ff0980ffff0bff3aff0bffff0bff12ff8080808080ff8080808080ffff010b80ff0180ffff0bff3affff0bff12ff1880ffff0bff3affff0bff3affff0bff12ff2a80ff0580ffff0bff3affff02ff16ffff04ff02ffff04ff07ffff04ffff0bff12ff1280ff8080808080ffff0bff12ff8080808080ff02ffff03ffff07ff0580ffff01ff0bffff0102ffff02ff3effff04ff02ffff04ff09ff80808080ffff02ff3effff04ff02ffff04ff0dff8080808080ffff01ff0bff12ff058080ff0180ff018080 \ No newline at end of file diff --git a/chia/wallet/puzzles/database_layer.clvm.hex.sha256tree b/chia/wallet/puzzles/database_layer.clvm.hex.sha256tree deleted file mode 100644 index a4e6742686..0000000000 --- a/chia/wallet/puzzles/database_layer.clvm.hex.sha256tree +++ /dev/null @@ -1 +0,0 @@ -dff4698ae98ada214a7e7f3b37482fab7927532f90fabe02ab6f5229df284f8f diff --git a/chia/wallet/puzzles/database_offer.clvm b/chia/wallet/puzzles/database_offer.clvm deleted file mode 100644 index 86cbbd1914..0000000000 --- a/chia/wallet/puzzles/database_offer.clvm +++ /dev/null @@ -1,70 +0,0 @@ -(mod ( - DB_LAYER_MOD_HASH - DB_SINGLETON_STRUCT - LEAF_REVEAL - CLAIM_TARGET - RECOVERY_TARGET - RECOVERY_TIMELOCK - spend_type ; 1 is claim, 0 is recover - my_amount - db_innerpuz_hash - current_root - inclusion_proof - ) - ; DB_SINGLETON_STRUCT is (MOD_HASH . (LAUNCHER_ID . LAUNCHER_PUZZLE_HASH)) - ; current_root is a bytes32 of the top of the merkle tree in the db_host - also known as current_state - ; inclusion_proof is a list where first entry is a binary representation of path, and subsequent entries are corresponding merkle hashes - (include condition_codes.clvm) - (include curry-and-treehash.clinc) - - (defun-inline tree_branch_hash (left right) (sha256 2 left right)) - - (defun calculate_merkle_root (path current_hash additional_steps) - (if additional_steps - (calculate_merkle_root - (lsh path -1) - (if (logand path 1) - (tree_branch_hash - (f additional_steps) - current_hash - ) - (tree_branch_hash - current_hash - (f additional_steps) - ) - ) - (r additional_steps) - ) - current_hash - ) - ) - - (defun calculate_merkle_root_for_merkle_proof (LEAF_REVEAL proof_of_inclusion) - (calculate_merkle_root - (f proof_of_inclusion) - (sha256tree LEAF_REVEAL) ; should this be tree-hashed when curried in? - (if (r proof_of_inclusion) (f (r proof_of_inclusion)) ()) - ) - ) - - (defun calculate_db_current_puzzlehash (DB_SINGLETON_STRUCT DB_LAYER_MOD_HASH current_root db_innerpuz_hash) - (puzzle-hash-of-curried-function (f DB_SINGLETON_STRUCT) - (puzzle-hash-of-curried-function DB_LAYER_MOD_HASH - (sha256 1 db_innerpuz_hash) - (sha256 1 current_root) - (sha256 1 DB_LAYER_MOD_HASH) - ) - (sha256tree DB_SINGLETON_STRUCT) - ) - ) - - (if spend_type - ; CLAIM - (if (= current_root (calculate_merkle_root_for_merkle_proof LEAF_REVEAL inclusion_proof)) ; current root is validated in assert_puzzle_announcement below - (list (list CREATE_COIN CLAIM_TARGET my_amount) (list ASSERT_MY_AMOUNT my_amount) (list ASSERT_PUZZLE_ANNOUNCEMENT (sha256 (calculate_db_current_puzzlehash DB_SINGLETON_STRUCT DB_LAYER_MOD_HASH current_root db_innerpuz_hash) current_root))) - (x) - ) - ; RECOVER - (list (list CREATE_COIN RECOVERY_TARGET my_amount) (list ASSERT_MY_AMOUNT my_amount) (list ASSERT_SECONDS_RELATIVE RECOVERY_TIMELOCK)) - ) -) diff --git a/chia/wallet/puzzles/database_offer.clvm.hex b/chia/wallet/puzzles/database_offer.clvm.hex deleted file mode 100644 index 6fa7a696df..0000000000 --- a/chia/wallet/puzzles/database_offer.clvm.hex +++ /dev/null @@ -1 +0,0 @@ -ff02ffff01ff02ffff03ff82017fffff01ff02ffff03ffff09ff820bffffff02ff36ffff04ff02ffff04ff17ffff04ff8217ffff808080808080ffff01ff04ffff04ff34ffff04ff2fffff04ff8202ffff80808080ffff04ffff04ff10ffff04ff8202ffff808080ffff04ffff04ff28ffff04ffff0bffff02ff3affff04ff02ffff04ff0bffff04ff05ffff04ff820bffffff04ff8205ffff80808080808080ff820bff80ff808080ff80808080ffff01ff088080ff0180ffff01ff04ffff04ff34ffff04ff5fffff04ff8202ffff80808080ffff04ffff04ff10ffff04ff8202ffff808080ffff04ffff04ff38ffff04ff81bfff808080ff8080808080ff0180ffff04ffff01ffffff49ff3f50ffff0233ff0401ffffff0102ffff02ffff03ff05ffff01ff02ff2affff04ff02ffff04ff0dffff04ffff0bff32ffff0bff3cff2c80ffff0bff32ffff0bff32ffff0bff3cff2280ff0980ffff0bff32ff0bffff0bff3cff8080808080ff8080808080ffff010b80ff0180ff02ff2effff04ff02ffff04ff09ffff04ffff02ff2effff04ff02ffff04ff0bffff04ffff0bffff0101ff2f80ffff04ffff0bffff0101ff1780ffff04ffff0bffff0101ff0b80ff80808080808080ffff04ffff02ff3effff04ff02ffff04ff05ff80808080ff808080808080ffffff02ffff03ff17ffff01ff02ff26ffff04ff02ffff04ffff17ff05ffff0181ff80ffff04ffff02ffff03ffff18ff05ffff010180ffff01ff0bffff0102ff27ff0b80ffff01ff0bffff0102ff0bff278080ff0180ffff04ff37ff808080808080ffff010b80ff0180ff02ff26ffff04ff02ffff04ff13ffff04ffff02ff3effff04ff02ffff04ff05ff80808080ffff04ffff02ffff03ff1bffff012bff8080ff0180ff808080808080ffff0bff32ffff0bff3cff2480ffff0bff32ffff0bff32ffff0bff3cff2280ff0580ffff0bff32ffff02ff2affff04ff02ffff04ff07ffff04ffff0bff3cff3c80ff8080808080ffff0bff3cff8080808080ff02ffff03ffff07ff0580ffff01ff0bffff0102ffff02ff3effff04ff02ffff04ff09ff80808080ffff02ff3effff04ff02ffff04ff0dff8080808080ffff01ff0bffff0101ff058080ff0180ff018080 \ No newline at end of file diff --git a/chia/wallet/puzzles/database_offer.clvm.hex.sha256tree b/chia/wallet/puzzles/database_offer.clvm.hex.sha256tree deleted file mode 100644 index a86d8760a2..0000000000 --- a/chia/wallet/puzzles/database_offer.clvm.hex.sha256tree +++ /dev/null @@ -1 +0,0 @@ -6b5c5137fddf15424f97953a00111589838d42eb2bf314f1cc37e71144009881 diff --git a/chia/wallet/puzzles/singleton_top_layer_atari_only.clvm b/chia/wallet/puzzles/singleton_top_layer_atari_only.clvm deleted file mode 100644 index 3e4ade403c..0000000000 --- a/chia/wallet/puzzles/singleton_top_layer_atari_only.clvm +++ /dev/null @@ -1,173 +0,0 @@ -(mod (SINGLETON_STRUCT INNER_PUZZLE lineage_proof my_amount inner_solution) - -;; SINGLETON_STRUCT = (MOD_HASH . (LAUNCHER_ID . LAUNCHER_PUZZLE_HASH)) - -; SINGLETON_STRUCT, INNER_PUZZLE are curried in by the wallet - -; EXAMPLE SOLUTION '(0xfadeddab 0xdeadbeef 1 (0xdeadbeef 200) 50 ((51 0xfadeddab 100) (60 "trash") (51 deadbeef 0)))' - - -; This puzzle is a wrapper around an inner smart puzzle which guarantees uniqueness. -; It takes its singleton identity from a coin with a launcher puzzle which guarantees that it is unique. - - (include condition_codes.clvm) - (include curry-and-treehash.clinc) ; also imports the constant ONE == 1 - (include singleton_truths.clib) - - ; takes a lisp tree and returns the hash of it - (defun sha256tree1 (TREE) - (if (l TREE) - (sha256 2 (sha256tree1 (f TREE)) (sha256tree1 (r TREE))) - (sha256 ONE TREE) - ) - ) - - ; "assert" is a macro that wraps repeated instances of "if" - ; usage: (assert A0 A1 ... An R) - ; all of A0, A1, ... An must evaluate to non-null, or an exception is raised - ; return the value of R (if we get that far) - - (defmacro assert items - (if (r items) - (list if (f items) (c assert (r items)) (q . (x))) - (f items) - ) - ) - - (defun-inline mod_hash_for_singleton_struct (SINGLETON_STRUCT) (f SINGLETON_STRUCT)) - (defun-inline launcher_id_for_singleton_struct (SINGLETON_STRUCT) (f (r SINGLETON_STRUCT))) - (defun-inline launcher_puzzle_hash_for_singleton_struct (SINGLETON_STRUCT) (r (r SINGLETON_STRUCT))) - - ;; return the full puzzlehash for a singleton with the innerpuzzle curried in - ; puzzle-hash-of-curried-function is imported from curry-and-treehash.clinc - (defun-inline calculate_full_puzzle_hash (SINGLETON_STRUCT inner_puzzle_hash) - (puzzle-hash-of-curried-function (mod_hash_for_singleton_struct SINGLETON_STRUCT) - inner_puzzle_hash - (sha256tree1 SINGLETON_STRUCT) - ) - ) - - ; assembles information from the solution to create our own full ID including asserting our parent is a singleton - (defun-inline create_my_ID (SINGLETON_STRUCT full_puzzle_hash parent_parent parent_inner_puzzle_hash parent_amount my_amount) - (sha256 (sha256 parent_parent (calculate_full_puzzle_hash SINGLETON_STRUCT parent_inner_puzzle_hash) parent_amount) - full_puzzle_hash - my_amount) - ) - - ;; take a boolean and a non-empty list of conditions - ;; strip off the first condition if a boolean is set - ;; this is used to remove `(CREATE_COIN xxx -113)` - ;; pretty sneaky, eh? - (defun strip_first_condition_if (boolean condition_list) - (if boolean - (r condition_list) - condition_list - ) - ) - - (defun-inline morph_condition (condition SINGLETON_STRUCT) - (c (f condition) (c (calculate_full_puzzle_hash SINGLETON_STRUCT (f (r condition))) (r (r condition)))) - ) - - ;; return the value of the coin created if this is a `CREATE_COIN` condition, or 0 otherwise - (defun-inline created_coin_value_or_0 (condition) - (if (= (f condition) CREATE_COIN) - (f (r (r condition))) - 0 - ) - ) - - ;; Returns a (bool . bool) - (defun odd_cons_m113 (output_amount) - (c - (= (logand output_amount ONE) ONE) ;; is it odd? - (= output_amount -113) ;; is it the escape value? - ) - ) - - ; Assert exactly one output with odd value exists - ignore it if value is -113 - - ;; this function iterates over the output conditions from the inner puzzle & solution - ;; and both checks that exactly one unique singleton child is created (with odd valued output), - ;; and wraps the inner puzzle with this same singleton wrapper puzzle - ;; - ;; The special case where the output value is -113 means a child singleton is intentionally - ;; *NOT* being created, thus forever ending this singleton's existence - - (defun check_and_morph_conditions_for_singleton (SINGLETON_STRUCT conditions has_odd_output_been_found) - (if conditions - (morph_next_condition SINGLETON_STRUCT conditions has_odd_output_been_found (odd_cons_m113 (created_coin_value_or_0 (f conditions)))) - (if has_odd_output_been_found - 0 - (x) ;; no odd output found - ) - ) - ) - - ;; a continuation of `check_and_morph_conditions_for_singleton` with booleans `is_output_odd` and `is_output_m113` - ;; precalculated - (defun morph_next_condition (SINGLETON_STRUCT conditions has_odd_output_been_found (is_output_odd . is_output_m113)) - (assert - (not (all is_output_odd has_odd_output_been_found)) - (strip_first_condition_if - is_output_m113 - (c (if is_output_odd - (morph_condition (f conditions) SINGLETON_STRUCT) - (f conditions) - ) - (check_and_morph_conditions_for_singleton SINGLETON_STRUCT (r conditions) (any is_output_odd has_odd_output_been_found)) - ) - ) - ) - ) - - ; this final stager asserts our ID - ; it also runs the innerpuz with the innersolution with the "truths" added - ; it then passes that output conditions from the innerpuz to the morph conditions function - (defun-inline stager_three (SINGLETON_STRUCT my_id INNER_PUZZLE inner_solution) - (c (list ASSERT_MY_COIN_ID my_id) (check_and_morph_conditions_for_singleton SINGLETON_STRUCT (a INNER_PUZZLE inner_solution) 0)) - ) - - ; this checks whether we are an eve spend or not and calculates our full coin ID appropriately and passes it on to the final stager - ; if we are the eve spend it also adds the additional checks that our parent's puzzle is the standard launcher format and that out parent ID is the same as our singleton ID - - (defun-inline stager_two (SINGLETON_STRUCT lineage_proof full_puzhash my_amount INNER_PUZZLE inner_solution) - (stager_three - SINGLETON_STRUCT - (if (is_not_eve_proof lineage_proof) - (create_my_ID - SINGLETON_STRUCT - full_puzhash - (parent_info_for_lineage_proof lineage_proof) - (puzzle_hash_for_lineage_proof lineage_proof) - (amount_for_lineage_proof lineage_proof) - my_amount - ) - (if (= - (launcher_id_for_singleton_struct SINGLETON_STRUCT) - (sha256 (parent_info_for_eve_proof lineage_proof) (launcher_puzzle_hash_for_singleton_struct SINGLETON_STRUCT) (amount_for_eve_proof lineage_proof)) - ) - (sha256 (launcher_id_for_singleton_struct SINGLETON_STRUCT) full_puzhash my_amount) - (x) - ) - ) - INNER_PUZZLE - inner_solution - ) - ) - - ; this calculates our current full puzzle hash and passes it to stager two - (defun-inline stager_one (SINGLETON_STRUCT lineage_proof my_innerpuzhash my_amount INNER_PUZZLE inner_solution) - (stager_two SINGLETON_STRUCT lineage_proof (calculate_full_puzzle_hash SINGLETON_STRUCT my_innerpuzhash) my_amount INNER_PUZZLE inner_solution) - ) - - - ; main - - ; if our value is not an odd amount then we are invalid - ; this calculates my_innerpuzhash and passes all values to stager_one - (if (logand my_amount ONE) - (stager_one SINGLETON_STRUCT lineage_proof (sha256tree1 INNER_PUZZLE) my_amount INNER_PUZZLE inner_solution) - (x) - ) -) diff --git a/chia/wallet/puzzles/singleton_top_layer_atari_only.clvm.hex b/chia/wallet/puzzles/singleton_top_layer_atari_only.clvm.hex deleted file mode 100644 index d76cb2876e..0000000000 --- a/chia/wallet/puzzles/singleton_top_layer_atari_only.clvm.hex +++ /dev/null @@ -1 +0,0 @@ -ff02ffff01ff02ffff03ffff18ff2fff3480ffff01ff04ffff04ff10ffff04ffff02ffff03ff77ffff01ff0bffff0bff27ffff02ff36ffff04ff02ffff04ff09ffff04ff57ffff04ffff02ff2effff04ff02ffff04ff05ff80808080ff808080808080ff81b780ffff02ff36ffff04ff02ffff04ff09ffff04ffff02ff2effff04ff02ffff04ff0bff80808080ffff04ffff02ff2effff04ff02ffff04ff05ff80808080ff808080808080ff2f80ffff01ff02ffff03ffff09ff15ffff0bff27ff1dff578080ffff01ff0bff15ffff02ff36ffff04ff02ffff04ff09ffff04ffff02ff2effff04ff02ffff04ff0bff80808080ffff04ffff02ff2effff04ff02ffff04ff05ff80808080ff808080808080ff2f80ffff01ff088080ff018080ff0180ff808080ffff02ff2affff04ff02ffff04ff05ffff04ffff02ff0bff5f80ffff01ff80808080808080ffff01ff088080ff0180ffff04ffff01ffffff46ff0233ffff0401ff0102ffffff02ffff03ff05ffff01ff02ff12ffff04ff02ffff04ff0dffff04ffff0bff3cffff0bff34ff2480ffff0bff3cffff0bff3cffff0bff34ff2c80ff0980ffff0bff3cff0bffff0bff34ff8080808080ff8080808080ffff010b80ff0180ffff02ffff03ff0bffff01ff02ff3affff04ff02ffff04ff05ffff04ff0bffff04ff17ffff04ffff02ff26ffff04ff02ffff04ffff02ffff03ffff09ff23ff3880ffff0181b3ff8080ff0180ff80808080ff80808080808080ffff01ff02ffff03ff17ff80ffff01ff088080ff018080ff0180ff02ffff03ffff20ffff22ff4fff178080ffff01ff02ff3effff04ff02ffff04ff6fffff04ffff04ffff02ffff03ff4fffff01ff04ff23ffff04ffff02ff36ffff04ff02ffff04ff09ffff04ff53ffff04ffff02ff2effff04ff02ffff04ff05ff80808080ff808080808080ff738080ffff011380ff0180ffff02ff2affff04ff02ffff04ff05ffff04ff1bffff04ffff21ff4fff1780ff80808080808080ff8080808080ffff01ff088080ff0180ffffff04ffff09ffff18ff05ff3480ff3480ffff09ff05ffff01818f8080ff0bff3cffff0bff34ff2880ffff0bff3cffff0bff3cffff0bff34ff2c80ff0580ffff0bff3cffff02ff12ffff04ff02ffff04ff07ffff04ffff0bff34ff3480ff8080808080ffff0bff34ff8080808080ffff02ffff03ffff07ff0580ffff01ff0bffff0102ffff02ff2effff04ff02ffff04ff09ff80808080ffff02ff2effff04ff02ffff04ff0dff8080808080ffff01ff0bff34ff058080ff0180ff02ffff03ff05ffff011bffff010b80ff0180ff018080 \ No newline at end of file diff --git a/chia/wallet/puzzles/singleton_top_layer_atari_only.clvm.hex.sha256tree b/chia/wallet/puzzles/singleton_top_layer_atari_only.clvm.hex.sha256tree deleted file mode 100644 index 01f28166d0..0000000000 --- a/chia/wallet/puzzles/singleton_top_layer_atari_only.clvm.hex.sha256tree +++ /dev/null @@ -1 +0,0 @@ -ffc0846eea5898e9aefea956b7d0e08df97a1c080910648270d979a4900d88aa From 38f87a17d078a8d280c64ed11cf1e4b0ad0010d7 Mon Sep 17 00:00:00 2001 From: Matt Hauff Date: Mon, 11 Jul 2022 15:34:49 -0500 Subject: [PATCH 42/62] get rid of a bunch of stuff --- chia/data_layer/data_layer_wallet.py | 111 ---------- chia/wallet/db_wallet/db_wallet_puzzles.py | 69 ------- tests/wallet/db_wallet/test_db_clvm.py | 188 +---------------- tests/wallet/db_wallet/test_dl_wallet.py | 228 +-------------------- 4 files changed, 6 insertions(+), 590 deletions(-) diff --git a/chia/data_layer/data_layer_wallet.py b/chia/data_layer/data_layer_wallet.py index f65bf7b4f1..a8c25e0f62 100644 --- a/chia/data_layer/data_layer_wallet.py +++ b/chia/data_layer/data_layer_wallet.py @@ -516,96 +516,6 @@ class DataLayerWallet: ) return txs - async def create_report_spend( - self, - launcher_id: bytes32, - fee: uint64 = uint64(0), - ) -> Tuple[List[TransactionRecord], Announcement]: - singleton_record, parent_lineage = await self.get_spendable_singleton_info(launcher_id) - - # Create the puzzle - current_full_puz = create_host_fullpuz( - singleton_record.inner_puzzle_hash, - singleton_record.root, - launcher_id, - ) - - # Create the solution - assert singleton_record.lineage_proof.parent_name is not None - assert singleton_record.lineage_proof.amount is not None - db_layer_sol = Program.to([1, singleton_record.lineage_proof.amount, []]) - full_sol = Program.to( - [ - parent_lineage.to_program(), - singleton_record.lineage_proof.amount, - db_layer_sol, - ] - ) - - # Create the spend - current_coin = Coin( - singleton_record.lineage_proof.parent_name, - current_full_puz.get_tree_hash(), - singleton_record.lineage_proof.amount, - ) - coin_spend = CoinSpend( - current_coin, - SerializedProgram.from_program(current_full_puz), - SerializedProgram.from_program(full_sol), - ) - spend_bundle = SpendBundle([coin_spend], G2Element()) - expected_announcement = Announcement(current_full_puz.get_tree_hash(), singleton_record.root) - - # Create the relevant records - dl_tx = TransactionRecord( - confirmed_at_height=uint32(0), - created_at_time=uint64(int(time.time())), - to_puzzle_hash=singleton_record.inner_puzzle_hash, - amount=uint64(singleton_record.lineage_proof.amount), - fee_amount=uint64(0), - confirmed=False, - sent=uint32(10), - spend_bundle=spend_bundle, - additions=spend_bundle.additions(), - removals=spend_bundle.removals(), - memos=list(compute_memos(spend_bundle).items()), - wallet_id=self.id(), - sent_to=[], - trade_id=None, - type=uint32(TransactionType.OUTGOING_TX.value), - name=singleton_record.coin_id, - ) - if fee > 0: - chia_tx = await self.create_tandem_xch_tx(fee, expected_announcement, coin_announcement=False) - aggregate_bundle = SpendBundle.aggregate([dl_tx.spend_bundle, chia_tx.spend_bundle]) - dl_tx = dataclasses.replace(dl_tx, spend_bundle=aggregate_bundle) - chia_tx = dataclasses.replace(chia_tx, spend_bundle=None) - txs: List[TransactionRecord] = [dl_tx, chia_tx] - else: - txs = [dl_tx] - new_singleton_record = SingletonRecord( - coin_id=Coin( - current_coin.name(), current_full_puz.get_tree_hash(), singleton_record.lineage_proof.amount - ).name(), - launcher_id=launcher_id, - root=singleton_record.root, - confirmed=False, - confirmed_at_height=uint32(0), - timestamp=uint64(0), - inner_puzzle_hash=singleton_record.inner_puzzle_hash, - lineage_proof=LineageProof( - singleton_record.coin_id, - create_host_layer_puzzle( - singleton_record.inner_puzzle_hash, - singleton_record.root, - ).get_tree_hash(), - singleton_record.lineage_proof.amount, - ), - generation=uint32(singleton_record.generation + 1), - ) - await self.wallet_state_manager.dl_store.add_singleton_record(new_singleton_record, False) - return txs, expected_announcement - async def get_spendable_singleton_info(self, launcher_id: bytes32) -> Tuple[SingletonRecord, LineageProof]: # First, let's make sure this is a singleton that we track and that we can spend singleton_record: Optional[SingletonRecord] = await self.get_latest_singleton(launcher_id) @@ -844,27 +754,6 @@ class DataLayerWallet: await self.wallet_state_manager.dl_store.delete_singleton_records_by_launcher_id(launcher_id) await self.wallet_state_manager.dl_store.delete_launcher(launcher_id) - ############# - # DL OFFERS # - ############# - - async def get_info_for_offer_claim( - self, - launcher_id: bytes32, - ) -> Tuple[Program, bytes32, bytes32]: - singleton_record: Optional[SingletonRecord] = await self.get_latest_singleton(launcher_id) - if singleton_record is None: - raise ValueError(f"Singleton with launcher ID {launcher_id} is not tracked by DL Wallet") - elif not singleton_record.confirmed: - raise ValueError(f"Singleton with launcher ID {launcher_id} is in an unconfirmed state") - - current_full_puz = create_host_fullpuz( - singleton_record.inner_puzzle_hash, - singleton_record.root, - launcher_id, - ) - return current_full_puz, singleton_record.inner_puzzle_hash, singleton_record.root - ########### # UTILITY # ########### diff --git a/chia/wallet/db_wallet/db_wallet_puzzles.py b/chia/wallet/db_wallet/db_wallet_puzzles.py index ae466671db..97beccc819 100644 --- a/chia/wallet/db_wallet/db_wallet_puzzles.py +++ b/chia/wallet/db_wallet/db_wallet_puzzles.py @@ -12,10 +12,6 @@ from chia.wallet.puzzles.load_clvm import load_clvm SINGLETON_TOP_LAYER_MOD = load_clvm("singleton_top_layer_atari_only.clvm") # TODO: need new data layer specific clvm SINGLETON_LAUNCHER = load_clvm("singleton_launcher.clvm") -DB_HOST_MOD = load_clvm("database_layer.clvm") -DB_OFFER_MOD = load_clvm("database_offer.clvm") - -DB_HOST_MOD_HASH = DB_HOST_MOD.get_tree_hash() def create_host_fullpuz(innerpuz_hash: bytes32, current_root: bytes32, genesis_id: bytes32) -> Program: @@ -31,71 +27,6 @@ def create_host_layer_puzzle(innerpuz_hash: bytes32, current_root: bytes32) -> P return db_layer -def solve_data_layer_to_report(amount: uint64) -> Program: - # https://github.com/Chia-Network/clvm/pull/102 - # https://github.com/Chia-Network/clvm/pull/106 - return Program.to( # type: ignore[no-any-return] - [ - 1, - amount, - [], - ] - ) - - -def solve_data_layer_to_update(inner_puzzle: Program, inner_solution: Program) -> Program: - # https://github.com/Chia-Network/clvm/pull/102 - # https://github.com/Chia-Network/clvm/pull/106 - return Program.to( # type: ignore[no-any-return] - [ - 0, - inner_solution, - inner_puzzle, - ] - ) - - -def create_offer_fullpuz( - leaf_reveal: bytes, - host_genesis_id: bytes32, - claim_target: bytes32, - recovery_target: bytes32, - recovery_timelock: uint64, -) -> Program: - mod_hash = SINGLETON_TOP_LAYER_MOD.get_tree_hash() - # singleton_struct = (MOD_HASH . (LAUNCHER_ID . LAUNCHER_PUZZLE_HASH)) - singleton_struct = Program.to((mod_hash, (host_genesis_id, SINGLETON_LAUNCHER.get_tree_hash()))) - full_puz = DB_OFFER_MOD.curry( - DB_HOST_MOD_HASH, singleton_struct, leaf_reveal, claim_target, recovery_target, recovery_timelock - ) - return full_puz - - -def solve_dl_offer_for_claim( - offer_amount: uint64, inner_puzzle_hash: bytes32, root: bytes32, proof_of_inclusion: Program -) -> Program: - # https://github.com/Chia-Network/clvm/pull/102 - # https://github.com/Chia-Network/clvm/pull/106 - return Program.to( # type: ignore[no-any-return] - [ - 1, - offer_amount, - inner_puzzle_hash, - root, - proof_of_inclusion, - ] - ) - - -def solve_dl_offer_for_recover(offer_amount: uint64) -> Program: - # https://github.com/Chia-Network/clvm/pull/102 - # https://github.com/Chia-Network/clvm/pull/106 - return Program.to( # type: ignore[no-any-return] - [ - 0, - offer_amount, - ] - ) def match_dl_singleton(puzzle: Program) -> Tuple[bool, Iterator[Program]]: diff --git a/tests/wallet/db_wallet/test_db_clvm.py b/tests/wallet/db_wallet/test_db_clvm.py index 49d2e7ff40..ba098a78c9 100644 --- a/tests/wallet/db_wallet/test_db_clvm.py +++ b/tests/wallet/db_wallet/test_db_clvm.py @@ -9,11 +9,6 @@ from chia.wallet.db_wallet.db_wallet_puzzles import ( create_host_fullpuz, create_offer_fullpuz, SINGLETON_LAUNCHER, - create_host_layer_puzzle, - solve_data_layer_to_report, - solve_data_layer_to_update, - solve_dl_offer_for_claim, - solve_dl_offer_for_recover, ) from chia.wallet.lineage_proof import LineageProof from chia.wallet.puzzles.singleton_top_layer import solution_for_singleton @@ -125,35 +120,6 @@ class TestDLLifecycle: bad_puzzle, ) - @pytest.mark.asyncio() - async def test_report(self, setup_sim_and_singleton: SetupArgs) -> None: - sim, sim_client, singleton, lineage_proof = setup_sim_and_singleton[0:4] - - try: - bundle = SpendBundle( - [ - CoinSpend( - singleton, - create_host_fullpuz(ACS_PH, self.get_merkle_root("init"), singleton.parent_coin_info), - solution_for_singleton( - lineage_proof, - singleton.amount, - solve_data_layer_to_report(singleton.amount), - ), - ) - ], - G2Element(), - ) - result = (await sim_client.push_tx(bundle))[0] - assert result == MempoolInclusionStatus.SUCCESS - self.cost["report spend"] = cost_of_spend_bundle(bundle) - await sim.farm_block() - new_singleton = (await sim_client.get_coin_records_by_parent_ids([singleton.name()]))[0].coin - assert new_singleton.puzzle_hash == singleton.puzzle_hash - finally: - # https://github.com/Chia-Network/chia-blockchain/pull/11819 - await sim.close() # type: ignore[no-untyped-call] - @pytest.mark.asyncio() async def test_update(self, setup_sim_and_singleton: SetupArgs) -> None: sim, sim_client, singleton, lineage_proof = setup_sim_and_singleton[0:4] @@ -167,19 +133,11 @@ class TestDLLifecycle: solution_for_singleton( lineage_proof, singleton.amount, - solve_data_layer_to_update( - ACS, - Program.to( - [ - [ - 51, - create_host_layer_puzzle( - ACS_PH, self.get_merkle_root("update") - ).get_tree_hash(), - singleton.amount, - ] - ] - ), + Program.to( + [[ + [51, ACS_PH, singleton.amount], + [-24, ACS, Program.to((self.get_merkle_root("update"), None))] + ]] ), ), ) @@ -201,142 +159,6 @@ class TestDLLifecycle: # https://github.com/Chia-Network/chia-blockchain/pull/11819 await sim.close() # type: ignore[no-untyped-call] - @pytest.mark.asyncio() - async def test_offer_cant_claim(self, setup_sim_and_singleton: SetupArgs) -> None: - ( - sim, - sim_client, - singleton, - lineage_proof, - good_offer_coin, - bad_offer_coin, - good_offer_puzzle, - bad_offer_puzzle, - ) = setup_sim_and_singleton - - try: - bundle = SpendBundle( - [ - CoinSpend( - singleton, - create_host_fullpuz(ACS_PH, self.get_merkle_root("init"), singleton.parent_coin_info), - solution_for_singleton( - lineage_proof, - singleton.amount, - solve_data_layer_to_report(singleton.amount), - ), - ), - CoinSpend( - bad_offer_coin, - bad_offer_puzzle, - solve_dl_offer_for_claim( - OFFER_AMOUNT, - ACS_PH, - self.get_merkle_root("init"), - self.get_merkle_proof("nope"), - ), - ), - ], - G2Element(), - ) - result = (await sim_client.push_tx(bundle))[0] - assert result == MempoolInclusionStatus.FAILED - offer_cs = bundle.coin_spends[1] - with pytest.raises(ValueError, match="clvm raise"): - offer_cs.puzzle_reveal.to_program().run(offer_cs.solution.to_program()) - finally: - # https://github.com/Chia-Network/chia-blockchain/pull/11819 - await sim.close() # type: ignore[no-untyped-call] - - @pytest.mark.asyncio() - async def test_offer_can_claim(self, setup_sim_and_singleton: SetupArgs) -> None: - ( - sim, - sim_client, - singleton, - lineage_proof, - good_offer_coin, - bad_offer_coin, - good_offer_puzzle, - bad_offer_puzzle, - ) = setup_sim_and_singleton - - try: - bundle = SpendBundle( - [ - CoinSpend( - singleton, - create_host_fullpuz(ACS_PH, self.get_merkle_root("init"), singleton.parent_coin_info), - solution_for_singleton( - lineage_proof, - singleton.amount, - solve_data_layer_to_report(singleton.amount), - ), - ), - CoinSpend( - good_offer_coin, - good_offer_puzzle, - solve_dl_offer_for_claim( - OFFER_AMOUNT, - ACS_PH, - self.get_merkle_root("init"), - self.get_merkle_proof("init"), - ), - ), - ], - G2Element(), - ) - result = (await sim_client.push_tx(bundle))[0] - assert result == MempoolInclusionStatus.SUCCESS - self.cost["offer claim"] = cost_of_spend_bundle(bundle) - await sim.farm_block() - offer_reward = (await sim_client.get_coin_records_by_parent_ids([good_offer_coin.name()]))[0].coin - assert offer_reward.puzzle_hash == ACS_2_PH - finally: - # https://github.com/Chia-Network/chia-blockchain/pull/11819 - await sim.close() # type: ignore[no-untyped-call] - - @pytest.mark.asyncio() - async def test_offer_recovery(self, setup_sim_and_singleton: SetupArgs) -> None: - ( - sim, - sim_client, - singleton, - lineage_proof, - good_offer_coin, - bad_offer_coin, - good_offer_puzzle, - bad_offer_puzzle, - ) = setup_sim_and_singleton - - try: - bundle = SpendBundle( - [ - CoinSpend( - bad_offer_coin, - bad_offer_puzzle, - solve_dl_offer_for_recover(OFFER_AMOUNT), - ), - ], - G2Element(), - ) - result = (await sim_client.push_tx(bundle))[0] - assert result == MempoolInclusionStatus.FAILED - - # Should work after a minute - sim.pass_time(uint64(60)) - await sim.farm_block() - result = (await sim_client.push_tx(bundle))[0] - assert result == MempoolInclusionStatus.SUCCESS - self.cost["offer recovery"] = cost_of_spend_bundle(bundle) - await sim.farm_block() - - offer_reward = (await sim_client.get_coin_records_by_parent_ids([bad_offer_coin.name()]))[0].coin - assert offer_reward.puzzle_hash == ACS_PH - finally: - # https://github.com/Chia-Network/chia-blockchain/pull/11819 - await sim.close() # type: ignore[no-untyped-call] - def test_cost(self) -> None: import json import logging diff --git a/tests/wallet/db_wallet/test_dl_wallet.py b/tests/wallet/db_wallet/test_dl_wallet.py index c2982f4a7d..b54a8788c9 100644 --- a/tests/wallet/db_wallet/test_dl_wallet.py +++ b/tests/wallet/db_wallet/test_dl_wallet.py @@ -306,24 +306,6 @@ class TestDLWallet: await time_out_assert(10, wallet_0.get_confirmed_balance, funds - 2000000000000) await asyncio.sleep(0.5) - for _ in range(0, 2): - current_record = await dl_wallet.get_latest_singleton(launcher_id) - txs, _ = await dl_wallet.create_report_spend(launcher_id, fee=uint64(2000000000000)) - new_record = await dl_wallet.get_latest_singleton(launcher_id) - assert new_record is not None - assert new_record != current_record - assert not new_record.confirmed - - for tx in txs: - await wallet_node_0.wallet_state_manager.add_pending_transaction(tx) - await full_node_api.process_transaction_records(records=txs) - - await time_out_assert(15, is_singleton_confirmed, True, dl_wallet, launcher_id) - await asyncio.sleep(0.5) - - await time_out_assert(10, wallet_0.get_unconfirmed_balance, funds - 6000000000000) - await time_out_assert(10, wallet_0.get_confirmed_balance, funds - 6000000000000) - previous_record = await dl_wallet.get_latest_singleton(launcher_id) new_root = MerkleTree([Program.to("new root").get_tree_hash()]).calculate_root() @@ -409,7 +391,7 @@ class TestDLWallet: await asyncio.sleep(0.5) # Because these have the same fee, the one that gets pushed first will win - report_txs, _ = await dl_wallet_1.create_report_spend(launcher_id, fee=uint64(2000000000000)) + report_txs, _ = await dl_wallet_1.create_update_state_spend(launcher_id, current_record.root, fee=uint64(2000000000000)) record_1 = await dl_wallet_1.get_latest_singleton(launcher_id) assert record_1 is not None assert current_record != record_1 @@ -513,211 +495,3 @@ class TestDLWallet: for tx in update_txs_0: assert await wallet_node_0.wallet_state_manager.tx_store.get_transaction_record(tx.name) is None assert await dl_wallet_0.get_singleton_record(record_0.coin_id) is None - - # @pytest.mark.skip(reason="DLO Wallet is not supported yet") - # @pytest.mark.asyncio - # async def test_dlo_wallet(self, three_wallet_nodes: SimulatorsAndWallets) -> None: - # raise # for ignoring mypy :) - # time_lock = uint64(10) - # full_nodes, wallets = three_wallet_nodes - # full_node_api = full_nodes[0] - # full_node_api.time_per_block = 2 * time_lock - # full_node_server = full_node_api.server - # wallet_node_0, server_0 = wallets[0] - # wallet_node_1, server_1 = wallets[1] - # wallet_node_2, server_2 = wallets[2] - # assert wallet_node_0.wallet_state_manager is not None - # assert wallet_node_1.wallet_state_manager is not None - # assert wallet_node_2.wallet_state_manager is not None - # wallet_0 = wallet_node_0.wallet_state_manager.main_wallet - # wallet_1 = wallet_node_1.wallet_state_manager.main_wallet - # wallet_2 = wallet_node_2.wallet_state_manager.main_wallet - # - # ph2 = await wallet_2.get_new_puzzlehash() - # - # await server_0.start_client(PeerInfo("localhost", uint16(full_node_server._port)), None) - # await server_1.start_client(PeerInfo("localhost", uint16(full_node_server._port)), None) - # await server_2.start_client(PeerInfo("localhost", uint16(full_node_server._port)), None) - # - # funds = await full_node_api.farm_blocks(count=1, wallet=wallet_0) - # - # await time_out_assert(10, wallet_0.get_unconfirmed_balance, funds) - # await time_out_assert(10, wallet_0.get_confirmed_balance, funds) - # - # nodes = [Program.to("thing").get_tree_hash(), Program.to([8]).get_tree_hash()] - # current_tree = MerkleTree(nodes) - # current_root = current_tree.calculate_root() - # - # # Wallet1 sets up DLWallet1 - # async with wallet_node_0.wallet_state_manager.lock: - # creation_record = await DataLayerWallet.create_new_dl_wallet( - # wallet_node_0.wallet_state_manager, wallet_0, uint64(101), current_root - # ) - # - # dl_wallet_0: DataLayerWallet = creation_record.item - # - # await full_node_api.process_transaction_records(records=creation_record.transaction_records) - # - # await time_out_assert(15, dl_wallet_0.get_confirmed_balance, 101) - # await time_out_assert(15, dl_wallet_0.get_unconfirmed_balance, 101) - # - # # Wallet1 sets up DLOWallet1 - # async with wallet_node_1.wallet_state_manager.lock: - # dlo_wallet_1: DLOWallet = await DLOWallet.create_new_dlo_wallet( - # wallet_node_1.wallet_state_manager, - # wallet_1, - # ) - # - # await full_node_api.farm_blocks(count=2, wallet=wallet_1) - # - # await time_out_assert(15, dlo_wallet_1.get_confirmed_balance, 0) - # await time_out_assert(15, dlo_wallet_1.get_unconfirmed_balance, 0) - # assert dl_wallet_0.dl_info.origin_coin is not None - # tr = await dlo_wallet_1.generate_datalayer_offer_spend( - # amount=uint64(201), - # leaf_reveal=Program.to("thing").get_tree_hash(), - # host_genesis_id=dl_wallet_0.dl_info.origin_coin.name(), - # claim_target=await wallet_2.get_new_puzzlehash(), - # recovery_target=await wallet_1.get_new_puzzlehash(), - # recovery_timelock=time_lock, - # ) - # await wallet_1.push_transaction(tr) - # await full_node_api.process_transaction_records(records=[tr]) - # - # await time_out_assert(15, dlo_wallet_1.get_confirmed_balance, 201) - # await time_out_assert(15, dlo_wallet_1.get_unconfirmed_balance, 201) - # - # # create a second DLO Wallet and claim the coin - # async with wallet_node_2.wallet_state_manager.lock: - # dlo_wallet_2: DLOWallet = await DLOWallet.create_new_dlo_wallet( - # wallet_node_2.wallet_state_manager, - # wallet_2, - # ) - # - # offer_coin = await dlo_wallet_1.get_coin() - # offer_full_puzzle = dlo_wallet_1.puzzle_for_pk(0x00) - # db_puzzle, db_innerpuz, current_root = await dl_wallet_0.get_info_for_offer_claim() - # inclusion_proof = current_tree.generate_proof(Program.to("thing").get_tree_hash()) - # assert db_innerpuz is not None - # sb2 = await dlo_wallet_2.claim_dl_offer( - # offer_coin, - # offer_full_puzzle, - # db_innerpuz.get_tree_hash(), - # current_root, - # inclusion_proof, - # ) - # sb = await dl_wallet_0.create_report_spend() - # sb = SpendBundle.aggregate([sb2, sb]) - # tr = TransactionRecord( - # confirmed_at_height=uint32(0), - # created_at_time=uint64(int(time.time())), - # to_puzzle_hash=ph2, - # amount=uint64(201), - # fee_amount=uint64(0), - # confirmed=False, - # sent=uint32(0), - # spend_bundle=sb, - # additions=sb.additions(), - # removals=sb.removals(), - # memos=list(sb.get_memos().items()), - # wallet_id=dl_wallet_0.id(), - # sent_to=[], - # trade_id=None, - # type=uint32(TransactionType.OUTGOING_TX.value), - # name=sb.name(), - # ) - # await wallet_2.push_transaction(tr) - # await full_node_api.process_transaction_records(records=[tr]) - # - # await time_out_assert(15, wallet_2.get_confirmed_balance, 201) - # await time_out_assert(15, wallet_2.get_unconfirmed_balance, 201) - # - # @pytest.mark.skip(reason="DLO wallet is not supported yet") - # @pytest.mark.asyncio - # async def test_dlo_wallet_reclaim(self, three_wallet_nodes: SimulatorsAndWallets) -> None: - # raise # for ignoring mypy :) - # time_lock = uint64(10) - # - # full_nodes, wallets = three_wallet_nodes - # full_node_api = full_nodes[0] - # full_node_api.time_per_block = 2 * time_lock - # full_node_server = full_node_api.server - # wallet_node_0, server_0 = wallets[0] - # wallet_node_1, server_1 = wallets[1] - # wallet_node_2, server_2 = wallets[2] - # assert wallet_node_0.wallet_state_manager is not None - # assert wallet_node_1.wallet_state_manager is not None - # assert wallet_node_2.wallet_state_manager is not None - # wallet_0 = wallet_node_0.wallet_state_manager.main_wallet - # wallet_1 = wallet_node_1.wallet_state_manager.main_wallet - # wallet_2 = wallet_node_2.wallet_state_manager.main_wallet - # - # await server_0.start_client(PeerInfo("localhost", uint16(full_node_server._port)), None) - # await server_1.start_client(PeerInfo("localhost", uint16(full_node_server._port)), None) - # await server_2.start_client(PeerInfo("localhost", uint16(full_node_server._port)), None) - # - # funds = await full_node_api.farm_blocks(count=1, wallet=wallet_0) - # - # await time_out_assert(10, wallet_0.get_unconfirmed_balance, funds) - # await time_out_assert(10, wallet_0.get_confirmed_balance, funds) - # - # nodes = [Program.to("thing").get_tree_hash(), Program.to([8]).get_tree_hash()] - # current_tree = MerkleTree(nodes) - # current_root = current_tree.calculate_root() - # - # # Wallet1 sets up DLWallet1 - # async with wallet_node_0.wallet_state_manager.lock: - # creation_record = await DataLayerWallet.create_new_dl_wallet( - # wallet_node_0.wallet_state_manager, wallet_0, uint64(101), current_root - # ) - # - # dl_wallet_0: DataLayerWallet = creation_record.item - # - # await full_node_api.process_transaction_records(records=creation_record.transaction_records) - # - # await time_out_assert(15, dl_wallet_0.get_confirmed_balance, 101) - # await time_out_assert(15, dl_wallet_0.get_unconfirmed_balance, 101) - # - # # Wallet1 sets up DLOWallet1 - # async with wallet_node_1.wallet_state_manager.lock: - # dlo_wallet_1: DLOWallet = await DLOWallet.create_new_dlo_wallet( - # wallet_node_1.wallet_state_manager, - # wallet_1, - # ) - # - # wallet_1_funds = await full_node_api.farm_blocks(count=1, wallet=wallet_1) - # offer_amount = 201 - # - # await time_out_assert(15, dlo_wallet_1.get_confirmed_balance, 0) - # await time_out_assert(15, dlo_wallet_1.get_unconfirmed_balance, 0) - # assert dl_wallet_0.dl_info.origin_coin is not None - # tr = await dlo_wallet_1.generate_datalayer_offer_spend( - # amount=uint64(offer_amount), - # leaf_reveal=Program.to("thing").get_tree_hash(), - # host_genesis_id=dl_wallet_0.dl_info.origin_coin.name(), - # claim_target=await wallet_2.get_new_puzzlehash(), - # recovery_target=await wallet_1.get_new_puzzlehash(), - # recovery_timelock=time_lock, - # ) - # await wallet_1.push_transaction(tr) - # await full_node_api.process_transaction_records(records=[tr]) - # - # await time_out_assert(15, dlo_wallet_1.get_confirmed_balance, offer_amount) - # await time_out_assert(15, dlo_wallet_1.get_unconfirmed_balance, offer_amount) - # wallet_1_funds -= offer_amount - # - # await time_out_assert(15, wallet_1.get_confirmed_balance, wallet_1_funds) - # await time_out_assert(15, wallet_1.get_unconfirmed_balance, wallet_1_funds) - # - # transaction_record = await dlo_wallet_1.create_recover_dl_offer_spend() - # # Process a block to make sure the time lock for the offer has passed - # await full_node_api.process_blocks(count=1) - # - # await full_node_api.process_transaction_records(records=[transaction_record]) - # - # wallet_1_funds += offer_amount - # - # await time_out_assert(15, dlo_wallet_1.get_confirmed_balance, 0) - # await time_out_assert(15, dlo_wallet_1.get_unconfirmed_balance, 0) - # await time_out_assert(15, wallet_1.get_confirmed_balance, wallet_1_funds) - # await time_out_assert(15, wallet_1.get_unconfirmed_balance, wallet_1_funds) From 483e4d3e34c85e7fd9562be525a82a8da5779c40 Mon Sep 17 00:00:00 2001 From: Matt Hauff Date: Mon, 11 Jul 2022 15:35:11 -0500 Subject: [PATCH 43/62] Fix CLVM tests --- tests/clvm/test_clvm_compilation.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/clvm/test_clvm_compilation.py b/tests/clvm/test_clvm_compilation.py index b3119337d4..6c2e4b9ba6 100644 --- a/tests/clvm/test_clvm_compilation.py +++ b/tests/clvm/test_clvm_compilation.py @@ -50,9 +50,6 @@ wallet_program_files = set( "chia/wallet/puzzles/nft_state_layer.clvm", "chia/wallet/puzzles/nft_ownership_layer.clvm", "chia/wallet/puzzles/nft_ownership_transfer_program_one_way_claim_with_royalties.clvm", - "chia/wallet/puzzles/database_offer.clvm", - "chia/wallet/puzzles/database_layer.clvm", - "chia/wallet/puzzles/singleton_top_layer_atari_only.clvm", "chia/wallet/puzzles/graftroot_dl_offers.clvm", ] ) From 1bb162a02383edb4b1fb029add88bed1d3a410d5 Mon Sep 17 00:00:00 2001 From: Matt Hauff Date: Mon, 11 Jul 2022 15:46:34 -0500 Subject: [PATCH 44/62] Pivot DL -> NFT --- chia/data_layer/data_layer_wallet.py | 29 +++++++++++++--------- chia/wallet/db_wallet/db_wallet_puzzles.py | 29 +++++++++++----------- tests/wallet/db_wallet/test_db_clvm.py | 6 ++--- 3 files changed, 34 insertions(+), 30 deletions(-) diff --git a/chia/data_layer/data_layer_wallet.py b/chia/data_layer/data_layer_wallet.py index a8c25e0f62..5bf6b73ff9 100644 --- a/chia/data_layer/data_layer_wallet.py +++ b/chia/data_layer/data_layer_wallet.py @@ -12,6 +12,8 @@ from blspy import G2Element from chia.consensus.block_record import BlockRecord from chia.protocols.wallet_protocol import PuzzleSolutionResponse, CoinState from chia.wallet.db_wallet.db_wallet_puzzles import ( + ACS, + ACS_PH, create_host_fullpuz, SINGLETON_LAUNCHER, create_host_layer_puzzle, @@ -151,7 +153,7 @@ class DataLayerWallet: # Now let's check that the full puzzle is an odd data layer singleton if ( - full_puzhash != create_host_fullpuz(inner_puzhash, root, launcher_spend.coin.name()).get_tree_hash() + full_puzhash != create_host_fullpuz(inner_puzhash, root, launcher_spend.coin.name()).get_tree_hash(inner_puzhash) or amount % 2 == 0 ): return False, None @@ -267,7 +269,7 @@ class DataLayerWallet: timestamp=timestamp, lineage_proof=LineageProof( launcher_id, - create_host_layer_puzzle(inner_puzhash, root).get_tree_hash(), + create_host_layer_puzzle(inner_puzhash, root).get_tree_hash(inner_puzhash), amount, ), generation=uint32(0), @@ -311,7 +313,7 @@ class DataLayerWallet: launcher_coin: Coin = Coin(launcher_parent.name(), SINGLETON_LAUNCHER.get_tree_hash(), uint64(1)) inner_puzzle: Program = await self.standard_wallet.get_new_puzzle() - full_puzzle: Program = create_host_fullpuz(inner_puzzle.get_tree_hash(), initial_root, launcher_coin.name()) + full_puzzle: Program = create_host_fullpuz(inner_puzzle, initial_root, launcher_coin.name()) genesis_launcher_solution: Program = Program.to( [full_puzzle.get_tree_hash(), 1, [initial_root, inner_puzzle.get_tree_hash()]] @@ -368,7 +370,7 @@ class DataLayerWallet: timestamp=uint64(0), lineage_proof=LineageProof( launcher_coin.name(), - create_host_layer_puzzle(inner_puzzle.get_tree_hash(), initial_root).get_tree_hash(), + create_host_layer_puzzle(inner_puzzle, initial_root).get_tree_hash(), uint64(1), ), generation=uint32(0), @@ -417,13 +419,13 @@ class DataLayerWallet: # Make the child's puzzles next_inner_puzzle: Program = await self.standard_wallet.get_new_puzzle(in_transaction=in_transaction) - next_db_layer_puzzle: Program = create_host_layer_puzzle(next_inner_puzzle.get_tree_hash(), root_hash) - next_full_puz = create_host_fullpuz(next_inner_puzzle.get_tree_hash(), root_hash, launcher_id) + next_db_layer_puzzle: Program = create_host_layer_puzzle(next_inner_puzzle, root_hash) + next_full_puz = create_host_fullpuz(next_inner_puzzle, root_hash, launcher_id) # Construct the current puzzles current_inner_puzzle: Program = self.standard_wallet.puzzle_for_pk(inner_puzzle_derivation.pubkey) current_full_puz = create_host_fullpuz( - current_inner_puzzle.get_tree_hash(), + current_inner_puzzle, singleton_record.root, launcher_id, ) @@ -442,6 +444,9 @@ class DataLayerWallet: primaries=primaries, coin_announcements={b"$"} if fee > 0 else None, ) + magic_condition = [-24, ACS, [[Program.to((root_hash, None)), ACS_PH], None]] + # TODO: This line is a hack, make_solution should allow us to pass extra conditions to it + innersol = Program.to([[], (1, magic_condition.cons(innersol.at("rfr"))), []]) db_layer_sol = Program.to([0, inner_sol, current_inner_puzzle]) full_sol = Program.to( [ @@ -579,7 +584,7 @@ class DataLayerWallet: puzzle = parent_spend.puzzle_reveal solution = parent_spend.solution - matched, curried_args = match_dl_singleton(puzzle.to_program()) + matched, _ = match_dl_singleton(puzzle.to_program()) if matched: self.log.info(f"DL singleton removed: {parent_spend.coin}") singleton_record: Optional[SingletonRecord] = await self.wallet_state_manager.dl_store.get_singleton_record( @@ -590,11 +595,11 @@ class DataLayerWallet: return # First let's create the singleton's full puz to check if it's the same (report spend) - current_full_puz: Program = create_host_fullpuz( + current_full_puz_hash: bytes32 = create_host_fullpuz( singleton_record.inner_puzzle_hash, singleton_record.root, singleton_record.launcher_id, - ) + ).get_tree_hash(singleton_record.inner_puzzle_hash) # Information we need to create the singleton record full_puzzle_hash: bytes32 @@ -610,7 +615,7 @@ class DataLayerWallet: if condition[0] == ConditionOpcode.CREATE_COIN and int.from_bytes(condition[2], "big") % 2 == 1: full_puzzle_hash = bytes32(condition[1]) amount = uint64(int.from_bytes(condition[2], "big")) - if current_full_puz.get_tree_hash() == full_puzzle_hash: + if current_full_puz_hash == full_puzzle_hash: root = singleton_record.root inner_puzzle_hash = singleton_record.inner_puzzle_hash else: @@ -643,7 +648,7 @@ class DataLayerWallet: timestamp=timestamp, lineage_proof=LineageProof( parent_name, - create_host_layer_puzzle(inner_puzzle_hash, root).get_tree_hash(), + create_host_layer_puzzle(inner_puzzle_hash, root).get_tree_hash(inner_puzzle_hash), amount, ), generation=uint32(singleton_record.generation + 1), diff --git a/chia/wallet/db_wallet/db_wallet_puzzles.py b/chia/wallet/db_wallet/db_wallet_puzzles.py index 97beccc819..12310f4f2a 100644 --- a/chia/wallet/db_wallet/db_wallet_puzzles.py +++ b/chia/wallet/db_wallet/db_wallet_puzzles.py @@ -1,32 +1,31 @@ -from typing import Tuple, Iterator +from typing import Union, Tuple, Iterator from chia.types.blockchain_format.sized_bytes import bytes32 from chia.types.blockchain_format.program import Program from chia.util.ints import uint64 +from chia.wallet.nft_wallet.nft_puzzles import create_nft_layer_puzzle_with_curry_params, NFT_STATE_LAYER_MOD from chia.wallet.puzzles.load_clvm import load_clvm # from chia.types.condition_opcodes import ConditionOpcode # from chia.wallet.util.merkle_tree import MerkleTree, TreeType - -SINGLETON_TOP_LAYER_MOD = load_clvm("singleton_top_layer_atari_only.clvm") +ACS = Program.to(1) +ACS_PH = ACS.get_tree_hash() +SINGLETON_TOP_LAYER_MOD = load_clvm("singleton_top_layer_v1_1.clvm") # TODO: need new data layer specific clvm SINGLETON_LAUNCHER = load_clvm("singleton_launcher.clvm") -def create_host_fullpuz(innerpuz_hash: bytes32, current_root: bytes32, genesis_id: bytes32) -> Program: - db_layer = create_host_layer_puzzle(innerpuz_hash, current_root) +def create_host_fullpuz(innerpuz: Union[Program, bytes32], current_root: bytes32, genesis_id: bytes32) -> Program: + db_layer = create_host_layer_puzzle(innerpuz, current_root) mod_hash = SINGLETON_TOP_LAYER_MOD.get_tree_hash() singleton_struct = Program.to((mod_hash, (genesis_id, SINGLETON_LAUNCHER.get_tree_hash()))) return SINGLETON_TOP_LAYER_MOD.curry(singleton_struct, db_layer) -def create_host_layer_puzzle(innerpuz_hash: bytes32, current_root: bytes32) -> Program: - # singleton_struct = (MOD_HASH . (LAUNCHER_ID . LAUNCHER_PUZZLE_HASH)) - db_layer = DB_HOST_MOD.curry(DB_HOST_MOD_HASH, current_root, innerpuz_hash) - return db_layer - - +def create_host_layer_puzzle(innerpuz: Union[Program, bytes32], current_root: bytes32) -> Program: + # some hard coded metadata formatting and metadata updater for now + return create_nft_layer_puzzle_with_curry_params(Program.to((current_root, None)), ACS_PH, innerpuz) def match_dl_singleton(puzzle: Program) -> Tuple[bool, Iterator[Program]]: @@ -36,11 +35,11 @@ def match_dl_singleton(puzzle: Program) -> Tuple[bool, Iterator[Program]]: mod, singleton_curried_args = puzzle.uncurry() if mod == SINGLETON_TOP_LAYER_MOD: mod, dl_curried_args = singleton_curried_args.at("rf").uncurry() - if mod == DB_HOST_MOD: + if mod == NFT_STATE_LAYER_MOD and dl_curried_args.at("rrf") == ACS_PH: launcher_id = singleton_curried_args.at("frf") - root = dl_curried_args.at("rf") - innerpuz_hash = dl_curried_args.at("rrf") - return True, iter((innerpuz_hash, root, launcher_id)) + root = dl_curried_args.at("rff") + innerpuz = dl_curried_args.at("rrrf") + return True, iter((innerpuz, root, launcher_id)) return False, iter(()) diff --git a/tests/wallet/db_wallet/test_db_clvm.py b/tests/wallet/db_wallet/test_db_clvm.py index ba098a78c9..202cdb19c5 100644 --- a/tests/wallet/db_wallet/test_db_clvm.py +++ b/tests/wallet/db_wallet/test_db_clvm.py @@ -66,7 +66,7 @@ class TestDLLifecycle: await sim.farm_block(ACS_PH) fund_coin = (await sim_client.get_coin_records_by_puzzle_hash(ACS_PH))[0].coin launcher_coin = Coin(fund_coin.name(), SINGLETON_LAUNCHER_HASH, uint64(1)) - singleton_puzzle = create_host_fullpuz(ACS_PH, self.get_merkle_root("init"), launcher_coin.name()) + singleton_puzzle = create_host_fullpuz(ACS, self.get_merkle_root("init"), launcher_coin.name()) good_puzzle = create_offer_fullpuz( self.hash_merkle_value("init"), launcher_coin.name(), @@ -129,7 +129,7 @@ class TestDLLifecycle: [ CoinSpend( singleton, - create_host_fullpuz(ACS_PH, self.get_merkle_root("init"), singleton.parent_coin_info), + create_host_fullpuz(ACS, self.get_merkle_root("init"), singleton.parent_coin_info), solution_for_singleton( lineage_proof, singleton.amount, @@ -152,7 +152,7 @@ class TestDLLifecycle: assert ( new_singleton.puzzle_hash == create_host_fullpuz( - ACS_PH, self.get_merkle_root("update"), singleton.parent_coin_info + ACS, self.get_merkle_root("update"), singleton.parent_coin_info ).get_tree_hash() ) finally: From f768378ae718bcc92dc5a9792bb832aaf4f921ec Mon Sep 17 00:00:00 2001 From: Matt Hauff Date: Mon, 11 Jul 2022 15:48:40 -0500 Subject: [PATCH 45/62] Remove tests of old CLVM --- tests/wallet/db_wallet/test_db_clvm.py | 167 ------------------------- 1 file changed, 167 deletions(-) delete mode 100644 tests/wallet/db_wallet/test_db_clvm.py diff --git a/tests/wallet/db_wallet/test_db_clvm.py b/tests/wallet/db_wallet/test_db_clvm.py deleted file mode 100644 index 202cdb19c5..0000000000 --- a/tests/wallet/db_wallet/test_db_clvm.py +++ /dev/null @@ -1,167 +0,0 @@ -import pytest -import pytest_asyncio - -from blspy import G2Element -from typing import Dict, Tuple - -from chia.clvm.spend_sim import SpendSim, SimClient -from chia.wallet.db_wallet.db_wallet_puzzles import ( - create_host_fullpuz, - create_offer_fullpuz, - SINGLETON_LAUNCHER, -) -from chia.wallet.lineage_proof import LineageProof -from chia.wallet.puzzles.singleton_top_layer import solution_for_singleton -from chia.types.blockchain_format.program import Program -from chia.types.blockchain_format.coin import Coin -from chia.types.blockchain_format.sized_bytes import bytes32 -from chia.types.mempool_inclusion_status import MempoolInclusionStatus -from chia.types.spend_bundle import SpendBundle -from chia.types.coin_spend import CoinSpend -from chia.util.ints import uint64 -from chia.wallet.util.merkle_tree import MerkleTree - -from tests.clvm.benchmark_costs import cost_of_spend_bundle - - -ACS = Program.to(1) -ACS_2 = Program.to([3, "2", 1, []]) # (if "2" 1 ()) == 1 -ACS_PH = ACS.get_tree_hash() -ACS_2_PH = ACS_2.get_tree_hash() -SINGLETON_LAUNCHER_HASH = SINGLETON_LAUNCHER.get_tree_hash() -OFFER_AMOUNT = uint64(13) - -pytestmark = pytest.mark.data_layer - - -SetupArgs = Tuple[SpendSim, SimClient, Coin, LineageProof, Coin, Coin, Program, Program] - - -class TestDLLifecycle: - cost: Dict[str, int] = {} - - def get_merkle_tree(self, string: str) -> MerkleTree: - return MerkleTree([Program.to(string).get_tree_hash(), Program.to(string).get_tree_hash()]) - - def get_merkle_root(self, string: str) -> bytes32: - return self.get_merkle_tree(string).calculate_root() - - def hash_merkle_value(self, string: str) -> bytes32: - # https://github.com/Chia-Network/clvm/pull/102 - # https://github.com/Chia-Network/clvm/pull/106 - return Program.to(string).get_tree_hash() # type: ignore[no-any-return] - - def get_merkle_proof(self, string: str) -> Program: - proof = self.get_merkle_tree(string).generate_proof(self.hash_merkle_value(string)) - # https://github.com/Chia-Network/clvm/pull/102 - # https://github.com/Chia-Network/clvm/pull/106 - return Program.to(proof) # type: ignore[no-any-return] - - @pytest_asyncio.fixture(scope="function") - async def setup_sim_and_singleton(self) -> SetupArgs: - # https://github.com/Chia-Network/chia-blockchain/pull/11819 - sim = await SpendSim.create() # type: ignore[no-untyped-call] - sim_client = SimClient(sim) # type: ignore[no-untyped-call] - await sim.farm_block() - await sim.farm_block(ACS_PH) - fund_coin = (await sim_client.get_coin_records_by_puzzle_hash(ACS_PH))[0].coin - launcher_coin = Coin(fund_coin.name(), SINGLETON_LAUNCHER_HASH, uint64(1)) - singleton_puzzle = create_host_fullpuz(ACS, self.get_merkle_root("init"), launcher_coin.name()) - good_puzzle = create_offer_fullpuz( - self.hash_merkle_value("init"), - launcher_coin.name(), - ACS_2_PH, - ACS_PH, - uint64(60), # 60 seconds - ) - bad_puzzle = create_offer_fullpuz( - self.hash_merkle_value("nope"), - launcher_coin.name(), - ACS_2_PH, - ACS_PH, - uint64(60), # 60 seconds - ) - bundle = SpendBundle( - [ - CoinSpend( - fund_coin, - ACS, - Program.to( - [ - [51, SINGLETON_LAUNCHER_HASH, 1], - [51, good_puzzle.get_tree_hash(), OFFER_AMOUNT], - [51, bad_puzzle.get_tree_hash(), OFFER_AMOUNT], - ] - ), - ), - CoinSpend( - launcher_coin, - SINGLETON_LAUNCHER, - Program.to([singleton_puzzle.get_tree_hash(), launcher_coin.amount, []]), - ), - ], - G2Element(), - ) - result = (await sim_client.push_tx(bundle))[0] - assert result == MempoolInclusionStatus.SUCCESS - self.cost["launch singleton and create two coins"] = cost_of_spend_bundle(bundle) - await sim.farm_block() - singleton = (await sim_client.get_coin_records_by_puzzle_hashes([singleton_puzzle.get_tree_hash()]))[0].coin - good_offer_coin = (await sim_client.get_coin_records_by_puzzle_hashes([good_puzzle.get_tree_hash()]))[0].coin - bad_offer_coin = (await sim_client.get_coin_records_by_puzzle_hashes([bad_puzzle.get_tree_hash()]))[0].coin - return ( - sim, - sim_client, - singleton, - LineageProof(parent_name=launcher_coin.parent_coin_info, amount=launcher_coin.amount), - good_offer_coin, - bad_offer_coin, - good_puzzle, - bad_puzzle, - ) - - @pytest.mark.asyncio() - async def test_update(self, setup_sim_and_singleton: SetupArgs) -> None: - sim, sim_client, singleton, lineage_proof = setup_sim_and_singleton[0:4] - - try: - bundle = SpendBundle( - [ - CoinSpend( - singleton, - create_host_fullpuz(ACS, self.get_merkle_root("init"), singleton.parent_coin_info), - solution_for_singleton( - lineage_proof, - singleton.amount, - Program.to( - [[ - [51, ACS_PH, singleton.amount], - [-24, ACS, Program.to((self.get_merkle_root("update"), None))] - ]] - ), - ), - ) - ], - G2Element(), - ) - result = (await sim_client.push_tx(bundle))[0] - assert result == MempoolInclusionStatus.SUCCESS - self.cost["update spend"] = cost_of_spend_bundle(bundle) - await sim.farm_block() - new_singleton = (await sim_client.get_coin_records_by_parent_ids([singleton.name()]))[0].coin - assert ( - new_singleton.puzzle_hash - == create_host_fullpuz( - ACS, self.get_merkle_root("update"), singleton.parent_coin_info - ).get_tree_hash() - ) - finally: - # https://github.com/Chia-Network/chia-blockchain/pull/11819 - await sim.close() # type: ignore[no-untyped-call] - - def test_cost(self) -> None: - import json - import logging - - log = logging.getLogger(__name__) - log.warning(json.dumps(self.cost)) From 21c1352ec37fb771c63a97feb4d7c9767f1396b6 Mon Sep 17 00:00:00 2001 From: Matt Hauff Date: Mon, 11 Jul 2022 15:50:37 -0500 Subject: [PATCH 46/62] Get rid of some bothersome debugs --- chia/wallet/nft_wallet/nft_puzzles.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/chia/wallet/nft_wallet/nft_puzzles.py b/chia/wallet/nft_wallet/nft_puzzles.py index c25ac6e836..fc5fb3b60a 100644 --- a/chia/wallet/nft_wallet/nft_puzzles.py +++ b/chia/wallet/nft_wallet/nft_puzzles.py @@ -35,19 +35,6 @@ def create_nft_layer_puzzle_with_curry_params( METADATA METADATA_UPDATER_PUZZLE_HASH INNER_PUZZLE""" - log.debug( - "Creating nft layer puzzle curry: mod_hash: %s, metadata: %r, metadata_hash: %s", - NFT_STATE_LAYER_MOD_HASH, - metadata, - metadata_updater_hash, - ) - log.debug( - "Currying with: %s %s %s %s", - NFT_STATE_LAYER_MOD_HASH, - inner_puzzle.get_tree_hash(), - metadata_updater_hash, - metadata.get_tree_hash(), - ) return NFT_STATE_LAYER_MOD.curry(NFT_STATE_LAYER_MOD_HASH, metadata, metadata_updater_hash, inner_puzzle) From 1dfd2b542e6f09052d312f1297906eba42f1d66c Mon Sep 17 00:00:00 2001 From: Matt Hauff Date: Tue, 12 Jul 2022 09:56:11 -0500 Subject: [PATCH 47/62] minor test fixes --- chia/data_layer/data_layer_wallet.py | 35 ++++++++++------------ chia/wallet/db_wallet/db_wallet_puzzles.py | 8 ++--- 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/chia/data_layer/data_layer_wallet.py b/chia/data_layer/data_layer_wallet.py index 5bf6b73ff9..0f198d116f 100644 --- a/chia/data_layer/data_layer_wallet.py +++ b/chia/data_layer/data_layer_wallet.py @@ -12,8 +12,8 @@ from blspy import G2Element from chia.consensus.block_record import BlockRecord from chia.protocols.wallet_protocol import PuzzleSolutionResponse, CoinState from chia.wallet.db_wallet.db_wallet_puzzles import ( - ACS, - ACS_PH, + ACS_MU, + ACS_MU_PH, create_host_fullpuz, SINGLETON_LAUNCHER, create_host_layer_puzzle, @@ -419,7 +419,6 @@ class DataLayerWallet: # Make the child's puzzles next_inner_puzzle: Program = await self.standard_wallet.get_new_puzzle(in_transaction=in_transaction) - next_db_layer_puzzle: Program = create_host_layer_puzzle(next_inner_puzzle, root_hash) next_full_puz = create_host_fullpuz(next_inner_puzzle, root_hash, launcher_id) # Construct the current puzzles @@ -435,7 +434,7 @@ class DataLayerWallet: assert singleton_record.lineage_proof.amount is not None primaries: List[AmountWithPuzzlehash] = [ { - "puzzlehash": next_db_layer_puzzle.get_tree_hash(), + "puzzlehash": next_inner_puzzle.get_tree_hash(), "amount": singleton_record.lineage_proof.amount, "memos": [launcher_id, root_hash, next_inner_puzzle.get_tree_hash()], } @@ -444,10 +443,10 @@ class DataLayerWallet: primaries=primaries, coin_announcements={b"$"} if fee > 0 else None, ) - magic_condition = [-24, ACS, [[Program.to((root_hash, None)), ACS_PH], None]] + magic_condition = Program.to([-24, ACS_MU, [[Program.to((root_hash, None)), ACS_MU_PH], None]]) # TODO: This line is a hack, make_solution should allow us to pass extra conditions to it - innersol = Program.to([[], (1, magic_condition.cons(innersol.at("rfr"))), []]) - db_layer_sol = Program.to([0, inner_sol, current_inner_puzzle]) + inner_sol = Program.to([[], (1, magic_condition.cons(inner_sol.at("rfr"))), []]) + db_layer_sol = Program.to([inner_sol]) full_sol = Program.to( [ parent_lineage.to_program(), @@ -615,19 +614,15 @@ class DataLayerWallet: if condition[0] == ConditionOpcode.CREATE_COIN and int.from_bytes(condition[2], "big") % 2 == 1: full_puzzle_hash = bytes32(condition[1]) amount = uint64(int.from_bytes(condition[2], "big")) - if current_full_puz_hash == full_puzzle_hash: - root = singleton_record.root - inner_puzzle_hash = singleton_record.inner_puzzle_hash - else: - try: - root = bytes32(condition[3][1]) - inner_puzzle_hash = bytes32(condition[3][2]) - except IndexError: - self.log.warning( - f"Parent {parent_name} with launcher {singleton_record.launcher_id} " - "did not hint its child properly" - ) - return + try: + root = bytes32(condition[3][1]) + inner_puzzle_hash = bytes32(condition[3][2]) + except IndexError: + self.log.warning( + f"Parent {parent_name} with launcher {singleton_record.launcher_id} " + "did not hint its child properly" + ) + return found_singleton = True break diff --git a/chia/wallet/db_wallet/db_wallet_puzzles.py b/chia/wallet/db_wallet/db_wallet_puzzles.py index 12310f4f2a..0f68b59f56 100644 --- a/chia/wallet/db_wallet/db_wallet_puzzles.py +++ b/chia/wallet/db_wallet/db_wallet_puzzles.py @@ -9,8 +9,8 @@ from chia.wallet.puzzles.load_clvm import load_clvm # from chia.types.condition_opcodes import ConditionOpcode # from chia.wallet.util.merkle_tree import MerkleTree, TreeType -ACS = Program.to(1) -ACS_PH = ACS.get_tree_hash() +ACS_MU = Program.to(11) # returns the third argument a.k.a the full solution +ACS_MU_PH = ACS_MU.get_tree_hash() SINGLETON_TOP_LAYER_MOD = load_clvm("singleton_top_layer_v1_1.clvm") # TODO: need new data layer specific clvm SINGLETON_LAUNCHER = load_clvm("singleton_launcher.clvm") @@ -25,7 +25,7 @@ def create_host_fullpuz(innerpuz: Union[Program, bytes32], current_root: bytes32 def create_host_layer_puzzle(innerpuz: Union[Program, bytes32], current_root: bytes32) -> Program: # some hard coded metadata formatting and metadata updater for now - return create_nft_layer_puzzle_with_curry_params(Program.to((current_root, None)), ACS_PH, innerpuz) + return create_nft_layer_puzzle_with_curry_params(Program.to((current_root, None)), ACS_MU_PH, innerpuz) def match_dl_singleton(puzzle: Program) -> Tuple[bool, Iterator[Program]]: @@ -35,7 +35,7 @@ def match_dl_singleton(puzzle: Program) -> Tuple[bool, Iterator[Program]]: mod, singleton_curried_args = puzzle.uncurry() if mod == SINGLETON_TOP_LAYER_MOD: mod, dl_curried_args = singleton_curried_args.at("rf").uncurry() - if mod == NFT_STATE_LAYER_MOD and dl_curried_args.at("rrf") == ACS_PH: + if mod == NFT_STATE_LAYER_MOD and dl_curried_args.at("rrf") == ACS_MU_PH: launcher_id = singleton_curried_args.at("frf") root = dl_curried_args.at("rff") innerpuz = dl_curried_args.at("rrrf") From ecded9f9a27aca51b93a57867a463621f8c2e986 Mon Sep 17 00:00:00 2001 From: Matt Hauff Date: Tue, 12 Jul 2022 09:56:41 -0500 Subject: [PATCH 48/62] skip the rebase test --- tests/wallet/db_wallet/test_dl_wallet.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/wallet/db_wallet/test_dl_wallet.py b/tests/wallet/db_wallet/test_dl_wallet.py index b54a8788c9..45ec295392 100644 --- a/tests/wallet/db_wallet/test_dl_wallet.py +++ b/tests/wallet/db_wallet/test_dl_wallet.py @@ -322,6 +322,7 @@ class TestDLWallet: await time_out_assert(15, is_singleton_confirmed, True, dl_wallet, launcher_id) await asyncio.sleep(0.5) + @pytest.mark.skip(reason="maybe no longer relevant, needs to be rewritten at least") @pytest.mark.parametrize( "trusted", [True, False], From 231fa227de3c28ba20fe9c61643ba95138c64685 Mon Sep 17 00:00:00 2001 From: Matt Hauff Date: Tue, 12 Jul 2022 10:30:00 -0500 Subject: [PATCH 49/62] Remove dlo_wallet folder --- chia/wallet/dlo_wallet/__init__.py | 0 chia/wallet/dlo_wallet/dlo_wallet.py | 317 --------------------------- 2 files changed, 317 deletions(-) delete mode 100644 chia/wallet/dlo_wallet/__init__.py delete mode 100644 chia/wallet/dlo_wallet/dlo_wallet.py diff --git a/chia/wallet/dlo_wallet/__init__.py b/chia/wallet/dlo_wallet/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/chia/wallet/dlo_wallet/dlo_wallet.py b/chia/wallet/dlo_wallet/dlo_wallet.py deleted file mode 100644 index 2a68c32532..0000000000 --- a/chia/wallet/dlo_wallet/dlo_wallet.py +++ /dev/null @@ -1,317 +0,0 @@ -from dataclasses import dataclass -import logging -import time -from typing import Any, Dict, List, Optional, Set, Tuple, Type, TypeVar - -from blspy import AugSchemeMPL - -from chia.util.streamable import Streamable, streamable -from chia.types.blockchain_format.coin import Coin -from chia.wallet.db_wallet.db_wallet_puzzles import create_offer_fullpuz -from chia.types.blockchain_format.program import Program -from chia.types.blockchain_format.sized_bytes import bytes32 -from chia.types.coin_spend import CoinSpend -from chia.types.spend_bundle import SpendBundle -from chia.util.ints import uint8, uint32, uint64 -from chia.wallet.transaction_record import TransactionRecord -from chia.wallet.util.compute_memos import compute_memos -from chia.wallet.util.transaction_type import TransactionType -from chia.wallet.util.wallet_types import WalletType -from chia.wallet.wallet import Wallet -from chia.wallet.wallet_coin_record import WalletCoinRecord -from chia.wallet.wallet_info import WalletInfo -from chia.wallet.wallet_state_manager import WalletStateManager - - -_T_DLOWallet = TypeVar("_T_DLOWallet", bound="DLOWallet") - - -@dataclass(frozen=True) -@streamable -class DLOInfo(Streamable): - leaf_reveal: Optional[bytes] - host_genesis_id: Optional[bytes32] - claim_target: Optional[bytes32] - recovery_target: Optional[bytes32] - recovery_timelock: Optional[uint64] - active_offer: Optional[Coin] - - -@dataclass -class DLOWallet: - wallet_state_manager: WalletStateManager - log: logging.Logger - wallet_id: uint32 - wallet_info: WalletInfo - dlo_info: DLOInfo - standard_wallet: Wallet - base_puzzle_program: Optional[bytes] - base_inner_puzzle_hash: Optional[bytes32] - cost_of_single_tx: Optional[int] - - @classmethod - def type(cls) -> uint8: - return uint8(WalletType.DATA_LAYER_OFFER) - - @classmethod - async def create_new_dlo_wallet( - cls: Type[_T_DLOWallet], - wallet_state_manager: Any, - wallet: Wallet, - ) -> _T_DLOWallet: - dlo_info = DLOInfo(None, None, None, None, None, None) - info_as_string = bytes(dlo_info).hex() - wallet_info = await wallet_state_manager.user_store.create_wallet( - "DLO Wallet", WalletType.DATA_LAYER_OFFER.value, info_as_string - ) - if wallet_info is None: - # TODO: this should be an exception way down at the source, not here - raise ValueError("Internal Error") - - self = cls( - cost_of_single_tx=None, - base_puzzle_program=None, - base_inner_puzzle_hash=None, - standard_wallet=wallet, - log=logging.getLogger(__name__), - wallet_state_manager=wallet_state_manager, - dlo_info=dlo_info, - wallet_info=wallet_info, - wallet_id=wallet_info.id, - ) - - await self.wallet_state_manager.add_new_wallet(self, self.wallet_info.id) - return self - - def puzzle_for_pk(self, pubkey: bytes) -> Program: - if self.dlo_info.leaf_reveal is not None: - return create_offer_fullpuz( - self.dlo_info.leaf_reveal, - self.dlo_info.host_genesis_id, # type: ignore[arg-type] - self.dlo_info.claim_target, # type: ignore[arg-type] - self.dlo_info.recovery_target, # type: ignore[arg-type] - self.dlo_info.recovery_timelock, # type: ignore[arg-type] - ) - return Program.to(pubkey) # type: ignore[no-any-return] - - def id(self) -> uint32: - return self.wallet_info.id - - async def generate_datalayer_offer_spend( - self, - amount: uint64, - leaf_reveal: bytes, - host_genesis_id: bytes32, - claim_target: bytes32, - recovery_target: bytes32, - recovery_timelock: uint64, - ) -> TransactionRecord: - full_puzzle: Program = create_offer_fullpuz( - leaf_reveal, - host_genesis_id, - claim_target, - recovery_target, - recovery_timelock, - ) - tr: TransactionRecord = await self.standard_wallet.generate_signed_transaction( - amount, full_puzzle.get_tree_hash() - ) - await self.wallet_state_manager.add_interested_puzzle_hashes( - [full_puzzle.get_tree_hash()], [self.wallet_id], True - ) - - active_coin = None - spend_bundle = tr.spend_bundle - if spend_bundle is not None: - for coin in spend_bundle.additions(): - if coin.puzzle_hash == full_puzzle.get_tree_hash(): - active_coin = coin - if active_coin is None: - raise ValueError("Unable to find created coin") - - await self.standard_wallet.push_transaction(tr) - dlo_info = DLOInfo( - leaf_reveal, - host_genesis_id, - claim_target, - recovery_target, - recovery_timelock, - active_coin, - ) - await self.save_info(dlo_info, True) - return tr - - async def claim_dl_offer( - self, - offer_coin: Coin, - offer_full_puzzle: Program, - db_innerpuz_hash: bytes32, - current_root: bytes32, - inclusion_proof: Tuple[Optional[int], List[Optional[List[bytes32]]]], - fee: uint64 = uint64(0), - ) -> SpendBundle: - solution = Program.to([1, offer_coin.amount, db_innerpuz_hash, current_root, inclusion_proof]) - sb = SpendBundle([CoinSpend(offer_coin, offer_full_puzzle, solution)], AugSchemeMPL.aggregate([])) - # ret = uncurry_offer_puzzle(offer_full_puzzle) - # singleton_struct, leaf_reveal, claim_target, recovery_target, recovery_timelock = ret - # tr = TransactionRecord( - # confirmed_at_height=uint32(0), - # created_at_time=uint64(int(time.time())), - # to_puzzle_hash=claim_target, - # amount=uint64(offer_coin.amount), - # fee_amount=uint64(fee), - # confirmed=False, - # sent=uint32(0), - # spend_bundle=sb, - # additions=list(sb.additions()), - # removals=list(sb.removals()), - # wallet_id=self.id(), - # sent_to=[], - # trade_id=None, - # type=uint32(TransactionType.OUTGOING_TX.value), - # name=sb.name(), - # ) - # self.standard_wallet.push_transaction(tr) - return sb - - async def create_recover_dl_offer_spend( - self, - leaf_reveal: Optional[bytes] = None, - host_genesis_id: Optional[bytes32] = None, - claim_target: Optional[bytes32] = None, - recovery_target: Optional[bytes32] = None, - recovery_timelock: Optional[uint64] = None, - fee: uint64 = uint64(0), - ) -> TransactionRecord: - coin = self.dlo_info.active_offer - if coin is None: - raise ValueError("Active offer coin unexpectedly None") - - solution = Program.to([0, coin.amount]) - - if leaf_reveal is None: - leaf_reveal = self.dlo_info.leaf_reveal - host_genesis_id = self.dlo_info.host_genesis_id - claim_target = self.dlo_info.claim_target - recovery_target = self.dlo_info.recovery_target - recovery_timelock = self.dlo_info.recovery_timelock - full_puzzle: Program = create_offer_fullpuz( - leaf_reveal, host_genesis_id, claim_target, recovery_target, recovery_timelock # type: ignore[arg-type] - ) - coin_spend = CoinSpend(coin, full_puzzle, solution) - sb = SpendBundle([coin_spend], AugSchemeMPL.aggregate([])) - # TODO: fix optionality issue with to_puzzle_hash - tr = TransactionRecord( - confirmed_at_height=uint32(0), - created_at_time=uint64(int(time.time())), - to_puzzle_hash=recovery_target, # type: ignore[arg-type] - amount=uint64(coin.amount), - fee_amount=uint64(fee), - confirmed=False, - sent=uint32(0), - spend_bundle=sb, - additions=list(sb.additions()), - removals=list(sb.removals()), - memos=list(compute_memos(sb).items()), - wallet_id=self.id(), - sent_to=[], - trade_id=None, - type=uint32(TransactionType.OUTGOING_TX.value), - name=sb.name(), - ) - await self.standard_wallet.push_transaction(tr) - return tr - - async def get_coin(self) -> Coin: - try: - coins = await self.select_coins(uint64(1)) - except ValueError: - # TODO: not really right exactly the same since there are two cases of ValueError... - coins = set() - - if len(coins) > 1: - return coins.pop() - - if self.dlo_info.active_offer is None: - # TODO: is this what we want here? - raise ValueError("Unable to get a coin, no coins selected and no active offer.") - - return self.dlo_info.active_offer - - async def get_confirmed_balance(self, record_list: Optional[Set[WalletCoinRecord]] = None) -> uint64: - if record_list is None: - record_list = await self.wallet_state_manager.coin_store.get_unspent_coins_for_wallet(self.id()) - - amount: uint64 = uint64(0) - for record in record_list: - amount = uint64(amount + record.coin.amount) - - self.log.info(f"Confirmed balance for dlo wallet is {amount}") - return uint64(amount) - - async def get_unconfirmed_balance(self, record_list: Optional[Set[WalletCoinRecord]] = None) -> uint64: - # TODO: should the uint128 be changed? - return await self.wallet_state_manager.get_unconfirmed_balance(self.id(), record_list) # type: ignore[return-value] # noqa: E501 - - async def get_spendable_balance(self, unspent_records: Optional[Set[WalletCoinRecord]] = None) -> uint64: - spendable_am = await self.wallet_state_manager.get_confirmed_spendable_balance_for_wallet( - self.wallet_info.id, unspent_records - ) - # TODO: should the uint128 be changed? - return spendable_am # type: ignore[return-value] - - async def select_coins(self, amount: uint64, exclude: Optional[List[Coin]] = None) -> Set[Coin]: - """Returns a set of coins that can be used for generating a new transaction.""" - if exclude is None: - exclude = [] - - spendable_amount = await self.get_spendable_balance() - if amount > spendable_amount: - error_msg = f"Can't select {amount}, from spendable {spendable_amount} for wallet id {self.id()}" - self.log.warning(error_msg) - raise ValueError(error_msg) - - self.log.info(f"About to select coins for amount {amount}") - unspent: List[WalletCoinRecord] = list( - await self.wallet_state_manager.get_spendable_coins_for_wallet(self.wallet_info.id) - ) - sum_value = 0 - used_coins: Set[Coin] = set() - - # Use older coins first - unspent.sort(key=lambda r: r.confirmed_block_height) - - # Try to use coins from the store, if there isn't enough of "unused" - # coins use change coins that are not confirmed yet - unconfirmed_removals: Dict[bytes32, Coin] = await self.wallet_state_manager.unconfirmed_removals_for_wallet( - self.wallet_info.id - ) - for coinrecord in unspent: - if sum_value >= amount and len(used_coins) > 0: - break - if coinrecord.coin.name() in unconfirmed_removals: - continue - if coinrecord.coin in exclude: - continue - sum_value += coinrecord.coin.amount - used_coins.add(coinrecord.coin) - - # This happens when we couldn't use one of the coins because it's already used - # but unconfirmed, and we are waiting for the change. (unconfirmed_additions) - if sum_value < amount: - raise ValueError( - "Can't make this transaction at the moment. Waiting for the change from the previous transaction." - ) - - self.log.info(f"Successfully selected coins: {used_coins}") - return used_coins - - async def save_info(self, dlo_info: DLOInfo, in_transaction: bool) -> None: - self.dlo_info = dlo_info - current_info = self.wallet_info - info_as_string = bytes(self.dlo_info).hex() - wallet_info = WalletInfo(current_info.id, current_info.name, current_info.type, info_as_string) - self.wallet_info = wallet_info - await self.wallet_state_manager.user_store.update_wallet(wallet_info, in_transaction) - await self.wallet_state_manager.update_wallet_puzzle_hashes(self.wallet_info.id) - return From d0fc7ff329311ebcf093d0bce0b5ca3053b29487 Mon Sep 17 00:00:00 2001 From: Matt Hauff Date: Tue, 12 Jul 2022 10:34:26 -0500 Subject: [PATCH 50/62] Lint --- chia/data_layer/data_layer_wallet.py | 10 ++-------- chia/wallet/db_wallet/db_wallet_puzzles.py | 7 ++++++- tests/wallet/db_wallet/test_dl_wallet.py | 4 +++- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/chia/data_layer/data_layer_wallet.py b/chia/data_layer/data_layer_wallet.py index 0f198d116f..8dcf320d1b 100644 --- a/chia/data_layer/data_layer_wallet.py +++ b/chia/data_layer/data_layer_wallet.py @@ -153,7 +153,8 @@ class DataLayerWallet: # Now let's check that the full puzzle is an odd data layer singleton if ( - full_puzhash != create_host_fullpuz(inner_puzhash, root, launcher_spend.coin.name()).get_tree_hash(inner_puzhash) + full_puzhash + != create_host_fullpuz(inner_puzhash, root, launcher_spend.coin.name()).get_tree_hash(inner_puzhash) or amount % 2 == 0 ): return False, None @@ -593,13 +594,6 @@ class DataLayerWallet: self.log.warning(f"DL wallet received coin it does not have parent for. Expected parent {parent_name}.") return - # First let's create the singleton's full puz to check if it's the same (report spend) - current_full_puz_hash: bytes32 = create_host_fullpuz( - singleton_record.inner_puzzle_hash, - singleton_record.root, - singleton_record.launcher_id, - ).get_tree_hash(singleton_record.inner_puzzle_hash) - # Information we need to create the singleton record full_puzzle_hash: bytes32 amount: uint64 diff --git a/chia/wallet/db_wallet/db_wallet_puzzles.py b/chia/wallet/db_wallet/db_wallet_puzzles.py index 0f68b59f56..ff7b0ad18f 100644 --- a/chia/wallet/db_wallet/db_wallet_puzzles.py +++ b/chia/wallet/db_wallet/db_wallet_puzzles.py @@ -25,7 +25,12 @@ def create_host_fullpuz(innerpuz: Union[Program, bytes32], current_root: bytes32 def create_host_layer_puzzle(innerpuz: Union[Program, bytes32], current_root: bytes32) -> Program: # some hard coded metadata formatting and metadata updater for now - return create_nft_layer_puzzle_with_curry_params(Program.to((current_root, None)), ACS_MU_PH, innerpuz) + return create_nft_layer_puzzle_with_curry_params( + Program.to((current_root, None)), + ACS_MU_PH, + # TODO: the nft driver doesn't like the Union yet, but changing that is out of scope for me rn - Quex + innerpuz, # type: ignore + ) def match_dl_singleton(puzzle: Program) -> Tuple[bool, Iterator[Program]]: diff --git a/tests/wallet/db_wallet/test_dl_wallet.py b/tests/wallet/db_wallet/test_dl_wallet.py index 45ec295392..e73c6cb2cc 100644 --- a/tests/wallet/db_wallet/test_dl_wallet.py +++ b/tests/wallet/db_wallet/test_dl_wallet.py @@ -392,7 +392,9 @@ class TestDLWallet: await asyncio.sleep(0.5) # Because these have the same fee, the one that gets pushed first will win - report_txs, _ = await dl_wallet_1.create_update_state_spend(launcher_id, current_record.root, fee=uint64(2000000000000)) + report_txs = await dl_wallet_1.create_update_state_spend( + launcher_id, current_record.root, fee=uint64(2000000000000) + ) record_1 = await dl_wallet_1.get_latest_singleton(launcher_id) assert record_1 is not None assert current_record != record_1 From e6ed51179b91a5586c9095cc0146a9a3e84f6f4f Mon Sep 17 00:00:00 2001 From: Matt Hauff Date: Tue, 12 Jul 2022 10:44:49 -0500 Subject: [PATCH 51/62] Remove some more dlo stuff --- .isort.cfg | 1 - setup.py | 1 - tests/wallet/db_wallet/test_dl_wallet.py | 1 - 3 files changed, 3 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index c44cb67e31..6f2e941e33 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -117,7 +117,6 @@ extend_skip= chia/wallet/did_wallet/did_info.py chia/wallet/did_wallet/did_wallet_puzzles.py chia/wallet/did_wallet/did_wallet.py - chia/wallet/dlo_wallet/dlo_wallet.py chia/wallet/lineage_proof.py chia/wallet/payment.py chia/wallet/puzzles/load_clvm.py diff --git a/setup.py b/setup.py index e78707929b..0e37ae5da9 100644 --- a/setup.py +++ b/setup.py @@ -107,7 +107,6 @@ kwargs = dict( "chia.util", "chia.wallet", "chia.wallet.db_wallet", - "chia.wallet.dlo_wallet", "chia.wallet.puzzles", "chia.wallet.rl_wallet", "chia.wallet.cat_wallet", diff --git a/tests/wallet/db_wallet/test_dl_wallet.py b/tests/wallet/db_wallet/test_dl_wallet.py index e73c6cb2cc..731d248a22 100644 --- a/tests/wallet/db_wallet/test_dl_wallet.py +++ b/tests/wallet/db_wallet/test_dl_wallet.py @@ -10,7 +10,6 @@ from chia.simulator.simulator_protocol import FarmNewBlockProtocol from tests.setup_nodes import setup_simulators_and_wallets from chia.data_layer.data_layer_wallet import DataLayerWallet -# from chia.wallet.dlo_wallet.dlo_wallet import DLOWallet from chia.types.blockchain_format.program import Program from tests.time_out_assert import time_out_assert from chia.wallet.util.merkle_tree import MerkleTree From a8309d221d6000a40a65ab56ec444ffc2a3b457d Mon Sep 17 00:00:00 2001 From: Matt Hauff Date: Tue, 12 Jul 2022 12:30:38 -0500 Subject: [PATCH 52/62] pre-commit --- .github/workflows/build-test-macos-wallet-db_wallet.yml | 2 +- .github/workflows/build-test-ubuntu-wallet-db_wallet.yml | 2 +- tests/wallet/db_wallet/test_dl_wallet.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test-macos-wallet-db_wallet.yml b/.github/workflows/build-test-macos-wallet-db_wallet.yml index 053caba0f0..539ec29560 100644 --- a/.github/workflows/build-test-macos-wallet-db_wallet.yml +++ b/.github/workflows/build-test-macos-wallet-db_wallet.yml @@ -95,7 +95,7 @@ jobs: - name: Test wallet-db_wallet code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc --module pytest --durations=10 -n 0 -m "not benchmark" tests/wallet/db_wallet/test_db_clvm.py tests/wallet/db_wallet/test_db_graftroot.py tests/wallet/db_wallet/test_dl_wallet.py + venv/bin/coverage run --rcfile=.coveragerc --module pytest --durations=10 -n 0 -m "not benchmark" tests/wallet/db_wallet/test_db_graftroot.py tests/wallet/db_wallet/test_dl_wallet.py - name: Process coverage data run: | diff --git a/.github/workflows/build-test-ubuntu-wallet-db_wallet.yml b/.github/workflows/build-test-ubuntu-wallet-db_wallet.yml index 09fb1f8fba..140639f93e 100644 --- a/.github/workflows/build-test-ubuntu-wallet-db_wallet.yml +++ b/.github/workflows/build-test-ubuntu-wallet-db_wallet.yml @@ -94,7 +94,7 @@ jobs: - name: Test wallet-db_wallet code with pytest run: | . ./activate - venv/bin/coverage run --rcfile=.coveragerc --module pytest --durations=10 -n 0 -m "not benchmark" tests/wallet/db_wallet/test_db_clvm.py tests/wallet/db_wallet/test_db_graftroot.py tests/wallet/db_wallet/test_dl_wallet.py + venv/bin/coverage run --rcfile=.coveragerc --module pytest --durations=10 -n 0 -m "not benchmark" tests/wallet/db_wallet/test_db_graftroot.py tests/wallet/db_wallet/test_dl_wallet.py - name: Process coverage data run: | diff --git a/tests/wallet/db_wallet/test_dl_wallet.py b/tests/wallet/db_wallet/test_dl_wallet.py index 731d248a22..7f0933e982 100644 --- a/tests/wallet/db_wallet/test_dl_wallet.py +++ b/tests/wallet/db_wallet/test_dl_wallet.py @@ -27,6 +27,7 @@ def event_loop() -> Iterator[asyncio.AbstractEventLoop]: async def is_singleton_confirmed(dl_wallet: DataLayerWallet, lid: bytes32) -> bool: rec = await dl_wallet.get_latest_singleton(lid) + print("HEY") if rec is None: return False if rec.confirmed is True: From 81e213d95cd7bb59773a7a76ed7e517bf47c782d Mon Sep 17 00:00:00 2001 From: Matt Hauff Date: Tue, 12 Jul 2022 12:36:20 -0500 Subject: [PATCH 53/62] Remove accidental print --- tests/wallet/db_wallet/test_dl_wallet.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/wallet/db_wallet/test_dl_wallet.py b/tests/wallet/db_wallet/test_dl_wallet.py index 7f0933e982..731d248a22 100644 --- a/tests/wallet/db_wallet/test_dl_wallet.py +++ b/tests/wallet/db_wallet/test_dl_wallet.py @@ -27,7 +27,6 @@ def event_loop() -> Iterator[asyncio.AbstractEventLoop]: async def is_singleton_confirmed(dl_wallet: DataLayerWallet, lid: bytes32) -> bool: rec = await dl_wallet.get_latest_singleton(lid) - print("HEY") if rec is None: return False if rec.confirmed is True: From 807ae8feda1213da7d31dd146e8932c35220eaf8 Mon Sep 17 00:00:00 2001 From: Matt Hauff Date: Tue, 12 Jul 2022 15:22:10 -0500 Subject: [PATCH 54/62] Add generate_signed_transaction to DL --- chia/data_layer/data_layer_wallet.py | 98 ++++++++++++++++++++---- tests/wallet/db_wallet/test_dl_wallet.py | 15 +++- 2 files changed, 94 insertions(+), 19 deletions(-) diff --git a/chia/data_layer/data_layer_wallet.py b/chia/data_layer/data_layer_wallet.py index 8dcf320d1b..d16882ad23 100644 --- a/chia/data_layer/data_layer_wallet.py +++ b/chia/data_layer/data_layer_wallet.py @@ -404,12 +404,20 @@ class DataLayerWallet: async def create_update_state_spend( self, launcher_id: bytes32, - root_hash: bytes32, + root_hash: Optional[bytes32], + new_puz_hash: Optional[bytes32] = None, + new_amount: Optional[uint64] = None, fee: uint64 = uint64(0), + coin_announcements_to_consume: Optional[Set[Announcement]] = None, + puzzle_announcements_to_consume: Optional[Set[Announcement]] = None, + sign: bool = True, in_transaction: bool = False, ) -> List[TransactionRecord]: singleton_record, parent_lineage = await self.get_spendable_singleton_info(launcher_id) + if root_hash is None: + root_hash = singleton_record.root + inner_puzzle_derivation: Optional[ DerivationRecord ] = await self.wallet_state_manager.puzzle_store.get_derivation_record_for_puzzle_hash( @@ -419,8 +427,12 @@ class DataLayerWallet: raise ValueError(f"DL Wallet does not have permission to update Singleton with launcher ID {launcher_id}") # Make the child's puzzles - next_inner_puzzle: Program = await self.standard_wallet.get_new_puzzle(in_transaction=in_transaction) - next_full_puz = create_host_fullpuz(next_inner_puzzle, root_hash, launcher_id) + if new_puz_hash is None: + new_puz_hash = (await self.standard_wallet.get_new_puzzle(in_transaction=in_transaction)).get_tree_hash() + assert new_puz_hash is not None + next_full_puz_hash: bytes32 = create_host_fullpuz(new_puz_hash, root_hash, launcher_id).get_tree_hash( + new_puz_hash + ) # Construct the current puzzles current_inner_puzzle: Program = self.standard_wallet.puzzle_for_pk(inner_puzzle_derivation.pubkey) @@ -435,18 +447,25 @@ class DataLayerWallet: assert singleton_record.lineage_proof.amount is not None primaries: List[AmountWithPuzzlehash] = [ { - "puzzlehash": next_inner_puzzle.get_tree_hash(), - "amount": singleton_record.lineage_proof.amount, - "memos": [launcher_id, root_hash, next_inner_puzzle.get_tree_hash()], + "puzzlehash": new_puz_hash, + "amount": singleton_record.lineage_proof.amount if new_amount is None else new_amount, + "memos": [launcher_id, root_hash, new_puz_hash], } ] inner_sol: Program = self.standard_wallet.make_solution( primaries=primaries, coin_announcements={b"$"} if fee > 0 else None, + coin_announcements_to_assert={a.name() for a in coin_announcements_to_consume} + if coin_announcements_to_consume is not None + else None, + puzzle_announcements_to_assert={a.name() for a in puzzle_announcements_to_consume} + if puzzle_announcements_to_consume is not None + else None, ) - magic_condition = Program.to([-24, ACS_MU, [[Program.to((root_hash, None)), ACS_MU_PH], None]]) - # TODO: This line is a hack, make_solution should allow us to pass extra conditions to it - inner_sol = Program.to([[], (1, magic_condition.cons(inner_sol.at("rfr"))), []]) + if root_hash != singleton_record.root: + magic_condition = Program.to([-24, ACS_MU, [[Program.to((root_hash, None)), ACS_MU_PH], None]]) + # TODO: This line is a hack, make_solution should allow us to pass extra conditions to it + inner_sol = Program.to([[], (1, magic_condition.cons(inner_sol.at("rfr"))), []]) db_layer_sol = Program.to([inner_sol]) full_sol = Program.to( [ @@ -468,12 +487,16 @@ class DataLayerWallet: SerializedProgram.from_program(full_sol), ) await self.standard_wallet.hack_populate_secret_key_for_puzzle_hash(current_inner_puzzle.get_tree_hash()) - spend_bundle = await self.sign(coin_spend) + + if sign: + spend_bundle = await self.sign(coin_spend) + else: + spend_bundle = SpendBundle([coin_spend], G2Element()) dl_tx = TransactionRecord( confirmed_at_height=uint32(0), created_at_time=uint64(int(time.time())), - to_puzzle_hash=next_inner_puzzle.get_tree_hash(), + to_puzzle_hash=new_puz_hash, amount=uint64(singleton_record.lineage_proof.amount), fee_amount=fee, confirmed=False, @@ -499,18 +522,16 @@ class DataLayerWallet: else: txs = [dl_tx] new_singleton_record = SingletonRecord( - coin_id=Coin( - current_coin.name(), next_full_puz.get_tree_hash(), singleton_record.lineage_proof.amount - ).name(), + coin_id=Coin(current_coin.name(), next_full_puz_hash, singleton_record.lineage_proof.amount).name(), launcher_id=launcher_id, root=root_hash, - inner_puzzle_hash=next_inner_puzzle.get_tree_hash(), + inner_puzzle_hash=new_puz_hash, confirmed=False, confirmed_at_height=uint32(0), timestamp=uint64(0), lineage_proof=LineageProof( singleton_record.coin_id, - next_inner_puzzle.get_tree_hash(), + new_puz_hash, singleton_record.lineage_proof.amount, ), generation=uint32(singleton_record.generation + 1), @@ -521,6 +542,49 @@ class DataLayerWallet: ) return txs + async def generate_signed_transaction( + self, + amounts: List[uint64], + puzzle_hashes: List[bytes32], + fee: uint64 = uint64(0), + coins: Set[Coin] = set(), + memos: Optional[List[List[bytes]]] = None, # ignored + coin_announcements_to_consume: Optional[Set[Announcement]] = None, + puzzle_announcements_to_consume: Optional[Set[Announcement]] = None, + ignore_max_send_amount: bool = False, # ignored + # This wallet only + launcher_id: Optional[bytes32] = None, + new_root_hash: Optional[bytes32] = None, + sign: bool = True, # This only prevent signing of THIS wallet's part of the tx (fee will still be signed) + ) -> List[TransactionRecord]: + # Figure out the launcher ID + if len(coins) == 0: + if launcher_id is None: + raise ValueError("Not enough info to know which DL coin to send") + else: + if len(coins) != 1: + raise ValueError("The wallet can only send one DL coin at a time") + else: + record = await self.wallet_state_manager.dl_store.get_singleton_record(next(iter(coins)).name()) + if record is None: + raise ValueError("The specified coin is not a tracked DL") + else: + launcher_id = record.launcher_id + + if len(amounts) != 1 or len(puzzle_hashes) != 1: + raise ValueError("The wallet can only send one DL coin to one place at a time") + + return await self.create_update_state_spend( + launcher_id, + new_root_hash, + puzzle_hashes[0], + amounts[0], + fee, + coin_announcements_to_consume, + puzzle_announcements_to_consume, + sign, + ) + async def get_spendable_singleton_info(self, launcher_id: bytes32) -> Tuple[SingletonRecord, LineageProof]: # First, let's make sure this is a singleton that we track and that we can spend singleton_record: Optional[SingletonRecord] = await self.get_latest_singleton(launcher_id) @@ -732,7 +796,7 @@ class DataLayerWallet: await self.create_update_state_spend( launcher_id, singleton.root, - fee, + fee=fee, in_transaction=in_transaction, ) ) diff --git a/tests/wallet/db_wallet/test_dl_wallet.py b/tests/wallet/db_wallet/test_dl_wallet.py index 731d248a22..41696dedea 100644 --- a/tests/wallet/db_wallet/test_dl_wallet.py +++ b/tests/wallet/db_wallet/test_dl_wallet.py @@ -287,9 +287,20 @@ class TestDLWallet: new_root = MerkleTree([Program.to("root").get_tree_hash()]).calculate_root() - txs = await dl_wallet.create_update_state_spend(launcher_id, new_root, fee=uint64(1999999999999)) + txs = await dl_wallet.generate_signed_transaction( + [previous_record.lineage_proof.amount], + [previous_record.inner_puzzle_hash], + launcher_id=previous_record.launcher_id, + new_root_hash=new_root, + fee=uint64(1999999999999), + ) with pytest.raises(ValueError, match="is currently pending"): - await dl_wallet.create_update_state_spend(launcher_id, new_root) + await dl_wallet.generate_signed_transaction( + [previous_record.lineage_proof.amount], + [previous_record.inner_puzzle_hash], + coins=[txs[0].spend_bundle.removals()[0]], + fee=uint64(1999999999999), + ) new_record = await dl_wallet.get_latest_singleton(launcher_id) assert new_record is not None From ad697dad24b34a9f60daa63e0ca00e5255f4b251 Mon Sep 17 00:00:00 2001 From: Matt Hauff Date: Wed, 13 Jul 2022 07:07:41 -0500 Subject: [PATCH 55/62] mypy --- tests/wallet/db_wallet/test_dl_wallet.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/wallet/db_wallet/test_dl_wallet.py b/tests/wallet/db_wallet/test_dl_wallet.py index 41696dedea..483e4da73a 100644 --- a/tests/wallet/db_wallet/test_dl_wallet.py +++ b/tests/wallet/db_wallet/test_dl_wallet.py @@ -284,6 +284,8 @@ class TestDLWallet: await asyncio.sleep(0.5) previous_record = await dl_wallet.get_latest_singleton(launcher_id) + assert previous_record is not None + assert previous_record.lineage_proof.amount is not None new_root = MerkleTree([Program.to("root").get_tree_hash()]).calculate_root() @@ -294,11 +296,12 @@ class TestDLWallet: new_root_hash=new_root, fee=uint64(1999999999999), ) + assert txs[0].spend_bundle is not None with pytest.raises(ValueError, match="is currently pending"): await dl_wallet.generate_signed_transaction( [previous_record.lineage_proof.amount], [previous_record.inner_puzzle_hash], - coins=[txs[0].spend_bundle.removals()[0]], + coins=set([txs[0].spend_bundle.removals()[0]]), fee=uint64(1999999999999), ) From 59663b2f76f4b5bdb4e53867790f6e6db00cfd40 Mon Sep 17 00:00:00 2001 From: Matt Hauff Date: Wed, 20 Jul 2022 14:24:49 -0500 Subject: [PATCH 56/62] Delete an old test file --- tests/wallet/db_wallet/test_db_clvm.py | 345 ------------------------- 1 file changed, 345 deletions(-) delete mode 100644 tests/wallet/db_wallet/test_db_clvm.py diff --git a/tests/wallet/db_wallet/test_db_clvm.py b/tests/wallet/db_wallet/test_db_clvm.py deleted file mode 100644 index 988edee392..0000000000 --- a/tests/wallet/db_wallet/test_db_clvm.py +++ /dev/null @@ -1,345 +0,0 @@ -import pytest -import pytest_asyncio - -from blspy import G2Element -from typing import Dict, Tuple - -from chia.clvm.spend_sim import SpendSim, SimClient -from chia.wallet.db_wallet.db_wallet_puzzles import ( - create_host_fullpuz, - create_offer_fullpuz, - SINGLETON_LAUNCHER, - create_host_layer_puzzle, - solve_data_layer_to_report, - solve_data_layer_to_update, - solve_dl_offer_for_claim, - solve_dl_offer_for_recover, -) -from chia.wallet.lineage_proof import LineageProof -from chia.wallet.puzzles.singleton_top_layer import solution_for_singleton -from chia.types.blockchain_format.program import Program -from chia.types.blockchain_format.coin import Coin -from chia.types.blockchain_format.sized_bytes import bytes32 -from chia.types.mempool_inclusion_status import MempoolInclusionStatus -from chia.types.spend_bundle import SpendBundle -from chia.types.coin_spend import CoinSpend -from chia.util.ints import uint64 -from chia.wallet.util.merkle_tree import MerkleTree - -from tests.clvm.benchmark_costs import cost_of_spend_bundle - - -ACS = Program.to(1) -ACS_2 = Program.to([3, "2", 1, []]) # (if "2" 1 ()) == 1 -ACS_PH = ACS.get_tree_hash() -ACS_2_PH = ACS_2.get_tree_hash() -SINGLETON_LAUNCHER_HASH = SINGLETON_LAUNCHER.get_tree_hash() -OFFER_AMOUNT = uint64(13) - -pytestmark = pytest.mark.data_layer - - -SetupArgs = Tuple[SpendSim, SimClient, Coin, LineageProof, Coin, Coin, Program, Program] - - -class TestDLLifecycle: - cost: Dict[str, int] = {} - - def get_merkle_tree(self, string: str) -> MerkleTree: - return MerkleTree([Program.to(string).get_tree_hash(), Program.to(string).get_tree_hash()]) - - def get_merkle_root(self, string: str) -> bytes32: - return self.get_merkle_tree(string).calculate_root() - - def hash_merkle_value(self, string: str) -> bytes32: - # https://github.com/Chia-Network/clvm/pull/102 - # https://github.com/Chia-Network/clvm/pull/106 - return Program.to(string).get_tree_hash() # type: ignore[no-any-return] - - def get_merkle_proof(self, string: str) -> Program: - proof = self.get_merkle_tree(string).generate_proof(self.hash_merkle_value(string)) - # https://github.com/Chia-Network/clvm/pull/102 - # https://github.com/Chia-Network/clvm/pull/106 - return Program.to(proof) # type: ignore[no-any-return] - - @pytest_asyncio.fixture(scope="function") - async def setup_sim_and_singleton(self) -> SetupArgs: - # https://github.com/Chia-Network/chia-blockchain/pull/11819 - sim = await SpendSim.create() # type: ignore[no-untyped-call] - sim_client = SimClient(sim) # type: ignore[no-untyped-call] - await sim.farm_block() - await sim.farm_block(ACS_PH) - fund_coin = (await sim_client.get_coin_records_by_puzzle_hash(ACS_PH))[0].coin - launcher_coin = Coin(fund_coin.name(), SINGLETON_LAUNCHER_HASH, uint64(1)) - singleton_puzzle = create_host_fullpuz(ACS_PH, self.get_merkle_root("init"), launcher_coin.name()) - good_puzzle = create_offer_fullpuz( - self.hash_merkle_value("init"), - launcher_coin.name(), - ACS_2_PH, - ACS_PH, - uint64(60), # 60 seconds - ) - bad_puzzle = create_offer_fullpuz( - self.hash_merkle_value("nope"), - launcher_coin.name(), - ACS_2_PH, - ACS_PH, - uint64(60), # 60 seconds - ) - bundle = SpendBundle( - [ - CoinSpend( - fund_coin, - ACS, - Program.to( - [ - [51, SINGLETON_LAUNCHER_HASH, 1], - [51, good_puzzle.get_tree_hash(), OFFER_AMOUNT], - [51, bad_puzzle.get_tree_hash(), OFFER_AMOUNT], - ] - ), - ), - CoinSpend( - launcher_coin, - SINGLETON_LAUNCHER, - Program.to([singleton_puzzle.get_tree_hash(), launcher_coin.amount, []]), - ), - ], - G2Element(), - ) - result = (await sim_client.push_tx(bundle))[0] - assert result == MempoolInclusionStatus.SUCCESS - self.cost["launch singleton and create two coins"] = cost_of_spend_bundle(bundle) - await sim.farm_block() - singleton = (await sim_client.get_coin_records_by_puzzle_hashes([singleton_puzzle.get_tree_hash()]))[0].coin - good_offer_coin = (await sim_client.get_coin_records_by_puzzle_hashes([good_puzzle.get_tree_hash()]))[0].coin - bad_offer_coin = (await sim_client.get_coin_records_by_puzzle_hashes([bad_puzzle.get_tree_hash()]))[0].coin - return ( - sim, - sim_client, - singleton, - LineageProof(parent_name=launcher_coin.parent_coin_info, amount=uint64(launcher_coin.amount)), - good_offer_coin, - bad_offer_coin, - good_puzzle, - bad_puzzle, - ) - - @pytest.mark.asyncio() - async def test_report(self, setup_sim_and_singleton: SetupArgs) -> None: - sim, sim_client, singleton, lineage_proof = setup_sim_and_singleton[0:4] - - try: - bundle = SpendBundle( - [ - CoinSpend( - singleton, - create_host_fullpuz(ACS_PH, self.get_merkle_root("init"), singleton.parent_coin_info), - solution_for_singleton( - lineage_proof, - uint64(singleton.amount), - solve_data_layer_to_report(uint64(singleton.amount)), - ), - ) - ], - G2Element(), - ) - result = (await sim_client.push_tx(bundle))[0] - assert result == MempoolInclusionStatus.SUCCESS - self.cost["report spend"] = cost_of_spend_bundle(bundle) - await sim.farm_block() - new_singleton = (await sim_client.get_coin_records_by_parent_ids([singleton.name()]))[0].coin - assert new_singleton.puzzle_hash == singleton.puzzle_hash - finally: - # https://github.com/Chia-Network/chia-blockchain/pull/11819 - await sim.close() # type: ignore[no-untyped-call] - - @pytest.mark.asyncio() - async def test_update(self, setup_sim_and_singleton: SetupArgs) -> None: - sim, sim_client, singleton, lineage_proof = setup_sim_and_singleton[0:4] - - try: - bundle = SpendBundle( - [ - CoinSpend( - singleton, - create_host_fullpuz(ACS_PH, self.get_merkle_root("init"), singleton.parent_coin_info), - solution_for_singleton( - lineage_proof, - uint64(singleton.amount), - solve_data_layer_to_update( - ACS, - Program.to( - [ - [ - 51, - create_host_layer_puzzle( - ACS_PH, self.get_merkle_root("update") - ).get_tree_hash(), - singleton.amount, - ] - ] - ), - ), - ), - ) - ], - G2Element(), - ) - result = (await sim_client.push_tx(bundle))[0] - assert result == MempoolInclusionStatus.SUCCESS - self.cost["update spend"] = cost_of_spend_bundle(bundle) - await sim.farm_block() - new_singleton = (await sim_client.get_coin_records_by_parent_ids([singleton.name()]))[0].coin - assert ( - new_singleton.puzzle_hash - == create_host_fullpuz( - ACS_PH, self.get_merkle_root("update"), singleton.parent_coin_info - ).get_tree_hash() - ) - finally: - # https://github.com/Chia-Network/chia-blockchain/pull/11819 - await sim.close() # type: ignore[no-untyped-call] - - @pytest.mark.asyncio() - async def test_offer_cant_claim(self, setup_sim_and_singleton: SetupArgs) -> None: - ( - sim, - sim_client, - singleton, - lineage_proof, - good_offer_coin, - bad_offer_coin, - good_offer_puzzle, - bad_offer_puzzle, - ) = setup_sim_and_singleton - - try: - bundle = SpendBundle( - [ - CoinSpend( - singleton, - create_host_fullpuz(ACS_PH, self.get_merkle_root("init"), singleton.parent_coin_info), - solution_for_singleton( - lineage_proof, - uint64(singleton.amount), - solve_data_layer_to_report(uint64(singleton.amount)), - ), - ), - CoinSpend( - bad_offer_coin, - bad_offer_puzzle, - solve_dl_offer_for_claim( - OFFER_AMOUNT, - ACS_PH, - self.get_merkle_root("init"), - self.get_merkle_proof("nope"), - ), - ), - ], - G2Element(), - ) - result = (await sim_client.push_tx(bundle))[0] - assert result == MempoolInclusionStatus.FAILED - offer_cs = bundle.coin_spends[1] - with pytest.raises(ValueError, match="clvm raise"): - offer_cs.puzzle_reveal.to_program().run(offer_cs.solution.to_program()) - finally: - # https://github.com/Chia-Network/chia-blockchain/pull/11819 - await sim.close() # type: ignore[no-untyped-call] - - @pytest.mark.asyncio() - async def test_offer_can_claim(self, setup_sim_and_singleton: SetupArgs) -> None: - ( - sim, - sim_client, - singleton, - lineage_proof, - good_offer_coin, - bad_offer_coin, - good_offer_puzzle, - bad_offer_puzzle, - ) = setup_sim_and_singleton - - try: - bundle = SpendBundle( - [ - CoinSpend( - singleton, - create_host_fullpuz(ACS_PH, self.get_merkle_root("init"), singleton.parent_coin_info), - solution_for_singleton( - lineage_proof, - uint64(singleton.amount), - solve_data_layer_to_report(uint64(singleton.amount)), - ), - ), - CoinSpend( - good_offer_coin, - good_offer_puzzle, - solve_dl_offer_for_claim( - OFFER_AMOUNT, - ACS_PH, - self.get_merkle_root("init"), - self.get_merkle_proof("init"), - ), - ), - ], - G2Element(), - ) - result = (await sim_client.push_tx(bundle))[0] - assert result == MempoolInclusionStatus.SUCCESS - self.cost["offer claim"] = cost_of_spend_bundle(bundle) - await sim.farm_block() - offer_reward = (await sim_client.get_coin_records_by_parent_ids([good_offer_coin.name()]))[0].coin - assert offer_reward.puzzle_hash == ACS_2_PH - finally: - # https://github.com/Chia-Network/chia-blockchain/pull/11819 - await sim.close() # type: ignore[no-untyped-call] - - @pytest.mark.asyncio() - async def test_offer_recovery(self, setup_sim_and_singleton: SetupArgs) -> None: - ( - sim, - sim_client, - singleton, - lineage_proof, - good_offer_coin, - bad_offer_coin, - good_offer_puzzle, - bad_offer_puzzle, - ) = setup_sim_and_singleton - - try: - bundle = SpendBundle( - [ - CoinSpend( - bad_offer_coin, - bad_offer_puzzle, - solve_dl_offer_for_recover(OFFER_AMOUNT), - ), - ], - G2Element(), - ) - result = (await sim_client.push_tx(bundle))[0] - assert result == MempoolInclusionStatus.FAILED - - # Should work after a minute - sim.pass_time(uint64(60)) - await sim.farm_block() - result = (await sim_client.push_tx(bundle))[0] - assert result == MempoolInclusionStatus.SUCCESS - self.cost["offer recovery"] = cost_of_spend_bundle(bundle) - await sim.farm_block() - - offer_reward = (await sim_client.get_coin_records_by_parent_ids([bad_offer_coin.name()]))[0].coin - assert offer_reward.puzzle_hash == ACS_PH - finally: - # https://github.com/Chia-Network/chia-blockchain/pull/11819 - await sim.close() # type: ignore[no-untyped-call] - - def test_cost(self) -> None: - import json - import logging - - log = logging.getLogger(__name__) - log.warning(json.dumps(self.cost)) From 4ae2b755bc58642fa222caa1f1fe4a16810e7ec8 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 25 Jul 2022 22:51:43 -0700 Subject: [PATCH 57/62] make init_data_layer a context manager instead --- tests/core/data_layer/test_data_rpc.py | 30 +++++++++++++++----------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/tests/core/data_layer/test_data_rpc.py b/tests/core/data_layer/test_data_rpc.py index b13310d0d3..8318e4a61a 100644 --- a/tests/core/data_layer/test_data_rpc.py +++ b/tests/core/data_layer/test_data_rpc.py @@ -1,4 +1,5 @@ import asyncio +import contextlib from pathlib import Path from typing import AsyncIterator, Dict, List, Tuple, Set import pytest @@ -32,6 +33,7 @@ nodes = Tuple[WalletNode, FullNodeSimulator] nodes_with_port = Tuple[WalletNode, FullNodeSimulator, int] +@contextlib.asynccontextmanager async def init_data_layer(wallet_rpc_port: int, bt: BlockTools, db_path: Path) -> AsyncIterator[DataLayer]: config = bt.config config["data_layer"]["wallet_peer"]["port"] = wallet_rpc_port @@ -45,9 +47,11 @@ async def init_data_layer(wallet_rpc_port: int, bt: BlockTools, db_path: Path) - kwargs.update(parse_cli_args=False) service = Service(**kwargs, running_new_process=False) await service.start() - yield service._api.data_layer - service.stop() - await service.wait_closed() + try: + yield service._api.data_layer + finally: + service.stop() + await service.wait_closed() @pytest_asyncio.fixture(scope="function") @@ -94,7 +98,7 @@ async def test_create_insert_get(one_wallet_node_and_rpc: nodes_with_port, bt: B ) await time_out_assert(15, wallet_node.wallet_state_manager.main_wallet.get_confirmed_balance, funds) wallet_rpc_api = WalletRpcApi(wallet_node) - async for data_layer in init_data_layer(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path): + async with init_data_layer(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path) as data_layer: # test insert data_rpc_api = DataLayerRpcApi(data_layer) key = b"a" @@ -162,7 +166,7 @@ async def test_upsert(one_wallet_node_and_rpc: nodes_with_port, bt: BlockTools, ) await time_out_assert(15, wallet_node.wallet_state_manager.main_wallet.get_confirmed_balance, funds) wallet_rpc_api = WalletRpcApi(wallet_node) - async for data_layer in init_data_layer(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path): + async with init_data_layer(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path) as data_layer: # test insert data_rpc_api = DataLayerRpcApi(data_layer) key = b"a" @@ -207,7 +211,7 @@ async def test_create_double_insert(one_wallet_node_and_rpc: nodes_with_port, bt ) await time_out_assert(15, wallet_node.wallet_state_manager.main_wallet.get_confirmed_balance, funds) wallet_rpc_api = WalletRpcApi(wallet_node) - async for data_layer in init_data_layer(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path): + async with init_data_layer(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path) as data_layer: data_rpc_api = DataLayerRpcApi(data_layer) res = await data_rpc_api.create_data_store({}) assert res is not None @@ -269,7 +273,7 @@ async def test_keys_values_ancestors(one_wallet_node_and_rpc: nodes_with_port, b await time_out_assert(15, wallet_node.wallet_state_manager.main_wallet.get_confirmed_balance, funds) wallet_rpc_api = WalletRpcApi(wallet_node) # TODO: with this being a pseudo context manager'ish thing it doesn't actually handle shutdown - async for data_layer in init_data_layer(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path): + async with init_data_layer(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path) as data_layer: data_rpc_api = DataLayerRpcApi(data_layer) res = await data_rpc_api.create_data_store({}) assert res is not None @@ -352,7 +356,7 @@ async def test_get_roots(one_wallet_node_and_rpc: nodes_with_port, bt: BlockTool ) await time_out_assert(15, wallet_node.wallet_state_manager.main_wallet.get_confirmed_balance, funds) wallet_rpc_api = WalletRpcApi(wallet_node) - async for data_layer in init_data_layer(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path): + async with init_data_layer(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path) as data_layer: data_rpc_api = DataLayerRpcApi(data_layer) res = await data_rpc_api.create_data_store({}) assert res is not None @@ -426,7 +430,7 @@ async def test_get_root_history(one_wallet_node_and_rpc: nodes_with_port, bt: Bl ) await time_out_assert(15, wallet_node.wallet_state_manager.main_wallet.get_confirmed_balance, funds) wallet_rpc_api = WalletRpcApi(wallet_node) - async for data_layer in init_data_layer(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path): + async with init_data_layer(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path) as data_layer: data_rpc_api = DataLayerRpcApi(data_layer) res = await data_rpc_api.create_data_store({}) assert res is not None @@ -503,7 +507,7 @@ async def test_get_kv_diff(one_wallet_node_and_rpc: nodes_with_port, bt: BlockTo ) await time_out_assert(15, wallet_node.wallet_state_manager.main_wallet.get_confirmed_balance, funds) wallet_rpc_api = WalletRpcApi(wallet_node) - async for data_layer in init_data_layer(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path): + async with init_data_layer(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path) as data_layer: data_rpc_api = DataLayerRpcApi(data_layer) res = await data_rpc_api.create_data_store({}) assert res is not None @@ -595,7 +599,7 @@ async def test_batch_update_matches_single_operations( ) await time_out_assert(15, wallet_node.wallet_state_manager.main_wallet.get_confirmed_balance, funds) wallet_rpc_api = WalletRpcApi(wallet_node) - async for data_layer in init_data_layer(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path): + async with init_data_layer(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path) as data_layer: data_rpc_api = DataLayerRpcApi(data_layer) res = await data_rpc_api.create_data_store({}) assert res is not None @@ -705,7 +709,7 @@ async def test_get_owned_stores(one_wallet_node_and_rpc: nodes_with_port, bt: Bl ) await time_out_assert(15, wallet_node.wallet_state_manager.main_wallet.get_confirmed_balance, funds) - async for data_layer in init_data_layer(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path): + async with init_data_layer(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path) as data_layer: data_rpc_api = DataLayerRpcApi(data_layer) expected_store_ids = [] @@ -741,7 +745,7 @@ async def test_subscriptions(one_wallet_node_and_rpc: nodes_with_port, bt: Block ) await time_out_assert(15, wallet_node.wallet_state_manager.main_wallet.get_confirmed_balance, funds) - async for data_layer in init_data_layer(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path): + async with init_data_layer(wallet_rpc_port=wallet_rpc_port, bt=bt, db_path=tmp_path) as data_layer: data_rpc_api = DataLayerRpcApi(data_layer) res = await data_rpc_api.create_data_store({}) From d6ccba7a99bf17846321bdc8c39d710a68eac93d Mon Sep 17 00:00:00 2001 From: Amine Khaldi Date: Wed, 27 Jul 2022 16:55:16 +0100 Subject: [PATCH 58/62] Reflect the change on datalayer related code. --- tests/wallet/db_wallet/test_db_clvm.py | 20 +++++++------------- tests/wallet/db_wallet/test_db_graftroot.py | 2 +- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/tests/wallet/db_wallet/test_db_clvm.py b/tests/wallet/db_wallet/test_db_clvm.py index 988edee392..a6f0f492d4 100644 --- a/tests/wallet/db_wallet/test_db_clvm.py +++ b/tests/wallet/db_wallet/test_db_clvm.py @@ -64,9 +64,8 @@ class TestDLLifecycle: @pytest_asyncio.fixture(scope="function") async def setup_sim_and_singleton(self) -> SetupArgs: - # https://github.com/Chia-Network/chia-blockchain/pull/11819 - sim = await SpendSim.create() # type: ignore[no-untyped-call] - sim_client = SimClient(sim) # type: ignore[no-untyped-call] + sim = await SpendSim.create() + sim_client = SimClient(sim) await sim.farm_block() await sim.farm_block(ACS_PH) fund_coin = (await sim_client.get_coin_records_by_puzzle_hash(ACS_PH))[0].coin @@ -151,8 +150,7 @@ class TestDLLifecycle: new_singleton = (await sim_client.get_coin_records_by_parent_ids([singleton.name()]))[0].coin assert new_singleton.puzzle_hash == singleton.puzzle_hash finally: - # https://github.com/Chia-Network/chia-blockchain/pull/11819 - await sim.close() # type: ignore[no-untyped-call] + await sim.close() @pytest.mark.asyncio() async def test_update(self, setup_sim_and_singleton: SetupArgs) -> None: @@ -198,8 +196,7 @@ class TestDLLifecycle: ).get_tree_hash() ) finally: - # https://github.com/Chia-Network/chia-blockchain/pull/11819 - await sim.close() # type: ignore[no-untyped-call] + await sim.close() @pytest.mark.asyncio() async def test_offer_cant_claim(self, setup_sim_and_singleton: SetupArgs) -> None: @@ -245,8 +242,7 @@ class TestDLLifecycle: with pytest.raises(ValueError, match="clvm raise"): offer_cs.puzzle_reveal.to_program().run(offer_cs.solution.to_program()) finally: - # https://github.com/Chia-Network/chia-blockchain/pull/11819 - await sim.close() # type: ignore[no-untyped-call] + await sim.close() @pytest.mark.asyncio() async def test_offer_can_claim(self, setup_sim_and_singleton: SetupArgs) -> None: @@ -293,8 +289,7 @@ class TestDLLifecycle: offer_reward = (await sim_client.get_coin_records_by_parent_ids([good_offer_coin.name()]))[0].coin assert offer_reward.puzzle_hash == ACS_2_PH finally: - # https://github.com/Chia-Network/chia-blockchain/pull/11819 - await sim.close() # type: ignore[no-untyped-call] + await sim.close() @pytest.mark.asyncio() async def test_offer_recovery(self, setup_sim_and_singleton: SetupArgs) -> None: @@ -334,8 +329,7 @@ class TestDLLifecycle: offer_reward = (await sim_client.get_coin_records_by_parent_ids([bad_offer_coin.name()]))[0].coin assert offer_reward.puzzle_hash == ACS_PH finally: - # https://github.com/Chia-Network/chia-blockchain/pull/11819 - await sim.close() # type: ignore[no-untyped-call] + await sim.close() def test_cost(self) -> None: import json diff --git a/tests/wallet/db_wallet/test_db_graftroot.py b/tests/wallet/db_wallet/test_db_graftroot.py index 47b5c4ab63..1592e66612 100644 --- a/tests/wallet/db_wallet/test_db_graftroot.py +++ b/tests/wallet/db_wallet/test_db_graftroot.py @@ -124,4 +124,4 @@ async def test_graftroot(setup_sim: Tuple[SpendSim, SimClient]) -> None: with pytest.raises(ValueError, match="clvm raise"): graftroot_puzzle.run(graftroot_spend.solution.to_program()) finally: - await sim.close() # type: ignore[no-untyped-call] + await sim.close() From 3047bc262aa153a40e50add52a51accf21679566 Mon Sep 17 00:00:00 2001 From: Amine Khaldi Date: Fri, 29 Jul 2022 21:42:45 +0100 Subject: [PATCH 59/62] Update service construction in init_data_layer(). --- tests/core/data_layer/test_data_rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/data_layer/test_data_rpc.py b/tests/core/data_layer/test_data_rpc.py index 8318e4a61a..a945890897 100644 --- a/tests/core/data_layer/test_data_rpc.py +++ b/tests/core/data_layer/test_data_rpc.py @@ -45,7 +45,7 @@ async def init_data_layer(wallet_rpc_port: int, bt: BlockTools, db_path: Path) - save_config(bt.root_path, "config.yaml", config) kwargs = service_kwargs_for_data_layer(root_path=bt.root_path, config=config) kwargs.update(parse_cli_args=False) - service = Service(**kwargs, running_new_process=False) + service = Service(**kwargs) await service.start() try: yield service._api.data_layer From 597743fc157a2b0c10b8351a6a795feb6421f8b6 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 30 Jul 2022 21:48:13 -0700 Subject: [PATCH 60/62] correct parameter order in call to WalletNode.fetch_puzzle_solution() --- chia/wallet/wallet_state_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chia/wallet/wallet_state_manager.py b/chia/wallet/wallet_state_manager.py index e4bf5e6ef1..ce984c6e74 100644 --- a/chia/wallet/wallet_state_manager.py +++ b/chia/wallet/wallet_state_manager.py @@ -1093,7 +1093,7 @@ class WalletStateManager: curr_coin_state = new_coin_state[0] if record.wallet_type == WalletType.DATA_LAYER: singleton_spend = await self.wallet_node.fetch_puzzle_solution( - peer, coin_state.spent_height, coin_state.coin + coin_state.spent_height, coin_state.coin, peer ) dl_wallet = self.wallets[uint32(record.wallet_id)] await dl_wallet.singleton_removed( From c45d1e0407b03658253344016e8f8ba82657feb1 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 30 Jul 2022 22:01:35 -0700 Subject: [PATCH 61/62] make flake8 5 happy (cherry picked from commit a8c3723b9ae1583ed25afc1f6b13f5a82862097a) --- chia/cmds/passphrase.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/chia/cmds/passphrase.py b/chia/cmds/passphrase.py index dff0fbeb29..35ad05d6c5 100644 --- a/chia/cmds/passphrase.py +++ b/chia/cmds/passphrase.py @@ -12,9 +12,9 @@ def passphrase_cmd(): @passphrase_cmd.command( "set", - help="""Sets or updates the keyring passphrase. If --passphrase-file and/or --current-passphrase-file options are provided, - the passphrases will be read from the specified files. Otherwise, a prompt will be provided to enter the - passphrase.""", + help="""Sets or updates the keyring passphrase. If --passphrase-file and/or --current-passphrase-file options are + provided, the passphrases will be read from the specified files. Otherwise, a prompt will be provided to + enter the passphrase.""", short_help="Set or update the keyring passphrase", ) @click.option("--passphrase-file", type=click.File("r"), help="File or descriptor to read the passphrase from") @@ -70,8 +70,8 @@ def set_cmd( @passphrase_cmd.command( "remove", - help="""Remove the keyring passphrase. If the --current-passphrase-file option is provided, the passphrase will be read from - the specified file. Otherwise, a prompt will be provided to enter the passphrase.""", + help="""Remove the keyring passphrase. If the --current-passphrase-file option is provided, the passphrase will be + read from the specified file. Otherwise, a prompt will be provided to enter the passphrase.""", short_help="Remove the keyring passphrase", ) @click.option( From da67da07e84c391a15d7656f05f9b8a45ba80f92 Mon Sep 17 00:00:00 2001 From: Amine Khaldi Date: Sun, 31 Jul 2022 20:29:58 +0100 Subject: [PATCH 62/62] Reflect the change on datalayer related code. --- chia/rpc/data_layer_rpc_api.py | 36 +++++++++++++++++----------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/chia/rpc/data_layer_rpc_api.py b/chia/rpc/data_layer_rpc_api.py index 6518759ef7..9c57fde9e2 100644 --- a/chia/rpc/data_layer_rpc_api.py +++ b/chia/rpc/data_layer_rpc_api.py @@ -4,7 +4,7 @@ from pathlib import Path from chia.data_layer.data_layer import DataLayer from chia.data_layer.data_layer_util import Side, Subscription -from chia.rpc.rpc_server import Endpoint +from chia.rpc.rpc_server import Endpoint, EndpointResult from chia.types.blockchain_format.sized_bytes import bytes32 from chia.util.byte_types import hexstr_to_bytes @@ -73,20 +73,20 @@ class DataLayerRpcApi: "/add_missing_files": self.add_missing_files, } - async def create_data_store(self, request: Dict[str, Any]) -> Dict[str, Any]: + async def create_data_store(self, request: Dict[str, Any]) -> EndpointResult: if self.service is None: raise Exception("Data layer not created") fee = get_fee(self.service.config, request) txs, value = await self.service.create_store(uint64(fee)) return {"txs": txs, "id": value.hex()} - async def get_owned_stores(self, request: Dict[str, Any]) -> Dict[str, Any]: + async def get_owned_stores(self, request: Dict[str, Any]) -> EndpointResult: if self.service is None: raise Exception("Data layer not created") singleton_records = await self.service.get_owned_stores() return {"store_ids": [singleton.launcher_id.hex() for singleton in singleton_records]} - async def get_value(self, request: Dict[str, Any]) -> Dict[str, Any]: + async def get_value(self, request: Dict[str, Any]) -> EndpointResult: store_id = bytes32.from_hexstr(request["id"]) key = hexstr_to_bytes(request["key"]) if self.service is None: @@ -97,7 +97,7 @@ class DataLayerRpcApi: hex = value.hex() return {"value": hex} - async def get_keys_values(self, request: Dict[str, Any]) -> Dict[str, Any]: + async def get_keys_values(self, request: Dict[str, Any]) -> EndpointResult: store_id = bytes32(hexstr_to_bytes(request["id"])) root_hash = request.get("root_hash") if root_hash is not None: @@ -111,7 +111,7 @@ class DataLayerRpcApi: json_nodes.append(json) return {"keys_values": json_nodes} - async def get_ancestors(self, request: Dict[str, Any]) -> Dict[str, Any]: + async def get_ancestors(self, request: Dict[str, Any]) -> EndpointResult: store_id = bytes32(hexstr_to_bytes(request["id"])) node_hash = bytes32.from_hexstr(request["hash"]) if self.service is None: @@ -119,7 +119,7 @@ class DataLayerRpcApi: value = await self.service.get_ancestors(node_hash, store_id) return {"ancestors": value} - async def batch_update(self, request: Dict[str, Any]) -> Dict[str, Any]: + async def batch_update(self, request: Dict[str, Any]) -> EndpointResult: """ id - the id of the store we are operating on changelist - a list of changes to apply on store @@ -135,7 +135,7 @@ class DataLayerRpcApi: raise Exception(f"Batch update failed for: {store_id}") return {"tx_id": transaction_record.name} - async def insert(self, request: Dict[str, Any]) -> Dict[str, Any]: + async def insert(self, request: Dict[str, Any]) -> EndpointResult: """ rows_to_add a list of clvm objects as bytes to add to talbe rows_to_remove a list of row hashes to remove @@ -151,7 +151,7 @@ class DataLayerRpcApi: transaction_record = await self.service.batch_update(store_id, changelist, uint64(fee)) return {"tx_id": transaction_record.name} - async def delete_key(self, request: Dict[str, Any]) -> Dict[str, Any]: + async def delete_key(self, request: Dict[str, Any]) -> EndpointResult: """ rows_to_add a list of clvm objects as bytes to add to talbe rows_to_remove a list of row hashes to remove @@ -166,7 +166,7 @@ class DataLayerRpcApi: transaction_record = await self.service.batch_update(store_id, changelist, uint64(fee)) return {"tx_id": transaction_record.name} - async def get_root(self, request: Dict[str, Any]) -> Dict[str, Any]: + async def get_root(self, request: Dict[str, Any]) -> EndpointResult: """get hash of latest tree root""" store_id = bytes32(hexstr_to_bytes(request["id"])) # todo input checks @@ -177,7 +177,7 @@ class DataLayerRpcApi: raise Exception(f"Failed to get root for {store_id.hex()}") return {"hash": rec.root, "confirmed": rec.confirmed, "timestamp": rec.timestamp} - async def get_local_root(self, request: Dict[str, Any]) -> Dict[str, Any]: + async def get_local_root(self, request: Dict[str, Any]) -> EndpointResult: """get hash of latest tree root saved in our local datastore""" store_id = bytes32(hexstr_to_bytes(request["id"])) # todo input checks @@ -186,7 +186,7 @@ class DataLayerRpcApi: res = await self.service.get_local_root(store_id) return {"hash": res} - async def get_roots(self, request: Dict[str, Any]) -> Dict[str, Any]: + async def get_roots(self, request: Dict[str, Any]) -> EndpointResult: """ get state hashes for a list of roots """ @@ -202,7 +202,7 @@ class DataLayerRpcApi: roots.append({"id": id_bytes, "hash": rec.root, "confirmed": rec.confirmed, "timestamp": rec.timestamp}) return {"root_hashes": roots} - async def subscribe(self, request: Dict[str, Any]) -> Dict[str, Any]: + async def subscribe(self, request: Dict[str, Any]) -> EndpointResult: """ subscribe to singleton """ @@ -217,7 +217,7 @@ class DataLayerRpcApi: await self.service.subscribe(store_id=store_id_bytes, urls=urls) return {} - async def unsubscribe(self, request: Dict[str, Any]) -> Dict[str, Any]: + async def unsubscribe(self, request: Dict[str, Any]) -> EndpointResult: """ unsubscribe from singleton """ @@ -230,7 +230,7 @@ class DataLayerRpcApi: await self.service.unsubscribe(store_id_bytes) return {} - async def subscriptions(self, request: Dict[str, Any]) -> Dict[str, Any]: + async def subscriptions(self, request: Dict[str, Any]) -> EndpointResult: """ List current subscriptions """ @@ -239,7 +239,7 @@ class DataLayerRpcApi: subscriptions: List[Subscription] = await self.service.get_subscriptions() return {"store_ids": [sub.tree_id.hex() for sub in subscriptions]} - async def add_missing_files(self, request: Dict[str, Any]) -> Dict[str, Any]: + async def add_missing_files(self, request: Dict[str, Any]) -> EndpointResult: """ complete the data server files. """ @@ -257,7 +257,7 @@ class DataLayerRpcApi: await self.service.add_missing_files(tree_id, override, foldername) return {} - async def get_root_history(self, request: Dict[str, Any]) -> Dict[str, Any]: + async def get_root_history(self, request: Dict[str, Any]) -> EndpointResult: """ get history of state hashes for a store """ @@ -271,7 +271,7 @@ class DataLayerRpcApi: res.insert(0, {"root_hash": rec.root, "confirmed": rec.confirmed, "timestamp": rec.timestamp}) return {"root_history": res} - async def get_kv_diff(self, request: Dict[str, Any]) -> Dict[str, Any]: + async def get_kv_diff(self, request: Dict[str, Any]) -> EndpointResult: """ get kv diff between two root hashes """