plotting|tests|github: Fix and test re-loading/removal of plots where we missed the keys (#9578)

* tests: Allow to use custom keys in `BlockTools.new_plot`

* tests: Create three plots with `G1Element()` as farmer/pool key

But not add them as default plots. Just to have plots in the test cache with different keys from
within and outside the keychain for plot loading tests.

create

* test|github: Bump `test-cache` version to `0.28.0`

* plotting: Drop plots from `no_key_filenames` after the keys appeared

* plotting: Drop plots from `no_key_filenames` if the plot was removed

* tests: Test loading of plots where we miss the keys

* plotting: Use `x.copy()` instead of `list(x)`

* plotting: Add a comment about the missing keys plot removal

* tests: Fix after #9595
This commit is contained in:
dustinface
2021-12-17 14:36:29 -08:00
committed by GitHub
parent 9de8235691
commit 2ab146bf2d
53 changed files with 137 additions and 63 deletions
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
+1 -1
View File
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
+1 -1
View File
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
+1 -1
View File
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
+1 -1
View File
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
+1 -1
View File
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
@@ -68,7 +68,7 @@ jobs:
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory
+9
View File
@@ -247,6 +247,10 @@ class PlotManager:
if plot_removed(path):
del self.failed_to_open_filenames[path]
for path in self.no_key_filenames.copy():
if plot_removed(path):
self.no_key_filenames.remove(path)
filenames_to_remove: List[str] = []
for plot_filename, paths_entry in self.plot_filename_paths.items():
loaded_path, duplicated_paths = paths_entry
@@ -395,6 +399,11 @@ class PlotManager:
if not self.open_no_key_filenames:
return None
# If a plot is in `no_key_filenames` the keys were missing in earlier refresh cycles. We can remove
# the current plot from that list if its in there since we passed the key checks above.
if file_path in self.no_key_filenames:
self.no_key_filenames.remove(file_path)
local_sk = master_sk_to_local_sk(local_master_sk)
plot_public_key: G1Element = ProofOfSpace.generate_plot_public_key(
+28 -13
View File
@@ -237,10 +237,22 @@ class BlockTools:
# Pool Plots
for i in range(5):
await self.new_plot(self.pool_ph)
# Some plots with keys that are not in the keychain
for i in range(3):
await self.new_plot(
path=self.plot_dir / "not_in_keychain",
plot_keys=PlotKeys(G1Element(), G1Element(), None),
exclude_final_dir=True,
)
await self.refresh_plots()
async def new_plot(
self, pool_contract_puzzle_hash: Optional[bytes32] = None, path: Path = None
self,
pool_contract_puzzle_hash: Optional[bytes32] = None,
path: Path = None,
plot_keys: Optional[PlotKeys] = None,
exclude_final_dir: bool = False,
) -> Optional[bytes32]:
final_dir = self.plot_dir
if path is not None:
@@ -263,19 +275,21 @@ class BlockTools:
args.nobitfield = False
args.exclude_final_dir = False
args.list_duplicates = False
args.exclude_final_dir = exclude_final_dir
try:
pool_pk: Optional[G1Element] = None
pool_address: Optional[str] = None
if pool_contract_puzzle_hash is None:
pool_pk = self.pool_pk
else:
pool_address = encode_puzzle_hash(pool_contract_puzzle_hash, "xch")
if plot_keys is None:
pool_pk: Optional[G1Element] = None
pool_address: Optional[str] = None
if pool_contract_puzzle_hash is None:
pool_pk = self.pool_pk
else:
pool_address = encode_puzzle_hash(pool_contract_puzzle_hash, "xch")
keys = PlotKeys(self.farmer_pk, pool_pk, pool_address)
plot_keys = PlotKeys(self.farmer_pk, pool_pk, pool_address)
# No datetime in the filename, to get deterministic filenames and not re-plot
created, existed = await create_plots(
args,
keys,
plot_keys,
self.root_path,
use_datetime=False,
test_private_keys=[AugSchemeMPL.key_gen(std_hash(len(self.expected_plots).to_bytes(2, "big")))],
@@ -292,10 +306,11 @@ class BlockTools:
assert len(created) == 0
plot_id_new, path_new = list(existed.items())[0]
# TODO: address hint error and remove ignore
# error: Invalid index type "Optional[bytes32]" for "Dict[bytes32, Path]"; expected type "bytes32"
# [index]
self.expected_plots[plot_id_new] = path_new # type: ignore[index]
if not exclude_final_dir:
# TODO: address hint error and remove ignore
# error: Invalid index type "Optional[bytes32]" for "Dict[bytes32, Path]"; expected type "bytes32"
# [index]
self.expected_plots[plot_id_new] = path_new # type: ignore[index]
# create_plots() updates plot_directories. Ensure we refresh our config to reflect the updated value
self._config["harvester"]["plot_directories"] = load_config(self.root_path, "config.yaml", "harvester")[
+41 -1
View File
@@ -4,6 +4,7 @@ from pathlib import Path
from shutil import copy, move
from typing import Callable, Iterator, List, Optional
import pytest
from blspy import G1Element
from dataclasses import dataclass
from chia.plotting.util import (
@@ -19,6 +20,7 @@ from chia.util.config import create_default_chia_config
from chia.util.path import mkdir
from chia.plotting.manager import PlotManager
from tests.block_tools import get_plot_dir
from tests.plotting.util import get_test_plots
from tests.setup_nodes import bt
from tests.time_out_assert import time_out_assert
@@ -135,7 +137,7 @@ class TestEnvironment:
def test_environment(tmp_path) -> Iterator[TestEnvironment]:
dir_1_count: int = 7
dir_2_count: int = 3
plots: List[Path] = list(sorted(get_plot_dir().glob("*.plot")))
plots: List[Path] = get_test_plots()
assert len(plots) >= dir_1_count + dir_2_count
dir_1: TestDirectory = TestDirectory(tmp_path / "plots" / "1", plots[0:dir_1_count])
@@ -402,6 +404,44 @@ async def test_invalid_plots(test_environment):
assert retry_test_plot not in env.refresh_tester.plot_manager.failed_to_open_filenames
@pytest.mark.asyncio
async def test_keys_missing(test_environment: TestEnvironment) -> None:
env: TestEnvironment = test_environment
not_in_keychain_plots: List[Path] = get_test_plots("not_in_keychain")
dir_not_in_keychain: TestDirectory = TestDirectory(
env.root_path / "plots" / "not_in_keychain", not_in_keychain_plots
)
expected_result = PlotRefreshResult()
# The plots in "not_in_keychain" directory have infinity g1 elements as farmer/pool key so they should be plots
# with missing keys for now
add_plot_directory(env.root_path, str(dir_not_in_keychain.path))
expected_result.loaded = []
expected_result.removed = []
expected_result.processed = len(dir_not_in_keychain)
expected_result.remaining = 0
for i in range(2):
await env.refresh_tester.run(expected_result)
assert len(env.refresh_tester.plot_manager.no_key_filenames) == len(dir_not_in_keychain)
for path in env.refresh_tester.plot_manager.no_key_filenames:
assert path in dir_not_in_keychain.plots
# Delete one of the plots and make sure it gets dropped from the no key filenames list
drop_plot = dir_not_in_keychain.path_list()[0]
dir_not_in_keychain.drop(drop_plot)
drop_plot.unlink()
assert drop_plot in env.refresh_tester.plot_manager.no_key_filenames
expected_result.processed -= 1
await env.refresh_tester.run(expected_result)
assert drop_plot not in env.refresh_tester.plot_manager.no_key_filenames
# Now add the missing keys to the plot manager's key lists and make sure the plots are getting loaded
env.refresh_tester.plot_manager.farmer_public_keys.append(G1Element())
env.refresh_tester.plot_manager.pool_public_keys.append(G1Element())
expected_result.loaded = dir_not_in_keychain.plot_info_list() # type: ignore[assignment]
expected_result.processed = len(dir_not_in_keychain)
await env.refresh_tester.run(expected_result)
# And make sure they are dropped from the list of plots with missing keys
assert len(env.refresh_tester.plot_manager.no_key_filenames) == 0
@pytest.mark.asyncio
async def test_plot_info_caching(test_environment):
env: TestEnvironment = test_environment
+10
View File
@@ -0,0 +1,10 @@
from typing import List
from pathlib import Path
from tests.block_tools import get_plot_dir
def get_test_plots(sub_dir: str = "") -> List[Path]:
path = get_plot_dir()
if sub_dir != "":
path = path / sub_dir
return list(sorted(path.glob("*.plot")))
@@ -3,7 +3,7 @@
with:
repository: 'Chia-Network/test-cache'
path: '.chia'
ref: '0.27.0'
ref: '0.28.0'
fetch-depth: 1
- name: Link home directory