filelock for test- plots and blocks (#20193)

* introduce file lock for creating the test-plots in tests. This avoids having every process in pytest-xdist create the same plots, clobbering each other

* add lockfile to the test fixtures that generate the test blockchains. These are quite expensive to generate, we don't want every pytest-xdist process to have to do it

* improve error message when the plot manager has an unexpected number of plots, in the simulator

* review comments: use FileLock instead of custom made

Author: arvidn

chia/_tests/conftest.py

@@ -230,9 +230,11 @@ def blockchain_constants(consensus_mode: ConsensusMode) -> ConsensusConstants:
 
 
 @pytest.fixture(scope="session", name="bt")
-async def block_tools_fixture(get_keychain, blockchain_constants, anyio_backend) -> BlockTools:
+async def block_tools_fixture(get_keychain, blockchain_constants, anyio_backend, testrun_uid: str) -> BlockTools:
     # Note that this causes a lot of CPU and disk traffic - disk, DB, ports, process creation ...
-    shared_block_tools = await create_block_tools_async(constants=blockchain_constants, keychain=get_keychain)
+    shared_block_tools = await create_block_tools_async(
+        constants=blockchain_constants, keychain=get_keychain, testrun_uid=testrun_uid
+    )
     return shared_block_tools
 
 
@@ -950,13 +952,17 @@ def get_temp_keyring():
 
 
 @pytest.fixture(scope="function")
-async def get_b_tools_1(get_temp_keyring):
-    return await create_block_tools_async(constants=test_constants_modified, keychain=get_temp_keyring)
+async def get_b_tools_1(get_temp_keyring, testrun_uid: str):
+    return await create_block_tools_async(
+        constants=test_constants_modified, keychain=get_temp_keyring, testrun_uid=testrun_uid
+    )
 
 
 @pytest.fixture(scope="function")
-async def get_b_tools(get_temp_keyring):
-    local_b_tools = await create_block_tools_async(constants=test_constants_modified, keychain=get_temp_keyring)
+async def get_b_tools(get_temp_keyring, testrun_uid):
+    local_b_tools = await create_block_tools_async(
+        constants=test_constants_modified, keychain=get_temp_keyring, testrun_uid=testrun_uid
+    )
     new_config = local_b_tools._config
     local_b_tools.change_config(new_config)
     return local_b_tools
@@ -1258,7 +1264,9 @@ def populated_temp_file_keyring_fixture() -> Iterator[TempKeyring]:
 
 @pytest.fixture(scope="function")
 async def farmer_harvester_2_simulators_zero_bits_plot_filter(
-    tmp_path: Path, get_temp_keyring: Keychain
+    tmp_path: Path,
+    get_temp_keyring: Keychain,
+    testrun_uid,
 ) -> AsyncIterator[
     tuple[
         FarmerService,
@@ -1277,6 +1285,7 @@ async def farmer_harvester_2_simulators_zero_bits_plot_filter(
         bt = await create_block_tools_async(
             zero_bit_plot_filter_consts,
             keychain=get_temp_keyring,
+            testrun_uid=testrun_uid,
         )
 
         config_overrides: dict[str, int] = {"full_node.max_sync_wait": 0}
@@ -1289,6 +1298,7 @@ async def farmer_harvester_2_simulators_zero_bits_plot_filter(
                 num_pool_plots=0,
                 num_non_keychain_plots=0,
                 config_overrides=config_overrides,
+                testrun_uid=testrun_uid,
             )
             for _ in range(2)
         ]

chia/_tests/util/blockchain.py

@@ -9,6 +9,7 @@
 
 from chia_rs import ConsensusConstants, FullBlock
 from chia_rs.sized_ints import uint64
+from filelock import FileLock
 
 from chia.consensus.block_height_map import BlockHeightMap
 from chia.consensus.blockchain import Blockchain
@@ -59,47 +60,49 @@ def persistent_blocks(
         block_list_input = []
     block_path_dir = DEFAULT_ROOT_PATH.parent.joinpath("blocks")
     file_path = block_path_dir.joinpath(db_name)
+    lock_file_path = block_path_dir / (db_name + ".lockfile")
 
     ci = os.environ.get("CI")
     if ci is not None and not file_path.exists():
         raise Exception(f"Running in CI and expected path not found: {file_path!r}")
 
     block_path_dir.mkdir(parents=True, exist_ok=True)
 
-    if file_path.exists():
-        print(f"File found at: {file_path}")
-        try:
-            bytes_list = file_path.read_bytes()
-            # TODO: use explicit serialization instead of pickle
-            block_bytes_list: list[bytes] = pickle.loads(bytes_list)  # noqa: S301
-            blocks: list[FullBlock] = []
-            for block_bytes in block_bytes_list:
-                blocks.append(FullBlock.from_bytes_unchecked(block_bytes))
-            if len(blocks) == num_of_blocks + len(block_list_input):
-                print(f"\n loaded {file_path} with {len(blocks)} blocks")
+    with FileLock(lock_file_path):
+        if file_path.exists():
+            print(f"File found at: {file_path}")
+            try:
+                bytes_list = file_path.read_bytes()
+                # TODO: use explicit serialization instead of pickle
+                block_bytes_list: list[bytes] = pickle.loads(bytes_list)  # noqa: S301
+                blocks: list[FullBlock] = []
+                for block_bytes in block_bytes_list:
+                    blocks.append(FullBlock.from_bytes_unchecked(block_bytes))
+                if len(blocks) == num_of_blocks + len(block_list_input):
+                    print(f"\n loaded {file_path} with {len(blocks)} blocks")
 
-                return blocks
-        except EOFError:
-            print("\n error reading db file")
-    else:
-        print(f"File not found at: {file_path}")
+                    return blocks
+            except EOFError:
+                print("\n error reading db file")
+        else:
+            print(f"File not found at: {file_path}")
 
-    print("Creating a new test db")
-    return new_test_db(
-        file_path,
-        num_of_blocks,
-        seed,
-        empty_sub_slots,
-        bt,
-        block_list_input,
-        time_per_block,
-        normalized_to_identity_cc_eos=normalized_to_identity_cc_eos,
-        normalized_to_identity_icc_eos=normalized_to_identity_icc_eos,
-        normalized_to_identity_cc_sp=normalized_to_identity_cc_sp,
-        normalized_to_identity_cc_ip=normalized_to_identity_cc_ip,
-        dummy_block_references=dummy_block_references,
-        include_transactions=include_transactions,
-    )
+        print("Creating a new test db")
+        return new_test_db(
+            file_path,
+            num_of_blocks,
+            seed,
+            empty_sub_slots,
+            bt,
+            block_list_input,
+            time_per_block,
+            normalized_to_identity_cc_eos=normalized_to_identity_cc_eos,
+            normalized_to_identity_icc_eos=normalized_to_identity_icc_eos,
+            normalized_to_identity_cc_sp=normalized_to_identity_cc_sp,
+            normalized_to_identity_cc_ip=normalized_to_identity_cc_ip,
+            dummy_block_references=dummy_block_references,
+            include_transactions=include_transactions,
+        )
 
 
 def new_test_db(

chia/simulator/block_tools.py

@@ -41,6 +41,7 @@
 )
 from chia_rs.sized_bytes import bytes32
 from chia_rs.sized_ints import uint8, uint16, uint32, uint64, uint128
+from filelock import FileLock
 
 from chia.consensus.block_creation import create_unfinished_block, unfinished_block_to_full_block
 from chia.consensus.block_record import BlockRecordProtocol
@@ -338,6 +339,29 @@ def test_callback(event: PlotRefreshEvents, update_result: PlotRefreshResult) ->
                 assert self.total_result.processed == update_result.processed
                 assert self.total_result.duration == update_result.duration
                 assert update_result.remaining == 0
+
+                expected_plots: set[str] = set()
+                found_plots: set[str] = set()
+                if len(self.plot_manager.plots) != len(self.expected_plots):  # pragma: no cover
+                    for pid, filename in self.expected_plots.items():
+                        expected_plots.add(filename.name)
+                    for filename, _ in self.plot_manager.plots.items():
+                        found_plots.add(filename.name)
+                    print(f"directory: {self.plot_dir}")
+                    print(f"expected: {len(expected_plots)}")
+                    for f in expected_plots:
+                        print(f)
+                    print(f"plot manager: {len(found_plots)}")
+                    for f in found_plots:
+                        print(f)
+                    diff = found_plots.difference(expected_plots)
+                    print(f"found unexpected: {len(diff)}")
+                    for f in diff:
+                        print(f)
+                    diff = expected_plots.difference(found_plots)
+                    print(f"not found: {len(diff)}")
+                    for f in diff:
+                        print(f)
                 assert len(self.plot_manager.plots) == len(self.expected_plots)
 
         self.plot_manager: PlotManager = PlotManager(
@@ -500,34 +524,41 @@ async def setup_plots(
         num_non_keychain_plots: int = 3,
         plot_size: int = 20,
         bitfield: bool = True,
+        testrun_uid: Optional[str] = None,
     ) -> bool:
-        self.add_plot_directory(self.plot_dir)
-        assert self.created_plots == 0
-        existing_plots: bool = True
-        # OG Plots
-        for i in range(num_og_plots):
-            plot = await self.new_plot(plot_size=plot_size, bitfield=bitfield)
-            if plot.new_plot:
-                existing_plots = False
-        # Pool Plots
-        for i in range(num_pool_plots):
-            plot = await self.new_plot(self.pool_ph, plot_size=plot_size, bitfield=bitfield)
-            if plot.new_plot:
-                existing_plots = False
-        # Some plots with keys that are not in the keychain
-        for i in range(num_non_keychain_plots):
-            plot = await self.new_plot(
-                path=self.plot_dir / "not_in_keychain",
-                plot_keys=PlotKeys(G1Element(), G1Element(), None),
-                exclude_plots=True,
-                plot_size=plot_size,
-                bitfield=bitfield,
-            )
-            if plot.new_plot:
-                existing_plots = False
-        await self.refresh_plots()
-        assert len(self.plot_manager.plots) == len(self.expected_plots)
-        return existing_plots
+        if testrun_uid is None:
+            lock_file_name = self.plot_dir / ".lockfile"
+        else:
+            lock_file_name = self.plot_dir / (testrun_uid + ".lockfile")
+
+        with FileLock(lock_file_name):
+            self.add_plot_directory(self.plot_dir)
+            assert self.created_plots == 0
+            existing_plots: bool = True
+            # OG Plots
+            for i in range(num_og_plots):
+                plot = await self.new_plot(plot_size=plot_size, bitfield=bitfield)
+                if plot.new_plot:
+                    existing_plots = False
+            # Pool Plots
+            for i in range(num_pool_plots):
+                plot = await self.new_plot(self.pool_ph, plot_size=plot_size, bitfield=bitfield)
+                if plot.new_plot:
+                    existing_plots = False
+            # Some plots with keys that are not in the keychain
+            for i in range(num_non_keychain_plots):
+                plot = await self.new_plot(
+                    path=self.plot_dir / "not_in_keychain",
+                    plot_keys=PlotKeys(G1Element(), G1Element(), None),
+                    exclude_plots=True,
+                    plot_size=plot_size,
+                    bitfield=bitfield,
+                )
+                if plot.new_plot:
+                    existing_plots = False
+            await self.refresh_plots()
+            assert len(self.plot_manager.plots) == len(self.expected_plots)
+            return existing_plots
 
     async def new_plot(
         self,
@@ -2080,6 +2111,7 @@ async def create_block_tools_async(
     num_og_plots: int = 15,
     num_pool_plots: int = 5,
     num_non_keychain_plots: int = 3,
+    testrun_uid: Optional[str] = None,
 ) -> BlockTools:
     global create_block_tools_async_count
     create_block_tools_async_count += 1
@@ -2090,6 +2122,7 @@ async def create_block_tools_async(
         num_og_plots=num_og_plots,
         num_pool_plots=num_pool_plots,
         num_non_keychain_plots=num_non_keychain_plots,
+        testrun_uid=testrun_uid,
     )
 
     return bt