Merge pull request #4097 from ethereum/dev

Release v1.5.0-beta.1

Author: jtraglia

.circleci/config.yml

@@ -35,14 +35,14 @@ commands:
     description: "Restore the cache with pyspec keys"
     steps:
       - restore_cached_venv:
-          venv_name: v32-pyspec
-          reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "requirements_preinstallation.txt" }}
+          venv_name: v33-pyspec
+          reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "pyproject.toml" }}
   save_pyspec_cached_venv:
     description: Save a venv into a cache with pyspec keys"
     steps:
       - save_cached_venv:
-          venv_name: v32-pyspec
-          reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "requirements_preinstallation.txt" }}
+          venv_name: v33-pyspec
+          reqs_checksum: cache-{{ checksum "setup.py" }}-{{ checksum "pyproject.toml" }}
           venv_path: ./venv
 jobs:
   checkout_specs:
@@ -75,7 +75,7 @@ jobs:
       - restore_pyspec_cached_venv
       - run:
           name: Install pyspec requirements
-          command: make eth2spec
+          command: make pyspec
       - save_pyspec_cached_venv
   test-phase0:
     docker:

Makefile

@@ -19,7 +19,6 @@ ALL_EXECUTABLE_SPEC_NAMES = \
 	clean         \
 	coverage      \
 	detect_errors \
-	eth2spec      \
 	gen_all       \
 	gen_list      \
 	help          \
@@ -41,7 +40,6 @@ help:
 	@echo "make $(BOLD)clean$(NORM)         -- delete all untracked files"
 	@echo "make $(BOLD)coverage$(NORM)      -- run pyspec tests with coverage"
 	@echo "make $(BOLD)detect_errors$(NORM) -- detect generator errors"
-	@echo "make $(BOLD)eth2spec$(NORM)      -- force rebuild eth2spec package"
 	@echo "make $(BOLD)gen_<gen>$(NORM)     -- run a single generator"
 	@echo "make $(BOLD)gen_all$(NORM)       -- run all generators"
 	@echo "make $(BOLD)gen_list$(NORM)      -- list all generator targets"
@@ -60,33 +58,23 @@ PYTHON_VENV = $(VENV)/bin/python3
 PIP_VENV = $(VENV)/bin/pip3
 CODESPELL_VENV = $(VENV)/bin/codespell
 
-# Make a virtual environment will all of the necessary dependencies.
-$(VENV): requirements_preinstallation.txt
+# Make a virtual environment.
+$(VENV):
 	@echo "Creating virtual environment"
 	@python3 -m venv $(VENV)
-	@$(PIP_VENV) install -r requirements_preinstallation.txt
+	@$(PIP_VENV) install --quiet uv==0.5.24
 
 ###############################################################################
 # Specification
 ###############################################################################
 
 TEST_LIBS_DIR = $(CURDIR)/tests/core
 PYSPEC_DIR = $(TEST_LIBS_DIR)/pyspec
-SITE_PACKAGES := $(wildcard $(VENV)/lib/python*/site-packages)
-ETH2SPEC := $(SITE_PACKAGES)/eth2spec
-
-# Install the eth2spec package.
-# The pipe indicates that venv is an order-only prerequisite.
-# When restoring venv cache, its timestamp is newer than eth2spec.
-$(ETH2SPEC): setup.py | $(VENV)
-	@$(PIP_VENV) install .[docs,lint,test,generator]
-
-# Force rebuild/install the eth2spec package.
-eth2spec:
-	@$(MAKE) --always-make $(ETH2SPEC)
 
 # Create the pyspec for all phases.
-pyspec: $(VENV) setup.py
+pyspec: $(VENV) setup.py pyproject.toml
+	@echo "Building eth2spec"
+	@$(PYTHON_VENV) -m uv pip install .[docs,lint,test,generator]
 	@echo "Building all pyspecs"
 	@$(PYTHON_VENV) setup.py pyspecdev
 
@@ -113,7 +101,7 @@ test: MAYBE_TEST := $(if $(k),-k=$(k))
 test: MAYBE_FORK := $(if $(fork),--fork=$(fork))
 test: PRESET := --preset=$(if $(preset),$(preset),minimal)
 test: BLS := --bls-type=$(if $(bls),$(bls),fastest)
-test: $(ETH2SPEC) pyspec
+test: pyspec
 	@mkdir -p $(TEST_REPORT_DIR)
 	@$(PYTHON_VENV) -m pytest \
 		-n auto \
@@ -137,7 +125,7 @@ COVERAGE_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), --cov=eth2spec.$S.$(
 # Run pytest with coverage tracking
 _test_with_coverage: MAYBE_TEST := $(if $(k),-k=$(k))
 _test_with_coverage: MAYBE_FORK := $(if $(fork),--fork=$(fork))
-_test_with_coverage: $(ETH2SPEC) pyspec
+_test_with_coverage: pyspec
 	@$(PYTHON_VENV) -m pytest \
 		-n auto \
 		$(MAYBE_TEST) \
@@ -211,7 +199,7 @@ _check_toc: $(MARKDOWN_FILES:=.toc)
 	@[ "$$(find . -name '*.md.tmp' -print -quit)" ] && exit 1 || exit 0
 
 # Check for mistakes.
-lint: $(ETH2SPEC) pyspec _check_toc
+lint: pyspec _check_toc
 	@$(CODESPELL_VENV) . --skip "./.git,$(VENV),$(PYSPEC_DIR)/.mypy_cache" -I .codespell-whitelist
 	@$(PYTHON_VENV) -m flake8 --config $(FLAKE8_CONFIG) $(PYSPEC_DIR)/eth2spec
 	@$(PYTHON_VENV) -m flake8 --config $(FLAKE8_CONFIG) $(TEST_GENERATORS_DIR)
@@ -236,11 +224,11 @@ gen_list:
 	done
 
 # Run one generator.
-# This will forcibly rebuild eth2spec just in case.
+# This will forcibly rebuild pyspec just in case.
 # To check modules for a generator, append modcheck=true, eg:
 #   make gen_genesis modcheck=true
 gen_%: MAYBE_MODCHECK := $(if $(filter true,$(modcheck)),--modcheck)
-gen_%: eth2spec
+gen_%: pyspec
 	@mkdir -p $(TEST_VECTOR_DIR)
 	@$(PYTHON_VENV) $(GENERATOR_DIR)/$*/main.py \
 		--output $(TEST_VECTOR_DIR) \
@@ -260,7 +248,7 @@ detect_errors: $(TEST_VECTOR_DIR)
 	fi
 
 # Generate KZG trusted setups for testing.
-kzg_setups: $(ETH2SPEC)
+kzg_setups: pyspec
 	@for preset in minimal mainnet; do \
 		$(PYTHON_VENV) $(SCRIPTS_DIR)/gen_kzg_trusted_setups.py \
 			--secret=1337 \

README.md

@@ -81,7 +81,6 @@ Documentation on the different components used during spec writing can be found
 
 Conformance tests built from the executable python spec are available in the [Ethereum Proof-of-Stake Consensus Spec Tests](https://github.com/ethereum/consensus-spec-tests) repo. Compressed tarballs are available in [releases](https://github.com/ethereum/consensus-spec-tests/releases).
 
-
 ## Installation and Usage
 The consensus-specs repo can be used by running the tests locally or inside a docker container.
 

configs/mainnet.yaml

@@ -171,8 +171,7 @@ MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384
 SAMPLES_PER_SLOT: 8
 CUSTODY_REQUIREMENT: 4
 MAX_BLOBS_PER_BLOCK_FULU: 12
-# `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK_FULU`
-MAX_REQUEST_BLOB_SIDECARS_FULU: 1536
+MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 4096
 
 # Whisk
 # `Epoch(2**8)`

configs/minimal.yaml

@@ -172,8 +172,7 @@ MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384
 SAMPLES_PER_SLOT: 8
 CUSTODY_REQUIREMENT: 4
 MAX_BLOBS_PER_BLOCK_FULU: 12
-# `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK_FULU`
-MAX_REQUEST_BLOB_SIDECARS_FULU: 1536
+MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 4096
 
 # Whisk
 WHISK_EPOCHS_PER_SHUFFLING_PHASE: 4

docker/README.md

@@ -1,10 +1,11 @@
 ## Docker related information
 
 This dockerfile sets up the dependencies required to run consensus-spec tests. The docker image can be locally built with:
-- `docker build ./ -t $IMAGE_NAME -f ./docker/Dockerfile`
 
+- `docker build ./ -t $IMAGE_NAME -f ./docker/Dockerfile`
 
 Handy commands:
+
 - `docker run -it $IMAGE_NAME /bin/sh` will give you a shell inside the docker container to manually run any tests
 - `docker run $IMAGE_NAME make test` will run the make test command inside the docker container
 
@@ -13,6 +14,7 @@ Ideally manual running of docker containers is for advanced users, we recommend
 The `scripts/build_run_docker_tests.sh` script will cover most use cases. The script allows the user to configure the fork(altair/bellatrix/capella..), `$IMAGE_NAME` (specifies the container to use), preset type (mainnet/minimal), and test all forks flags. Ideally, this is the main way that users interact with the spec tests instead of running it locally with varying versions of dependencies.
 
 E.g:
+
 - `./build_run_docker_tests.sh --p mainnet` will run the mainnet preset tests
 - `./build_run_docker_tests.sh --a` will run all the tests across all the forks
 - `./build_run_docker_tests.sh --f deneb` will only run deneb tests

docs/docs/new-feature.md

@@ -1,8 +1,9 @@
 # How to add a new feature proposal in consensus-specs
 
+## Table of contents
+
 <!-- START doctoc generated TOC please keep comment here to allow auto update -->
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
-## Table of Contents
 
 - [A. Make it executable for linter checks](#a-make-it-executable-for-linter-checks)
   - [1. Create a folder under `./specs/_features`](#1-create-a-folder-under-specs_features)
@@ -23,7 +24,6 @@
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
-
 ## A. Make it executable for linter checks
 
 ### 1. Create a folder under `./specs/_features`
@@ -35,6 +35,7 @@ For example, if it's an `EIP-9999` CL spec, you can create a `./specs/_features/
 For example, if the latest fork is Capella, use `./specs/capella` content as your "previous fork".
 
 ### 3. Write down your proposed `beacon-chain.md` change
+
 - You can either use [Beacon Chain Spec Template](./templates/beacon-chain-template.md), or make a copy of the latest fork content and then edit it.
 - Tips:
     - We use [`doctoc`](https://www.npmjs.com/package/doctoc) tool to generate the table of content.
@@ -50,8 +51,11 @@ For example, if the latest fork is Capella, use `./specs/capella` content as you
         - Use simple Python rather than the fancy Python dark magic.
 
 ### 4. Add `fork.md`
+
 You can refer to the previous fork's `fork.md` file.
+
 ### 5. Make it executable
+
 - Update Pyspec [`constants.py`](https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/test/helpers/constants.py) with the new feature name.
 - Update helpers for [`setup.py`](https://github.com/ethereum/consensus-specs/blob/dev/setup.py) for building the spec:
     - Update [`pysetup/constants.py`](https://github.com/ethereum/consensus-specs/blob/dev/pysetup/constants.py) with the new feature name as Pyspec `constants.py` defined.
@@ -63,17 +67,21 @@ You can refer to the previous fork's `fork.md` file.
 ## B: Make it executable for pytest and test generator
 
 ### 1. [Optional] Add `light-client/*` docs if you updated the content of `BeaconBlock`
+
 - You can refer to the previous fork's `light-client/*` file.
 - Add the path of the new markdown files in [`pysetup/md_doc_paths.py`](https://github.com/ethereum/consensus-specs/blob/dev/pysetup/md_doc_paths.py)'s `get_md_doc_paths` function.
 
 ### 2. Add the mainnet and minimal presets and update the configs
+
 - Add presets: `presets/mainnet/<new-feature-name>.yaml` and `presets/minimal/<new-feature-name>.yaml`
 - Update configs: `configs/mainnet.yaml` and `configs/minimal.yaml`
 
 ### 3. Update [`context.py`](https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/test/context.py)
+
 - [Optional] Add `with_<new-feature-name>_and_later` decorator for writing pytest cases. e.g., `with_capella_and_later`.
 
 ### 4. Update [`constants.py`](https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/test/helpers/constants.py)
+
 - Add `<NEW_FEATURE>` to `ALL_PHASES` and `TESTGEN_FORKS`
 
 ### 5. Update [`genesis.py`](https://github.com/ethereum/consensus-specs/blob/dev/tests/core/pyspec/eth2spec/test/helpers/genesis.py):
@@ -94,6 +102,7 @@ def create_genesis_state(spec, validator_balances, activation_threshold):
 - If the given feature changes `ExecutionPayload` fields, you have to set the initial values by updating `get_sample_genesis_execution_payload_header` helper.
 
 ### 6. Update CI configurations
+
 - Update [GitHub Actions config](https://github.com/ethereum/consensus-specs/blob/dev/.github/workflows/run-tests.yml)
     - Update `pyspec-tests.strategy.matrix.version` list by adding new feature to it
 - Update [CircleCI config](https://github.com/ethereum/consensus-specs/blob/dev/.circleci/config.yml)
@@ -102,7 +111,9 @@ def create_genesis_state(spec, validator_balances, activation_threshold):
 ## Others
 
 ### Bonus
+
 - Add `validator.md` if honest validator behavior changes with the new feature.
 
 ### Need help?
+
 You can tag spec elves for cleaning up your PR. 🧚