feat: struct based live chain logging

.github/workflows/qa-clean-exit-block-downloading.yml

@@ -49,7 +49,7 @@ jobs:
         set +e # Disable exit on error
 
         # Run Erigon, send ctrl-c and check logs
-        python3 $ERIGON_QA_PATH/test_system/qa-tests/clean-exit/run_and_check_clean_exit.py ${{ github.workspace }}/build/bin $ERIGON_TESTBED_DATA_DIR $WORKING_TIME_SECONDS Erigon3
+        python3 $ERIGON_QA_PATH/test_system/qa-tests/clean-exit/run_and_check_clean_exit.py ${{ github.workspace }}/build/bin $ERIGON_TESTBED_DATA_DIR $WORKING_TIME_SECONDS
 
         # Capture monitoring script exit status
         test_exit_status=$?

.github/workflows/qa-clean-exit-snapshot-downloading.yml

@@ -45,7 +45,7 @@ jobs:
         set +e # Disable exit on error
 
         # Run Erigon, send ctrl-c and check logs
-        python3 $ERIGON_QA_PATH/test_system/qa-tests/clean-exit/run_and_check_clean_exit.py ${{ github.workspace }}/build/bin $ERIGON_DATA_DIR $WORKING_TIME_SECONDS Erigon3
+        python3 $ERIGON_QA_PATH/test_system/qa-tests/clean-exit/run_and_check_clean_exit.py ${{ github.workspace }}/build/bin $ERIGON_DATA_DIR $WORKING_TIME_SECONDS
 
         # Capture monitoring script exit status
         test_exit_status=$?

.github/workflows/qa-snap-download.yml

@@ -39,7 +39,7 @@ jobs:
         set +e # Disable exit on error
 
         # Run Erigon, monitor snapshot downloading and check logs
-        python3 $ERIGON_QA_PATH/test_system/qa-tests/snap-download/run_and_check_snap_download.py ${{ github.workspace }}/build/bin $ERIGON_DATA_DIR $TOTAL_TIME_SECONDS Erigon3
+        python3 $ERIGON_QA_PATH/test_system/qa-tests/snap-download/run_and_check_snap_download.py ${{ github.workspace }}/build/bin $ERIGON_DATA_DIR $TOTAL_TIME_SECONDS
 
         # Capture monitoring script exit status
         test_exit_status=$?

.github/workflows/qa-tip-tracking.yml

@@ -45,7 +45,7 @@ jobs:
         # 1. Launch the testbed Erigon instance
         # 2. Allow time for the Erigon to achieve synchronization
         # 3. Begin timing the duration that Erigon maintains synchronization
-        python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/run_and_check_tip_tracking.py ${{ github.workspace }}/build/bin $ERIGON_TESTBED_DATA_DIR $TRACKING_TIME_SECONDS $TOTAL_TIME_SECONDS Erigon3
+        python3 $ERIGON_QA_PATH/test_system/qa-tests/tip-tracking/run_and_check_tip_tracking.py ${{ github.workspace }}/build/bin $ERIGON_TESTBED_DATA_DIR $TRACKING_TIME_SECONDS $TOTAL_TIME_SECONDS
 
         # Capture monitoring script exit status
         test_exit_status=$?

Makefile

@@ -211,7 +211,7 @@ mocks: mocks-clean
 
 ## mocks-clean:                       cleans all generated test mocks
 mocks-clean:
-	grep -r -l --exclude-dir="erigon-lib" --exclude-dir="tests" --exclude-dir="*$(GOBINREL)*" "^// Code generated by MockGen. DO NOT EDIT.$$" . | xargs rm -r
+	grep -r -l --exclude-dir="erigon-lib" --exclude-dir="*$(GOBINREL)*" "^// Code generated by MockGen. DO NOT EDIT.$$" . | xargs rm -r
 
 ## solc:                              generate all solidity contracts
 solc:

README.md

@@ -329,8 +329,7 @@ Engine API.
 
 #### Caplin's Usage.
 
-Caplin is be enabled by default. to disable it and enable the Engine API, use the `--externalcl` flag. from that point
-on, an external Consensus Layer will not be need
+Caplin is be enabled by default. to disable it and enable the Engine API, use the `--externalcl` flag. from that point on, an external Consensus Layer will not be need
 anymore.
 
 Caplin also has an archivial mode for historical states and blocks. it can be enabled through the `--caplin.archive`
@@ -603,6 +602,7 @@ In order to configure the ports, use:
 |-----------|------|----------|---------|---------------|
 | REST      | 5555 | TCP      | REST    | Public        |
 
+
 #### `shared` ports
 
 | Component | Port | Protocol | Purpose | Should Expose |
@@ -786,14 +786,12 @@ Supported networks: all (except Mumbai).
   stage_trace_index
 - E3 can execute 1 historical transaction - without executing it's block - because history/indices have
   transaction-granularity, instead of block-granularity.
-- E3 doesn't store Logs (aka Receipts) - it always re-executing historical txn (but it's cheaper then in E2 - see point
-  above). Also Logs LRU added in E2 (release/2.60) and E3: https://github.com/ledgerwatch/erigon/pull/10112
+- Doesn't store Receipts/Logs - it always re-executing historical transactions - but re-execution is cheaper (see point
+  above). We would like to see how it will impact users - welcome feedback. Likely we will try add some small LRU-cache
   here. Likely later we will add optional flag "to persist receipts".
-- `--sync.loop.block.limit` is enabled by default. (Default: `2_000`.
-  Set `--sync.loop.block.limit=10_000_000 --batchSize=1g` to increase sync speed on good hardware).
-- datadir/chaindata is small now - to prevent it's grow: we recommend set `--batchSize <= 1G`. And it's fine
-  to `rm -rf chaindata`
-- can symlink/mount latest state to fast drive and history to cheap drive
+- More cold-start-friendly and os-pre-fetch-friendly.
+- datadir/chaindata is small now - to prevent it's grow: we recommend set --batchSize <= 1G. Probably 512mb is
+  enough.
 
 ### E3 datadir structure
 
@@ -810,7 +808,7 @@ datadir
 # There is 4 domains: account, storage, code, commitment 
 ```
 
-### E3 can store state on fast disk and history on cheap disk
+### E3 can store state on fast disk and history on slow disk
 
 If you can afford store datadir on 1 nvme-raid - great. If can't - it's possible to store history on cheap drive.
 
@@ -867,24 +865,3 @@ du -hsc /erigon/snapshots/*
 1.3T	/erigon/snapshots/idx
 3.7T	total
 ```
-
-### E3 other perf trics
-
-- `--sync.loop.block.limit=10_000_000 --batchSize=1g` - likely will help for sync speed.
-- on cloud-drives (good throughput, bad latency) - can enable OS's brain to pre-fetch some data (`madv_normal` instead
-  of `madv_random`). For `snapshots/domain` folder (latest
-  state) `KV_MADV_NORMAL_NO_LAST_LVL=accounts,storage,commitment` (or if have enough
-  RAM:  `KV_MADV_NORMAL=accounts,storage,commitment`). For `chaindata` folder (latest updates) `MDBX_READAHEAD=true`.
-  For all files - `SNAPSHOT_MADV_RND=false`
-
-- can lock latest state in RAM - to prevent from eviction (node may face high historical RPC traffic without impacting
-  Chain-Tip perf):
-
-```
-vmtouch -vdlw /mnt/erigon/snapshots/domain/*bt
-ls /mnt/erigon/snapshots/domain/*.kv | parallel vmtouch -vdlw
-
-# if it failing with "can't allocate memory", try: 
-sync && sudo sysctl vm.drop_caches=3
-echo 1 > /proc/sys/vm/compact_memory
-```

accounts/abi/bind/backends/simulated.go

@@ -46,6 +46,7 @@ import (
 	"github.com/ledgerwatch/erigon/core"
 	"github.com/ledgerwatch/erigon/core/rawdb"
 	"github.com/ledgerwatch/erigon/core/state"
+	"github.com/ledgerwatch/erigon/core/tracing"
 	"github.com/ledgerwatch/erigon/core/types"
 	"github.com/ledgerwatch/erigon/core/vm"
 	"github.com/ledgerwatch/erigon/event"
@@ -714,7 +715,7 @@ func (b *SimulatedBackend) callContract(_ context.Context, call ethereum.CallMsg
 	}
 	// Set infinite balance to the fake caller account.
 	from := statedb.GetOrNewStateObject(call.From)
-	from.SetBalance(uint256.NewInt(0).SetAllOne())
+	from.SetBalance(uint256.NewInt(0).SetAllOne(), tracing.BalanceChangeUnspecified)
 	// Execute the call.
 	msg := callMsg{call}
 

cl/abstract/beacon_state.go

@@ -61,7 +61,6 @@ type BeaconStateSSZ interface {
 	HashSSZ() (out [32]byte, err error)
 }
 
-//go:generate mockgen -typed=true -destination=./mock_services/beacon_state_mutator_mock.go -package=mock_services . BeaconStateMutator
 type BeaconStateMutator interface {
 	SetVersion(version clparams.StateVersion)
 	SetSlot(slot uint64)
@@ -105,7 +104,7 @@ type BeaconStateMutator interface {
 	SetValidatorInactivityScore(index int, score uint64) error
 	SetCurrentEpochParticipationFlags(flags []cltypes.ParticipationFlags)
 	SetPreviousEpochParticipationFlags(flags []cltypes.ParticipationFlags)
-	SetPreviousEpochAttestations(attestations *solid.ListSSZ[*solid.PendingAttestation]) // temporarily skip this mock
+	SetPreviousEpochAttestations(attestations *solid.ListSSZ[*solid.PendingAttestation])
 
 	AddEth1DataVote(vote *cltypes.Eth1Data)
 	AddValidator(validator solid.Validator, balance uint64)
@@ -193,14 +192,8 @@ type BeaconStateMinimal interface {
 	PreviousEpochAttestationsLength() int
 }
 
-// BeaconStateReader is an interface for reading the beacon state.
-//
-//go:generate mockgen -typed=true -destination=./mock_services/beacon_state_reader_mock.go -package=mock_services . BeaconStateReader
-type BeaconStateReader interface {
-	ValidatorPublicKey(index int) (common.Bytes48, error)
-	GetDomain(domainType [4]byte, epoch uint64) ([]byte, error)
-	CommitteeCount(epoch uint64) uint64
-	ValidatorForValidatorIndex(index int) (solid.Validator, error)
-	Version() clparams.StateVersion
-	GenesisValidatorsRoot() common.Hash
+// TODO figure this out
+type BeaconStateCopying interface {
+	//CopyInto(dst *raw.BeaconState) error
+	//Copy() (*raw.BeaconState, error)
 }