Merge branch 'feature/erigon-live-tracer-hook-port-based-on-main' into feat/firehose-live-tracer-hook-port-based-on-main

Author: dhyaniarun1993

.github/workflows/ci-cd-main-branch-docker-images.yml

@@ -0,0 +1,177 @@
+name: CI-CD build and deploy docker images based on the last commit in the main branch
+run-name: "Commit id ${{ github.sha }}: CI-CD build and deploy docker images based on the commit id in the main branch"
+
+env:
+  APPLICATION: "erigon"
+  BUILDER_IMAGE: "golang:1.23.3-alpine3.20"
+  TARGET_BASE_IMAGE: "alpine:3.20.3"
+  APP_REPO: "erigontech/erigon"
+  CHECKOUT_REF: "main"
+  DOCKERHUB_REPOSITORY: "erigontech/erigon"
+  LABEL_DESCRIPTION: "[docker image built on a last commit id from the main branch] Erigon is an implementation of Ethereum (execution layer with embeddable consensus layer), on the efficiency frontier. Archive Node by default."
+  KEEP_IMAGES: 100
+
+on:
+  push:
+    branches:
+      - 'main'
+    paths-ignore:
+      - '.github/**'
+  workflow_dispatch:
+
+jobs:
+  define_matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      os: ${{ steps.os.outputs.os }}
+
+    steps:
+     - name: Define os
+       id: os
+       run: echo 'os=ubuntu-latest' >> "$GITHUB_OUTPUT"
+
+  Build:
+    needs: define_matrix
+    runs-on: ${{ needs.define_matrix.outputs.os }}
+    timeout-minutes: 45
+    outputs:
+      docker_build_tag: ${{ steps.built_tag_export.outputs.docker_build_tag }}
+
+    steps:
+      - name: Fast checkout git repository
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 ## 4.1.7 release
+        with:
+          repository: ${{ env.APP_REPO }}
+          fetch-depth: 1
+          ref: ${{ env.CHECKOUT_REF }}
+          path: 'erigon'
+
+      - name: Setup go env and cache
+        uses: actions/setup-go@v5
+        with:
+          go-version: '>=1.22'
+          go-version-file: 'erigon/go.mod'
+          cache-dependency-path: |
+            erigon/go.sum
+
+      - name: Get commit id
+        id: getCommitId
+        run: |
+          cd erigon
+          echo "id=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+          echo "short_commit_id=$(git rev-parse --short=7 HEAD)" >> $GITHUB_OUTPUT
+          cd ..
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567  ## v3.3.0
+        with:
+          username: ${{ secrets.ORG_DOCKERHUB_ERIGONTECH_USERNAME }}
+          password: ${{ secrets.ORG_DOCKERHUB_ERIGONTECH_TOKEN }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf ## v3.2.0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db ## v3.6.1
+
+      - name: Build arm64
+        run: |
+          docker run --platform linux/arm64 \
+          --rm -v $(pwd)/erigon:/erigon:ro \
+          -v $(pwd)/build-arm64:/erigon-build \
+          -v ${HOME}/.cache/go-build/arm64:/root/.cache/go-build \
+          -v ${HOME}/go/pkg/mod:/go/pkg/mod \
+          -w /erigon --entrypoint /bin/sh \
+          ${{ env.BUILDER_IMAGE }} \
+          -c "apk update; apk add make git gcc libstdc++ build-base linux-headers bash ca-certificates; git config --global --add safe.directory /erigon; make GOARCH=arm64 GOBIN=/erigon-build BUILD_TAGS=nosqlite,noboltdb,nosilkworm erigon integration rpcdaemon"
+
+      - name: Build amd64
+        run: |
+          docker run --platform linux/amd64 \
+          --rm -v $(pwd)/erigon:/erigon:ro \
+          -v $(pwd)/build-amd64:/erigon-build \
+          -v ${HOME}/.cache/go-build/amd64:/root/.cache/go-build \
+          -v ${HOME}/go/pkg/mod:/go/pkg/mod \
+          -w /erigon --entrypoint /bin/sh \
+          ${{ env.BUILDER_IMAGE }} \
+          -c "apk update; apk add make git gcc libstdc++ build-base linux-headers bash ca-certificates; git config --global --add safe.directory /erigon; make GOARCH=amd64 GOAMD64=v2 GOBIN=/erigon-build BUILD_TAGS=nosqlite,noboltdb,nosilkworm erigon integration rpcdaemon"
+
+      - name: Build and push multi-platform docker image based on the commit id ${{ steps.getCommitId.outputs.short_commit_id }} in the main branch
+        env:
+          BUILD_VERSION: "main-${{ steps.getCommitId.outputs.short_commit_id }}"
+          DOCKER_URL: ${{ env.DOCKERHUB_REPOSITORY }}
+          DOCKERFILE_PATH: Dockerfile.release
+        run: |
+          cp -vr build-amd64 erigon/
+          cp -vr build-arm64 erigon/
+          cd erigon
+          docker buildx build \
+          --file ${{ env.DOCKERFILE_PATH }} \
+          --target ci-cd-main-branch \
+          --attest type=provenance,mode=max \
+          --sbom=true \
+          --build-arg CI_CD_MAIN_TARGET_BASE_IMAGE=${{ env.TARGET_BASE_IMAGE }} \
+          --build-arg CI_CD_MAIN_BUILDER_IMAGE=${{ env.BUILDER_IMAGE }} \
+          --tag ${{ env.DOCKER_URL }}:${{ env.BUILD_VERSION }} \
+          --tag ${{ env.DOCKER_URL }}:main-latest \
+          --label org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \
+          --label org.opencontainers.image.authors="https://github.com/erigontech/erigon/graphs/contributors" \
+          --label org.opencontainers.image.url="https://github.com/erigontech/erigon/blob/main/Dockerfile" \
+          --label org.opencontainers.image.documentation="https://github.com/erigontech/erigon/blob/main/Dockerfile" \
+          --label org.opencontainers.image.source="https://github.com/erigontech/erigon/blob/main/Dockerfile" \
+          --label org.opencontainers.image.version=${{ steps.getCommitId.outputs.id }} \
+          --label org.opencontainers.image.revision=${{ steps.getCommitId.outputs.id }} \
+          --label org.opencontainers.image.vcs-ref-short=${{ steps.getCommitId.outputs.short_commit_id }} \
+          --label org.opencontainers.image.vendor="${{ github.repository_owner }}" \
+          --label org.opencontainers.image.description="${{ env.LABEL_DESCRIPTION }}" \
+          --label org.opencontainers.image.base.name="${{ env.TARGET_BASE_IMAGE }}" \
+          --push \
+          --platform linux/amd64,linux/arm64 .
+
+      - name: export and print docker build tag, cleanup old docker images
+        id: built_tag_export
+        env:
+          BUILD_VERSION: "main-${{ steps.getCommitId.outputs.short_commit_id }}"
+        run: |
+          echo "docker_build_tag=${{ env.BUILD_VERSION }}" >> $GITHUB_OUTPUT
+          echo The following docker images have been published:
+          echo "${{ env.DOCKERHUB_REPOSITORY }}:main-${{ env.BUILD_VERSION }}"
+          echo "${{ env.DOCKERHUB_REPOSITORY }}:main-latest"
+          echo
+          echo "Cleanup old docker images matching pattern tag ~= main-XXXXXXX"
+          curl_cmd="curl -s -H \"Authorization: JWT ${{ secrets.ORG_DOCKERHUB_ERIGONTECH_TOKEN }}\" "
+          dockerhub_url='https://hub.docker.com/v2/namespaces/erigontech/repositories/erigon'
+          my_list () {
+            # First page:
+            next_page="$dockerhub_url/tags?page=1&page_size=100"
+            while [ "$next_page" != "null" ]
+              do
+              # Print tags and push dates for tags matching "main-":
+              $curl_cmd $next_page | jq -r '.results|.[]|.name + " " + .tag_last_pushed' | grep 'main-'
+              next_page=`$curl_cmd $next_page | jq '.next' | sed -e 's/^\"//' -e 's/\"$//'`
+            done
+            }
+
+            my_list | tail -n+${{ env.KEEP_IMAGES }} | while read line; do
+              echo -n "Removing docker image/published - $line "
+              current_image=$(echo $line | sed -e 's/^\(main-.\{7\}\) .*/\1/')
+              output_code=$(curl --write-out %{http_code} --output curl-output.log \
+                    -s -X DELETE -H "Accept: application/json" \
+                    -H "Authorization: JWT ${{ secrets.ORG_DOCKERHUB_ERIGONTECH_TOKEN }}" \
+                    https://hub.docker.com/v2/repositories/erigontech/erigon/tags/${current_image} )
+              if [ $output_code -ne 204 ]; then
+                echo "ERROR: failed to remove docker image erigon:${current_image}"
+                echo "ERROR: API response: $(cat curl-output.log)."
+              else
+                echo -n " - removed. "
+              fi
+              echo "Done."
+            done
+
+  run-kurtosis-assertoor:
+    needs: [define_matrix, Build]
+    uses: erigontech/erigon/.github/workflows/test-kurtosis-assertoor.yml@main
+    with:
+      checkout_ref: ${{ github.sha }}
+      os: ${{ needs.define_matrix.outputs.os }}
+      docker_build_tag: ${{ needs.Build.outputs.docker_build_tag }}

.github/workflows/ci.yml

@@ -6,8 +6,7 @@ on:
       - 'release/**'
   pull_request:
     branches:
-      - main
-      - 'release/**'
+      - '**'
     types:
       - opened
       - reopened
@@ -17,7 +16,11 @@ on:
 
 concurrency:
   group: ${{ github.ref }}
-  cancel-in-progress: true
+  cancel-in-progress: ${{ !contains(fromJSON('[
+      "refs/heads/release/2.60",
+      "refs/heads/release/2.61",
+      "refs/heads/main"
+    ]'), github.ref) }}
 
 jobs:
   tests:
@@ -34,9 +37,16 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/setup-go@v4
+
+      - uses: actions/setup-go@v5
         with:
-          go-version: '1.21'
+          go-version: '1.22'
+          cache: ${{ contains(fromJSON('[
+            "refs/heads/release/2.60",
+            "refs/heads/release/2.61",
+            "refs/heads/main"
+            ]'), github.ref) }}
+
       - name: Install dependencies on Linux
         if: runner.os == 'Linux'
         run: sudo apt update && sudo apt install build-essential
@@ -54,24 +64,12 @@ jobs:
             echo >&2 "Reproducible build broken"; cat erigon1.sha256; cat erigon2.sha256; exit 1
           fi
 
-      - name: Install golangci-lint
-        if: runner.os == 'Linux'
-        uses: golangci/golangci-lint-action@v4
-        with:
-          version: v1.59.1
-          skip-build-cache: true
-          args: --help
-
-      - name: Lint
-        if: runner.os == 'Linux'
-        run: make lint
-
       - name: Test
         run: make test
 
       - name: SonarCloud
         if: runner.os == 'Linux'
-        uses: SonarSource/sonarcloud-github-action@v2.1.1
+        uses: SonarSource/sonarcloud-github-action@v2.3.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
@@ -90,11 +88,16 @@ jobs:
         with:
           minimum-size: 8GB
       - uses: actions/checkout@v4
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: '1.21'
-
-      - uses: actions/cache@v3
+          go-version: '1.22'
+          cache: ${{ contains(fromJSON('[
+            "refs/heads/release/2.60",
+            "refs/heads/release/2.61",
+            "refs/heads/main"
+            ]'), github.ref) }}
+
+      - uses: actions/cache@v4
         with:
           path: |
             C:\ProgramData\chocolatey\lib\mingw
@@ -113,31 +116,3 @@ jobs:
 
       - name: Test erigon-lib
         run: cd erigon-lib && make test-no-fuzz
-
-  docker-build-check:
-    # don't run this on main - the PR must have run it to be merged and it misleads that this pushes the docker image
-    if: (${{ github.event_name == 'push' || !github.event.pull_request.draft }}) && ${{ github.ref != 'refs/heads/main' }}
-    runs-on: ubuntu-22.04
-
-    steps:
-      - uses: AutoModality/action-clean@v1
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0 # fetch git tags for "git describe"
-
-      - name: make docker (see dockerhub for image builds)
-        run: DOCKER_TAG=thorax/erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker
-
-      # check with root permissions, should be cached from previous build
-      - name: sudo make docker
-        run: sudo DOCKER_TAG=thorax/erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker
-
-#  automated-tests:
-#    runs-on:
-#      ubuntu-22.04
-#    if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
-#    steps:
-#      - uses: actions/checkout@v4
-#
-#      - name: run automated testing
-#        run: BUILD_ERIGON=1 ./tests/automated-testing/run.sh

.github/workflows/docker-image-remove.yml

@@ -0,0 +1,44 @@
+name: Remove docker image
+run-name: Request to remove erigontech/erigon:${{ inputs.docker_image_tag }} by @${{ github.actor }}
+
+env:
+  DOCKERHUB_REPOSITORY: "erigontech/erigon"
+  API_URL: "https://hub.docker.com/v2/repositories/erigontech/erigon/tags"
+
+on:
+  push:
+    branches-ignore:
+      - '**'
+  workflow_dispatch:
+    inputs:
+      docker_image_tag:
+        required: true
+        type: string
+        default: 'not_yet_defined'
+        description: 'Docker image tag to remove from hub.docker.com. Works only for erigontech/erigon'
+
+jobs:
+
+  build-release:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    name: Remove docker image
+
+    steps:
+
+      - name: Run API Call
+        env:
+          TOKEN: ${{ secrets.ORG_DOCKERHUB_ERIGONTECH_TOKEN }}
+        run: |
+          output_code=$(curl --write-out %{http_code} --output curl-output.log \
+                    -s -X DELETE -H "Accept: application/json" \
+                    -H "Authorization: JWT ${{ env.TOKEN }}" \
+                    ${{ env.API_URL }}/${{ inputs.docker_image_tag }} )
+          if [ $output_code -ne 204 ]; then
+            echo "ERROR: failed to remove docker image ${{ env.DOCKERHUB_REPOSITORY }}:${{ inputs.docker_image_tag }}"
+            echo "ERROR: API response: $(cat curl-output.log)."
+            exit 1
+          else
+            echo "SUCCESS: docker image ${{ env.DOCKERHUB_REPOSITORY }}:${{ inputs.docker_image_tag }} removed."
+            exit 0
+          fi

.github/workflows/docker-tags.yml

@@ -16,26 +16,24 @@ jobs:
           fetch-depth: 0
 
       - name: dockerhub-login
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB }}
           password: ${{ secrets.DOCKERHUB_KEY }}
       - name: ghcr-login
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-qemu-action@v3
 
       - run: |
           make release-dry-run
           docker images
-        #          docker image push --all-tags thorax/erigon
-        #          docker image push --all-tags ghcr.io/erigontech/erigon
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VERSION: ${GITHUB_REF#refs/tags/}
           DOCKER_USERNAME: ${{ secrets.DOCKERHUB }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_KEY }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_KEY }}