Add ready fn (#58)

* Add ready fn

* Fix

* Fix CI

* Fix logs upload

* Fix ci

* Add checks

* Add

* Add more

* One more

* Explicit log

* More

* God

* Go

* Add

* Try again

* Add last?

* Cleaning

Author: ferranbt

.github/workflows/checks.yaml

@@ -36,14 +36,14 @@ jobs:
       - name: Run playground
         run: go run main.go cook ${{ matrix.flags }} --output /tmp/playground --timeout 4m --watchdog
 
-      - name: Move playground logs
+      - name: Copy playground logs
         if: ${{ failure() }}
-        run: mv /tmp/playground.log /tmp/playground/logs
+        run: ./scripts/ci-copy-playground-logs.sh /tmp/playground /tmp/playground-logs
 
       - name: Archive playground logs
         uses: actions/upload-artifact@v4
         if: ${{ failure() }}
         with:
           name: playground-logs-${{ matrix.flags }}
-          path: /tmp/playground/logs
+          path: /tmp/playground-logs
           retention-days: 5

internal/components.go

@@ -1,5 +1,12 @@
 package internal
 
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+)
+
 var defaultJWTToken = "04592280e1778419b7aa954d43871cb2cfb2ebda754fb735e8adeb293a88f9bf"
 
 var (
@@ -267,6 +274,17 @@ func (l *LighthouseBeaconNode) Run(svc *service, ctx *ExContext) {
 	}
 }
 
+var _ ServiceReady = &LighthouseBeaconNode{}
+
+func (l *LighthouseBeaconNode) Ready(logOutput io.Writer, service *service, ctx context.Context) error {
+	beaconNodeURL := fmt.Sprintf("http://localhost:%d", service.MustGetPort("http").HostPort)
+
+	if err := waitForChainAlive(logOutput, beaconNodeURL, 30*time.Second); err != nil {
+		return err
+	}
+	return nil
+}
+
 type LighthouseValidator struct {
 	BeaconNode string
 }

internal/local_runner.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"runtime"
 	"strings"
 	"sync"
 	"text/template"
@@ -374,6 +375,16 @@ func (d *LocalRunner) toDockerComposeService(s *service) (map[string]interface{}
 		"labels": map[string]string{"playground": "true"},
 	}
 
+	if runtime.GOOS == "linux" {
+		// We rely on host.docker.internal as the DNS address for the host inside
+		// the container. But, this is only available on Macos and Windows.
+		// On Linux, you can use the IP address 172.17.0.1 to access the host.
+		// Thus, if we are running on Linux, we need to add an extra host entry.
+		service["extra_hosts"] = map[string]string{
+			"host.docker.internal": "172.17.0.1",
+		}
+	}
+
 	if s.entrypoint != "" {
 		service["entrypoint"] = s.entrypoint
 	}
@@ -428,7 +439,6 @@ func (d *LocalRunner) generateDockerCompose() ([]byte, error) {
 	}
 
 	compose["services"] = services
-
 	yamlData, err := yaml.Marshal(compose)
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal docker compose: %w", err)

internal/manifest.go

@@ -1,8 +1,11 @@
 package internal
 
 import (
+	"context"
 	"fmt"
+	"io"
 	"strings"
+	"sync"
 	"text/template"
 
 	flag "github.com/spf13/pflag"
@@ -74,6 +77,43 @@ type Service interface {
 	Run(service *service, ctx *ExContext)
 }
 
+type ServiceReady interface {
+	Ready(out io.Writer, service *service, ctx context.Context) error
+}
+
+func WaitForReady(manifest *Manifest) error {
+	var wg sync.WaitGroup
+	readyErr := make(chan error, len(manifest.Services()))
+
+	output, err := manifest.out.LogOutput("ready")
+	if err != nil {
+		return fmt.Errorf("failed to create log output: %w", err)
+	}
+
+	for _, s := range manifest.Services() {
+		if readyFn, ok := s.component.(ServiceReady); ok {
+			wg.Add(1)
+
+			go func() {
+				defer wg.Done()
+
+				if err := readyFn.Ready(output, s, context.Background()); err != nil {
+					readyErr <- fmt.Errorf("service %s failed to start: %w", s.Name, err)
+				}
+			}()
+		}
+	}
+	wg.Wait()
+
+	close(readyErr)
+	for err := range readyErr {
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (s *Manifest) Services() []*service {
 	return s.services
 }

internal/recipe_l1.go

@@ -38,7 +38,7 @@ func (l *L1Recipe) Flags() *flag.FlagSet {
 	flags := flag.NewFlagSet("l1", flag.ContinueOnError)
 	flags.BoolVar(&l.latestFork, "latest-fork", false, "use the latest fork")
 	flags.BoolVar(&l.useRethForValidation, "use-reth-for-validation", false, "use reth for validation")
-	flags.Uint64Var(&l.secondaryELPort, "secondary-el", 1234, "port to use for the secondary builder")
+	flags.Uint64Var(&l.secondaryELPort, "secondary-el", 0, "port to use for the secondary builder")
 	flags.BoolVar(&l.useNativeReth, "use-native-reth", false, "use the native reth binary")
 	return flags
 }
@@ -100,9 +100,6 @@ func (l *L1Recipe) Watchdog(manifest *Manifest, out *output) error {
 	}
 
 	beaconNodeURL := fmt.Sprintf("http://localhost:%d", beaconNode.MustGetPort("http").HostPort)
-	if err := waitForChainAlive(watchDogOut, beaconNodeURL, 30*time.Second); err != nil {
-		return err
-	}
 	beaconNodeELURL := fmt.Sprintf("http://localhost:%d", beaconNodeEL.MustGetPort("http").HostPort)
 
 	watchGroup := newWatchGroup()

internal/releases.go

@@ -5,6 +5,7 @@ import (
 	"compress/gzip"
 	"fmt"
 	"io"
+	"log"
 	"net/http"
 	"os"
 	"os/exec"
@@ -40,17 +41,17 @@ func downloadRelease(outputFolder string, artifact *release) (string, error) {
 	archVersion := artifact.Arch(goos, goarch)
 	if archVersion == "" {
 		// Case 2. The architecture is not supported.
-		fmt.Printf("unsupported OS/Arch: %s/%s\n", goos, goarch)
+		log.Printf("unsupported OS/Arch: %s/%s\n", goos, goarch)
 		if _, err := exec.LookPath(artifact.Name); err != nil {
 			return "", fmt.Errorf("error looking up binary in PATH: %v", err)
 		} else {
 			outPath = artifact.Name
-			fmt.Printf("Using %s from PATH\n", artifact.Name)
+			log.Printf("Using %s from PATH\n", artifact.Name)
 		}
 	} else {
 		// Case 3. Download the binary from the release page
 		releasesURL := fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s-%s-%s.tar.gz", artifact.Org, artifact.Name, artifact.Version, artifact.Name, artifact.Version, archVersion)
-		fmt.Printf("Downloading %s: %s\n", outPath, releasesURL)
+		log.Printf("Downloading %s: %s\n", outPath, releasesURL)
 
 		if err := downloadArtifact(releasesURL, artifact.Name, outPath); err != nil {
 			return "", fmt.Errorf("error downloading artifact: %v", err)

internal/watchers.go

@@ -190,6 +190,7 @@ func watchChainHead(logOutput io.Writer, elURL string, blockTime time.Duration)
 			if latestBlock != nil && num <= *latestBlock {
 				continue
 			}
+			log.Infof("Chain head: %d", num)
 			latestBlock = &num
 
 			// Reset timeout since we saw a new block

main.go

@@ -138,6 +138,12 @@ func runIt(recipe internal.Recipe) error {
 		}
 	}
 
+	// wait for service readiness
+	if err := internal.WaitForReady(svcManager); err != nil {
+		dockerRunner.Stop()
+		return fmt.Errorf("failed to wait for service readiness: %w", err)
+	}
+
 	watchdogErr := make(chan error, 1)
 	if watchdog {
 		go func() {

scripts/ci-copy-playground-logs.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+if [ "$#" -ne 2 ]; then
+    echo "Usage: $0 source_folder destination_folder"
+    exit 1
+fi
+
+source_dir="$1"
+dest_dir="$2"
+
+# Check if source directory exists
+if [ ! -d "$source_dir" ]; then
+    echo "Error: Source directory '$source_dir' does not exist"
+    exit 1
+fi
+
+# Create destination directory if it doesn't exist
+mkdir -p "$dest_dir"
+
+# First, copy everything
+cp -r "$source_dir"/* "$dest_dir"/ 2>/dev/null || true
+
+# Remove any data_* directories from the destination
+find "$dest_dir" -type d -name "data_*" -exec rm -rf {} +
+
+echo "Copied contents from '$source_dir' to '$dest_dir' (excluding data_* folders)"