Implement observability stack with Alloy-based logging

Dockerfile

@@ -56,5 +56,9 @@ RUN mkdir -p /app/src/lib/cli && ln -sf /app/src/esm/cli/alto.js /app/src/lib/cl
 # Always use JSON logs in production
 ENV ALTO_JSON=true
 
+# Add labels for log collection
+LABEL logging="alloy"
+LABEL environment="production"
+
 # start app
 ENTRYPOINT ["pnpm", "start"]

docker-compose.observability.yml

@@ -0,0 +1,100 @@
+version: "3.8"
+
+services:
+  # Prometheus - Metrics collection and storage
+  prometheus:
+    image: prom/prometheus:v2.48.1
+    container_name: alto-prometheus
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--web.console.libraries=/usr/share/prometheus/console_libraries"
+      - "--web.console.templates=/usr/share/prometheus/consoles"
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./observability/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
+    networks:
+      - alto-observability
+    restart: unless-stopped
+
+  # Grafana Tempo - Distributed tracing backend
+  tempo:
+    image: grafana/tempo:2.3.1
+    container_name: alto-tempo
+    command: ["-config.file=/etc/tempo.yml"]
+    ports:
+      - "3200:3200" # Tempo query frontend
+      - "4318:4318" # OTLP HTTP receiver
+      - "4317:4317" # OTLP gRPC receiver
+    volumes:
+      - ./observability/tempo.yml:/etc/tempo.yml:ro
+      - tempo-data:/var/tempo
+    networks:
+      - alto-observability
+    restart: unless-stopped
+
+  # Grafana Loki - Log aggregation
+  loki:
+    image: grafana/loki:2.9.3
+    container_name: alto-loki
+    command: ["-config.file=/etc/loki/local-config.yaml"]
+    ports:
+      - "3100:3100"
+    volumes:
+      - ./observability/loki.yml:/etc/loki/local-config.yaml:ro
+      - loki-data:/loki
+    networks:
+      - alto-observability
+    restart: unless-stopped
+
+  # Grafana Alloy - Log collector
+  alloy:
+    image: grafana/alloy:v1.0.0
+    container_name: alto-alloy
+    command: run --server.http.listen-addr=0.0.0.0:12345 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy
+    environment:
+      - ALTO_CLUSTER_NAME=${ALTO_CLUSTER_NAME:-alto-bundler}
+    ports:
+      - "12345:12345"
+    volumes:
+      - ./observability/alloy.alloy:/etc/alloy/config.alloy:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - alloy-data:/var/lib/alloy
+    networks:
+      - alto-observability
+    depends_on:
+      - loki
+    restart: unless-stopped
+
+  # Grafana - Unified observability dashboard
+  grafana:
+    image: grafana/grafana:10.2.3
+    container_name: alto-grafana
+    ports:
+      - "3003:3000"
+    environment:
+      # Default credentials: admin/admin (change in production)
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+    volumes:
+      - ./observability/grafana-provisioning:/etc/grafana/provisioning:ro
+      - grafana-data:/var/lib/grafana
+    networks:
+      - alto-observability
+    depends_on:
+      - prometheus
+      - tempo
+      - loki
+    restart: unless-stopped
+
+networks:
+  alto-observability:
+    driver: bridge
+
+volumes:
+  prometheus-data:
+  tempo-data:
+  loki-data:
+  alloy-data:
+  grafana-data:

docker-compose.yml

@@ -0,0 +1,26 @@
+version: "3.8"
+
+services:
+  # Alto ERC-4337 Bundler
+  ultra-relay-provider:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: ultra-relay-provider
+    labels:
+      logging: "alloy"
+    networks:
+      - alto-observability
+    environment:
+      # Override with your configuration
+      - ALTO_RPC_URL=${ALTO_RPC_URL}
+      - ALTO_PORT=3000
+      - ALTO_ENABLE_TELEMETRY=true
+      - ALTO_OTLP_ENDPOINT=http://tempo:4318
+    ports:
+      - "3000:3000"
+    restart: unless-stopped
+
+networks:
+  alto-observability:
+    name: alto-observability

observability/alloy.alloy

@@ -0,0 +1,63 @@
+// Grafana Alloy configuration for Docker log collection
+
+// Discover containers with the logging="alloy" label
+discovery.docker "containers" {
+  host = "unix:///var/run/docker.sock"
+
+  filter {
+    name   = "label"
+    values = ["logging=alloy"]
+  }
+}
+
+// Define relabel rules first
+loki.relabel "docker_labels" {
+  forward_to = [loki.write.loki.receiver]
+
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    target_label  = "container"
+    regex         = "/(.*)"
+    replacement   = "$1"
+  }
+
+  rule {
+    source_labels = ["__meta_docker_container_label_environment"]
+    target_label  = "environment"
+  }
+
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+    target_label  = "service"
+  }
+
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_project"]
+    target_label  = "project"
+  }
+
+  rule {
+    source_labels = ["__meta_docker_container_id"]
+    target_label  = "container_id"
+    regex         = "(.{12}).*"
+    replacement   = "$1"
+  }
+}
+
+// Collect Docker logs
+loki.source.docker "docker_logs" {
+  host       = "unix:///var/run/docker.sock"
+  targets    = discovery.docker.containers.targets
+  forward_to = [loki.relabel.docker_labels.receiver]
+}
+
+loki.write "loki" {
+  endpoint {
+    url = "http://loki:3100/loki/api/v1/push"
+  }
+
+  external_labels = {
+    cluster = env("ALTO_CLUSTER_NAME"),
+    source  = "alloy",
+  }
+}

observability/grafana-provisioning/datasources/datasources.yml

@@ -0,0 +1,47 @@
+apiVersion: 1
+
+datasources:
+  # Prometheus datasource for metrics
+  - name: Prometheus
+    type: prometheus
+    uid: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false
+    jsonData:
+      timeInterval: "15s"
+
+  # Tempo datasource for distributed tracing
+  - name: Tempo
+    type: tempo
+    uid: tempo
+    access: proxy
+    url: http://tempo:3200
+    editable: false
+    jsonData:
+      tracesToLogsV2:
+        datasourceUid: "loki"
+        spanStartTimeShift: "-1m"
+        spanEndTimeShift: "1m"
+        filterByTraceID: true
+        filterBySpanID: false
+        tags: ["trace_id"]
+      serviceMap:
+        datasourceUid: "prometheus"
+      nodeGraph:
+        enabled: true
+
+  # Loki datasource for logs
+  - name: Loki
+    type: loki
+    uid: loki
+    access: proxy
+    url: http://loki:3100
+    editable: false
+    jsonData:
+      derivedFields:
+        - datasourceUid: "tempo"
+          matcherRegex: "trace_id=(\\w+)"
+          name: "TraceID"
+          url: "$${__value.raw}"

observability/loki.yml

@@ -0,0 +1,39 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+common:
+  path_prefix: /loki
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+limits_config:
+  enforce_metric_name: false
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  max_cache_freshness_per_query: 10m
+  split_queries_by_interval: 15m
+
+query_range:
+  align_queries_with_step: true
+  cache_results: true
+
+ruler:
+  alertmanager_url: http://localhost:9093

observability/prometheus.yml

@@ -0,0 +1,18 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+  external_labels:
+    cluster: "alto-bundler"
+    environment: "local"
+
+scrape_configs:
+  # Alto Bundler metrics
+  # Using service name (ultra-relay-provider) for cross-platform Docker compatibility
+  # This works on Linux, macOS, and Windows when bundler runs in the same Docker network
+  - job_name: "alto-bundler"
+    static_configs:
+      - targets: ["ultra-relay-provider:3000"]
+        labels:
+          service: "alto-bundler"
+    metrics_path: "/metrics"
+    scrape_interval: 10s

observability/tempo.yml

@@ -0,0 +1,34 @@
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        http:
+          endpoint: 0.0.0.0:4318
+        grpc:
+          endpoint: 0.0.0.0:4317
+
+ingester:
+  trace_idle_period: 10s
+  max_block_bytes: 1_000_000
+  max_block_duration: 5m
+
+compactor:
+  compaction:
+    compaction_window: 1h
+    max_block_bytes: 100_000_000
+    block_retention: 1h
+    compacted_block_retention: 10m
+
+storage:
+  trace:
+    backend: local
+    local:
+      path: /var/tempo/traces
+    wal:
+      path: /var/tempo/wal
+    pool:
+      max_workers: 100
+      queue_depth: 10000

package.json

@@ -22,7 +22,7 @@
         "build:contracts:EPSimulations07": "forge build --quiet --root contracts --evm-version paris --out ../src/contracts/ src/v07/EntryPointSimulations.sol",
         "build:contracts:EPSimulations08": "forge build --quiet --root contracts --evm-version cancun --out ../src/contracts/ src/v08/EntryPointSimulations.sol",
         "build:contracts": "pnpm run build:contracts:PimlicoSimulations && pnpm run build:contracts:EPFilterOpsOverride06 && pnpm run build:contracts:EPFilterOpsOverride07 && pnpm run build:contracts:EPFilterOpsOverride08 && pnpm run build:contracts:EPGasEstimationOverride06 && pnpm run build:contracts:EPSimulations07 && pnpm run build:contracts:EPSimulations08",
-        "start": "node src/esm/cli/alto.js run",
+        "start": "node --require ./src/esm/cli/instrumentation.js src/esm/cli/alto.js run",
         "dev": "nodemon --ext ts,js,json --watch src --exec DOTENV_CONFIG_PATH=$(pwd)/.env tsx --tsconfig src/tsconfig.json src/cli/alto.ts run",
         "test": "pnpm --filter e2e run test",
         "test:ci": "pnpm --filter e2e run test:ci",

src/cli/config/bundler.ts

@@ -340,9 +340,6 @@ export type IGasEstimationArgsInput = z.input<typeof gasEstimationArgsSchema>
 export type IMempoolArgs = z.infer<typeof mempoolArgsSchema>
 export type IMempoolArgsInput = z.input<typeof mempoolArgsSchema>
 
-export type IOptions = z.infer<typeof optionArgsSchema>
-export type IOptionsInput = z.input<typeof optionArgsSchema>
-
 export type IRedisArgs = z.infer<typeof redisArgsSchema>
 export type IRedisArgsInput = z.input<typeof redisArgsSchema>
 
@@ -358,3 +355,6 @@ export const optionArgsSchema = z.object({
     ...mempoolArgsSchema.shape,
     ...redisArgsSchema.shape
 })
+
+export type IOptions = z.infer<typeof optionArgsSchema>
+export type IOptionsInput = z.input<typeof optionArgsSchema>