OpenTelemetry monitoring

app.py

@@ -1,3 +1,5 @@
+import os
+import logging
 from fastapi import FastAPI, responses, status
 
 
@@ -14,8 +16,24 @@
 from submodules.model.business_objects import general
 from submodules.model import enums
 from submodules.model import session
+from submodules.model import telemetry
 
-app = FastAPI()
+
+OTLP_GRPC_ENDPOINT = os.getenv("OTLP_GRPC_ENDPOINT", "tempo:4317")
+
+app_name = "refinery-tokenizer"
+app = FastAPI(title=app_name)
+
+if telemetry.ENABLE_TELEMETRY:
+    print("WARNING:  Running telemetry.", flush=True)
+    telemetry.setting_otlp(app, app_name=app_name, endpoint=OTLP_GRPC_ENDPOINT)
+    app.add_middleware(telemetry.PrometheusMiddleware, app_name=app_name)
+    app.add_route("/metrics", telemetry.metrics)
+
+    # Filter out /metrics
+    logging.getLogger("uvicorn.access").addFilter(
+        lambda record: "GET /metrics" not in record.getMessage()
+    )
 
 
 @app.middleware("http")

start

@@ -2,11 +2,13 @@
 
 DEBUG_MODE=false
 DEBUG_PORT=15671
+ENABLE_TELEMETRY=false
 
-while getopts d flag
+while getopts dg flag
 do
     case "${flag}" in
         d) DEBUG_MODE=true;;
+        g) ENABLE_TELEMETRY=true;;
     esac
 done
 
@@ -52,6 +54,7 @@ docker run -d --rm \
 -e S3_USE_SSL=0 \
 -e WS_NOTIFY_ENDPOINT=http://refinery-websocket:8080 \
 -e POSTGRES=postgresql://postgres:kern@graphql-postgres:5432 \
+-e ENABLE_TELEMETRY=$ENABLE_TELEMETRY \
 -v "$INFERENCE_DIR":/inference \
 --mount type=bind,source="$(pwd)"/,target=/app \
 -v /var/run/docker.sock:/var/run/docker.sock \