diff --git a/cla-backend-go/cmd/server.go b/cla-backend-go/cmd/server.go index 94400a08d..86cb3d824 100644 --- a/cla-backend-go/cmd/server.go +++ b/cla-backend-go/cmd/server.go @@ -81,6 +81,7 @@ import ( "github.com/linuxfoundation/easycla/cla-backend-go/api_logs" "github.com/linuxfoundation/easycla/cla-backend-go/signatures" + "github.com/linuxfoundation/easycla/cla-backend-go/telemetry" v2Signatures "github.com/linuxfoundation/easycla/cla-backend-go/v2/signatures" ini "github.com/linuxfoundation/easycla/cla-backend-go/init" @@ -146,6 +147,45 @@ type combinedRepo struct { projects_cla_groups.Repository } +const ( + envDDBAPILogging = "DDB_API_LOGGING" + envOtelDatadogAPILogging = "OTEL_DATADOG_API_LOGGING" +) + +// parseBoolish parses common "boolean-ish" env var values. +// Returns (value, ok). ok=false means "unknown/invalid". +func parseBoolish(v string) (bool, bool) { + s := strings.TrimSpace(strings.ToLower(v)) + switch s { + case "1", "true", "yes", "y", "on": + return true, true + case "0", "false", "no", "n", "off": + return false, true + default: + return false, false + } +} + +// enabledByEnvOrStage implements: +// - if env var set to true/1/yes -> enabled +// - if env var set to false/0/no -> disabled +// - if env var unset/empty -> defaultByStage[idx] +// - idx 0 = dev/default stage, index 1 = prod stage +func enabledByEnvOrStage(envVar, stage string, defaultByStage [2]bool) bool { + if raw, ok := os.LookupEnv(envVar); ok && strings.TrimSpace(raw) != "" { + if b, ok2 := parseBoolish(raw); ok2 { + return b + } + log.Warnf("LG:api-log-flag-invalid:%s value=%q (falling back to STAGE default)", envVar, raw) + } + st := strings.TrimSpace(strings.ToLower(stage)) + if st == "prod" { + return defaultByStage[1] + } + // dev and all non-prod stages default to enabled + return defaultByStage[0] +} + // apiPathLoggerWithDB creates a middleware that logs API requests to DynamoDB func apiPathLoggerWithDB(apiLogsRepo api_logs.Repository) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { @@ -177,6 +217,8 @@ func apiPathLoggerWithDB(apiLogsRepo api_logs.Repository) func(http.Handler) htt } // server function called by environment specific server functions +// +//nolint:gocyclo func server(localMode bool) http.Handler { f := logrus.Fields{ "functionName": "cmd.server", @@ -208,6 +250,28 @@ func server(localMode bool) http.Handler { stage := viper.GetString("STAGE") dynamodbRegion := ini.GetProperty("DYNAMODB_AWS_REGION") + ddbAPILoggingEnabled := enabledByEnvOrStage(envDDBAPILogging, stage, [2]bool{true, false}) + otelDatadogEnabled := enabledByEnvOrStage(envOtelDatadogAPILogging, stage, [2]bool{true, true}) + + // Initialize OTel SDK -> Datadog Lambda Extension (OTLP) once at cold start. + // If init fails, disable OTel logging but never fail startup. + if otelDatadogEnabled { + version := strings.TrimSpace(Commit) + if strings.TrimSpace(version) == "" { + version = Version + } + if strings.TrimSpace(version) == "" { + version = "unknown" + } + if er := telemetry.InitDatadogOTel(telemetry.DatadogOTelConfig{ + Stage: stage, + Service: "easycla-backend", + Version: version, + }); er != nil { + log.Infof("LG:otel-datadog-disabled err=%v", er) + otelDatadogEnabled = false + } + } log.WithFields(f).Infof("Service %s starting...", ini.ServiceName) @@ -221,6 +285,8 @@ func server(localMode bool) http.Handler { log.Infof("Golang OS : %s", runtime.GOOS) log.Infof("Golang Arch : %s", runtime.GOARCH) log.Infof("DYANAMODB_AWS_REGION : %s", dynamodbRegion) + log.Infof("DDB_API_LOGGING : %t", ddbAPILoggingEnabled) + log.Infof("OTEL_DATADOG_API_LOGGING: %t", otelDatadogEnabled) log.Infof("GH_ORG_VALIDATION : %t", githubOrgValidation) log.Infof("COMPANY_USER_VALIDATION : %t", companyUserValidation) log.Infof("STAGE : %s", stage) @@ -239,6 +305,8 @@ func server(localMode bool) http.Handler { f["companyUserValidation"] = companyUserValidation f["stage"] = stage f["serviceHost"] = host + f["ddbAPILogging"] = ddbAPILoggingEnabled + f["otelDatadog"] = otelDatadogEnabled log.WithFields(f).Info("config") } @@ -305,7 +373,14 @@ func server(localMode bool) http.Handler { approvalListRepo := approval_list.NewRepository(awsSession, stage) v1CompanyRepo := v1Company.NewRepository(awsSession, stage) eventsRepo := events.NewRepository(awsSession, stage) - apiLogsRepo := api_logs.NewRepository(stage, dynamodb.New(awsSession)) + + var apiLogsRepo api_logs.Repository + if ddbAPILoggingEnabled { + apiLogsRepo = api_logs.NewRepository(stage, dynamodb.New(awsSession)) + } else { + apiLogsRepo = nil + } + v1ProjectClaGroupRepo := projects_cla_groups.NewRepository(awsSession, stage) v1CLAGroupRepo := repository.NewRepository(awsSession, stage, gitV1Repository, gerritRepo, v1ProjectClaGroupRepo) metricsRepo := metrics.NewRepository(awsSession, stage, configFile.APIGatewayURL, v1ProjectClaGroupRepo) @@ -497,6 +572,12 @@ func server(localMode bool) http.Handler { v2API.Serve(middlewareSetupfunc), v2SwaggerSpec.BasePath()), configFile.AllowedOrigins) } + + // OTel/Datadog (OTLP -> Datadog Lambda Extension) - enabled by flag + if otelDatadogEnabled { + apiHandler = telemetry.WrapHTTPHandler(apiHandler) + } + return apiHandler } diff --git a/cla-backend-go/go.mod b/cla-backend-go/go.mod index 0c1646df8..6bd6be18c 100644 --- a/cla-backend-go/go.mod +++ b/cla-backend-go/go.mod @@ -29,7 +29,7 @@ require ( github.com/gofrs/uuid v4.0.0+incompatible github.com/golang/mock v1.6.0 github.com/google/go-github/v37 v37.0.0 - github.com/google/uuid v1.1.4 + github.com/google/uuid v1.6.0 github.com/gorilla/sessions v1.2.1 // indirect github.com/imroc/req v0.3.0 github.com/jessevdk/go-flags v1.4.0 @@ -37,7 +37,7 @@ require ( github.com/jmoiron/sqlx v1.2.0 github.com/juju/mempool v0.0.0-20160205104927-24974d6c264f // indirect github.com/juju/zip v0.0.0-20160205105221-f6b1e93fa2e2 - github.com/kr/pretty v0.3.0 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mitchellh/mapstructure v1.5.0 @@ -51,18 +51,18 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/spf13/viper v1.12.0 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.11.1 github.com/verdverm/frisby v0.0.0-20170604211311-b16556248a9a github.com/xanzy/go-gitlab v0.50.1 go.uber.org/ratelimit v0.1.0 - golang.org/x/crypto v0.7.0 // indirect + golang.org/x/crypto v0.47.0 // indirect golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d // indirect - golang.org/x/net v0.8.0 - golang.org/x/oauth2 v0.6.0 - golang.org/x/sync v0.2.0 - golang.org/x/sys v0.33.0 // indirect + golang.org/x/net v0.49.0 + golang.org/x/oauth2 v0.34.0 + golang.org/x/sync v0.19.0 + golang.org/x/sys v0.40.0 // indirect golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect ) require ( @@ -70,26 +70,36 @@ require ( github.com/bradleyfalzon/ghinstallation/v2 v2.2.0 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/golang-jwt/jwt/v4 v4.5.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 + go.opentelemetry.io/otel v1.40.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 + go.opentelemetry.io/otel/sdk v1.40.0 ) require ( github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/aws/smithy-go v1.20.2 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/circl v1.3.2 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/fatih/color v1.15.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/analysis v0.21.4 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-playground/locales v0.13.0 // indirect github.com/go-playground/universal-translator v0.17.0 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-github/v50 v50.2.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/gorilla/securecookie v1.1.1 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect github.com/hashicorp/hcl v1.0.0 // indirect @@ -110,7 +120,7 @@ require ( github.com/pelletier/go-toml/v2 v2.0.5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/rogpeppe/go-internal v1.6.1 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/spf13/afero v1.9.5 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect @@ -118,8 +128,15 @@ require ( github.com/subosito/gotenv v1.4.1 // indirect github.com/ugorji/go/codec v1.2.6 // indirect go.mongodb.org/mongo-driver v1.10.1 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/appengine v1.6.7 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect + go.opentelemetry.io/otel/metric v1.40.0 // indirect + go.opentelemetry.io/otel/trace v1.40.0 // indirect + go.opentelemetry.io/proto/otlp v1.9.0 // indirect + golang.org/x/text v0.33.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect + google.golang.org/grpc v1.78.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/cla-backend-go/go.sum b/cla-backend-go/go.sum index 5d80d5f1d..31aeada23 100644 --- a/cla-backend-go/go.sum +++ b/cla-backend-go/go.sum @@ -83,7 +83,11 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2 github.com/bradleyfalzon/ghinstallation/v2 v2.2.0 h1:AVvVU33rE8wdTS1aNnenwpigEBA9mvzI5OhjhZfH/LU= github.com/bradleyfalzon/ghinstallation/v2 v2.2.0/go.mod h1:xo3iIfK0lDKECe0s19nbxT0KKvk7LsrGc4NxR5ckKMA= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -113,6 +117,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= @@ -129,6 +135,11 @@ github.com/go-chi/chi v0.0.0-20180202194135-e223a795a06a/go.mod h1:eB3wogJHnLi3x github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= @@ -306,9 +317,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -321,10 +331,10 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-github/v37 v37.0.0 h1:rCspN8/6kB1BAJWZfuafvHhyfIo5fkAulaP/3bOQ/tM= github.com/google/go-github/v37 v37.0.0/go.mod h1:LM7in3NmXDrX58GbEHy7FtNLbI2JijX93RnMKvWG3m4= github.com/google/go-github/v50 v50.1.0/go.mod h1:Ev4Tre8QoKiolvbpOSG3FIi4Mlon3S2Nt9W5JYqKiwA= @@ -353,8 +363,8 @@ github.com/google/uuid v0.0.0-20171129191014-dec09d789f3d/go.mod h1:TIyPZe4Mgqvf github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.4 h1:0ecGp3skIrHWPNGPJDaBIghfA6Sp7Ruo2Io8eLKzWm0= -github.com/google/uuid v1.1.4/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= @@ -366,6 +376,8 @@ github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyC github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= @@ -427,8 +439,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -479,7 +491,6 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mozillazg/request v0.8.0 h1:TbXeQUdBWr1J1df5Z+lQczDFzX9JD71kTCl7Zu/9rNM= github.com/mozillazg/request v0.8.0/go.mod h1:weoQ/mVFNbWgRBtivCGF1tUT9lwneFesues+CleXMWc= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nightlyone/lockfile v1.0.0/go.mod h1:rywoIealpdNse2r832aiD9jRk8ErCatROs6LzC841CI= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= @@ -499,6 +510,7 @@ github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3v github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -510,8 +522,9 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -560,8 +573,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= @@ -613,8 +626,30 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= +go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= +go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 h1:wVZXIWjQSeSmMoxF74LzAnpVQOAFDo3pPji9Y4SOFKc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0/go.mod h1:khvBS2IggMFNwZK/6lEeHg/W57h/IX6J4URh57fuI40= +go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= +go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= +go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= +go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= +go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= +go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= +go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= +go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= +go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= +go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/ratelimit v0.1.0 h1:U2AruXqeTb4Eh9sYQSTrMhH8Cb7M0Ian2ibBOnBcnAw= go.uber.org/ratelimit v0.1.0/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -634,8 +669,9 @@ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= +golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -720,8 +756,9 @@ golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -732,8 +769,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= -golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -748,8 +785,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -807,8 +844,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -824,8 +861,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -894,6 +931,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -920,7 +959,6 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -958,6 +996,10 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 h1:JLQynH/LBHfCTSbDWl+py8C+Rg/k1OVH3xfcaiANuF0= +google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:kSJwQxqmFXeo79zOmbrALdflXQeAYcUbgS7PbpMknCY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -974,6 +1016,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -984,14 +1028,13 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= diff --git a/cla-backend-go/serverless.yml b/cla-backend-go/serverless.yml index 76bc17f37..b8a77e1aa 100644 --- a/cla-backend-go/serverless.yml +++ b/cla-backend-go/serverless.yml @@ -32,6 +32,16 @@ custom: dynamodb: # Region where dynamodb is installed region: us-east-1 + + # Datadog tagging conventions + placeholders (CloudOps ticket LFIT-7051 will provide real values) + datadog: + dd_env: + dev: dev + staging: staging + prod: prod + site: ${file(./env.json):dd-site-${opt:stage}, ssm:/cla-dd-site-${opt:stage}} + apiKeySecretArn: ${file(./env.json):dd-api-key-secret-arn-${opt:stage}, ssm:/cla-dd-api-key-secret-arn-${opt:stage}} + extensionLayerArn: ${file(./env.json):dd-extension-layer-arn-${opt:stage}, ssm:/cla-dd-extension-layer-arn-${opt:stage}} # Config for serverless-prune-plugin - remove all but the 10 most recent # versions to avoid the "Code storage limit exceeded" error prune: @@ -63,6 +73,11 @@ provider: - "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" - "arn:aws:iam::aws:policy/service-role/AWSLambdaDynamoDBExecutionRole" statements: + - Effect: Allow + Action: + - secretsmanager:GetSecretValue + Resource: + - ${self:custom.datadog.apiKeySecretArn} - Effect: Allow Action: - cloudwatch:* @@ -274,6 +289,19 @@ provider: # Turn on USER_AUTH_TRACING to see additional debug of user scopes for the authenticated users - output is verbose USER_AUTH_TRACING: true + DDB_API_LOGGING: ${file(./env.json):ddb-api-logging-${opt:stage}, ssm:/cla-ddb-api-logging-${opt:stage}} + OTEL_DATADOG_API_LOGGING: ${file(./env.json):otel-datadog-api-logging-${opt:stage}, ssm:/cla-otel-datadog-api-logging-${opt:stage}} + + # Datadog Lambda Extension (OTLP/HTTP) - Go backend uses pure OTel SDK -> DD Extension. + DD_ENV: ${self:custom.datadog.dd_env.${opt:stage}, self:custom.datadog.dd_env.dev} + DD_SERVICE: easycla-backend + # DD_SITE: ${self:custom.datadog.site} + # DD_API_KEY_SECRET_ARN: ${self:custom.datadog.apiKeySecretArn} + DD_SITE: ${file(./env.json):dd-site-${opt:stage}, ssm:/cla-dd-site-${opt:stage}} + DD_API_KEY_SECRET_ARN: ${file(./env.json):dd-api-key-secret-arn-${opt:stage}, ssm:/cla-dd-api-key-secret-arn-${opt:stage}} + DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_HTTP_ENDPOINT: localhost:4318 + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: http://localhost:4318/v1/traces + stackTags: Name: ${self:service} stage: ${self:provider.stage} @@ -322,3 +350,5 @@ functions: patterns: - '!**' - 'bin/backend-aws-lambda' + layers: + - ${self:custom.datadog.extensionLayerArn} diff --git a/cla-backend-go/telemetry/datadog_otlp.go b/cla-backend-go/telemetry/datadog_otlp.go new file mode 100644 index 000000000..b990833b5 --- /dev/null +++ b/cla-backend-go/telemetry/datadog_otlp.go @@ -0,0 +1,266 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package telemetry + +import ( + "context" + "fmt" + "net/http" + "net/url" + "os" + "regexp" + "strings" + "sync" + "time" + + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +// DatadogOTelConfig configures OTel SDK for exporting traces to the Datadog Lambda Extension. +type DatadogOTelConfig struct { + Stage string + Service string + Version string +} + +var ( + ddInitOnce sync.Once + ddInitErr error +) + +// InitDatadogOTel initializes the global OTel SDK (tracer provider + OTLP exporter). +// Safe to call multiple times (sync.Once). Never panics. +func InitDatadogOTel(cfg DatadogOTelConfig) error { + ddInitOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Tags (prefer explicit DD_* env vars; fallback to stage/config). + ddEnv := strings.TrimSpace(os.Getenv("DD_ENV")) + if ddEnv == "" { + ddEnv = stageToDDEnv(cfg.Stage) + } + + ddService := strings.TrimSpace(os.Getenv("DD_SERVICE")) + if ddService == "" { + ddService = cfg.Service + } + + ddVersion := strings.TrimSpace(os.Getenv("DD_VERSION")) + if ddVersion == "" { + ddVersion = cfg.Version + } + + exporter, err := newOTLPHTTPExporter(ctx) + if err != nil { + ddInitErr = err + return + } + + // Vendor-neutral resource attributes (Datadog maps these automatically). + res, err := resource.New(ctx, + resource.WithFromEnv(), + resource.WithTelemetrySDK(), + resource.WithAttributes( + attribute.String("service.name", ddService), + attribute.String("service.version", ddVersion), + attribute.String("deployment.environment.name", ddEnv), + ), + ) + if err != nil { + ddInitErr = err + return + } + + tp := sdktrace.NewTracerProvider( + sdktrace.WithResource(res), + // Batch exporter => async export (no per-request network IO). + sdktrace.WithBatcher(exporter), + ) + + otel.SetTracerProvider(tp) + otel.SetTextMapPropagator( + propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + ), + ) + }) + + if ddInitErr != nil { + log.Infof("LG:otel-datadog-init-failed err=%v", ddInitErr) + } + return ddInitErr +} + +// WrapHTTPHandler instruments inbound HTTP requests using otelhttp and produces spans. +func WrapHTTPHandler(next http.Handler) http.Handler { + // Regexes mirror ./utils/count_apis.sh so OTel span names group the same way as the offline API log rollups: + // - collapse multiple slashes + // - trim trailing slash + // - mask common asset extensions -> ".{asset}" + // - normalize Swagger assets "/vN/swagger.{asset}" -> "/vN/swagger" (keep version; do NOT map to /v*) + // - mask UUIDs, numeric IDs, Salesforce IDs, LFX IDs, and literal "null" segments + reMultiSlash := regexp.MustCompile(`/{2,}`) + reAssetExt := regexp.MustCompile(`\.(png|svg|css|js|json|xml|htm|html)$`) + reSwaggerAsset := regexp.MustCompile(`^(/v[0-9]+)/swagger\.\{asset\}$`) + reUUID := regexp.MustCompile(`[0-9a-fA-F-]{36}`) + reNumericID := regexp.MustCompile(`/[0-9]+(/|$)`) + reSFID := regexp.MustCompile(`/(?:00|a0)[A-Za-z0-9]{13,16}(/|$)`) + reLFXID := regexp.MustCompile(`/lf[A-Za-z0-9]{16,22}(/|$)`) + reNull := regexp.MustCompile(`/null(/|$)`) + + sanitize := func(path string) string { + p := strings.TrimSpace(path) + if p == "" { + return "/" + } + if !strings.HasPrefix(p, "/") { + p = "/" + p + } + + p = reMultiSlash.ReplaceAllString(p, "/") + if len(p) > 1 && strings.HasSuffix(p, "/") { + p = strings.TrimSuffix(p, "/") + } + + // Asset extensions (including swagger.json/xml/html) -> ".{asset}" + p = reAssetExt.ReplaceAllString(p, ".{asset}") + + // Keep the version (/v1, /v2, ...) but normalize swagger asset paths. + if m := reSwaggerAsset.FindStringSubmatch(p); m != nil { + p = m[1] + "/swagger" + } + + // Dynamic segment masking (use template placeholders, not "*") + p = reUUID.ReplaceAllString(p, "{uuid}") + p = reNumericID.ReplaceAllString(p, "/{id}$1") + p = reSFID.ReplaceAllString(p, "/{sfid}$1") + p = reLFXID.ReplaceAllString(p, "/{lfxid}$1") + p = reNull.ReplaceAllString(p, "/{null}$1") + + if p == "" { + return "/" + } + return p + } + + return otelhttp.NewHandler( + next, + "easycla-http", + otelhttp.WithSpanNameFormatter(func(_ string, r *http.Request) string { + // LG: use this to have per distinct API URL datapoints + // return fmt.Sprintf("%s %s", r.Method, r.URL.Path) + return fmt.Sprintf("%s %s", r.Method, sanitize(r.URL.Path)) + }), + ) +} + +func newOTLPHTTPExporter(ctx context.Context) (sdktrace.SpanExporter, error) { + // Standard overrides; default to Datadog Lambda Extension OTLP/HTTP. + // + // OTLP/HTTP env var rules: + // - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT is per-signal. If set, preserve its path verbatim + // (default to "/" if no path). + // - OTEL_EXPORTER_OTLP_ENDPOINT is a base endpoint. If set (and per-signal is not), + // append "/v1/traces" (handling trailing slashes). + tracesEndpoint := strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")) + baseEndpoint := strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")) + + var ( + endpoint string + usedTracesEndpoint bool + usedBaseEndpoint bool + ) + + if tracesEndpoint != "" { + endpoint = tracesEndpoint + usedTracesEndpoint = true + } else if baseEndpoint != "" { + endpoint = baseEndpoint + usedBaseEndpoint = true + } else { + // Datadog Lambda Extension default (OTLP/HTTP). + endpoint = "http://localhost:4318/v1/traces" + // Default is already the full traces endpoint => treat like per-signal. + usedTracesEndpoint = true + } + + var host string + parsedPath := "" + insecure := true + + // Accept full URL or host:port[/path] + if strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://") { + u, err := url.Parse(endpoint) + if err != nil { + return nil, err + } + host = u.Host + parsedPath = u.Path + insecure = (u.Scheme == "http") + } else { + host = endpoint + if strings.Contains(endpoint, "/") { + parts := strings.SplitN(endpoint, "/", 2) + host = parts[0] + // Preserve remainder as path (empty remainder => "/") + parsedPath = "/" + parts[1] + } + } + + // Normalize empty/missing paths to "/" (URL semantics) + if strings.TrimSpace(parsedPath) == "" { + parsedPath = "/" + } else if !strings.HasPrefix(parsedPath, "/") { + // Defensive (shouldn't happen with url.Parse) + parsedPath = "/" + parsedPath + } + + path := parsedPath + if usedBaseEndpoint { + // Base endpoint: append OTLP/HTTP traces path, handling trailing slashes. + base := strings.TrimRight(parsedPath, "/") + path = base + "/v1/traces" + } else if usedTracesEndpoint { + // Per-signal endpoint: preserve path verbatim (already normalized above) + path = parsedPath + } + + if strings.TrimSpace(host) == "" { + return nil, fmt.Errorf("invalid OTLP endpoint: %q", endpoint) + } + + opts := []otlptracehttp.Option{ + otlptracehttp.WithEndpoint(host), + otlptracehttp.WithURLPath(path), + otlptracehttp.WithTimeout(2 * time.Second), + } + if insecure { + opts = append(opts, otlptracehttp.WithInsecure()) + } + + return otlptracehttp.New(ctx, opts...) +} + +func stageToDDEnv(stage string) string { + const prod = "prod" + const staging = "staging" + switch strings.ToLower(strings.TrimSpace(stage)) { + case prod: + return prod + case staging: + return staging + default: + return "dev" + } +} diff --git a/cla-backend/cla/routes.py b/cla-backend/cla/routes.py index f3e4a2386..0adbc56a1 100755 --- a/cla-backend/cla/routes.py +++ b/cla-backend/cla/routes.py @@ -6,10 +6,16 @@ """ import hug +import os +import re +import time +from urllib.parse import urlparse import requests from falcon import HTTP_401, HTTP_400, HTTP_OK, HTTP_500, Response from hug.middleware import LogMiddleware +import pdb + import cla import cla.auth import cla.controllers.company @@ -40,6 +46,393 @@ _APILOG_CLS = None _APILOG_IMPORT_ERROR = None +_FEATURE_FLAG_CACHE = {} + + +# --- OTel/Datadog (OTLP/HTTP -> Datadog Lambda Extension) state --- +_OTEL_TRACER = None +_OTEL_TRACER_PROVIDER = None +_OTEL_INIT_ERROR = None +_OTEL_DISABLED = False +_OTEL_DISABLED_REASON = None + +def _disable_otel(reason): + global _OTEL_DISABLED, _OTEL_DISABLED_REASON + if _OTEL_DISABLED: + return + _OTEL_DISABLED = True + _OTEL_DISABLED_REASON = reason + try: + pdb.set_trace() + cla.log.info(f"LG:otel-datadog-disabled reason={reason}") + except Exception: + pass + +# --- Path sanitizer regexes (mirror ./utils/count_apis.sh, but keep /vN versions intact) --- +_RE_MULTI_SLASH = re.compile(r"/{2,}") +_RE_ASSET_EXT = re.compile(r"\.(png|svg|css|js|json|xml|htm|html)$") +_RE_SWAGGER_ASSET = re.compile(r"^(/v[0-9]+)/swagger\.\{asset\}$") +_RE_UUID = re.compile(r"[0-9a-fA-F-]{36}") +_RE_NUMERIC_ID = re.compile(r"/[0-9]+(/|$)") +_RE_SFID = re.compile(r"/(?:00|a0)[A-Za-z0-9]{13,16}(/|$)") +_RE_LFXID = re.compile(r"/lf[A-Za-z0-9]{16,22}(/|$)") +_RE_NULL = re.compile(r"/null(/|$)") + +def _sanitize_api_path(path: str) -> str: + """ + Low-cardinality path template matching ./utils/count_apis.sh behavior, + except we DO NOT collapse /v1,/v2,... into /v* (version is preserved). + """ + p = (path or "").strip() + if p == "": + return "/" + if not p.startswith("/"): + p = "/" + p + + p = _RE_MULTI_SLASH.sub("/", p) + if len(p) > 1 and p.endswith("/"): + p = p[:-1] + + # Assets -> ".{asset}" + p = _RE_ASSET_EXT.sub(".{asset}", p) + + # /vN/swagger.{asset} -> /vN/swagger (keep version) + p = _RE_SWAGGER_ASSET.sub(r"\1/swagger", p) + + # Dynamic IDs -> placeholders + p = _RE_UUID.sub("{uuid}", p) + p = _RE_NUMERIC_ID.sub(r"/{id}\1", p) + p = _RE_SFID.sub(r"/{sfid}\1", p) + p = _RE_LFXID.sub(r"/{lfxid}\1", p) + p = _RE_NULL.sub(r"/{null}\1", p) + + pdb.set_trace() + return p or "/" + +def _stage_to_dd_env(stage: str) -> str: + st = (stage or "dev").strip().lower() + if st == "prod": + return "prod" + if st == "staging": + return "staging" + return "dev" + +def _build_otlp_traces_endpoint() -> str: + """ + Match the Go exporter selection logic: + - prefer OTEL_EXPORTER_OTLP_TRACES_ENDPOINT if set (preserve its path verbatim; default "/" if missing) + - else use OTEL_EXPORTER_OTLP_ENDPOINT as base and append "/v1/traces" (handling trailing slashes) + - else default to "http://localhost:4318/v1/traces" + + Accept full URL or host:port[/path]. + Returns a full URL including scheme + path (suitable for OTLPSpanExporter(endpoint=...)). + """ + traces_ep = (os.getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") or "").strip() + base_ep = (os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT") or "").strip() + + used_base = False + if traces_ep: + raw = traces_ep + elif base_ep: + raw = base_ep + used_base = True + else: + raw = "http://localhost:4318/v1/traces" + + scheme = "http" + host = "" + path = "/" + + if raw.startswith("http://") or raw.startswith("https://"): + u = urlparse(raw) + scheme = (u.scheme or "http") + host = u.netloc + path = u.path or "/" + else: + # host:port[/path] (default scheme http) + if "/" in raw: + host, rest = raw.split("/", 1) + path = "/" + rest if rest else "/" + else: + host = raw + path = "/" + + if not path.startswith("/"): + path = "/" + path + + if used_base: + base_path = path.rstrip("/") + path = base_path + "/v1/traces" + + if not host or host.strip() == "": + raise ValueError(f"invalid OTLP endpoint: {raw!r}") + + pdb.set_trace() + return f"{scheme}://{host}{path}" + +def _init_otel_datadog() -> None: + """ + Initialize a minimal OTel SDK pipeline for exporting spans to the Datadog Lambda Extension via OTLP/HTTP. + Never raises; on failure it caches the error and becomes a no-op. + """ + global _OTEL_TRACER, _OTEL_TRACER_PROVIDER, _OTEL_INIT_ERROR + + if _OTEL_DISABLED: + return + + if _OTEL_TRACER is not None or _OTEL_INIT_ERROR is not None: + return + try: + # Lazy import so we never fail module import / Lambda cold start if deps are missing. + pdb.set_trace() + from opentelemetry import trace as otel_trace + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + pdb.set_trace() + except Exception as e: + _OTEL_INIT_ERROR = e + _disable_otel(f"init-import-1 err={e}") + return + + try: + stage = (os.getenv("STAGE", "dev") or "dev").strip() + dd_env = (os.getenv("DD_ENV") or "").strip() or _stage_to_dd_env(stage) + dd_service = (os.getenv("DD_SERVICE") or "").strip() or "easycla-backend" + dd_version = (os.getenv("DD_VERSION") or "").strip() or (os.getenv("VERSION") or "").strip() or "unknown" + + endpoint = _build_otlp_traces_endpoint() + + exporter = OTLPSpanExporter(endpoint=endpoint, timeout=0.5) + + # Wrap exporter so any export failure disables tracing for this container + from opentelemetry.sdk.trace.export import SpanExportResult + pdb.set_trace() + class _FailFastExporter: + def __init__(self, inner): + self._inner = inner + def export(self, spans): + if _OTEL_DISABLED: + return SpanExportResult.FAILURE + try: + res = self._inner.export(spans) + except Exception as ex: + _disable_otel(f"export err={ex}") + return SpanExportResult.FAILURE + if res != SpanExportResult.SUCCESS: + _disable_otel(f"export result={res}") + return res + def shutdown(self): + try: + return self._inner.shutdown() + except Exception: + return + + resource = Resource.create({ + # Vendor-neutral resource attrs (Datadog maps these automatically). + "service.name": dd_service, + "service.version": dd_version, + "deployment.environment.name": dd_env, + }) + + provider = TracerProvider(resource=resource) + # In Lambda, synchronous export is safest; FailFastExporter prevents repeated latency on failure. + provider.add_span_processor(SimpleSpanProcessor(_FailFastExporter(exporter))) + + otel_trace.set_tracer_provider(provider) + + _OTEL_TRACER_PROVIDER = provider + _OTEL_TRACER = otel_trace.get_tracer("easycla-http") + pdb.set_trace() + except Exception as e: + pdb.set_trace() + _OTEL_INIT_ERROR = e + _disable_otel(f"init-import-2 err={e}") + +def _parse_http_status_code(status): + """ + Falcon typically stores response.status like "200 OK". + Return int status code or None. + """ + pdb.set_trace() + if status is None: + return None + try: + s = str(status).strip() + if s == "": + return None + # "200 OK" -> 200 + pdb.set_trace() + return int(s.split()[0]) + except Exception: + return None + +def _otel_start_request_span(request) -> None: + """ + Start a SERVER span for the inbound request and store it in request.context. + Never raises. + """ + try: + req_ctx = getattr(request, "context", None) + pdb.set_trace() + if req_ctx is None: + return + # Defensive: don't double-start + if req_ctx.get("_otel_span") is not None: + return + except Exception: + pdb.set_trace() + return + + try: + _init_otel_datadog() + if _OTEL_TRACER is None: + return + + from opentelemetry import context as otel_context + from opentelemetry.propagate import extract + from opentelemetry.trace import SpanKind, Status, StatusCode, set_span_in_context + pdb.set_trace() + except Exception as e: + pdb.set_trace() + try: + cla.log.info(f"LG:api-log-otel-datadog-init-missing err={e}") + except Exception: + pass + return + + method = (getattr(request, "method", "GET") or "GET").strip().upper() + raw_path = getattr(request, "path", "/") + route = _sanitize_api_path(raw_path) + span_name = f"{method} {route}" + + try: + # Extract W3C parent context from inbound headers (low cost). + headers = getattr(request, "headers", None) or {} + carrier = {} + for k in ("traceparent", "tracestate", "baggage"): + try: + v = headers.get(k) + except Exception: + v = None + if v: + carrier[k] = v + + parent_ctx = extract(carrier) + pdb.set_trace() + + span = _OTEL_TRACER.start_span(span_name, context=parent_ctx, kind=SpanKind.SERVER) + # Low-cardinality attrs + span.set_attribute("http.method", method) + span.set_attribute("http.route", route) + + # Make span current for the remainder of the request (so any future child spans attach correctly). + ctx_with_span = set_span_in_context(span, parent_ctx) + token = otel_context.attach(ctx_with_span) + + # Persist for response middleware. + request.context["_otel_span"] = span + request.context["_otel_ctx_token"] = token + request.context["_otel_route"] = route + pdb.set_trace() + except Exception as e: + pdb.set_trace() + try: + cla.log.info(f"LG:api-log-otel-datadog-failed:{route} err={e}") + except Exception: + pass + +def _otel_end_request_span(request, response) -> None: + """ + End the SERVER span for the inbound request, set status code, and detach context. + Never raises. + """ + span = None + token = None + route = None + pdb.set_trace() + + try: + ctx = getattr(request, "context", None) + if ctx is None: + return + span = ctx.pop("_otel_span", None) + token = ctx.pop("_otel_ctx_token", None) + route = ctx.pop("_otel_route", None) + pdb.set_trace() + except Exception: + # If request.context isn't mutable/dict-like for some reason, just bail. + pdb.set_trace() + return + + try: + if span is None: + return + + status_code = _parse_http_status_code(getattr(response, "status", None)) + if status_code is not None: + span.set_attribute("http.status_code", status_code) + # Mark 5xx as errors (4xx are usually client errors, not service faults) + if status_code >= 500: + from opentelemetry.trace import Status, StatusCode + span.set_status(Status(StatusCode.ERROR)) + + span.end() + pdb.set_trace() + except Exception as e: + pdb.set_trace() + try: + if route is None: + route = _sanitize_api_path(getattr(request, "path", "/")) + cla.log.info(f"LG:api-log-otel-datadog-failed:{route} err={e}") + except Exception: + pass + finally: + # Always detach if we attached. + pdb.set_trace() + if token is not None: + try: + from opentelemetry import context as otel_context + otel_context.detach(token) + except Exception: + pass + +def _parse_boolish(value): + pdb.set_trace() + if value is None: + return None + v = str(value).strip().lower() + if v in ("1", "true", "yes", "y", "on"): + return True + if v in ("0", "false", "no", "n", "off"): + return False + return None + +def _enabled_by_env_or_stage(env_var: str, default_by_stage: tuple[bool, bool]) -> bool: + # cache (env vars don't change during a lambda container lifetime) + if env_var in _FEATURE_FLAG_CACHE: + pdb.set_trace() + return _FEATURE_FLAG_CACHE[env_var] + + raw = os.getenv(env_var) + if raw is not None and raw.strip() != "": + parsed = _parse_boolish(raw) + pdb.set_trace() + if parsed is not None: + _FEATURE_FLAG_CACHE[env_var] = parsed + return parsed + try: + cla.log.info(f"LG:api-log-flag-invalid:{env_var} value={raw} (falling back to STAGE default)") + except Exception: + pass + + stage = (os.getenv("STAGE", "dev") or "dev").strip().lower() + is_prod = stage == "prod" + enabled = default_by_stage[1] if is_prod else default_by_stage[0] + _FEATURE_FLAG_CACHE[env_var] = enabled + pdb.set_trace() + return enabled def _get_apilog_cls(): """ @@ -76,13 +469,23 @@ def process_data_api_logs(request, response): """ cla.log.info('LG:api-request-path:' + request.path) - # Log API request to DynamoDB table - apilog_cls = _get_apilog_cls() - if apilog_cls is not None: - try: - apilog_cls.log_api_request(request.path) - except Exception as e: - cla.log.info(f"LG:api-log-dynamo-failed:{request.path} err={e}") + # DynamoDB API logging (conditional) + if _enabled_by_env_or_stage("DDB_API_LOGGING", default_by_stage=(True, False)): + apilog_cls = _get_apilog_cls() + pdb.set_trace() + if apilog_cls is not None: + try: + apilog_cls.log_api_request(request.path) + pdb.set_trace() + except Exception as e: + pdb.set_trace() + cla.log.info(f"LG:api-log-dynamo-failed:{request.path} err={e}") + + # OTel/Datadog API logging (OTLP/HTTP -> Datadog Lambda Extension) + if _enabled_by_env_or_stage("OTEL_DATADOG_API_LOGGING", default_by_stage=(True, True)): + pdb.set_trace() + _otel_start_request_span(request) + pdb.set_trace() if "/github/activity" in request.path: body = request.bounded_stream.read() @@ -98,6 +501,10 @@ def process_data(request, response, resource): response.set_header("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS") response.set_header("Access-Control-Allow-Headers", "Content-Type, Authorization") + # Close the OTel span after handlers run (captures final status code). + if _enabled_by_env_or_stage("OTEL_DATADOG_API_LOGGING", default_by_stage=(True, True)): + _otel_end_request_span(request, response) + @hug.directive() def check_auth(request=None, **kwargs): @@ -108,9 +515,14 @@ def check_auth(request=None, **kwargs): @hug.exception(cla.auth.AuthError) -def handle_auth_error(exception, response=None, **kwargs): +def handle_auth_error(exception, request=None, response=None, **kwargs): """Handles authentication errors""" response.status = HTTP_401 + + # Ensure OTel span closes even if response middleware isn't invoked for exceptions. + if _enabled_by_env_or_stage("OTEL_DATADOG_API_LOGGING", default_by_stage=(True, True)): + pdb.set_trace() + _otel_end_request_span(request, response) return exception.response diff --git a/cla-backend/requirements.txt b/cla-backend/requirements.txt index baa8af478..0afe6b431 100644 --- a/cla-backend/requirements.txt +++ b/cla-backend/requirements.txt @@ -57,3 +57,7 @@ astroid==3.3.8 pluggy==1.5.0 gunicorn==22.0.0 PyNaCl==1.5.0 +# OpenTelemetry (OTLP/HTTP exporter) for Datadog Lambda Extension +opentelemetry-api<2,>=1 +opentelemetry-sdk<2,>=1 +opentelemetry-exporter-otlp-proto-http<2,>=1 diff --git a/cla-backend/serverless.yml b/cla-backend/serverless.yml index 68cc9bfd5..a9a1bc045 100644 --- a/cla-backend/serverless.yml +++ b/cla-backend/serverless.yml @@ -29,6 +29,14 @@ package: custom: allowed_origins: ${file(./env.json):cla-allowed-origins-${sls:stage}, ssm:/cla-allowed-origins-${sls:stage}} + datadog: + dd_env: + dev: dev + staging: staging + prod: prod + site: ${file(./env.json):dd-site-${sls:stage}, ssm:/cla-dd-site-${sls:stage}} + apiKeySecretArn: ${file(./env.json):dd-api-key-secret-arn-${sls:stage}, ssm:/cla-dd-api-key-secret-arn-${sls:stage}} + extensionLayerArn: ${file(./env.json):dd-extension-layer-arn-${sls:stage}, ssm:/cla-dd-extension-layer-arn-${sls:stage}} wsgi: app: cla.routes.__hug_wsgi__ pythonBin: python @@ -138,6 +146,11 @@ provider: - "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" - "arn:aws:iam::aws:policy/service-role/AWSLambdaDynamoDBExecutionRole" statements: + - Effect: Allow + Action: + - secretsmanager:GetSecretValue + Resource: + - ${self:custom.datadog.apiKeySecretArn} - Effect: Allow Action: - cloudwatch:* @@ -361,6 +374,20 @@ provider: # https://github.com/pypa/setuptools/issues/2232 SETUPTOOLS_USE_DISTUTILS: stdlib + # API usage logging toggles: + DDB_API_LOGGING: ${file(./env.json):ddb-api-logging-${sls:stage}, ssm:/cla-ddb-api-logging-${sls:stage}} + OTEL_DATADOG_API_LOGGING: ${file(./env.json):otel-datadog-api-logging-${sls:stage}, ssm:/cla-otel-datadog-api-logging-${sls:stage}} + + # Datadog/OTel placeholders (Python OTel is stubbed for now) + DD_ENV: ${self:custom.datadog.dd_env.${sls:stage}, self:custom.datadog.dd_env.dev} + DD_SERVICE: easycla-backend + # DD_SITE: ${self:custom.datadog.site} + # DD_API_KEY_SECRET_ARN: ${self:custom.datadog.apiKeySecretArn} + DD_SITE: ${file(./env.json):dd-site-${sls:stage}, ssm:/cla-dd-site-${sls:stage}} + DD_API_KEY_SECRET_ARN: ${file(./env.json):dd-api-key-secret-arn-${sls:stage}, ssm:/cla-dd-api-key-secret-arn-${sls:stage}} + DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_HTTP_ENDPOINT: localhost:4318 + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: http://localhost:4318/v1/traces + stackTags: Name: ${self:service} stage: ${sls:stage} @@ -405,6 +432,8 @@ functions: individually: true patterns: - 'auth/bin/**' + layers: + - ${self:custom.datadog.extensionLayerArn} api-v3-lambda: name: ${self:service}-${sls:stage, 'dev'}-api-v3-lambda @@ -420,6 +449,8 @@ functions: individually: true patterns: - 'bin/backend-aws-lambda' + layers: + - ${self:custom.datadog.extensionLayerArn} dynamo-projects-events-lambda: name: ${self:service}-${sls:stage, 'dev'}-dynamo-projects-lambda @@ -430,6 +461,8 @@ functions: individually: true patterns: - 'bin/dynamo-events-lambda' + layers: + - ${self:custom.datadog.extensionLayerArn} dynamo-signatures-events-lambda: handler: 'bin/dynamo-events-lambda' @@ -440,6 +473,8 @@ functions: individually: true patterns: - 'bin/dynamo-events-lambda' + layers: + - ${self:custom.datadog.extensionLayerArn} dynamo-events-events-lambda: handler: 'bin/dynamo-events-lambda' @@ -450,6 +485,8 @@ functions: individually: true patterns: - 'bin/dynamo-events-lambda' + layers: + - ${self:custom.datadog.extensionLayerArn} dynamo-repositories-events-lambda: handler: 'bin/dynamo-events-lambda' @@ -460,6 +497,8 @@ functions: individually: true patterns: - 'bin/dynamo-events-lambda' + layers: + - ${self:custom.datadog.extensionLayerArn} dynamo-projects-cla-groups-events-lambda: handler: 'bin/dynamo-events-lambda' @@ -470,6 +509,8 @@ functions: individually: true patterns: - 'bin/dynamo-events-lambda' + layers: + - ${self:custom.datadog.extensionLayerArn} dynamo-github-orgs-events-lambda: handler: 'bin/dynamo-events-lambda' @@ -480,6 +521,8 @@ functions: individually: true patterns: - 'bin/dynamo-events-lambda' + layers: + - ${self:custom.datadog.extensionLayerArn} save-metrics-lambda: name: ${self:service}-${sls:stage, 'dev'}-save-metrics-lambda @@ -496,6 +539,8 @@ functions: individually: true patterns: - 'bin/metrics-aws-lambda' + layers: + - ${self:custom.datadog.extensionLayerArn} report-metrics-lambda: name: ${self:service}-${sls:stage, 'dev'}-report-metrics-lambda @@ -512,6 +557,8 @@ functions: individually: true patterns: - 'bin/metrics-report-lambda' + layers: + - ${self:custom.datadog.extensionLayerArn} zip-builder-scheduler-lambda: name: ${self:service}-${sls:stage, 'dev'}-zip-builder-scheduler-lambda @@ -528,6 +575,8 @@ functions: individually: true patterns: - 'bin/zipbuilder-scheduler-lambda' + layers: + - ${self:custom.datadog.extensionLayerArn} zip-builder-lambda: handler: 'bin/zipbuilder-lambda' @@ -540,6 +589,8 @@ functions: individually: true patterns: - 'bin/zipbuilder-lambda' + layers: + - ${self:custom.datadog.extensionLayerArn} gitlab-repository-check-lambda: handler: 'bin/gitlab-repository-check-lambda' @@ -557,6 +608,8 @@ functions: individually: true patterns: - 'bin/gitlab-repository-check-lambda' + layers: + - ${self:custom.datadog.extensionLayerArn} # User Subscribe event for dynamodb cla-stage-users table. easycla-user-event-handler-lambda: @@ -572,6 +625,8 @@ functions: events: - sns: arn: ${self:custom.userEventsSNSTopicARN} + layers: + - ${self:custom.datadog.extensionLayerArn} apiv1: handler: wsgi_handler.handler @@ -581,6 +636,8 @@ functions: method: ANY path: v1/{proxy+} cors: true + layers: + - ${self:custom.datadog.extensionLayerArn} apiv2: handler: wsgi_handler.handler @@ -590,6 +647,8 @@ functions: method: ANY path: v2/{proxy+} cors: true + layers: + - ${self:custom.datadog.extensionLayerArn} salesforceprojects: handler: cla.salesforce.get_projects @@ -599,6 +658,8 @@ functions: method: ANY path: v1/salesforce/projects cors: true + layers: + - ${self:custom.datadog.extensionLayerArn} salesforceprojectbyID: handler: cla.salesforce.get_project @@ -608,6 +669,8 @@ functions: method: ANY path: v1/salesforce/project cors: true + layers: + - ${self:custom.datadog.extensionLayerArn} # GitHub callback handler githubinstall: @@ -617,6 +680,8 @@ functions: - http: method: ANY path: v2/github/installation + layers: + - ${self:custom.datadog.extensionLayerArn} # GitHub callback handler githubactivity: @@ -626,6 +691,8 @@ functions: - http: method: POST path: v2/github/activity + layers: + - ${self:custom.datadog.extensionLayerArn} resources: diff --git a/setenv.sh b/setenv.sh index 1d8af99ab..c160dad84 100644 --- a/setenv.sh +++ b/setenv.sh @@ -32,3 +32,17 @@ export GH_ORG_VALIDATION=false export DISABLE_LOCAL_PERMISSION_CHECKS=true export COMPANY_USER_VALIDATION=false export CLA_SIGNATURE_FILES_BUCKET=cla-signature-files-dev + +# Logging +export DDB_API_LOGGING=true +export OTEL_DATADOG_API_LOGGING=true +export DD_ENV=dev +export DD_SERVICE='easycla-backend' +export DD_SITE='datadoghq.com' +# export DD_SITE='app.datadoghq.com' +export DD_API_KEY_SECRET_ARN="$(cat ./DD_API_KEY_SECRET_ARN.secret)" +# Get via aws --profile lfproduct-dev --region us-east-2 secretsmanager get-secret-value --secret-id "$DD_API_KEY_SECRET_ARN" --query SecretString --output text +export DD_API_KEY="$(cat ./DD_API_KEY.secret)" +export DD_EXTENSION_LAYER_ARN_DEV="$(cat ./DD_EXTENSION_LAYER_ARN_DEV.secret)" +export DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_HTTP_ENDPOINT='localhost:4318' +export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT='http://localhost:4318/v1/traces' diff --git a/utils/get_ssm_value.sh b/utils/get_ssm_value.sh index 7c7d1d952..b8aede8e0 100755 --- a/utils/get_ssm_value.sh +++ b/utils/get_ssm_value.sh @@ -1,11 +1,11 @@ #!/bin/bash -set -euo pipefail -export AWS_PAGER="" if [ -z "$1" ] then echo "Usage: $0 " exit 1 fi +set -euo pipefail +export AWS_PAGER="" if [ -z "$REGION" ] then REGION="us-east-2" diff --git a/utils/otel_dd/check_spans.sh b/utils/otel_dd/check_spans.sh new file mode 100755 index 000000000..ddf82494a --- /dev/null +++ b/utils/otel_dd/check_spans.sh @@ -0,0 +1,2 @@ +#!/bin/bash +curl -s http://localhost:8888/metrics | egrep 'otelcol_(receiver_accepted_spans|exporter_sent_spans|exporter_send_failed_spans)' diff --git a/utils/otel_dd/check_spans_in_ddog.sh b/utils/otel_dd/check_spans_in_ddog.sh new file mode 100755 index 000000000..e08afe8fa --- /dev/null +++ b/utils/otel_dd/check_spans_in_ddog.sh @@ -0,0 +1,18 @@ +curl -sS -X POST "https://api.${DD_SITE}/api/v2/spans/events/search" \ + -H "Content-Type: application/json" \ + -H "DD-API-KEY: ${DD_API_KEY}" \ + -H "DD-APPLICATION-KEY: ${DD_APP_KEY}" \ + -d '{ + "data": { + "type": "search_request", + "attributes": { + "filter": { + "from": "now-15m", + "to": "now", + "query": "service:easycla-backend env:dev" + }, + "sort": "timestamp", + "page": { "limit": 10 } + } + } + }' | jq -r diff --git a/utils/otel_dd/otelcol-dd.yaml b/utils/otel_dd/otelcol-dd.yaml new file mode 100644 index 000000000..22a361df5 --- /dev/null +++ b/utils/otel_dd/otelcol-dd.yaml @@ -0,0 +1,30 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 +processors: + batch: +exporters: + debug: + verbosity: detailed + datadog: + api: + key: ${DD_API_KEY} + site: ${DD_SITE} +service: + telemetry: + logs: + level: info + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [datadog] diff --git a/utils/otel_dd/run_collector.sh b/utils/otel_dd/run_collector.sh new file mode 100755 index 000000000..2e432ff44 --- /dev/null +++ b/utils/otel_dd/run_collector.sh @@ -0,0 +1,19 @@ +#!/bin/bash +if [ -z "$DD_API_KEY_SECRET_ARN" ] +then + source setenv.sh +fi +if [ -z "$DD_API_KEY_SECRET_ARN" ] +then + echo "DD_API_KEY_SECRET_ARN is not set. Please set it in setenv.sh and try again." + exit 1 +fi + +docker run --rm -it \ + -p 4318:4318 \ + -p 8888:8888 \ + -e DD_API_KEY -e DD_SITE \ + -v "$PWD/utils/otel_dd/otelcol-dd.yaml:/etc/otelcol/config.yaml:ro" \ + -v /etc/passwd:/etc/passwd:ro \ + otel/opentelemetry-collector-contrib:latest \ + --config /etc/otelcol/config.yaml diff --git a/utils/otel_dd/validate_dd_keys.sh b/utils/otel_dd/validate_dd_keys.sh new file mode 100755 index 000000000..ba0606da1 --- /dev/null +++ b/utils/otel_dd/validate_dd_keys.sh @@ -0,0 +1,2 @@ +#!/bin/bash +curl -sS -H "DD-API-KEY: ${DD_API_KEY}" -H "DD-APPLICATION-KEY: ${DD_APP_KEY}" https://api.datadoghq.com/api/v2/validate_keys | jq -r diff --git a/utils/otel_dd_go/build.sh b/utils/otel_dd_go/build.sh new file mode 100755 index 000000000..c248c9f5e --- /dev/null +++ b/utils/otel_dd_go/build.sh @@ -0,0 +1,2 @@ +#!/bin/bash +go fmt otel_dd.go && go vet otel_dd.go && go build otel_dd.go diff --git a/utils/otel_dd_go/go.mod b/utils/otel_dd_go/go.mod new file mode 100644 index 000000000..206ccd48d --- /dev/null +++ b/utils/otel_dd_go/go.mod @@ -0,0 +1,30 @@ +module github.com/linuxfoundation/easycla/utils/otel_dd_go + +go 1.24.4 + +require ( + go.opentelemetry.io/otel v1.40.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 + go.opentelemetry.io/otel/sdk v1.40.0 +) + +require ( + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect + go.opentelemetry.io/otel/metric v1.40.0 // indirect + go.opentelemetry.io/otel/trace v1.40.0 // indirect + go.opentelemetry.io/proto/otlp v1.9.0 // indirect + golang.org/x/net v0.49.0 // indirect + golang.org/x/sys v0.40.0 // indirect + golang.org/x/text v0.33.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/grpc v1.78.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect +) diff --git a/utils/otel_dd_go/go.sum b/utils/otel_dd_go/go.sum new file mode 100644 index 000000000..66b49f2e8 --- /dev/null +++ b/utils/otel_dd_go/go.sum @@ -0,0 +1,61 @@ +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= +go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 h1:wVZXIWjQSeSmMoxF74LzAnpVQOAFDo3pPji9Y4SOFKc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0/go.mod h1:khvBS2IggMFNwZK/6lEeHg/W57h/IX6J4URh57fuI40= +go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= +go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= +go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= +go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= +go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= +go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= +go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= +go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= +go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= +go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M= +google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/utils/otel_dd_go/otel_dd b/utils/otel_dd_go/otel_dd new file mode 100755 index 000000000..fcc185c91 Binary files /dev/null and b/utils/otel_dd_go/otel_dd differ diff --git a/utils/otel_dd_go/otel_dd.go b/utils/otel_dd_go/otel_dd.go new file mode 100644 index 000000000..82ed8e6d6 --- /dev/null +++ b/utils/otel_dd_go/otel_dd.go @@ -0,0 +1,233 @@ +package main + +import ( + "context" + "fmt" + "net/url" + "os" + "regexp" + "strings" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +var ( + reMultiSlash = regexp.MustCompile(`/{2,}`) + reAssetExt = regexp.MustCompile(`\.(png|svg|css|js|json|xml|htm|html)$`) + reSwaggerAsset = regexp.MustCompile(`^(/v[0-9]+)/swagger\.\{asset\}$`) + reUUID = regexp.MustCompile(`[0-9a-fA-F-]{36}`) + reNumericID = regexp.MustCompile(`/[0-9]+(/|$)`) + reSFID = regexp.MustCompile(`/(?:00|a0)[A-Za-z0-9]{13,16}(/|$)`) + reLFXID = regexp.MustCompile(`/lf[A-Za-z0-9]{16,22}(/|$)`) + reNull = regexp.MustCompile(`/null(/|$)`) +) + +func sanitizeAPIPath(path string) string { + p := strings.TrimSpace(path) + if p == "" { + return "/" + } + if !strings.HasPrefix(p, "/") { + p = "/" + p + } + + p = reMultiSlash.ReplaceAllString(p, "/") + if len(p) > 1 && strings.HasSuffix(p, "/") { + p = strings.TrimSuffix(p, "/") + } + + p = reAssetExt.ReplaceAllString(p, ".{asset}") + if m := reSwaggerAsset.FindStringSubmatch(p); m != nil { + p = m[1] + "/swagger" + } + + p = reUUID.ReplaceAllString(p, "{uuid}") + p = reNumericID.ReplaceAllString(p, "/{id}$1") + p = reSFID.ReplaceAllString(p, "/{sfid}$1") + p = reLFXID.ReplaceAllString(p, "/{lfxid}$1") + p = reNull.ReplaceAllString(p, "/{null}$1") + + if p == "" { + return "/" + } + return p +} + +func stageToDDEnv(stage string) string { + st := strings.ToLower(strings.TrimSpace(stage)) + switch st { + case "prod", "production": + return "prod" + case "staging": + return "staging" + default: + return "dev" + } +} + +// buildOTLPTracesEndpoint matches the same rules as your Go backend: +// 1. OTEL_EXPORTER_OTLP_TRACES_ENDPOINT (preserve path) +// 2. OTEL_EXPORTER_OTLP_ENDPOINT (append /v1/traces) +// 3. default http://localhost:4318/v1/traces +func buildOTLPTracesEndpoint() (host string, path string, insecure bool, err error) { + traces := strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")) + base := strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")) + + raw := "" + usedBase := false + if traces != "" { + raw = traces + } else if base != "" { + raw = base + usedBase = true + } else { + raw = "http://localhost:4318/v1/traces" + } + + insecure = true + parsedPath := "/" + + if strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") { + u, e := url.Parse(raw) + if e != nil { + return "", "", false, e + } + host = u.Host + parsedPath = u.Path + insecure = (u.Scheme == "http") + } else { + // host:port[/path] + host = raw + if strings.Contains(raw, "/") { + parts := strings.SplitN(raw, "/", 2) + host = parts[0] + if parts[1] != "" { + parsedPath = "/" + parts[1] + } else { + parsedPath = "/" + } + } + } + + if strings.TrimSpace(parsedPath) == "" { + parsedPath = "/" + } + if !strings.HasPrefix(parsedPath, "/") { + parsedPath = "/" + parsedPath + } + + if usedBase { + parsedPath = strings.TrimRight(parsedPath, "/") + "/v1/traces" + } + + if strings.TrimSpace(host) == "" { + return "", "", false, fmt.Errorf("invalid OTLP endpoint: %q", raw) + } + + return host, parsedPath, insecure, nil +} + +func main() { + if len(os.Args) < 2 { + fmt.Fprintf(os.Stderr, "usage: %s [METHOD]\n", os.Args[0]) + os.Exit(2) + } + + raw := os.Args[1] + method := "GET" + if len(os.Args) >= 3 { + method = strings.ToUpper(strings.TrimSpace(os.Args[2])) + if method == "" { + method = "GET" + } + } + + pathOnly := raw + if strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") { + if u, err := url.Parse(raw); err == nil && u.Path != "" { + pathOnly = u.Path + } else { + pathOnly = "/" + } + } + + route := sanitizeAPIPath(pathOnly) + spanName := fmt.Sprintf("%s %s", method, route) + + ddEnv := strings.TrimSpace(os.Getenv("DD_ENV")) + if ddEnv == "" { + ddEnv = stageToDDEnv(os.Getenv("STAGE")) + } + ddService := strings.TrimSpace(os.Getenv("DD_SERVICE")) + if ddService == "" { + ddService = "easycla-backend" + } + ddVersion := strings.TrimSpace(os.Getenv("DD_VERSION")) + if ddVersion == "" { + ddVersion = strings.TrimSpace(os.Getenv("VERSION")) + } + if ddVersion == "" { + ddVersion = "unknown" + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + host, urlPath, insecure, err := buildOTLPTracesEndpoint() + if err != nil { + fmt.Fprintf(os.Stderr, "endpoint error: %v\n", err) + os.Exit(3) + } + + opts := []otlptracehttp.Option{ + otlptracehttp.WithEndpoint(host), + otlptracehttp.WithURLPath(urlPath), + otlptracehttp.WithTimeout(2 * time.Second), + } + if insecure { + opts = append(opts, otlptracehttp.WithInsecure()) + } + + exp, err := otlptracehttp.New(ctx, opts...) + if err != nil { + fmt.Fprintf(os.Stderr, "exporter init error: %v\n", err) + os.Exit(4) + } + + res, _ := resource.New(ctx, + resource.WithAttributes( + attribute.String("service.name", ddService), + attribute.String("service.version", ddVersion), + attribute.String("deployment.environment.name", ddEnv), + ), + ) + + // Sync export so the span is sent before process exit. + tp := sdktrace.NewTracerProvider( + sdktrace.WithResource(res), + sdktrace.WithSyncer(exp), + ) + defer func() { + _ = tp.Shutdown(ctx) + }() + + otel.SetTracerProvider(tp) + + tr := otel.Tracer("easycla-otlp-poc") + _, span := tr.Start(ctx, spanName) + span.SetAttributes( + attribute.String("http.method", method), + attribute.String("http.route", route), + attribute.String("http.url", raw), + ) + span.End() + + _ = tp.ForceFlush(ctx) + + fmt.Printf("sent span: %s -> %s%s\n", spanName, host, urlPath) +} diff --git a/utils/otel_dd_py/otel_dd.py b/utils/otel_dd_py/otel_dd.py new file mode 100755 index 000000000..9ded398f4 --- /dev/null +++ b/utils/otel_dd_py/otel_dd.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 + +# Example: +# ./cla-backend/.venv/bin/python ./utils/otel_dd_py/otel_dd.py 'https://example.com/v2/project/123' + +import os +import re +import sys +from urllib.parse import urlparse + +# --- Path sanitizer (same intent as your backend code; keeps /vN intact) --- +_RE_MULTI_SLASH = re.compile(r"/{2,}") +_RE_ASSET_EXT = re.compile(r"\.(png|svg|css|js|json|xml|htm|html)$") +_RE_SWAGGER_ASSET = re.compile(r"^(/v[0-9]+)/swagger\.\{asset\}$") +_RE_UUID = re.compile(r"[0-9a-fA-F-]{36}") +_RE_NUMERIC_ID = re.compile(r"/[0-9]+(/|$)") +_RE_SFID = re.compile(r"/(?:00|a0)[A-Za-z0-9]{13,16}(/|$)") +_RE_LFXID = re.compile(r"/lf[A-Za-z0-9]{16,22}(/|$)") +_RE_NULL = re.compile(r"/null(/|$)") + +def sanitize_api_path(path: str) -> str: + p = (path or "").strip() + if not p: + return "/" + if not p.startswith("/"): + p = "/" + p + + p = _RE_MULTI_SLASH.sub("/", p) + if len(p) > 1 and p.endswith("/"): + p = p[:-1] + + p = _RE_ASSET_EXT.sub(".{asset}", p) + p = _RE_SWAGGER_ASSET.sub(r"\1/swagger", p) + + p = _RE_UUID.sub("{uuid}", p) + p = _RE_NUMERIC_ID.sub(r"/{id}\1", p) + p = _RE_SFID.sub(r"/{sfid}\1", p) + p = _RE_LFXID.sub(r"/{lfxid}\1", p) + p = _RE_NULL.sub(r"/{null}\1", p) + return p or "/" + +def stage_to_dd_env(stage: str) -> str: + st = (stage or "dev").strip().lower() + if st in ("prod", "production"): + return "prod" + if st == "staging": + return "staging" + return "dev" + +def build_otlp_traces_endpoint() -> str: + """ + Preference order: + 1) OTEL_EXPORTER_OTLP_TRACES_ENDPOINT (preserve path) + 2) OTEL_EXPORTER_OTLP_ENDPOINT (append /v1/traces) + 3) default http://localhost:4318/v1/traces + Accepts full URL or host:port[/path]. + """ + traces_ep = (os.getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") or "").strip() + base_ep = (os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT") or "").strip() + + used_base = False + if traces_ep: + raw = traces_ep + elif base_ep: + raw = base_ep + used_base = True + else: + raw = "http://localhost:4318/v1/traces" + + scheme = "http" + host = "" + path = "/" + + if raw.startswith("http://") or raw.startswith("https://"): + u = urlparse(raw) + scheme = u.scheme or "http" + host = u.netloc + path = u.path or "/" + else: + # host:port[/path] + if "/" in raw: + host, rest = raw.split("/", 1) + path = "/" + rest if rest else "/" + else: + host = raw + path = "/" + + if not path.startswith("/"): + path = "/" + path + if used_base: + path = path.rstrip("/") + "/v1/traces" + + if not host.strip(): + raise ValueError(f"invalid OTLP endpoint: {raw!r}") + + return f"{scheme}://{host}{path}" + +def main(argv: list[str]) -> int: + if len(argv) < 2: + print(f"usage: {argv[0]} [METHOD]", file=sys.stderr) + return 2 + + raw = argv[1] + method = (argv[2] if len(argv) >= 3 else "GET").strip().upper() or "GET" + + # Extract path from URL if needed + if raw.startswith("http://") or raw.startswith("https://"): + path = urlparse(raw).path or "/" + else: + path = raw + + route = sanitize_api_path(path) + span_name = f"{method} {route}" + + dd_env = (os.getenv("DD_ENV") or "").strip() or stage_to_dd_env(os.getenv("STAGE", "dev")) + dd_service = (os.getenv("DD_SERVICE") or "").strip() or "easycla-backend" + dd_version = (os.getenv("DD_VERSION") or "").strip() or (os.getenv("VERSION") or "").strip() or "unknown" + + endpoint = build_otlp_traces_endpoint() + + # Lazy imports (so the script can still show a clean error if deps are missing) + try: + from opentelemetry import trace as otel_trace + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + except Exception as e: + print(f"otel deps missing: {e}", file=sys.stderr) + return 3 + + resource = Resource.create({ + "service.name": dd_service, + "service.version": dd_version, + "deployment.environment.name": dd_env, + }) + + provider = TracerProvider(resource=resource) + exporter = OTLPSpanExporter(endpoint=endpoint, timeout=2.0) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + otel_trace.set_tracer_provider(provider) + tracer = otel_trace.get_tracer("easycla-otlp-poc") + + # Emit one span + try: + with tracer.start_as_current_span(span_name) as span: + span.set_attribute("http.method", method) + span.set_attribute("http.route", route) + span.set_attribute("http.url", raw) + finally: + # Ensure it gets pushed before exit + try: + provider.force_flush() + except Exception: + pass + try: + provider.shutdown() + except Exception: + pass + + print(f"sent span: {span_name} -> {endpoint}") + return 0 + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) + diff --git a/utils/set_ssm_logging_params.sh b/utils/set_ssm_logging_params.sh new file mode 100755 index 000000000..c614c1ac5 --- /dev/null +++ b/utils/set_ssm_logging_params.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# REGION=us-east-1 +# STAGE=dev +# SET=1 +# STAGE=dev REGION=us-east-1 SET=1 ./set_ssm_logging_params.sh +if [ -z "$STAGE" ] +then + export STAGE=dev +fi +if [ -z "$REGION" ] +then + export REGION=us-east-1 +fi +if [ ! -z "$SET" ] +then + ./utils/set_ssm_value.sh cla-dd-site-dev 'datadoghq.com' String + # ./utils/set_ssm_value.sh cla-dd-site-dev 'app.datadoghq.com' String + ./utils/set_ssm_value.sh cla-dd-api-key-secret-arn-dev "$(cat ./DD_API_KEY_SECRET_ARN.secret)" String + ./utils/set_ssm_value.sh cla-dd-extension-layer-arn-dev "$(cat ./DD_EXTENSION_LAYER_ARN_DEV.secret)" String + ./utils/set_ssm_value.sh cla-ddb-api-logging-dev true String + ./utils/set_ssm_value.sh cla-otel-datadog-api-logging-dev true String +fi + +./utils/get_ssm_value.sh cla-dd-site-dev +./utils/get_ssm_value.sh cla-dd-api-key-secret-arn-dev +./utils/get_ssm_value.sh cla-dd-extension-layer-arn-dev +./utils/get_ssm_value.sh cla-ddb-api-logging-dev +./utils/get_ssm_value.sh cla-otel-datadog-api-logging-dev diff --git a/utils/set_ssm_value.sh b/utils/set_ssm_value.sh new file mode 100755 index 000000000..60dd5c728 --- /dev/null +++ b/utils/set_ssm_value.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +if [ $# -lt 2 ] +then + echo "Usage: $0 [type]" + echo "Example: STAGE=dev REGION=us-east-1 $0 cla-dd-site-dev dev String" + exit 1 +fi + +set -euo pipefail +export AWS_PAGER="" + +PARAM_NAME="$1" +PARAM_VALUE="$2" +PARAM_TYPE="${3:-String}" # Default to String if not provided + +if [ -z "${REGION:-}" ] +then + REGION="us-east-2" +fi + +if [ -z "${STAGE:-}" ] +then + STAGE="dev" +fi + +echo "Setting SSM parameter:" +echo " Name: ${PARAM_NAME}" +echo " Value: ${PARAM_VALUE}" +echo " Type: ${PARAM_TYPE}" +echo " Region: ${REGION}" +echo " Stage: ${STAGE}" +echo + +aws ssm put-parameter \ + --region "${REGION}" \ + --profile "lfproduct-${STAGE}" \ + --name "${PARAM_NAME}" \ + --value "${PARAM_VALUE}" \ + --type "${PARAM_TYPE}" \ + --overwrite + +echo "Done." +