Add secret redaction for all secrets using buildkite-agent redactor (#125)

* Add secret redaction using buildkite-agent

Implements batch redaction of secrets using buildkite-agent's redactor command. Features JSON batching for v3.66.0+ and warns to upgrade for redaction on lower versions.

- Refactors Config and Run to support secret collection
- Updates environment and secret handlers for redaction workflow
- Adds agent capability detection and 1MB JSON chunking for efficiency
- Updates tests to support the refactor of Config and Run
- Updated `README.md` to reflect the changes
- Updated pipeline tests

Author: JoeColeman95

.buildkite/docker-compose.yml

@@ -11,6 +11,13 @@ services:
     environment:
       - GOOS
       - GOARCH
+  tests:
+    build:
+      context: .
+      dockerfile: Dockerfile-compile
+    volumes:
+      - ../:/work:cached
+    working_dir: /work
   release:
     build:
       dockerfile: Dockerfile-release

.buildkite/pipeline.yml

@@ -72,10 +72,12 @@ steps:
 
   - wait
 
-  - label: ":bash: :hammer:"
+  - label: ":go: Unit Tests"
     plugins:
-      docker-compose#v2.2.0:
+      docker-compose#v5.10.0:
+        config: .buildkite/docker-compose.yml
         run: tests
+    command: .buildkite/steps/tests.sh
 
   - label: "㊙️ git-credentials test"
     command: .buildkite/test_credentials.sh
@@ -120,3 +122,14 @@ steps:
           env:
             - GITHUB_RELEASE_ACCESS_TOKEN
 
+  - label: ":shell: Plugin Tests"
+    plugins:
+      - plugin-tester#v1.2.0:
+          folders:
+            - tests
+
+  - label: ":shell: Shellcheck"
+    plugins:
+      - shellcheck#v1.4.0:
+          files:
+            - hooks/**

.buildkite/steps/tests.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "--- Running Go tests"
+
+cd s3secrets-helper
+
+go version
+echo arch is "$(uname -m)"
+
+go install gotest.tools/[email protected]
+
+if [[ "$(go env GOOS)" == "windows" ]]; then
+  gotestsum --junitfile="junit-${BUILDKITE_JOB_ID:-local}.xml" -- -count=1 -race ./...
+else
+  gotestsum --junitfile="junit-${BUILDKITE_JOB_ID:-local}.xml" -- -count=1 -race -cover ./...
+fi
+
+echo "--- Go tests completed successfully"

README.md

@@ -69,6 +69,12 @@ The private key is exposed to both the checkout and the command as an ssh-agent
 The secrets in the env file are exposed as environment variables, as are individual secret files.
 The locations of git-credentials are passed via `GIT_CONFIG_PARAMETERS` environment to git.
 
+## Secret Redaction
+
+When using Buildkite Agent v3.67.0 or later, secrets are automatically redacted from build logs to prevent accidental exposure. The plugin will detect the agent version and use the built-in redactor feature when available.
+
+For agents running older versions, a warning will be displayed recommending an upgrade for enhanced security.
+
 ## Uploading Secrets
 
 ### SSH Keys

s3secrets-helper/env/env.go

@@ -1,11 +1,11 @@
 package env
 
 const (
-	EnvBucket     = "BUILDKITE_PLUGIN_S3_SECRETS_BUCKET"
-	EnvRegion     = "BUILDKITE_PLUGIN_S3_SECRETS_REGION"
-	EnvPrefix     = "BUILDKITE_PLUGIN_S3_SECRETS_BUCKET_PREFIX"
-	EnvPipeline   = "BUILDKITE_PIPELINE_SLUG"
-	EnvRepo       = "BUILDKITE_REPO"
-	EnvCredHelper = "BUILDKITE_PLUGIN_S3_SECRETS_CREDHELPER"
+	EnvBucket                    = "BUILDKITE_PLUGIN_S3_SECRETS_BUCKET"
+	EnvRegion                    = "BUILDKITE_PLUGIN_S3_SECRETS_REGION"
+	EnvPrefix                    = "BUILDKITE_PLUGIN_S3_SECRETS_BUCKET_PREFIX"
+	EnvPipeline                  = "BUILDKITE_PIPELINE_SLUG"
+	EnvRepo                      = "BUILDKITE_REPO"
+	EnvCredHelper                = "BUILDKITE_PLUGIN_S3_SECRETS_CREDHELPER"
 	EnvSkipSSHKeyNotFoundWarning = "BUILDKITE_PLUGIN_S3_SECRETS_SKIP_SSH_KEY_NOT_FOUND_WARNING"
-)
+)

s3secrets-helper/go.mod

@@ -26,4 +26,5 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sso v1.27.0 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.32.0 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.36.0 // indirect
+	github.com/joho/godotenv v1.5.1 // indirect
 )

s3secrets-helper/go.sum

@@ -38,3 +38,5 @@ github.com/aws/smithy-go v1.22.5 h1:P9ATCXPMb2mPjYBgueqJNCA5S9UfktsW0tTxi+a7eqw=
 github.com/aws/smithy-go v1.22.5/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
 github.com/awsdocs/aws-doc-sdk-examples/gov2/testtools v0.0.0-20250305205910-f85b847ca6da h1:+SYXpcEy9JKkpaJp9JM1pKTBIi++DnNbwykRx7MEsn8=
 github.com/awsdocs/aws-doc-sdk-examples/gov2/testtools v0.0.0-20250305205910-f85b847ca6da/go.mod h1:9Oj/8PZn3D5Ftp/Z1QWrIEFE0daERMqfJawL9duHRfc=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=

s3secrets-helper/main.go

@@ -48,7 +48,7 @@ func mainWithError(log *log.Logger) error {
 		return fmt.Errorf("The %s environment variable is required, set it to the path of the git-credential-s3-secrets script.", env.EnvCredHelper)
 	}
 
-	return secrets.Run(secrets.Config{
+	return secrets.Run(&secrets.Config{
 		Repo:                      os.Getenv(env.EnvRepo),
 		Bucket:                    bucket,
 		Prefix:                    prefix,