Support newline-delimited JSON and fix YAML multi-doc vs. sequence

acceptance.bats

@@ -461,6 +461,12 @@ EOF"
   [ "$status" -eq 0 ]
 }
 
+@test "Can validate newline-delimited JSON log files" {
+  run ./conftest test -p examples/ndjson/policy/ examples/ndjson/sample.ndjson
+  [ "$status" -eq 1 ]
+  [[ "$output" =~ "2 tests, 1 passed, 0 warnings, 1 failure, 0 exceptions" ]]
+}
+
 @test "Should fail if strict is set and there are unused variables in the policy" {
   run ./conftest test -p examples/strict-rules/policy/ examples/kubernetes/deployment.yaml --strict
   [ "$status" -eq 1 ]

builtins/parse_config.go

@@ -1,14 +1,14 @@
 package builtins
 
 import (
-	"encoding/json"
+	"bytes"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 
 	"github.com/open-policy-agent/conftest/parser"
 	"github.com/open-policy-agent/opa/v1/ast"
-	"github.com/open-policy-agent/opa/v1/ast/location"
 	"github.com/open-policy-agent/opa/v1/rego"
 	"github.com/open-policy-agent/opa/v1/types"
 )
@@ -53,25 +53,40 @@ func registerParseCombinedConfigFiles() {
 }
 
 // parseConfig takes a parser name and configuration as strings and returns the
-// parsed configuration as a Rego object. This can be used to parse all of the
+// parsed configuration as a Rego object. This can be used to parse all the
 // configuration formats conftest supports in-line in Rego policies.
 func parseConfig(bctx rego.BuiltinContext, op1, op2 *ast.Term) (*ast.Term, error) {
 	args, err := decodeArgs[string](op1, op2)
 	if err != nil {
 		return nil, fmt.Errorf("decode args: %w", err)
 	}
 	parserName, config := args[0], args[1]
-
-	parser, err := parser.New(parserName)
+	configParser, err := parser.New(parserName)
 	if err != nil {
 		return nil, fmt.Errorf("create config parser: %w", err)
 	}
-	var cfg any
-	if err := parser.Unmarshal([]byte(config), &cfg); err != nil {
-		return nil, fmt.Errorf("unmarshal config: %w", err)
+	if bctx.Location == nil {
+		bctx.Location = &ast.Location{
+			File: "-", // stdin (inline)
+			Text: []byte(config),
+		}
 	}
+	return parseConfigToAST(bctx, configParser, bytes.NewBufferString(config))
+}
 
-	return toAST(bctx, cfg, []byte(config))
+func parseConfigToAST(bctx rego.BuiltinContext, configParser parser.Parser, config io.Reader) (*ast.Term, error) {
+	parsed, err := configParser.Parse(config)
+	if err != nil {
+		return nil, fmt.Errorf("unmarshal config: %w", err)
+	}
+	// maintain backwards compatibility and provide single document as non-slice cfg
+	var cfg any
+	if len(parsed) == 1 {
+		cfg = parsed[0]
+	} else {
+		cfg = parsed
+	}
+	return toAST(bctx, cfg)
 }
 
 // parseConfigFile takes a config file path, parses the config file, and
@@ -83,21 +98,15 @@ func parseConfigFile(bctx rego.BuiltinContext, op1 *ast.Term) (*ast.Term, error)
 	}
 	filePath := filepath.Join(filepath.Dir(bctx.Location.File), path)
 
-	parser, err := parser.NewFromPath(filePath)
+	configParser, err := parser.NewFromPath(filePath)
 	if err != nil {
 		return nil, fmt.Errorf("create config parser: %w", err)
 	}
-	contents, err := os.ReadFile(filePath)
+	file, err := os.Open(filePath)
 	if err != nil {
-		return nil, fmt.Errorf("read config file %s: %w", filePath, err)
+		return nil, fmt.Errorf("open config file %s: %w", filePath, err)
 	}
-
-	var cfg any
-	if err := parser.Unmarshal(contents, &cfg); err != nil {
-		return nil, fmt.Errorf("unmarshal config: %w", err)
-	}
-
-	return toAST(bctx, cfg, contents)
+	return parseConfigToAST(bctx, configParser, file)
 }
 
 // parseCombinedConfigFiles takes multiple config file paths, parses the configs,
@@ -116,12 +125,7 @@ func parseCombinedConfigFiles(bctx rego.BuiltinContext, op1 *ast.Term) (*ast.Ter
 		return nil, fmt.Errorf("parse combine configurations: %w", err)
 	}
 	combined := parser.CombineConfigurations(cfg)
-	content, err := json.Marshal(combined)
-	if err != nil {
-		return nil, fmt.Errorf("marshal combined content: %w", err)
-	}
-
-	return toAST(bctx, combined["Combined"], content)
+	return toAST(bctx, combined["Combined"])
 }
 
 func decodeSliceArg[T any](arg *ast.Term) ([]T, error) {
@@ -173,20 +177,10 @@ func decodeArgs[T any](args ...*ast.Term) ([]T, error) {
 	return decoded, nil
 }
 
-func toAST(bctx rego.BuiltinContext, cfg any, contents []byte) (*ast.Term, error) {
+func toAST(bctx rego.BuiltinContext, cfg any) (*ast.Term, error) {
 	val, err := ast.InterfaceToValue(cfg)
 	if err != nil {
 		return nil, fmt.Errorf("convert config to ast.Value: %w", err)
 	}
-	var loc *location.Location
-	if bctx.Location != nil {
-		loc = bctx.Location
-	} else {
-		loc = &ast.Location{
-			File: "-", // stdin
-			Text: contents,
-		}
-	}
-
-	return &ast.Term{Value: val, Location: loc}, nil
+	return &ast.Term{Value: val, Location: bctx.Location}, nil
 }

docs/examples.md

@@ -23,6 +23,7 @@ You can find examples using various other tools in the `examples` directory, inc
 * [Jsonnet](https://github.com/open-policy-agent/conftest/tree/master/examples/jsonnet)
 * [Kubernetes](https://github.com/open-policy-agent/conftest/tree/master/examples/kubernetes)
 * [Kustomize](https://github.com/open-policy-agent/conftest/tree/master/examples/kustomize)
+* [Newline-delimited JSON](https://github.com/open-policy-agent/conftest/tree/master/examples/ndjson)
 * [Properties](https://github.com/open-policy-agent/conftest/tree/master/examples/properties)
 * [Report](https://github.com/open-policy-agent/conftest/tree/master/examples/report)
 * [Serverless Framework](https://github.com/open-policy-agent/conftest/tree/master/examples/serverless)

examples/ndjson/policy/logs.rego

@@ -0,0 +1,7 @@
+package main
+import rego.v1
+
+deny contains msg if {
+	input.level == "ERROR"
+	msg = sprintf("error log found, message '%s'", [input.message])
+}

examples/ndjson/sample.ndjson

@@ -0,0 +1,2 @@
+{"level": "INFO", "message": "some message"}
+{"level": "ERROR", "message":  "some failure"}

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/KeisukeYamashita/go-vcl v0.4.0
 	github.com/basgys/goxml2json v1.1.0
 	github.com/bufbuild/protocompile v0.6.0
+	github.com/dimchansky/utfbom v1.1.1
 	github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665
 	github.com/go-ini/ini v1.67.0
 	github.com/google/go-cmp v0.7.0
@@ -24,18 +25,17 @@ require (
 	github.com/open-policy-agent/opa v1.8.0
 	github.com/opencontainers/image-spec v1.1.1
 	github.com/owenrumney/go-sarif/v2 v2.3.3
-	github.com/pkg/errors v0.9.1
 	github.com/shteou/go-ignore v0.3.1
 	github.com/spdx/tools-golang v0.5.5
 	github.com/spf13/cobra v1.9.1
 	github.com/spf13/viper v1.20.1
 	github.com/subosito/gotenv v1.6.0
 	github.com/tmccombs/hcl2json v0.6.7
+	go.yaml.in/yaml/v4 v4.0.0-rc.2
 	golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0
 	google.golang.org/protobuf v1.36.6
 	olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3
 	oras.land/oras-go/v2 v2.6.0
-	sigs.k8s.io/yaml v1.6.0
 )
 
 require (
@@ -101,6 +101,7 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/prometheus/client_golang v1.23.0 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
@@ -155,4 +156,5 @@ require (
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect
 	google.golang.org/grpc v1.74.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	sigs.k8s.io/yaml v1.6.0 // indirect
 )

go.sum

@@ -733,6 +733,8 @@ github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINA
 github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI=
 github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54 h1:SG7nF6SRlWhcT7cNTs5R6Hk4V2lcmLz2NsG2VnInyNo=
 github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
+github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
+github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
@@ -1251,6 +1253,8 @@ go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
 go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 go.yaml.in/yaml/v3 v3.0.3 h1:bXOww4E/J3f66rav3pX3m8w6jDE4knZjGOw8b5Y6iNE=
 go.yaml.in/yaml/v3 v3.0.3/go.mod h1:tBHosrYAkRZjRAOREWbDnBXUf08JOwYq++0QNwQiWzI=
+go.yaml.in/yaml/v4 v4.0.0-rc.2 h1:/FrI8D64VSr4HtGIlUtlFMGsm7H7pWTbj6vOLVZcA6s=
+go.yaml.in/yaml/v4 v4.0.0-rc.2/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=

internal/commands/parse.go

@@ -1,7 +1,6 @@
 package commands
 
 import (
-	"encoding/json"
 	"fmt"
 
 	"github.com/open-policy-agent/conftest/parser"
@@ -43,7 +42,7 @@ func NewParseCommand() *cobra.Command {
 			return nil
 		},
 		RunE: func(_ *cobra.Command, files []string) error {
-			var configurations map[string]any
+			var configurations map[string][]any
 			var err error
 			if viper.GetString("parser") != "" {
 				configurations, err = parser.ParseConfigurationsAs(files, viper.GetString("parser"))
@@ -57,10 +56,8 @@ func NewParseCommand() *cobra.Command {
 			var output string
 			if viper.GetBool("combine") {
 				output, err = parser.FormatCombined(configurations)
-			} else if len(configurations) == 1 {
-				output, err = formatSingleJSON(configurations)
 			} else {
-				output, err = parser.FormatJSON(configurations)
+				output, err = parser.Format(configurations)
 			}
 			if err != nil {
 				return fmt.Errorf("format output: %w", err)
@@ -76,19 +73,3 @@ func NewParseCommand() *cobra.Command {
 
 	return &cmd
 }
-
-func formatSingleJSON(configurations map[string]any) (string, error) {
-	if len(configurations) != 1 {
-		return "", fmt.Errorf("formatSingleJSON: only supports one configuration")
-	}
-	var config any
-	for _, cfg := range configurations {
-		config = cfg
-	}
-	marshaled, err := json.MarshalIndent(config, "", "  ")
-	if err != nil {
-		return "", err
-	}
-
-	return string(marshaled), nil
-}

parser/cue/cue.go

@@ -3,6 +3,7 @@ package cue
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 
 	"cuelang.org/go/cue/cuecontext"
 	cformat "cuelang.org/go/cue/format"
@@ -11,24 +12,29 @@ import (
 // Parser is a CUE parser.
 type Parser struct{}
 
-// Unmarshal unmarshals CUE files.
-func (*Parser) Unmarshal(p []byte, v any) error {
+// Parse parses CUE files.
+func (*Parser) Parse(r io.Reader) ([]any, error) {
+	p, err := io.ReadAll(r)
+	if err != nil {
+		return nil, fmt.Errorf("read: %w", err)
+	}
 	out, err := cformat.Source(p)
 	if err != nil {
-		return fmt.Errorf("format cue: %w", err)
+		return nil, fmt.Errorf("format cue: %w", err)
 	}
 
 	cueContext := cuecontext.New()
 	cueBytes := cueContext.CompileBytes(out)
 
 	cueJSON, err := cueBytes.MarshalJSON()
 	if err != nil {
-		return fmt.Errorf("marshal json: %w", err)
+		return nil, fmt.Errorf("marshal json: %w", err)
 	}
 
-	if err := json.Unmarshal(cueJSON, v); err != nil {
-		return fmt.Errorf("unmarshal cue json: %w", err)
+	var v any
+	if err := json.Unmarshal(cueJSON, &v); err != nil {
+		return nil, fmt.Errorf("unmarshal cue json: %w", err)
 	}
 
-	return nil
+	return []any{v}, nil
 }

parser/cue/cue_test.go

@@ -1,6 +1,7 @@
 package cue
 
 import (
+	"bytes"
 	"testing"
 )
 
@@ -25,16 +26,16 @@ func TestCueParser(t *testing.T) {
 
 	parser := &Parser{}
 
-	var input any
-	if err := parser.Unmarshal([]byte(p), &input); err != nil {
+	input, err := parser.Parse(bytes.NewReader([]byte(p)))
+	if err != nil {
 		t.Fatalf("parser should not have thrown an error: %v", err)
 	}
 
-	if input == nil {
+	if len(input) != 1 {
 		t.Error("There should be information parsed but its nil")
 	}
 
-	inputMap := input.(map[string]any)
+	inputMap := input[0].(map[string]any)
 	kind := inputMap["kind"]
 	if kind != "Deployment" {
 		t.Error("Parsed cuelang file should be a deployment, but was not")

parser/cyclonedx/cyclonedx.go

@@ -5,15 +5,20 @@ import (
 	"encoding/json"
 	"encoding/xml"
 	"fmt"
+	"io"
 
 	"github.com/CycloneDX/cyclonedx-go"
 )
 
 // Parser is a CycloneDX parser.
 type Parser struct{}
 
-// Unmarshal unmarshals CycloneDX files.
-func (*Parser) Unmarshal(p []byte, v any) error {
+// Parse parses CycloneDX files.
+func (*Parser) Parse(r io.Reader) ([]any, error) {
+	p, err := io.ReadAll(r)
+	if err != nil {
+		return nil, fmt.Errorf("read: %w", err)
+	}
 	bomFileFormat := cyclonedx.BOMFileFormatJSON
 	if !json.Valid(p) {
 		bomFileFormat = cyclonedx.BOMFileFormatXML
@@ -29,19 +34,19 @@ func (*Parser) Unmarshal(p []byte, v any) error {
 	if bomFileFormat == cyclonedx.BOMFileFormatXML {
 		var data cyclonedx.BOM
 		if err := xml.Unmarshal(p, &data); err != nil {
-			return fmt.Errorf("unmarshaling XML error: %v", err)
+			return nil, fmt.Errorf("unmarshaling XML error: %v", err)
 		}
 		if d, err := json.Marshal(data); err == nil {
 			temp = d
 		} else {
-			return fmt.Errorf("marshaling JSON error: %v", err)
+			return nil, fmt.Errorf("marshaling JSON error: %v", err)
 		}
 	}
 
-	err := json.Unmarshal(temp, v)
-	if err != nil {
-		return fmt.Errorf("unmarshaling JSON error: %v", err)
+	var v any
+	if err := json.Unmarshal(temp, &v); err != nil {
+		return nil, fmt.Errorf("unmarshaling JSON error: %v", err)
 	}
 
-	return nil
+	return []any{v}, nil
 }