Strict YAML parsing (#6652)

Adds a custom YAML unmarshaller in the `//strictyaml` package based on
`go-yaml/yaml v3` with unique key detection enabled and ensures that
target struct is able to contain all target fields.

Fixes https://github.com/letsencrypt/boulder/issues/3344.

ca/ecdsa_allow_list.go

@@ -5,8 +5,8 @@ import (
 
 	"github.com/letsencrypt/boulder/log"
 	"github.com/letsencrypt/boulder/reloader"
+	"github.com/letsencrypt/boulder/strictyaml"
 	"github.com/prometheus/client_golang/prometheus"
-	"gopkg.in/yaml.v3"
 )
 
 // ECDSAAllowList acts as a container for a map of Registration IDs, a
@@ -24,7 +24,7 @@ type ECDSAAllowList struct {
 // of a YAML list (as bytes).
 func (e *ECDSAAllowList) Update(contents []byte) error {
 	var regIDs []int64
-	err := yaml.Unmarshal(contents, &regIDs)
+	err := strictyaml.Unmarshal(contents, &regIDs)
 	if err != nil {
 		return err
 	}

cmd/boulder-observer/main.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/letsencrypt/boulder/cmd"
 	"github.com/letsencrypt/boulder/observer"
-	"gopkg.in/yaml.v3"
+	"github.com/letsencrypt/boulder/strictyaml"
 )
 
 func main() {
@@ -19,7 +19,8 @@ func main() {
 
 	// Parse the YAML config file.
 	var config observer.ObsConf
-	err = yaml.Unmarshal(configYAML, &config)
+	err = strictyaml.Unmarshal(configYAML, &config)
+
 	if err != nil {
 		cmd.FailOnError(err, "failed to parse YAML config")
 	}

cmd/ceremony/main.go

@@ -17,6 +17,7 @@ import (
 	"github.com/letsencrypt/boulder/cmd"
 	"github.com/letsencrypt/boulder/linter"
 	"github.com/letsencrypt/boulder/pkcs11helpers"
+	"github.com/letsencrypt/boulder/strictyaml"
 	"golang.org/x/crypto/ocsp"
 	"gopkg.in/yaml.v3"
 )
@@ -465,11 +466,8 @@ func signAndWriteCert(tbs, issuer *x509.Certificate, subjectPubKey crypto.Public
 }
 
 func rootCeremony(configBytes []byte) error {
-	d := yaml.NewDecoder(bytes.NewReader(configBytes))
-	d.KnownFields(true)
-
 	var config rootConfig
-	err := d.Decode(&config)
+	err := strictyaml.Unmarshal(configBytes, &config)
 	if err != nil {
 		return fmt.Errorf("failed to parse config: %s", err)
 	}
@@ -504,11 +502,8 @@ func rootCeremony(configBytes []byte) error {
 }
 
 func intermediateCeremony(configBytes []byte, ct certType) error {
-	d := yaml.NewDecoder(bytes.NewReader(configBytes))
-	d.KnownFields(true)
-
 	var config intermediateConfig
-	err := d.Decode(&config)
+	err := strictyaml.Unmarshal(configBytes, &config)
 	if err != nil {
 		return fmt.Errorf("failed to parse config: %s", err)
 	}
@@ -554,11 +549,8 @@ func intermediateCeremony(configBytes []byte, ct certType) error {
 }
 
 func csrCeremony(configBytes []byte) error {
-	d := yaml.NewDecoder(bytes.NewReader(configBytes))
-	d.KnownFields(true)
-
 	var config csrConfig
-	err := d.Decode(&config)
+	err := strictyaml.Unmarshal(configBytes, &config)
 	if err != nil {
 		return fmt.Errorf("failed to parse config: %s", err)
 	}
@@ -600,11 +592,8 @@ func csrCeremony(configBytes []byte) error {
 }
 
 func keyCeremony(configBytes []byte) error {
-	d := yaml.NewDecoder(bytes.NewReader(configBytes))
-	d.KnownFields(true)
-
 	var config keyConfig
-	err := d.Decode(&config)
+	err := strictyaml.Unmarshal(configBytes, &config)
 	if err != nil {
 		return fmt.Errorf("failed to parse config: %s", err)
 	}
@@ -636,11 +625,8 @@ func keyCeremony(configBytes []byte) error {
 }
 
 func ocspRespCeremony(configBytes []byte) error {
-	d := yaml.NewDecoder(bytes.NewReader(configBytes))
-	d.KnownFields(true)
-
 	var config ocspRespConfig
-	err := d.Decode(&config)
+	err := strictyaml.Unmarshal(configBytes, &config)
 	if err != nil {
 		return fmt.Errorf("failed to parse config: %s", err)
 	}
@@ -709,11 +695,8 @@ func ocspRespCeremony(configBytes []byte) error {
 }
 
 func crlCeremony(configBytes []byte) error {
-	d := yaml.NewDecoder(bytes.NewReader(configBytes))
-	d.KnownFields(true)
-
 	var config crlConfig
-	err := d.Decode(&config)
+	err := strictyaml.Unmarshal(configBytes, &config)
 	if err != nil {
 		return fmt.Errorf("failed to parse config: %s", err)
 	}
@@ -794,6 +777,11 @@ func main() {
 	var ct struct {
 		CeremonyType string `yaml:"ceremony-type"`
 	}
+
+	// We are intentionally using non-strict unmarshaling to read the top level
+	// tags to populate the "ct" struct for use in the switch statement below.
+	// Further strict processing of each yaml node is done on a case by case basis
+	// inside the switch statement.
 	err = yaml.Unmarshal(configBytes, &ct)
 	if err != nil {
 		log.Fatalf("Failed to parse config: %s", err)

goodkey/blocked.go

@@ -9,8 +9,7 @@ import (
 	"os"
 
 	"github.com/letsencrypt/boulder/core"
-
-	yaml "gopkg.in/yaml.v3"
+	"github.com/letsencrypt/boulder/strictyaml"
 )
 
 // blockedKeys is a type for maintaining a map of SHA256 hashes
@@ -58,7 +57,7 @@ func loadBlockedKeysList(filename string) (*blockedKeys, error) {
 		BlockedHashes    []string `yaml:"blocked"`
 		BlockedHashesHex []string `yaml:"blockedHashesHex"`
 	}
-	err = yaml.Unmarshal(yamlBytes, &list)
+	err = strictyaml.Unmarshal(yamlBytes, &list)
 	if err != nil {
 		return nil, err
 	}

observer/probers/crl/crl_conf.go

@@ -5,8 +5,8 @@ import (
 	"net/url"
 
 	"github.com/letsencrypt/boulder/observer/probers"
+	"github.com/letsencrypt/boulder/strictyaml"
 	"github.com/prometheus/client_golang/prometheus"
-	"gopkg.in/yaml.v3"
 )
 
 const (
@@ -28,7 +28,8 @@ func (c CRLConf) Kind() string {
 // UnmarshalSettings constructs a CRLConf object from YAML as bytes.
 func (c CRLConf) UnmarshalSettings(settings []byte) (probers.Configurer, error) {
 	var conf CRLConf
-	err := yaml.Unmarshal(settings, &conf)
+	err := strictyaml.Unmarshal(settings, &conf)
+
 	if err != nil {
 		return nil, err
 	}

observer/probers/dns/dns_conf.go

@@ -7,9 +7,9 @@ import (
 	"strings"
 
 	"github.com/letsencrypt/boulder/observer/probers"
+	"github.com/letsencrypt/boulder/strictyaml"
 	"github.com/miekg/dns"
 	"github.com/prometheus/client_golang/prometheus"
-	"gopkg.in/yaml.v3"
 )
 
 var (
@@ -33,7 +33,7 @@ func (c DNSConf) Kind() string {
 // UnmarshalSettings constructs a DNSConf object from YAML as bytes.
 func (c DNSConf) UnmarshalSettings(settings []byte) (probers.Configurer, error) {
 	var conf DNSConf
-	err := yaml.Unmarshal(settings, &conf)
+	err := strictyaml.Unmarshal(settings, &conf)
 	if err != nil {
 		return nil, err
 	}

observer/probers/http/http_conf.go

@@ -5,8 +5,8 @@ import (
 	"net/url"
 
 	"github.com/letsencrypt/boulder/observer/probers"
+	"github.com/letsencrypt/boulder/strictyaml"
 	"github.com/prometheus/client_golang/prometheus"
-	"gopkg.in/yaml.v3"
 )
 
 // HTTPConf is exported to receive YAML configuration.
@@ -26,7 +26,7 @@ func (c HTTPConf) Kind() string {
 // HTTPConf object.
 func (c HTTPConf) UnmarshalSettings(settings []byte) (probers.Configurer, error) {
 	var conf HTTPConf
-	err := yaml.Unmarshal(settings, &conf)
+	err := strictyaml.Unmarshal(settings, &conf)
 	if err != nil {
 		return nil, err
 	}

observer/probers/mock/mock_conf.go

@@ -5,8 +5,8 @@ import (
 
 	"github.com/letsencrypt/boulder/cmd"
 	"github.com/letsencrypt/boulder/observer/probers"
+	"github.com/letsencrypt/boulder/strictyaml"
 	"github.com/prometheus/client_golang/prometheus"
-	"gopkg.in/yaml.v3"
 )
 
 type MockConfigurer struct {
@@ -25,7 +25,7 @@ func (c MockConfigurer) Kind() string {
 
 func (c MockConfigurer) UnmarshalSettings(settings []byte) (probers.Configurer, error) {
 	var conf MockConfigurer
-	err := yaml.Unmarshal(settings, &conf)
+	err := strictyaml.Unmarshal(settings, &conf)
 	if err != nil {
 		return nil, err
 	}

observer/probers/tls/tls_conf.go

@@ -6,8 +6,8 @@ import (
 	"strings"
 
 	"github.com/letsencrypt/boulder/observer/probers"
+	"github.com/letsencrypt/boulder/strictyaml"
 	"github.com/prometheus/client_golang/prometheus"
-	"gopkg.in/yaml.v3"
 )
 
 const (
@@ -32,7 +32,7 @@ func (c TLSConf) Kind() string {
 // object.
 func (c TLSConf) UnmarshalSettings(settings []byte) (probers.Configurer, error) {
 	var conf TLSConf
-	err := yaml.Unmarshal(settings, &conf)
+	err := strictyaml.Unmarshal(settings, &conf)
 	if err != nil {
 		return nil, err
 	}

policy/pa.go

@@ -22,7 +22,7 @@ import (
 	"github.com/letsencrypt/boulder/identifier"
 	blog "github.com/letsencrypt/boulder/log"
 	"github.com/letsencrypt/boulder/reloader"
-	"gopkg.in/yaml.v3"
+	"github.com/letsencrypt/boulder/strictyaml"
 )
 
 // AuthorityImpl enforces CA policy decisions.
@@ -89,7 +89,7 @@ func (pa *AuthorityImpl) loadHostnamePolicy(contents []byte) error {
 	hash := sha256.Sum256(contents)
 	pa.log.Infof("loading hostname policy, sha256: %s", hex.EncodeToString(hash[:]))
 	var policy blockedNamesPolicy
-	err := yaml.Unmarshal(contents, &policy)
+	err := strictyaml.Unmarshal(contents, &policy)
 	if err != nil {
 		return err
 	}

ratelimit/rate-limits.go

@@ -4,9 +4,8 @@ import (
 	"sync"
 	"time"
 
-	"gopkg.in/yaml.v3"
-
 	"github.com/letsencrypt/boulder/cmd"
+	"github.com/letsencrypt/boulder/strictyaml"
 )
 
 // Limits is defined to allow mock implementations be provided during unit
@@ -118,7 +117,7 @@ func (r *limitsImpl) NewOrdersPerAccount() RateLimitPolicy {
 // YAML configuration (typically read from disk by a reloader)
 func (r *limitsImpl) LoadPolicies(contents []byte) error {
 	var newPolicy rateLimitConfig
-	err := yaml.Unmarshal(contents, &newPolicy)
+	err := strictyaml.Unmarshal(contents, &newPolicy)
 	if err != nil {
 		return err
 	}

strictyaml/yaml.go

@@ -0,0 +1,46 @@
+// Package strictyaml provides a strict YAML unmarshaller based on `go-yaml/yaml`
+package strictyaml
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+
+	"gopkg.in/yaml.v3"
+)
+
+// Unmarshal takes a byte array and an interface passed by reference. The
+// d.Decode will read the next YAML-encoded value from its input and store it in
+// the value pointed to by yamlObj. Any config keys from the incoming YAML
+// document which do not correspond to expected keys in the config struct will
+// result in errors.
+//
+// TODO(https://github.com/go-yaml/yaml/issues/639): Replace this function with
+// yaml.Unmarshal once a more ergonomic way to set unmarshal options is added
+// upstream.
+func Unmarshal(b []byte, yamlObj interface{}) error {
+	r := bytes.NewReader(b)
+
+	d := yaml.NewDecoder(r)
+	d.KnownFields(true)
+
+	// d.Decode will mutate yamlObj
+	err := d.Decode(yamlObj)
+
+	if err != nil {
+		// io.EOF is returned when the YAML document is empty.
+		if errors.Is(err, io.EOF) {
+			return fmt.Errorf("unmarshalling YAML, bytes cannot be nil: %w", err)
+		}
+		return fmt.Errorf("unmarshalling YAML: %w", err)
+	}
+
+	// As bytes are read by the decoder, the length of the byte buffer should
+	// decrease. If it doesn't, there's a problem.
+	if r.Len() != 0 {
+		return fmt.Errorf("yaml object of size %d bytes had %d bytes of unexpected unconsumed trailers", r.Size(), r.Len())
+	}
+
+	return nil
+}

strictyaml/yaml_test.go

@@ -0,0 +1,47 @@
+package strictyaml
+
+import (
+	"io"
+	"testing"
+
+	"github.com/letsencrypt/boulder/test"
+)
+
+var (
+	emptyConfig = []byte(``)
+	validConfig = []byte(`
+a: c
+d: c
+`)
+	invalidConfig1 = []byte(`
+x: y
+`)
+
+	invalidConfig2 = []byte(`
+a: c
+d: c
+x:
+  - hey
+`)
+)
+
+func TestStrictYAMLUnmarshal(t *testing.T) {
+	var config struct {
+		A string `yaml:"a"`
+		D string `yaml:"d"`
+	}
+
+	err := Unmarshal(validConfig, &config)
+	test.AssertNotError(t, err, "yaml: unmarshal errors")
+	test.AssertNotError(t, err, "EOF")
+
+	err = Unmarshal(invalidConfig1, &config)
+	test.AssertError(t, err, "yaml: unmarshal errors")
+
+	err = Unmarshal(invalidConfig2, &config)
+	test.AssertError(t, err, "yaml: unmarshal errors")
+
+	// Test an empty buffer (config file)
+	err = Unmarshal(emptyConfig, &config)
+	test.AssertErrorIs(t, err, io.EOF)
+}