diff --git a/.chloggen/support-nested-URIs-breaking.yaml b/.chloggen/support-nested-URIs-breaking.yaml new file mode 100644 index 0000000000..b71a3c8119 --- /dev/null +++ b/.chloggen/support-nested-URIs-breaking.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: breaking + +# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver) +component: confmap + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Using an Invalid Scheme in a URI will throw an error. + +# One or more tracking issues or pull requests related to the change +issues: [7504] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/.chloggen/support-nested-URIs.yaml b/.chloggen/support-nested-URIs.yaml new file mode 100755 index 0000000000..fd99168043 --- /dev/null +++ b/.chloggen/support-nested-URIs.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver) +component: confmap + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add support for nested URIs. + +# One or more tracking issues or pull requests related to the change +issues: [7117] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/confmap/README.md b/confmap/README.md index 91aca8c73d..42dcf548d8 100644 --- a/confmap/README.md +++ b/confmap/README.md @@ -37,8 +37,9 @@ that can be used by code that is oblivious to the usage of `Providers` and `Conv or an individual value (partial configuration) when the `configURI` is embedded into the `Conf` as a values using the syntax `${configURI}`. -**Limitation:** when embed a `${configURI}` the uri cannot contain dollar sign ("$") character. This is to allow the -current implementation to evolve in the future to support embedded uri within uri, e.g. `${http://my.domain.com?os=${OS}}`. +**Limitation:** +- When embedding a `${configURI}` the uri cannot contain dollar sign ("$") character unless it embeds another uri. +- The number of URIs is limited to 100. ```terminal Resolver Provider diff --git a/confmap/expand.go b/confmap/expand.go new file mode 100644 index 0000000000..65730302cc --- /dev/null +++ b/confmap/expand.go @@ -0,0 +1,184 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package confmap // import "go.opentelemetry.io/collector/confmap" + +import ( + "context" + "errors" + "fmt" + "reflect" + "regexp" + "strconv" + "strings" +) + +// schemePattern defines the regexp pattern for scheme names. +// Scheme name consist of a sequence of characters beginning with a letter and followed by any +// combination of letters, digits, plus ("+"), period ("."), or hyphen ("-"). +const schemePattern = `[A-Za-z][A-Za-z0-9+.-]+` + +var ( + // Need to match new line as well in the OpaqueValue, so setting the "s" flag. See https://pkg.go.dev/regexp/syntax. + uriRegexp = regexp.MustCompile(`(?s:^(?P` + schemePattern + `):(?P.*)$)`) + + errTooManyRecursiveExpansions = errors.New("too many recursive expansions") +) + +func (mr *Resolver) expandValueRecursively(ctx context.Context, value any) (any, error) { + for i := 0; i < 100; i++ { + val, changed, err := mr.expandValue(ctx, value) + if err != nil { + return nil, err + } + if !changed { + return val, nil + } + value = val + } + return nil, errTooManyRecursiveExpansions +} + +func (mr *Resolver) expandValue(ctx context.Context, value any) (any, bool, error) { + switch v := value.(type) { + case string: + if !strings.Contains(v, "${") || !strings.Contains(v, "}") { + // No URIs to expand. + return value, false, nil + } + // Embedded or nested URIs. + return mr.findAndExpandURI(ctx, v) + case []any: + nslice := make([]any, 0, len(v)) + nchanged := false + for _, vint := range v { + val, changed, err := mr.expandValue(ctx, vint) + if err != nil { + return nil, false, err + } + nslice = append(nslice, val) + nchanged = nchanged || changed + } + return nslice, nchanged, nil + case map[string]any: + nmap := map[string]any{} + nchanged := false + for mk, mv := range v { + val, changed, err := mr.expandValue(ctx, mv) + if err != nil { + return nil, false, err + } + nmap[mk] = val + nchanged = nchanged || changed + } + return nmap, nchanged, nil + } + return value, false, nil +} + +// findURI attempts to find the first expandable URI in input. It returns an expandable +// URI, or an empty string if none are found. +// Note: findURI is only called when input contains a closing bracket. +func findURI(input string) string { + closeIndex := strings.Index(input, "}") + remaining := input[closeIndex+1:] + openIndex := strings.LastIndex(input[:closeIndex+1], "${") + + // if there is a missing "${" or the uri does not contain ":", check the next URI. + if openIndex < 0 || !strings.Contains(input[openIndex:closeIndex+1], ":") { + // if remaining does not contain "}", there are no URIs left: stop recursion. + if !strings.Contains(remaining, "}") { + return "" + } + return findURI(remaining) + } + + return input[openIndex : closeIndex+1] +} + +// findAndExpandURI attempts to find and expand the first occurrence of an expandable URI in input. If an expandable URI is found it +// returns the input with the URI expanded, true and nil. Otherwise, it returns the unchanged input, false and the expanding error. +func (mr *Resolver) findAndExpandURI(ctx context.Context, input string) (any, bool, error) { + uri := findURI(input) + if uri == "" { + // No URI found, return. + return input, false, nil + } + if uri == input { + // If the value is a single URI, then the return value can be anything. + // This is the case `foo: ${file:some_extra_config.yml}`. + return mr.expandURI(ctx, input) + } + expanded, changed, err := mr.expandURI(ctx, uri) + if err != nil { + return input, false, err + } + repl, err := toString(uri, expanded) + if err != nil { + return input, false, err + } + return strings.ReplaceAll(input, uri, repl), changed, err +} + +// toString attempts to convert input to a string. +func toString(strURI string, input any) (string, error) { + // This list must be kept in sync with checkRawConfType. + val := reflect.ValueOf(input) + switch val.Kind() { + case reflect.String: + return val.String(), nil + case reflect.Int, reflect.Int32, reflect.Int64: + return strconv.FormatInt(val.Int(), 10), nil + case reflect.Float32, reflect.Float64: + return strconv.FormatFloat(val.Float(), 'f', -1, 64), nil + case reflect.Bool: + return strconv.FormatBool(val.Bool()), nil + default: + return "", fmt.Errorf("expanding %v, expected convertable to string value type, got %q(%T)", strURI, input, input) + } +} + +func (mr *Resolver) expandURI(ctx context.Context, uri string) (any, bool, error) { + lURI, err := newLocation(uri[2 : len(uri)-1]) + if err != nil { + return nil, false, err + } + if strings.Contains(lURI.opaqueValue, "$") { + return nil, false, fmt.Errorf("the uri %q contains unsupported characters ('$')", lURI.asString()) + } + ret, err := mr.retrieveValue(ctx, lURI) + if err != nil { + return nil, false, err + } + mr.closers = append(mr.closers, ret.Close) + val, err := ret.AsRaw() + return val, true, err +} + +type location struct { + scheme string + opaqueValue string +} + +func (c location) asString() string { + return c.scheme + ":" + c.opaqueValue +} + +func newLocation(uri string) (location, error) { + submatches := uriRegexp.FindStringSubmatch(uri) + if len(submatches) != 3 { + return location{}, fmt.Errorf("invalid uri: %q", uri) + } + return location{scheme: submatches[1], opaqueValue: submatches[2]}, nil +} diff --git a/confmap/expand_test.go b/confmap/expand_test.go new file mode 100644 index 0000000000..8c6aa5e597 --- /dev/null +++ b/confmap/expand_test.go @@ -0,0 +1,472 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package confmap // import "go.opentelemetry.io/collector/confmap" + +import ( + "context" + "errors" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolverExpandEnvVars(t *testing.T) { + var testCases = []struct { + name string // test case name (also file name containing config yaml) + }{ + {name: "expand-with-no-env.yaml"}, + {name: "expand-with-partial-env.yaml"}, + {name: "expand-with-all-env.yaml"}, + } + + envs := map[string]string{ + "EXTRA": "some string", + "EXTRA_MAP_VALUE_1": "some map value_1", + "EXTRA_MAP_VALUE_2": "some map value_2", + "EXTRA_LIST_MAP_VALUE_1": "some list map value_1", + "EXTRA_LIST_MAP_VALUE_2": "some list map value_2", + "EXTRA_LIST_VALUE_1": "some list value_1", + "EXTRA_LIST_VALUE_2": "some list value_2", + } + + expectedCfgMap := newConfFromFile(t, filepath.Join("testdata", "expand-with-no-env.yaml")) + fileProvider := newFakeProvider("file", func(_ context.Context, uri string, _ WatcherFunc) (*Retrieved, error) { + return NewRetrieved(newConfFromFile(t, uri[5:])) + }) + envProvider := newFakeProvider("env", func(_ context.Context, uri string, _ WatcherFunc) (*Retrieved, error) { + return NewRetrieved(envs[uri[4:]]) + }) + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + resolver, err := NewResolver(ResolverSettings{URIs: []string{filepath.Join("testdata", test.name)}, Providers: makeMapProvidersMap(fileProvider, envProvider), Converters: nil}) + require.NoError(t, err) + + // Test that expanded configs are the same with the simple config with no env vars. + cfgMap, err := resolver.Resolve(context.Background()) + require.NoError(t, err) + assert.Equal(t, expectedCfgMap, cfgMap.ToStringMap()) + }) + } +} + +func TestResolverDoneNotExpandOldEnvVars(t *testing.T) { + expectedCfgMap := map[string]any{"test.1": "${EXTRA}", "test.2": "$EXTRA", "test.3": "${EXTRA}:${EXTRA}"} + fileProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) { + return NewRetrieved(expectedCfgMap) + }) + envProvider := newFakeProvider("env", func(context.Context, string, WatcherFunc) (*Retrieved, error) { + return NewRetrieved("some string") + }) + emptySchemeProvider := newFakeProvider("", func(context.Context, string, WatcherFunc) (*Retrieved, error) { + return NewRetrieved("some string") + }) + + resolver, err := NewResolver(ResolverSettings{URIs: []string{"test:"}, Providers: makeMapProvidersMap(fileProvider, envProvider, emptySchemeProvider), Converters: nil}) + require.NoError(t, err) + + // Test that expanded configs are the same with the simple config with no env vars. + cfgMap, err := resolver.Resolve(context.Background()) + require.NoError(t, err) + assert.Equal(t, expectedCfgMap, cfgMap.ToStringMap()) +} + +func TestResolverExpandMapAndSliceValues(t *testing.T) { + provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) { + return NewRetrieved(map[string]any{ + "test_map": map[string]any{"recv": "${test:MAP_VALUE}"}, + "test_slice": []any{"${test:MAP_VALUE}"}}) + }) + + const receiverExtraMapValue = "some map value" + testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) { + return NewRetrieved(receiverExtraMapValue) + }) + + resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil}) + require.NoError(t, err) + + cfgMap, err := resolver.Resolve(context.Background()) + require.NoError(t, err) + expectedMap := map[string]any{ + "test_map": map[string]any{"recv": receiverExtraMapValue}, + "test_slice": []any{receiverExtraMapValue}} + assert.Equal(t, expectedMap, cfgMap.ToStringMap()) +} + +func TestResolverExpandStringValues(t *testing.T) { + tests := []struct { + name string + input string + output any + }{ + // Embedded. + { + name: "NoMatchOldStyle", + input: "${HOST}:${PORT}", + output: "${HOST}:${PORT}", + }, + { + name: "NoMatchOldStyleNoBrackets", + input: "${HOST}:$PORT", + output: "${HOST}:$PORT", + }, + { + name: "ComplexValue", + input: "${env:COMPLEX_VALUE}", + output: []any{"localhost:3042"}, + }, + { + name: "Embedded", + input: "${env:HOST}:3043", + output: "localhost:3043", + }, + { + name: "EmbeddedMulti", + input: "${env:HOST}:${env:PORT}", + output: "localhost:3044", + }, + { + name: "EmbeddedConcat", + input: "https://${env:HOST}:3045", + output: "https://localhost:3045", + }, + { + name: "EmbeddedNewAndOldStyle", + input: "${env:HOST}:${PORT}", + output: "localhost:${PORT}", + }, + { + name: "Int", + input: "test_${env:INT}", + output: "test_1", + }, + { + name: "Int32", + input: "test_${env:INT32}", + output: "test_32", + }, + { + name: "Int64", + input: "test_${env:INT64}", + output: "test_64", + }, + { + name: "Float32", + input: "test_${env:FLOAT32}", + output: "test_3.25", + }, + { + name: "Float64", + input: "test_${env:FLOAT64}", + output: "test_6.4", + }, + { + name: "Bool", + input: "test_${env:BOOL}", + output: "test_true", + }, + { + name: "MultipleSameMatches", + input: "test_${env:BOOL}_test_${env:BOOL}", + output: "test_true_test_true", + }, + + // Nested. + { + name: "Nested", + input: "${test:localhost:${env:PORT}}", + output: "localhost:3044", + }, + { + name: "EmbeddedInNested", + input: "${test:${env:HOST}:${env:PORT}}", + output: "localhost:3044", + }, + { + name: "EmbeddedAndNested", + input: "${test:localhost:${env:PORT}}?os=${env:OS}", + output: "localhost:3044?os=ubuntu", + }, + { + name: "NestedMultiple", + input: "${test:1${test:2${test:3${test:4${test:5${test:6}}}}}}", + output: "123456", + }, + // No expand. + { + name: "NoMatchMissingOpeningBracket", + input: "env:HOST}", + output: "env:HOST}", + }, + { + name: "NoMatchMissingClosingBracket", + input: "${HOST", + output: "${HOST", + }, + { + name: "NoMatchBracketsWithout$", + input: "HO{ST}", + output: "HO{ST}", + }, + { + name: "NoMatchOnlyMissingClosingBracket", + input: "${env:HOST${env:PORT?os=${env:OS", + output: "${env:HOST${env:PORT?os=${env:OS", + }, + { + name: "NoMatchOnlyMissingOpeningBracket", + input: "env:HOST}env:PORT}?os=env:OS}", + output: "env:HOST}env:PORT}?os=env:OS}", + }, + { + name: "NoMatchCloseBeforeOpen", + input: "env:HOST}${env:PORT", + output: "env:HOST}${env:PORT", + }, + { + name: "NoMatchOldStyleNested", + input: "${test:localhost:${PORT}}", + output: "${test:localhost:${PORT}}", + }, + // Partial expand. + { + name: "PartialMatchMissingOpeningBracketFirst", + input: "env:HOST}${env:PORT}", + output: "env:HOST}3044", + }, + { + name: "PartialMatchMissingOpeningBracketLast", + input: "${env:HOST}env:PORT}", + output: "localhostenv:PORT}", + }, + { + name: "PartialMatchMissingClosingBracketFirst", + input: "${env:HOST${env:PORT}", + output: "${env:HOST3044", + }, + { + name: "PartialMatchMissingClosingBracketLast", + input: "${env:HOST}${env:PORT", + output: "localhost${env:PORT", + }, + { + name: "PartialMatchMultipleMissingOpen", + input: "env:HOST}env:PORT}?os=${env:OS}", + output: "env:HOST}env:PORT}?os=ubuntu", + }, + { + name: "PartialMatchMultipleMissingClosing", + input: "${env:HOST${env:PORT?os=${env:OS}", + output: "${env:HOST${env:PORT?os=ubuntu", + }, + { + name: "PartialMatchMoreClosingBrackets", + input: "${env:HOST}}}}}}${env:PORT?os=${env:OS}", + output: "localhost}}}}}${env:PORT?os=ubuntu", + }, + { + name: "PartialMatchMoreOpeningBrackets", + input: "${env:HOST}${${${${${env:PORT}?os=${env:OS}", + output: "localhost${${${${3044?os=ubuntu", + }, + { + name: "PartialMatchAlternatingMissingOpening", + input: "env:HOST}${env:PORT}?os=env:OS}&pr=${env:PR}", + output: "env:HOST}3044?os=env:OS}&pr=amd", + }, + { + name: "PartialMatchAlternatingMissingClosing", + input: "${env:HOST${env:PORT}?os=${env:OS&pr=${env:PR}", + output: "${env:HOST3044?os=${env:OS&pr=amd", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) { + return NewRetrieved(map[string]any{tt.name: tt.input}) + }) + + testProvider := newFakeProvider("test", func(_ context.Context, uri string, _ WatcherFunc) (*Retrieved, error) { + return NewRetrieved(uri[5:]) + }) + + resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, newEnvProvider(), testProvider), Converters: nil}) + require.NoError(t, err) + + cfgMap, err := resolver.Resolve(context.Background()) + require.NoError(t, err) + assert.Equal(t, map[string]any{tt.name: tt.output}, cfgMap.ToStringMap()) + }) + } +} + +func newEnvProvider() Provider { + return newFakeProvider("env", func(_ context.Context, uri string, _ WatcherFunc) (*Retrieved, error) { + switch uri { + case "env:COMPLEX_VALUE": + return NewRetrieved([]any{"localhost:3042"}) + case "env:HOST": + return NewRetrieved("localhost") + case "env:OS": + return NewRetrieved("ubuntu") + case "env:PR": + return NewRetrieved("amd") + case "env:PORT": + return NewRetrieved(3044) + case "env:INT": + return NewRetrieved(1) + case "env:INT32": + return NewRetrieved(32) + case "env:INT64": + return NewRetrieved(64) + case "env:FLOAT32": + return NewRetrieved(float32(3.25)) + case "env:FLOAT64": + return NewRetrieved(float64(6.4)) + case "env:BOOL": + return NewRetrieved(true) + } + return nil, errors.New("impossible") + }) +} + +func TestResolverExpandReturnError(t *testing.T) { + tests := []struct { + name string + input any + }{ + { + name: "string_value", + input: "${test:VALUE}", + }, + { + name: "slice_value", + input: []any{"${test:VALUE}"}, + }, + { + name: "map_value", + input: map[string]any{"test": "${test:VALUE}"}, + }, + { + name: "string_embedded_value", + input: "https://${test:HOST}:3045", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) { + return NewRetrieved(map[string]any{tt.name: tt.input}) + }) + + myErr := errors.New(tt.name) + testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) { + return nil, myErr + }) + + resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil}) + require.NoError(t, err) + + _, err = resolver.Resolve(context.Background()) + assert.ErrorIs(t, err, myErr) + }) + } +} + +func TestResolverInfiniteExpand(t *testing.T) { + const receiverValue = "${test:VALUE}" + provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) { + return NewRetrieved(map[string]any{"test": receiverValue}) + }) + + testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) { + return NewRetrieved(receiverValue) + }) + + resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil}) + require.NoError(t, err) + + _, err = resolver.Resolve(context.Background()) + assert.ErrorIs(t, err, errTooManyRecursiveExpansions) +} + +func TestResolverExpandInvalidScheme(t *testing.T) { + provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) { + return NewRetrieved(map[string]any{"test": "${g_c_s:VALUE}"}) + }) + + testProvider := newFakeProvider("g_c_s", func(context.Context, string, WatcherFunc) (*Retrieved, error) { + panic("must not be called") + }) + + resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil}) + require.NoError(t, err) + + _, err = resolver.Resolve(context.Background()) + + assert.EqualError(t, err, `invalid uri: "g_c_s:VALUE"`) +} + +func TestResolverExpandInvalidOpaqueValue(t *testing.T) { + provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) { + return NewRetrieved(map[string]any{"test": []any{map[string]any{"test": "${test:$VALUE}"}}}) + }) + + testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) { + panic("must not be called") + }) + + resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil}) + require.NoError(t, err) + + _, err = resolver.Resolve(context.Background()) + assert.EqualError(t, err, `the uri "test:$VALUE" contains unsupported characters ('$')`) +} + +func TestResolverExpandUnsupportedScheme(t *testing.T) { + provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) { + return NewRetrieved(map[string]any{"test": "${unsupported:VALUE}"}) + }) + + testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) { + panic("must not be called") + }) + + resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil}) + require.NoError(t, err) + + _, err = resolver.Resolve(context.Background()) + assert.EqualError(t, err, `scheme "unsupported" is not supported for uri "unsupported:VALUE"`) +} + +func TestResolverExpandStringValueInvalidReturnValue(t *testing.T) { + provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) { + return NewRetrieved(map[string]any{"test": "localhost:${test:PORT}"}) + }) + + testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) { + return NewRetrieved([]any{1243}) + }) + + resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil}) + require.NoError(t, err) + + _, err = resolver.Resolve(context.Background()) + assert.EqualError(t, err, `expanding ${test:PORT}, expected convertable to string value type, got ['ӛ']([]interface {})`) +} diff --git a/confmap/resolver.go b/confmap/resolver.go index b6755abfc2..5a2ce32ff6 100644 --- a/confmap/resolver.go +++ b/confmap/resolver.go @@ -18,9 +18,7 @@ import ( "context" "errors" "fmt" - "reflect" "regexp" - "strconv" "strings" "go.uber.org/multierr" @@ -28,24 +26,9 @@ import ( "go.opentelemetry.io/collector/featuregate" ) -// schemePattern defines the regexp pattern for scheme names. -// Scheme name consist of a sequence of characters beginning with a letter and followed by any -// combination of letters, digits, plus ("+"), period ("."), or hyphen ("-"). -const schemePattern = `[A-Za-z][A-Za-z0-9+.-]+` - -var ( - // follows drive-letter specification: - // https://datatracker.ietf.org/doc/html/draft-kerwin-file-scheme-07.html#section-2.2 - driverLetterRegexp = regexp.MustCompile("^[A-z]:") - - // Need to match new line as well in the OpaqueValue, so setting the "s" flag. See https://pkg.go.dev/regexp/syntax. - uriRegexp = regexp.MustCompile(`(?s:^(?P` + schemePattern + `):(?P.*)$)`) - - // embeddedURI matches "embedded" provider uris into a string value. - embeddedURI = regexp.MustCompile(`\${` + schemePattern + `:.*?}`) - - errTooManyRecursiveExpansions = errors.New("too many recursive expansions") -) +// follows drive-letter specification: +// https://datatracker.ietf.org/doc/html/draft-kerwin-file-scheme-07.html#section-2.2 +var driverLetterRegexp = regexp.MustCompile("^[A-z]:") var _ = featuregate.GlobalRegistry().MustRegister( "confmap.expandEnabled", @@ -225,123 +208,6 @@ func (mr *Resolver) closeIfNeeded(ctx context.Context) error { return err } -func (mr *Resolver) expandValueRecursively(ctx context.Context, value any) (any, error) { - for i := 0; i < 100; i++ { - val, changed, err := mr.expandValue(ctx, value) - if err != nil { - return nil, err - } - if !changed { - return val, nil - } - value = val - } - return nil, errTooManyRecursiveExpansions -} - -func (mr *Resolver) expandValue(ctx context.Context, value any) (any, bool, error) { - switch v := value.(type) { - case string: - // If no embedded "uris" no need to expand. embeddedURI regexp matches uriRegexp as well. - if !embeddedURI.MatchString(v) { - return value, false, nil - } - - // If the value is a single URI, then the return value can be anything. - // This is the case `foo: ${file:some_extra_config.yml}`. - if embeddedURI.FindString(v) == v { - return mr.expandStringURI(ctx, v) - } - - // If the URI is embedded into the string, return value must be a string, and we have to concatenate all strings. - var nerr error - var nchanged bool - nv := embeddedURI.ReplaceAllStringFunc(v, func(s string) string { - ret, changed, err := mr.expandStringURI(ctx, s) - nchanged = nchanged || changed - nerr = multierr.Append(nerr, err) - if err != nil { - return "" - } - // This list must be kept in sync with checkRawConfType. - val := reflect.ValueOf(ret) - switch val.Kind() { - case reflect.String: - return val.String() - case reflect.Int, reflect.Int32, reflect.Int64: - return strconv.FormatInt(val.Int(), 10) - case reflect.Float32, reflect.Float64: - return strconv.FormatFloat(val.Float(), 'f', -1, 64) - case reflect.Bool: - return strconv.FormatBool(val.Bool()) - default: - nerr = multierr.Append(nerr, fmt.Errorf("expanding %v, expected string value type, got %T", s, ret)) - return v - } - }) - return nv, nchanged, nerr - case []any: - nslice := make([]any, 0, len(v)) - nchanged := false - for _, vint := range v { - val, changed, err := mr.expandValue(ctx, vint) - if err != nil { - return nil, false, err - } - nslice = append(nslice, val) - nchanged = nchanged || changed - } - return nslice, nchanged, nil - case map[string]any: - nmap := map[string]any{} - nchanged := false - for mk, mv := range v { - val, changed, err := mr.expandValue(ctx, mv) - if err != nil { - return nil, false, err - } - nmap[mk] = val - nchanged = nchanged || changed - } - return nmap, nchanged, nil - } - return value, false, nil -} - -func (mr *Resolver) expandStringURI(ctx context.Context, uri string) (any, bool, error) { - lURI, err := newLocation(uri[2 : len(uri)-1]) - if err != nil { - return nil, false, err - } - if strings.Contains(lURI.opaqueValue, "$") { - return nil, false, fmt.Errorf("the uri %q contains unsupported characters ('$')", lURI.asString()) - } - ret, err := mr.retrieveValue(ctx, lURI) - if err != nil { - return nil, false, err - } - mr.closers = append(mr.closers, ret.Close) - val, err := ret.AsRaw() - return val, true, err -} - -type location struct { - scheme string - opaqueValue string -} - -func (c location) asString() string { - return c.scheme + ":" + c.opaqueValue -} - -func newLocation(uri string) (location, error) { - submatches := uriRegexp.FindStringSubmatch(uri) - if len(submatches) != 3 { - return location{}, fmt.Errorf("invalid uri: %q", uri) - } - return location{scheme: submatches[1], opaqueValue: submatches[2]}, nil -} - func (mr *Resolver) retrieveValue(ctx context.Context, uri location) (*Retrieved, error) { p, ok := mr.providers[uri.scheme] if !ok { diff --git a/confmap/resolver_test.go b/confmap/resolver_test.go index 2bd92944b0..f62f90278e 100644 --- a/confmap/resolver_test.go +++ b/confmap/resolver_test.go @@ -96,6 +96,14 @@ func (m *mockConverter) Convert(context.Context, *Conf) error { return errors.New("converter_err") } +func makeMapProvidersMap(providers ...Provider) map[string]Provider { + ret := make(map[string]Provider, len(providers)) + for _, provider := range providers { + ret[provider.Scheme()] = provider + } + return ret +} + func TestNewResolverInvalidScheme(t *testing.T) { _, err := NewResolver(ResolverSettings{URIs: []string{"s_3:has invalid char"}, Providers: makeMapProvidersMap(&mockProvider{scheme: "s_3"})}) assert.EqualError(t, err, `invalid uri: "s_3:has invalid char"`) @@ -350,333 +358,3 @@ func TestResolverShutdownClosesWatch(t *testing.T) { assert.NoError(t, resolver.Shutdown(context.Background())) watcherWG.Wait() } - -func TestResolverExpandEnvVars(t *testing.T) { - var testCases = []struct { - name string // test case name (also file name containing config yaml) - }{ - {name: "expand-with-no-env.yaml"}, - {name: "expand-with-partial-env.yaml"}, - {name: "expand-with-all-env.yaml"}, - } - - envs := map[string]string{ - "EXTRA": "some string", - "EXTRA_MAP_VALUE_1": "some map value_1", - "EXTRA_MAP_VALUE_2": "some map value_2", - "EXTRA_LIST_MAP_VALUE_1": "some list map value_1", - "EXTRA_LIST_MAP_VALUE_2": "some list map value_2", - "EXTRA_LIST_VALUE_1": "some list value_1", - "EXTRA_LIST_VALUE_2": "some list value_2", - } - - expectedCfgMap := newConfFromFile(t, filepath.Join("testdata", "expand-with-no-env.yaml")) - fileProvider := newFakeProvider("file", func(_ context.Context, uri string, _ WatcherFunc) (*Retrieved, error) { - return NewRetrieved(newConfFromFile(t, uri[5:])) - }) - envProvider := newFakeProvider("env", func(_ context.Context, uri string, _ WatcherFunc) (*Retrieved, error) { - return NewRetrieved(envs[uri[4:]]) - }) - - for _, test := range testCases { - t.Run(test.name, func(t *testing.T) { - resolver, err := NewResolver(ResolverSettings{URIs: []string{filepath.Join("testdata", test.name)}, Providers: makeMapProvidersMap(fileProvider, envProvider), Converters: nil}) - require.NoError(t, err) - - // Test that expanded configs are the same with the simple config with no env vars. - cfgMap, err := resolver.Resolve(context.Background()) - require.NoError(t, err) - assert.Equal(t, expectedCfgMap, cfgMap.ToStringMap()) - }) - } -} - -func TestResolverDoneNotExpandOldEnvVars(t *testing.T) { - expectedCfgMap := map[string]any{"test.1": "${EXTRA}", "test.2": "$EXTRA", "test.3": "${EXTRA}:${EXTRA}"} - fileProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) { - return NewRetrieved(expectedCfgMap) - }) - envProvider := newFakeProvider("env", func(context.Context, string, WatcherFunc) (*Retrieved, error) { - return NewRetrieved("some string") - }) - emptySchemeProvider := newFakeProvider("", func(context.Context, string, WatcherFunc) (*Retrieved, error) { - return NewRetrieved("some string") - }) - - resolver, err := NewResolver(ResolverSettings{URIs: []string{"test:"}, Providers: makeMapProvidersMap(fileProvider, envProvider, emptySchemeProvider), Converters: nil}) - require.NoError(t, err) - - // Test that expanded configs are the same with the simple config with no env vars. - cfgMap, err := resolver.Resolve(context.Background()) - require.NoError(t, err) - assert.Equal(t, expectedCfgMap, cfgMap.ToStringMap()) -} - -func TestResolverExpandMapAndSliceValues(t *testing.T) { - provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) { - return NewRetrieved(map[string]any{ - "test_map": map[string]any{"recv": "${test:MAP_VALUE}"}, - "test_slice": []any{"${test:MAP_VALUE}"}}) - }) - - const receiverExtraMapValue = "some map value" - testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) { - return NewRetrieved(receiverExtraMapValue) - }) - - resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil}) - require.NoError(t, err) - - cfgMap, err := resolver.Resolve(context.Background()) - require.NoError(t, err) - expectedMap := map[string]any{ - "test_map": map[string]any{"recv": receiverExtraMapValue}, - "test_slice": []any{receiverExtraMapValue}} - assert.Equal(t, expectedMap, cfgMap.ToStringMap()) -} - -func TestResolverExpandStringValues(t *testing.T) { - tests := []struct { - name string - input string - output any - }{ - { - name: "test_no_match_old", - input: "${HOST}:${PORT}", - output: "${HOST}:${PORT}", - }, - { - name: "test_no_match_old_no_brackets", - input: "${HOST}:$PORT", - output: "${HOST}:$PORT", - }, - { - name: "test_match_value", - input: "${env:COMPLEX_VALUE}", - output: []any{"localhost:3042"}, - }, - { - name: "test_match_embedded", - input: "${env:HOST}:3043", - output: "localhost:3043", - }, - { - name: "test_match_embedded_multi", - input: "${env:HOST}:${env:PORT}", - output: "localhost:3044", - }, - { - name: "test_match_embedded_concat", - input: "https://${env:HOST}:3045", - output: "https://localhost:3045", - }, - { - name: "test_match_embedded_new_and_old", - input: "${env:HOST}:${PORT}", - output: "localhost:${PORT}", - }, - { - name: "test_match_int", - input: "test_${env:INT}", - output: "test_1", - }, - { - name: "test_match_int32", - input: "test_${env:INT32}", - output: "test_32", - }, - { - name: "test_match_int64", - input: "test_${env:INT64}", - output: "test_64", - }, - { - name: "test_match_float32", - input: "test_${env:FLOAT32}", - output: "test_3.25", - }, - { - name: "test_match_float64", - input: "test_${env:FLOAT64}", - output: "test_6.4", - }, - { - name: "test_match_bool", - input: "test_${env:BOOL}", - output: "test_true", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) { - return NewRetrieved(map[string]any{tt.name: tt.input}) - }) - - testProvider := newFakeProvider("env", func(_ context.Context, uri string, _ WatcherFunc) (*Retrieved, error) { - switch uri { - case "env:COMPLEX_VALUE": - return NewRetrieved([]any{"localhost:3042"}) - case "env:HOST": - return NewRetrieved("localhost") - case "env:PORT": - return NewRetrieved(3044) - case "env:INT": - return NewRetrieved(1) - case "env:INT32": - return NewRetrieved(32) - case "env:INT64": - return NewRetrieved(64) - case "env:FLOAT32": - return NewRetrieved(float32(3.25)) - case "env:FLOAT64": - return NewRetrieved(float64(6.4)) - case "env:BOOL": - return NewRetrieved(true) - } - return nil, errors.New("impossible") - }) - - resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil}) - require.NoError(t, err) - - cfgMap, err := resolver.Resolve(context.Background()) - require.NoError(t, err) - assert.Equal(t, map[string]any{tt.name: tt.output}, cfgMap.ToStringMap()) - }) - } -} - -func TestResolverExpandReturnError(t *testing.T) { - tests := []struct { - name string - input any - }{ - { - name: "string_value", - input: "${test:VALUE}", - }, - { - name: "slice_value", - input: []any{"${test:VALUE}"}, - }, - { - name: "map_value", - input: map[string]any{"test": "${test:VALUE}"}, - }, - { - name: "string_embedded_value", - input: "https://${test:HOST}:3045", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) { - return NewRetrieved(map[string]any{tt.name: tt.input}) - }) - - myErr := errors.New(tt.name) - testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) { - return nil, myErr - }) - - resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil}) - require.NoError(t, err) - - _, err = resolver.Resolve(context.Background()) - assert.ErrorIs(t, err, myErr) - }) - } -} -func TestResolverInfiniteExpand(t *testing.T) { - const receiverValue = "${test:VALUE}" - provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) { - return NewRetrieved(map[string]any{"test": receiverValue}) - }) - - testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) { - return NewRetrieved(receiverValue) - }) - - resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil}) - require.NoError(t, err) - - _, err = resolver.Resolve(context.Background()) - assert.ErrorIs(t, err, errTooManyRecursiveExpansions) -} - -func TestResolverExpandInvalidScheme(t *testing.T) { - provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) { - return NewRetrieved(map[string]any{"test": "${g_c_s:VALUE}"}) - }) - - testProvider := newFakeProvider("g_c_s", func(context.Context, string, WatcherFunc) (*Retrieved, error) { - panic("must not be called") - }) - - resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil}) - require.NoError(t, err) - - _, err = resolver.Resolve(context.Background()) - // TODO: Investigate how to return an error like `invalid uri: "g_c_s:VALUE"` and keep the legacy behavior - // for cases like "${HOST}:${PORT}" - assert.NoError(t, err) -} - -func TestResolverExpandInvalidOpaqueValue(t *testing.T) { - provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) { - return NewRetrieved(map[string]any{"test": []any{map[string]any{"test": "${test:$VALUE}"}}}) - }) - - testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) { - panic("must not be called") - }) - - resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil}) - require.NoError(t, err) - - _, err = resolver.Resolve(context.Background()) - assert.EqualError(t, err, `the uri "test:$VALUE" contains unsupported characters ('$')`) -} - -func TestResolverExpandUnsupportedScheme(t *testing.T) { - provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) { - return NewRetrieved(map[string]any{"test": "${unsupported:VALUE}"}) - }) - - testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) { - panic("must not be called") - }) - - resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil}) - require.NoError(t, err) - - _, err = resolver.Resolve(context.Background()) - assert.EqualError(t, err, `scheme "unsupported" is not supported for uri "unsupported:VALUE"`) -} - -func TestResolverExpandStringValueInvalidReturnValue(t *testing.T) { - provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) { - return NewRetrieved(map[string]any{"test": "localhost:${test:PORT}"}) - }) - - testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) { - return NewRetrieved([]any{1243}) - }) - - resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil}) - require.NoError(t, err) - - _, err = resolver.Resolve(context.Background()) - assert.EqualError(t, err, `expanding ${test:PORT}, expected string value type, got []interface {}`) -} - -func makeMapProvidersMap(providers ...Provider) map[string]Provider { - ret := make(map[string]Provider, len(providers)) - for _, provider := range providers { - ret[provider.Scheme()] = provider - } - return ret -}