open-telemetry
/
opentelemetry-collector
mirror of https://github.com/open-telemetry/opentelemetry-collector.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

confmap: Add support for nested URIs (#7504) 

* confmap: Add support for nested URIs

Addresses #7117.

This PR adds support for nesting URIs, e.g.
```
test: "${http://example.com/?os=${env:OS}}"
```
**Breaking change:**
In the case of an InvalidScheme, error `invalid uri: "g_c_s:VALUE"` will be thrown. Previously, no error was thrown (see [test case](https://github.com/open-telemetry/opentelemetry-collector/blob/v0.74.0/confmap/resolver_test.go#L623-L625)).

Other than the above, there will be no breaking changes. Although the parser now provides the ability to understand why parsing a URI fails (e.g. missing opening `${`), no errors will be thrown to preserve the previous behaviour.

The old way to expand env vars (e.g. `${HOST}`) is not supported in nested URIs, as expanding the old way is done in a converter. This has been documented.

* add issue # to changelog

* address feedback

* address feedback

* address feedback

* Address feedback: change expandStringURI & preserve comment

* Address feedback: move recursion to findURI

* Address feedback: avoid duplicate calls to findURI

* Update confmap/expand.go

Co-authored-by: Bogdan Drutu <lazy@splunk.com>

* Address Feedback

* remove unused errURILimit

* update findURI documentation

* fix test

* Update confmap/expand.go

* Address feedback: remove named return values

---------

Co-authored-by: Bogdan Drutu <lazy@splunk.com>
This commit is contained in:
Mackenzie 2023-04-15 21:47:19 +02:00 committed by GitHub
parent cc471de9bd
commit fe508cfe83
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 702 additions and 469 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.chloggen/support-nested-URIs-breaking.yaml Normal file
View File

@ -0,0 +1,16 @@
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
change_type: breaking
# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
component: confmap
# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
note: Using an Invalid Scheme in a URI will throw an error.
# One or more tracking issues or pull requests related to the change
issues: [7504]
# (Optional) One or more lines of additional information to render under the primary note.
# These lines will be padded with 2 spaces and then inserted directly into the document.
# Use pipe (|) for multiline entries.
subtext:

16
.chloggen/support-nested-URIs.yaml Executable file
View File

@ -0,0 +1,16 @@
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
change_type: enhancement
# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
component: confmap
# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
note: Add support for nested URIs.
# One or more tracking issues or pull requests related to the change
issues: [7117]
# (Optional) One or more lines of additional information to render under the primary note.
# These lines will be padded with 2 spaces and then inserted directly into the document.
# Use pipe (|) for multiline entries.
subtext:

5
confmap/README.md
View File

@ -37,8 +37,9 @@ that can be used by code that is oblivious to the usage of `Providers` and `Conv
or an individual value (partial configuration) when the `configURI` is embedded into the `Conf` as a values using
the syntax `${configURI}`.
**Limitation:** when embed a `${configURI}` the uri cannot contain dollar sign ("$") character. This is to allow the
current implementation to evolve in the future to support embedded uri within uri, e.g. `${http://my.domain.com?os=${OS}}`.
**Limitation:**
- When embedding a `${configURI}` the uri cannot contain dollar sign ("$") character unless it embeds another uri.
- The number of URIs is limited to 100.
```terminal
Resolver Provider

184
confmap/expand.go Normal file
View File

@ -0,0 +1,184 @@
// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package confmap // import "go.opentelemetry.io/collector/confmap"
import (
"context"
"errors"
"fmt"
"reflect"
"regexp"
"strconv"
"strings"
)
// schemePattern defines the regexp pattern for scheme names.
// Scheme name consist of a sequence of characters beginning with a letter and followed by any
// combination of letters, digits, plus ("+"), period ("."), or hyphen ("-").
const schemePattern = `[A-Za-z][A-Za-z0-9+.-]+`
var (
// Need to match new line as well in the OpaqueValue, so setting the "s" flag. See https://pkg.go.dev/regexp/syntax.
uriRegexp = regexp.MustCompile(`(?s:^(?P<Scheme>` + schemePattern + `):(?P<OpaqueValue>.*)$)`)
errTooManyRecursiveExpansions = errors.New("too many recursive expansions")
)
func (mr *Resolver) expandValueRecursively(ctx context.Context, value any) (any, error) {
for i := 0; i < 100; i++ {
val, changed, err := mr.expandValue(ctx, value)
if err != nil {
return nil, err
}
if !changed {
return val, nil
}
value = val
}
return nil, errTooManyRecursiveExpansions
}
func (mr *Resolver) expandValue(ctx context.Context, value any) (any, bool, error) {
switch v := value.(type) {
case string:
if !strings.Contains(v, "${") || !strings.Contains(v, "}") {
// No URIs to expand.
return value, false, nil
}
// Embedded or nested URIs.
return mr.findAndExpandURI(ctx, v)
case []any:
nslice := make([]any, 0, len(v))
nchanged := false
for _, vint := range v {
val, changed, err := mr.expandValue(ctx, vint)
if err != nil {
return nil, false, err
}
nslice = append(nslice, val)
nchanged = nchanged || changed
}
return nslice, nchanged, nil
case map[string]any:
nmap := map[string]any{}
nchanged := false
for mk, mv := range v {
val, changed, err := mr.expandValue(ctx, mv)
if err != nil {
return nil, false, err
}
nmap[mk] = val
nchanged = nchanged || changed
}
return nmap, nchanged, nil
}
return value, false, nil
}
// findURI attempts to find the first expandable URI in input. It returns an expandable
// URI, or an empty string if none are found.
// Note: findURI is only called when input contains a closing bracket.
func findURI(input string) string {
closeIndex := strings.Index(input, "}")
remaining := input[closeIndex+1:]
openIndex := strings.LastIndex(input[:closeIndex+1], "${")
// if there is a missing "${" or the uri does not contain ":", check the next URI.
if openIndex < 0 || !strings.Contains(input[openIndex:closeIndex+1], ":") {
// if remaining does not contain "}", there are no URIs left: stop recursion.
if !strings.Contains(remaining, "}") {
return ""
}
return findURI(remaining)
}
return input[openIndex : closeIndex+1]
}
// findAndExpandURI attempts to find and expand the first occurrence of an expandable URI in input. If an expandable URI is found it
// returns the input with the URI expanded, true and nil. Otherwise, it returns the unchanged input, false and the expanding error.
func (mr *Resolver) findAndExpandURI(ctx context.Context, input string) (any, bool, error) {
uri := findURI(input)
if uri == "" {
// No URI found, return.
return input, false, nil
}
if uri == input {
// If the value is a single URI, then the return value can be anything.
// This is the case `foo: ${file:some_extra_config.yml}`.
return mr.expandURI(ctx, input)
}
expanded, changed, err := mr.expandURI(ctx, uri)
if err != nil {
return input, false, err
}
repl, err := toString(uri, expanded)
if err != nil {
return input, false, err
}
return strings.ReplaceAll(input, uri, repl), changed, err
}
// toString attempts to convert input to a string.
func toString(strURI string, input any) (string, error) {
// This list must be kept in sync with checkRawConfType.
val := reflect.ValueOf(input)
switch val.Kind() {
case reflect.String:
return val.String(), nil
case reflect.Int, reflect.Int32, reflect.Int64:
return strconv.FormatInt(val.Int(), 10), nil
case reflect.Float32, reflect.Float64:
return strconv.FormatFloat(val.Float(), 'f', -1, 64), nil
case reflect.Bool:
return strconv.FormatBool(val.Bool()), nil
default:
return "", fmt.Errorf("expanding %v, expected convertable to string value type, got %q(%T)", strURI, input, input)
}
}
func (mr *Resolver) expandURI(ctx context.Context, uri string) (any, bool, error) {
lURI, err := newLocation(uri[2 : len(uri)-1])
if err != nil {
return nil, false, err
}
if strings.Contains(lURI.opaqueValue, "$") {
return nil, false, fmt.Errorf("the uri %q contains unsupported characters ('$')", lURI.asString())
}
ret, err := mr.retrieveValue(ctx, lURI)
if err != nil {
return nil, false, err
}
mr.closers = append(mr.closers, ret.Close)
val, err := ret.AsRaw()
return val, true, err
}
type location struct {
scheme string
opaqueValue string
}
func (c location) asString() string {
return c.scheme + ":" + c.opaqueValue
}
func newLocation(uri string) (location, error) {
submatches := uriRegexp.FindStringSubmatch(uri)
if len(submatches) != 3 {
return location{}, fmt.Errorf("invalid uri: %q", uri)
}
return location{scheme: submatches[1], opaqueValue: submatches[2]}, nil
}

472
confmap/expand_test.go Normal file
View File

@ -0,0 +1,472 @@
// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package confmap // import "go.opentelemetry.io/collector/confmap"
import (
"context"
"errors"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestResolverExpandEnvVars(t *testing.T) {
var testCases = []struct {
name string // test case name (also file name containing config yaml)
}{
{name: "expand-with-no-env.yaml"},
{name: "expand-with-partial-env.yaml"},
{name: "expand-with-all-env.yaml"},
}
envs := map[string]string{
"EXTRA": "some string",
"EXTRA_MAP_VALUE_1": "some map value_1",
"EXTRA_MAP_VALUE_2": "some map value_2",
"EXTRA_LIST_MAP_VALUE_1": "some list map value_1",
"EXTRA_LIST_MAP_VALUE_2": "some list map value_2",
"EXTRA_LIST_VALUE_1": "some list value_1",
"EXTRA_LIST_VALUE_2": "some list value_2",
}
expectedCfgMap := newConfFromFile(t, filepath.Join("testdata", "expand-with-no-env.yaml"))
fileProvider := newFakeProvider("file", func(_ context.Context, uri string, _ WatcherFunc) (*Retrieved, error) {
return NewRetrieved(newConfFromFile(t, uri[5:]))
})
envProvider := newFakeProvider("env", func(_ context.Context, uri string, _ WatcherFunc) (*Retrieved, error) {
return NewRetrieved(envs[uri[4:]])
})
for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
resolver, err := NewResolver(ResolverSettings{URIs: []string{filepath.Join("testdata", test.name)}, Providers: makeMapProvidersMap(fileProvider, envProvider), Converters: nil})
require.NoError(t, err)
// Test that expanded configs are the same with the simple config with no env vars.
cfgMap, err := resolver.Resolve(context.Background())
require.NoError(t, err)
assert.Equal(t, expectedCfgMap, cfgMap.ToStringMap())
})
}
}
func TestResolverDoneNotExpandOldEnvVars(t *testing.T) {
expectedCfgMap := map[string]any{"test.1": "${EXTRA}", "test.2": "$EXTRA", "test.3": "${EXTRA}:${EXTRA}"}
fileProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(expectedCfgMap)
})
envProvider := newFakeProvider("env", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved("some string")
})
emptySchemeProvider := newFakeProvider("", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved("some string")
})
resolver, err := NewResolver(ResolverSettings{URIs: []string{"test:"}, Providers: makeMapProvidersMap(fileProvider, envProvider, emptySchemeProvider), Converters: nil})
require.NoError(t, err)
// Test that expanded configs are the same with the simple config with no env vars.
cfgMap, err := resolver.Resolve(context.Background())
require.NoError(t, err)
assert.Equal(t, expectedCfgMap, cfgMap.ToStringMap())
}
func TestResolverExpandMapAndSliceValues(t *testing.T) {
provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(map[string]any{
"test_map": map[string]any{"recv": "${test:MAP_VALUE}"},
"test_slice": []any{"${test:MAP_VALUE}"}})
})
const receiverExtraMapValue = "some map value"
testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(receiverExtraMapValue)
})
resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil})
require.NoError(t, err)
cfgMap, err := resolver.Resolve(context.Background())
require.NoError(t, err)
expectedMap := map[string]any{
"test_map": map[string]any{"recv": receiverExtraMapValue},
"test_slice": []any{receiverExtraMapValue}}
assert.Equal(t, expectedMap, cfgMap.ToStringMap())
}
func TestResolverExpandStringValues(t *testing.T) {
tests := []struct {
name string
input string
output any
}{
// Embedded.
{
name: "NoMatchOldStyle",
input: "${HOST}:${PORT}",
output: "${HOST}:${PORT}",
},
{
name: "NoMatchOldStyleNoBrackets",
input: "${HOST}:$PORT",
output: "${HOST}:$PORT",
},
{
name: "ComplexValue",
input: "${env:COMPLEX_VALUE}",
output: []any{"localhost:3042"},
},
{
name: "Embedded",
input: "${env:HOST}:3043",
output: "localhost:3043",
},
{
name: "EmbeddedMulti",
input: "${env:HOST}:${env:PORT}",
output: "localhost:3044",
},
{
name: "EmbeddedConcat",
input: "https://${env:HOST}:3045",
output: "https://localhost:3045",
},
{
name: "EmbeddedNewAndOldStyle",
input: "${env:HOST}:${PORT}",
output: "localhost:${PORT}",
},
{
name: "Int",
input: "test_${env:INT}",
output: "test_1",
},
{
name: "Int32",
input: "test_${env:INT32}",
output: "test_32",
},
{
name: "Int64",
input: "test_${env:INT64}",
output: "test_64",
},
{
name: "Float32",
input: "test_${env:FLOAT32}",
output: "test_3.25",
},
{
name: "Float64",
input: "test_${env:FLOAT64}",
output: "test_6.4",
},
{
name: "Bool",
input: "test_${env:BOOL}",
output: "test_true",
},
{
name: "MultipleSameMatches",
input: "test_${env:BOOL}_test_${env:BOOL}",
output: "test_true_test_true",
},
// Nested.
{
name: "Nested",
input: "${test:localhost:${env:PORT}}",
output: "localhost:3044",
},
{
name: "EmbeddedInNested",
input: "${test:${env:HOST}:${env:PORT}}",
output: "localhost:3044",
},
{
name: "EmbeddedAndNested",
input: "${test:localhost:${env:PORT}}?os=${env:OS}",
output: "localhost:3044?os=ubuntu",
},
{
name: "NestedMultiple",
input: "${test:1${test:2${test:3${test:4${test:5${test:6}}}}}}",
output: "123456",
},
// No expand.
{
name: "NoMatchMissingOpeningBracket",
input: "env:HOST}",
output: "env:HOST}",
},
{
name: "NoMatchMissingClosingBracket",
input: "${HOST",
output: "${HOST",
},
{
name: "NoMatchBracketsWithout$",
input: "HO{ST}",
output: "HO{ST}",
},
{
name: "NoMatchOnlyMissingClosingBracket",
input: "${env:HOST${env:PORT?os=${env:OS",
output: "${env:HOST${env:PORT?os=${env:OS",
},
{
name: "NoMatchOnlyMissingOpeningBracket",
input: "env:HOST}env:PORT}?os=env:OS}",
output: "env:HOST}env:PORT}?os=env:OS}",
},
{
name: "NoMatchCloseBeforeOpen",
input: "env:HOST}${env:PORT",
output: "env:HOST}${env:PORT",
},
{
name: "NoMatchOldStyleNested",
input: "${test:localhost:${PORT}}",
output: "${test:localhost:${PORT}}",
},
// Partial expand.
{
name: "PartialMatchMissingOpeningBracketFirst",
input: "env:HOST}${env:PORT}",
output: "env:HOST}3044",
},
{
name: "PartialMatchMissingOpeningBracketLast",
input: "${env:HOST}env:PORT}",
output: "localhostenv:PORT}",
},
{
name: "PartialMatchMissingClosingBracketFirst",
input: "${env:HOST${env:PORT}",
output: "${env:HOST3044",
},
{
name: "PartialMatchMissingClosingBracketLast",
input: "${env:HOST}${env:PORT",
output: "localhost${env:PORT",
},
{
name: "PartialMatchMultipleMissingOpen",
input: "env:HOST}env:PORT}?os=${env:OS}",
output: "env:HOST}env:PORT}?os=ubuntu",
},
{
name: "PartialMatchMultipleMissingClosing",
input: "${env:HOST${env:PORT?os=${env:OS}",
output: "${env:HOST${env:PORT?os=ubuntu",
},
{
name: "PartialMatchMoreClosingBrackets",
input: "${env:HOST}}}}}}${env:PORT?os=${env:OS}",
output: "localhost}}}}}${env:PORT?os=ubuntu",
},
{
name: "PartialMatchMoreOpeningBrackets",
input: "${env:HOST}${${${${${env:PORT}?os=${env:OS}",
output: "localhost${${${${3044?os=ubuntu",
},
{
name: "PartialMatchAlternatingMissingOpening",
input: "env:HOST}${env:PORT}?os=env:OS}&pr=${env:PR}",
output: "env:HOST}3044?os=env:OS}&pr=amd",
},
{
name: "PartialMatchAlternatingMissingClosing",
input: "${env:HOST${env:PORT}?os=${env:OS&pr=${env:PR}",
output: "${env:HOST3044?os=${env:OS&pr=amd",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(map[string]any{tt.name: tt.input})
})
testProvider := newFakeProvider("test", func(_ context.Context, uri string, _ WatcherFunc) (*Retrieved, error) {
return NewRetrieved(uri[5:])
})
resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, newEnvProvider(), testProvider), Converters: nil})
require.NoError(t, err)
cfgMap, err := resolver.Resolve(context.Background())
require.NoError(t, err)
assert.Equal(t, map[string]any{tt.name: tt.output}, cfgMap.ToStringMap())
})
}
}
func newEnvProvider() Provider {
return newFakeProvider("env", func(_ context.Context, uri string, _ WatcherFunc) (*Retrieved, error) {
switch uri {
case "env:COMPLEX_VALUE":
return NewRetrieved([]any{"localhost:3042"})
case "env:HOST":
return NewRetrieved("localhost")
case "env:OS":
return NewRetrieved("ubuntu")
case "env:PR":
return NewRetrieved("amd")
case "env:PORT":
return NewRetrieved(3044)
case "env:INT":
return NewRetrieved(1)
case "env:INT32":
return NewRetrieved(32)
case "env:INT64":
return NewRetrieved(64)
case "env:FLOAT32":
return NewRetrieved(float32(3.25))
case "env:FLOAT64":
return NewRetrieved(float64(6.4))
case "env:BOOL":
return NewRetrieved(true)
}
return nil, errors.New("impossible")
})
}
func TestResolverExpandReturnError(t *testing.T) {
tests := []struct {
name string
input any
}{
{
name: "string_value",
input: "${test:VALUE}",
},
{
name: "slice_value",
input: []any{"${test:VALUE}"},
},
{
name: "map_value",
input: map[string]any{"test": "${test:VALUE}"},
},
{
name: "string_embedded_value",
input: "https://${test:HOST}:3045",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(map[string]any{tt.name: tt.input})
})
myErr := errors.New(tt.name)
testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return nil, myErr
})
resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil})
require.NoError(t, err)
_, err = resolver.Resolve(context.Background())
assert.ErrorIs(t, err, myErr)
})
}
}
func TestResolverInfiniteExpand(t *testing.T) {
const receiverValue = "${test:VALUE}"
provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(map[string]any{"test": receiverValue})
})
testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(receiverValue)
})
resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil})
require.NoError(t, err)
_, err = resolver.Resolve(context.Background())
assert.ErrorIs(t, err, errTooManyRecursiveExpansions)
}
func TestResolverExpandInvalidScheme(t *testing.T) {
provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(map[string]any{"test": "${g_c_s:VALUE}"})
})
testProvider := newFakeProvider("g_c_s", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
panic("must not be called")
})
resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil})
require.NoError(t, err)
_, err = resolver.Resolve(context.Background())
assert.EqualError(t, err, `invalid uri: "g_c_s:VALUE"`)
}
func TestResolverExpandInvalidOpaqueValue(t *testing.T) {
provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(map[string]any{"test": []any{map[string]any{"test": "${test:$VALUE}"}}})
})
testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
panic("must not be called")
})
resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil})
require.NoError(t, err)
_, err = resolver.Resolve(context.Background())
assert.EqualError(t, err, `the uri "test:$VALUE" contains unsupported characters ('$')`)
}
func TestResolverExpandUnsupportedScheme(t *testing.T) {
provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(map[string]any{"test": "${unsupported:VALUE}"})
})
testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
panic("must not be called")
})
resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil})
require.NoError(t, err)
_, err = resolver.Resolve(context.Background())
assert.EqualError(t, err, `scheme "unsupported" is not supported for uri "unsupported:VALUE"`)
}
func TestResolverExpandStringValueInvalidReturnValue(t *testing.T) {
provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(map[string]any{"test": "localhost:${test:PORT}"})
})
testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved([]any{1243})
})
resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil})
require.NoError(t, err)
_, err = resolver.Resolve(context.Background())
assert.EqualError(t, err, `expanding ${test:PORT}, expected convertable to string value type, got ['ӛ']([]interface {})`)
}

140
confmap/resolver.go
View File

@ -18,9 +18,7 @@ import (
"context"
"errors"
"fmt"
"reflect"
"regexp"
"strconv"
"strings"
"go.uber.org/multierr"
@ -28,24 +26,9 @@ import (
"go.opentelemetry.io/collector/featuregate"
)
// schemePattern defines the regexp pattern for scheme names.
// Scheme name consist of a sequence of characters beginning with a letter and followed by any
// combination of letters, digits, plus ("+"), period ("."), or hyphen ("-").
const schemePattern = `[A-Za-z][A-Za-z0-9+.-]+`
var (
// follows drive-letter specification:
// https://datatracker.ietf.org/doc/html/draft-kerwin-file-scheme-07.html#section-2.2
driverLetterRegexp = regexp.MustCompile("^[A-z]:")
// Need to match new line as well in the OpaqueValue, so setting the "s" flag. See https://pkg.go.dev/regexp/syntax.
uriRegexp = regexp.MustCompile(`(?s:^(?P<Scheme>` + schemePattern + `):(?P<OpaqueValue>.*)$)`)
// embeddedURI matches "embedded" provider uris into a string value.
embeddedURI = regexp.MustCompile(`\${` + schemePattern + `:.*?}`)
errTooManyRecursiveExpansions = errors.New("too many recursive expansions")
)
// follows drive-letter specification:
// https://datatracker.ietf.org/doc/html/draft-kerwin-file-scheme-07.html#section-2.2
var driverLetterRegexp = regexp.MustCompile("^[A-z]:")
var _ = featuregate.GlobalRegistry().MustRegister(
"confmap.expandEnabled",
@ -225,123 +208,6 @@ func (mr *Resolver) closeIfNeeded(ctx context.Context) error {
return err
}
func (mr *Resolver) expandValueRecursively(ctx context.Context, value any) (any, error) {
for i := 0; i < 100; i++ {
val, changed, err := mr.expandValue(ctx, value)
if err != nil {
return nil, err
}
if !changed {
return val, nil
}
value = val
}
return nil, errTooManyRecursiveExpansions
}
func (mr *Resolver) expandValue(ctx context.Context, value any) (any, bool, error) {
switch v := value.(type) {
case string:
// If no embedded "uris" no need to expand. embeddedURI regexp matches uriRegexp as well.
if !embeddedURI.MatchString(v) {
return value, false, nil
}
// If the value is a single URI, then the return value can be anything.
// This is the case `foo: ${file:some_extra_config.yml}`.
if embeddedURI.FindString(v) == v {
return mr.expandStringURI(ctx, v)
}
// If the URI is embedded into the string, return value must be a string, and we have to concatenate all strings.
var nerr error
var nchanged bool
nv := embeddedURI.ReplaceAllStringFunc(v, func(s string) string {
ret, changed, err := mr.expandStringURI(ctx, s)
nchanged = nchanged || changed
nerr = multierr.Append(nerr, err)
if err != nil {
return ""
}
// This list must be kept in sync with checkRawConfType.
val := reflect.ValueOf(ret)
switch val.Kind() {
case reflect.String:
return val.String()
case reflect.Int, reflect.Int32, reflect.Int64:
return strconv.FormatInt(val.Int(), 10)
case reflect.Float32, reflect.Float64:
return strconv.FormatFloat(val.Float(), 'f', -1, 64)
case reflect.Bool:
return strconv.FormatBool(val.Bool())
default:
nerr = multierr.Append(nerr, fmt.Errorf("expanding %v, expected string value type, got %T", s, ret))
return v
}
})
return nv, nchanged, nerr
case []any:
nslice := make([]any, 0, len(v))
nchanged := false
for _, vint := range v {
val, changed, err := mr.expandValue(ctx, vint)
if err != nil {
return nil, false, err
}
nslice = append(nslice, val)
nchanged = nchanged || changed
}
return nslice, nchanged, nil
case map[string]any:
nmap := map[string]any{}
nchanged := false
for mk, mv := range v {
val, changed, err := mr.expandValue(ctx, mv)
if err != nil {
return nil, false, err
}
nmap[mk] = val
nchanged = nchanged || changed
}
return nmap, nchanged, nil
}
return value, false, nil
}
func (mr *Resolver) expandStringURI(ctx context.Context, uri string) (any, bool, error) {
lURI, err := newLocation(uri[2 : len(uri)-1])
if err != nil {
return nil, false, err
}
if strings.Contains(lURI.opaqueValue, "$") {
return nil, false, fmt.Errorf("the uri %q contains unsupported characters ('$')", lURI.asString())
}
ret, err := mr.retrieveValue(ctx, lURI)
if err != nil {
return nil, false, err
}
mr.closers = append(mr.closers, ret.Close)
val, err := ret.AsRaw()
return val, true, err
}
type location struct {
scheme string
opaqueValue string
}
func (c location) asString() string {
return c.scheme + ":" + c.opaqueValue
}
func newLocation(uri string) (location, error) {
submatches := uriRegexp.FindStringSubmatch(uri)
if len(submatches) != 3 {
return location{}, fmt.Errorf("invalid uri: %q", uri)
}
return location{scheme: submatches[1], opaqueValue: submatches[2]}, nil
}
func (mr *Resolver) retrieveValue(ctx context.Context, uri location) (*Retrieved, error) {
p, ok := mr.providers[uri.scheme]
if !ok {

338
confmap/resolver_test.go
View File

@ -96,6 +96,14 @@ func (m *mockConverter) Convert(context.Context, *Conf) error {
return errors.New("converter_err")
}
func makeMapProvidersMap(providers ...Provider) map[string]Provider {
ret := make(map[string]Provider, len(providers))
for _, provider := range providers {
ret[provider.Scheme()] = provider
}
return ret
}
func TestNewResolverInvalidScheme(t *testing.T) {
_, err := NewResolver(ResolverSettings{URIs: []string{"s_3:has invalid char"}, Providers: makeMapProvidersMap(&mockProvider{scheme: "s_3"})})
assert.EqualError(t, err, `invalid uri: "s_3:has invalid char"`)
@ -350,333 +358,3 @@ func TestResolverShutdownClosesWatch(t *testing.T) {
assert.NoError(t, resolver.Shutdown(context.Background()))
watcherWG.Wait()
}
func TestResolverExpandEnvVars(t *testing.T) {
var testCases = []struct {
name string // test case name (also file name containing config yaml)
}{
{name: "expand-with-no-env.yaml"},
{name: "expand-with-partial-env.yaml"},
{name: "expand-with-all-env.yaml"},
}
envs := map[string]string{
"EXTRA": "some string",
"EXTRA_MAP_VALUE_1": "some map value_1",
"EXTRA_MAP_VALUE_2": "some map value_2",
"EXTRA_LIST_MAP_VALUE_1": "some list map value_1",
"EXTRA_LIST_MAP_VALUE_2": "some list map value_2",
"EXTRA_LIST_VALUE_1": "some list value_1",
"EXTRA_LIST_VALUE_2": "some list value_2",
}
expectedCfgMap := newConfFromFile(t, filepath.Join("testdata", "expand-with-no-env.yaml"))
fileProvider := newFakeProvider("file", func(_ context.Context, uri string, _ WatcherFunc) (*Retrieved, error) {
return NewRetrieved(newConfFromFile(t, uri[5:]))
})
envProvider := newFakeProvider("env", func(_ context.Context, uri string, _ WatcherFunc) (*Retrieved, error) {
return NewRetrieved(envs[uri[4:]])
})
for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
resolver, err := NewResolver(ResolverSettings{URIs: []string{filepath.Join("testdata", test.name)}, Providers: makeMapProvidersMap(fileProvider, envProvider), Converters: nil})
require.NoError(t, err)
// Test that expanded configs are the same with the simple config with no env vars.
cfgMap, err := resolver.Resolve(context.Background())
require.NoError(t, err)
assert.Equal(t, expectedCfgMap, cfgMap.ToStringMap())
})
}
}
func TestResolverDoneNotExpandOldEnvVars(t *testing.T) {
expectedCfgMap := map[string]any{"test.1": "${EXTRA}", "test.2": "$EXTRA", "test.3": "${EXTRA}:${EXTRA}"}
fileProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(expectedCfgMap)
})
envProvider := newFakeProvider("env", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved("some string")
})
emptySchemeProvider := newFakeProvider("", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved("some string")
})
resolver, err := NewResolver(ResolverSettings{URIs: []string{"test:"}, Providers: makeMapProvidersMap(fileProvider, envProvider, emptySchemeProvider), Converters: nil})
require.NoError(t, err)
// Test that expanded configs are the same with the simple config with no env vars.
cfgMap, err := resolver.Resolve(context.Background())
require.NoError(t, err)
assert.Equal(t, expectedCfgMap, cfgMap.ToStringMap())
}
func TestResolverExpandMapAndSliceValues(t *testing.T) {
provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(map[string]any{
"test_map": map[string]any{"recv": "${test:MAP_VALUE}"},
"test_slice": []any{"${test:MAP_VALUE}"}})
})
const receiverExtraMapValue = "some map value"
testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(receiverExtraMapValue)
})
resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil})
require.NoError(t, err)
cfgMap, err := resolver.Resolve(context.Background())
require.NoError(t, err)
expectedMap := map[string]any{
"test_map": map[string]any{"recv": receiverExtraMapValue},
"test_slice": []any{receiverExtraMapValue}}
assert.Equal(t, expectedMap, cfgMap.ToStringMap())
}
func TestResolverExpandStringValues(t *testing.T) {
tests := []struct {
name string
input string
output any
}{
{
name: "test_no_match_old",
input: "${HOST}:${PORT}",
output: "${HOST}:${PORT}",
},
{
name: "test_no_match_old_no_brackets",
input: "${HOST}:$PORT",
output: "${HOST}:$PORT",
},
{
name: "test_match_value",
input: "${env:COMPLEX_VALUE}",
output: []any{"localhost:3042"},
},
{
name: "test_match_embedded",
input: "${env:HOST}:3043",
output: "localhost:3043",
},
{
name: "test_match_embedded_multi",
input: "${env:HOST}:${env:PORT}",
output: "localhost:3044",
},
{
name: "test_match_embedded_concat",
input: "https://${env:HOST}:3045",
output: "https://localhost:3045",
},
{
name: "test_match_embedded_new_and_old",
input: "${env:HOST}:${PORT}",
output: "localhost:${PORT}",
},
{
name: "test_match_int",
input: "test_${env:INT}",
output: "test_1",
},
{
name: "test_match_int32",
input: "test_${env:INT32}",
output: "test_32",
},
{
name: "test_match_int64",
input: "test_${env:INT64}",
output: "test_64",
},
{
name: "test_match_float32",
input: "test_${env:FLOAT32}",
output: "test_3.25",
},
{
name: "test_match_float64",
input: "test_${env:FLOAT64}",
output: "test_6.4",
},
{
name: "test_match_bool",
input: "test_${env:BOOL}",
output: "test_true",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(map[string]any{tt.name: tt.input})
})
testProvider := newFakeProvider("env", func(_ context.Context, uri string, _ WatcherFunc) (*Retrieved, error) {
switch uri {
case "env:COMPLEX_VALUE":
return NewRetrieved([]any{"localhost:3042"})
case "env:HOST":
return NewRetrieved("localhost")
case "env:PORT":
return NewRetrieved(3044)
case "env:INT":
return NewRetrieved(1)
case "env:INT32":
return NewRetrieved(32)
case "env:INT64":
return NewRetrieved(64)
case "env:FLOAT32":
return NewRetrieved(float32(3.25))
case "env:FLOAT64":
return NewRetrieved(float64(6.4))
case "env:BOOL":
return NewRetrieved(true)
}
return nil, errors.New("impossible")
})
resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil})
require.NoError(t, err)
cfgMap, err := resolver.Resolve(context.Background())
require.NoError(t, err)
assert.Equal(t, map[string]any{tt.name: tt.output}, cfgMap.ToStringMap())
})
}
}
func TestResolverExpandReturnError(t *testing.T) {
tests := []struct {
name string
input any
}{
{
name: "string_value",
input: "${test:VALUE}",
},
{
name: "slice_value",
input: []any{"${test:VALUE}"},
},
{
name: "map_value",
input: map[string]any{"test": "${test:VALUE}"},
},
{
name: "string_embedded_value",
input: "https://${test:HOST}:3045",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(map[string]any{tt.name: tt.input})
})
myErr := errors.New(tt.name)
testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return nil, myErr
})
resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil})
require.NoError(t, err)
_, err = resolver.Resolve(context.Background())
assert.ErrorIs(t, err, myErr)
})
}
}
func TestResolverInfiniteExpand(t *testing.T) {
const receiverValue = "${test:VALUE}"
provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(map[string]any{"test": receiverValue})
})
testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(receiverValue)
})
resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil})
require.NoError(t, err)
_, err = resolver.Resolve(context.Background())
assert.ErrorIs(t, err, errTooManyRecursiveExpansions)
}
func TestResolverExpandInvalidScheme(t *testing.T) {
provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(map[string]any{"test": "${g_c_s:VALUE}"})
})
testProvider := newFakeProvider("g_c_s", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
panic("must not be called")
})
resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil})
require.NoError(t, err)
_, err = resolver.Resolve(context.Background())
// TODO: Investigate how to return an error like `invalid uri: "g_c_s:VALUE"` and keep the legacy behavior
// for cases like "${HOST}:${PORT}"
assert.NoError(t, err)
}
func TestResolverExpandInvalidOpaqueValue(t *testing.T) {
provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(map[string]any{"test": []any{map[string]any{"test": "${test:$VALUE}"}}})
})
testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
panic("must not be called")
})
resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil})
require.NoError(t, err)
_, err = resolver.Resolve(context.Background())
assert.EqualError(t, err, `the uri "test:$VALUE" contains unsupported characters ('$')`)
}
func TestResolverExpandUnsupportedScheme(t *testing.T) {
provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(map[string]any{"test": "${unsupported:VALUE}"})
})
testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
panic("must not be called")
})
resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil})
require.NoError(t, err)
_, err = resolver.Resolve(context.Background())
assert.EqualError(t, err, `scheme "unsupported" is not supported for uri "unsupported:VALUE"`)
}
func TestResolverExpandStringValueInvalidReturnValue(t *testing.T) {
provider := newFakeProvider("input", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved(map[string]any{"test": "localhost:${test:PORT}"})
})
testProvider := newFakeProvider("test", func(context.Context, string, WatcherFunc) (*Retrieved, error) {
return NewRetrieved([]any{1243})
})
resolver, err := NewResolver(ResolverSettings{URIs: []string{"input:"}, Providers: makeMapProvidersMap(provider, testProvider), Converters: nil})
require.NoError(t, err)
_, err = resolver.Resolve(context.Background())
assert.EqualError(t, err, `expanding ${test:PORT}, expected string value type, got []interface {}`)
}
func makeMapProvidersMap(providers ...Provider) map[string]Provider {
ret := make(map[string]Provider, len(providers))
for _, provider := range providers {
ret[provider.Scheme()] = provider
}
return ret
}