HTTP Binding: add "maxResponseBodySize" metadata option (#3040)

Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com>
Signed-off-by: Bernd Verst <github@bernd.dev>
Co-authored-by: Bernd Verst <github@bernd.dev>
This commit is contained in:
Alessandro (Ale) Segala 2023-08-14 21:06:30 -07:00 committed by GitHub
parent 80a16418c8
commit da50003dd4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 330 additions and 65 deletions

View File

@ -14,6 +14,7 @@ require (
require (
github.com/dapr/kit v0.11.4-0.20230807225040-b6b141aa3e32 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect
github.com/inconshreveable/mousetrap v1.0.1 // indirect
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect
@ -21,7 +22,9 @@ require (
github.com/spf13/pflag v1.0.5 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
k8s.io/apimachinery v0.26.3 // indirect
)
replace github.com/dapr/components-contrib => ../

View File

@ -5,17 +5,23 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk=
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=
github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/invopop/jsonschema v0.6.0 h1:8e+xY8ZEn8gDHUYylSlLHy22P+SLeIRIHv3nM3hCbmY=
github.com/invopop/jsonschema v0.6.0/go.mod h1:O9uiLokuu0+MGFlyiaqtWxwqJm41/+8Nj0lD7A36YH0=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 h1:BpfhmLKZf+SjVanKKhCgf3bg+511DmU9eDQTen7LLbY=
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
@ -37,13 +43,44 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
github.com/xeipuuv/gojsonschema v1.2.1-0.20201027075954-b076d39a02e5 h1:ImnGIsrcG8vwbovhYvvSY8fagVV6QhCWSWXfzwGDLVs=
github.com/xeipuuv/gojsonschema v1.2.1-0.20201027075954-b076d39a02e5/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI=
golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
k8s.io/apimachinery v0.26.3 h1:dQx6PNETJ7nODU3XPtrwkfuubs6w7sX0M8n61zHIV/k=
k8s.io/apimachinery v0.26.3/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I=
sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

View File

@ -20,7 +20,7 @@ type ComponentMetadata struct {
// Version of the component metadata schema.
SchemaVersion string `json:"schemaVersion" yaml:"schemaVersion" jsonschema:"enum=v1"`
// Component type, of one of the allowed values.
Type string `json:"type" yaml:"type" jsonschema:"enum=bindings,enum=state,enum=secretstores,enum=pubsub,enum=workflows,enum=configuration,enum=lock,enum=middleware"`
Type string `json:"type" yaml:"type" jsonschema:"enum=bindings,enum=state,enum=secretstores,enum=pubsub,enum=workflows,enum=configuration,enum=lock,enum=middleware,enum=crypto"`
// Name of the component (without the inital type, e.g. "http" instead of "bindings.http").
Name string `json:"name" yaml:"name"`
// Version of the component, with the leading "v", e.g. "v1".
@ -86,7 +86,7 @@ type Metadata struct {
Sensitive bool `json:"sensitive,omitempty" yaml:"sensitive,omitempty"`
// Type of the property.
// If this is empty, it's interpreted as "string".
Type string `json:"type,omitempty" yaml:"type,omitempty" jsonschema:"enum=string,enum=number,enum=bool,enum=duration"`
Type string `json:"type,omitempty" yaml:"type,omitempty" jsonschema:"enum=string,enum=number,enum=bool,enum=duration,enum=bytesize"`
// Default value for the property.
// If it's a string, don't forget to add quotes.
Default string `json:"default,omitempty" yaml:"default,omitempty"`

View File

@ -46,6 +46,7 @@ const (
TraceMetadataKey = "traceHeaders"
securityToken = "securityToken"
securityTokenHeader = "securityTokenHeader"
defaultMaxResponseBodySizeBytes = 100 << 20 // 100 MB
)
// HTTPSource is a binding for an http url endpoint invocation
@ -67,17 +68,29 @@ type httpMetadata struct {
SecurityToken string `mapstructure:"securityToken"`
SecurityTokenHeader string `mapstructure:"securityTokenHeader"`
ResponseTimeout *time.Duration `mapstructure:"responseTimeout"`
// Maximum response to read from HTTP response bodies.
// This can either be an integer which is interpreted in bytes, or a string with an added unit such as Mi.
// A value <= 0 means no limit.
// Default: 100MB
MaxResponseBodySize metadata.ByteSize `mapstructure:"maxResponseBodySize"`
maxResponseBodySizeBytes int64
}
// NewHTTP returns a new HTTPSource.
func NewHTTP(logger logger.Logger) bindings.OutputBinding {
return &HTTPSource{logger: logger}
return &HTTPSource{
logger: logger,
}
}
// Init performs metadata parsing.
func (h *HTTPSource) Init(_ context.Context, meta bindings.Metadata) error {
var err error
if err = metadata.DecodeMetadata(meta.Properties, &h.metadata); err != nil {
h.metadata = httpMetadata{
MaxResponseBodySize: metadata.NewByteSize(defaultMaxResponseBodySizeBytes),
}
err := metadata.DecodeMetadata(meta.Properties, &h.metadata)
if err != nil {
return err
}
@ -98,6 +111,11 @@ func (h *HTTPSource) Init(_ context.Context, meta bindings.Metadata) error {
}
}
h.metadata.maxResponseBodySizeBytes, err = h.metadata.MaxResponseBodySize.GetBytes()
if err != nil {
return fmt.Errorf("invalid value for maxResponseBodySize: %w", err)
}
// See guidance on proper HTTP client settings here:
// https://medium.com/@nate510/don-t-use-go-s-default-http-client-4804cb19f779
dialer := &net.Dialer{
@ -226,7 +244,7 @@ func (h *HTTPSource) Invoke(parentCtx context.Context, req *bindings.InvokeReque
if req.Metadata == nil {
// Prevent things below from failing if req.Metadata is nil.
req.Metadata = make(map[string]string)
req.Metadata = make(map[string]string, 0)
}
if req.Metadata["path"] != "" {
@ -306,11 +324,20 @@ func (h *HTTPSource) Invoke(parentCtx context.Context, req *bindings.InvokeReque
if err != nil {
return nil, err
}
defer resp.Body.Close()
defer func() {
// Drain before closing
_, _ = io.Copy(io.Discard, resp.Body)
resp.Body.Close()
}()
var respBody io.Reader = resp.Body
if h.metadata.maxResponseBodySizeBytes > 0 {
respBody = io.LimitReader(resp.Body, h.metadata.maxResponseBodySizeBytes)
}
// Read the response body. For empty responses (e.g. 204 No Content)
// `b` will be an empty slice.
b, err := io.ReadAll(resp.Body)
b, err := io.ReadAll(respBody)
if err != nil {
return nil, err
}

View File

@ -68,6 +68,7 @@ func (tc TestCase) ToInvokeRequest() bindings.InvokeRequest {
}
requestMetadata["X-Status-Code"] = strconv.Itoa(tc.statusCode)
requestMetadata["path"] = tc.path
return bindings.InvokeRequest{
Data: []byte(tc.input),
@ -83,6 +84,14 @@ type HTTPHandler struct {
func (h *HTTPHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
h.Path = req.URL.Path
if strings.TrimPrefix(h.Path, "/") == "large" {
// Write 5KB
for i := 0; i < 1<<10; i++ {
fmt.Fprint(w, "12345")
}
return
}
h.Headers = make(map[string]string)
for headerKey, headerValue := range req.Header {
h.Headers[headerKey] = headerValue[0]
@ -728,3 +737,27 @@ func verifyTimeoutBehavior(t *testing.T, hs bindings.OutputBinding, handler *HTT
})
}
}
func TestMaxBodySizeHonored(t *testing.T) {
handler := NewHTTPHandler()
s := httptest.NewServer(handler)
defer s.Close()
hs, err := InitBinding(s, map[string]string{"maxResponseBodySize": "1Ki"})
require.NoError(t, err)
tc := TestCase{
input: "GET",
operation: "get",
path: "/large",
err: "context deadline exceeded",
statusCode: 200,
}
req := tc.ToInvokeRequest()
response, err := hs.Invoke(context.Background(), &req)
require.NoError(t, err)
// Should have only read 1KB
assert.Len(t, response.Data, 1<<10)
}

View File

@ -25,7 +25,7 @@ binding:
- name: patch
description: "Sometimes used to update a subset of fields of a record"
- name: delete
description: "Delete a data/record"
description: "Delete data/record"
- name: options
description: "Requests for information about the communication options available (not commonly used)"
- name: trace
@ -37,32 +37,28 @@ metadata:
description: "The base URL of the HTTP endpoint to invoke"
example: '"http://host:port/path", "http://myservice:8000/customer"'
# If omitted, uses the same values as "<root>.binding"
binding:
output: true
- name: responseTimeout
required: false
description: "The duration after which HTTP requests should be canceled."
example: '"10s", "5m"'
binding:
output: true
- name: maxResponseBodySize
required: false
description: "Max amount of data to read from the response body, as a resource quantity. A value <= 0 means no limit."
type: bytesize
default: '"100Mi"'
example: '"100" (as bytes), "1k", "10Ki", "1M", "1G"'
- name: MTLSRootCA
required: false
description: "Path to root ca certificate or pem encoded string"
example: "ca.pem"
binding:
output: true
description: "CA certificate: either a PEM-encoded string, or a path to a certificate on disk"
example: '"/path/to/ca.pem"'
- name: MTLSClientCert
required: false
description: "Path to client certificate or pem encoded string"
example: "client.pem"
binding:
output: true
description: "Client certificate for mTLS: either a PEM-encoded string, or a path to a certificate on disk"
example: '"/path/to/client.pem"'
- name: MTLSClientKey
required: false
description: "Path to client private key or pem encoded string"
example: "client.key"
binding:
output: true
description: "Client key for mTLS: either a PEM-encoded string, or a path to a certificate on disk"
example: '"/path/to/client.key"'
- name: MTLSRenegotiation
required: false
description: "Set TLS renegotiation setting"
@ -70,18 +66,12 @@ metadata:
- "RenegotiateNever"
- "RenegotiateOnceAsClient"
- "RenegotiateFreelyAsClient"
example: "RenegotiateOnceAsClient"
binding:
output: true
example: '"RenegotiateOnceAsClient"'
- name: securityToken
required: false
description: "The security token to include on an outgoing HTTP request as a header"
example: "this-value-is-preferably-injected-from-a-secret-store"
binding:
output: true
example: '"this-value-is-preferably-injected-from-a-secret-store"'
- name: securityTokenHeader
required: false
description: "The header name on an outgoing HTTP request for a security token"
example: "X-Security-Token"
binding:
output: true
example: '"X-Security-Token"'

View File

@ -117,7 +117,8 @@
"string",
"number",
"bool",
"duration"
"duration",
"bytesize"
],
"description": "Type of the property.\nIf this is empty, it's interpreted as \"string\"."
},
@ -211,7 +212,8 @@
"workflows",
"configuration",
"lock",
"middleware"
"middleware",
"crypto"
],
"description": "Component type, of one of the allowed values."
},

93
metadata/bytesize.go Normal file
View File

@ -0,0 +1,93 @@
/*
Copyright 2021 The Dapr Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package metadata
import (
"fmt"
"reflect"
"github.com/mitchellh/mapstructure"
"github.com/spf13/cast"
"k8s.io/apimachinery/pkg/api/resource"
)
// ByteSize contains a quantity for a resource that is measured in bytes.
// This extends the resource.Quantity struct from k8s.io/apimachinery to add some utility methods specific for Dapr.
// Although the library from K8s supports other kinds of resource quantities, our focus is on sizes in bytes.
type ByteSize struct {
resource.Quantity
}
// NewByteSize returns a new ByteSize with a default value in bytes.
func NewByteSize(defaultValue int64) ByteSize {
return ByteSize{
Quantity: *resource.NewQuantity(defaultValue, resource.BinarySI),
}
}
// GetBytes returns the number of bytes in the quantity.
// Note: this operation is expensive, so it's recommended to cache the returned value.
func (q *ByteSize) GetBytes() (int64, error) {
if q == nil || q.IsZero() {
return 0, nil
}
val, ok := q.AsInt64()
if !ok {
return 0, fmt.Errorf("cannot get bytes from resource quantity value '%v'", q)
}
return val, nil
}
func toByteSizeHookFunc() mapstructure.DecodeHookFunc {
bytesizeType := reflect.TypeOf(ByteSize{})
bytesizePtrType := reflect.TypeOf(&ByteSize{})
return func(
f reflect.Type,
t reflect.Type,
data any,
) (any, error) {
var isPtr bool
switch t {
case bytesizeType:
// Nop
case bytesizePtrType:
isPtr = true
default:
// Not a type we support with this hook
return data, nil
}
// First, cast to string
str, err := cast.ToStringE(data)
if err != nil {
return nil, fmt.Errorf("failed to cast value to string: %w", err)
}
// Parse as quantity
q, err := resource.ParseQuantity(str)
if err != nil {
return nil, fmt.Errorf("value is not a valid quantity: %w", err)
}
// Return a pointer if desired
res := ByteSize{Quantity: q}
if isPtr {
return &res, nil
}
return res, nil
}
}

View File

@ -170,6 +170,7 @@ func DecodeMetadata(input any, result any) error {
toTimeDurationHookFunc(),
toTruthyBoolHookFunc(),
toStringArrayHookFunc(),
toByteSizeHookFunc(),
),
Metadata: nil,
Result: result,
@ -250,36 +251,40 @@ func resolveAliases(md map[string]string, result any) error {
}
func toTruthyBoolHookFunc() mapstructure.DecodeHookFunc {
stringType := reflect.TypeOf("")
boolType := reflect.TypeOf(true)
boolPtrType := reflect.TypeOf(ptr.Of(true))
return func(
f reflect.Type,
t reflect.Type,
data any,
) (any, error) {
if f == reflect.TypeOf("") && t == reflect.TypeOf(true) {
val := data.(string)
return utils.IsTruthy(val), nil
if f == stringType && t == boolType {
return utils.IsTruthy(data.(string)), nil
}
if f == reflect.TypeOf("") && t == reflect.TypeOf(reflect.TypeOf(ptr.Of(true))) {
val := data.(string)
return ptr.Of(utils.IsTruthy(val)), nil
if f == stringType && t == boolPtrType {
return ptr.Of(utils.IsTruthy(data.(string))), nil
}
return data, nil
}
}
func toStringArrayHookFunc() mapstructure.DecodeHookFunc {
stringType := reflect.TypeOf("")
stringSliceType := reflect.TypeOf([]string{})
stringSlicePtrType := reflect.TypeOf(ptr.Of([]string{}))
return func(
f reflect.Type,
t reflect.Type,
data any,
) (any, error) {
if f == reflect.TypeOf("") && t == reflect.TypeOf([]string{}) {
val := data.(string)
return strings.Split(val, ","), nil
if f == stringType && t == stringSliceType {
return strings.Split(data.(string), ","), nil
}
if f == reflect.TypeOf("") && t == reflect.TypeOf(ptr.Of([]string{})) {
val := data.(string)
return ptr.Of(strings.Split(val, ",")), nil
if f == stringType && t == stringSlicePtrType {
return ptr.Of(strings.Split(data.(string), ",")), nil
}
return data, nil
}
@ -287,8 +292,9 @@ func toStringArrayHookFunc() mapstructure.DecodeHookFunc {
func toTimeDurationArrayHookFunc() mapstructure.DecodeHookFunc {
convert := func(input string) ([]time.Duration, error) {
res := make([]time.Duration, 0)
for _, v := range strings.Split(input, ",") {
parts := strings.Split(input, ",")
res := make([]time.Duration, 0, len(parts))
for _, v := range parts {
input := strings.TrimSpace(v)
if input == "" {
continue
@ -307,16 +313,20 @@ func toTimeDurationArrayHookFunc() mapstructure.DecodeHookFunc {
return res, nil
}
stringType := reflect.TypeOf("")
durationSliceType := reflect.TypeOf([]time.Duration{})
durationSlicePtrType := reflect.TypeOf(ptr.Of([]time.Duration{}))
return func(
f reflect.Type,
t reflect.Type,
data any,
) (any, error) {
if f == reflect.TypeOf("") && t == reflect.TypeOf([]time.Duration{}) {
if f == stringType && t == durationSliceType {
inputArrayString := data.(string)
return convert(inputArrayString)
}
if f == reflect.TypeOf("") && t == reflect.TypeOf(ptr.Of([]time.Duration{})) {
if f == stringType && t == durationSlicePtrType {
inputArrayString := data.(string)
res, err := convert(inputArrayString)
if err != nil {

View File

@ -30,14 +30,14 @@ func TestIsRawPayload(t *testing.T) {
})
assert.Equal(t, false, val)
assert.Nil(t, err)
assert.NoError(t, err)
})
t.Run("Metadata map is nil", func(t *testing.T) {
val, err := IsRawPayload(nil)
assert.Equal(t, false, val)
assert.Nil(t, err)
assert.NoError(t, err)
})
t.Run("Metadata with bad value", func(t *testing.T) {
@ -55,7 +55,7 @@ func TestIsRawPayload(t *testing.T) {
})
assert.Equal(t, false, val)
assert.Nil(t, err)
assert.NoError(t, err)
})
t.Run("Metadata with correct value as true", func(t *testing.T) {
@ -64,7 +64,7 @@ func TestIsRawPayload(t *testing.T) {
})
assert.Equal(t, true, val)
assert.Nil(t, err)
assert.NoError(t, err)
})
}
@ -143,7 +143,7 @@ func TestMetadataDecode(t *testing.T) {
err := DecodeMetadata(testData, &m)
assert.Nil(t, err)
require.NoError(t, err)
assert.Equal(t, true, *m.Mybool)
assert.Equal(t, "test", m.Mystring)
assert.Equal(t, 1, m.Myinteger)
@ -185,7 +185,7 @@ func TestMetadataDecode(t *testing.T) {
testData["boolvaluenonsense"] = "nonsense"
err := DecodeMetadata(testData, &m)
assert.NoError(t, err)
require.NoError(t, err)
assert.True(t, *m.BoolPointer)
assert.True(t, m.BoolValueOn)
assert.True(t, m.BoolValue1)
@ -225,7 +225,7 @@ func TestMetadataDecode(t *testing.T) {
testData["emptystringarraypointerwithcomma"] = ","
err := DecodeMetadata(testData, &m)
assert.NoError(t, err)
require.NoError(t, err)
assert.Equal(t, []string{"one", "two", "three"}, m.StringArray)
assert.Equal(t, []string{"one", "two", "three"}, *m.StringArrayPointer)
assert.Equal(t, []string{""}, m.EmptyStringArray)
@ -237,6 +237,37 @@ func TestMetadataDecode(t *testing.T) {
assert.Equal(t, []string{"", ""}, m.EmptyStringArrayWithComma)
assert.Equal(t, []string{"", ""}, *m.EmptyStringArrayPointerWithComma)
})
t.Run("Test metadata decode hook for byte sizes", func(t *testing.T) {
type testMetadata struct {
BytesizeValue1 ByteSize
BytesizeValue2 ByteSize
BytesizeValue3 ByteSize
BytesizeValue4 ByteSize
BytesizeValueNotProvided ByteSize
BytesizeValuePtr *ByteSize
BytesizeValuePtrNotProvided *ByteSize
}
var m testMetadata
testData := make(map[string]any)
testData["bytesizevalue1"] = "100"
testData["bytesizevalue2"] = 100
testData["bytesizevalue3"] = "1Ki"
testData["bytesizevalue4"] = "1000k"
testData["bytesizevalueptr"] = "1Gi"
err := DecodeMetadata(testData, &m)
require.NoError(t, err)
assert.Equal(t, "100", m.BytesizeValue1.String())
assert.Equal(t, "100", m.BytesizeValue2.String())
assert.Equal(t, "1Ki", m.BytesizeValue3.String())
assert.Equal(t, "1M", m.BytesizeValue4.String())
assert.Equal(t, "1Gi", m.BytesizeValuePtr.String())
assert.Nil(t, m.BytesizeValuePtrNotProvided)
assert.Equal(t, "0", m.BytesizeValueNotProvided.String())
})
}
func TestMetadataStructToStringMap(t *testing.T) {

View File

@ -11,6 +11,7 @@ require (
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.14.0 // indirect
github.com/cloudevents/sdk-go/v2 v2.14.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
@ -27,6 +28,8 @@ require (
golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b // indirect
golang.org/x/sys v0.11.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
k8s.io/apimachinery v0.26.3 // indirect
)
replace github.com/dapr/components-contrib => ../../../../

View File

@ -10,16 +10,21 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@ -47,26 +52,57 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA=
github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI=
golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
k8s.io/apimachinery v0.26.3 h1:dQx6PNETJ7nODU3XPtrwkfuubs6w7sX0M8n61zHIV/k=
k8s.io/apimachinery v0.26.3/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I=