diff --git a/bake/hcl_test.go b/bake/hcl_test.go index 88972e8c..db1ca9c6 100644 --- a/bake/hcl_test.go +++ b/bake/hcl_test.go @@ -423,6 +423,63 @@ func TestHCLNullVariables(t *testing.T) { require.Equal(t, ptrstr("bar"), c.Targets[0].Args["foo"]) } +func TestHCLTypedNullVariables(t *testing.T) { + types := []string{ + "any", + "string", "number", "bool", + "list(string)", "set(string)", "map(string)", + "tuple([string])", "object({val: string})", + } + for _, varType := range types { + tName := fmt.Sprintf("variable typed %q with null default remains null", varType) + t.Run(tName, func(t *testing.T) { + dt := fmt.Sprintf(` + variable "FOO" { + type = %s + default = null + } + + target "default" { + args = { + foo = equal(FOO, null) + } + }`, varType) + c, err := ParseFile([]byte(dt), "docker-bake.hcl") + require.NoError(t, err) + require.Equal(t, 1, len(c.Targets)) + require.Equal(t, "true", *c.Targets[0].Args["foo"]) + }) + } +} + +func TestHCLTypedValuelessVariables(t *testing.T) { + types := []string{ + "any", + "string", "number", "bool", + "list(string)", "set(string)", "map(string)", + "tuple([string])", "object({val: string})", + } + for _, varType := range types { + tName := fmt.Sprintf("variable typed %q with no default is null", varType) + t.Run(tName, func(t *testing.T) { + dt := fmt.Sprintf(` + variable "FOO" { + type = %s + } + + target "default" { + args = { + foo = equal(FOO, null) + } + }`, varType) + c, err := ParseFile([]byte(dt), "docker-bake.hcl") + require.NoError(t, err) + require.Equal(t, 1, len(c.Targets)) + require.Equal(t, "true", *c.Targets[0].Args["foo"]) + }) + } +} + func TestJSONNullVariables(t *testing.T) { dt := []byte(`{ "variable": { @@ -1565,6 +1622,20 @@ target "two" { require.Equal(t, map[string]*string{"b": ptrstr("pre-jkl")}, c.Targets[1].Args) } +func TestEmptyVariable(t *testing.T) { + dt := []byte(` + variable "FOO" {} + target "default" { + args = { + foo = equal(FOO, "") + } + }`) + c, err := ParseFile(dt, "docker-bake.hcl") + require.NoError(t, err) + require.Equal(t, 1, len(c.Targets)) + require.Equal(t, "true", *c.Targets[0].Args["foo"]) +} + func TestEmptyVariableJSON(t *testing.T) { dt := []byte(`{ "variable": { @@ -1877,19 +1948,6 @@ func TestTypedVarOverrides(t *testing.T) { override: `"hello"`, wantValue: `"hello"`, }, - { - name: "any", - varType: "any", - override: "[1,2]", - wantValue: "[1,2]", - }, - { - name: "any never convert to complex types", - varType: "any", - override: "[1,2]", - argValue: "length(FOO)", - wantErrorMsg: "collection must be a list", - }, { name: "proper CSV list of strings", varType: "list(string)", @@ -2090,19 +2148,6 @@ func TestTypedVarOverrides_JSON(t *testing.T) { override: `"hello"`, wantValue: "hello", }, - { - name: "any", - varType: "any", - override: "[1,2]", - wantValue: "[1,2]", - }, - { - name: "any never convert to complex types", - varType: "any", - override: "[1,2]", - argValue: "length(FOO)", - wantErrorMsg: "collection must be a list", - }, { name: "list of strings", varType: "list(string)", @@ -2313,6 +2358,7 @@ func TestJSONOverridePriority(t *testing.T) { dt := []byte(` variable "foo" { type = number + default = 101 } target "default" { @@ -2325,8 +2371,7 @@ func TestJSONOverridePriority(t *testing.T) { c, err := ParseFile(dt, "docker-bake.hcl") require.NoError(t, err) require.Equal(t, 1, len(c.Targets)) - // a variable with no value has always resulted in an empty string - require.Equal(t, "", *c.Targets[0].Args["bar"]) + require.Equal(t, "101", *c.Targets[0].Args["bar"]) t.Setenv("foo_JSON", "42") c, err = ParseFile(dt, "docker-bake.hcl") diff --git a/bake/hclparser/hclparser.go b/bake/hclparser/hclparser.go index d132e966..d8f13459 100644 --- a/bake/hclparser/hclparser.go +++ b/bake/hclparser/hclparser.go @@ -282,7 +282,7 @@ func (p *parser) resolveValue(ectx *hcl.EvalContext, name string) (err error) { } var diags hcl.Diagnostics - varType := cty.DynamicPseudoType + varType, typeSpecified := cty.DynamicPseudoType, false def, ok := p.attrs[name] if !ok { vr, ok := p.vars[name] @@ -295,12 +295,13 @@ func (p *parser) resolveValue(ectx *hcl.EvalContext, name string) (err error) { if diags.HasErrors() { return diags } + typeSpecified = !varType.Equals(cty.DynamicPseudoType) || hcl.ExprAsKeyword(vr.Type) == "any" } if def == nil { - // lack of specified value is considered to have an empty string value, - // but any overrides get type checked - if _, ok, _ := p.valueHasOverride(name, false); !ok { + // Lack of specified value, when untyped is considered to have an empty string value. + // A typed variable with no value will result in (typed) nil. + if _, ok, _ := p.valueHasOverride(name, false); !ok && !typeSpecified { vv := cty.StringVal("") v = &vv return @@ -322,9 +323,6 @@ func (p *parser) resolveValue(ectx *hcl.EvalContext, name string) (err error) { } } - // Not entirely true... this doesn't differentiate between a user that specified 'any' - // and a user that specified nothing. But the result is the same; both are treated as strings. - typeSpecified := !varType.Equals(cty.DynamicPseudoType) envv, hasEnv, jsonEnv := p.valueHasOverride(name, typeSpecified) _, isVar := p.vars[name] diff --git a/docs/bake-reference.md b/docs/bake-reference.md index 92f39994..5245f611 100644 --- a/docs/bake-reference.md +++ b/docs/bake-reference.md @@ -1081,6 +1081,7 @@ or interpolate them in attribute values in your Bake file. ```hcl variable "TAG" { + type = string default = "latest" } @@ -1102,6 +1103,206 @@ overriding the default `latest` value shown in the previous example. $ TAG=dev docker buildx bake webapp-dev ``` +Variables can also be assigned an explicit type. +If provided, it will be used to validate the default value (if set), as well as any overrides. +This is particularly useful when using complex types which are intended to be overridden. +The previous example could be expanded to apply an arbitrary series of tags. +```hcl +variable "TAGS" { + default = ["latest"] + type = list(string) +} + +target "webapp-dev" { + dockerfile = "Dockerfile.webapp" + tags = [for tag in TAGS: "docker.io/username/webapp:${tag}"] +} +``` + +This example shows how to generate three tags without changing the file +or using custom functions/parsing: +```console +$ TAGS=dev,latest,2 docker buildx bake webapp-dev +``` + +### Variable typing + +The following primitive types are available: +* `string` +* `number` +* `bool` + +The type is expressed like a keyword; it must be expressed as a literal: +```hcl +variable "OK" { + type = string +} + +# cannot be an actual string +variable "BAD" { + type = "string" +} + +# cannot be the result of an expression +variable "ALSO_BAD" { + type = lower("string") +} +``` +Specifying primitive types can be valuable to show intent (especially when a default is not provided), +but bake will generally behave as expected without explicit typing. + +Complex types are expressed with "type constructors"; they are: +* `tuple([,...])` +* `list()` +* `set()` +* `map()` +* `object({=},...})` + +The following are examples of each of those, as well as how the (optional) default value would be expressed: +```hcl +# structured way to express "1.2.3-alpha" +variable "MY_VERSION" { + type = tuple([number, number, number, string]) + default = [1, 2, 3, "alpha"] +} + +# JDK versions used in a matrix build +variable "JDK_VERSIONS" { + type = list(number) + default = [11, 17, 21] +} + +# better way to express the previous example; this will also +# enforce set semantics and allow use of set-based functions +variable "JDK_VERSIONS" { + type = set(number) + default = [11, 17, 21] +} + +# with the help of lookup(), translate a 'feature' to a tag +variable "FEATURE_TO_NAME" { + type = map(string) + default = {featureA = "slim", featureB = "tiny"} +} + +# map a branch name to a registry location +variable "PUSH_DESTINATION" { + type = object({branch = string, registry = string}) + default = {branch = "main", registry = "prod-registry.invalid.com"} +} + +# make the previous example more useful with composition +variable "PUSH_DESTINATIONS" { + type = list(object({branch = string, registry = string})) + default = [ + {branch = "develop", registry = "test-registry.invalid.com"}, + {branch = "main", registry = "prod-registry.invalid.com"}, + ] +} +``` +Note that in each example, the default value would be valid even if typing was not present. +If typing was omitted, the first three would all be considered `tuple`; +you would be restricted to functions that operate on `tuple` and, for example, not be able to add elements. +Similarly, the third and fourth would both be considered `object`, with the limits and semantics of that type. +In short, in the absence of a type, any value delimited with `[]` is a `tuple` +and value delimited with `{}` is an `object`. +Explicit typing for complex types not only opens up the ability to use functions applicable to that specialized type, +but is also a precondition for providing overrides. + +> [!NOTE] +> See [HCL Type Expressions][typeexpr] page for more details. + +### Overriding variables + +As mentioned in the [intro to variables](#variable), primitive types (`string`, `number`, and `bool`) +can be overridden without typing and will generally behave as expected. +(When explicit typing is not provided, a variable is assumed to be primitive when the default value lacks `{}` or `[]` delimiters; +a variable with neither typing nor a default value is treated as `string`.) +Naturally, these same overrides can be used alongside explicit typing too; +they may help in edge cases where you want `VAR=true` to be a `string`, where without typing, +it may be a `string` or a `bool` depending on how/where it's used. +Overriding a variable with a complex type can only be done when the type is provided. +This is still done via environment variables, but the values can be provided via CSV or JSON. + +#### CSV overrides + +This is considered the canonical method and is well suited to interactive usage. +It is assumed that `list` and `set` will be the most common complex type, +as well as the most common complex type designed to be overridden. +Thus, there is full CSV support for `list` and `set` +(and `tuple`; despite being considered a structural type, it is more like a collection type in this regard). + + +There is limited support for `map` and `object` and no support for composite types; +for these advanced cases, an alternative mechanism [using JSON](#json-overrides) is available. + +#### JSON overrides + +Overrides can also be provided via JSON. +This is the only method available for providing some complex types and may be convenient if overrides are already JSON +(for example, if they come from a JSON API). +It can also be used when dealing with values are difficult or impossible to specify using CSV (e.g., values containing quotes or commas). +To use JSON, simply append `_JSON` to the variable name. +In this contrived example, CSV cannot handle the second value; despite being a supported CSV type, JSON must be used: +```hcl +variable "VALS" { + type = list(string) + default = ["some", "list"] +} +``` +```console +$ cat data.json +["hello","with,comma","with\"quote"] +$ VALS_JSON=$(< data.json) docker buildx bake + +# CSV equivalent, though the second value cannot be expressed at all +$ VALS='hello,"with""quote"' docker buildx bake +``` + +This example illustrates some precedence and usage rules: +```hcl +variable "FOO" { + type = string + default = "foo" +} + +variable "FOO_JSON" { + type = string + default = "foo" +} +``` + +The variable `FOO` can *only* be overridden using CSV because `FOO_JSON`, which would typically used for a JSON override, +is already a defined variable. +Since `FOO_JSON` is an actual variable, setting that environment variable would be expected to a CSV value. +A JSON override *is* possible for this variable, using environment variable `FOO_JSON_JSON`. + +```Console +# These three are all equivalent, setting variable FOO=bar +$ FOO=bar docker buildx bake <...> +$ FOO='bar' docker buildx bake <...> +$ FOO="bar" docker buildx bake <...> + +# Sets *only* variable FOO_JSON; FOO is untouched +$ FOO_JSON=bar docker buildx bake <...> + +# This also sets FOO_JSON, but will fail due to not being valid JSON +$ FOO_JSON_JSON=bar docker buildx bake <...> + +# These are all equivalent +$ cat data.json +"bar" +$ FOO_JSON_JSON=$(< data.json) docker buildx bake <...> +$ FOO_JSON_JSON='"bar"' docker buildx bake <...> +$ FOO_JSON=bar docker buildx bake <...> + +# This results in setting two different variables, both specified as CSV (FOO=bar and FOO_JSON="baz") +$ FOO=bar FOO_JSON='"baz"' docker buildx bake <...> + +# These refer to the same variable with FOO_JSON_JSON having precedence and read as JSON (FOO_JSON=baz) +$ FOO_JSON=bar FOO_JSON_JSON='"baz"' docker buildx bake <...> +``` + ### Built-in variables The following variables are built-ins that you can use with Bake without having @@ -1239,4 +1440,5 @@ target "webapp-dev" { [ssh]: https://docs.docker.com/reference/cli/docker/buildx/build/#ssh [tag]: https://docs.docker.com/reference/cli/docker/image/build/#tag [target]: https://docs.docker.com/reference/cli/docker/image/build/#target +[typeexpr]: https://github.com/hashicorp/hcl/tree/main/ext/typeexpr [userfunc]: https://github.com/hashicorp/hcl/tree/main/ext/userfunc diff --git a/go.mod b/go.mod index a796b6c9..06aa6815 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/hashicorp/hcl/v2 v2.23.0 github.com/in-toto/in-toto-golang v0.5.0 github.com/mitchellh/hashstructure/v2 v2.0.2 - github.com/moby/buildkit v0.22.0-rc2 + github.com/moby/buildkit v0.22.0 github.com/moby/go-archive v0.1.0 github.com/moby/sys/atomicwriter v0.1.0 github.com/moby/sys/mountinfo v0.7.2 diff --git a/go.sum b/go.sum index a156b17f..2adf8018 100644 --- a/go.sum +++ b/go.sum @@ -253,8 +253,8 @@ github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZX github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/moby/buildkit v0.22.0-rc2 h1:B/1nFOqPO/ceJ8nzAbc+q3d0ckyGwlWtxVQXpeqXuwY= -github.com/moby/buildkit v0.22.0-rc2/go.mod h1:j4pP5hxiTWcz7xuTK2cyxQislHl/N2WWHzOy43DlLJw= +github.com/moby/buildkit v0.22.0 h1:aWN06w1YGSVN1XfeZbj2ZbgY+zi5xDAjEFI8Cy9fTjA= +github.com/moby/buildkit v0.22.0/go.mod h1:j4pP5hxiTWcz7xuTK2cyxQislHl/N2WWHzOy43DlLJw= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= diff --git a/vendor/modules.txt b/vendor/modules.txt index 7daf793c..9492d656 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -451,7 +451,7 @@ github.com/mitchellh/go-wordwrap # github.com/mitchellh/hashstructure/v2 v2.0.2 ## explicit; go 1.14 github.com/mitchellh/hashstructure/v2 -# github.com/moby/buildkit v0.22.0-rc2 +# github.com/moby/buildkit v0.22.0 ## explicit; go 1.23.0 github.com/moby/buildkit/api/services/control github.com/moby/buildkit/api/types