docker
/
buildx
mirror of https://github.com/docker/buildx.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #928 from tonistiigi/bake-named-contexts 

bake: add named contexts keys
This commit is contained in:
Tõnis Tiigi 2022-02-07 10:52:01 -08:00 committed by GitHub
parent fec415a8e0 c3dab802d8
commit 60a025b227
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 639 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
bake/bake.go
View File

@ -8,6 +8,7 @@ import (
"os"
"path"
"regexp"
"sort"
"strconv"
"strings"
@ -130,6 +131,12 @@ func ReadTargets(ctx context.Context, files []File, targets, overrides []string,
g = []*Group{{Targets: dedupString(gt)}}
}
for name, t := range m {
if err := c.loadLinks(name, t, m, o, nil); err != nil {
return nil, nil, err
}
}
return m, g, nil
}
@ -303,10 +310,45 @@ func (c Config) expandTargets(pattern string) ([]string, error) {
return names, nil
}
func (c Config) loadLinks(name string, t *Target, m map[string]*Target, o map[string]map[string]Override, visited []string) error {
visited = append(visited, name)
for _, v := range t.Contexts {
if strings.HasPrefix(v, "target:") {
target := strings.TrimPrefix(v, "target:")
if target == t.Name {
return errors.Errorf("target %s cannot link to itself", target)
}
for _, v := range visited {
if v == target {
return errors.Errorf("infinite loop from %s to %s", name, target)
}
}
t2, ok := m[target]
if !ok {
var err error
t2, err = c.ResolveTarget(target, o)
if err != nil {
return err
}
t2.Outputs = nil
m[target] = t2
}
if err := c.loadLinks(target, t2, m, o, visited); err != nil {
return err
}
if len(t.Platforms) > 1 && len(t2.Platforms) > 1 {
if !sliceEqual(t.Platforms, t2.Platforms) {
return errors.Errorf("target %s can't be used by %s because it is defined for different platforms %v and %v", target, name, t2.Platforms, t.Platforms)
}
}
}
}
return nil
}
func (c Config) newOverrides(v []string) (map[string]map[string]Override, error) {
m := map[string]map[string]Override{}
for _, v := range v {
parts := strings.SplitN(v, "=", 2)
keys := strings.SplitN(parts[0], ".", 3)
if len(keys) < 2 {
@ -351,6 +393,11 @@ func (c Config) newOverrides(v []string) (map[string]map[string]Override, error)
o.Value = v
}
fallthrough
case "contexts":
if len(keys) != 3 {
return nil, errors.Errorf("invalid key %s, contexts requires name", parts[0])
}
fallthrough
default:
if len(parts) == 2 {
o.Value = parts[1]
@ -461,6 +508,7 @@ type Target struct {
Inherits []string `json:"inherits,omitempty" hcl:"inherits,optional"`
Context *string `json:"context,omitempty" hcl:"context,optional"`
Contexts map[string]string `json:"contexts,omitempty" hcl:"contexts,optional"`
Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional"`
DockerfileInline *string `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional"`
Args map[string]string `json:"args,omitempty" hcl:"args,optional"`
@ -488,6 +536,15 @@ func (t *Target) normalize() {
t.CacheFrom = removeDupes(t.CacheFrom)
t.CacheTo = removeDupes(t.CacheTo)
t.Outputs = removeDupes(t.Outputs)
for k, v := range t.Contexts {
if v == "" {
delete(t.Contexts, k)
}
}
if len(t.Contexts) == 0 {
t.Contexts = nil
}
}
func (t *Target) Merge(t2 *Target) {
@ -506,6 +563,12 @@ func (t *Target) Merge(t2 *Target) {
}
t.Args[k] = v
}
for k, v := range t2.Contexts {
if t.Contexts == nil {
t.Contexts = map[string]string{}
}
t.Contexts[k] = v
}
for k, v := range t2.Labels {
if t.Labels == nil {
t.Labels = map[string]string{}
@ -565,7 +628,14 @@ func (t *Target) AddOverrides(overrides map[string]Override) error {
t.Args = map[string]string{}
}
t.Args[keys[1]] = value
case "contexts":
if len(keys) != 2 {
return errors.Errorf("contexts require name")
}
if t.Contexts == nil {
t.Contexts = map[string]string{}
}
t.Contexts[keys[1]] = value
case "labels":
if len(keys) != 2 {
return errors.Errorf("labels require name")
@ -693,6 +763,7 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
bi := build.Inputs{
ContextPath: contextPath,
DockerfilePath: dockerfilePath,
NamedContexts: t.Contexts,
}
if t.DockerfileInline != nil {
bi.DockerfileInline = *t.DockerfileInline
@ -811,3 +882,17 @@ func validateTargetName(name string) error {
}
return nil
}
func sliceEqual(s1, s2 []string) bool {
if len(s1) != len(s2) {
return false
}
sort.Strings(s1)
sort.Strings(s2)
for i := range s1 {
if s1[i] != s2[i] {
return false
}
}
return true
}

202
bake/bake_test.go
View File

@ -353,6 +353,208 @@ func TestOverrideMerge(t *testing.T) {
require.Equal(t, "type=registry", m["app"].Outputs[0])
}
func TestReadContexts(t *testing.T) {
fp := File{
Name: "docker-bake.hcl",
Data: []byte(`
target "base" {
contexts = {
foo: "bar"
abc: "def"
}
}
target "app" {
inherits = ["base"]
contexts = {
foo: "baz"
}
}
`),
}
ctx := context.TODO()
m, _, err := ReadTargets(ctx, []File{fp}, []string{"app"}, []string{}, nil)
require.NoError(t, err)
require.Equal(t, 1, len(m))
_, ok := m["app"]
require.True(t, ok)
bo, err := TargetsToBuildOpt(m, &Input{})
require.NoError(t, err)
ctxs := bo["app"].Inputs.NamedContexts
require.Equal(t, 2, len(ctxs))
require.Equal(t, "baz", ctxs["foo"])
require.Equal(t, "def", ctxs["abc"])
m, _, err = ReadTargets(ctx, []File{fp}, []string{"app"}, []string{"app.contexts.foo=bay", "base.contexts.ghi=jkl"}, nil)
require.NoError(t, err)
require.Equal(t, 1, len(m))
_, ok = m["app"]
require.True(t, ok)
bo, err = TargetsToBuildOpt(m, &Input{})
require.NoError(t, err)
ctxs = bo["app"].Inputs.NamedContexts
require.Equal(t, 3, len(ctxs))
require.Equal(t, "bay", ctxs["foo"])
require.Equal(t, "def", ctxs["abc"])
require.Equal(t, "jkl", ctxs["ghi"])
// test resetting base values
m, _, err = ReadTargets(ctx, []File{fp}, []string{"app"}, []string{"app.contexts.foo="}, nil)
require.NoError(t, err)
require.Equal(t, 1, len(m))
_, ok = m["app"]
require.True(t, ok)
bo, err = TargetsToBuildOpt(m, &Input{})
require.NoError(t, err)
ctxs = bo["app"].Inputs.NamedContexts
require.Equal(t, 1, len(ctxs))
require.Equal(t, "def", ctxs["abc"])
}
func TestReadContextFromTargetUnknown(t *testing.T) {
fp := File{
Name: "docker-bake.hcl",
Data: []byte(`
target "base" {
contexts = {
foo: "bar"
abc: "def"
}
}
target "app" {
contexts = {
foo: "baz"
bar: "target:bar"
}
}
`),
}
ctx := context.TODO()
_, _, err := ReadTargets(ctx, []File{fp}, []string{"app"}, []string{}, nil)
require.Error(t, err)
require.Contains(t, err.Error(), "failed to find target bar")
}
func TestReadContextFromTargetChain(t *testing.T) {
ctx := context.TODO()
fp := File{
Name: "docker-bake.hcl",
Data: []byte(`
target "base" {
}
target "mid" {
output = ["foo"]
contexts = {
parent: "target:base"
}
}
target "app" {
contexts = {
foo: "baz"
bar: "target:mid"
}
}
target "unused" {}
`),
}
m, _, err := ReadTargets(ctx, []File{fp}, []string{"app"}, []string{}, nil)
require.NoError(t, err)
require.Equal(t, 3, len(m))
app, ok := m["app"]
require.True(t, ok)
require.Equal(t, 2, len(app.Contexts))
mid, ok := m["mid"]
require.True(t, ok)
require.Equal(t, 0, len(mid.Outputs))
require.Equal(t, 1, len(mid.Contexts))
base, ok := m["base"]
require.True(t, ok)
require.Equal(t, 0, len(base.Contexts))
}
func TestReadContextFromTargetInfiniteLoop(t *testing.T) {
ctx := context.TODO()
fp := File{
Name: "docker-bake.hcl",
Data: []byte(`
target "mid" {
output = ["foo"]
contexts = {
parent: "target:app"
}
}
target "app" {
contexts = {
foo: "baz"
bar: "target:mid"
}
}
`),
}
_, _, err := ReadTargets(ctx, []File{fp}, []string{"app", "mid"}, []string{}, nil)
require.Error(t, err)
require.Contains(t, err.Error(), "infinite loop from")
}
func TestReadContextFromTargetMultiPlatform(t *testing.T) {
ctx := context.TODO()
fp := File{
Name: "docker-bake.hcl",
Data: []byte(`
target "mid" {
output = ["foo"]
platforms = ["linux/amd64", "linux/arm64"]
}
target "app" {
contexts = {
bar: "target:mid"
}
platforms = ["linux/amd64", "linux/arm64"]
}
`),
}
_, _, err := ReadTargets(ctx, []File{fp}, []string{"app"}, []string{}, nil)
require.NoError(t, err)
}
func TestReadContextFromTargetInvalidPlatforms(t *testing.T) {
ctx := context.TODO()
fp := File{
Name: "docker-bake.hcl",
Data: []byte(`
target "mid" {
output = ["foo"]
platforms = ["linux/amd64", "linux/riscv64"]
}
target "app" {
contexts = {
bar: "target:mid"
}
platforms = ["linux/amd64", "linux/arm64"]
}
`),
}
_, _, err := ReadTargets(ctx, []File{fp}, []string{"app"}, []string{}, nil)
require.Error(t, err)
require.Contains(t, err.Error(), "defined for different platforms")
}
func TestReadTargetsDefault(t *testing.T) {
t.Parallel()
ctx := context.TODO()

145
build/build.go
View File

@ -23,6 +23,7 @@ import (
"github.com/docker/buildx/util/imagetools"
"github.com/docker/buildx/util/progress"
"github.com/docker/buildx/util/resolver"
"github.com/docker/buildx/util/waitmap"
"github.com/docker/cli/opts"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api/types"
@ -34,6 +35,7 @@ import (
gateway "github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/upload/uploadprovider"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/apicaps"
"github.com/moby/buildkit/util/entitlements"
"github.com/moby/buildkit/util/progress/progresswriter"
@ -667,8 +669,35 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
}
}
// validate that all links between targets use same drivers
for name := range opt {
dps := m[name]
for _, dp := range dps {
for k, v := range dp.so.FrontendAttrs {
if strings.HasPrefix(k, "context:") && strings.HasPrefix(v, "target:") {
k2 := strings.TrimPrefix(v, "target:")
dps2, ok := m[k2]
if !ok {
return nil, errors.Errorf("failed to find target %s for context %s", k2, strings.TrimPrefix(k, "context:")) // should be validated before already
}
var found bool
for _, dp2 := range dps2 {
if dp2.driverIndex == dp.driverIndex {
found = true
break
}
}
if !found {
return nil, errors.Errorf("failed to use %s as context %s for %s because targets build with different drivers", k2, strings.TrimPrefix(k, "context:"), name)
}
}
}
}
}
resp = map[string]*client.SolveResponse{}
var respMu sync.Mutex
results := waitmap.New()
multiTarget := len(opt) > 1
@ -793,7 +822,6 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
for i, dp := range dps {
so := *dp.so
if multiDriver {
for i, e := range so.Exports {
switch e.Type {
@ -826,14 +854,42 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
pw := progress.WithPrefix(w, k, multiTarget)
c := clients[dp.driverIndex]
pw = progress.ResetTime(pw)
eg.Go(func() error {
if err := waitContextDeps(ctx, dp.driverIndex, results, &so); err != nil {
return err
}
pw = progress.ResetTime(pw)
defer wg.Done()
ch, done := progress.NewChannel(pw)
defer func() { <-done }()
rr, err := c.Solve(ctx, nil, so, ch)
frontendInputs := make(map[string]*pb.Definition)
for key, st := range so.FrontendInputs {
def, err := st.Marshal(ctx)
if err != nil {
return err
}
frontendInputs[key] = def.ToPB()
}
req := gateway.SolveRequest{
Frontend: so.Frontend,
FrontendOpt: so.FrontendAttrs,
FrontendInputs: frontendInputs,
}
so.Frontend = ""
so.FrontendAttrs = nil
so.FrontendInputs = nil
rr, err := c.Build(ctx, so, "buildx", func(ctx context.Context, c gateway.Client) (*gateway.Result, error) {
res, err := c.Solve(ctx, req)
if err != nil {
return nil, err
}
results.Set(resultKey(dp.driverIndex, k), res)
return res, nil
}, ch)
if err != nil {
return err
}
@ -1084,7 +1140,7 @@ func LoadInputs(ctx context.Context, d driver.Driver, inp Inputs, pw progress.Wr
for k, v := range inp.NamedContexts {
target.FrontendAttrs["frontend.caps"] = "moby.buildkit.frontend.contexts+forward"
if urlutil.IsGitURL(v) || urlutil.IsURL(v) || strings.HasPrefix(v, "docker-image://") {
if urlutil.IsGitURL(v) || urlutil.IsURL(v) || strings.HasPrefix(v, "docker-image://") || strings.HasPrefix(v, "target:") {
target.FrontendAttrs["context:"+k] = v
continue
}
@ -1111,6 +1167,83 @@ func LoadInputs(ctx context.Context, d driver.Driver, inp Inputs, pw progress.Wr
return release, nil
}
func resultKey(index int, name string) string {
return fmt.Sprintf("%d-%s", index, name)
}
func waitContextDeps(ctx context.Context, index int, results *waitmap.Map, so *client.SolveOpt) error {
m := map[string]string{}
for k, v := range so.FrontendAttrs {
if strings.HasPrefix(k, "context:") && strings.HasPrefix(v, "target:") {
target := resultKey(index, strings.TrimPrefix(v, "target:"))
m[target] = k
}
}
if len(m) == 0 {
return nil
}
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
res, err := results.Get(ctx, keys...)
if err != nil {
return err
}
for k, v := range m {
r, ok := res[k]
if !ok {
continue
}
rr, ok := r.(*gateway.Result)
if !ok {
return errors.Errorf("invalid result type %T", rr)
}
if so.FrontendAttrs == nil {
so.FrontendAttrs = map[string]string{}
}
if so.FrontendInputs == nil {
so.FrontendInputs = map[string]llb.State{}
}
if len(rr.Refs) > 0 {
for platform, r := range rr.Refs {
st, err := r.ToState()
if err != nil {
return err
}
so.FrontendInputs[k+"::"+platform] = st
so.FrontendAttrs[v+"::"+platform] = "input:" + k + "::" + platform
dt, ok := rr.Metadata["containerimage.config/"+platform]
if !ok {
continue
}
dt, err = json.Marshal(map[string][]byte{"containerimage.config": dt})
if err != nil {
return err
}
so.FrontendAttrs["input-metadata:"+k+"::"+platform] = string(dt)
}
}
if rr.Ref != nil {
st, err := rr.Ref.ToState()
if err != nil {
return err
}
so.FrontendInputs[k] = st
so.FrontendAttrs[v] = "input:" + k
if dt, ok := rr.Metadata["containerimage.config"]; ok {
dt, err = json.Marshal(map[string][]byte{"containerimage.config": dt})
if err != nil {
return err
}
so.FrontendAttrs["input-metadata:"+k] = string(dt)
}
}
}
return nil
}
func notSupported(d driver.Driver, f driver.Feature) error {
return errors.Errorf("%s feature is currently not supported for %s driver. Please switch to a different driver (eg. \"docker buildx create --use\")", f, d.Factory().Name())
}

74
docs/reference/buildx_bake.md
View File

@ -339,7 +339,7 @@ target "db" {
Complete list of valid target fields:
`args`, `cache-from`, `cache-to`, `context`, `dockerfile`, `inherits`, `labels`,
`args`, `cache-from`, `cache-to`, `context`, `contexts`, `dockerfile`, `inherits`, `labels`,
`no-cache`, `output`, `platform`, `pull`, `secrets`, `ssh`, `tags`, `target`
### Global scope attributes
@ -799,6 +799,78 @@ $ docker buildx bake --print app
}
```
### Defining additional build contexts and linking targets
In addition to the main `context` key that defines the build context each target can also define additional named contexts with a map defined with key `contexts`. These values map to the `--build-context` flag in the [build command](buildx_build.md#build-context).
Inside the Dockerfile these contexts can be used with the `FROM` instruction or `--from` flag.
The value can be a local source directory, container image (with docker-image:// prefix), Git URL, HTTP URL or a name of another target in the Bake file (with target: prefix).
#### Pinning alpine image
```Dockerfile
# Dockerfile
FROM alpine
RUN echo "Hello world"
```
```hcl
# docker-bake.hcl
target "app" {
contexts = {
alpine = "docker-image://alpine:3.13"
}
}
```
#### Using a secondary source directory
```Dockerfile
# Dockerfile
FROM scratch AS src
FROM golang
COPY --from=src . .
```
```hcl
# docker-bake.hcl
target "app" {
contexts = {
src = "../path/to/source"
}
}
```
#### Using a result of one target as a base image in another target
To use a result of one target as a build context of another, specity the target name with `target:` prefix.
```Dockerfile
# Dockerfile
FROM baseapp
RUN echo "Hello world"
```
```hcl
# docker-bake.hcl
target "base" {
dockerfile = "baseapp.Dockerfile"
}
target "app" {
contexts = {
baseapp = "target:base"
}
}
```
Please note that in most cases you should just use a single multi-stage Dockerfile with multiple targets for similar behavior. This case is recommended when you have multiple Dockerfiles that can't be easily merged into one.
### Extension field with Compose
[Special extension](https://github.com/compose-spec/compose-spec/blob/master/spec.md#extension)

74
util/waitmap/waitmap.go Normal file
View File

@ -0,0 +1,74 @@
package waitmap
import (
"context"
"sync"
)
type Map struct {
mu sync.RWMutex
m map[string]interface{}
ch map[string]chan struct{}
}
func New() *Map {
return &Map{
m: make(map[string]interface{}),
ch: make(map[string]chan struct{}),
}
}
func (m *Map) Set(key string, value interface{}) {
m.mu.Lock()
defer m.mu.Unlock()
m.m[key] = value
if ch, ok := m.ch[key]; ok {
if ch != nil {
close(ch)
}
}
m.ch[key] = nil
}
func (m *Map) Get(ctx context.Context, keys ...string) (map[string]interface{}, error) {
if len(keys) == 0 {
return map[string]interface{}{}, nil
}
if len(keys) > 1 {
out := make(map[string]interface{})
for _, key := range keys {
mm, err := m.Get(ctx, key)
if err != nil {
return nil, err
}
out[key] = mm[key]
}
return out, nil
}
key := keys[0]
m.mu.Lock()
ch, ok := m.ch[key]
if !ok {
ch = make(chan struct{})
m.ch[key] = ch
}
if ch != nil {
m.mu.Unlock()
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-ch:
m.mu.Lock()
}
}
res := m.m[key]
m.mu.Unlock()
return map[string]interface{}{key: res}, nil
}

64
util/waitmap/waitmap_test.go Normal file
View File

@ -0,0 +1,64 @@
package waitmap
import (
"context"
"errors"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestGetAfter(t *testing.T) {
m := New()
m.Set("foo", "bar")
m.Set("bar", "baz")
ctx := context.TODO()
v, err := m.Get(ctx, "foo", "bar")
require.NoError(t, err)
require.Equal(t, 2, len(v))
require.Equal(t, "bar", v["foo"])
require.Equal(t, "baz", v["bar"])
v, err = m.Get(ctx, "foo")
require.NoError(t, err)
require.Equal(t, 1, len(v))
require.Equal(t, "bar", v["foo"])
}
func TestTimeout(t *testing.T) {
m := New()
m.Set("foo", "bar")
ctx, cancel := context.WithTimeout(context.TODO(), 100*time.Millisecond)
defer cancel()
_, err := m.Get(ctx, "bar")
require.Error(t, err)
require.True(t, errors.Is(err, context.DeadlineExceeded))
}
func TestBlocking(t *testing.T) {
m := New()
m.Set("foo", "bar")
go func() {
time.Sleep(100 * time.Millisecond)
m.Set("bar", "baz")
time.Sleep(50 * time.Millisecond)
m.Set("baz", "abc")
}()
ctx := context.TODO()
v, err := m.Get(ctx, "foo", "bar", "baz")
require.NoError(t, err)
require.Equal(t, 3, len(v))
require.Equal(t, "bar", v["foo"])
require.Equal(t, "baz", v["bar"])
require.Equal(t, "abc", v["baz"])
}