docker
/
buildx
mirror of https://github.com/docker/buildx.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #398 from tonistiigi/remote-bake 

bake: remote inputs support
This commit is contained in:
Tõnis Tiigi 2020-12-07 23:07:24 -08:00 committed by GitHub
parent 0360668cc1 40fad4bbb5
commit 080e9981c7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 692 additions and 339 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
bake/bake.go
View File

@ -5,6 +5,7 @@ import (
"io/ioutil"
"os"
"path"
"regexp"
"strconv"
"strings"
@ -12,14 +13,55 @@ import (
"github.com/docker/buildx/util/platformutil"
"github.com/docker/docker/pkg/urlutil"
hcl "github.com/hashicorp/hcl/v2"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/session/auth/authprovider"
"github.com/pkg/errors"
)
func ReadTargets(ctx context.Context, files, targets, overrides []string) (map[string]*Target, error) {
var httpPrefix = regexp.MustCompile(`^https?://`)
var gitURLPathWithFragmentSuffix = regexp.MustCompile(`\.git(?:#.+)?$`)
type File struct {
Name string
Data []byte
}
func defaultFilenames() []string {
return []string{
"docker-compose.yml", // support app
"docker-compose.yaml", // support app
"docker-bake.json",
"docker-bake.override.json",
"docker-bake.hcl",
"docker-bake.override.hcl",
}
}
func ReadLocalFiles(names []string) ([]File, error) {
isDefault := false
if len(names) == 0 {
isDefault = true
names = defaultFilenames()
}
out := make([]File, 0, len(names))
for _, n := range names {
dt, err := ioutil.ReadFile(n)
if err != nil {
if isDefault && errors.Is(err, os.ErrNotExist) {
continue
}
return nil, err
}
out = append(out, File{Name: n, Data: dt})
}
return out, nil
}
func ReadTargets(ctx context.Context, files []File, targets, overrides []string) (map[string]*Target, error) {
var c Config
for _, f := range files {
cfg, err := ParseFile(f)
cfg, err := ParseFile(f.Data, f.Name)
if err != nil {
return nil, err
}
@ -44,12 +86,7 @@ func ReadTargets(ctx context.Context, files, targets, overrides []string) (map[s
return m, nil
}
func ParseFile(fn string) (*Config, error) {
dt, err := ioutil.ReadFile(fn)
if err != nil {
return nil, err
}
func ParseFile(dt []byte, fn string) (*Config, error) {
fnl := strings.ToLower(fn)
if strings.HasSuffix(fnl, ".yml") || strings.HasSuffix(fnl, ".yaml") {
return ParseCompose(dt)
@ -336,20 +373,22 @@ type Target struct {
// Inherits is the only field that cannot be overridden with --set
Inherits []string `json:"inherits,omitempty" hcl:"inherits,optional"`
Context *string `json:"context,omitempty" hcl:"context,optional"`
Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional"`
Args map[string]string `json:"args,omitempty" hcl:"args,optional"`
Labels map[string]string `json:"labels,omitempty" hcl:"labels,optional"`
Tags []string `json:"tags,omitempty" hcl:"tags,optional"`
CacheFrom []string `json:"cache-from,omitempty" hcl:"cache-from,optional"`
CacheTo []string `json:"cache-to,omitempty" hcl:"cache-to,optional"`
Target *string `json:"target,omitempty" hcl:"target,optional"`
Secrets []string `json:"secret,omitempty" hcl:"secret,optional"`
SSH []string `json:"ssh,omitempty" hcl:"ssh,optional"`
Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional"`
Outputs []string `json:"output,omitempty" hcl:"output,optional"`
Pull *bool `json:"pull,omitempty" hcl:"pull,optional"`
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional"`
Context *string `json:"context,omitempty" hcl:"context,optional"`
Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional"`
DockerfileInline *string `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional"`
Args map[string]string `json:"args,omitempty" hcl:"args,optional"`
Labels map[string]string `json:"labels,omitempty" hcl:"labels,optional"`
Tags []string `json:"tags,omitempty" hcl:"tags,optional"`
CacheFrom []string `json:"cache-from,omitempty" hcl:"cache-from,optional"`
CacheTo []string `json:"cache-to,omitempty" hcl:"cache-to,optional"`
Target *string `json:"target,omitempty" hcl:"target,optional"`
Secrets []string `json:"secret,omitempty" hcl:"secret,optional"`
SSH []string `json:"ssh,omitempty" hcl:"ssh,optional"`
Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional"`
Outputs []string `json:"output,omitempty" hcl:"output,optional"`
Pull *bool `json:"pull,omitempty" hcl:"pull,optional"`
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional"`
// IMPORTANT: if you add more fields here, do not forget to update newOverrides and README.
}
@ -363,10 +402,10 @@ func (t *Target) normalize() {
t.Outputs = removeDupes(t.Outputs)
}
func TargetsToBuildOpt(m map[string]*Target) (map[string]build.Options, error) {
func TargetsToBuildOpt(m map[string]*Target, inp *Input) (map[string]build.Options, error) {
m2 := make(map[string]build.Options, len(m))
for k, v := range m {
bo, err := toBuildOpt(v)
bo, err := toBuildOpt(v, inp)
if err != nil {
return nil, err
}
@ -375,7 +414,19 @@ func TargetsToBuildOpt(m map[string]*Target) (map[string]build.Options, error) {
return m2, nil
}
func toBuildOpt(t *Target) (*build.Options, error) {
func updateContext(t *build.Inputs, inp *Input) {
if inp == nil || inp.State == nil {
return
}
if t.ContextPath == "." {
t.ContextPath = inp.URL
return
}
st := llb.Scratch().File(llb.Copy(*inp.State, t.ContextPath, "/"), llb.WithCustomNamef("set context to %s", t.ContextPath))
t.ContextState = &st
}
func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
if v := t.Context; v != nil && *v == "-" {
return nil, errors.Errorf("context from stdin not allowed in bake")
}
@ -387,6 +438,7 @@ func toBuildOpt(t *Target) (*build.Options, error) {
if t.Context != nil {
contextPath = *t.Context
}
contextPath = path.Clean(contextPath)
dockerfilePath := "Dockerfile"
if t.Dockerfile != nil {
dockerfilePath = *t.Dockerfile
@ -405,11 +457,17 @@ func toBuildOpt(t *Target) (*build.Options, error) {
pull = *t.Pull
}
bi := build.Inputs{
ContextPath: contextPath,
DockerfilePath: dockerfilePath,
}
if t.DockerfileInline != nil {
bi.DockerfileInline = *t.DockerfileInline
}
updateContext(&bi, inp)
bo := &build.Options{
Inputs: build.Inputs{
ContextPath: contextPath,
DockerfilePath: dockerfilePath,
},
Inputs: bi,
Tags: t.Tags,
BuildArgs: t.Args,
Labels: t.Labels,
@ -473,6 +531,9 @@ func merge(t1, t2 *Target) *Target {
if t2.Dockerfile != nil {
t1.Dockerfile = t2.Dockerfile
}
if t2.DockerfileInline != nil {
t1.DockerfileInline = t2.DockerfileInline
}
for k, v := range t2.Args {
if t1.Args == nil {
t1.Args = map[string]string{}

61
bake/bake_test.go
View File

@ -2,9 +2,7 @@ package bake
import (
"context"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
@ -12,12 +10,10 @@ import (
func TestReadTargets(t *testing.T) {
t.Parallel()
tmpdir, err := ioutil.TempDir("", "bake")
require.NoError(t, err)
defer os.RemoveAll(tmpdir)
fp := filepath.Join(tmpdir, "config.hcl")
err = ioutil.WriteFile(fp, []byte(`
fp := File{
Name: "config.hcl",
Data: []byte(`
target "webDEP" {
args = {
VAR_INHERITED = "webDEP"
@ -32,13 +28,13 @@ target "webapp" {
VAR_BOTH = "webapp"
}
inherits = ["webDEP"]
}`), 0600)
require.NoError(t, err)
}`),
}
ctx := context.TODO()
t.Run("NoOverrides", func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, nil)
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, nil)
require.NoError(t, err)
require.Equal(t, 1, len(m))
@ -50,7 +46,7 @@ target "webapp" {
})
t.Run("InvalidTargetOverrides", func(t *testing.T) {
_, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"nosuchtarget.context=foo"})
_, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"nosuchtarget.context=foo"})
require.NotNil(t, err)
require.Equal(t, err.Error(), "could not find any target matching 'nosuchtarget'")
})
@ -60,7 +56,7 @@ target "webapp" {
os.Setenv("VAR_FROMENV"+t.Name(), "fromEnv")
defer os.Unsetenv("VAR_FROM_ENV" + t.Name())
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{
"webapp.args.VAR_UNSET",
"webapp.args.VAR_EMPTY=",
"webapp.args.VAR_SET=bananas",
@ -89,7 +85,7 @@ target "webapp" {
// building leaf but overriding parent fields
t.Run("parent", func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{
"webDEP.args.VAR_INHERITED=override",
"webDEP.args.VAR_BOTH=override",
})
@ -100,23 +96,23 @@ target "webapp" {
})
t.Run("ContextOverride", func(t *testing.T) {
_, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.context"})
_, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.context"})
require.NotNil(t, err)
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.context=foo"})
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.context=foo"})
require.NoError(t, err)
require.Equal(t, "foo", *m["webapp"].Context)
})
t.Run("NoCacheOverride", func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.no-cache=false"})
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.no-cache=false"})
require.NoError(t, err)
require.Equal(t, false, *m["webapp"].NoCache)
})
t.Run("PullOverride", func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, []string{"webapp"}, []string{"webapp.pull=false"})
m, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.pull=false"})
require.NoError(t, err)
require.Equal(t, false, *m["webapp"].Pull)
})
@ -176,7 +172,7 @@ target "webapp" {
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
m, err := ReadTargets(ctx, []string{fp}, test.targets, test.overrides)
m, err := ReadTargets(ctx, []File{fp}, test.targets, test.overrides)
test.check(t, m, err)
})
}
@ -185,14 +181,11 @@ target "webapp" {
func TestReadTargetsCompose(t *testing.T) {
t.Parallel()
tmpdir, err := ioutil.TempDir("", "bake")
require.NoError(t, err)
defer os.RemoveAll(tmpdir)
fp := filepath.Join(tmpdir, "docker-compose.yml")
err = ioutil.WriteFile(fp, []byte(`
version: "3"
fp := File{
Name: "docker-compose.yml",
Data: []byte(
`version: "3"
services:
db:
build: .
@ -203,13 +196,13 @@ services:
dockerfile: Dockerfile.webapp
args:
buildno: 1
`), 0600)
require.NoError(t, err)
fp2 := filepath.Join(tmpdir, "docker-compose2.yml")
err = ioutil.WriteFile(fp2, []byte(`
version: "3"
`),
}
fp2 := File{
Name: "docker-compose2.yml",
Data: []byte(
`version: "3"
services:
newservice:
build: .
@ -217,12 +210,12 @@ services:
build:
args:
buildno2: 12
`), 0600)
require.NoError(t, err)
`),
}
ctx := context.TODO()
m, err := ReadTargets(ctx, []string{fp, fp2}, []string{"default"}, nil)
m, err := ReadTargets(ctx, []File{fp, fp2}, []string{"default"}, nil)
require.NoError(t, err)
require.Equal(t, 3, len(m))

32
bake/hcl.go
View File

@ -107,6 +107,20 @@ type staticConfig struct {
}
func ParseHCL(dt []byte, fn string) (_ *Config, err error) {
if strings.HasSuffix(fn, ".json") || strings.HasSuffix(fn, ".hcl") {
return parseHCL(dt, fn)
}
cfg, err := parseHCL(dt, fn+".hcl")
if err != nil {
cfg2, err2 := parseHCL(dt, fn+".json")
if err2 == nil {
return cfg2, nil
}
}
return cfg, err
}
func parseHCL(dt []byte, fn string) (_ *Config, err error) {
defer func() {
err = formatHCLError(dt, err)
}()
@ -192,15 +206,17 @@ func formatHCLError(dt []byte, err error) error {
if d.Severity != hcl.DiagError {
continue
}
src := errdefs.Source{
Info: &pb.SourceInfo{
Filename: d.Subject.Filename,
Data: dt,
},
Ranges: []*pb.Range{toErrRange(d.Subject)},
if d.Subject != nil {
src := errdefs.Source{
Info: &pb.SourceInfo{
Filename: d.Subject.Filename,
Data: dt,
},
Ranges: []*pb.Range{toErrRange(d.Subject)},
}
err = errdefs.WithSource(err, src)
break
}
err = errdefs.WithSource(err, src)
break
}
return err
}

236
bake/remote.go Normal file
View File

@ -0,0 +1,236 @@
package bake
import (
"archive/tar"
"bytes"
"context"
"strings"
"github.com/docker/buildx/build"
"github.com/docker/buildx/driver"
"github.com/docker/buildx/util/progress"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
gwclient "github.com/moby/buildkit/frontend/gateway/client"
"github.com/pkg/errors"
)
type Input struct {
State *llb.State
URL string
}
func ReadRemoteFiles(ctx context.Context, dis []build.DriverInfo, url string, names []string, pw progress.Writer) ([]File, *Input, error) {
st, filename, ok := detectHttpContext(url)
if !ok {
st, ok = detectGitContext(url)
if !ok {
return nil, nil, errors.Errorf("not url context")
}
}
inp := &Input{State: st, URL: url}
var files []File
var di *build.DriverInfo
for _, d := range dis {
if d.Err == nil {
di = &d
continue
}
}
if di == nil {
return nil, nil, nil
}
c, err := driver.Boot(ctx, di.Driver, pw)
if err != nil {
return nil, nil, err
}
ch, done := progress.NewChannel(pw)
defer func() { <-done }()
_, err = c.Build(ctx, client.SolveOpt{}, "buildx", func(ctx context.Context, c gwclient.Client) (*gwclient.Result, error) {
def, err := st.Marshal(ctx)
if err != nil {
return nil, err
}
res, err := c.Solve(ctx, gwclient.SolveRequest{
Definition: def.ToPB(),
})
if err != nil {
return nil, err
}
ref, err := res.SingleRef()
if err != nil {
return nil, err
}
if filename != "" {
files, err = filesFromURLRef(ctx, c, ref, inp, filename, names)
} else {
files, err = filesFromRef(ctx, ref, names)
}
return nil, err
}, ch)
if err != nil {
return nil, nil, err
}
return files, inp, nil
}
func IsRemoteURL(url string) bool {
if _, _, ok := detectHttpContext(url); ok {
return true
}
if _, ok := detectGitContext(url); ok {
return true
}
return false
}
func detectHttpContext(url string) (*llb.State, string, bool) {
if httpPrefix.MatchString(url) {
httpContext := llb.HTTP(url, llb.Filename("context"), llb.WithCustomName("[internal] load remote build context"))
return &httpContext, "context", true
}
return nil, "", false
}
func detectGitContext(ref string) (*llb.State, bool) {
found := false
if httpPrefix.MatchString(ref) && gitURLPathWithFragmentSuffix.MatchString(ref) {
found = true
}
for _, prefix := range []string{"git://", "github.com/", "git@"} {
if strings.HasPrefix(ref, prefix) {
found = true
break
}
}
if !found {
return nil, false
}
parts := strings.SplitN(ref, "#", 2)
branch := ""
if len(parts) > 1 {
branch = parts[1]
}
gitOpts := []llb.GitOption{llb.WithCustomName("[internal] load git source " + ref)}
st := llb.Git(parts[0], branch, gitOpts...)
return &st, true
}
func isArchive(header []byte) bool {
for _, m := range [][]byte{
{0x42, 0x5A, 0x68}, // bzip2
{0x1F, 0x8B, 0x08}, // gzip
{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, // xz
} {
if len(header) < len(m) {
continue
}
if bytes.Equal(m, header[:len(m)]) {
return true
}
}
r := tar.NewReader(bytes.NewBuffer(header))
_, err := r.Next()
return err == nil
}
func filesFromURLRef(ctx context.Context, c gwclient.Client, ref gwclient.Reference, inp *Input, filename string, names []string) ([]File, error) {
stat, err := ref.StatFile(ctx, gwclient.StatRequest{Path: filename})
if err != nil {
return nil, err
}
dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{
Filename: filename,
Range: &gwclient.FileRange{
Length: 1024,
},
})
if err != nil {
return nil, err
}
if isArchive(dt) {
bc := llb.Scratch().File(llb.Copy(inp.State, filename, "/", &llb.CopyInfo{
AttemptUnpack: true,
}))
inp.State = &bc
inp.URL = ""
def, err := bc.Marshal(ctx)
if err != nil {
return nil, err
}
res, err := c.Solve(ctx, gwclient.SolveRequest{
Definition: def.ToPB(),
})
if err != nil {
return nil, err
}
ref, err := res.SingleRef()
if err != nil {
return nil, err
}
return filesFromRef(ctx, ref, names)
}
inp.State = nil
name := inp.URL
inp.URL = ""
if len(dt) > stat.Size() {
if stat.Size() > 1024*512 {
return nil, errors.Errorf("non-archive definition URL bigger than maximum allowed size")
}
dt, err = ref.ReadFile(ctx, gwclient.ReadRequest{
Filename: filename,
})
if err != nil {
return nil, err
}
}
return []File{{Name: name, Data: dt}}, nil
}
func filesFromRef(ctx context.Context, ref gwclient.Reference, names []string) ([]File, error) {
// TODO: auto-remove parent dir in needed
var files []File
isDefault := false
if len(names) == 0 {
isDefault = true
names = defaultFilenames()
}
for _, name := range names {
_, err := ref.StatFile(ctx, gwclient.StatRequest{Path: name})
if err != nil {
if isDefault {
continue
}
return nil, err
}
dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{Filename: name})
if err != nil {
return nil, err
}
files = append(files, File{Name: name, Data: dt})
}
return files, nil
}

119
build/build.go
View File

@ -22,6 +22,7 @@ import (
dockerclient "github.com/docker/docker/client"
"github.com/docker/docker/pkg/urlutil"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/upload/uploadprovider"
"github.com/moby/buildkit/util/entitlements"
@ -61,9 +62,11 @@ type Options struct {
}
type Inputs struct {
ContextPath string
DockerfilePath string
InStream io.Reader
ContextPath string
DockerfilePath string
InStream io.Reader
ContextState *llb.State
DockerfileInline string
}
type DriverInfo struct {
@ -173,7 +176,6 @@ func splitToDriverPairs(availablePlatforms map[string]int, opt map[string]Option
}
func resolveDrivers(ctx context.Context, drivers []DriverInfo, auth Auth, opt map[string]Options, pw progress.Writer) (map[string][]driverPair, []*client.Client, error) {
availablePlatforms := map[string]int{}
for i, d := range drivers {
for _, p := range d.Platform {
@ -278,14 +280,7 @@ func toRepoOnly(in string) (string, error) {
return strings.Join(out, ","), nil
}
func isDefaultMobyDriver(d driver.Driver) bool {
_, ok := d.(interface {
IsDefaultMobyDriver()
})
return ok
}
func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCallback) (solveOpt *client.SolveOpt, release func(), err error) {
func toSolveOpt(ctx context.Context, d driver.Driver, multiDriver bool, opt Options, pw progress.Writer, dl dockerLoadCallback) (solveOpt *client.SolveOpt, release func(), err error) {
defers := make([]func(), 0, 2)
releaseF := func() {
for _, f := range defers {
@ -336,15 +331,11 @@ func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCal
so.FrontendAttrs["multi-platform"] = "true"
}
_, isDefaultMobyDriver := d.(interface {
IsDefaultMobyDriver()
})
switch len(opt.Exports) {
case 1:
// valid
case 0:
if isDefaultMobyDriver && !noDefaultLoad() {
if d.IsMobyDriver() && !noDefaultLoad() {
// backwards compat for docker driver only:
// this ensures the build results in a docker image.
opt.Exports = []client.ExportEntry{{Type: "image", Attrs: map[string]string{}}}
@ -398,7 +389,7 @@ func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCal
}
if e.Type == "docker" {
if e.Output == nil {
if isDefaultMobyDriver {
if d.IsMobyDriver() {
e.Type = "image"
} else {
w, cancel, err := dl(e.Attrs["context"])
@ -412,7 +403,7 @@ func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCal
return nil, nil, notSupported(d, driver.DockerExporter)
}
}
if e.Type == "image" && isDefaultMobyDriver {
if e.Type == "image" && d.IsMobyDriver() {
opt.Exports[i].Type = "moby"
if e.Attrs["push"] != "" {
if ok, _ := strconv.ParseBool(e.Attrs["push"]); ok {
@ -425,7 +416,7 @@ func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCal
so.Exports = opt.Exports
so.Session = opt.Session
releaseLoad, err := LoadInputs(opt.Inputs, &so)
releaseLoad, err := LoadInputs(ctx, d, opt.Inputs, pw, &so)
if err != nil {
return nil, nil, err
}
@ -479,7 +470,7 @@ func toSolveOpt(d driver.Driver, multiDriver bool, opt Options, dl dockerLoadCal
return &so, releaseF, nil
}
func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, auth Auth, pw progress.Writer) (resp map[string]*client.SolveResponse, err error) {
func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, auth Auth, w progress.Writer) (resp map[string]*client.SolveResponse, err error) {
if len(drivers) == 0 {
return nil, errors.Errorf("driver required for build")
}
@ -491,7 +482,7 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
var noMobyDriver driver.Driver
for _, d := range drivers {
if !isDefaultMobyDriver(d.Driver) {
if !d.Driver.IsMobyDriver() {
noMobyDriver = d.Driver
break
}
@ -506,10 +497,8 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
}
}
m, clients, err := resolveDrivers(ctx, drivers, auth, opt, pw)
m, clients, err := resolveDrivers(ctx, drivers, auth, opt, w)
if err != nil {
close(pw.Status())
<-pw.Done()
return nil, err
}
@ -522,7 +511,6 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
}
}()
mw := progress.NewMultiWriter(pw)
eg, ctx := errgroup.WithContext(ctx)
for k, opt := range opt {
@ -530,8 +518,8 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
for i, dp := range m[k] {
d := drivers[dp.driverIndex].Driver
opt.Platforms = dp.platforms
so, release, err := toSolveOpt(d, multiDriver, opt, func(name string) (io.WriteCloser, func(), error) {
return newDockerLoader(ctx, docker, name, mw)
so, release, err := toSolveOpt(ctx, d, multiDriver, opt, w, func(name string) (io.WriteCloser, func(), error) {
return newDockerLoader(ctx, docker, name, w)
})
if err != nil {
return nil, err
@ -559,8 +547,7 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
var pushNames string
eg.Go(func() error {
pw := mw.WithPrefix("default", false)
defer close(pw.Status())
pw := progress.WithPrefix(w, "default", false)
wg.Wait()
select {
case <-ctx.Done():
@ -663,23 +650,17 @@ func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, do
}
func(i int, dp driverPair, so client.SolveOpt) {
pw := mw.WithPrefix(k, multiTarget)
pw := progress.WithPrefix(w, k, multiTarget)
c := clients[dp.driverIndex]
var statusCh chan *client.SolveStatus
if pw != nil {
pw = progress.ResetTime(pw)
statusCh = pw.Status()
eg.Go(func() error {
<-pw.Done()
return pw.Err()
})
}
pw = progress.ResetTime(pw)
eg.Go(func() error {
defer wg.Done()
rr, err := c.Solve(ctx, nil, so, statusCh)
ch, done := progress.NewChannel(pw)
defer func() { <-done }()
rr, err := c.Solve(ctx, nil, so, ch)
if err != nil {
return err
}
@ -720,7 +701,7 @@ func createTempDockerfile(r io.Reader) (string, error) {
return dir, err
}
func LoadInputs(inp Inputs, target *client.SolveOpt) (func(), error) {
func LoadInputs(ctx context.Context, d driver.Driver, inp Inputs, pw progress.Writer, target *client.SolveOpt) (func(), error) {
if inp.ContextPath == "" {
return nil, errors.New("please specify build context (e.g. \".\" for the current directory)")
}
@ -736,6 +717,12 @@ func LoadInputs(inp Inputs, target *client.SolveOpt) (func(), error) {
)
switch {
case inp.ContextState != nil:
if target.FrontendInputs == nil {
target.FrontendInputs = make(map[string]llb.State)
}
target.FrontendInputs["context"] = *inp.ContextState
target.FrontendInputs["dockerfile"] = *inp.ContextState
case inp.ContextPath == "-":
if inp.DockerfilePath == "-" {
return nil, errStdinConflict
@ -746,21 +733,22 @@ func LoadInputs(inp Inputs, target *client.SolveOpt) (func(), error) {
if err != nil && err != io.EOF {
return nil, errors.Wrap(err, "failed to peek context header from STDIN")
}
if isArchive(magic) {
// stdin is context
up := uploadprovider.New()
target.FrontendAttrs["context"] = up.Add(buf)
target.Session = append(target.Session, up)
} else {
if inp.DockerfilePath != "" {
return nil, errDockerfileConflict
if !(err == io.EOF && len(magic) == 0) {
if isArchive(magic) {
// stdin is context
up := uploadprovider.New()
target.FrontendAttrs["context"] = up.Add(buf)
target.Session = append(target.Session, up)
} else {
if inp.DockerfilePath != "" {
return nil, errDockerfileConflict
}
// stdin is dockerfile
dockerfileReader = buf
inp.ContextPath, _ = ioutil.TempDir("", "empty-dir")
toRemove = append(toRemove, inp.ContextPath)
target.LocalDirs["context"] = inp.ContextPath
}
// stdin is dockerfile
dockerfileReader = buf
inp.ContextPath, _ = ioutil.TempDir("", "empty-dir")
toRemove = append(toRemove, inp.ContextPath)
target.LocalDirs["context"] = inp.ContextPath
}
case isLocalDir(inp.ContextPath):
@ -784,6 +772,10 @@ func LoadInputs(inp Inputs, target *client.SolveOpt) (func(), error) {
return nil, errors.Errorf("unable to prepare context: path %q not found", inp.ContextPath)
}
if inp.DockerfileInline != "" {
dockerfileReader = strings.NewReader(inp.DockerfileInline)
}
if dockerfileReader != nil {
dockerfileDir, err = createTempDockerfile(dockerfileReader)
if err != nil {
@ -791,6 +783,17 @@ func LoadInputs(inp Inputs, target *client.SolveOpt) (func(), error) {
}
toRemove = append(toRemove, dockerfileDir)
dockerfileName = "Dockerfile"
target.FrontendAttrs["dockerfilekey"] = "dockerfile"
}
if urlutil.IsURL(inp.DockerfilePath) {
dockerfileDir, err = createTempDockerfileFromURL(ctx, d, inp.DockerfilePath, pw)
if err != nil {
return nil, err
}
toRemove = append(toRemove, dockerfileDir)
dockerfileName = "Dockerfile"
target.FrontendAttrs["dockerfilekey"] = "dockerfile"
delete(target.FrontendInputs, "dockerfile")
}
if dockerfileName == "" {
@ -818,7 +821,7 @@ func notSupported(d driver.Driver, f driver.Feature) error {
type dockerLoadCallback func(name string) (io.WriteCloser, func(), error)
func newDockerLoader(ctx context.Context, d DockerAPI, name string, mw *progress.MultiWriter) (io.WriteCloser, func(), error) {
func newDockerLoader(ctx context.Context, d DockerAPI, name string, status progress.Writer) (io.WriteCloser, func(), error) {
c, err := d.DockerAPI(name)
if err != nil {
return nil, nil, err
@ -841,7 +844,7 @@ func newDockerLoader(ctx context.Context, d DockerAPI, name string, mw *progress
w.mu.Unlock()
return
}
prog := mw.WithPrefix("", false)
prog := progress.WithPrefix(status, "", false)
progress.FromReader(prog, "importing to docker", resp.Body)
},
done: done,

71
build/url.go Normal file
View File

@ -0,0 +1,71 @@
package build
import (
"context"
"io/ioutil"
"path/filepath"
"github.com/docker/buildx/driver"
"github.com/docker/buildx/util/progress"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
gwclient "github.com/moby/buildkit/frontend/gateway/client"
"github.com/pkg/errors"
)
func createTempDockerfileFromURL(ctx context.Context, d driver.Driver, url string, pw progress.Writer) (string, error) {
c, err := driver.Boot(ctx, d, pw)
if err != nil {
return "", err
}
var out string
ch, done := progress.NewChannel(pw)
defer func() { <-done }()
_, err = c.Build(ctx, client.SolveOpt{}, "buildx", func(ctx context.Context, c gwclient.Client) (*gwclient.Result, error) {
def, err := llb.HTTP(url, llb.Filename("Dockerfile"), llb.WithCustomNamef("[internal] load %s", url)).Marshal(ctx)
if err != nil {
return nil, err
}
res, err := c.Solve(ctx, gwclient.SolveRequest{
Definition: def.ToPB(),
})
if err != nil {
return nil, err
}
ref, err := res.SingleRef()
if err != nil {
return nil, err
}
stat, err := ref.StatFile(ctx, gwclient.StatRequest{
Path: "Dockerfile",
})
if err != nil {
return nil, err
}
if stat.Size() > 512*1024 {
return nil, errors.Errorf("Dockerfile %s bigger than allowed max size", url)
}
dt, err := ref.ReadFile(ctx, gwclient.ReadRequest{
Filename: "Dockerfile",
})
if err != nil {
return nil, err
}
dir, err := ioutil.TempDir("", "buildx")
if err != nil {
return nil, err
}
if err := ioutil.WriteFile(filepath.Join(dir, "Dockerfile"), dt, 0600); err != nil {
return nil, err
}
out = dir
return nil, nil
}, ch)
if err != nil {
return "", err
}
return out, nil
}

85
commands/bake.go
View File

@ -1,11 +1,14 @@
package commands
import (
"context"
"encoding/json"
"fmt"
"os"
"github.com/docker/buildx/bake"
"github.com/docker/buildx/build"
"github.com/docker/buildx/util/progress"
"github.com/docker/cli/cli/command"
"github.com/moby/buildkit/util/appcontext"
"github.com/pkg/errors"
@ -19,18 +22,16 @@ type bakeOptions struct {
commonOptions
}
func runBake(dockerCli command.Cli, targets []string, in bakeOptions) error {
func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error) {
ctx := appcontext.Context()
if len(in.files) == 0 {
files, err := defaultFiles()
if err != nil {
return err
var url string
if len(targets) > 0 {
if bake.IsRemoteURL(targets[0]) {
url = targets[0]
targets = targets[1:]
}
if len(files) == 0 {
return errors.Errorf("no docker-compose.yml or docker-bake.hcl found, specify build file with -f/--file")
}
in.files = files
}
if len(targets) == 0 {
@ -52,8 +53,38 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) error {
if in.pull != nil {
overrides = append(overrides, fmt.Sprintf("*.pull=%t", *in.pull))
}
contextPathHash, _ := os.Getwd()
m, err := bake.ReadTargets(ctx, in.files, targets, overrides)
ctx2, cancel := context.WithCancel(context.TODO())
defer cancel()
printer := progress.NewPrinter(ctx2, os.Stderr, in.progress)
defer func() {
if printer != nil {
err1 := printer.Wait()
if err == nil {
err = err1
}
}
}()
dis, err := getInstanceOrDefault(ctx, dockerCli, in.builder, contextPathHash)
if err != nil {
return err
}
var files []bake.File
var inp *bake.Input
if url != "" {
files, inp, err = bake.ReadRemoteFiles(ctx, dis, url, in.files, printer)
} else {
files, err = bake.ReadLocalFiles(in.files)
}
if err != nil {
return err
}
m, err := bake.ReadTargets(ctx, files, targets, overrides)
if err != nil {
return err
}
@ -63,40 +94,22 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) error {
if err != nil {
return err
}
err = printer.Wait()
printer = nil
if err != nil {
return err
}
fmt.Fprintln(dockerCli.Out(), string(dt))
return nil
}
bo, err := bake.TargetsToBuildOpt(m)
bo, err := bake.TargetsToBuildOpt(m, inp)
if err != nil {
return err
}
contextPathHash, _ := os.Getwd()
return buildTargets(ctx, dockerCli, bo, in.progress, contextPathHash, in.builder)
}
func defaultFiles() ([]string, error) {
fns := []string{
"docker-compose.yml", // support app
"docker-compose.yaml", // support app
"docker-bake.json",
"docker-bake.override.json",
"docker-bake.hcl",
"docker-bake.override.hcl",
}
out := make([]string, 0, len(fns))
for _, f := range fns {
if _, err := os.Stat(f); err != nil {
if os.IsNotExist(errors.Cause(err)) {
continue
}
return nil, err
}
out = append(out, f)
}
return out, nil
_, err = build.Build(ctx, dis, bo, dockerAPI(dockerCli), dockerCli.ConfigFile(), printer)
return err
}
func bakeCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {

9
commands/build.go
View File

@ -203,9 +203,14 @@ func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]bu
ctx2, cancel := context.WithCancel(context.TODO())
defer cancel()
pw := progress.NewPrinter(ctx2, os.Stderr, progressMode)
printer := progress.NewPrinter(ctx2, os.Stderr, progressMode)
_, err = build.Build(ctx, dis, opts, dockerAPI(dockerCli), dockerCli.ConfigFile(), printer)
err1 := printer.Wait()
if err == nil {
err = err1
}
_, err = build.Build(ctx, dis, opts, dockerAPI(dockerCli), dockerCli.ConfigFile(), pw)
return err
}

16
commands/inspect.go
View File

@ -171,25 +171,27 @@ func boot(ctx context.Context, ngi *nginfo, dockerCli command.Cli) (bool, error)
return false, nil
}
pw := progress.NewPrinter(context.TODO(), os.Stderr, "auto")
mw := progress.NewMultiWriter(pw)
printer := progress.NewPrinter(context.TODO(), os.Stderr, "auto")
eg, _ := errgroup.WithContext(ctx)
for _, idx := range toBoot {
func(idx int) {
eg.Go(func() error {
pw := mw.WithPrefix(ngi.ng.Nodes[idx].Name, len(toBoot) > 1)
pw := progress.WithPrefix(printer, ngi.ng.Nodes[idx].Name, len(toBoot) > 1)
_, err := driver.Boot(ctx, ngi.drivers[idx].di.Driver, pw)
if err != nil {
ngi.drivers[idx].err = err
}
close(pw.Status())
<-pw.Done()
return nil
})
}(idx)
}
return true, eg.Wait()
err := eg.Wait()
err1 := printer.Wait()
if err == nil {
err = err1
}
return true, err
}

4
driver/docker-container/driver.go
View File

@ -32,6 +32,10 @@ type Driver struct {
env []string
}
func (d *Driver) IsMobyDriver() bool {
return false
}
func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error {
return progress.Wrap("[internal] booting buildkit", l, func(sub progress.SubLogger) error {
_, err := d.DockerAPI.ContainerInspect(ctx, d.Name)

4
driver/docker/driver.go
View File

@ -57,4 +57,6 @@ func (d *Driver) Factory() driver.Factory {
return d.factory
}
func (d *Driver) IsDefaultMobyDriver() {}
func (d *Driver) IsMobyDriver() bool {
return true
}

7
driver/driver.go
View File

@ -57,6 +57,7 @@ type Driver interface {
Rm(ctx context.Context, force bool) error
Client(ctx context.Context) (*client.Client, error)
Features() map[Feature]bool
IsMobyDriver() bool
}
func Boot(ctx context.Context, d Driver, pw progress.Writer) (*client.Client, error) {
@ -71,11 +72,7 @@ func Boot(ctx context.Context, d Driver, pw progress.Writer) (*client.Client, er
if try > 2 {
return nil, errors.Errorf("failed to bootstrap %T driver in attempts", d)
}
if err := d.Bootstrap(ctx, func(s *client.SolveStatus) {
if pw != nil {
pw.Status() <- s
}
}); err != nil {
if err := d.Bootstrap(ctx, pw.Write); err != nil {
return nil, err
}
}

4
driver/kubernetes/driver.go
View File

@ -44,6 +44,10 @@ type Driver struct {
podChooser podchooser.PodChooser
}
func (d *Driver) IsMobyDriver() bool {
return false
}
func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error {
return progress.Wrap("[internal] booting buildkit", l, func(sub progress.SubLogger) error {
_, err := d.deploymentClient.Get(ctx, d.deployment.Name, metav1.GetOptions{})

22
driver/manager.go
View File

@ -5,10 +5,12 @@ import (
"io/ioutil"
"sort"
"strings"
"sync"
"k8s.io/client-go/rest"
dockerclient "github.com/docker/docker/client"
"github.com/moby/buildkit/client"
"github.com/pkg/errors"
)
@ -117,9 +119,27 @@ func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.API
return nil, err
}
}
return f.New(ctx, ic)
d, err := f.New(ctx, ic)
if err != nil {
return nil, err
}
return &cachedDriver{Driver: d}, nil
}
func GetFactories() map[string]Factory {
return drivers
}
type cachedDriver struct {
Driver
client *client.Client
err error
once sync.Once
}
func (d *cachedDriver) Client(ctx context.Context) (*client.Client, error) {
d.once.Do(func() {
d.client, d.err = d.Driver.Client(ctx)
})
return d.client, d.err
}

10
util/progress/fromreader.go
View File

@ -11,7 +11,6 @@ import (
)
func FromReader(w Writer, name string, rc io.ReadCloser) {
status := w.Status()
dgst := digest.FromBytes([]byte(identity.NewID()))
tm := time.Now()
@ -21,9 +20,9 @@ func FromReader(w Writer, name string, rc io.ReadCloser) {
Started: &tm,
}
status <- &client.SolveStatus{
w.Write(&client.SolveStatus{
Vertexes: []*client.Vertex{&vtx},
}
})
_, err := io.Copy(ioutil.Discard, rc)
@ -33,8 +32,7 @@ func FromReader(w Writer, name string, rc io.ReadCloser) {
if err != nil {
vtx2.Error = err.Error()
}
status <- &client.SolveStatus{
w.Write(&client.SolveStatus{
Vertexes: []*client.Vertex{&vtx2},
}
close(status)
})
}

97
util/progress/multiwriter.go
View File

@ -1,101 +1,32 @@
package progress
import (
"context"
"strings"
"sync"
"github.com/moby/buildkit/client"
"golang.org/x/sync/errgroup"
)
type MultiWriter struct {
w Writer
eg *errgroup.Group
once sync.Once
ready chan struct{}
}
func (mw *MultiWriter) WithPrefix(pfx string, force bool) Writer {
in := make(chan *client.SolveStatus)
out := mw.w.Status()
p := &prefixed{
main: mw.w,
in: in,
func WithPrefix(w Writer, pfx string, force bool) Writer {
return &prefixed{
main: w,
pfx: pfx,
force: force,
}
mw.eg.Go(func() error {
mw.once.Do(func() {
close(mw.ready)
})
for {
select {
case v, ok := <-in:
if ok {
if force {
for _, v := range v.Vertexes {
v.Name = addPrefix(pfx, v.Name)
}
}
out <- v
} else {
return nil
}
case <-mw.Done():
return mw.Err()
}
}
})
return p
}
func (mw *MultiWriter) Done() <-chan struct{} {
return mw.w.Done()
}
func (mw *MultiWriter) Err() error {
return mw.w.Err()
}
func (mw *MultiWriter) Status() chan *client.SolveStatus {
return nil
}
type prefixed struct {
main Writer
in chan *client.SolveStatus
main Writer
pfx string
force bool
}
func (p *prefixed) Done() <-chan struct{} {
return p.main.Done()
}
func (p *prefixed) Err() error {
return p.main.Err()
}
func (p *prefixed) Status() chan *client.SolveStatus {
return p.in
}
func NewMultiWriter(pw Writer) *MultiWriter {
if pw == nil {
return nil
}
eg, _ := errgroup.WithContext(context.TODO())
ready := make(chan struct{})
go func() {
<-ready
eg.Wait()
close(pw.Status())
}()
return &MultiWriter{
w: pw,
eg: eg,
ready: ready,
func (p *prefixed) Write(v *client.SolveStatus) {
if p.force {
for _, v := range v.Vertexes {
v.Name = addPrefix(p.pfx, v.Name)
}
}
p.main.Write(v)
}
func addPrefix(pfx, name string) string {

21
util/progress/printer.go
View File

@ -9,32 +9,27 @@ import (
"github.com/moby/buildkit/util/progress/progressui"
)
type printer struct {
type Printer struct {
status chan *client.SolveStatus
done <-chan struct{}
err error
}
func (p *printer) Done() <-chan struct{} {
return p.done
}
func (p *printer) Err() error {
func (p *Printer) Wait() error {
close(p.status)
<-p.done
return p.err
}
func (p *printer) Status() chan *client.SolveStatus {
if p == nil {
return nil
}
return p.status
func (p *Printer) Write(s *client.SolveStatus) {
p.status <- s
}
func NewPrinter(ctx context.Context, out console.File, mode string) Writer {
func NewPrinter(ctx context.Context, out console.File, mode string) *Printer {
statusCh := make(chan *client.SolveStatus)
doneCh := make(chan struct{})
pw := &printer{
pw := &Printer{
status: statusCh,
done: doneCh,
}

85
util/progress/reset.go
View File

@ -7,56 +7,45 @@ import (
)
func ResetTime(in Writer) Writer {
w := &pw{Writer: in, status: make(chan *client.SolveStatus), tm: time.Now()}
go func() {
for {
select {
case <-in.Done():
return
case st, ok := <-w.status:
if !ok {
close(in.Status())
return
}
if w.diff == nil {
for _, v := range st.Vertexes {
if v.Started != nil {
d := v.Started.Sub(w.tm)
w.diff = &d
}
}
}
if w.diff != nil {
for _, v := range st.Vertexes {
if v.Started != nil {
d := v.Started.Add(-*w.diff)
v.Started = &d
}
if v.Completed != nil {
d := v.Completed.Add(-*w.diff)
v.Completed = &d
}
}
for _, v := range st.Statuses {
if v.Started != nil {
d := v.Started.Add(-*w.diff)
v.Started = &d
}
if v.Completed != nil {
d := v.Completed.Add(-*w.diff)
v.Completed = &d
}
v.Timestamp = v.Timestamp.Add(-*w.diff)
}
for _, v := range st.Logs {
v.Timestamp = v.Timestamp.Add(-*w.diff)
}
}
in.Status() <- st
return &pw{Writer: in, status: make(chan *client.SolveStatus), tm: time.Now()}
}
func (w *pw) Write(st *client.SolveStatus) {
if w.diff == nil {
for _, v := range st.Vertexes {
if v.Started != nil {
d := v.Started.Sub(w.tm)
w.diff = &d
}
}
}()
return w
}
if w.diff != nil {
for _, v := range st.Vertexes {
if v.Started != nil {
d := v.Started.Add(-*w.diff)
v.Started = &d
}
if v.Completed != nil {
d := v.Completed.Add(-*w.diff)
v.Completed = &d
}
}
for _, v := range st.Statuses {
if v.Started != nil {
d := v.Started.Add(-*w.diff)
v.Started = &d
}
if v.Completed != nil {
d := v.Completed.Add(-*w.diff)
v.Completed = &d
}
v.Timestamp = v.Timestamp.Add(-*w.diff)
}
for _, v := range st.Logs {
v.Timestamp = v.Timestamp.Add(-*w.diff)
}
}
w.Writer.Write(st)
}
type pw struct {

29
util/progress/writer.go
View File

@ -9,13 +9,10 @@ import (
)
type Writer interface {
Done() <-chan struct{}
Err() error
Status() chan *client.SolveStatus
Write(*client.SolveStatus)
}
func Write(w Writer, name string, f func() error) {
status := w.Status()
dgst := digest.FromBytes([]byte(identity.NewID()))
tm := time.Now()
@ -25,9 +22,9 @@ func Write(w Writer, name string, f func() error) {
Started: &tm,
}
status <- &client.SolveStatus{
w.Write(&client.SolveStatus{
Vertexes: []*client.Vertex{&vtx},
}
})
err := f()
@ -37,7 +34,23 @@ func Write(w Writer, name string, f func() error) {
if err != nil {
vtx2.Error = err.Error()
}
status <- &client.SolveStatus{
w.Write(&client.SolveStatus{
Vertexes: []*client.Vertex{&vtx2},
}
})
}
func NewChannel(w Writer) (chan *client.SolveStatus, chan struct{}) {
ch := make(chan *client.SolveStatus)
done := make(chan struct{})
go func() {
for {
v, ok := <-ch
if !ok {
close(done)
return
}
w.Write(v)
}
}()
return ch, done
}