Handle podman-remote --arch, --platform, --os

Podman remote should be able to handle remote specification of
arches.

Requires: https://github.com/containers/buildah/pull/3116

Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/checkpoint-restore/go-criu v0.0.0-20190109184317-bdb7599cd87b
 	github.com/containernetworking/cni v0.8.1
 	github.com/containernetworking/plugins v0.9.1
-	github.com/containers/buildah v1.20.0
+	github.com/containers/buildah v1.20.1-0.20210402144408-36a37402d0c8
 	github.com/containers/common v0.35.4
 	github.com/containers/conmon v2.0.20+incompatible
 	github.com/containers/image/v5 v5.10.5

go.sum

@@ -174,9 +174,8 @@ github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHV
 github.com/containernetworking/plugins v0.8.7/go.mod h1:R7lXeZaBzpfqapcAbHRW8/CYwm0dHzbz0XEjofx0uB0=
 github.com/containernetworking/plugins v0.9.1 h1:FD1tADPls2EEi3flPc2OegIY1M9pUa9r2Quag7HMLV8=
 github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8=
-github.com/containers/buildah v1.20.0 h1:H8db/d2uSGm947mqjX0Iup6F0T9AnK3kS/ff9RCemZA=
+github.com/containers/buildah v1.20.1-0.20210402144408-36a37402d0c8 h1:RlqbDlfE3+qrq4bNTZG7NVPqCDzfZrgE/yicu0VAykQ=
-github.com/containers/buildah v1.20.0/go.mod h1:8V3UBoTKBWU9AxNHb1MAKnZZ9oSoz/IsYyjeymrpl1s=
+github.com/containers/buildah v1.20.1-0.20210402144408-36a37402d0c8/go.mod h1:iowyscoAC5jwNDhs3c5CLGdBZ9FJk5UOoN2I5TdmXFs=
-github.com/containers/common v0.35.3/go.mod h1:rMzxgD7nMGw++cEbsp+NZv0UJO4rgXbm7F7IbJPTwIE=
 github.com/containers/common v0.35.4 h1:szyWRncsHkBwCVpu1dkEOXUjkwCetlfcLmKJTwo1Sp8=
 github.com/containers/common v0.35.4/go.mod h1:rMzxgD7nMGw++cEbsp+NZv0UJO4rgXbm7F7IbJPTwIE=
 github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg=

pkg/api/handlers/compat/images_build.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/containers/buildah"
 	buildahDefine "github.com/containers/buildah/define"
+	"github.com/containers/buildah/pkg/parse"
 	"github.com/containers/buildah/util"
 	"github.com/containers/image/v5/types"
 	"github.com/containers/podman/v3/libpod"
@@ -445,6 +446,17 @@ func BuildImage(w http.ResponseWriter, r *http.Request) {
 		},
 	}
 
+	if len(query.Platform) > 0 {
+		variant := ""
+		buildOptions.OS, buildOptions.Architecture, variant, err = parse.Platform(query.Platform)
+		if err != nil {
+			utils.BadRequest(w, "platform", query.Platform, err)
+			return
+		}
+		buildOptions.SystemContext.OSChoice = buildOptions.OS
+		buildOptions.SystemContext.ArchitectureChoice = buildOptions.Architecture
+		buildOptions.SystemContext.VariantChoice = variant
+	}
 	if _, found := r.URL.Query()["timestamp"]; found {
 		ts := time.Unix(query.Timestamp, 0)
 		buildOptions.Timestamp = &ts

pkg/bindings/images/build.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"strconv"
 	"strings"
 
@@ -190,6 +191,10 @@ func Build(ctx context.Context, containerFiles []string, options entities.BuildO
 			platform = "linux"
 		}
 		platform += "/" + options.Architecture
+	} else {
+		if len(platform) > 0 {
+			platform += "/" + runtime.GOARCH
+		}
 	}
 	if len(platform) > 0 {
 		params.Set("platform", platform)

test/e2e/build_test.go

@@ -566,4 +566,42 @@ RUN echo hello`, ALPINE)
 		Expect(session.OutputToString()).To(ContainSubstring("(user)"))
 		Expect(session.OutputToString()).To(ContainSubstring("(elapsed)"))
 	})
+
+	It("podman build --arch --os flag", func() {
+		containerfile := `FROM scratch`
+		containerfilePath := filepath.Join(podmanTest.TempDir, "Containerfile")
+		err := ioutil.WriteFile(containerfilePath, []byte(containerfile), 0755)
+		Expect(err).To(BeNil())
+		session := podmanTest.Podman([]string{"build", "--pull-never", "-t", "test", "--arch", "foo", "--os", "bar", "--file", containerfilePath, podmanTest.TempDir})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		inspect := podmanTest.Podman([]string{"image", "inspect", "--format", "{{ .Architecture }}", "test"})
+		inspect.WaitWithDefaultTimeout()
+		Expect(inspect.OutputToString()).To(Equal("foo"))
+
+		inspect = podmanTest.Podman([]string{"image", "inspect", "--format", "{{ .Os }}", "test"})
+		inspect.WaitWithDefaultTimeout()
+		Expect(inspect.OutputToString()).To(Equal("bar"))
+
+	})
+
+	It("podman build --os windows flag", func() {
+		containerfile := `FROM scratch`
+		containerfilePath := filepath.Join(podmanTest.TempDir, "Containerfile")
+		err := ioutil.WriteFile(containerfilePath, []byte(containerfile), 0755)
+		Expect(err).To(BeNil())
+		session := podmanTest.Podman([]string{"build", "--pull-never", "-t", "test", "--os", "windows", "--file", containerfilePath, podmanTest.TempDir})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		inspect := podmanTest.Podman([]string{"image", "inspect", "--format", "{{ .Architecture }}", "test"})
+		inspect.WaitWithDefaultTimeout()
+		Expect(inspect.OutputToString()).To(Equal(runtime.GOARCH))
+
+		inspect = podmanTest.Podman([]string{"image", "inspect", "--format", "{{ .Os }}", "test"})
+		inspect.WaitWithDefaultTimeout()
+		Expect(inspect.OutputToString()).To(Equal("windows"))
+
+	})
 })

vendor/github.com/containers/buildah/.cirrus.yml

@@ -164,7 +164,7 @@ conformance_task:
     gce_instance:
         image_name: "${UBUNTU_CACHE_IMAGE_NAME}"
 
-    timeout_in: 20m
+    timeout_in: 25m
 
     setup_script: '${SCRIPT_BASE}/setup.sh |& ${_TIMESTAMP}'
     conformance_test_script: '${SCRIPT_BASE}/test.sh conformance |& ${_TIMESTAMP}'

vendor/github.com/containers/buildah/Makefile

@@ -149,7 +149,7 @@ install.runc:
 
 .PHONY: test-conformance
 test-conformance:
-	$(GO_TEST) -v -tags "$(STORAGETAGS) $(SECURITYTAGS)" -cover -timeout 15m ./tests/conformance
+	$(GO_TEST) -v -tags "$(STORAGETAGS) $(SECURITYTAGS)" -cover -timeout 20m ./tests/conformance
 
 .PHONY: test-integration
 test-integration: install.tools

vendor/github.com/containers/buildah/chroot/run.go

@@ -20,6 +20,7 @@ import (
 	"unsafe"
 
 	"github.com/containers/buildah/bind"
+	"github.com/containers/buildah/copier"
 	"github.com/containers/buildah/util"
 	"github.com/containers/storage/pkg/ioutils"
 	"github.com/containers/storage/pkg/mount"
@@ -1161,7 +1162,18 @@ func setupChrootBindMounts(spec *specs.Spec, bundlePath string) (undoBinds func(
 			}
 		}
 		target := filepath.Join(spec.Root.Path, m.Destination)
-		if _, err := os.Stat(target); err != nil {
+		// Check if target is a symlink
+		stat, err := os.Lstat(target)
+		// If target is a symlink, follow the link and ensure the destination exists
+		if err == nil && stat != nil && (stat.Mode()&os.ModeSymlink != 0) {
+			target, err = copier.Eval(spec.Root.Path, m.Destination, copier.EvalOptions{})
+			if err != nil {
+				return nil, errors.Wrapf(err, "evaluating symlink %q", target)
+			}
+			// Stat the destination of the evaluated symlink
+			_, err = os.Stat(target)
+		}
+		if err != nil {
 			// If the target can't be stat()ted, check the error.
 			if !os.IsNotExist(err) {
 				return undoBinds, errors.Wrapf(err, "error examining %q for mounting in mount namespace", target)

vendor/github.com/containers/buildah/define/types.go

@@ -28,7 +28,7 @@ const (
 	Package = "buildah"
 	// Version for the Package.  Bump version in contrib/rpm/buildah.spec
 	// too.
-	Version = "1.20.0"
+	Version = "1.20.1-dev"
 
 	// DefaultRuntime if containers.conf fails.
 	DefaultRuntime = "runc"
@@ -166,7 +166,7 @@ func cloneToDirectory(url, dir string) error {
 		cmd = exec.Command("git", "clone", url, dir)
 	} else {
 		logrus.Debugf("cloning repo %q and branch %q to %q", gitBranch[0], gitBranch[1], dir)
-		cmd = exec.Command("git", "clone", "-b", gitBranch[1], gitBranch[0], dir)
+		cmd = exec.Command("git", "clone", "--recurse-submodules", "-b", gitBranch[1], gitBranch[0], dir)
 	}
 	return cmd.Run()
 }

vendor/github.com/containers/buildah/go.mod

@@ -4,7 +4,7 @@ go 1.12
 
 require (
 	github.com/containernetworking/cni v0.8.1
-	github.com/containers/common v0.35.3
+	github.com/containers/common v0.35.4
 	github.com/containers/image/v5 v5.10.5
 	github.com/containers/ocicrypt v1.1.0
 	github.com/containers/storage v1.28.1

vendor/github.com/containers/buildah/go.sum

@@ -165,8 +165,8 @@ github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ
 github.com/containernetworking/cni v0.8.1 h1:7zpDnQ3T3s4ucOuJ/ZCLrYBxzkg0AELFfII3Epo9TmI=
 github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
 github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM=
-github.com/containers/common v0.35.3 h1:6tEBSIHlJzpmt35zA1ZcjBqbtUilAHDWaa7buPvaqWY=
+github.com/containers/common v0.35.4 h1:szyWRncsHkBwCVpu1dkEOXUjkwCetlfcLmKJTwo1Sp8=
-github.com/containers/common v0.35.3/go.mod h1:rMzxgD7nMGw++cEbsp+NZv0UJO4rgXbm7F7IbJPTwIE=
+github.com/containers/common v0.35.4/go.mod h1:rMzxgD7nMGw++cEbsp+NZv0UJO4rgXbm7F7IbJPTwIE=
 github.com/containers/image/v5 v5.10.5 h1:VK1UbsZMzjdw5Xqr3Im9h4iOqHWU0naFs+I78kavc7I=
 github.com/containers/image/v5 v5.10.5/go.mod h1:SgIbWEedCNBbn2FI5cH0/jed1Ecy2s8XK5zTxvJTzII=
 github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b h1:Q8ePgVfHDplZ7U33NwHZkrVELsZP5fYj9pM5WBZB2GE=

vendor/github.com/containers/buildah/pkg/blobcache/blobcache.go

@@ -13,6 +13,7 @@ import (
 	"github.com/containers/image/v5/docker/reference"
 	"github.com/containers/image/v5/image"
 	"github.com/containers/image/v5/manifest"
+	"github.com/containers/image/v5/pkg/compression"
 	"github.com/containers/image/v5/transports"
 	"github.com/containers/image/v5/types"
 	"github.com/containers/storage/pkg/archive"
@@ -301,25 +302,32 @@ func (s *blobCacheSource) LayerInfosForCopy(ctx context.Context, instanceDigest
 				alternate = filepath.Join(filepath.Dir(alternate), makeFilename(digest.Digest(replaceDigest), false))
 				fileInfo, err := os.Stat(alternate)
 				if err == nil {
-					logrus.Debugf("suggesting cached blob with digest %q and compression %v in place of blob with digest %q", string(replaceDigest), s.reference.compress, info.Digest.String())
-					info.Digest = digest.Digest(replaceDigest)
-					info.Size = fileInfo.Size()
 					switch info.MediaType {
 					case v1.MediaTypeImageLayer, v1.MediaTypeImageLayerGzip:
 						switch s.reference.compress {
 						case types.Compress:
 							info.MediaType = v1.MediaTypeImageLayerGzip
+							info.CompressionAlgorithm = &compression.Gzip
 						case types.Decompress:
 							info.MediaType = v1.MediaTypeImageLayer
+							info.CompressionAlgorithm = nil
 						}
 					case docker.V2S2MediaTypeUncompressedLayer, manifest.DockerV2Schema2LayerMediaType:
 						switch s.reference.compress {
 						case types.Compress:
 							info.MediaType = manifest.DockerV2Schema2LayerMediaType
+							info.CompressionAlgorithm = &compression.Gzip
 						case types.Decompress:
-							info.MediaType = docker.V2S2MediaTypeUncompressedLayer
+							// nope, not going to suggest anything, it's not allowed by the spec
+							replacedInfos = append(replacedInfos, info)
+							continue
 						}
 					}
+					logrus.Debugf("suggesting cached blob with digest %q, type %q, and compression %v in place of blob with digest %q", string(replaceDigest), info.MediaType, s.reference.compress, info.Digest.String())
+					info.CompressionOperation = s.reference.compress
+					info.Digest = digest.Digest(replaceDigest)
+					info.Size = fileInfo.Size()
+					logrus.Debugf("info = %#v", info)
 				}
 			}
 			replacedInfos = append(replacedInfos, info)
@@ -422,8 +430,9 @@ func (d *blobCacheDestination) PutBlob(ctx context.Context, stream io.Reader, in
 	var err error
 	var n int
 	var alternateDigest digest.Digest
+	var closer io.Closer
 	wg := new(sync.WaitGroup)
-	defer wg.Wait()
+	needToWait := false
 	compression := archive.Uncompressed
 	if inputInfo.Digest != "" {
 		filename := filepath.Join(d.reference.directory, makeFilename(inputInfo.Digest, isConfig))
@@ -458,7 +467,7 @@ func (d *blobCacheDestination) PutBlob(ctx context.Context, stream io.Reader, in
 				if n >= len(initial) {
 					compression = archive.DetectCompression(initial[:n])
 				}
-				if compression != archive.Uncompressed {
+				if compression == archive.Gzip {
 					// The stream is compressed, so create a file which we'll
 					// use to store a decompressed copy.
 					decompressedTemp, err2 := ioutil.TempFile(d.reference.directory, makeFilename(inputInfo.Digest, isConfig))
@@ -470,10 +479,11 @@ func (d *blobCacheDestination) PutBlob(ctx context.Context, stream io.Reader, in
 						// closing the writing end of the pipe after
 						// PutBlob() returns.
 						decompressReader, decompressWriter := io.Pipe()
-						defer decompressWriter.Close()
+						closer = decompressWriter
 						stream = io.TeeReader(stream, decompressWriter)
 						// Let saveStream() close the reading end and handle the temporary file.
 						wg.Add(1)
+						needToWait = true
 						go saveStream(wg, decompressReader, decompressedTemp, filename, inputInfo.Digest, isConfig, &alternateDigest)
 					}
 				}
@@ -481,6 +491,12 @@ func (d *blobCacheDestination) PutBlob(ctx context.Context, stream io.Reader, in
 		}
 	}
 	newBlobInfo, err := d.destination.PutBlob(ctx, stream, inputInfo, cache, isConfig)
+	if closer != nil {
+		closer.Close()
+	}
+	if needToWait {
+		wg.Wait()
+	}
 	if err != nil {
 		return newBlobInfo, errors.Wrapf(err, "error storing blob to image destination for cache %q", transports.ImageName(d.reference))
 	}

vendor/github.com/containers/buildah/pkg/parse/parse.go

@@ -629,7 +629,7 @@ func SystemContextFromOptions(c *cobra.Command) (*types.SystemContext, error) {
 	}
 	if c.Flag("platform") != nil && c.Flag("platform").Changed {
 		if platform, err := c.Flags().GetString("platform"); err == nil {
-			os, arch, variant, err := parsePlatform(platform)
+			os, arch, variant, err := Platform(platform)
 			if err != nil {
 				return nil, err
 			}
@@ -672,7 +672,7 @@ func PlatformFromOptions(c *cobra.Command) (os, arch string, err error) {
 
 	if c.Flag("platform").Changed {
 		if pf, err := c.Flags().GetString("platform"); err == nil {
-			selectedOS, selectedArch, _, err := parsePlatform(pf)
+			selectedOS, selectedArch, _, err := Platform(pf)
 			if err != nil {
 				return "", "", errors.Wrap(err, "unable to parse platform")
 			}
@@ -691,7 +691,8 @@ func DefaultPlatform() string {
 	return runtime.GOOS + platformSep + runtime.GOARCH
 }
 
-func parsePlatform(platform string) (os, arch, variant string, err error) {
+// Platform separates the platform string into os, arch and variant
+func Platform(platform string) (os, arch, variant string, err error) {
 	split := strings.Split(platform, platformSep)
 	if len(split) < 2 {
 		return "", "", "", errors.Errorf("invalid platform syntax for %q (use OS/ARCH)", platform)

vendor/github.com/containers/buildah/run_linux.go

@@ -359,7 +359,17 @@ func runSetupBuiltinVolumes(mountLabel, mountPoint, containerDir string, builtin
 			}
 			initializeVolume = true
 		}
-		stat, err := os.Stat(srcPath)
+		// Check if srcPath is a symlink
+		stat, err := os.Lstat(srcPath)
+		// If srcPath is a symlink, follow the link and ensure the destination exists
+		if err == nil && stat != nil && (stat.Mode()&os.ModeSymlink != 0) {
+			srcPath, err = copier.Eval(mountPoint, volume, copier.EvalOptions{})
+			if err != nil {
+				return nil, errors.Wrapf(err, "evaluating symlink %q", srcPath)
+			}
+			// Stat the destination of the evaluated symlink
+			stat, err = os.Stat(srcPath)
+		}
 		if err != nil {
 			if !os.IsNotExist(err) {
 				return nil, err
@@ -519,8 +529,9 @@ func (b *Builder) setupMounts(mountPoint string, spec *specs.Spec, bundlePath st
 		return err
 	}
 
+	allMounts := util.SortMounts(append(append(append(append(append(volumes, builtins...), secretMounts...), bindFileMounts...), specMounts...), sysfsMount...))
 	// Add them all, in the preferred order, except where they conflict with something that was previously added.
-	for _, mount := range append(append(append(append(append(volumes, builtins...), secretMounts...), bindFileMounts...), specMounts...), sysfsMount...) {
+	for _, mount := range allMounts {
 		if haveMount(mount.Destination) {
 			// Already mounting something there, no need to bother with this one.
 			continue

vendor/github.com/containers/buildah/util/util.go

@@ -6,6 +6,8 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"path/filepath"
+	"sort"
 	"strings"
 	"sync"
 	"syscall"
@@ -474,3 +476,26 @@ func MergeEnv(defaults, overrides []string) []string {
 	}
 	return s
 }
+
+type byDestination []specs.Mount
+
+func (m byDestination) Len() int {
+	return len(m)
+}
+
+func (m byDestination) Less(i, j int) bool {
+	return m.parts(i) < m.parts(j)
+}
+
+func (m byDestination) Swap(i, j int) {
+	m[i], m[j] = m[j], m[i]
+}
+
+func (m byDestination) parts(i int) int {
+	return strings.Count(filepath.Clean(m[i].Destination), string(os.PathSeparator))
+}
+
+func SortMounts(m []specs.Mount) []specs.Mount {
+	sort.Sort(byDestination(m))
+	return m
+}

vendor/modules.txt

@@ -74,7 +74,7 @@ github.com/containernetworking/plugins/pkg/utils/hwaddr
 github.com/containernetworking/plugins/pkg/utils/sysctl
 github.com/containernetworking/plugins/plugins/ipam/host-local/backend
 github.com/containernetworking/plugins/plugins/ipam/host-local/backend/allocator
-# github.com/containers/buildah v1.20.0
+# github.com/containers/buildah v1.20.1-0.20210402144408-36a37402d0c8
 github.com/containers/buildah
 github.com/containers/buildah/bind
 github.com/containers/buildah/chroot