Merge pull request #23949 from l0rd/wsl-disk-oci-pull

Get WSL disk as an OCI artifact

pkg/machine/applehv/stubber.go

@@ -14,7 +14,6 @@ import (
 	"github.com/containers/podman/v5/pkg/machine/apple/vfkit"
 	"github.com/containers/podman/v5/pkg/machine/define"
 	"github.com/containers/podman/v5/pkg/machine/ignition"
-	"github.com/containers/podman/v5/pkg/machine/shim/diskpull"
 	"github.com/containers/podman/v5/pkg/machine/vmconfigs"
 	"github.com/containers/podman/v5/utils"
 	vfConfig "github.com/crc-org/vfkit/pkg/config"
@@ -152,10 +151,6 @@ func (a AppleHVStubber) PostStartNetworking(mc *vmconfigs.MachineConfig, noInfo
 	return nil
 }
 
-func (a AppleHVStubber) GetDisk(userInputPath string, dirs *define.MachineDirs, mc *vmconfigs.MachineConfig) error {
-	return diskpull.GetDisk(userInputPath, dirs, mc.ImagePath, a.VMType(), mc.Name)
-}
-
 func (a *AppleHVStubber) GetRosetta(mc *vmconfigs.MachineConfig) (bool, error) {
 	rosetta := mc.AppleHypervisor.Vfkit.Rosetta
 	return rosetta, nil

pkg/machine/e2e/machine_test.go

@@ -62,17 +62,13 @@ var _ = BeforeSuite(func() {
 	if err != nil {
 		Fail("unable to create testProvider")
 	}
-	if testProvider.VMType() == define.WSLVirt {
+
-		pullError = pullWSLDisk()
+	testDiskProvider := testProvider.VMType()
-	} else {
+	if testDiskProvider == define.LibKrun {
-		// This is a one-off and a little messy but once WSL switches
+		testDiskProvider = define.AppleHvVirt // libkrun uses the applehv image for testing
-		// to use OCI disk artifacts, we can make all the conditionals cleaner.
-		testDiskProvider := testProvider.VMType()
-		if testDiskProvider == define.LibKrun {
-			testDiskProvider = define.AppleHvVirt // libkrun uses the applehv image for testing
-		}
-		pullError = pullOCITestDisk(tmpDir, testDiskProvider)
 	}
+	pullError = pullOCITestDisk(tmpDir, testDiskProvider)
+
 	if pullError != nil {
 		Fail(fmt.Sprintf("failed to pull disk: %q", pullError))
 	}

pkg/machine/e2e/machine_wsl_test.go

@@ -1,55 +0,0 @@
-package e2e_test
-
-import (
-	"errors"
-	"fmt"
-	url2 "net/url"
-	"os"
-	"path"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"github.com/containers/podman/v5/pkg/machine"
-	"github.com/containers/podman/v5/pkg/machine/compression"
-	"github.com/containers/podman/v5/pkg/machine/define"
-	"github.com/containers/podman/v5/pkg/machine/wsl"
-	. "github.com/onsi/ginkgo/v2"
-)
-
-func pullWSLDisk() error {
-	downloadLocation := os.Getenv("MACHINE_IMAGE")
-	if downloadLocation == "" {
-		dl, _, _, _, err := wsl.GetFedoraDownloadForWSL()
-		if err != nil {
-			return errors.New("unable to determine WSL download")
-		}
-		downloadLocation = dl.String()
-	}
-
-	if downloadLocation == "" {
-		return errors.New("machine tests require a file reference to a disk image right now")
-	}
-	compressionExtension := ".zst"
-	suiteImageName = strings.TrimSuffix(path.Base(downloadLocation), compressionExtension)
-	fqImageName = filepath.Join(tmpDir, suiteImageName)
-	getMe, err := url2.Parse(downloadLocation)
-	if err != nil {
-		return fmt.Errorf("unable to create url for download: %q", err)
-	}
-	now := time.Now()
-	if err := machine.DownloadVMImage(getMe, suiteImageName, fqImageName+compressionExtension); err != nil {
-		return fmt.Errorf("unable to download machine image: %q", err)
-	}
-	GinkgoWriter.Println("Download took: ", time.Since(now).String())
-	diskImage, err := define.NewMachineFile(fqImageName+compressionExtension, nil)
-	if err != nil {
-		return fmt.Errorf("unable to create vmfile %q: %v", fqImageName+compressionExtension, err)
-	}
-	compressionStart := time.Now()
-	if err := compression.Decompress(diskImage, fqImageName); err != nil {
-		return fmt.Errorf("unable to decompress image file: %q", err)
-	}
-	GinkgoWriter.Println("compression took: ", time.Since(compressionStart))
-	return nil
-}

pkg/machine/hyperv/stubber.go

@@ -19,7 +19,6 @@ import (
 	"github.com/containers/podman/v5/pkg/machine/env"
 	"github.com/containers/podman/v5/pkg/machine/hyperv/vsock"
 	"github.com/containers/podman/v5/pkg/machine/ignition"
-	"github.com/containers/podman/v5/pkg/machine/shim/diskpull"
 	"github.com/containers/podman/v5/pkg/machine/vmconfigs"
 	"github.com/containers/podman/v5/pkg/systemd/parser"
 	"github.com/sirupsen/logrus"
@@ -451,10 +450,6 @@ func (h HyperVStubber) UpdateSSHPort(mc *vmconfigs.MachineConfig, port int) erro
 	return nil
 }
 
-func (h HyperVStubber) GetDisk(userInputPath string, dirs *define.MachineDirs, mc *vmconfigs.MachineConfig) error {
-	return diskpull.GetDisk(userInputPath, dirs, mc.ImagePath, h.VMType(), mc.Name)
-}
-
 func resizeDisk(newSize strongunits.GiB, imagePath *define.VMFile) error {
 	resize := exec.Command("powershell", []string{"-command", fmt.Sprintf("Resize-VHD %s %d", imagePath.GetPath(), newSize.ToBytes())}...)
 	logrus.Debug(resize.Args)

pkg/machine/libkrun/stubber.go

@@ -12,7 +12,6 @@ import (
 	"github.com/containers/podman/v5/pkg/machine/apple/vfkit"
 	"github.com/containers/podman/v5/pkg/machine/define"
 	"github.com/containers/podman/v5/pkg/machine/ignition"
-	"github.com/containers/podman/v5/pkg/machine/shim/diskpull"
 	"github.com/containers/podman/v5/pkg/machine/vmconfigs"
 	"github.com/containers/podman/v5/utils"
 	vfConfig "github.com/crc-org/vfkit/pkg/config"
@@ -55,10 +54,6 @@ func (l LibKrunStubber) CreateVM(opts define.CreateVMOpts, mc *vmconfigs.Machine
 	return apple.ResizeDisk(mc, mc.Resources.DiskSize)
 }
 
-func (l LibKrunStubber) GetDisk(userInputPath string, dirs *define.MachineDirs, mc *vmconfigs.MachineConfig) error {
-	return diskpull.GetDisk(userInputPath, dirs, mc.ImagePath, l.VMType(), mc.Name)
-}
-
 func (l LibKrunStubber) PrepareIgnition(mc *vmconfigs.MachineConfig, ignBuilder *ignition.IgnitionBuilder) (*ignition.ReadyUnitOpts, error) {
 	return nil, nil
 }

pkg/machine/ocipull/ociartifact.go

@@ -27,6 +27,7 @@ const (
 	artifactRegistry     = "quay.io"
 	artifactRepo         = "podman"
 	artifactImageName    = "machine-os"
+	artifactImageNameWSL = "machine-os-wsl"
 	artifactOriginalName = "org.opencontainers.image.title"
 	machineOS            = "linux"
 )
@@ -94,7 +95,14 @@ func NewOCIArtifactPull(ctx context.Context, dirs *define.MachineDirs, endpoint
 
 	cache := false
 	if endpoint == "" {
-		endpoint = fmt.Sprintf("docker://%s/%s/%s:%s", artifactRegistry, artifactRepo, artifactImageName, artifactVersion.majorMinor())
+		// The OCI artifact containing the OS image for WSL has a different
+		// image name. This should be temporary and dropped as soon as the
+		// OS image for WSL is built from fedora-coreos too (c.f. RUN-2178).
+		imageName := artifactImageName
+		if vmType == define.WSLVirt {
+			imageName = artifactImageNameWSL
+		}
+		endpoint = fmt.Sprintf("docker://%s/%s/%s:%s", artifactRegistry, artifactRepo, imageName, artifactVersion.majorMinor())
 		cache = true
 	}
 

pkg/machine/qemu/stubber.go

@@ -20,7 +20,6 @@ import (
 	"github.com/containers/podman/v5/pkg/machine/define"
 	"github.com/containers/podman/v5/pkg/machine/ignition"
 	"github.com/containers/podman/v5/pkg/machine/qemu/command"
-	"github.com/containers/podman/v5/pkg/machine/shim/diskpull"
 	"github.com/containers/podman/v5/pkg/machine/sockets"
 	"github.com/containers/podman/v5/pkg/machine/vmconfigs"
 	"github.com/sirupsen/logrus"
@@ -390,10 +389,6 @@ func (q *QEMUStubber) UpdateSSHPort(mc *vmconfigs.MachineConfig, port int) error
 	return nil
 }
 
-func (q *QEMUStubber) GetDisk(userInputPath string, dirs *define.MachineDirs, mc *vmconfigs.MachineConfig) error {
-	return diskpull.GetDisk(userInputPath, dirs, mc.ImagePath, q.VMType(), mc.Name)
-}
-
 func (q *QEMUStubber) GetRosetta(mc *vmconfigs.MachineConfig) (bool, error) {
 	return false, nil
 }

pkg/machine/shim/host.go

@@ -17,6 +17,7 @@ import (
 	"github.com/containers/podman/v5/pkg/machine/ignition"
 	"github.com/containers/podman/v5/pkg/machine/lock"
 	"github.com/containers/podman/v5/pkg/machine/proxyenv"
+	"github.com/containers/podman/v5/pkg/machine/shim/diskpull"
 	"github.com/containers/podman/v5/pkg/machine/vmconfigs"
 	"github.com/containers/podman/v5/utils"
 	"github.com/hashicorp/go-multierror"
@@ -151,7 +152,7 @@ func Init(opts machineDefine.InitOptions, mp vmconfigs.VMProvider) error {
 	// "/path
 	// "docker://quay.io/something/someManifest
 
-	if err := mp.GetDisk(opts.Image, dirs, mc); err != nil {
+	if err := diskpull.GetDisk(opts.Image, dirs, mc.ImagePath, mp.VMType(), mc.Name); err != nil {
 		return err
 	}
 

pkg/machine/vmconfigs/config.go

@@ -79,10 +79,6 @@ func (o OCIMachineImage) download() error {
 
 type VMProvider interface { //nolint:interfacebloat
 	CreateVM(opts define.CreateVMOpts, mc *MachineConfig, builder *ignition.IgnitionBuilder) error
-	// GetDisk should be only temporary.  It is largely here only because WSL disk pulling is different
-	// TODO
-	// Let's deprecate this ASAP
-	GetDisk(userInputPath string, dirs *define.MachineDirs, mc *MachineConfig) error
 	PrepareIgnition(mc *MachineConfig, ignBuilder *ignition.IgnitionBuilder) (*ignition.ReadyUnitOpts, error)
 	Exists(name string) (bool, error)
 	MountType() VolumeMountType

pkg/machine/wsl/fedora.go

@@ -1,95 +0,0 @@
-package wsl
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"os"
-	"path"
-	"strings"
-	"time"
-
-	"github.com/containers/podman/v5/pkg/machine"
-	"github.com/containers/podman/v5/version"
-	"github.com/sirupsen/logrus"
-)
-
-const (
-	latestReleaseURL = "https://github.com/containers/podman-machine-wsl-os/releases/latest/download"
-)
-
-type FedoraDownload struct {
-	machine.Download
-}
-
-func (f FedoraDownload) Get() *machine.Download {
-	return &f.Download
-}
-
-func (f FedoraDownload) HasUsableCache() (bool, error) {
-	info, err := os.Stat(f.LocalPath)
-	if err != nil {
-		if errors.Is(err, os.ErrNotExist) {
-			return false, nil
-		}
-		return false, err
-	}
-	return info.Size() == f.Size, nil
-}
-
-func (f FedoraDownload) CleanCache() error {
-	// Set cached image to expire after 2 weeks
-	expire := 14 * 24 * time.Hour
-	return machine.RemoveImageAfterExpire(f.CacheDir, expire)
-}
-
-func GetFedoraDownloadForWSL() (*url.URL, string, string, int64, error) {
-	arch := machine.DetermineMachineArch()
-	if arch != "amd64" && arch != "arm64" {
-		return nil, "", "", -1, fmt.Errorf("CPU architecture %q is not supported", arch)
-	}
-
-	releaseURL, err := url.Parse(latestReleaseURL)
-	if err != nil {
-		return nil, "", "", -1, fmt.Errorf("could not parse release URL: %s: %w", releaseURL, err)
-	}
-
-	rootFs := fmt.Sprintf("%d.%d-rootfs-%s.tar.zst", version.Version.Major, version.Version.Minor, arch)
-	rootFsURL := appendToURL(releaseURL, rootFs)
-
-	resp, err := http.Head(rootFsURL.String())
-	if err != nil {
-		return nil, "", "", -1, fmt.Errorf("head request failed: %s: %w", releaseURL, err)
-	}
-	_ = resp.Body.Close()
-	contentLen := resp.ContentLength
-
-	if resp.StatusCode != http.StatusOK {
-		return nil, "", "", -1, fmt.Errorf("head request failed: %s: %w", rootFsURL, err)
-	}
-
-	verURL := appendToURL(releaseURL, "version")
-	resp, err = http.Get(verURL.String())
-	if err != nil {
-		return nil, "", "", -1, fmt.Errorf("get request failed: %s: %w", verURL.String(), err)
-	}
-
-	defer func() {
-		if err := resp.Body.Close(); err != nil {
-			logrus.Errorf("error closing http body: %q", err)
-		}
-	}()
-	b, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 1024})
-	if err != nil {
-		return nil, "", "", -1, fmt.Errorf("failed reading: %s: %w", verURL.String(), err)
-	}
-	return rootFsURL, strings.TrimSpace(string(b)), arch, contentLen, nil
-}
-
-func appendToURL(url *url.URL, elem string) *url.URL {
-	newURL := *url
-	newURL.Path = path.Join(url.Path, elem)
-	return &newURL
-}

pkg/machine/wsl/stubber.go

@@ -8,15 +8,10 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
-	"path/filepath"
 	"strings"
 
 	"github.com/containers/podman/v5/pkg/machine/env"
-	"github.com/containers/podman/v5/pkg/machine/ocipull"
-	"github.com/containers/podman/v5/pkg/machine/shim/diskpull"
-	"github.com/containers/podman/v5/pkg/machine/stdpull"
 	"github.com/containers/podman/v5/pkg/machine/wsl/wutil"
-	"github.com/containers/podman/v5/utils"
 
 	gvproxy "github.com/containers/gvisor-tap-vsock/pkg/types"
 	"github.com/containers/podman/v5/pkg/machine"
@@ -290,64 +285,6 @@ func (w WSLStubber) VMType() define.VMType {
 	return define.WSLVirt
 }
 
-func (w WSLStubber) GetDisk(userInputPath string, dirs *define.MachineDirs, mc *vmconfigs.MachineConfig) error {
-	var (
-		myDisk ocipull.Disker
-	)
-
-	if userInputPath != "" {
-		return diskpull.GetDisk(userInputPath, dirs, mc.ImagePath, w.VMType(), mc.Name)
-	}
-
-	// check github for the latest version of the WSL dist
-	downloadURL, downloadVersion, _, _, err := GetFedoraDownloadForWSL()
-	if err != nil {
-		return err
-	}
-
-	// we now save the "cached" rootfs in the form of "v<version-number>-rootfs.tar.xz"
-	// i.e.v39.0.31-rootfs.tar.xz
-	versionedBase := fmt.Sprintf("%s-%s", downloadVersion, filepath.Base(downloadURL.Path))
-
-	cachedFile, err := dirs.ImageCacheDir.AppendToNewVMFile(versionedBase, nil)
-	if err != nil {
-		return err
-	}
-
-	// if we find the same file cached (determined by filename only), then dont pull
-	if _, err = os.Stat(cachedFile.GetPath()); err == nil {
-		logrus.Debugf("%q already exists locally", cachedFile.GetPath())
-		myDisk, err = stdpull.NewStdDiskPull(cachedFile.GetPath(), mc.ImagePath)
-		if err != nil {
-			return err
-		}
-	} else {
-		files, err := os.ReadDir(dirs.ImageCacheDir.GetPath())
-		if err != nil {
-			logrus.Warn("failed to clean machine image cache: ", err)
-		} else {
-			defer func() {
-				for _, file := range files {
-					path := filepath.Join(dirs.ImageCacheDir.GetPath(), file.Name())
-					logrus.Debugf("cleaning cached image: %s", path)
-					err := utils.GuardedRemoveAll(path)
-					if err != nil && !errors.Is(err, os.ErrNotExist) {
-						logrus.Warn("failed to clean machine image cache: ", err)
-					}
-				}
-			}()
-		}
-
-		myDisk, err = stdpull.NewDiskFromURL(downloadURL.String(), mc.ImagePath, dirs.ImageCacheDir, &versionedBase, true)
-		if err != nil {
-			return err
-		}
-	}
-	// up until now, nothing has really happened
-	// pull if needed and decompress to image location
-	return myDisk.Get()
-}
-
 func (w WSLStubber) GetRosetta(mc *vmconfigs.MachineConfig) (bool, error) {
 	return false, nil
 }