Podman 5 machine refactor - applehv

this is the second provider done (qemu first).  all tests pass on arm64 hardware locally ... the hybrid pull from oci registries limit this to arm64 only.

calling gvproxy, waiting for it, and then vfkit seems to still be problematic.  this would be an area that should be cleaned up once all providers are implemented.

Signed-off-by: Brent Baude <bbaude@redhat.com>

pkg/machine/applehv/claim.go

@@ -1,83 +0,0 @@
-//go:build darwin
-
-package applehv
-
-import (
-	"fmt"
-	"io"
-	"io/fs"
-	"net"
-	"os"
-	"os/user"
-	"path/filepath"
-	"time"
-)
-
-// TODO the following functions were taken from pkg/qemu/claim_darwin.go and
-// should be refactored.  I'm thinking even something in pkg/machine/
-
-func dockerClaimSupported() bool {
-	return true
-}
-
-func dockerClaimHelperInstalled() bool {
-	u, err := user.Current()
-	if err != nil {
-		return false
-	}
-
-	labelName := fmt.Sprintf("com.github.containers.podman.helper-%s", u.Username)
-	fileName := filepath.Join("/Library", "LaunchDaemons", labelName+".plist")
-	info, err := os.Stat(fileName)
-	return err == nil && info.Mode().IsRegular()
-}
-
-func claimDockerSock() bool {
-	u, err := user.Current()
-	if err != nil {
-		return false
-	}
-
-	helperSock := fmt.Sprintf("/var/run/podman-helper-%s.socket", u.Username)
-	con, err := net.DialTimeout("unix", helperSock, time.Second*5)
-	if err != nil {
-		return false
-	}
-	_ = con.SetWriteDeadline(time.Now().Add(time.Second * 5))
-	_, err = fmt.Fprintln(con, "GO")
-	if err != nil {
-		return false
-	}
-	_ = con.SetReadDeadline(time.Now().Add(time.Second * 5))
-	read, err := io.ReadAll(con)
-
-	return err == nil && string(read) == "OK"
-}
-
-func findClaimHelper() string {
-	exe, err := os.Executable()
-	if err != nil {
-		return ""
-	}
-
-	exe, err = filepath.EvalSymlinks(exe)
-	if err != nil {
-		return ""
-	}
-
-	return filepath.Join(filepath.Dir(exe), "podman-mac-helper")
-}
-
-func checkSockInUse(sock string) bool {
-	if info, err := os.Stat(sock); err == nil && info.Mode()&fs.ModeSocket == fs.ModeSocket {
-		_, err = net.DialTimeout("unix", dockerSock, dockerConnectTimeout)
-		return err == nil
-	}
-
-	return false
-}
-
-func alreadyLinked(target string, link string) bool {
-	read, err := os.Readlink(link)
-	return err == nil && read == target
-}

pkg/machine/applehv/config.go

@@ -2,200 +2,7 @@
 
 package applehv
 
-import (
-	"errors"
-	"fmt"
-	"io/fs"
-	"path/filepath"
-	"time"
-
-	"github.com/containers/podman/v4/pkg/machine"
-	"github.com/containers/podman/v4/pkg/machine/compression"
-	"github.com/containers/podman/v4/pkg/machine/define"
-	"github.com/containers/podman/v4/pkg/machine/ignition"
-	"github.com/containers/podman/v4/pkg/machine/vmconfigs"
-	vfConfig "github.com/crc-org/vfkit/pkg/config"
-	"github.com/docker/go-units"
-	"golang.org/x/sys/unix"
-)
-
 const (
 	localhostURI       = "http://localhost"
 	ignitionSocketName = "ignition.sock"
 )
-
-type AppleHVVirtualization struct {
-	machine.Virtualization
-}
-
-type MMHardwareConfig struct {
-	CPUs     uint16
-	DiskPath string
-	DiskSize uint64
-	Memory   int32
-}
-
-func VirtualizationProvider() machine.VirtProvider {
-	return &AppleHVVirtualization{
-		machine.NewVirtualization(define.AppleHV, compression.Xz, define.Raw, vmtype),
-	}
-}
-
-func (v AppleHVVirtualization) CheckExclusiveActiveVM() (bool, string, error) {
-	fsVms, err := getVMInfos()
-	if err != nil {
-		return false, "", err
-	}
-	for _, vm := range fsVms {
-		if vm.Running || vm.Starting {
-			return true, vm.Name, nil
-		}
-	}
-
-	return false, "", nil
-}
-
-func (v AppleHVVirtualization) IsValidVMName(name string) (bool, error) {
-	configDir, err := machine.GetConfDir(define.AppleHvVirt)
-	if err != nil {
-		return false, err
-	}
-	fqName := filepath.Join(configDir, fmt.Sprintf("%s.json", name))
-	if _, err := loadMacMachineFromJSON(fqName); err != nil {
-		return false, err
-	}
-	return true, nil
-}
-
-func (v AppleHVVirtualization) List(opts machine.ListOptions) ([]*machine.ListResponse, error) {
-	var (
-		response []*machine.ListResponse
-	)
-
-	mms, err := v.loadFromLocalJson()
-	if err != nil {
-		return nil, err
-	}
-
-	for _, mm := range mms {
-		vmState, err := mm.Vfkit.State()
-		if err != nil {
-			if errors.Is(err, unix.ECONNREFUSED) {
-				vmState = define.Stopped
-			} else {
-				return nil, err
-			}
-		}
-
-		mlr := machine.ListResponse{
-			Name:           mm.Name,
-			CreatedAt:      mm.Created,
-			LastUp:         mm.LastUp,
-			Running:        vmState == define.Running,
-			Starting:       vmState == define.Starting,
-			Stream:         mm.ImageStream,
-			VMType:         define.AppleHvVirt.String(),
-			CPUs:           mm.CPUs,
-			Memory:         mm.Memory * units.MiB,
-			DiskSize:       mm.DiskSize * units.GiB,
-			Port:           mm.Port,
-			RemoteUsername: mm.RemoteUsername,
-			IdentityPath:   mm.IdentityPath,
-		}
-		response = append(response, &mlr)
-	}
-	return response, nil
-}
-
-func (v AppleHVVirtualization) LoadVMByName(name string) (machine.VM, error) {
-	m := MacMachine{Name: name}
-	return m.loadFromFile()
-}
-
-func (v AppleHVVirtualization) NewMachine(opts define.InitOptions) (machine.VM, error) {
-	m := MacMachine{Name: opts.Name}
-
-	if len(opts.USBs) > 0 {
-		return nil, fmt.Errorf("USB host passthrough is not supported for applehv machines")
-	}
-
-	configDir, err := machine.GetConfDir(define.AppleHvVirt)
-	if err != nil {
-		return nil, err
-	}
-
-	configPath, err := define.NewMachineFile(getVMConfigPath(configDir, opts.Name), nil)
-	if err != nil {
-		return nil, err
-	}
-	m.ConfigPath = *configPath
-
-	dataDir, err := machine.GetDataDir(define.AppleHvVirt)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := ignition.SetIgnitionFile(&m.IgnitionFile, vmtype, m.Name, configDir); err != nil {
-		return nil, err
-	}
-
-	// Set creation time
-	m.Created = time.Now()
-
-	m.ResourceConfig = vmconfigs.ResourceConfig{
-		CPUs:     opts.CPUS,
-		DiskSize: opts.DiskSize,
-		// Diskpath will be needed
-		Memory: opts.Memory,
-	}
-	bl := vfConfig.NewEFIBootloader(fmt.Sprintf("%s/%ss", dataDir, opts.Name), true)
-	m.Vfkit.VirtualMachine = vfConfig.NewVirtualMachine(uint(opts.CPUS), opts.Memory, bl)
-
-	if err := m.writeConfig(); err != nil {
-		return nil, err
-	}
-	return m.loadFromFile()
-}
-
-func (v AppleHVVirtualization) RemoveAndCleanMachines() error {
-	// This can be implemented when host networking is completed.
-	return define.ErrNotImplemented
-}
-
-func (v AppleHVVirtualization) VMType() define.VMType {
-	return vmtype
-}
-
-func (v AppleHVVirtualization) loadFromLocalJson() ([]*MacMachine, error) {
-	var (
-		jsonFiles []string
-		mms       []*MacMachine
-	)
-	configDir, err := machine.GetConfDir(v.VMType())
-	if err != nil {
-		return nil, err
-	}
-	if err := filepath.WalkDir(configDir, func(input string, d fs.DirEntry, e error) error {
-		if e != nil {
-			return e
-		}
-		if filepath.Ext(d.Name()) == ".json" {
-			jsonFiles = append(jsonFiles, input)
-		}
-		return nil
-	}); err != nil {
-		return nil, err
-	}
-
-	for _, jsonFile := range jsonFiles {
-		mm, err := loadMacMachineFromJSON(jsonFile)
-		if err != nil {
-			return nil, err
-		}
-		if err != nil {
-			return nil, err
-		}
-		mms = append(mms, mm)
-	}
-	return mms, nil
-}

pkg/machine/applehv/ignition.go

@@ -7,14 +7,20 @@ import (
 	"net/http"
 
 	"github.com/containers/podman/v4/pkg/machine/define"
+	"github.com/containers/podman/v4/pkg/machine/vmconfigs"
 	"github.com/sirupsen/logrus"
 )
 
 // serveIgnitionOverSock allows podman to open a small httpd instance on the vsock between the host
 // and guest to inject the ignitionfile into fcos
-func (m *MacMachine) serveIgnitionOverSock(ignitionSocket *define.VMFile) error {
+func serveIgnitionOverSock(ignitionSocket *define.VMFile, mc *vmconfigs.MachineConfig) error {
-	logrus.Debugf("reading ignition file: %s", m.IgnitionFile.GetPath())
+	ignitionFile, err := mc.IgnitionFile()
-	ignFile, err := m.IgnitionFile.Read()
+	if err != nil {
+		return err
+	}
+
+	logrus.Debugf("reading ignition file: %s", ignitionFile.GetPath())
+	ignFile, err := ignitionFile.Read()
 	if err != nil {
 		return err
 	}
@@ -22,7 +28,7 @@ func (m *MacMachine) serveIgnitionOverSock(ignitionSocket *define.VMFile) error
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		_, err := w.Write(ignFile)
 		if err != nil {
-			logrus.Error("failed to serve ignition file: %v", err)
+			logrus.Errorf("failed to serve ignition file: %v", err)
 		}
 	})
 	listener, err := net.Listen("unix", ignitionSocket.GetPath())

pkg/machine/applehv/stubber.go

@@ -0,0 +1,311 @@
+//go:build darwin
+
+package applehv
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strconv"
+	"time"
+
+	"github.com/containers/common/pkg/config"
+	gvproxy "github.com/containers/gvisor-tap-vsock/pkg/types"
+	"github.com/containers/podman/v4/pkg/machine"
+	"github.com/containers/podman/v4/pkg/machine/applehv/vfkit"
+	"github.com/containers/podman/v4/pkg/machine/define"
+	"github.com/containers/podman/v4/pkg/machine/ignition"
+	"github.com/containers/podman/v4/pkg/machine/sockets"
+	"github.com/containers/podman/v4/pkg/machine/vmconfigs"
+	"github.com/containers/podman/v4/pkg/strongunits"
+	"github.com/containers/podman/v4/utils"
+	vfConfig "github.com/crc-org/vfkit/pkg/config"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
+)
+
+// applehcMACAddress is a pre-defined mac address that vfkit recognizes
+// and is required for network flow
+const applehvMACAddress = "5a:94:ef:e4:0c:ee"
+
+var (
+	vfkitCommand              = "vfkit"
+	gvProxyWaitBackoff        = 500 * time.Millisecond
+	gvProxyMaxBackoffAttempts = 6
+)
+
+type AppleHVStubber struct {
+	vmconfigs.AppleHVConfig
+}
+
+func (a AppleHVStubber) CreateVM(opts define.CreateVMOpts, mc *vmconfigs.MachineConfig, ignBuilder *ignition.IgnitionBuilder) error {
+	mc.AppleHypervisor = new(vmconfigs.AppleHVConfig)
+	mc.AppleHypervisor.Vfkit = vfkit.VfkitHelper{}
+	bl := vfConfig.NewEFIBootloader(fmt.Sprintf("%s/efi-bl-%s", opts.Dirs.DataDir.GetPath(), opts.Name), true)
+	mc.AppleHypervisor.Vfkit.VirtualMachine = vfConfig.NewVirtualMachine(uint(mc.Resources.CPUs), mc.Resources.Memory, bl)
+
+	randPort, err := utils.GetRandomPort()
+	if err != nil {
+		return err
+	}
+	mc.AppleHypervisor.Vfkit.Endpoint = localhostURI + ":" + strconv.Itoa(randPort)
+
+	var virtiofsMounts []machine.VirtIoFs
+	for _, mnt := range mc.Mounts {
+		virtiofsMounts = append(virtiofsMounts, machine.MountToVirtIOFs(mnt))
+	}
+
+	// Populate the ignition file with virtiofs stuff
+	ignBuilder.WithUnit(generateSystemDFilesForVirtiofsMounts(virtiofsMounts)...)
+
+	return resizeDisk(mc, strongunits.GiB(mc.Resources.DiskSize))
+}
+
+func (a AppleHVStubber) GetHyperVisorVMs() ([]string, error) {
+	// not applicable for applehv
+	return nil, nil
+}
+
+func (a AppleHVStubber) MountType() vmconfigs.VolumeMountType {
+	return vmconfigs.VirtIOFS
+}
+
+func (a AppleHVStubber) MountVolumesToVM(_ *vmconfigs.MachineConfig, _ bool) error {
+	// virtiofs: nothing to do here
+	return nil
+}
+
+func (a AppleHVStubber) RemoveAndCleanMachines(_ *define.MachineDirs) error {
+	return nil
+}
+
+func (a AppleHVStubber) SetProviderAttrs(mc *vmconfigs.MachineConfig, cpus, memory *uint64, newDiskSize *strongunits.GiB) error {
+	if newDiskSize != nil {
+		if err := resizeDisk(mc, *newDiskSize); err != nil {
+			return err
+		}
+	}
+	// VFKit does not require saving memory, disk, or cpu
+	return nil
+}
+
+func (a AppleHVStubber) StartNetworking(mc *vmconfigs.MachineConfig, cmd *gvproxy.GvproxyCommand) error {
+	gvProxySock, err := mc.GVProxySocket()
+	if err != nil {
+		return err
+	}
+	// make sure it does not exist before gvproxy is called
+	if err := gvProxySock.Delete(); err != nil {
+		logrus.Error(err)
+	}
+	cmd.AddVfkitSocket(fmt.Sprintf("unixgram://%s", gvProxySock.GetPath()))
+	return nil
+}
+
+func (a AppleHVStubber) StartVM(mc *vmconfigs.MachineConfig) (func() error, func() error, error) {
+	var (
+		ignitionSocket *define.VMFile
+	)
+
+	if bl := mc.AppleHypervisor.Vfkit.VirtualMachine.Bootloader; bl == nil {
+		return nil, nil, fmt.Errorf("unable to determine boot loader for this machine")
+	}
+
+	// Add networking
+	netDevice, err := vfConfig.VirtioNetNew(applehvMACAddress)
+	if err != nil {
+		return nil, nil, err
+	}
+	// Set user networking with gvproxy
+
+	gvproxySocket, err := mc.GVProxySocket()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Wait on gvproxy to be running and aware
+	if err := waitForGvProxy(gvproxySocket); err != nil {
+		return nil, nil, err
+	}
+
+	netDevice.SetUnixSocketPath(gvproxySocket.GetPath())
+
+	readySocket, err := mc.ReadySocket()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	logfile, err := mc.LogFile()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// create a one-time virtual machine for starting because we dont want all this information in the
+	// machineconfig if possible.  the preference was to derive this stuff
+	vm := vfConfig.NewVirtualMachine(uint(mc.Resources.CPUs), mc.Resources.Memory, mc.AppleHypervisor.Vfkit.VirtualMachine.Bootloader)
+
+	defaultDevices, err := getDefaultDevices(mc.ImagePath.GetPath(), logfile.GetPath(), readySocket.GetPath())
+	if err != nil {
+		return nil, nil, err
+	}
+
+	vm.Devices = append(vm.Devices, defaultDevices...)
+	vm.Devices = append(vm.Devices, netDevice)
+
+	mounts, err := virtIOFsToVFKitVirtIODevice(mc.Mounts)
+	if err != nil {
+		return nil, nil, err
+	}
+	vm.Devices = append(vm.Devices, mounts...)
+
+	// To start the VM, we need to call vfkit
+	cfg, err := config.Default()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	vfkitBinaryPath, err := cfg.FindHelperBinary(vfkitCommand, true)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	logrus.Debugf("vfkit path is: %s", vfkitBinaryPath)
+
+	cmd, err := vm.Cmd(vfkitBinaryPath)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	vfkitEndpointArgs, err := getVfKitEndpointCMDArgs(mc.AppleHypervisor.Vfkit.Endpoint)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	machineDataDir, err := mc.DataDir()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cmd.Args = append(cmd.Args, vfkitEndpointArgs...)
+
+	firstBoot, err := mc.IsFirstBoot()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if logrus.IsLevelEnabled(logrus.DebugLevel) {
+		debugDevArgs, err := getDebugDevicesCMDArgs()
+		if err != nil {
+			return nil, nil, err
+		}
+		cmd.Args = append(cmd.Args, debugDevArgs...)
+		cmd.Args = append(cmd.Args, "--gui") // add command line switch to pop the gui open
+	}
+
+	if firstBoot {
+		// If this is the first boot of the vm, we need to add the vsock
+		// device to vfkit so we can inject the ignition file
+		socketName := fmt.Sprintf("%s-%s", mc.Name, ignitionSocketName)
+		ignitionSocket, err = machineDataDir.AppendToNewVMFile(socketName, &socketName)
+		if err != nil {
+			return nil, nil, err
+		}
+		if err := ignitionSocket.Delete(); err != nil {
+			logrus.Errorf("unable to delete ignition socket: %q", err)
+		}
+
+		ignitionVsockDeviceCLI, err := getIgnitionVsockDeviceAsCLI(ignitionSocket.GetPath())
+		if err != nil {
+			return nil, nil, err
+		}
+		cmd.Args = append(cmd.Args, ignitionVsockDeviceCLI...)
+
+		logrus.Debug("first boot detected")
+		logrus.Debugf("serving ignition file over %s", ignitionSocket.GetPath())
+		go func() {
+			if err := serveIgnitionOverSock(ignitionSocket, mc); err != nil {
+				logrus.Error(err)
+			}
+			logrus.Debug("ignition vsock server exited")
+		}()
+	}
+
+	logrus.Debugf("listening for ready on: %s", readySocket.GetPath())
+	if err := readySocket.Delete(); err != nil {
+		logrus.Warnf("unable to delete previous ready socket: %q", err)
+	}
+	readyListen, err := net.Listen("unix", readySocket.GetPath())
+	if err != nil {
+		return nil, nil, err
+	}
+
+	logrus.Debug("waiting for ready notification")
+	readyChan := make(chan error)
+	go sockets.ListenAndWaitOnSocket(readyChan, readyListen)
+
+	logrus.Debugf("vfkit command-line: %v", cmd.Args)
+
+	if err := cmd.Start(); err != nil {
+		return nil, nil, err
+	}
+
+	returnFunc := func() error {
+		processErrChan := make(chan error)
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		go func() {
+			defer close(processErrChan)
+			for {
+				select {
+				case <-ctx.Done():
+					return
+				default:
+				}
+				if err := checkProcessRunning("vfkit", cmd.Process.Pid); err != nil {
+					processErrChan <- err
+					return
+				}
+				// lets poll status every half second
+				time.Sleep(500 * time.Millisecond)
+			}
+		}()
+
+		// wait for either socket or to be ready or process to have exited
+		select {
+		case err := <-processErrChan:
+			if err != nil {
+				return err
+			}
+		case err := <-readyChan:
+			if err != nil {
+				return err
+			}
+			logrus.Debug("ready notification received")
+		}
+		return nil
+	}
+	return cmd.Process.Release, returnFunc, nil
+}
+
+func (a AppleHVStubber) StopHostNetworking() error {
+	// TODO implement me
+	panic("implement me")
+}
+
+func (a AppleHVStubber) VMType() define.VMType {
+	return define.AppleHvVirt
+}
+
+func waitForGvProxy(gvproxySocket *define.VMFile) error {
+	backoffWait := gvProxyWaitBackoff
+	logrus.Debug("checking that gvproxy is running")
+	for i := 0; i < gvProxyMaxBackoffAttempts; i++ {
+		err := unix.Access(gvproxySocket.GetPath(), unix.W_OK)
+		if err == nil {
+			return nil
+		}
+		time.Sleep(backoffWait)
+		backoffWait *= 2
+	}
+	return fmt.Errorf("unable to connect to gvproxy %q", gvproxySocket.GetPath())
+}

pkg/machine/applehv/vfkit.go

@@ -3,10 +3,11 @@
 package applehv
 
 import (
-	"github.com/containers/podman/v4/pkg/machine"
+	"github.com/containers/podman/v4/pkg/machine/vmconfigs"
 	vfConfig "github.com/crc-org/vfkit/pkg/config"
 )
 
+// TODO this signature could be an machineconfig
 func getDefaultDevices(imagePath, logPath, readyPath string) ([]vfConfig.VirtioDevice, error) {
 	var devices []vfConfig.VirtioDevice
 
@@ -53,11 +54,14 @@ func getIgnitionVsockDevice(path string) (vfConfig.VirtioDevice, error) {
 	return vfConfig.VirtioVsockNew(1024, path, true)
 }
 
-func VirtIOFsToVFKitVirtIODevice(fs machine.VirtIoFs) vfConfig.VirtioFs {
+func virtIOFsToVFKitVirtIODevice(mounts []vmconfigs.Mount) ([]vfConfig.VirtioDevice, error) {
-	return vfConfig.VirtioFs{
+	var virtioDevices []vfConfig.VirtioDevice
-		DirectorySharingConfig: vfConfig.DirectorySharingConfig{
+	for _, vol := range mounts {
-			MountTag: fs.Tag,
+		virtfsDevice, err := vfConfig.VirtioFsNew(vol.Source, vol.Tag)
-		},
+		if err != nil {
-		SharedDir: fs.Source,
+			return nil, err
 		}
+		virtioDevices = append(virtioDevices, virtfsDevice)
+	}
+	return virtioDevices, nil
 }

pkg/machine/applehv/vfkit/config.go

@@ -57,6 +57,9 @@ func (vf *VfkitHelper) getRawState() (define.Status, error) {
 	if err != nil {
 		return "", err
 	}
+	if err := serverResponse.Body.Close(); err != nil {
+		logrus.Error(err)
+	}
 	return ToMachineStatus(response.State)
 }
 
@@ -66,7 +69,7 @@ func (vf *VfkitHelper) getRawState() (define.Status, error) {
 func (vf *VfkitHelper) State() (define.Status, error) {
 	vmState, err := vf.getRawState()
 	if err == nil {
-		return vmState, err
+		return vmState, nil
 	}
 	if errors.Is(err, unix.ECONNREFUSED) {
 		return define.Stopped, nil
@@ -107,7 +110,7 @@ func (vf *VfkitHelper) Stop(force, wait bool) error {
 			waitErr = nil
 			break
 		}
-		waitDuration = waitDuration * 2
+		waitDuration *= 2
 		logrus.Debugf("backoff wait time: %s", waitDuration.String())
 		time.Sleep(waitDuration)
 	}

pkg/machine/define/vmfile.go

@@ -73,6 +73,7 @@ func NewMachineFile(path string, symlink *string) (*VMFile, error) {
 		return nil, errors.New("invalid symlink path")
 	}
 	mf := VMFile{Path: path}
+	logrus.Debugf("socket length for %s is %d", path, len(path))
 	if symlink != nil && len(path) > MaxSocketPathLength {
 		if err := mf.makeSymlink(symlink); err != nil && !errors.Is(err, os.ErrExist) {
 			return nil, err
@@ -100,5 +101,5 @@ func (m *VMFile) makeSymlink(symlink *string) error {
 // AppendToNewVMFile takes a given path and appends it to the existing vmfile path.  The new
 // VMFile is returned
 func (m *VMFile) AppendToNewVMFile(additionalPath string, symlink *string) (*VMFile, error) {
-	return NewMachineFile(filepath.Join(m.GetPath(), additionalPath), symlink)
+	return NewMachineFile(filepath.Join(m.Path, additionalPath), symlink)
 }

pkg/machine/e2e/machine_test.go

@@ -59,7 +59,7 @@ var _ = BeforeSuite(func() {
 
 	downloadLocation := os.Getenv("MACHINE_IMAGE")
 	if downloadLocation == "" {
-		downloadLocation, err = GetDownload()
+		downloadLocation, err = GetDownload(testProvider.VMType())
 		if err != nil {
 			Fail("unable to derive download disk from fedora coreos")
 		}
@@ -69,9 +69,15 @@ var _ = BeforeSuite(func() {
 		Fail("machine tests require a file reference to a disk image right now")
 	}
 
-	// TODO Fix or remove - this only works for qemu rn
+	var compressionExtension string
-	// compressionExtension := fmt.Sprintf(".%s", testProvider.Compression().String())
+	switch testProvider.VMType() {
-	compressionExtension := ".xz"
+	case define.AppleHvVirt:
+		compressionExtension = ".gz"
+	case define.HyperVVirt:
+		compressionExtension = ".zip"
+	default:
+		compressionExtension = ".xz"
+	}
 
 	suiteImageName = strings.TrimSuffix(path.Base(downloadLocation), compressionExtension)
 	fqImageName = filepath.Join(tmpDir, suiteImageName)

pkg/machine/e2e/pull_test.go

@@ -7,14 +7,16 @@ import (
 	"net/http"
 
 	"github.com/containers/podman/v4/pkg/machine"
+	"github.com/containers/podman/v4/pkg/machine/define"
 	"github.com/coreos/stream-metadata-go/fedoracoreos"
 	"github.com/coreos/stream-metadata-go/stream"
 	"github.com/sirupsen/logrus"
 )
 
-func GetDownload() (string, error) {
+func GetDownload(vmType define.VMType) (string, error) {
 	var (
 		fcosstable           stream.Stream
+		artifactType, format string
 	)
 	url := fedoracoreos.GetStreamURL("testing")
 	resp, err := http.Get(url.String())
@@ -34,6 +36,19 @@ func GetDownload() (string, error) {
 	if err := json.Unmarshal(body, &fcosstable); err != nil {
 		return "", err
 	}
+
+	switch vmType {
+	case define.AppleHvVirt:
+		artifactType = "applehv"
+		format = "raw.gz"
+	case define.HyperVVirt:
+		artifactType = "hyperv"
+		format = "vhdx.zip"
+	default:
+		artifactType = "qemu"
+		format = "qcow2.xz"
+	}
+
 	arch, ok := fcosstable.Architectures[machine.GetFcosArch()]
 	if !ok {
 		return "", fmt.Errorf("unable to pull VM image: no targetArch in stream")
@@ -42,17 +57,17 @@ func GetDownload() (string, error) {
 	if upstreamArtifacts == nil {
 		return "", fmt.Errorf("unable to pull VM image: no artifact in stream")
 	}
-	upstreamArtifact, ok := upstreamArtifacts["qemu"]
+	upstreamArtifact, ok := upstreamArtifacts[artifactType]
 	if !ok {
-		return "", fmt.Errorf("unable to pull VM image: no %s artifact in stream", "qemu")
+		return "", fmt.Errorf("unable to pull VM image: no %s artifact in stream", artifactType)
 	}
 	formats := upstreamArtifact.Formats
 	if formats == nil {
 		return "", fmt.Errorf("unable to pull VM image: no formats in stream")
 	}
-	formatType, ok := formats["qcow2.xz"]
+	formatType, ok := formats[format]
 	if !ok {
-		return "", fmt.Errorf("unable to pull VM image: no %s format in stream", "qcow2.xz")
+		return "", fmt.Errorf("unable to pull VM image: no %s format in stream", format)
 	}
 	disk := formatType.Disk
 	return disk.Location, nil

pkg/machine/provider/platform_darwin.go

@@ -5,14 +5,13 @@ import (
 	"os"
 
 	"github.com/containers/common/pkg/config"
-	"github.com/containers/podman/v4/pkg/machine"
 	"github.com/containers/podman/v4/pkg/machine/applehv"
 	"github.com/containers/podman/v4/pkg/machine/define"
-	"github.com/containers/podman/v4/pkg/machine/qemu"
+	"github.com/containers/podman/v4/pkg/machine/vmconfigs"
 	"github.com/sirupsen/logrus"
 )
 
-func Get() (machine.VirtProvider, error) {
+func Get() (vmconfigs.VMProvider, error) {
 	cfg, err := config.Default()
 	if err != nil {
 		return nil, err
@@ -28,10 +27,8 @@ func Get() (machine.VirtProvider, error) {
 
 	logrus.Debugf("Using Podman machine with `%s` virtualization provider", resolvedVMType.String())
 	switch resolvedVMType {
-	case define.QemuVirt:
-		return qemu.VirtualizationProvider(), nil
 	case define.AppleHvVirt:
-		return applehv.VirtualizationProvider(), nil
+		return new(applehv.AppleHVStubber), nil
 	default:
 		return nil, fmt.Errorf("unsupported virtualization provider: `%s`", resolvedVMType.String())
 	}

pkg/machine/qemu/stubber.go

@@ -11,6 +11,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/containers/podman/v4/pkg/machine/ignition"
+
 	"github.com/containers/common/pkg/config"
 	"github.com/containers/common/pkg/strongunits"
 	gvproxy "github.com/containers/gvisor-tap-vsock/pkg/types"
@@ -68,7 +70,7 @@ func (q *QEMUStubber) setQEMUCommandLine(mc *vmconfigs.MachineConfig) error {
 	return nil
 }
 
-func (q *QEMUStubber) CreateVM(opts define.CreateVMOpts, mc *vmconfigs.MachineConfig) error {
+func (q *QEMUStubber) CreateVM(opts define.CreateVMOpts, mc *vmconfigs.MachineConfig, _ *ignition.IgnitionBuilder) error {
 	monitor, err := command.NewQMPMonitor(opts.Name, opts.Dirs.RuntimeDir)
 	if err != nil {
 		return err

pkg/machine/shim/claim_unsupported.go

@@ -1,4 +1,4 @@
-//build: !darwin
+//go:build !darwin
 
 package shim
 

pkg/machine/shim/host.go

@@ -182,7 +182,7 @@ func Init(opts machineDefine.InitOptions, mp vmconfigs.VMProvider) (*vmconfigs.M
 		return nil, err
 	}
 
-	readyUnitFile, err := ignition.CreateReadyUnitFile(machineDefine.QemuVirt, nil)
+	readyUnitFile, err := ignition.CreateReadyUnitFile(mp.VMType(), nil)
 	if err != nil {
 		return nil, err
 	}
@@ -194,12 +194,8 @@ func Init(opts machineDefine.InitOptions, mp vmconfigs.VMProvider) (*vmconfigs.M
 	}
 	ignBuilder.WithUnit(readyUnit)
 
-	if err := ignBuilder.Build(); err != nil {
-		return nil, err
-	}
-
 	// Mounts
-	mc.Mounts = vmconfigs.CmdLineVolumesToMounts(opts.Volumes, mp.MountType())
+	mc.Mounts = CmdLineVolumesToMounts(opts.Volumes, mp.MountType())
 
 	// TODO AddSSHConnectionToPodmanSocket could take an machineconfig instead
 	if err := connection.AddSSHConnectionsToPodmanSocket(mc.HostUser.UID, mc.SSH.Port, mc.SSH.IdentityPath, mc.Name, mc.SSH.RemoteUsername, opts); err != nil {
@@ -211,7 +207,11 @@ func Init(opts machineDefine.InitOptions, mp vmconfigs.VMProvider) (*vmconfigs.M
 	}
 	callbackFuncs.Add(cleanup)
 
-	if err := mp.CreateVM(createOpts, mc); err != nil {
+	if err := mp.CreateVM(createOpts, mc, &ignBuilder); err != nil {
+		return nil, err
+	}
+
+	if err := ignBuilder.Build(); err != nil {
 		return nil, err
 	}
 
@@ -327,7 +327,6 @@ func Start(mc *vmconfigs.MachineConfig, mp vmconfigs.VMProvider, dirs *machineDe
 	if err != nil {
 		return err
 	}
-
 	// if there are generic things that need to be done, a preStart function could be added here
 	// should it be extensive
 

pkg/machine/shim/networking.go

@@ -6,6 +6,7 @@ import (
 	"net"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/containers/common/pkg/config"
@@ -101,6 +102,8 @@ func startNetworking(mc *vmconfigs.MachineConfig, provider vmconfigs.VMProvider)
 	}
 
 	c := cmd.Cmd(binary)
+
+	logrus.Debugf("gvproxy command-line: %s %s", binary, strings.Join(cmd.ToCmdline(), " "))
 	if err := c.Start(); err != nil {
 		return forwardSock, 0, fmt.Errorf("unable to execute: %q: %w", cmd.ToCmdline(), err)
 	}

pkg/machine/shim/volume.go

@@ -0,0 +1,30 @@
+package shim
+
+import (
+	"github.com/containers/podman/v4/pkg/machine"
+	"github.com/containers/podman/v4/pkg/machine/vmconfigs"
+)
+
+func CmdLineVolumesToMounts(volumes []string, volumeType vmconfigs.VolumeMountType) []vmconfigs.Mount {
+	mounts := []vmconfigs.Mount{}
+	for i, volume := range volumes {
+		var mount vmconfigs.Mount
+		tag, source, target, readOnly, _ := vmconfigs.SplitVolume(i, volume)
+		switch volumeType {
+		case vmconfigs.VirtIOFS:
+			virtioMount := machine.NewVirtIoFsMount(source, target, readOnly)
+			mount = virtioMount.ToMount()
+		default:
+			mount = vmconfigs.Mount{
+				Type:          volumeType.String(),
+				Tag:           tag,
+				Source:        source,
+				Target:        target,
+				ReadOnly:      readOnly,
+				OriginalInput: volume,
+			}
+		}
+		mounts = append(mounts, mount)
+	}
+	return mounts
+}

pkg/machine/sockets/sockets.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/containers/podman/v4/pkg/machine/define"
+	"github.com/sirupsen/logrus"
 )
 
 // SetSocket creates a new machine file for the socket and assigns it to
@@ -33,10 +34,12 @@ func ReadySocketPath(runtimeDir, machineName string) string {
 func ListenAndWaitOnSocket(errChan chan<- error, listener net.Listener) {
 	conn, err := listener.Accept()
 	if err != nil {
+		logrus.Debug("failed to connect to ready socket")
 		errChan <- err
 		return
 	}
 	_, err = bufio.NewReader(conn).ReadString('\n')
+	logrus.Debug("ready ack received")
 
 	if closeErr := conn.Close(); closeErr != nil {
 		errChan <- closeErr

pkg/machine/vmconfigs/config.go

@@ -8,6 +8,7 @@ import (
 	"github.com/containers/common/pkg/strongunits"
 	gvproxy "github.com/containers/gvisor-tap-vsock/pkg/types"
 	"github.com/containers/podman/v4/pkg/machine/define"
+	"github.com/containers/podman/v4/pkg/machine/ignition"
 	"github.com/containers/podman/v4/pkg/machine/qemu/command"
 	"github.com/containers/storage/pkg/lockfile"
 )
@@ -106,7 +107,7 @@ func (f fcosMachineImage) path() string {
 }
 
 type VMProvider interface { //nolint:interfacebloat
-	CreateVM(opts define.CreateVMOpts, mc *MachineConfig) error
+	CreateVM(opts define.CreateVMOpts, mc *MachineConfig, builder *ignition.IgnitionBuilder) error
 	GetHyperVisorVMs() ([]string, error)
 	MountType() VolumeMountType
 	MountVolumesToVM(mc *MachineConfig, quiet bool) error

pkg/machine/vmconfigs/machine.go

@@ -10,14 +10,12 @@ import (
 	"strings"
 	"time"
 
-	"github.com/containers/podman/v4/pkg/machine/connection"
-
-	"github.com/sirupsen/logrus"
-
 	define2 "github.com/containers/podman/v4/libpod/define"
+	"github.com/containers/podman/v4/pkg/machine/connection"
 	"github.com/containers/podman/v4/pkg/machine/define"
 	"github.com/containers/podman/v4/pkg/machine/lock"
 	"github.com/containers/podman/v4/utils"
+	"github.com/sirupsen/logrus"
 )
 
 /*
@@ -235,7 +233,15 @@ func (mc *MachineConfig) ReadySocket() (*define.VMFile, error) {
 	if err != nil {
 		return nil, err
 	}
-	return rtDir.AppendToNewVMFile(mc.Name+".sock", nil)
+	return readySocket(mc.Name, rtDir)
+}
+
+func (mc *MachineConfig) GVProxySocket() (*define.VMFile, error) {
+	machineRuntimeDir, err := mc.RuntimeDir()
+	if err != nil {
+		return nil, err
+	}
+	return gvProxySocket(mc.Name, machineRuntimeDir)
 }
 
 func (mc *MachineConfig) LogFile() (*define.VMFile, error) {
@@ -264,6 +270,14 @@ func (mc *MachineConfig) Kind() (define.VMType, error) {
 	return define.UnknownVirt, nil
 }
 
+func (mc *MachineConfig) IsFirstBoot() (bool, error) {
+	never, err := time.Parse(time.RFC3339, "0001-01-01T00:00:00Z")
+	if err != nil {
+		return false, err
+	}
+	return mc.LastUp == never, nil
+}
+
 // LoadMachineByName returns a machine config based on the vm name and provider
 func LoadMachineByName(name string, dirs *define.MachineDirs) (*MachineConfig, error) {
 	fullPath, err := dirs.ConfigDir.AppendToNewVMFile(name+".json", nil)

pkg/machine/vmconfigs/sockets.go

@@ -0,0 +1,17 @@
+//go:build !darwin
+
+package vmconfigs
+
+import (
+	"fmt"
+
+	"github.com/containers/podman/v4/pkg/machine/define"
+)
+
+func gvProxySocket(name string, machineRuntimeDir *define.VMFile) (*define.VMFile, error) {
+	return machineRuntimeDir.AppendToNewVMFile(fmt.Sprintf("%s-gvproxy.sock", name), nil)
+}
+
+func readySocket(name string, machineRuntimeDir *define.VMFile) (*define.VMFile, error) {
+	return machineRuntimeDir.AppendToNewVMFile(name+".sock", nil)
+}

pkg/machine/vmconfigs/sockets_darwin.go

@@ -0,0 +1,17 @@
+package vmconfigs
+
+import (
+	"fmt"
+
+	"github.com/containers/podman/v4/pkg/machine/define"
+)
+
+func gvProxySocket(name string, machineRuntimeDir *define.VMFile) (*define.VMFile, error) {
+	socketName := fmt.Sprintf("%s-gvproxy.sock", name)
+	return machineRuntimeDir.AppendToNewVMFile(socketName, &socketName)
+}
+
+func readySocket(name string, machineRuntimeDir *define.VMFile) (*define.VMFile, error) {
+	socketName := name + ".sock"
+	return machineRuntimeDir.AppendToNewVMFile(socketName, &socketName)
+}

pkg/machine/vmconfigs/volumes.go

@@ -58,20 +58,3 @@ func SplitVolume(idx int, volume string) (string, string, string, bool, string)
 	readonly, securityModel := extractMountOptions(paths)
 	return tag, source, target, readonly, securityModel
 }
-
-func CmdLineVolumesToMounts(volumes []string, volumeType VolumeMountType) []Mount {
-	mounts := []Mount{}
-	for i, volume := range volumes {
-		tag, source, target, readOnly, _ := SplitVolume(i, volume)
-		mount := Mount{
-			Type:          volumeType.String(),
-			Tag:           tag,
-			Source:        source,
-			Target:        target,
-			ReadOnly:      readOnly,
-			OriginalInput: volume,
-		}
-		mounts = append(mounts, mount)
-	}
-	return mounts
-}

pkg/machine/volumes.go

@@ -61,3 +61,13 @@ func NewVirtIoFsMount(src, target string, readOnly bool) VirtIoFs {
 	vfs.Tag = vfs.unitName()
 	return vfs
 }
+
+func MountToVirtIOFs(mnt vmconfigs.Mount) VirtIoFs {
+	return VirtIoFs{
+		VolumeKind: VirtIOFsVk,
+		ReadOnly:   mnt.ReadOnly,
+		Source:     mnt.Source,
+		Tag:        mnt.Tag,
+		Target:     mnt.Target,
+	}
+}