diff --git a/cmd/podman/machine/rm.go b/cmd/podman/machine/rm.go
index dff2cd86ec..a14d4784db 100644
--- a/cmd/podman/machine/rm.go
+++ b/cmd/podman/machine/rm.go
@@ -120,14 +120,12 @@ func rm(_ *cobra.Command, args []string) error {
 	// All actual removal of files and vms should occur after this
 	//
 
-	// TODO Should this be a hard error?
 	if err := providerRm(); err != nil {
-		logrus.Errorf("failed to remove virtual machine from provider for %q", vmName)
+		logrus.Errorf("failed to remove virtual machine from provider for %q: %v", vmName, err)
 	}
 
-	// TODO Should this be a hard error?
 	if err := genericRm(); err != nil {
-		logrus.Error("failed to remove machines files")
+		return fmt.Errorf("failed to remove machines files: %v", err)
 	}
 	newMachineEvent(events.Remove, events.Event{Name: vmName})
 	return nil
diff --git a/pkg/machine/qemu/machine.go b/pkg/machine/qemu/machine.go
index 19e8368325..ce71243c94 100644
--- a/pkg/machine/qemu/machine.go
+++ b/pkg/machine/qemu/machine.go
@@ -16,6 +16,7 @@ import (
 	"time"
 
 	"github.com/containers/common/pkg/config"
+	"github.com/containers/podman/v5/pkg/errorhandling"
 	"github.com/containers/podman/v5/pkg/machine/define"
 	"github.com/containers/podman/v5/pkg/machine/vmconfigs"
 	"github.com/digitalocean/go-qemu/qmp"
@@ -237,7 +238,15 @@ func (q *QEMUStubber) Remove(mc *vmconfigs.MachineConfig) ([]string, func() erro
 	}
 
 	return qemuRmFiles, func() error {
-		return nil
+		var errs []error
+		if err := mc.QEMUHypervisor.QEMUPidPath.Delete(); err != nil {
+			errs = append(errs, err)
+		}
+
+		if err := mc.QEMUHypervisor.QMPMonitor.Address.Delete(); err != nil {
+			errs = append(errs, err)
+		}
+		return errorhandling.JoinErrors(errs)
 	}, nil
 }
 
diff --git a/pkg/machine/vmconfigs/machine.go b/pkg/machine/vmconfigs/machine.go
index b18f6c49ac..ceb710a353 100644
--- a/pkg/machine/vmconfigs/machine.go
+++ b/pkg/machine/vmconfigs/machine.go
@@ -11,6 +11,7 @@ import (
 	"time"
 
 	define2 "github.com/containers/podman/v5/libpod/define"
+	"github.com/containers/podman/v5/pkg/errorhandling"
 	"github.com/containers/podman/v5/pkg/machine/connection"
 	"github.com/containers/podman/v5/pkg/machine/define"
 	"github.com/containers/podman/v5/pkg/machine/lock"
@@ -184,28 +185,32 @@ func (mc *MachineConfig) Remove(saveIgnition, saveImage bool) ([]string, func()
 	}
 
 	mcRemove := func() error {
+		var errs []error
+		if err := connection.RemoveConnections(mc.Name, mc.Name+"-root"); err != nil {
+			errs = append(errs, err)
+		}
+
 		if !saveIgnition {
 			if err := ignitionFile.Delete(); err != nil {
-				logrus.Error(err)
+				errs = append(errs, err)
 			}
 		}
 		if !saveImage {
 			if err := mc.ImagePath.Delete(); err != nil {
-				logrus.Error(err)
+				errs = append(errs, err)
 			}
 		}
-		if err := mc.configPath.Delete(); err != nil {
-			logrus.Error(err)
-		}
 		if err := readySocket.Delete(); err != nil {
-			logrus.Error()
+			errs = append(errs, err)
 		}
 		if err := logPath.Delete(); err != nil {
-			logrus.Error(err)
+			errs = append(errs, err)
 		}
-		// TODO This should be bumped up into delete and called out in the text given then
-		// are not technically files per'se
-		return connection.RemoveConnections(mc.Name, mc.Name+"-root")
+
+		if err := mc.configPath.Delete(); err != nil {
+			errs = append(errs, err)
+		}
+		return errorhandling.JoinErrors(errs)
 	}
 
 	return rmFiles, mcRemove, nil