diff --git a/libpod/runtime.go b/libpod/runtime.go
index 73c607ad48..1aea7b4fe6 100644
--- a/libpod/runtime.go
+++ b/libpod/runtime.go
@@ -218,11 +218,6 @@ func newRuntimeFromConfig(ctx context.Context, conf *config.Config, options ...R
 		if runtime.store != nil {
 			_, _ = runtime.store.Shutdown(false)
 		}
-		// For `systemctl stop podman.service` support, exit code should be 0
-		if sig == syscall.SIGTERM {
-			os.Exit(0)
-		}
-		os.Exit(1)
 		return nil
 	}); err != nil && !errors.Is(err, shutdown.ErrHandlerExists) {
 		logrus.Errorf("Registering shutdown handler for libpod: %v", err)
diff --git a/libpod/shutdown/handler.go b/libpod/shutdown/handler.go
index d7fd5b5323..ac41899e0e 100644
--- a/libpod/shutdown/handler.go
+++ b/libpod/shutdown/handler.go
@@ -28,10 +28,18 @@ var (
 	shutdownInhibit sync.RWMutex
 	logrus          = logrusImport.WithField("PID", os.Getpid())
 	ErrNotStarted   = errors.New("shutdown signal handler has not yet been started")
+	// exitCode used to exit once we are done with all signal handlers, by default 1
+	exitCode = 1
 )
 
+// SetExitCode when we exit after we ran all shutdown handlers, it should be positive.
+func SetExitCode(i int) {
+	exitCode = i
+}
+
 // Start begins handling SIGTERM and SIGINT and will run the given on-signal
-// handlers when one is called. This can be cancelled by calling Stop().
+// handlers when one is called and then exit with the exit code of 1 if not
+// overwritten with SetExitCode(). This can be cancelled by calling Stop().
 func Start() error {
 	if sigChan != nil {
 		// Already running, do nothing.
@@ -75,6 +83,7 @@ func Start() error {
 			}
 			handlerLock.Unlock()
 			shutdownInhibit.Unlock()
+			os.Exit(exitCode)
 			return
 		}
 	}()
diff --git a/pkg/api/server/server.go b/pkg/api/server/server.go
index 4c07390d0d..41dff40cef 100644
--- a/pkg/api/server/server.go
+++ b/pkg/api/server/server.go
@@ -197,7 +197,13 @@ func (s *APIServer) Serve() error {
 	s.setupPprof()
 
 	if err := shutdown.Register("service", func(sig os.Signal) error {
-		return s.Shutdown(true)
+		err := s.Shutdown(true)
+		if err == nil {
+			// For `systemctl stop podman.service` support, exit code should be 0
+			// but only if we did indeed gracefully shutdown
+			shutdown.SetExitCode(0)
+		}
+		return err
 	}); err != nil {
 		return err
 	}