diff --git a/cmd/podman/exec.go b/cmd/podman/exec.go
index deff44a922..0684da842e 100644
--- a/cmd/podman/exec.go
+++ b/cmd/podman/exec.go
@@ -125,5 +125,10 @@ func execCmd(c *cliconfig.ExecValues) error {
 	streams.AttachError = true
 	streams.AttachInput = true
 
-	return ctr.Exec(c.Tty, c.Privileged, envs, cmd, c.User, c.Workdir, streams, c.PreserveFDs)
+	err = ctr.Exec(c.Tty, c.Privileged, envs, cmd, c.User, c.Workdir, streams, c.PreserveFDs)
+	if errors.Cause(err) == libpod.ErrCtrStateInvalid {
+		exitCode = 126
+	}
+
+	return err
 }
diff --git a/libpod/container_api.go b/libpod/container_api.go
index 1894780de3..52d3afc0ac 100644
--- a/libpod/container_api.go
+++ b/libpod/container_api.go
@@ -241,7 +241,7 @@ func (c *Container) Exec(tty, privileged bool, env, cmd []string, user, workDir
 
 	// TODO can probably relax this once we track exec sessions
 	if conState != ContainerStateRunning {
-		return errors.Errorf("cannot exec into container that is not running")
+		return errors.Wrapf(ErrCtrStateInvalid, "cannot exec into container that is not running")
 	}
 	if privileged || c.config.Privileged {
 		capList = caps.GetAllCapabilities()