pkg/machine/e2e: improve timeout handling

In case of timeouts actually log the command again and make sure to send
SIGABRT to the process as go will create a useful stack strace where we
can see where things are hanging. It also kill the process unlike the
default Eventually().Should(Exit()) call the leaves the process around.

The output will be captured by default in the log so we just see the
stack trace there.

And while at it bump the timout up to 10 mins, we are hitting hard
flakes in CI where machine init takes longer than 5 mins for unknown
reasons but this seems to be good enough.

Signed-off-by: Paul Holzinger <pholzing@redhat.com>
This commit is contained in:
Paul Holzinger 2024-07-02 11:26:49 +02:00
parent cf98506546
commit ada4e1a8c1
No known key found for this signature in database
GPG Key ID: EB145DD938A3CAF2
1 changed files with 12 additions and 2 deletions

View File

@ -9,6 +9,7 @@ import (
"slices" "slices"
"strconv" "strconv"
"strings" "strings"
"syscall"
"time" "time"
"github.com/containers/podman/v5/pkg/machine" "github.com/containers/podman/v5/pkg/machine"
@ -24,7 +25,7 @@ import (
var originalHomeDir = os.Getenv("HOME") var originalHomeDir = os.Getenv("HOME")
const ( const (
defaultTimeout = 240 * time.Second defaultTimeout = 10 * time.Minute
) )
type machineCommand interface { type machineCommand interface {
@ -53,7 +54,16 @@ type machineTestBuilder struct {
// waitWithTimeout waits for a command to complete for a given // waitWithTimeout waits for a command to complete for a given
// number of seconds // number of seconds
func (ms *machineSession) waitWithTimeout(timeout time.Duration) { func (ms *machineSession) waitWithTimeout(timeout time.Duration) {
Eventually(ms, timeout).Should(Exit()) Eventually(ms, timeout).Should(Exit(), func() string {
// Note eventually does not kill the command as such the command is leaked forever without killing it
// Also let's use SIGABRT to create a go stack trace so in case there is a deadlock we see it.
ms.Signal(syscall.SIGABRT)
// Give some time to let the command print the output so it is not printed much later
// in the log at the wrong place.
time.Sleep(1 * time.Second)
return fmt.Sprintf("command timed out after %fs: %v",
timeout.Seconds(), ms.Command.Args)
})
} }
func (ms *machineSession) Bytes() []byte { func (ms *machineSession) Bytes() []byte {