libgit2/managed: fix race issues in ssh transport

Race conditions in ssh smart subtransport caused some goroutines to
panic, resulting in crashing the whole controller, mostly evident in
image-automation-controller CI runs. Panic recovery in the main thread
do not handle goroutine panics. So, the existing panic recovery code in
libgit2 Checkout() methods weren't able to handle it.

This change groups the fields in ssh smart subtransport that may be
accessed by multiple goroutines into a new struct with a mutex. Also
adds panic recovery in the created goroutine to handle any other
possible panics.

Signed-off-by: Sunny <darkowlzz@protonmail.com>

pkg/git/libgit2/managed/ssh.go

@@ -46,12 +46,14 @@ package managed
 import (
 	"context"
 	"crypto/sha256"
+	"errors"
 	"fmt"
 	"io"
 	"net"
 	"net/url"
 	"runtime"
 	"strings"
+	"sync"
 	"time"
 
 	"golang.org/x/crypto/ssh"
@@ -83,16 +85,22 @@ func sshSmartSubtransportFactory(remote *git2go.Remote, transport *git2go.Transp
 type sshSmartSubtransport struct {
 	transport *git2go.Transport
 
-	lastAction    git2go.SmartServiceAction
+	lastAction git2go.SmartServiceAction
+	stdin      io.WriteCloser
+	stdout     io.Reader
+	addr       string
+	ctx        context.Context
+
+	con connection
+}
+
+type connection struct {
 	conn          net.Conn
 	client        *ssh.Client
 	session       *ssh.Session
-	stdin         io.WriteCloser
-	stdout        io.Reader
 	currentStream *sshSmartSubtransportStream
-	addr          string
 	connected     bool
-	ctx           context.Context
+	m             sync.Mutex
 }
 
 func (t *sshSmartSubtransport) Action(transportOptionsURL string, action git2go.SmartServiceAction) (git2go.SmartSubtransportStream, error) {
@@ -128,17 +136,17 @@ func (t *sshSmartSubtransport) Action(transportOptionsURL string, action git2go.
 	var cmd string
 	switch action {
 	case git2go.SmartServiceActionUploadpackLs, git2go.SmartServiceActionUploadpack:
-		if t.currentStream != nil {
+		if t.con.currentStream != nil {
 			if t.lastAction == git2go.SmartServiceActionUploadpackLs {
-				return t.currentStream, nil
+				return t.con.currentStream, nil
 			}
 		}
 		cmd = fmt.Sprintf("git-upload-pack '%s'", uPath)
 
 	case git2go.SmartServiceActionReceivepackLs, git2go.SmartServiceActionReceivepack:
-		if t.currentStream != nil {
+		if t.con.currentStream != nil {
 			if t.lastAction == git2go.SmartServiceActionReceivepackLs {
-				return t.currentStream, nil
+				return t.con.currentStream, nil
 			}
 		}
 		cmd = fmt.Sprintf("git-receive-pack '%s'", uPath)
@@ -147,7 +155,7 @@ func (t *sshSmartSubtransport) Action(transportOptionsURL string, action git2go.
 		return nil, fmt.Errorf("unexpected action: %v", action)
 	}
 
-	if t.connected {
+	if t.con.connected {
 		// Disregard errors from previous stream, futher details inside Close().
 		_ = t.Close()
 	}
@@ -185,21 +193,23 @@ func (t *sshSmartSubtransport) Action(transportOptionsURL string, action git2go.
 	if err != nil {
 		return nil, err
 	}
-	t.connected = true
+	t.con.m.Lock()
+	t.con.connected = true
+	t.con.m.Unlock()
 
 	traceLog.Info("[ssh]: creating new ssh session")
-	if t.session, err = t.client.NewSession(); err != nil {
+	if t.con.session, err = t.con.client.NewSession(); err != nil {
 		return nil, err
 	}
 
-	if t.stdin, err = t.session.StdinPipe(); err != nil {
+	if t.stdin, err = t.con.session.StdinPipe(); err != nil {
 		return nil, err
 	}
 
 	var w *io.PipeWriter
 	var reader io.Reader
 	t.stdout, w = io.Pipe()
-	if reader, err = t.session.StdoutPipe(); err != nil {
+	if reader, err = t.con.session.StdoutPipe(); err != nil {
 		return nil, err
 	}
 
@@ -208,7 +218,15 @@ func (t *sshSmartSubtransport) Action(transportOptionsURL string, action git2go.
 	//
 	// xref: https://github.com/golang/crypto/blob/eb4f295cb31f7fb5d52810411604a2638c9b19a2/ssh/session.go#L553-L558
 	go func() error {
-		defer w.Close()
+		defer func() {
+			w.Close()
+
+			// In case this goroutine panics, handle recovery.
+			if r := recover(); r != nil {
+				traceLog.Error(errors.New(r.(string)),
+					"[ssh]: recovered from libgit2 ssh smart subtransport panic", "address", t.addr)
+			}
+		}()
 
 		var cancel context.CancelFunc
 		ctx := t.ctx
@@ -226,9 +244,12 @@ func (t *sshSmartSubtransport) Action(transportOptionsURL string, action git2go.
 				return nil
 
 			default:
-				if !t.connected {
+				t.con.m.Lock()
+				if !t.con.connected {
+					t.con.m.Unlock()
 					return nil
 				}
+				t.con.m.Unlock()
 
 				_, err := io.Copy(w, reader)
 				if err != nil {
@@ -240,16 +261,16 @@ func (t *sshSmartSubtransport) Action(transportOptionsURL string, action git2go.
 	}()
 
 	traceLog.Info("[ssh]: run on remote", "cmd", cmd)
-	if err := t.session.Start(cmd); err != nil {
+	if err := t.con.session.Start(cmd); err != nil {
 		return nil, err
 	}
 
 	t.lastAction = action
-	t.currentStream = &sshSmartSubtransportStream{
+	t.con.currentStream = &sshSmartSubtransportStream{
 		owner: t,
 	}
 
-	return t.currentStream, nil
+	return t.con.currentStream, nil
 }
 
 func (t *sshSmartSubtransport) createConn(addr string, sshConfig *ssh.ClientConfig) error {
@@ -265,8 +286,8 @@ func (t *sshSmartSubtransport) createConn(addr string, sshConfig *ssh.ClientConf
 		return err
 	}
 
-	t.conn = conn
-	t.client = ssh.NewClient(c, chans, reqs)
+	t.con.conn = conn
+	t.con.client = ssh.NewClient(c, chans, reqs)
 
 	return nil
 }
@@ -282,31 +303,35 @@ func (t *sshSmartSubtransport) createConn(addr string, sshConfig *ssh.ClientConf
 // SmartSubTransport (i.e. unreleased resources, staled connections).
 func (t *sshSmartSubtransport) Close() error {
 	traceLog.Info("[ssh]: sshSmartSubtransport.Close()", "server", t.addr)
-	t.currentStream = nil
-	if t.client != nil && t.stdin != nil {
+	t.con.m.Lock()
+	defer t.con.m.Unlock()
+	t.con.currentStream = nil
+	if t.con.client != nil && t.stdin != nil {
 		_ = t.stdin.Close()
 	}
-	t.client = nil
+	t.con.client = nil
 
-	if t.session != nil {
+	if t.con.session != nil {
 		traceLog.Info("[ssh]: session.Close()", "server", t.addr)
-		_ = t.session.Close()
+		_ = t.con.session.Close()
 	}
-	t.session = nil
+	t.con.session = nil
 
 	return nil
 }
 
 func (t *sshSmartSubtransport) Free() {
 	traceLog.Info("[ssh]: sshSmartSubtransport.Free()")
-	if t.client != nil {
-		_ = t.client.Close()
+	if t.con.client != nil {
+		_ = t.con.client.Close()
 	}
 
-	if t.conn != nil {
-		_ = t.conn.Close()
+	if t.con.conn != nil {
+		_ = t.con.conn.Close()
 	}
-	t.connected = false
+	t.con.m.Lock()
+	t.con.connected = false
+	t.con.m.Unlock()
 }
 
 type sshSmartSubtransportStream struct {