Merge pull request #13 from spiffe/dependabot/go_modules/github.com/go-jose/go-jose/v4-4.0.5

Bump github.com/go-jose/go-jose/v4 from 4.0.4 to 4.0.5

.goreleaser.yaml

@@ -15,21 +15,55 @@ builds:
       - CGO_ENABLED=0
     goos:
       - linux
-    main: ./cmd
+    main: ./cmd/spire-ha-agent
     binary: spire-ha-agent
+    id: spire-ha-agent
+  - env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+    main: ./cmd/spire-trust-sync-helper
+    binary: spire-trust-sync-helper
+    id: spire-trust-sync-helper
 
 archives:
   - format: tar.gz
+    builds:
+      - spire-ha-agent
+    id: spire-ha-agent
     # this name template makes the OS and Arch compatible with the results of `uname`.
     name_template: >-
-      {{ .ProjectName }}_
+      spire-ha-agent_
       {{- title .Os }}_
       {{- if eq .Arch "amd64" }}x86_64
       {{- else if eq .Arch "386" }}i386
       {{- else }}{{ .Arch }}{{ end }}
       {{- if .Arm }}v{{ .Arm }}{{ end }}
     files:
-      - systemd/*
+      - systemd/spire-ha-agent@.service
+      - systemd/spire-socat@.service
+      - config/socat/*
+      - README.md
+      - LICENSE
+    # use zip for windows archives
+    format_overrides:
+      - goos: windows
+        format: zip
+  - format: tar.gz
+    builds:
+      - spire-trust-sync-helper
+    id: spire-trust-sync-helper
+    # this name template makes the OS and Arch compatible with the results of `uname`.
+    name_template: >-
+      spire-trust-sync-helper_
+      {{- title .Os }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else if eq .Arch "386" }}i386
+      {{- else }}{{ .Arch }}{{ end }}
+      {{- if .Arm }}v{{ .Arm }}{{ end }}
+    files:
+      - systemd/spire-trust-sync@.service  
+      - config/trust-sync/default.conf
       - README.md
       - LICENSE
     # use zip for windows archives
@@ -45,7 +79,8 @@ changelog:
       - "^test:"
 
 kos:
-  - repository: ghcr.io/spiffe/spire-ha-agent
+  - repositories:
+      - ghcr.io/spiffe/spire-ha-agent
     tags:
       - "{{.Version}}"
       - latest

Dockerfile

@@ -1,10 +1,11 @@
 FROM docker.io/library/golang:1.23.2 as build
 
-COPY * /build/
+COPY . /build/
 WORKDIR /build
 
 RUN \
-  GOPROXY=direct CGO_ENABLED=0 go build .
+  GOPROXY=direct CGO_ENABLED=0 go build cmd/spire-ha-agent/main.go && \
+  mv main spire-ha-agent
 
 FROM gcr.io/distroless/static-debian12
 COPY --from=build /build/spire-ha-agent /usr/bin/spire-ha-agent

README.md

@@ -10,11 +10,22 @@ An agent to setup a SPIRE HA TrustDomain using two independent SPIRE Servers
 This code is very early in development and is very experimental. Please do not use it in production yet. Please do consider testing it out, provide feedback,
 and maybe provide fixes.
 
-## Diagram
-
-![diagram](diagram.png)
-
 ## How it Works
 
 If the trust bundles of both servers are presented to the workload, it will not care which server instance a certificate is issued from. This agent provides
-both trust bundles to the end user as one trust bundle, and will contact whichever server is responding to respond to x509 certificate or jwt token requests.
+both trust bundles to the end user as one trust bundle, and will contact whichever server is responding to respond to x509 certificate or JWT token requests.
+
+# Basic Setup
+
+## Simple Diagram
+
+![diagram](diagram.png)
+
+# Advanced setup
+
+While the basic setup allows a server to go down and workloads to continue to operate normally, it has a drawback. It requires both servers to be up during spire-ha-agent startup. This restriction can be eliminated by making the trust bundle of the other server available. The spire-trust-sync service can be used to do so.
+
+## Cross Linked Trust Diagram
+
+![diagram](diagram2.png)
+

cmd/spire-ha-agent/main.go

@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"crypto/x509"
 	"reflect"
+	"slices"
 	"sync"
 	"strconv"
 	"os"
@@ -64,8 +65,10 @@ type clientSet struct {
 	clientOK bool
 	debugClient agentdebug.DebugClient
 	delegatedClient agentdelegated.DelegatedIdentityClient
-	bundle *x509bundle.Set
-	jwtBundles map[string]jose.JSONWebKeySet
+	ourX509Bundle *x509bundle.Bundle
+	haX509Bundle *x509bundle.Bundle
+	ourJWTBundle *jose.JSONWebKeySet
+	haJWTBundle *jose.JSONWebKeySet
 }
 
 func ConcatRawCertsFromCerts(certs []*x509.Certificate) []byte {
@@ -460,7 +463,7 @@ func parseX509Bundles(bun map[string][]byte) (*x509bundle.Set, error) {
 	return x509bundle.NewSet(bundles...), nil
 }
 
-func setupClient(ls *server, clientName string, id int, mainSockName string, adminSocketName string, cs *clientSet) {
+func setupClient(ls *server, clientName string, id int, adminSocketName string, cs *clientSet) {
 	//Raw client code: https://github.com/spiffe/go-spiffe/blob/main/v2/workloadapi/client.go#L255
 	var dialOptions []grpc.DialOption
 //	var conn *grpc.ClientConn
@@ -471,8 +474,6 @@ func setupClient(ls *server, clientName string, id int, mainSockName string, adm
                 log.Fatalf("Failed to dial context: %v", err)
         }
 
-	ls.x509BundleUpdate = make(chan x509BundleUpdated)
-	ls.jwtBundleUpdate = make(chan jwtBundleUpdated)
 	cs.delegatedClient = agentdelegated.NewDelegatedIdentityClient(dconn)
 	cs.debugClient = agentdebug.NewDebugClient(dconn)
 	go func() {
@@ -528,6 +529,7 @@ func setupClient(ls *server, clientName string, id int, mainSockName string, adm
 				}
 				log.Printf("Pushing x509 bundle")
 				ls.x509BundleUpdate <- x509BundleUpdated{id, bundles}
+
 			}
 		}
 	}()
@@ -552,8 +554,8 @@ func setupClient(ls *server, clientName string, id int, mainSockName string, adm
 				bundles := resp.GetBundles()
 				jwksBundles := make(map[string]jose.JSONWebKeySet)
 				for td, bundle := range bundles {
-					log.Printf("jwt Bundle: %s %s", td, string(bundle))
-					//log.Printf("jwt Bundle: %s %d", td, len(bundle))
+					//log.Printf("jwt Bundle: %s %s", td, string(bundle))
+					log.Printf("jwt Bundle: %s %d", td, len(bundle))
 					jwks := new(jose.JSONWebKeySet)
 					if err := json.NewDecoder(bytes.NewReader(bundle)).Decode(jwks); err != nil {
 						log.Printf("failed to decode key set: %v", err)
@@ -610,8 +612,24 @@ func main() {
 		grpc.StreamInterceptor(streamInterceptor),
 	)
 
-	setupClient(ls, "clientA", 0, "unix:///var/run/spire/agent/sockets/a/public/api.sock", "unix:///var/run/spire/agent/sockets/a/private/admin.sock", &ls.clients[0])
-	setupClient(ls, "clientB", 1, "unix:///var/run/spire/agent/sockets/b/public/api.sock", "unix:///var/run/spire/agent/sockets/b/private/admin.sock", &ls.clients[1])
+	apath := "unix:///var/run/spire/agent/sockets/a/private/admin.sock"
+	bpath := "unix:///var/run/spire/agent/sockets/b/private/admin.sock"
+	aname := "SPIRE_HA_AGENT_SOCKET"
+	if ls.multi {
+		aname = "SPIRE_HA_AGENT_SOCKET_A"
+	}
+	if os.Getenv(aname) != "" {
+		apath = os.Getenv(aname)
+	}
+	ls.x509BundleUpdate = make(chan x509BundleUpdated)
+	ls.jwtBundleUpdate = make(chan jwtBundleUpdated)
+	go setupClient(ls, "clientA", 0, apath, &ls.clients[0])
+	if ls.multi {
+		if os.Getenv("SPIRE_HA_AGENT_SOCKET_B") != "" {
+			bpath = os.Getenv("SPIRE_HA_AGENT_SOCKET_B")
+		}
+		go setupClient(ls, "clientB", 1, bpath, &ls.clients[1])
+	}
 
 	go func() {
 		for {
@@ -622,25 +640,53 @@ func main() {
 	}()
 
 	go func() {
+		var ourTD *spiffeid.TrustDomain
+		haTD, _ := spiffeid.TrustDomainFromString("spiffe://spire-ha")
 		log.Printf("Listening for x509 bundle updates\n")
 		for u := range ls.x509BundleUpdate {
 			log.Printf("Got update for %d\n", u.id)
-			ls.clients[u.id].bundle = u.bundle
-			if ls.clients[0].bundle != nil && ls.clients[1].bundle != nil {
-				log.Printf("We got two bundles\n")
-				var rawBundles map[string][]byte = make(map[string][]byte)
-				for _, bundle := range ls.clients[0].bundle.Bundles() {
-					td := bundle.TrustDomain()
-					if tdb, ok := ls.clients[1].bundle.Get(td); ok {
-						for _, cert := range tdb.X509Authorities() {
-							if !bundle.HasX509Authority(cert) {
-								bundle.AddX509Authority(cert)
-							}
-						}
-					}
-					rawBundles[td.String()] = ConcatRawCertsFromCerts(bundle.X509Authorities())
+			bl := u.bundle.Len()
+			log.Printf("Bundle count on update: %d\n", bl)
+			if bl < 1 {
+				log.Printf("Bad bundle pushed by the spire-agent.\n")
+				os.Exit(1)
+			}
+			if bl > 2 {
+				log.Printf("Too many federated bundles in the trust bundle. Please reconfigure the spire-ha-agent entry.\n")
+				os.Exit(1)
+			}
+			if bl == 2 && !u.bundle.Has(haTD) {
+				log.Printf("spire-ha trust bundle not found. Please reconfigure the spire-ha-agent entry.\n")
+				os.Exit(1)
+			}
+			for _, bundle := range u.bundle.Bundles() {
+				td := bundle.TrustDomain()
+				if td.Name() == "spire-ha" {
+					ls.clients[u.id].haX509Bundle = bundle
+					continue
 				}
+				if ourTD == nil {
+					ourTD = &td
+					log.Printf("Our trust domain detected as: %s\n", ourTD.Name())
+				}
+				ls.clients[u.id].ourX509Bundle = bundle
+			}
+			bundles := slices.DeleteFunc([]*x509bundle.Bundle{ls.clients[0].ourX509Bundle, ls.clients[0].haX509Bundle, ls.clients[1].ourX509Bundle, ls.clients[1].haX509Bundle}, func(b *x509bundle.Bundle) bool {
+				return b == nil
+			})
+			totalBundles := len(bundles)
+			if totalBundles > 1 || !ls.multi {
+				log.Printf("We got %d x509 bundles\n", totalBundles)
+				var rawBundles map[string][]byte = make(map[string][]byte)
+				bundle := x509bundle.New(*ourTD)
+				for _, tb := range bundles {
+					for _, cert := range tb.X509Authorities() {
+						bundle.AddX509Authority(cert)
+					}
+				}
+				rawBundles[ourTD.String()] = ConcatRawCertsFromCerts(bundle.X509Authorities())
 				if initBundle {
+					log.Printf("x509 inited")
 					wg.Done()
 					initBundle = false
 				}
@@ -661,39 +707,68 @@ func main() {
 	}()
 
 	go func() {
+		var ourTD *spiffeid.TrustDomain
+		//haTD, _ := spiffeid.TrustDomainFromString("spiffe://spire-ha")
 		log.Printf("Listening for jwt bundle updates\n")
 		for u := range ls.jwtBundleUpdate {
 			log.Printf("Got update for %d\n", u.id)
-			ls.clients[u.id].jwtBundles = u.bundle
-			if ls.clients[0].jwtBundles != nil && ls.clients[1].jwtBundles != nil {
-				log.Printf("We got two jwt bundles\n")
-				tmpBundles := make(map[string]jose.JSONWebKeySet)
+			bl := len(u.bundle)
+			log.Printf("JWT bundle count on update: %d\n", bl)
+			if bl < 1 {
+				log.Printf("Bad JWT bundle pushed by the spire-agent.\n")
+				os.Exit(1)
+			}
+			if bl > 2 {
+				log.Printf("Too many federated bundles in the JWT trust bundle. Please reconfigure the spire-ha-agent entry.\n")
+				os.Exit(1)
+			}
+			if _, ok := u.bundle["spiffe://spire-ha"]; bl == 2 && !ok {
+				log.Printf("spire-ha trust bundle not found in JWT trust bundle. Please reconfigure the spire-ha-agent entry. %s\n", u.bundle)
+				os.Exit(1)
+			}
+			for tdSTR, bundle := range u.bundle {
+				td, err := spiffeid.TrustDomainFromString(tdSTR)
+				if err != nil {
+					log.Printf("Failed to parse JWT trust bundle string. This should not happen.\n")
+					os.Exit(1)
+				}
+				//td := bundle.TrustDomain()
+				if td.Name() == "spire-ha" {
+					ls.clients[u.id].haJWTBundle = &bundle
+					continue
+				}
+				if ourTD == nil {
+					ourTD = &td
+					log.Printf("Our trust domain detected as: %s\n", ourTD.Name())
+				}
+				ls.clients[u.id].ourJWTBundle = &bundle
+			}
+			bundles := slices.DeleteFunc([]*jose.JSONWebKeySet{ls.clients[0].ourJWTBundle, ls.clients[0].haJWTBundle, ls.clients[1].ourJWTBundle, ls.clients[1].haJWTBundle}, func(b *jose.JSONWebKeySet) bool {
+				return b == nil
+			})
+			totalBundles := len(bundles)
+			if totalBundles > 1 || !ls.multi {
+				log.Printf("We got %d jwt bundles\n", totalBundles)
 				var rawBundles map[string][]byte = make(map[string][]byte)
-				for td, bundle := range ls.clients[0].jwtBundles {
-					kids := make(map[string]bool)
-					var set jose.JSONWebKeySet
+				kids := make(map[string]bool)
+				var set jose.JSONWebKeySet
+				for _, bundle := range bundles {
 					for _, b := range bundle.Keys {
-						kids[b.KeyID] = true
-						set.Keys = append(set.Keys, b)
-					}
-					if tdb, ok := ls.clients[1].jwtBundles[td]; ok {
-						for _, b := range tdb.Keys {
-							if _, ok := kids[b.KeyID]; !ok {
-								set.Keys = append(set.Keys, b)
-							}
+						if _, ok := kids[b.KeyID]; !ok {
+							kids[b.KeyID] = true
+							set.Keys = append(set.Keys, b)
 						}
 					}
-					tmpBundles[td] = set
-//FIXME td's in 1 but not 0. Maybe same with x509?
-					res, err := json.Marshal(tmpBundles[td])
-					if err != nil {
-//FIXME what is the best way to handle this
-						log.Printf("Failed to marchal. %v", err)
-						continue
-					}
-					rawBundles[td] = res
 				}
+				res, err := json.Marshal(set)
+				if err != nil {
+//FIXME what is the best way to handle this
+					log.Printf("Failed to marshal. %v", err)
+					continue
+				}
+				rawBundles[ourTD.Name()] = res
 				if jwtInitBundle {
+					log.Printf("jwt inited")
 					jwtWg.Done()
 					jwtInitBundle = false
 				}

cmd/spire-trust-sync-helper/main.go

@@ -0,0 +1,60 @@
+package main
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/spiffe/go-spiffe/v2/bundle/jwtbundle"
+	"github.com/spiffe/go-spiffe/v2/bundle/spiffebundle"
+	"github.com/spiffe/go-spiffe/v2/bundle/x509bundle"
+	"github.com/spiffe/go-spiffe/v2/spiffeid"
+)
+
+func main() {
+	var rawBundles map[string]string
+	if os.Getenv("SPIFFE_TRUST_DOMAIN") == "" {
+		fmt.Printf("SPIFFE_TRUST_DOMAIN must be set.")
+		os.Exit(1)
+	}
+	tde := os.Getenv("SPIFFE_TRUST_DOMAIN")
+	data, err := os.ReadFile("jwt_bundle.json")
+	if err != nil {
+		fmt.Printf("Failed to read jwt_bundle.json: %s", err)
+		os.Exit(2)
+	}
+	json.Unmarshal(data, &rawBundles)
+	decBundle := make([]byte, base64.StdEncoding.DecodedLen(len(rawBundles[tde])))
+	n, err := base64.StdEncoding.Decode(decBundle, []byte(rawBundles[tde]))
+	if err != nil {
+		fmt.Printf("Failed to decode jwt_bundle.json: %s\n", err)
+		os.Exit(3)
+	}
+	bundle := decBundle[:n]
+	td, err := spiffeid.TrustDomainFromString("spiffe://spire-ha")
+	if err != nil {
+		fmt.Printf("Could not build trust domain object: %s\n", err)
+		os.Exit(4)
+	}
+	jbundle, err := jwtbundle.Parse(td, bundle)
+	if err != nil {
+		fmt.Printf("Failed to parse jwt_bundle.json: %s\n", err)
+		os.Exit(5)
+	}
+	sb := spiffebundle.FromJWTAuthorities(td, jbundle.JWTAuthorities())
+	xb, err := x509bundle.Load(td, "ca.crt")
+	if err != nil {
+		fmt.Printf("Failed to load ca.crt: %s\n", err)
+		os.Exit(6)
+	}
+	for _, a := range xb.X509Authorities() {
+		sb.AddX509Authority(a)
+	}
+	final, err := sb.Marshal()
+	if err != nil {
+		fmt.Printf("Failed to marshal the bundle: %s\n", err)
+		os.Exit(7)
+	}
+	fmt.Printf("%s\n", final)
+}

config/socat/a.conf

@@ -0,0 +1 @@
+SPIRE_SOCAT_PORT=997

config/socat/b.conf

@@ -0,0 +1 @@
+SPIRE_SOCAT_PORT=998

config/socat/default.conf

@@ -0,0 +1 @@
+SPIRE_SOCAT_PORT=999

config/trust-sync/default.conf

@@ -0,0 +1,2 @@
+SPIRE_TRUST_SYNC_TRUSTDOMAIN=spire-ha
+SPIRE_SERVER_SOCKET=/var/run/spire/server/sockets/main/private/api.sock

diagram2.dot

@@ -0,0 +1,108 @@
+digraph G {
+  subgraph cluster_server2 {
+
+      label = "node name: n2"
+      #style = dashed
+        style="filled,solid,bold";
+        color="#b3b3b3";
+        fillcolor="#f5f5f5";
+        labeljust="l"; 
+      subgraph cluster_node2_systemd {
+          #label = "Systemd"
+          label = "systemd managed"
+          style = "dashed,filled"
+          color="#939393";
+          fillcolor="#d5d5d5";
+    spire_server_2[label=<<table border="0"><tr><td><b>SPIRE Server B</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestors: tpm</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#6c8ebf",fillcolor="#dae8fc"]
+          spire_agent_a[label=<<table border="0"><tr><td><b>SPIRE Agent A</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestor: tpm</font></td></tr><tr><td align="left"><font point-size="9">WorkloadAttestor: systemd</font></td></tr><tr><td align="left"><font point-size="9">systemd: spire-agent@.service</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#82b366",fillcolor="#d5e8d4"]
+          spire_trust_sync_a[label=<<table border="0"><tr><td><b>SPIRE Trust Sync A</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestor: tpm</font></td></tr><tr><td align="left"><font point-size="9">WorkloadAttestor: systemd</font></td></tr><tr><td align="left"><font point-size="9">systemd: spire-trust-sync@.service</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#82b366",fillcolor="#d5e8d4"]
+          spire_server_tb_a[label="Trust Bundle A", shape=note,style="rounded,dashed,filled,bold",fillcolor="#ffffff"]
+      }
+  }
+
+  subgraph cluster_server1 {
+      label = "node name: n1"
+      #style = dashed
+        style="filled,solid,bold";
+        color="#b3b3b3";
+        fillcolor="#f5f5f5";
+        labeljust="l"; 
+      subgraph cluster_node1_systemd {
+          #label = "Systemd"
+          label = "systemd managed"
+          style = "dashed,filled"
+          color="#939393";
+          fillcolor="#d5d5d5";
+
+    spire_server_1[label=<<table border="0"><tr><td><b>SPIRE Server A</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestors: tpm</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#6c8ebf",fillcolor="#dae8fc"]
+          spire_agent_b[label=<<table border="0"><tr><td><b>SPIRE Agent B</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestor: tpm</font></td></tr><tr><td align="left"><font point-size="9">WorkloadAttestor: systemd</font></td></tr><tr><td align="left"><font point-size="9">systemd: spire-agent@.service</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#82b366",fillcolor="#d5e8d4"]
+          spire_trust_sync_b[label=<<table border="0"><tr><td><b>SPIRE Trust Sync B</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestor: tpm</font></td></tr><tr><td align="left"><font point-size="9">WorkloadAttestor: systemd</font></td></tr><tr><td align="left"><font point-size="9">systemd: spire-trust-sync@.service</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#82b366",fillcolor="#d5e8d4"]
+          spire_server_tb_b[label="Trust Bundle B", shape=note,style="rounded,dashed,filled,bold",fillcolor="#ffffff"]
+      }
+  }
+
+  
+  subgraph cluster_node3 {
+      label = "node name: n3"
+      #style = dashed
+        style="filled,solid,bold";
+        color="#b3b3b3";
+        fillcolor="#f5f5f5";
+        labeljust="l"; 
+
+      subgraph cluster_node3_systemd {
+          #label = "Systemd"
+          label = "systemd managed"
+          style = "dashed,filled"
+          color="#939393";
+          fillcolor="#d5d5d5";
+
+       labeljust="l"; 
+
+          spire_agent1[label=<<table border="0"><tr><td><b>SPIRE Agent A</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestor: tpm</font></td></tr><tr><td align="left"><font point-size="9">WorkloadAttestor: systemd</font></td></tr><tr><td align="left"><font point-size="9">systemd: spire-agent@.service</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#82b366",fillcolor="#d5e8d4"]
+          spire_agent2[label=<<table border="0"><tr><td><b>SPIRE Agent B</b></td></tr><tr><td align="left"><font point-size="9">NodeAttestor: tpm</font></td></tr><tr><td align="left"><font point-size="9">WorkloadAttestor: systemd</font></td></tr><tr><td align="left"><font point-size="9">systemd: spire-agent@.service</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#82b366",fillcolor="#d5e8d4"]
+    subgraph cluster_tb {
+        label=""
+        style="invis"
+          spire_ha_agent[label=<<table border="0"><tr><td><b>SPIRE HA Agent</b></td></tr><tr><td align="left"><font point-size="9">systemd: spire-ha-agent@.service</font></td></tr></table>>,shape="record",style="rounded,solid,filled,bold",color="#82b366",fillcolor="#d5e8d4"]
+subgraph cluster_storage {
+          spire_ha_agent_state_a_b[label="Trust Bundle B", shape=note,style="rounded,dashed,filled,bold",fillcolor="#ffffff"]
+          spire_ha_agent_state_a[label="Trust Bundle A", shape=note,style="rounded,solid,filled,bold",fillcolor="#ffffff"]
+          spire_ha_agent_state_b_a[label="Trust Bundle A", shape=note,style="rounded,dashed,filled,bold",fillcolor="#ffffff"]
+          spire_ha_agent_state_b[label="Trust Bundle B", shape=note,style="rounded,solid,filled,bold",fillcolor="#ffffff"]
+}
+    }
+          sshd1[label="sshd",shape="box",style="rounded,solid,filled,bold",color="#d6b656",fillcolor="#fff2cc"]
+          kubelet1[label="kubelet",shape="box",style="rounded,solid,filled,bold",color="#d6b656",fillcolor="#fff2cc"]
+          dotdotdot1[label="...",shape="box",style="rounded,solid,filled,bold",color="#d6b656",fillcolor="#fff2cc"]
+          //k8s_spiffe_helper1[label=<<table border="0"><tr><td><b>K8s SPIFFE Helper</b></td></tr><tr><td align="left"><font point-size="9">systemd: k8s-spiffe-helper.service</font></td></tr><tr><td align="left"><font point-size="9">tool: spiffe-helper</font></td></tr></table>>,shape="box",style="rounded,solid,filled,bold",color="#d79b00",fillcolor="#ffe6cc"]
+      }
+  }
+  
+  spire_server_1 -> spire_agent1[dir=back]
+  spire_server_2 -> spire_agent2[dir=back]
+  spire_agent_b -> spire_server_2[constraint=false]
+  spire_agent_a -> spire_server_1[constraint=false]
+  spire_agent_b -> spire_trust_sync_b[dir=back]
+  spire_agent_a -> spire_trust_sync_a[dir=back]
+  spire_agent1 -> spire_ha_agent[dir=back]
+  spire_agent2 -> spire_ha_agent[dir=back]
+  spire_ha_agent -> sshd1[dir=back]
+  spire_ha_agent -> kubelet1[dir=back]
+  spire_ha_agent -> dotdotdot1[dir=back]
+//  spire_ha_agent -> spire_ha_agent_state[dir=both, constraint=false]
+ // spire_ha_agent_state_a -> spire_ha_agent_state_b 
+  spire_agent1 -> spire_ha_agent_state_a
+  spire_agent1 -> spire_ha_agent_state_a_b
+  spire_agent2 -> spire_ha_agent_state_b
+  spire_agent2 -> spire_ha_agent_state_b_a
+  spire_ha_agent_state_a -> spire_ha_agent
+  spire_ha_agent_state_a_b -> spire_ha_agent
+  spire_ha_agent_state_b -> spire_ha_agent
+  spire_ha_agent_state_b_a -> spire_ha_agent
+
+  spire_server_tb_b -> spire_server_1
+  spire_server_tb_a -> spire_server_2
+  spire_trust_sync_a -> spire_server_tb_a
+  spire_trust_sync_b -> spire_server_tb_b
+}

go.mod

@@ -4,14 +4,14 @@ go 1.23.2
 
 require (
 	github.com/Microsoft/go-winio v0.6.2
-	github.com/go-jose/go-jose/v4 v4.0.4
+	github.com/go-jose/go-jose/v4 v4.0.5
 	github.com/mdlayher/vsock v1.2.1
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spiffe/go-spiffe/v2 v2.4.0
 	github.com/spiffe/spire v1.11.0
 	github.com/spiffe/spire-api-sdk v1.11.0
-	github.com/stretchr/testify v1.9.0
-	golang.org/x/sys v0.28.0
+	github.com/stretchr/testify v1.10.0
+	golang.org/x/sys v0.31.0
 	google.golang.org/grpc v1.67.1
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -41,10 +41,10 @@ require (
 	github.com/uber-go/tally/v4 v4.1.16 // indirect
 	github.com/zeebo/errs v1.3.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
-	golang.org/x/crypto v0.31.0 // indirect
-	golang.org/x/net v0.30.0 // indirect
-	golang.org/x/sync v0.10.0 // indirect
-	golang.org/x/text v0.21.0 // indirect
+	golang.org/x/crypto v0.36.0 // indirect
+	golang.org/x/net v0.38.0 // indirect
+	golang.org/x/sync v0.12.0 // indirect
+	golang.org/x/text v0.23.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 // indirect
 	google.golang.org/protobuf v1.35.1 // indirect
 )

go.sum

@@ -42,8 +42,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
 github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E=
-github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc=
+github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE=
+github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -194,8 +194,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/twmb/murmur3 v1.1.5/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
 github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg=
@@ -215,8 +215,8 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
-golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
+golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -235,8 +235,8 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
-golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
+golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
+golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -247,8 +247,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
-golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
+golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -268,14 +268,14 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
-golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
+golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
-golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
+golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

pkg/peertracker/trace.go

@@ -1,6 +1,8 @@
 package peertracker
 
 import (
+	"os"
+
 	"github.com/mdlayher/vsock"
 )
 

systemd/spire-trust-sync@.service

@@ -0,0 +1,48 @@
+[Unit]
+Description=SPIRE Trust Bundle Sync %i
+PartOf=spire.target
+After=network-online.target local-fs.target time-sync.target
+Before=remote-fs-pre.target
+Wants=network-online.target local-fs.target time-sync.target remote-fs-pre.target spire-agent.target
+StartLimitIntervalSec=0
+
+[Service]
+WorkingDirectory=/var/run
+StateDirectory=spire/trust-sync/%i
+RuntimeDirectory=spire/trust-sync/%i
+RuntimeDirectoryPreserve=true
+ConfigurationDirectory=spire/trust-sync
+Environment="SPIRE_AGENT_ADDRESS=/var/run/spire/agent/sockets/%i/public/api.sock"
+Environment="SPIRE_TRUST_SYNC_WD=/var/run/spire/trust-sync/%i"
+EnvironmentFile=-/etc/spiffe/default-trust-domain.env
+EnvironmentFile=-/etc/spire/trust-sync/default.conf
+EnvironmentFile=-/etc/spire/trust-sync/%i.conf
+ExecStart=/bin/spiffe-helper -config /var/run/spire/trust-sync/%i/helper.conf
+ExecStartPre=mkdir -p /run/spire/trust-sync/%i/
+ExecStartPre=/bin/bash -c "echo Y2VydF9kaXIgPSAiQENEQCIKc3ZpZF9maWxlX25hbWUgPSAidGxzLmNydCIKc3ZpZF9rZXlfZmlsZV9uYW1lID0gInRscy5rZXkiCnN2aWRfYnVuZGxlX2ZpbGVfbmFtZSA9ICJjYS5jcnQiCmp3dF9idW5kbGVfZmlsZV9uYW1lID0gImp3dF9idW5kbGUuanNvbiIKY21kID0gImJhc2giCmNtZF9hcmdzID0gIi1lYyBcImNkICR7U1BJUkVfVFJVU1RfU1lOQ19XRH07IC91c3IvbGliZXhlYy9zcGlyZS90cnVzdC1zeW5jL3NwaXJlLXRydXN0LXN5bmMtaGVscGVyID4gYnVuZGxlLnNwaWZmZTsgc3BpcmUtc2VydmVyIGJ1bmRsZSBzZXQgLWlkIHNwaWZmZTovLyR7U1BJUkVfVFJVU1RfU1lOQ19UUlVTVERPTUFJTn0gLXNvY2tldFBhdGggJHtTUElSRV9TRVJWRVJfU09DS0VUfSAtZm9ybWF0IHNwaWZmZSA8IGJ1bmRsZS5zcGlmZmVcIiIKcmVuZXdfc2lnbmFsID0gIlNJR0hVUCIK | base64 -d > /var/run/spire/trust-sync/%i/helper.conf"
+ExecStartPre=/bin/sed -i "s^@CD@^/var/run/spire/trust-sync/%i^" /var/run/spire/trust-sync/%i/helper.conf
+# https://gist.github.com/ageis/f5595e59b1cddb1513d1b425a323db04
+LockPersonality=true
+MemoryDenyWriteExecute=true
+NoNewPrivileges=true
+PrivateDevices=false
+PrivateTmp=true
+ProtectControlGroups=true
+ProtectHome=true
+ProtectHostname=true
+ProtectKernelLogs=true
+ProtectKernelModules=true
+ProtectKernelTunables=true
+ProtectSystem=strict
+ReadOnlyPaths=/
+ReadWritePaths=/run/spire/agent
+Restart=always
+RestartSec=5s
+RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
+RestrictNamespaces=true
+RestrictRealtime=yes
+RestrictSUIDSGID=yes
+TasksMax=infinity
+
+[Install]
+WantedBy=spire.target