Make serialization of keyset items stable

upup/pkg/fi/ca.go

@@ -19,6 +19,7 @@ package fi
 import (
 	"bytes"
 	"fmt"
+	"math/big"
 	"strconv"
 
 	"k8s.io/kops/pkg/apis/kops"
@@ -182,3 +183,20 @@ func AddCert(keyset *Keyset, cert *pki.Certificate) {
 		Certificate: cert,
 	}
 }
+
+// KeysetItemIdOlder returns whether the KeysetItem Id a is older than b.
+func KeysetItemIdOlder(a, b string) bool {
+	aVersion, aOk := big.NewInt(0).SetString(a, 10)
+	bVersion, bOk := big.NewInt(0).SetString(b, 10)
+	if aOk {
+		if !bOk {
+			return false
+		}
+		return aVersion.Cmp(bVersion) < 0
+	} else {
+		if bOk {
+			return true
+		}
+		return a < b
+	}
+}

upup/pkg/fi/clientset_castore.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"fmt"
 	"math/big"
+	"sort"
 
 	"golang.org/x/crypto/ssh"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -312,7 +313,17 @@ func (c *ClientsetCAStore) storeKeyset(ctx context.Context, name string, keyset
 
 	kopsKeyset.Spec.Keys = nil
 	kopsKeyset.Spec.PrimaryId = keyset.Primary.Id
-	for _, item := range keyset.Items {
+
+	keys := make([]string, 0, len(keyset.Items))
+	for k := range keyset.Items {
+		keys = append(keys, k)
+	}
+	sort.Slice(keys, func(i, j int) bool {
+		return KeysetItemIdOlder(keyset.Items[keys[i]].Id, keyset.Items[keys[j]].Id)
+	})
+
+	for _, key := range keys {
+		item := keyset.Items[key]
 		var publicMaterial bytes.Buffer
 		if _, err := item.Certificate.WriteTo(&publicMaterial); err != nil {
 			return err

upup/pkg/fi/vfs_castore.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"math/big"
 	"os"
+	"sort"
 	"strings"
 	"sync"
 
@@ -138,7 +139,16 @@ func (k *Keyset) ToAPIObject(name string, includePrivateKeyMaterial bool) (*kops
 	o.Name = name
 	o.Spec.Type = kops.SecretTypeKeypair
 
-	for _, ki := range k.Items {
+	keys := make([]string, 0, len(k.Items))
+	for k := range k.Items {
+		keys = append(keys, k)
+	}
+	sort.Slice(keys, func(i, j int) bool {
+		return KeysetItemIdOlder(k.Items[keys[i]].Id, k.Items[keys[j]].Id)
+	})
+
+	for _, key := range keys {
+		ki := k.Items[key]
 		oki := kops.KeysetItem{
 			Id: ki.Id,
 		}