Merge pull request #10796 from olemarkus/annotate-all-the-things

Add a standardised set of labels on all resources

pkg/kubemanifest/visitor.go

@@ -72,7 +72,7 @@ func visit(visitor Visitor, data interface{}, path []string, mutator func(interf
 		if err != nil {
 			return err
 		}
-
+	case nil:
 	case map[string]interface{}:
 		m := data
 		for k, v := range m {

pkg/model/components/addonmanifests/BUILD.bazel

@@ -12,6 +12,7 @@ go_library(
         "//pkg/model:go_default_library",
         "//pkg/model/components/addonmanifests/dnscontroller:go_default_library",
         "//upup/pkg/fi:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/klog/v2:go_default_library",
     ],
 )

pkg/model/components/addonmanifests/remap.go

@@ -19,6 +19,7 @@ package addonmanifests
 import (
 	"fmt"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/klog/v2"
 	addonsapi "k8s.io/kops/channels/pkg/api"
 	"k8s.io/kops/pkg/assets"
@@ -37,21 +38,22 @@ func RemapAddonManifest(addon *addonsapi.AddonSpec, context *model.KopsModelCont
 			return nil, err
 		}
 
-		remapped := false
 		if name == "dns-controller.addons.k8s.io" {
 			if err := dnscontroller.Remap(context, addon, objects); err != nil {
 				return nil, err
 			}
-			remapped = true
 		}
 
-		if remapped {
+		err = addLabels(addon, objects)
-			b, err := objects.ToYAML()
+		if err != nil {
-			if err != nil {
+			return nil, fmt.Errorf("failed to annotate %q: %w", name, err)
-				return nil, err
-			}
-			manifest = b
 		}
+
+		b, err := objects.ToYAML()
+		if err != nil {
+			return nil, err
+		}
+		manifest = b
 	}
 
 	{
@@ -65,3 +67,34 @@ func RemapAddonManifest(addon *addonsapi.AddonSpec, context *model.KopsModelCont
 
 	return manifest, nil
 }
+
+func addLabels(addon *addonsapi.AddonSpec, objects kubemanifest.ObjectList) error {
+
+	for _, object := range objects {
+		meta := &metav1.ObjectMeta{}
+		err := object.Reparse(meta, "metadata")
+		if err != nil {
+			return fmt.Errorf("Failed to annotate %T", object)
+		}
+
+		if meta.Labels == nil {
+			meta.Labels = make(map[string]string)
+		}
+
+		meta.Labels["app.kubernetes.io/managed-by"] = "kops"
+		meta.Labels["addon.kops.k8s.io/name"] = *addon.Name
+		meta.Labels["addon.kops.k8s.io/version"] = *addon.Version
+
+		// ensure selector is set where applicable
+		for key, val := range addon.Selector {
+			existingVal, ok := meta.Labels[key]
+			if ok && existingVal != val {
+				return fmt.Errorf("label %q already set to %q while it should be %q", key, meta.Labels[key], val)
+			}
+
+			meta.Labels[key] = val
+		}
+		object.Set(meta, "metadata")
+	}
+	return nil
+}

upup/models/bindata.go

@@ -129,7 +129,7 @@ var _cloudupResourcesAddonsAnonymousIssuerDiscoveryAddonsK8sIoK8s116YamlTemplate
 kind: ClusterRoleBinding
 metadata:
   labels:
-    k8s-addon: anonymous-access.addons.k8s.io
+    k8s-addon: anonymous-issuer-discovery.addons.k8s.io
   name: anonymous:service-account-issuer-discovery
   namespace: kube-system
 roleRef:

upup/models/cloudup/resources/addons/anonymous-issuer-discovery.addons.k8s.io/k8s-1.16.yaml.template

@@ -2,7 +2,7 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   labels:
-    k8s-addon: anonymous-access.addons.k8s.io
+    k8s-addon: anonymous-issuer-discovery.addons.k8s.io
   name: anonymous:service-account-issuer-discovery
   namespace: kube-system
 roleRef:

upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go

@@ -528,7 +528,6 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
 					Name:     fi.String(key),
 					Version:  fi.String(version),
-					Selector: map[string]string{"app.kubernetes.io/name": "cert-manager"},
 					Manifest: fi.String(location),
 					Id:       id,
 				})

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc-containerd/manifest.yaml

@@ -7,47 +7,47 @@ spec:
   - id: k8s-1.16
     kubernetesVersion: '>=1.16.0-alpha.0'
     manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: 9ef65a8b68b39b5222609e3bd0f16e1beab7c561
+    manifestHash: 128ebaa5482d138a30c8e2fef4cf9a74643a9188
     name: kops-controller.addons.k8s.io
     selector:
       k8s-addon: kops-controller.addons.k8s.io
     version: 1.21.0-alpha.1
   - manifest: core.addons.k8s.io/v1.4.0.yaml
-    manifestHash: 3ffe9ac576f9eec72e2bdfbd2ea17d56d9b17b90
+    manifestHash: 75dd91a5b15ade4a61ebc1de8c35714376dfbed4
     name: core.addons.k8s.io
     selector:
       k8s-addon: core.addons.k8s.io
     version: 1.4.0
   - id: k8s-1.12
     manifest: kube-dns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: db49c98447b9d59dec4fa413461a6614bc6e43e9
+    manifestHash: 9115bb04f06321decd39b1588a93e31e48a40c06
     name: kube-dns.addons.k8s.io
     selector:
       k8s-addon: kube-dns.addons.k8s.io
     version: 1.15.13-kops.3
   - id: k8s-1.8
     manifest: rbac.addons.k8s.io/k8s-1.8.yaml
-    manifestHash: 5d53ce7b920cd1e8d65d2306d80a041420711914
+    manifestHash: a9ebd499f4d73dfa12cd6f0f762d3b143aecada6
     name: rbac.addons.k8s.io
     selector:
       k8s-addon: rbac.addons.k8s.io
     version: 1.8.0
   - id: k8s-1.9
     manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml
-    manifestHash: e1508d77cb4e527d7a2939babe36dc350dd83745
+    manifestHash: 1dbad74e01965afc2c32ca822d16c204d015db82
     name: kubelet-api.rbac.addons.k8s.io
     selector:
       k8s-addon: kubelet-api.rbac.addons.k8s.io
     version: v0.0.1
   - manifest: limit-range.addons.k8s.io/v1.5.0.yaml
-    manifestHash: 2ea50e23f1a5aa41df3724630ac25173738cc90c
+    manifestHash: 18871595294c46105ef2570f11b1b2318aecfb57
     name: limit-range.addons.k8s.io
     selector:
       k8s-addon: limit-range.addons.k8s.io
     version: 1.5.0
   - id: k8s-1.12
     manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: d8e045435539cc5353d3817f5998a55a7219fb10
+    manifestHash: e61e0c2c4d0d83cb95ee10d836ae8b77e334743b
     name: dns-controller.addons.k8s.io
     selector:
       k8s-addon: dns-controller.addons.k8s.io
@@ -55,7 +55,7 @@ spec:
   - id: v1.15.0
     kubernetesVersion: '>=1.15.0'
     manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
-    manifestHash: 16d85f6fe12023eea4853cbc718e60a2fd010dd8
+    manifestHash: cc7393f22cb59dc4e23b9220ee962243334f47f1
     name: storage-aws.addons.k8s.io
     selector:
       k8s-addon: storage-aws.addons.k8s.io
@@ -63,7 +63,7 @@ spec:
   - id: v1.7.0
     kubernetesVersion: <1.15.0
     manifest: storage-aws.addons.k8s.io/v1.7.0.yaml
-    manifestHash: ee50ae72b86ef8793d44defa6eec69ba92c72d5f
+    manifestHash: 0a699ecad09b62fd94da8b97d1c2204c716c2b8f
     name: storage-aws.addons.k8s.io
     selector:
       k8s-addon: storage-aws.addons.k8s.io
@@ -71,7 +71,7 @@ spec:
   - id: k8s-1.12
     kubernetesVersion: <1.16.0
     manifest: networking.amazon-vpc-routed-eni/k8s-1.12.yaml
-    manifestHash: 1b204a83ef58e8b268970861bb18ff2df597c86a
+    manifestHash: ae25f3b938e95a2649927d8b2047ed0e5f58539f
     name: networking.amazon-vpc-routed-eni
     selector:
       role.kubernetes.io/networking: "1"
@@ -79,7 +79,7 @@ spec:
   - id: k8s-1.16
     kubernetesVersion: '>=1.16.0'
     manifest: networking.amazon-vpc-routed-eni/k8s-1.16.yaml
-    manifestHash: 9d7b0aa5ab3df0170b85b53b9c4e087216d5ddeb
+    manifestHash: 1e2c852821287f009d36cb35d20b84e676a6c4f2
     name: networking.amazon-vpc-routed-eni
     selector:
       role.kubernetes.io/networking: "1"

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc-containerd/networking.amazon-vpc-routed-eni-k8s-1.12.yaml

@@ -1,6 +1,12 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: networking.amazon-vpc-routed-eni
+    addon.kops.k8s.io/version: 1.5.5-kops.1
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/networking: "1"
   name: aws-node
 rules:
 - apiGroups:
@@ -33,6 +39,12 @@ rules:
 apiVersion: v1
 kind: ServiceAccount
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: networking.amazon-vpc-routed-eni
+    addon.kops.k8s.io/version: 1.5.5-kops.1
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/networking: "1"
   name: aws-node
   namespace: kube-system
 
@@ -41,6 +53,12 @@ metadata:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: networking.amazon-vpc-routed-eni
+    addon.kops.k8s.io/version: 1.5.5-kops.1
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/networking: "1"
   name: aws-node
 roleRef:
   apiGroup: rbac.authorization.k8s.io
@@ -56,8 +74,13 @@ subjects:
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: networking.amazon-vpc-routed-eni
+    addon.kops.k8s.io/version: 1.5.5-kops.1
+    app.kubernetes.io/managed-by: kops
     k8s-app: aws-node
+    role.kubernetes.io/networking: "1"
   name: aws-node
   namespace: kube-system
 spec:
@@ -146,6 +169,12 @@ spec:
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: networking.amazon-vpc-routed-eni
+    addon.kops.k8s.io/version: 1.5.5-kops.1
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/networking: "1"
   name: eniconfigs.crd.k8s.amazonaws.com
 spec:
   group: crd.k8s.amazonaws.com

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc-containerd/networking.amazon-vpc-routed-eni-k8s-1.16.yaml

@@ -1,6 +1,12 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: networking.amazon-vpc-routed-eni
+    addon.kops.k8s.io/version: 1.7.8-kops.1
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/networking: "1"
   name: aws-node
 roleRef:
   apiGroup: rbac.authorization.k8s.io
@@ -16,6 +22,12 @@ subjects:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: networking.amazon-vpc-routed-eni
+    addon.kops.k8s.io/version: 1.7.8-kops.1
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/networking: "1"
   name: aws-node
 rules:
 - apiGroups:
@@ -58,6 +70,12 @@ rules:
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: networking.amazon-vpc-routed-eni
+    addon.kops.k8s.io/version: 1.7.8-kops.1
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/networking: "1"
   name: eniconfigs.crd.k8s.amazonaws.com
 spec:
   group: crd.k8s.amazonaws.com
@@ -76,8 +94,13 @@ spec:
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: networking.amazon-vpc-routed-eni
+    addon.kops.k8s.io/version: 1.7.8-kops.1
+    app.kubernetes.io/managed-by: kops
     k8s-app: aws-node
+    role.kubernetes.io/networking: "1"
   name: aws-node
   namespace: kube-system
 spec:
@@ -238,5 +261,11 @@ spec:
 apiVersion: v1
 kind: ServiceAccount
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: networking.amazon-vpc-routed-eni
+    addon.kops.k8s.io/version: 1.7.8-kops.1
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/networking: "1"
   name: aws-node
   namespace: kube-system

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml

@@ -7,47 +7,47 @@ spec:
   - id: k8s-1.16
     kubernetesVersion: '>=1.16.0-alpha.0'
     manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: 9ef65a8b68b39b5222609e3bd0f16e1beab7c561
+    manifestHash: 128ebaa5482d138a30c8e2fef4cf9a74643a9188
     name: kops-controller.addons.k8s.io
     selector:
       k8s-addon: kops-controller.addons.k8s.io
     version: 1.21.0-alpha.1
   - manifest: core.addons.k8s.io/v1.4.0.yaml
-    manifestHash: 3ffe9ac576f9eec72e2bdfbd2ea17d56d9b17b90
+    manifestHash: 75dd91a5b15ade4a61ebc1de8c35714376dfbed4
     name: core.addons.k8s.io
     selector:
       k8s-addon: core.addons.k8s.io
     version: 1.4.0
   - id: k8s-1.12
     manifest: kube-dns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: db49c98447b9d59dec4fa413461a6614bc6e43e9
+    manifestHash: 9115bb04f06321decd39b1588a93e31e48a40c06
     name: kube-dns.addons.k8s.io
     selector:
       k8s-addon: kube-dns.addons.k8s.io
     version: 1.15.13-kops.3
   - id: k8s-1.8
     manifest: rbac.addons.k8s.io/k8s-1.8.yaml
-    manifestHash: 5d53ce7b920cd1e8d65d2306d80a041420711914
+    manifestHash: a9ebd499f4d73dfa12cd6f0f762d3b143aecada6
     name: rbac.addons.k8s.io
     selector:
       k8s-addon: rbac.addons.k8s.io
     version: 1.8.0
   - id: k8s-1.9
     manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml
-    manifestHash: e1508d77cb4e527d7a2939babe36dc350dd83745
+    manifestHash: 1dbad74e01965afc2c32ca822d16c204d015db82
     name: kubelet-api.rbac.addons.k8s.io
     selector:
       k8s-addon: kubelet-api.rbac.addons.k8s.io
     version: v0.0.1
   - manifest: limit-range.addons.k8s.io/v1.5.0.yaml
-    manifestHash: 2ea50e23f1a5aa41df3724630ac25173738cc90c
+    manifestHash: 18871595294c46105ef2570f11b1b2318aecfb57
     name: limit-range.addons.k8s.io
     selector:
       k8s-addon: limit-range.addons.k8s.io
     version: 1.5.0
   - id: k8s-1.12
     manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: d8e045435539cc5353d3817f5998a55a7219fb10
+    manifestHash: e61e0c2c4d0d83cb95ee10d836ae8b77e334743b
     name: dns-controller.addons.k8s.io
     selector:
       k8s-addon: dns-controller.addons.k8s.io
@@ -55,7 +55,7 @@ spec:
   - id: v1.15.0
     kubernetesVersion: '>=1.15.0'
     manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
-    manifestHash: 16d85f6fe12023eea4853cbc718e60a2fd010dd8
+    manifestHash: cc7393f22cb59dc4e23b9220ee962243334f47f1
     name: storage-aws.addons.k8s.io
     selector:
       k8s-addon: storage-aws.addons.k8s.io
@@ -63,7 +63,7 @@ spec:
   - id: v1.7.0
     kubernetesVersion: <1.15.0
     manifest: storage-aws.addons.k8s.io/v1.7.0.yaml
-    manifestHash: ee50ae72b86ef8793d44defa6eec69ba92c72d5f
+    manifestHash: 0a699ecad09b62fd94da8b97d1c2204c716c2b8f
     name: storage-aws.addons.k8s.io
     selector:
       k8s-addon: storage-aws.addons.k8s.io
@@ -71,7 +71,7 @@ spec:
   - id: k8s-1.12
     kubernetesVersion: <1.16.0
     manifest: networking.amazon-vpc-routed-eni/k8s-1.12.yaml
-    manifestHash: 1b204a83ef58e8b268970861bb18ff2df597c86a
+    manifestHash: ae25f3b938e95a2649927d8b2047ed0e5f58539f
     name: networking.amazon-vpc-routed-eni
     selector:
       role.kubernetes.io/networking: "1"
@@ -79,7 +79,7 @@ spec:
   - id: k8s-1.16
     kubernetesVersion: '>=1.16.0'
     manifest: networking.amazon-vpc-routed-eni/k8s-1.16.yaml
-    manifestHash: bf1e49b5c32bf7e5af85266c8b17b848f8c965e1
+    manifestHash: d6ca46e4f4f742a2effbbfcecd3d01b5945ff1c3
     name: networking.amazon-vpc-routed-eni
     selector:
       role.kubernetes.io/networking: "1"

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/networking.amazon-vpc-routed-eni-k8s-1.12.yaml

@@ -1,6 +1,12 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: networking.amazon-vpc-routed-eni
+    addon.kops.k8s.io/version: 1.5.5-kops.1
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/networking: "1"
   name: aws-node
 rules:
 - apiGroups:
@@ -33,6 +39,12 @@ rules:
 apiVersion: v1
 kind: ServiceAccount
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: networking.amazon-vpc-routed-eni
+    addon.kops.k8s.io/version: 1.5.5-kops.1
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/networking: "1"
   name: aws-node
   namespace: kube-system
 
@@ -41,6 +53,12 @@ metadata:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: networking.amazon-vpc-routed-eni
+    addon.kops.k8s.io/version: 1.5.5-kops.1
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/networking: "1"
   name: aws-node
 roleRef:
   apiGroup: rbac.authorization.k8s.io
@@ -56,8 +74,13 @@ subjects:
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: networking.amazon-vpc-routed-eni
+    addon.kops.k8s.io/version: 1.5.5-kops.1
+    app.kubernetes.io/managed-by: kops
     k8s-app: aws-node
+    role.kubernetes.io/networking: "1"
   name: aws-node
   namespace: kube-system
 spec:
@@ -146,6 +169,12 @@ spec:
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: networking.amazon-vpc-routed-eni
+    addon.kops.k8s.io/version: 1.5.5-kops.1
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/networking: "1"
   name: eniconfigs.crd.k8s.amazonaws.com
 spec:
   group: crd.k8s.amazonaws.com

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/networking.amazon-vpc-routed-eni-k8s-1.16.yaml

@@ -1,6 +1,12 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: networking.amazon-vpc-routed-eni
+    addon.kops.k8s.io/version: 1.7.8-kops.1
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/networking: "1"
   name: aws-node
 roleRef:
   apiGroup: rbac.authorization.k8s.io
@@ -16,6 +22,12 @@ subjects:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: networking.amazon-vpc-routed-eni
+    addon.kops.k8s.io/version: 1.7.8-kops.1
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/networking: "1"
   name: aws-node
 rules:
 - apiGroups:
@@ -58,6 +70,12 @@ rules:
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: networking.amazon-vpc-routed-eni
+    addon.kops.k8s.io/version: 1.7.8-kops.1
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/networking: "1"
   name: eniconfigs.crd.k8s.amazonaws.com
 spec:
   group: crd.k8s.amazonaws.com
@@ -76,8 +94,13 @@ spec:
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: networking.amazon-vpc-routed-eni
+    addon.kops.k8s.io/version: 1.7.8-kops.1
+    app.kubernetes.io/managed-by: kops
     k8s-app: aws-node
+    role.kubernetes.io/networking: "1"
   name: aws-node
   namespace: kube-system
 spec:
@@ -238,5 +261,11 @@ spec:
 apiVersion: v1
 kind: ServiceAccount
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: networking.amazon-vpc-routed-eni
+    addon.kops.k8s.io/version: 1.7.8-kops.1
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/networking: "1"
   name: aws-node
   namespace: kube-system

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/aws-cloud-controller.addons.k8s.io-k8s-1.18.yaml

@@ -1,7 +1,12 @@
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.18.0-kops.1
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
     k8s-app: aws-cloud-controller-manager
   name: aws-cloud-controller-manager
   namespace: kube-system
@@ -42,6 +47,12 @@ spec:
 apiVersion: v1
 kind: ServiceAccount
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.18.0-kops.1
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
   name: cloud-controller-manager
   namespace: kube-system
 
@@ -50,6 +61,12 @@ metadata:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.18.0-kops.1
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
   name: cloud-controller-manager:apiserver-authentication-reader
   namespace: kube-system
 roleRef:
@@ -67,6 +84,12 @@ subjects:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.18.0-kops.1
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
   name: system:cloud-controller-manager
 rules:
 - apiGroups:
@@ -156,6 +179,12 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.18.0-kops.1
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
   name: system:cloud-controller-manager
 roleRef:
   apiGroup: rbac.authorization.k8s.io

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/manifest.yaml

@@ -7,40 +7,40 @@ spec:
   - id: k8s-1.16
     kubernetesVersion: '>=1.16.0-alpha.0'
     manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: 61e66ba2a661ddeb8cb7954b61e5ca1f3f6b4ce2
+    manifestHash: 6e587cee2a036bee3787e4d52ac597e89d92785e
     name: kops-controller.addons.k8s.io
     selector:
       k8s-addon: kops-controller.addons.k8s.io
     version: 1.21.0-alpha.1
   - manifest: core.addons.k8s.io/v1.4.0.yaml
-    manifestHash: 3ffe9ac576f9eec72e2bdfbd2ea17d56d9b17b90
+    manifestHash: 75dd91a5b15ade4a61ebc1de8c35714376dfbed4
     name: core.addons.k8s.io
     selector:
       k8s-addon: core.addons.k8s.io
     version: 1.4.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: c39ee62b44bc391d426293c951bfcdbbd28950c9
+    manifestHash: 70f2e607f0df7b9c80eb240984db4564b5de5ad8
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io
     version: 1.7.0-kops.3
   - id: k8s-1.9
     manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml
-    manifestHash: e1508d77cb4e527d7a2939babe36dc350dd83745
+    manifestHash: 1dbad74e01965afc2c32ca822d16c204d015db82
     name: kubelet-api.rbac.addons.k8s.io
     selector:
       k8s-addon: kubelet-api.rbac.addons.k8s.io
     version: v0.0.1
   - manifest: limit-range.addons.k8s.io/v1.5.0.yaml
-    manifestHash: 2ea50e23f1a5aa41df3724630ac25173738cc90c
+    manifestHash: 18871595294c46105ef2570f11b1b2318aecfb57
     name: limit-range.addons.k8s.io
     selector:
       k8s-addon: limit-range.addons.k8s.io
     version: 1.5.0
   - id: k8s-1.12
     manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: d8e045435539cc5353d3817f5998a55a7219fb10
+    manifestHash: e61e0c2c4d0d83cb95ee10d836ae8b77e334743b
     name: dns-controller.addons.k8s.io
     selector:
       k8s-addon: dns-controller.addons.k8s.io
@@ -48,7 +48,7 @@ spec:
   - id: v1.15.0
     kubernetesVersion: '>=1.15.0'
     manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
-    manifestHash: 16d85f6fe12023eea4853cbc718e60a2fd010dd8
+    manifestHash: cc7393f22cb59dc4e23b9220ee962243334f47f1
     name: storage-aws.addons.k8s.io
     selector:
       k8s-addon: storage-aws.addons.k8s.io
@@ -56,7 +56,7 @@ spec:
   - id: v1.7.0
     kubernetesVersion: <1.15.0
     manifest: storage-aws.addons.k8s.io/v1.7.0.yaml
-    manifestHash: ee50ae72b86ef8793d44defa6eec69ba92c72d5f
+    manifestHash: 0a699ecad09b62fd94da8b97d1c2204c716c2b8f
     name: storage-aws.addons.k8s.io
     selector:
       k8s-addon: storage-aws.addons.k8s.io
@@ -64,7 +64,7 @@ spec:
   - id: k8s-1.18
     kubernetesVersion: '>=1.18.0'
     manifest: aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml
-    manifestHash: 8a158e20cd289197a1378cf6cc95d1f81a2f5278
+    manifestHash: f3798709f4bc0eec2e211fda6f629fdae0e0b297
     name: aws-cloud-controller.addons.k8s.io
     selector:
       k8s-addon: aws-cloud-controller.addons.k8s.io
@@ -72,7 +72,7 @@ spec:
   - id: k8s-1.17
     kubernetesVersion: '>=1.17.0'
     manifest: aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml
-    manifestHash: 764e53dc640a307c42a075797e6307d2014a28b6
+    manifestHash: 7b0fe2a3ab1cafa4caa9d3128efe5a7c65bfb7e8
     name: aws-ebs-csi-driver.addons.k8s.io
     selector:
       k8s-addon: aws-ebs-csi-driver.addons.k8s.io

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/authentication.aws-k8s-1.12.yaml

@@ -1,6 +1,12 @@
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: authentication.aws
+    addon.kops.k8s.io/version: 0.5.1-kops.1
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/authentication: "1"
   name: iamidentitymappings.iamauthenticator.k8s.aws
 spec:
   group: iamauthenticator.k8s.aws
@@ -36,6 +42,12 @@ spec:
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: ClusterRole
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: authentication.aws
+    addon.kops.k8s.io/version: 0.5.1-kops.1
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/authentication: "1"
   name: aws-iam-authenticator
 rules:
 - apiGroups:
@@ -82,6 +94,12 @@ rules:
 apiVersion: v1
 kind: ServiceAccount
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: authentication.aws
+    addon.kops.k8s.io/version: 0.5.1-kops.1
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/authentication: "1"
   name: aws-iam-authenticator
   namespace: kube-system
 
@@ -90,6 +108,12 @@ metadata:
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: ClusterRoleBinding
 metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: authentication.aws
+    addon.kops.k8s.io/version: 0.5.1-kops.1
+    app.kubernetes.io/managed-by: kops
+    role.kubernetes.io/authentication: "1"
   name: aws-iam-authenticator
 roleRef:
   apiGroup: rbac.authorization.k8s.io
@@ -107,8 +131,13 @@ kind: DaemonSet
 metadata:
   annotations:
     seccomp.security.alpha.kubernetes.io/pod: runtime/default
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: authentication.aws
+    addon.kops.k8s.io/version: 0.5.1-kops.1
+    app.kubernetes.io/managed-by: kops
     k8s-app: aws-iam-authenticator
+    role.kubernetes.io/authentication: "1"
   name: aws-iam-authenticator
   namespace: kube-system
 spec:

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/manifest.yaml

@@ -7,40 +7,40 @@ spec:
   - id: k8s-1.16
     kubernetesVersion: '>=1.16.0-alpha.0'
     manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: 61e66ba2a661ddeb8cb7954b61e5ca1f3f6b4ce2
+    manifestHash: 6e587cee2a036bee3787e4d52ac597e89d92785e
     name: kops-controller.addons.k8s.io
     selector:
       k8s-addon: kops-controller.addons.k8s.io
     version: 1.21.0-alpha.1
   - manifest: core.addons.k8s.io/v1.4.0.yaml
-    manifestHash: 3ffe9ac576f9eec72e2bdfbd2ea17d56d9b17b90
+    manifestHash: 75dd91a5b15ade4a61ebc1de8c35714376dfbed4
     name: core.addons.k8s.io
     selector:
       k8s-addon: core.addons.k8s.io
     version: 1.4.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: c39ee62b44bc391d426293c951bfcdbbd28950c9
+    manifestHash: 70f2e607f0df7b9c80eb240984db4564b5de5ad8
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io
     version: 1.7.0-kops.3
   - id: k8s-1.9
     manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml
-    manifestHash: e1508d77cb4e527d7a2939babe36dc350dd83745
+    manifestHash: 1dbad74e01965afc2c32ca822d16c204d015db82
     name: kubelet-api.rbac.addons.k8s.io
     selector:
       k8s-addon: kubelet-api.rbac.addons.k8s.io
     version: v0.0.1
   - manifest: limit-range.addons.k8s.io/v1.5.0.yaml
-    manifestHash: 2ea50e23f1a5aa41df3724630ac25173738cc90c
+    manifestHash: 18871595294c46105ef2570f11b1b2318aecfb57
     name: limit-range.addons.k8s.io
     selector:
       k8s-addon: limit-range.addons.k8s.io
     version: 1.5.0
   - id: k8s-1.12
     manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: d8e045435539cc5353d3817f5998a55a7219fb10
+    manifestHash: e61e0c2c4d0d83cb95ee10d836ae8b77e334743b
     name: dns-controller.addons.k8s.io
     selector:
       k8s-addon: dns-controller.addons.k8s.io
@@ -48,7 +48,7 @@ spec:
   - id: v1.15.0
     kubernetesVersion: '>=1.15.0'
     manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
-    manifestHash: 16d85f6fe12023eea4853cbc718e60a2fd010dd8
+    manifestHash: cc7393f22cb59dc4e23b9220ee962243334f47f1
     name: storage-aws.addons.k8s.io
     selector:
       k8s-addon: storage-aws.addons.k8s.io
@@ -56,14 +56,14 @@ spec:
   - id: v1.7.0
     kubernetesVersion: <1.15.0
     manifest: storage-aws.addons.k8s.io/v1.7.0.yaml
-    manifestHash: ee50ae72b86ef8793d44defa6eec69ba92c72d5f
+    manifestHash: 0a699ecad09b62fd94da8b97d1c2204c716c2b8f
     name: storage-aws.addons.k8s.io
     selector:
       k8s-addon: storage-aws.addons.k8s.io
     version: 1.17.0
   - id: k8s-1.12
     manifest: authentication.aws/k8s-1.12.yaml
-    manifestHash: 7a5100a7a4938565f3bd64decc8c7767f57ae2cc
+    manifestHash: 685133bd71d677a5b299b5386eceffdbd3d60834
     name: authentication.aws
     selector:
       role.kubernetes.io/authentication: "1"

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/dns-controller.addons.k8s.io-k8s-1.12.yaml

@@ -1,7 +1,11 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: dns-controller.addons.k8s.io
     k8s-app: dns-controller
     version: v1.21.0-alpha.1
@@ -56,7 +60,11 @@ spec:
 apiVersion: v1
 kind: ServiceAccount
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: dns-controller.addons.k8s.io
   name: dns-controller
   namespace: kube-system
@@ -66,7 +74,11 @@ metadata:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: dns-controller.addons.k8s.io
   name: kops:dns-controller
 rules:
@@ -96,7 +108,11 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: dns-controller.addons.k8s.io
   name: kops:dns-controller
 roleRef:

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml

@@ -4,7 +4,11 @@ data:
     {"cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com"}
 kind: ConfigMap
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: kops-controller.addons.k8s.io
   name: kops-controller
   namespace: kube-system
@@ -14,7 +18,11 @@ metadata:
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: kops-controller.addons.k8s.io
     k8s-app: kops-controller
     version: v1.21.0-alpha.1
@@ -77,7 +85,11 @@ spec:
 apiVersion: v1
 kind: ServiceAccount
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: kops-controller.addons.k8s.io
   name: kops-controller
   namespace: kube-system
@@ -87,7 +99,11 @@ metadata:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: kops-controller.addons.k8s.io
   name: kops-controller
 rules:
@@ -106,7 +122,11 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: kops-controller.addons.k8s.io
   name: kops-controller
 roleRef:
@@ -123,7 +143,11 @@ subjects:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: kops-controller.addons.k8s.io
   name: kops-controller
   namespace: kube-system
@@ -162,7 +186,11 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: kops-controller.addons.k8s.io
   name: kops-controller
   namespace: kube-system

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml

@@ -7,47 +7,47 @@ spec:
   - id: k8s-1.16
     kubernetesVersion: '>=1.16.0-alpha.0'
     manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: 9ef65a8b68b39b5222609e3bd0f16e1beab7c561
+    manifestHash: 128ebaa5482d138a30c8e2fef4cf9a74643a9188
     name: kops-controller.addons.k8s.io
     selector:
       k8s-addon: kops-controller.addons.k8s.io
     version: 1.21.0-alpha.1
   - manifest: core.addons.k8s.io/v1.4.0.yaml
-    manifestHash: 3ffe9ac576f9eec72e2bdfbd2ea17d56d9b17b90
+    manifestHash: 75dd91a5b15ade4a61ebc1de8c35714376dfbed4
     name: core.addons.k8s.io
     selector:
       k8s-addon: core.addons.k8s.io
     version: 1.4.0
   - id: k8s-1.12
     manifest: kube-dns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: db49c98447b9d59dec4fa413461a6614bc6e43e9
+    manifestHash: 9115bb04f06321decd39b1588a93e31e48a40c06
     name: kube-dns.addons.k8s.io
     selector:
       k8s-addon: kube-dns.addons.k8s.io
     version: 1.15.13-kops.3
   - id: k8s-1.8
     manifest: rbac.addons.k8s.io/k8s-1.8.yaml
-    manifestHash: 5d53ce7b920cd1e8d65d2306d80a041420711914
+    manifestHash: a9ebd499f4d73dfa12cd6f0f762d3b143aecada6
     name: rbac.addons.k8s.io
     selector:
       k8s-addon: rbac.addons.k8s.io
     version: 1.8.0
   - id: k8s-1.9
     manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml
-    manifestHash: e1508d77cb4e527d7a2939babe36dc350dd83745
+    manifestHash: 1dbad74e01965afc2c32ca822d16c204d015db82
     name: kubelet-api.rbac.addons.k8s.io
     selector:
       k8s-addon: kubelet-api.rbac.addons.k8s.io
     version: v0.0.1
   - manifest: limit-range.addons.k8s.io/v1.5.0.yaml
-    manifestHash: 2ea50e23f1a5aa41df3724630ac25173738cc90c
+    manifestHash: 18871595294c46105ef2570f11b1b2318aecfb57
     name: limit-range.addons.k8s.io
     selector:
       k8s-addon: limit-range.addons.k8s.io
     version: 1.5.0
   - id: k8s-1.12
     manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: d8e045435539cc5353d3817f5998a55a7219fb10
+    manifestHash: e61e0c2c4d0d83cb95ee10d836ae8b77e334743b
     name: dns-controller.addons.k8s.io
     selector:
       k8s-addon: dns-controller.addons.k8s.io
@@ -55,7 +55,7 @@ spec:
   - id: v1.15.0
     kubernetesVersion: '>=1.15.0'
     manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
-    manifestHash: 16d85f6fe12023eea4853cbc718e60a2fd010dd8
+    manifestHash: cc7393f22cb59dc4e23b9220ee962243334f47f1
     name: storage-aws.addons.k8s.io
     selector:
       k8s-addon: storage-aws.addons.k8s.io
@@ -63,14 +63,14 @@ spec:
   - id: v1.7.0
     kubernetesVersion: <1.15.0
     manifest: storage-aws.addons.k8s.io/v1.7.0.yaml
-    manifestHash: ee50ae72b86ef8793d44defa6eec69ba92c72d5f
+    manifestHash: 0a699ecad09b62fd94da8b97d1c2204c716c2b8f
     name: storage-aws.addons.k8s.io
     selector:
       k8s-addon: storage-aws.addons.k8s.io
     version: 1.17.0
   - id: k8s-1.12
     manifest: networking.cilium.io/k8s-1.12-v1.9.yaml
-    manifestHash: 2831146cb6f181255c69a09d8e97cc406304a81a
+    manifestHash: a1d86d4d8501a5f4adfc7e6c356377730a507c86
     name: networking.cilium.io
     needsRollingUpdate: all
     selector:

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/public-jwks/anonymous-issuer-discovery.addons.k8s.io-k8s-1.16.yaml

@@ -1,8 +1,12 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
+  creationTimestamp: null
   labels:
-    k8s-addon: anonymous-access.addons.k8s.io
+    addon.kops.k8s.io/name: anonymous-issuer-discovery.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: anonymous-issuer-discovery.addons.k8s.io
   name: anonymous:service-account-issuer-discovery
   namespace: kube-system
 roleRef:

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/public-jwks/dns-controller.addons.k8s.io-k8s-1.12.yaml

@@ -1,7 +1,11 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: dns-controller.addons.k8s.io
     k8s-app: dns-controller
     version: v1.21.0-alpha.1
@@ -75,7 +79,11 @@ spec:
 apiVersion: v1
 kind: ServiceAccount
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: dns-controller.addons.k8s.io
   name: dns-controller
   namespace: kube-system
@@ -85,7 +93,11 @@ metadata:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: dns-controller.addons.k8s.io
   name: kops:dns-controller
 rules:
@@ -115,7 +127,11 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: dns-controller.addons.k8s.io
   name: kops:dns-controller
 roleRef:

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/public-jwks/kops-controller.addons.k8s.io-k8s-1.16.yaml

@@ -4,7 +4,11 @@ data:
     {"cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com"}
 kind: ConfigMap
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: kops-controller.addons.k8s.io
   name: kops-controller
   namespace: kube-system
@@ -14,7 +18,11 @@ metadata:
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: kops-controller.addons.k8s.io
     k8s-app: kops-controller
     version: v1.21.0-alpha.1
@@ -77,7 +85,11 @@ spec:
 apiVersion: v1
 kind: ServiceAccount
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: kops-controller.addons.k8s.io
   name: kops-controller
   namespace: kube-system
@@ -87,7 +99,11 @@ metadata:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: kops-controller.addons.k8s.io
   name: kops-controller
 rules:
@@ -106,7 +122,11 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: kops-controller.addons.k8s.io
   name: kops-controller
 roleRef:
@@ -123,7 +143,11 @@ subjects:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: kops-controller.addons.k8s.io
   name: kops-controller
   namespace: kube-system
@@ -162,7 +186,11 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: kops-controller.addons.k8s.io
   name: kops-controller
   namespace: kube-system

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/public-jwks/manifest.yaml

@@ -7,7 +7,7 @@ spec:
   - id: k8s-1.16
     kubernetesVersion: '>=1.16.0-alpha.0'
     manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: 9ef65a8b68b39b5222609e3bd0f16e1beab7c561
+    manifestHash: 128ebaa5482d138a30c8e2fef4cf9a74643a9188
     name: kops-controller.addons.k8s.io
     selector:
       k8s-addon: kops-controller.addons.k8s.io
@@ -15,47 +15,47 @@ spec:
   - id: k8s-1.16
     kubernetesVersion: '>=1.16.0-alpha.0'
     manifest: anonymous-issuer-discovery.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: d01bb2f3c12819e21bf0197624b95fb53dc0951a
+    manifestHash: 582cce6311b3b077b55fc6c97fa42524f12198c6
     name: anonymous-issuer-discovery.addons.k8s.io
     selector:
       k8s-addon: anonymous-issuer-discovery.addons.k8s.io
     version: 1.21.0-alpha.1
   - manifest: core.addons.k8s.io/v1.4.0.yaml
-    manifestHash: 3ffe9ac576f9eec72e2bdfbd2ea17d56d9b17b90
+    manifestHash: 75dd91a5b15ade4a61ebc1de8c35714376dfbed4
     name: core.addons.k8s.io
     selector:
       k8s-addon: core.addons.k8s.io
     version: 1.4.0
   - id: k8s-1.12
     manifest: kube-dns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: db49c98447b9d59dec4fa413461a6614bc6e43e9
+    manifestHash: 9115bb04f06321decd39b1588a93e31e48a40c06
     name: kube-dns.addons.k8s.io
     selector:
       k8s-addon: kube-dns.addons.k8s.io
     version: 1.15.13-kops.3
   - id: k8s-1.8
     manifest: rbac.addons.k8s.io/k8s-1.8.yaml
-    manifestHash: 5d53ce7b920cd1e8d65d2306d80a041420711914
+    manifestHash: a9ebd499f4d73dfa12cd6f0f762d3b143aecada6
     name: rbac.addons.k8s.io
     selector:
       k8s-addon: rbac.addons.k8s.io
     version: 1.8.0
   - id: k8s-1.9
     manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml
-    manifestHash: e1508d77cb4e527d7a2939babe36dc350dd83745
+    manifestHash: 1dbad74e01965afc2c32ca822d16c204d015db82
     name: kubelet-api.rbac.addons.k8s.io
     selector:
       k8s-addon: kubelet-api.rbac.addons.k8s.io
     version: v0.0.1
   - manifest: limit-range.addons.k8s.io/v1.5.0.yaml
-    manifestHash: 2ea50e23f1a5aa41df3724630ac25173738cc90c
+    manifestHash: 18871595294c46105ef2570f11b1b2318aecfb57
     name: limit-range.addons.k8s.io
     selector:
       k8s-addon: limit-range.addons.k8s.io
     version: 1.5.0
   - id: k8s-1.12
     manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: e1943d4ce081dd8fd7f0f4ee43281ac346179833
+    manifestHash: a88fe8ce7621db07076088a7a22cba2a482f22cd
     name: dns-controller.addons.k8s.io
     selector:
       k8s-addon: dns-controller.addons.k8s.io
@@ -63,7 +63,7 @@ spec:
   - id: v1.15.0
     kubernetesVersion: '>=1.15.0'
     manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
-    manifestHash: 16d85f6fe12023eea4853cbc718e60a2fd010dd8
+    manifestHash: cc7393f22cb59dc4e23b9220ee962243334f47f1
     name: storage-aws.addons.k8s.io
     selector:
       k8s-addon: storage-aws.addons.k8s.io
@@ -71,7 +71,7 @@ spec:
   - id: v1.7.0
     kubernetesVersion: <1.15.0
     manifest: storage-aws.addons.k8s.io/v1.7.0.yaml
-    manifestHash: ee50ae72b86ef8793d44defa6eec69ba92c72d5f
+    manifestHash: 0a699ecad09b62fd94da8b97d1c2204c716c2b8f
     name: storage-aws.addons.k8s.io
     selector:
       k8s-addon: storage-aws.addons.k8s.io

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/dns-controller.addons.k8s.io-k8s-1.12.yaml

@@ -1,7 +1,11 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: dns-controller.addons.k8s.io
     k8s-app: dns-controller
     version: v1.21.0-alpha.1
@@ -56,7 +60,11 @@ spec:
 apiVersion: v1
 kind: ServiceAccount
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: dns-controller.addons.k8s.io
   name: dns-controller
   namespace: kube-system
@@ -66,7 +74,11 @@ metadata:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: dns-controller.addons.k8s.io
   name: kops:dns-controller
 rules:
@@ -96,7 +108,11 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: dns-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: dns-controller.addons.k8s.io
   name: kops:dns-controller
 roleRef:

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml

@@ -4,7 +4,11 @@ data:
     {"cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["kops-custom-node-role","nodes.minimal.example.com"],"Region":"us-east-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["ca"],"certNames":["kubelet","kubelet-server","kube-proxy"]}}
 kind: ConfigMap
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: kops-controller.addons.k8s.io
   name: kops-controller
   namespace: kube-system
@@ -14,7 +18,11 @@ metadata:
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: kops-controller.addons.k8s.io
     k8s-app: kops-controller
     version: v1.21.0-alpha.1
@@ -79,7 +87,11 @@ spec:
 apiVersion: v1
 kind: ServiceAccount
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: kops-controller.addons.k8s.io
   name: kops-controller
   namespace: kube-system
@@ -89,7 +101,11 @@ metadata:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: kops-controller.addons.k8s.io
   name: kops-controller
 rules:
@@ -108,7 +124,11 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: kops-controller.addons.k8s.io
   name: kops-controller
 roleRef:
@@ -125,7 +145,11 @@ subjects:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: kops-controller.addons.k8s.io
   name: kops-controller
   namespace: kube-system
@@ -164,7 +188,11 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
+  creationTimestamp: null
   labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    addon.kops.k8s.io/version: 1.21.0-alpha.1
+    app.kubernetes.io/managed-by: kops
     k8s-addon: kops-controller.addons.k8s.io
   name: kops-controller
   namespace: kube-system

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml

@@ -7,40 +7,40 @@ spec:
   - id: k8s-1.16
     kubernetesVersion: '>=1.16.0-alpha.0'
     manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: 61e66ba2a661ddeb8cb7954b61e5ca1f3f6b4ce2
+    manifestHash: 6e587cee2a036bee3787e4d52ac597e89d92785e
     name: kops-controller.addons.k8s.io
     selector:
       k8s-addon: kops-controller.addons.k8s.io
     version: 1.21.0-alpha.1
   - manifest: core.addons.k8s.io/v1.4.0.yaml
-    manifestHash: 3ffe9ac576f9eec72e2bdfbd2ea17d56d9b17b90
+    manifestHash: 75dd91a5b15ade4a61ebc1de8c35714376dfbed4
     name: core.addons.k8s.io
     selector:
       k8s-addon: core.addons.k8s.io
     version: 1.4.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: c39ee62b44bc391d426293c951bfcdbbd28950c9
+    manifestHash: 70f2e607f0df7b9c80eb240984db4564b5de5ad8
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io
     version: 1.7.0-kops.3
   - id: k8s-1.9
     manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml
-    manifestHash: e1508d77cb4e527d7a2939babe36dc350dd83745
+    manifestHash: 1dbad74e01965afc2c32ca822d16c204d015db82
     name: kubelet-api.rbac.addons.k8s.io
     selector:
       k8s-addon: kubelet-api.rbac.addons.k8s.io
     version: v0.0.1
   - manifest: limit-range.addons.k8s.io/v1.5.0.yaml
-    manifestHash: 2ea50e23f1a5aa41df3724630ac25173738cc90c
+    manifestHash: 18871595294c46105ef2570f11b1b2318aecfb57
     name: limit-range.addons.k8s.io
     selector:
       k8s-addon: limit-range.addons.k8s.io
     version: 1.5.0
   - id: k8s-1.12
     manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: d8e045435539cc5353d3817f5998a55a7219fb10
+    manifestHash: e61e0c2c4d0d83cb95ee10d836ae8b77e334743b
     name: dns-controller.addons.k8s.io
     selector:
       k8s-addon: dns-controller.addons.k8s.io
@@ -48,7 +48,7 @@ spec:
   - id: v1.15.0
     kubernetesVersion: '>=1.15.0'
     manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
-    manifestHash: 16d85f6fe12023eea4853cbc718e60a2fd010dd8
+    manifestHash: cc7393f22cb59dc4e23b9220ee962243334f47f1
     name: storage-aws.addons.k8s.io
     selector:
       k8s-addon: storage-aws.addons.k8s.io
@@ -56,7 +56,7 @@ spec:
   - id: v1.7.0
     kubernetesVersion: <1.15.0
     manifest: storage-aws.addons.k8s.io/v1.7.0.yaml
-    manifestHash: ee50ae72b86ef8793d44defa6eec69ba92c72d5f
+    manifestHash: 0a699ecad09b62fd94da8b97d1c2204c716c2b8f
     name: storage-aws.addons.k8s.io
     selector:
       k8s-addon: storage-aws.addons.k8s.io

upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml

@@ -7,40 +7,40 @@ spec:
   - id: k8s-1.16
     kubernetesVersion: '>=1.16.0-alpha.0'
     manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: 61e66ba2a661ddeb8cb7954b61e5ca1f3f6b4ce2
+    manifestHash: 6e587cee2a036bee3787e4d52ac597e89d92785e
     name: kops-controller.addons.k8s.io
     selector:
       k8s-addon: kops-controller.addons.k8s.io
     version: 1.21.0-alpha.1
   - manifest: core.addons.k8s.io/v1.4.0.yaml
-    manifestHash: 3ffe9ac576f9eec72e2bdfbd2ea17d56d9b17b90
+    manifestHash: 75dd91a5b15ade4a61ebc1de8c35714376dfbed4
     name: core.addons.k8s.io
     selector:
       k8s-addon: core.addons.k8s.io
     version: 1.4.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: c39ee62b44bc391d426293c951bfcdbbd28950c9
+    manifestHash: 70f2e607f0df7b9c80eb240984db4564b5de5ad8
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io
     version: 1.7.0-kops.3
   - id: k8s-1.9
     manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml
-    manifestHash: e1508d77cb4e527d7a2939babe36dc350dd83745
+    manifestHash: 1dbad74e01965afc2c32ca822d16c204d015db82
     name: kubelet-api.rbac.addons.k8s.io
     selector:
       k8s-addon: kubelet-api.rbac.addons.k8s.io
     version: v0.0.1
   - manifest: limit-range.addons.k8s.io/v1.5.0.yaml
-    manifestHash: 2ea50e23f1a5aa41df3724630ac25173738cc90c
+    manifestHash: 18871595294c46105ef2570f11b1b2318aecfb57
     name: limit-range.addons.k8s.io
     selector:
       k8s-addon: limit-range.addons.k8s.io
     version: 1.5.0
   - id: k8s-1.12
     manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: d8e045435539cc5353d3817f5998a55a7219fb10
+    manifestHash: e61e0c2c4d0d83cb95ee10d836ae8b77e334743b
     name: dns-controller.addons.k8s.io
     selector:
       k8s-addon: dns-controller.addons.k8s.io
@@ -48,7 +48,7 @@ spec:
   - id: v1.15.0
     kubernetesVersion: '>=1.15.0'
     manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
-    manifestHash: 16d85f6fe12023eea4853cbc718e60a2fd010dd8
+    manifestHash: cc7393f22cb59dc4e23b9220ee962243334f47f1
     name: storage-aws.addons.k8s.io
     selector:
       k8s-addon: storage-aws.addons.k8s.io
@@ -56,14 +56,14 @@ spec:
   - id: v1.7.0
     kubernetesVersion: <1.15.0
     manifest: storage-aws.addons.k8s.io/v1.7.0.yaml
-    manifestHash: ee50ae72b86ef8793d44defa6eec69ba92c72d5f
+    manifestHash: 0a699ecad09b62fd94da8b97d1c2204c716c2b8f
     name: storage-aws.addons.k8s.io
     selector:
       k8s-addon: storage-aws.addons.k8s.io
     version: 1.17.0
   - id: k8s-1.12
     manifest: networking.weave/k8s-1.12.yaml
-    manifestHash: 538488bad36e59dbc5fa002960985b62de1aa9bd
+    manifestHash: 53d5d47f2765b73be6a77ca8b9c42740be6b454f
     name: networking.weave
     selector:
       role.kubernetes.io/networking: "1"