Minimal cluster-api integration

This only barely works, but we can start to boot machines and make
incremental progress.

clusterapi/Makefile

@@ -0,0 +1,17 @@
+# Copyright 2024 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apply:
+	go generate ./...
+	go run sigs.k8s.io/kustomize/kustomize/v5 build config/ | kubectl apply -f -

clusterapi/README.md

@@ -0,0 +1,50 @@
+This is experimental integration with the cluster-api.  It is very much not production ready (and currently barely works).
+
+We plug in our own bootstrap provider with the goal of enabling cluster-api nodes to join a kOps cluster.
+
+# Create a cluster on GCP
+
+*Note*: the name & zone matter, we need to match the values we'll create later in the CAPI resources.
+
+```
+kops create cluster clusterapi.k8s.local --zones us-east4-a
+kops update cluster clusterapi.k8s.local --yes --admin
+kops validate cluster --wait=10m
+```
+
+#cd cluster-api-provider-gcp
+#REGISTRY=${USER} make docker-build docker-push
+#REGISTRY=${USER} make install-management-cluster # Doesn't yet exist in capg
+
+
+
+# TODO: Install cert-manager
+
+# Install CAPI and CAPG
+```
+cd clusterapi
+kubectl apply ---server-side -f manifests/build
+```
+
+# Install our CRDs
+```
+kustomize build config | kubectl apply --server-side -f -
+```
+
+# Remove any stuff left over from previous runs
+```
+kubectl delete machinedeployment --all
+kubectl delete gcpmachinetemplate --all
+```
+
+```
+# Very carefully create a MachineDeployment matching our configuration
+cat examples/manifest.yaml | IMAGE_ID=projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts GCP_NODE_MACHINE_TYPE=e2-medium KUBERNETES_VERSION=v1.28.6 WORKER_MACHINE_COUNT=1  GCP_ZONE=us-east4-a GCP_REGION=us-east4 GCP_NETWORK_NAME=clusterapi-k8s-local GCP_SUBNET=us-east4-clusterapi-k8s-local GCP_PROJECT=$(gcloud config get project) CLUSTER_NAME=clusterapi-k8s-local envsubst | kubectl apply --server-side -n kube-system -f -
+```
+
+# IMAGE_ID=projects/debian-cloud/global/images/family/debian-12 doesn't work with user-data (????)
+
+# Run our controller, which populates the secret with the bootstrap script
+```
+go run .
+```

clusterapi/bootstrap/controllers/kopsconfig_controller.go

@@ -0,0 +1,322 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"sort"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/klog/v2"
+	api "k8s.io/kops/clusterapi/bootstrap/kops/api/v1beta1"
+	clusterv1 "k8s.io/kops/clusterapi/snapshot/cluster-api/api/v1beta1"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/assets"
+	"k8s.io/kops/pkg/client/simple/vfsclientset"
+	"k8s.io/kops/pkg/model"
+	"k8s.io/kops/pkg/model/resources"
+	"k8s.io/kops/pkg/wellknownservices"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/cloudup"
+	"k8s.io/kops/util/pkg/architectures"
+	"k8s.io/kops/util/pkg/mirrors"
+	"k8s.io/kops/util/pkg/vfs"
+	"k8s.io/utils/pointer"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+)
+
+// NewKopsConfigReconciler is the constructor for a KopsConfigReconciler
+func NewKopsConfigReconciler(mgr manager.Manager) error {
+	r := &KopsConfigReconciler{
+		client: mgr.GetClient(),
+	}
+
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&api.KopsConfig{}).
+		Complete(r)
+}
+
+// KopsConfigReconciler observes KopsConfig objects.
+type KopsConfigReconciler struct {
+	// client is the controller-runtime client
+	client client.Client
+}
+
+// +kubebuilder:rbac:groups=,resources=nodes,verbs=get;list;watch;patch
+
+// Reconcile is the main reconciler function that observes node changes.
+func (r *KopsConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	obj := &api.KopsConfig{}
+	if err := r.client.Get(ctx, req.NamespacedName, obj); err != nil {
+		klog.Warningf("unable to fetch object: %v", err)
+		if apierrors.IsNotFound(err) {
+			// we'll ignore not-found errors, since they can't be fixed by an immediate
+			// requeue (we'll need to wait for a new notification), and we can get them
+			// on deleted requests.
+			return ctrl.Result{}, nil
+		}
+		return ctrl.Result{}, err
+	}
+
+	data, err := r.buildBootstrapData(ctx)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	if err := r.storeBootstrapData(ctx, obj, data); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	if err := r.client.Status().Update(ctx, obj); err != nil {
+		return ctrl.Result{}, fmt.Errorf("error patching status: %w", err)
+	}
+	return ctrl.Result{}, nil
+}
+
+// storeBootstrapData creates a new secret with the data passed in as input,
+// sets the reference in the configuration status and ready to true.
+func (r *KopsConfigReconciler) storeBootstrapData(ctx context.Context, parent *api.KopsConfig, data []byte) error {
+	// log := ctrl.LoggerFrom(ctx)
+
+	clusterName := parent.Labels[clusterv1.ClusterNameLabel]
+
+	if clusterName == "" {
+		return fmt.Errorf("cluster name label %q not yet set", clusterv1.ClusterNameLabel)
+	}
+
+	secretName := types.NamespacedName{
+		Namespace: parent.GetNamespace(),
+		Name:      parent.GetName(),
+	}
+
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      secretName.Name,
+			Namespace: secretName.Namespace,
+			Labels: map[string]string{
+				clusterv1.ClusterNameLabel: clusterName,
+			},
+		},
+		Data: map[string][]byte{
+			"value": data,
+			// "format": []byte(scope.Config.Spec.Format),
+		},
+		Type: clusterv1.ClusterSecretType,
+	}
+
+	parentAPIVersion, parentKind := parent.GetObjectKind().GroupVersionKind().ToAPIVersionAndKind()
+	secret.OwnerReferences = []metav1.OwnerReference{
+		{
+			APIVersion: parentAPIVersion,
+			Kind:       parentKind,
+			Name:       parent.GetName(),
+			UID:        parent.GetUID(),
+			Controller: pointer.Bool(true),
+		},
+	}
+
+	var existing corev1.Secret
+	if err := r.client.Get(ctx, secretName, &existing); err != nil {
+		if apierrors.IsNotFound(err) {
+			if err := r.client.Create(ctx, secret); err != nil {
+				return fmt.Errorf("failed to create bootstrap data secret for KopsConfig %s/%s: %w", parent.GetNamespace(), parent.GetName(), err)
+			}
+		} else {
+			return fmt.Errorf("failed to get bootstrap data secret: %w", err)
+		}
+	} else {
+		// TODO: Verify that the existing secret "matches"
+		klog.Warningf("TODO: verify that the existing secret matches our expected value")
+	}
+
+	parent.Status.DataSecretName = pointer.String(secret.Name)
+	parent.Status.Ready = true
+	// conditions.MarkTrue(scope.Config, bootstrapv1.DataSecretAvailableCondition)
+	return nil
+}
+
+func (r *KopsConfigReconciler) buildBootstrapData(ctx context.Context) ([]byte, error) {
+	// tf := &TemplateFunctions{
+	// 	KopsModelContext: *modelContext,
+	// 	cloud:            cloud,
+	// }
+	// TODO: Make dynamic
+	clusterName := "clusterapi.k8s.local"
+	clusterStoreBasePath := "gs://kops-state-justinsb-root-20220725"
+
+	wellKnownAddresses := model.WellKnownAddresses{}
+	wellKnownAddresses[wellknownservices.KopsController] = append(wellKnownAddresses[wellknownservices.KopsController], "10.0.16.2")
+	wellKnownAddresses[wellknownservices.KubeAPIServer] = append(wellKnownAddresses[wellknownservices.KubeAPIServer], "10.0.16.2")
+
+	vfsContext := vfs.NewVFSContext()
+	basePath, err := vfsContext.BuildVfsPath(clusterStoreBasePath)
+	if err != nil {
+		return nil, fmt.Errorf("parsing vfs base path: %w", err)
+	}
+
+	// cluster := &kops.Cluster{}
+	// cluster.Spec.KubernetesVersion = "1.28.3"
+	// cluster.Spec.KubeAPIServer = &kops.KubeAPIServerConfig{}
+
+	vfsClientset := vfsclientset.NewVFSClientset(vfsContext, basePath)
+	cluster, err := vfsClientset.GetCluster(ctx, clusterName)
+	if err != nil {
+		return nil, fmt.Errorf("getting cluster %q: %w", clusterName, err)
+	}
+
+	if cluster.Spec.KubeAPIServer == nil {
+		cluster.Spec.KubeAPIServer = &kops.KubeAPIServerConfig{}
+	}
+
+	ig := &kops.InstanceGroup{}
+	ig.Spec.Role = kops.InstanceGroupRoleNode
+
+	getAssets := false
+	assetBuilder := assets.NewAssetBuilder(vfsContext, cluster.Spec.Assets, cluster.Spec.KubernetesVersion, getAssets)
+
+	encryptionConfigSecretHash := ""
+	// if fi.ValueOf(c.Cluster.Spec.EncryptionConfig) {
+	// 	secret, err := secretStore.FindSecret("encryptionconfig")
+	// 	if err != nil {
+	// 		return fmt.Errorf("could not load encryptionconfig secret: %v", err)
+	// 	}
+	// 	if secret == nil {
+	// 		fmt.Println("")
+	// 		fmt.Println("You have encryptionConfig enabled, but no encryptionconfig secret has been set.")
+	// 		fmt.Println("See `kops create secret encryptionconfig -h` and https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/")
+	// 		return fmt.Errorf("could not find encryptionconfig secret")
+	// 	}
+	// 	hashBytes := sha256.Sum256(secret.Data)
+	// 	encryptionConfigSecretHash = base64.URLEncoding.EncodeToString(hashBytes[:])
+	// }
+
+	nodeUpAssets := make(map[architectures.Architecture]*mirrors.MirroredAsset)
+	for _, arch := range architectures.GetSupported() {
+
+		asset, err := cloudup.NodeUpAsset(assetBuilder, arch)
+		if err != nil {
+			return nil, err
+		}
+		nodeUpAssets[arch] = asset
+	}
+
+	assets := make(map[architectures.Architecture][]*mirrors.MirroredAsset)
+	configBuilder, err := cloudup.NewNodeUpConfigBuilder(cluster, assetBuilder, assets, encryptionConfigSecretHash)
+	if err != nil {
+		return nil, err
+	}
+
+	// bootstrapScript := &model.BootstrapScript{
+	// 	// KopsModelContext:    modelContext,
+	// 	Lifecycle: fi.LifecycleSync,
+	// 	// NodeUpConfigBuilder: configBuilder,
+	// 	// NodeUpAssets:        c.NodeUpAssets,
+	// }
+
+	keysets := make(map[string]*fi.Keyset)
+
+	keystore, err := vfsClientset.KeyStore(cluster)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, keyName := range []string{"kubernetes-ca"} {
+		keyset, err := keystore.FindKeyset(ctx, keyName)
+		if err != nil {
+			return nil, fmt.Errorf("getting keyset %q: %w", keyName, err)
+		}
+
+		if keyset == nil {
+			return nil, fmt.Errorf("failed to find keyset %q", keyName)
+		}
+
+		keysets[keyName] = keyset
+	}
+
+	_, bootConfig, err := configBuilder.BuildConfig(ig, wellKnownAddresses, keysets)
+	if err != nil {
+		return nil, err
+	}
+
+	// configData, err := utils.YamlMarshal(config)
+	// if err != nil {
+	// 	return nil, fmt.Errorf("error converting nodeup config to yaml: %v", err)
+	// }
+	// sum256 := sha256.Sum256(configData)
+	// bootConfig.NodeupConfigHash = base64.StdEncoding.EncodeToString(sum256[:])
+	// b.nodeupConfig.Resource = fi.NewBytesResource(configData)
+
+	var nodeupScript resources.NodeUpScript
+	nodeupScript.NodeUpAssets = nodeUpAssets
+	nodeupScript.BootConfig = bootConfig
+
+	{
+		nodeupScript.EnvironmentVariables = func() (string, error) {
+			env := make(map[string]string)
+
+			// env, err := b.buildEnvironmentVariables()
+			// if err != nil {
+			// 	return "", err
+			// }
+
+			// Sort keys to have a stable sequence of "export xx=xxx"" statements
+			var keys []string
+			for k := range env {
+				keys = append(keys, k)
+			}
+			sort.Strings(keys)
+
+			var b bytes.Buffer
+			for _, k := range keys {
+				b.WriteString(fmt.Sprintf("export %s=%s\n", k, env[k]))
+			}
+			return b.String(), nil
+		}
+
+		nodeupScript.ProxyEnv = func() (string, error) {
+			return "", nil
+			// return b.createProxyEnv(cluster.Spec.Networking.EgressProxy)
+		}
+	}
+
+	// TODO: nodeupScript.CompressUserData = fi.ValueOf(b.ig.Spec.CompressUserData)
+
+	// By setting some sysctls early, we avoid broken configurations that prevent nodeup download.
+	// See https://github.com/kubernetes/kops/issues/10206 for details.
+	// TODO: nodeupScript.SetSysctls = setSysctls()
+
+	nodeupScript.CloudProvider = string(cluster.Spec.GetCloudProvider())
+
+	nodeupScriptResource, err := nodeupScript.Build()
+	if err != nil {
+		return nil, err
+	}
+
+	b, err := fi.ResourceAsBytes(nodeupScriptResource)
+	if err != nil {
+		return nil, err
+	}
+
+	return b, nil
+}

clusterapi/bootstrap/kops/api/v1beta1/doc.go

@@ -0,0 +1,17 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1

clusterapi/bootstrap/kops/api/v1beta1/groupversion_info.go

@@ -0,0 +1,36 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package v1beta1 contains API Schema definitions for the kops v1beta1 API group
+// +kubebuilder:object:generate=true
+// +groupName=bootstrap.cluster.x-k8s.io
+package v1beta1
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/scheme"
+)
+
+var (
+	// GroupVersion is group version used to register these objects.
+	GroupVersion = schema.GroupVersion{Group: "bootstrap.cluster.x-k8s.io", Version: "v1beta1"}
+
+	// SchemeBuilder is used to add go types to the GroupVersionKind scheme.
+	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
+
+	// AddToScheme adds the types in this group-version to the given scheme.
+	AddToScheme = SchemeBuilder.AddToScheme
+)

clusterapi/bootstrap/kops/api/v1beta1/kopsconfig_types.go

@@ -0,0 +1,82 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// KopsConfigSpec defines the desired state of KopsConfig.
+// Either ClusterConfiguration and InitConfiguration should be defined or the JoinConfiguration should be defined.
+type KopsConfigSpec struct {
+}
+
+// KopsConfigStatus defines the observed state of KopsConfig.
+type KopsConfigStatus struct {
+	// Ready indicates the BootstrapData field is ready to be consumed
+	// +optional
+	Ready bool `json:"ready"`
+
+	// DataSecretName is the name of the secret that stores the bootstrap data script.
+	// +optional
+	DataSecretName *string `json:"dataSecretName,omitempty"`
+
+	// // FailureReason will be set on non-retryable errors
+	// // +optional
+	// FailureReason string `json:"failureReason,omitempty"`
+
+	// // FailureMessage will be set on non-retryable errors
+	// // +optional
+	// FailureMessage string `json:"failureMessage,omitempty"`
+
+	// // ObservedGeneration is the latest generation observed by the controller.
+	// // +optional
+	// ObservedGeneration int64 `json:"observedGeneration,omitempty"`
+
+	// // Conditions defines current service state of the KopsConfig.
+	// // +optional
+	// Conditions clusterv1.Conditions `json:"conditions,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+// +kubebuilder:resource:path=kopsconfigs,scope=Namespaced,categories=cluster-api
+// +kubebuilder:storageversion
+// +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:name="Cluster",type="string",JSONPath=".metadata.labels['cluster\\.x-k8s\\.io/cluster-name']",description="Cluster"
+// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of KopsConfig"
+
+// KopsConfig is the Schema for the kopsconfigs API.
+type KopsConfig struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   KopsConfigSpec   `json:"spec,omitempty"`
+	Status KopsConfigStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// KopsConfigList contains a list of KopsConfig.
+type KopsConfigList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []KopsConfig `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&KopsConfig{}, &KopsConfigList{})
+}

clusterapi/bootstrap/kops/api/v1beta1/kopsconfigtemplate_types.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	clusterv1 "k8s.io/kops/clusterapi/snapshot/cluster-api/api/v1beta1"
+)
+
+// KopsConfigTemplateSpec defines the desired state of KopsConfigTemplate.
+type KopsConfigTemplateSpec struct {
+	Template KopsConfigTemplateResource `json:"template"`
+}
+
+// KopsConfigTemplateResource defines the Template structure.
+type KopsConfigTemplateResource struct {
+	// Standard object's metadata.
+	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
+	// +optional
+	ObjectMeta clusterv1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec KopsConfigSpec `json:"spec,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+// +kubebuilder:resource:path=kopsconfigtemplates,scope=Namespaced,categories=cluster-api
+// +kubebuilder:storageversion
+// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of KopsConfigTemplate"
+
+// KopsConfigTemplate is the Schema for the kopsconfigtemplates API.
+type KopsConfigTemplate struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec KopsConfigTemplateSpec `json:"spec,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// KopsConfigTemplateList contains a list of KopsConfigTemplate.
+type KopsConfigTemplateList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []KopsConfigTemplate `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&KopsConfigTemplate{}, &KopsConfigTemplateList{})
+}

clusterapi/config/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+commonLabels:
+  cluster.x-k8s.io/v1beta1: v1beta1
+
+resources:
+- crds/bootstrap.cluster.x-k8s.io_kopsconfigs.yaml
+- crds/bootstrap.cluster.x-k8s.io_kopsconfigtemplates.yaml
+- crds/controlplane.cluster.x-k8s.io_kopscontrolplanes.yaml
+- crds/controlplane.cluster.x-k8s.io_kopscontrolplanetemplates.yaml

clusterapi/controlplane/kops/api/v1beta1/doc.go

@@ -0,0 +1,17 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1

clusterapi/controlplane/kops/api/v1beta1/groupversion_info.go

@@ -0,0 +1,36 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package v1beta1 contains API Schema definitions for the kops v1beta1 API group
+// +kubebuilder:object:generate=true
+// +groupName=controlplane.cluster.x-k8s.io
+package v1beta1
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/scheme"
+)
+
+var (
+	// GroupVersion is group version used to register these objects.
+	GroupVersion = schema.GroupVersion{Group: "controlplane.cluster.x-k8s.io", Version: "v1beta1"}
+
+	// SchemeBuilder is used to add go types to the GroupVersionKind scheme.
+	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
+
+	// AddToScheme adds the types in this group-version to the given scheme.
+	AddToScheme = SchemeBuilder.AddToScheme
+)

clusterapi/controlplane/kops/api/v1beta1/kopscontrolplane_types.go

@@ -0,0 +1,81 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	clusterv1 "k8s.io/kops/clusterapi/snapshot/cluster-api/api/v1beta1"
+)
+
+// RolloutStrategyType defines the rollout strategies for a KopsControlPlane.
+type RolloutStrategyType string
+
+// KopsControlPlaneSpec defines the desired state of KopsControlPlane.
+type KopsControlPlaneSpec struct {
+}
+
+// KopsControlPlaneMachineTemplate defines the template for Machines
+// in a KopsControlPlane object.
+type KopsControlPlaneMachineTemplate struct {
+	// Standard object's metadata.
+	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
+	// +optional
+	ObjectMeta clusterv1.ObjectMeta `json:"metadata,omitempty"`
+}
+
+// KopsControlPlaneStatus defines the observed state of KopsControlPlane.
+type KopsControlPlaneStatus struct {
+}
+
+// +kubebuilder:object:root=true
+// +kubebuilder:resource:path=kopscontrolplanes,shortName=kcp,scope=Namespaced,categories=cluster-api
+// +kubebuilder:storageversion
+// +kubebuilder:subresource:status
+// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.selector
+// +kubebuilder:printcolumn:name="Cluster",type="string",JSONPath=".metadata.labels['cluster\\.x-k8s\\.io/cluster-name']",description="Cluster"
+// +kubebuilder:printcolumn:name="Initialized",type=boolean,JSONPath=".status.initialized",description="This denotes whether or not the control plane has the uploaded kops-config configmap"
+// +kubebuilder:printcolumn:name="API Server Available",type=boolean,JSONPath=".status.ready",description="KopsControlPlane API Server is ready to receive requests"
+// +kubebuilder:printcolumn:name="Desired",type=integer,JSONPath=".spec.replicas",description="Total number of machines desired by this control plane",priority=10
+// +kubebuilder:printcolumn:name="Replicas",type=integer,JSONPath=".status.replicas",description="Total number of non-terminated machines targeted by this control plane"
+// +kubebuilder:printcolumn:name="Ready",type=integer,JSONPath=".status.readyReplicas",description="Total number of fully running and ready control plane machines"
+// +kubebuilder:printcolumn:name="Updated",type=integer,JSONPath=".status.updatedReplicas",description="Total number of non-terminated machines targeted by this control plane that have the desired template spec"
+// +kubebuilder:printcolumn:name="Unavailable",type=integer,JSONPath=".status.unavailableReplicas",description="Total number of unavailable machines targeted by this control plane"
+// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of KopsControlPlane"
+// +kubebuilder:printcolumn:name="Version",type=string,JSONPath=".spec.version",description="Kubernetes version associated with this control plane"
+
+// KopsControlPlane is the Schema for the KopsControlPlane API.
+type KopsControlPlane struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   KopsControlPlaneSpec   `json:"spec,omitempty"`
+	Status KopsControlPlaneStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// KopsControlPlaneList contains a list of KopsControlPlane.
+type KopsControlPlaneList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []KopsControlPlane `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&KopsControlPlane{}, &KopsControlPlaneList{})
+}

clusterapi/controlplane/kops/api/v1beta1/kopscontrolplanetemplate_types.go

@@ -0,0 +1,86 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	clusterv1 "k8s.io/kops/clusterapi/snapshot/cluster-api/api/v1beta1"
+	// bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kops/api/v1beta1"
+)
+
+// KopsControlPlaneTemplateSpec defines the desired state of KopsControlPlaneTemplate.
+type KopsControlPlaneTemplateSpec struct {
+	Template KopsControlPlaneTemplateResource `json:"template"`
+}
+
+// +kubebuilder:object:root=true
+// +kubebuilder:resource:path=kopscontrolplanetemplates,scope=Namespaced,categories=cluster-api
+// +kubebuilder:storageversion
+// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of KopsControlPlaneTemplate"
+
+// KopsControlPlaneTemplate is the Schema for the kopscontrolplanetemplates API.
+type KopsControlPlaneTemplate struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec KopsControlPlaneTemplateSpec `json:"spec,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// KopsControlPlaneTemplateList contains a list of KopsControlPlaneTemplate.
+type KopsControlPlaneTemplateList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []KopsControlPlaneTemplate `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&KopsControlPlaneTemplate{}, &KopsControlPlaneTemplateList{})
+}
+
+// KopsControlPlaneTemplateResource describes the data needed to create a KopsControlPlane from a template.
+type KopsControlPlaneTemplateResource struct {
+	// Standard object's metadata.
+	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
+	// +optional
+	ObjectMeta clusterv1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec KopsControlPlaneTemplateResourceSpec `json:"spec"`
+}
+
+// KopsControlPlaneTemplateResourceSpec defines the desired state of KopsControlPlane.
+// NOTE: KopsControlPlaneTemplateResourceSpec is similar to KopsControlPlaneSpec but
+// omits Replicas and Version fields. These fields do not make sense on the KopsControlPlaneTemplate,
+// because they are calculated by the Cluster topology reconciler during reconciliation and thus cannot
+// be configured on the KopsControlPlaneTemplate.
+type KopsControlPlaneTemplateResourceSpec struct {
+}
+
+// KopsControlPlaneTemplateMachineTemplate defines the template for Machines
+// in a KopsControlPlaneTemplate object.
+// NOTE: KopsControlPlaneTemplateMachineTemplate is similar to KopsControlPlaneMachineTemplate but
+// omits ObjectMeta and InfrastructureRef fields. These fields do not make sense on the KopsControlPlaneTemplate,
+// because they are calculated by the Cluster topology reconciler during reconciliation and thus cannot
+// be configured on the KopsControlPlaneTemplate.
+type KopsControlPlaneTemplateMachineTemplate struct {
+	// Standard object's metadata.
+	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
+	// +optional
+	ObjectMeta clusterv1.ObjectMeta `json:"metadata,omitempty"`
+}

clusterapi/examples/manifest.yaml

@@ -0,0 +1,128 @@
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: "${CLUSTER_NAME}"
+spec:
+  #clusterNetwork:
+  #  pods:
+  #    cidrBlocks: ["192.168.0.0/16"]
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: GCPCluster
+    name: "${CLUSTER_NAME}"
+  controlPlaneRef:
+    kind: KopsControlPlane
+    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+    name: "${CLUSTER_NAME}-control-plane"
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: GCPCluster
+metadata:
+  name: "${CLUSTER_NAME}"
+spec:
+  project: "${GCP_PROJECT}"
+  region: "${GCP_REGION}"
+  network:
+    name: "${GCP_NETWORK_NAME}"
+# ---
+# kind: KubeadmControlPlane
+# apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+# metadata:
+#   name: "${CLUSTER_NAME}-control-plane"
+# spec:
+#   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
+#   machineTemplate:
+#     infrastructureRef:
+#       kind: GCPMachineTemplate
+#       apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+#       name: "${CLUSTER_NAME}-control-plane"
+#   kubeadmConfigSpec:
+#     initConfiguration:
+#       nodeRegistration:
+#         name: '{{ ds.meta_data.local_hostname.split(".")[0] }}'
+#         kubeletExtraArgs:
+#           cloud-provider: gce
+#     clusterConfiguration:
+#       apiServer:
+#         timeoutForControlPlane: 20m
+#         extraArgs:
+#           cloud-provider: gce
+#       controllerManager:
+#         extraArgs:
+#           cloud-provider: gce
+#           allocate-node-cidrs: "false"
+#     joinConfiguration:
+#       nodeRegistration:
+#         name: '{{ ds.meta_data.local_hostname.split(".")[0] }}'
+#         kubeletExtraArgs:
+#           cloud-provider: gce
+#   version: "${KUBERNETES_VERSION}"
+# ---
+# kind: GCPMachineTemplate
+# apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+# metadata:
+#   name: "${CLUSTER_NAME}-control-plane"
+# spec:
+#   template:
+#     spec:
+#       instanceType: "${GCP_CONTROL_PLANE_MACHINE_TYPE}"
+#       image: "${IMAGE_ID}"
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  name: "${CLUSTER_NAME}-md-0"
+spec:
+  clusterName: "${CLUSTER_NAME}"
+  replicas: ${WORKER_MACHINE_COUNT}
+  selector:
+    matchLabels:
+  template:
+    spec:
+      clusterName: "${CLUSTER_NAME}"
+      version: "${KUBERNETES_VERSION}"
+      failureDomain: "${GCP_ZONE}" 
+      bootstrap:
+        configRef:
+          name: "${CLUSTER_NAME}-md-0"
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          kind: KopsConfigTemplate
+      infrastructureRef:
+        name: "${CLUSTER_NAME}-md-0"
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: GCPMachineTemplate
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: GCPMachineTemplate
+metadata:
+  name: "${CLUSTER_NAME}-md-0"
+spec:
+  template:
+    spec:
+      instanceType: "${GCP_NODE_MACHINE_TYPE}"
+      image: "${IMAGE_ID}"
+      subnet: "${GCP_SUBNET}"
+      additionalNetworkTags:
+      - clusterapi-k8s-local-k8s-io-role-node
+      publicIP: true
+      additionalMetadata:
+      - key: kops-k8s-io-instance-group-name
+        value: nodes-us-east4-a
+      - key: cluster-name
+        value: clusterapi.k8s.local
+
+
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KopsConfigTemplate
+metadata:
+  name: "${CLUSTER_NAME}-md-0"
+spec:
+  template:
+    spec: {}
+      #joinConfiguration:
+      #  nodeRegistration:
+      #    name: '{{ ds.meta_data.local_hostname.split(".")[0] }}'
+      #    kubeletExtraArgs:
+      #      cloud-provider: gce

clusterapi/gen.go

@@ -0,0 +1,21 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+//go:generate go run sigs.k8s.io/controller-tools/cmd/controller-gen@v0.14.0 output:dir=config/crds crd:crdVersions=v1 paths=./bootstrap/kops/api/...;./controlplane/kops/api/...
+
+//go:generate go run sigs.k8s.io/controller-tools/cmd/controller-gen@v0.14.0 object paths=./snapshot/cluster-api/...;./bootstrap/kops/api/...;./controlplane/kops/api/...

clusterapi/main.go

@@ -0,0 +1,113 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+
+	coordinationv1 "k8s.io/api/coordination/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+	"k8s.io/klog/v2"
+	"k8s.io/klog/v2/klogr"
+	"k8s.io/kops/clusterapi/bootstrap/controllers"
+	bootstrapapi "k8s.io/kops/clusterapi/bootstrap/kops/api/v1beta1"
+	controlplaneapi "k8s.io/kops/clusterapi/controlplane/kops/api/v1beta1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
+	// +kubebuilder:scaffold:imports
+)
+
+var (
+	scheme = runtime.NewScheme()
+)
+
+func init() {
+	// +kubebuilder:scaffold:scheme
+}
+
+func main() {
+	ctx := context.Background()
+	if err := run(ctx); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}
+
+func run(ctx context.Context) error {
+	klog.InitFlags(nil)
+
+	// Disable metrics by default (avoid port conflicts, also risky because we are host network)
+	metricsAddress := ":0"
+
+	flag.Parse()
+
+	ctrl.SetLogger(klogr.New())
+
+	if err := buildScheme(); err != nil {
+		return fmt.Errorf("error building scheme: %w", err)
+	}
+
+	kubeConfig := ctrl.GetConfigOrDie()
+	options := ctrl.Options{
+		Scheme: scheme,
+		// MetricsBindAddress: metricsAddress,
+		// LeaderElection:      true,
+		// LeaderElectionID:    "kops-clusterapi-leader",
+	}
+	options.Metrics = server.Options{
+		BindAddress: metricsAddress,
+	}
+	mgr, err := ctrl.NewManager(kubeConfig, options)
+
+	if err != nil {
+		return fmt.Errorf("error starting manager: %w", err)
+	}
+
+	if err := controllers.NewKopsConfigReconciler(mgr); err != nil {
+		return fmt.Errorf("error creating controller: %w", err)
+	}
+
+	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+		return fmt.Errorf("error running manager: %w", err)
+	}
+	return nil
+}
+
+func buildScheme() error {
+	if err := corev1.AddToScheme(scheme); err != nil {
+		return fmt.Errorf("error registering corev1: %v", err)
+	}
+
+	if err := bootstrapapi.AddToScheme(scheme); err != nil {
+		return fmt.Errorf("error registering api: %w", err)
+	}
+
+	if err := controlplaneapi.AddToScheme(scheme); err != nil {
+		return fmt.Errorf("error registering api: %w", err)
+	}
+
+	// Needed so that the leader-election system can post events
+	if err := coordinationv1.AddToScheme(scheme); err != nil {
+		return fmt.Errorf("error registering coordinationv1: %v", err)
+	}
+	return nil
+}

pkg/nodeidentity/interfaces.go

@@ -38,5 +38,6 @@ type LegacyIdentifier interface {
 type LegacyInfo struct {
 	InstanceID    string
 	InstanceGroup string
+	// TODO: Remove
 	InstanceLifecycle string
 }

upup/pkg/fi/nodeup/command.go

@@ -153,8 +153,8 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
 		return fmt.Errorf("no instance group defined in nodeup config")
 	}
 
-	if want := bootConfig.NodeupConfigHash; want != "" {
+	if bootConfig.NodeupConfigHash != "" {
-		if got := base64.StdEncoding.EncodeToString(nodeupConfigHash[:]); got != want {
+		if want, got := bootConfig.NodeupConfigHash, base64.StdEncoding.EncodeToString(nodeupConfigHash[:]); got != want {
 			return fmt.Errorf("nodeup config hash mismatch (was %q, expected %q)", got, want)
 		}
 	}