From ea3122bf6738b718cdb3fc7cfa02d85567d14d06 Mon Sep 17 00:00:00 2001 From: justinsb Date: Mon, 19 Jun 2023 13:25:30 -0400 Subject: [PATCH] Minimal cluster-api integration This only barely works, but we can start to boot machines and make incremental progress. --- clusterapi/Makefile | 17 + clusterapi/README.md | 50 +++ .../controllers/kopsconfig_controller.go | 322 ++++++++++++++++++ clusterapi/bootstrap/kops/api/v1beta1/doc.go | 17 + .../kops/api/v1beta1/groupversion_info.go | 36 ++ .../kops/api/v1beta1/kopsconfig_types.go | 82 +++++ .../api/v1beta1/kopsconfigtemplate_types.go | 64 ++++ clusterapi/config/kustomization.yaml | 11 + .../controlplane/kops/api/v1beta1/doc.go | 17 + .../kops/api/v1beta1/groupversion_info.go | 36 ++ .../api/v1beta1/kopscontrolplane_types.go | 81 +++++ .../v1beta1/kopscontrolplanetemplate_types.go | 86 +++++ clusterapi/examples/manifest.yaml | 128 +++++++ clusterapi/gen.go | 21 ++ clusterapi/main.go | 113 ++++++ pkg/nodeidentity/interfaces.go | 5 +- upup/pkg/fi/nodeup/command.go | 4 +- 17 files changed, 1086 insertions(+), 4 deletions(-) create mode 100644 clusterapi/Makefile create mode 100644 clusterapi/README.md create mode 100644 clusterapi/bootstrap/controllers/kopsconfig_controller.go create mode 100644 clusterapi/bootstrap/kops/api/v1beta1/doc.go create mode 100644 clusterapi/bootstrap/kops/api/v1beta1/groupversion_info.go create mode 100644 clusterapi/bootstrap/kops/api/v1beta1/kopsconfig_types.go create mode 100644 clusterapi/bootstrap/kops/api/v1beta1/kopsconfigtemplate_types.go create mode 100644 clusterapi/config/kustomization.yaml create mode 100644 clusterapi/controlplane/kops/api/v1beta1/doc.go create mode 100644 clusterapi/controlplane/kops/api/v1beta1/groupversion_info.go create mode 100644 clusterapi/controlplane/kops/api/v1beta1/kopscontrolplane_types.go create mode 100644 clusterapi/controlplane/kops/api/v1beta1/kopscontrolplanetemplate_types.go create mode 100644 clusterapi/examples/manifest.yaml create mode 100644 clusterapi/gen.go create mode 100644 clusterapi/main.go diff --git a/clusterapi/Makefile b/clusterapi/Makefile new file mode 100644 index 0000000000..40b42213a8 --- /dev/null +++ b/clusterapi/Makefile @@ -0,0 +1,17 @@ +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apply: + go generate ./... + go run sigs.k8s.io/kustomize/kustomize/v5 build config/ | kubectl apply -f - diff --git a/clusterapi/README.md b/clusterapi/README.md new file mode 100644 index 0000000000..200fb092c0 --- /dev/null +++ b/clusterapi/README.md @@ -0,0 +1,50 @@ +This is experimental integration with the cluster-api. It is very much not production ready (and currently barely works). + +We plug in our own bootstrap provider with the goal of enabling cluster-api nodes to join a kOps cluster. + +# Create a cluster on GCP + +*Note*: the name & zone matter, we need to match the values we'll create later in the CAPI resources. + +``` +kops create cluster clusterapi.k8s.local --zones us-east4-a +kops update cluster clusterapi.k8s.local --yes --admin +kops validate cluster --wait=10m +``` + +#cd cluster-api-provider-gcp +#REGISTRY=${USER} make docker-build docker-push +#REGISTRY=${USER} make install-management-cluster # Doesn't yet exist in capg + + + +# TODO: Install cert-manager + +# Install CAPI and CAPG +``` +cd clusterapi +kubectl apply ---server-side -f manifests/build +``` + +# Install our CRDs +``` +kustomize build config | kubectl apply --server-side -f - +``` + +# Remove any stuff left over from previous runs +``` +kubectl delete machinedeployment --all +kubectl delete gcpmachinetemplate --all +``` + +``` +# Very carefully create a MachineDeployment matching our configuration +cat examples/manifest.yaml | IMAGE_ID=projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts GCP_NODE_MACHINE_TYPE=e2-medium KUBERNETES_VERSION=v1.28.6 WORKER_MACHINE_COUNT=1 GCP_ZONE=us-east4-a GCP_REGION=us-east4 GCP_NETWORK_NAME=clusterapi-k8s-local GCP_SUBNET=us-east4-clusterapi-k8s-local GCP_PROJECT=$(gcloud config get project) CLUSTER_NAME=clusterapi-k8s-local envsubst | kubectl apply --server-side -n kube-system -f - +``` + +# IMAGE_ID=projects/debian-cloud/global/images/family/debian-12 doesn't work with user-data (????) + +# Run our controller, which populates the secret with the bootstrap script +``` +go run . +``` diff --git a/clusterapi/bootstrap/controllers/kopsconfig_controller.go b/clusterapi/bootstrap/controllers/kopsconfig_controller.go new file mode 100644 index 0000000000..b0cfca0429 --- /dev/null +++ b/clusterapi/bootstrap/controllers/kopsconfig_controller.go @@ -0,0 +1,322 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "bytes" + "context" + "fmt" + "sort" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + api "k8s.io/kops/clusterapi/bootstrap/kops/api/v1beta1" + clusterv1 "k8s.io/kops/clusterapi/snapshot/cluster-api/api/v1beta1" + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/assets" + "k8s.io/kops/pkg/client/simple/vfsclientset" + "k8s.io/kops/pkg/model" + "k8s.io/kops/pkg/model/resources" + "k8s.io/kops/pkg/wellknownservices" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup" + "k8s.io/kops/util/pkg/architectures" + "k8s.io/kops/util/pkg/mirrors" + "k8s.io/kops/util/pkg/vfs" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +// NewKopsConfigReconciler is the constructor for a KopsConfigReconciler +func NewKopsConfigReconciler(mgr manager.Manager) error { + r := &KopsConfigReconciler{ + client: mgr.GetClient(), + } + + return ctrl.NewControllerManagedBy(mgr). + For(&api.KopsConfig{}). + Complete(r) +} + +// KopsConfigReconciler observes KopsConfig objects. +type KopsConfigReconciler struct { + // client is the controller-runtime client + client client.Client +} + +// +kubebuilder:rbac:groups=,resources=nodes,verbs=get;list;watch;patch + +// Reconcile is the main reconciler function that observes node changes. +func (r *KopsConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + obj := &api.KopsConfig{} + if err := r.client.Get(ctx, req.NamespacedName, obj); err != nil { + klog.Warningf("unable to fetch object: %v", err) + if apierrors.IsNotFound(err) { + // we'll ignore not-found errors, since they can't be fixed by an immediate + // requeue (we'll need to wait for a new notification), and we can get them + // on deleted requests. + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + data, err := r.buildBootstrapData(ctx) + if err != nil { + return ctrl.Result{}, err + } + + if err := r.storeBootstrapData(ctx, obj, data); err != nil { + return ctrl.Result{}, err + } + + if err := r.client.Status().Update(ctx, obj); err != nil { + return ctrl.Result{}, fmt.Errorf("error patching status: %w", err) + } + return ctrl.Result{}, nil +} + +// storeBootstrapData creates a new secret with the data passed in as input, +// sets the reference in the configuration status and ready to true. +func (r *KopsConfigReconciler) storeBootstrapData(ctx context.Context, parent *api.KopsConfig, data []byte) error { + // log := ctrl.LoggerFrom(ctx) + + clusterName := parent.Labels[clusterv1.ClusterNameLabel] + + if clusterName == "" { + return fmt.Errorf("cluster name label %q not yet set", clusterv1.ClusterNameLabel) + } + + secretName := types.NamespacedName{ + Namespace: parent.GetNamespace(), + Name: parent.GetName(), + } + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName.Name, + Namespace: secretName.Namespace, + Labels: map[string]string{ + clusterv1.ClusterNameLabel: clusterName, + }, + }, + Data: map[string][]byte{ + "value": data, + // "format": []byte(scope.Config.Spec.Format), + }, + Type: clusterv1.ClusterSecretType, + } + + parentAPIVersion, parentKind := parent.GetObjectKind().GroupVersionKind().ToAPIVersionAndKind() + secret.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: parentAPIVersion, + Kind: parentKind, + Name: parent.GetName(), + UID: parent.GetUID(), + Controller: pointer.Bool(true), + }, + } + + var existing corev1.Secret + if err := r.client.Get(ctx, secretName, &existing); err != nil { + if apierrors.IsNotFound(err) { + if err := r.client.Create(ctx, secret); err != nil { + return fmt.Errorf("failed to create bootstrap data secret for KopsConfig %s/%s: %w", parent.GetNamespace(), parent.GetName(), err) + } + } else { + return fmt.Errorf("failed to get bootstrap data secret: %w", err) + } + } else { + // TODO: Verify that the existing secret "matches" + klog.Warningf("TODO: verify that the existing secret matches our expected value") + } + + parent.Status.DataSecretName = pointer.String(secret.Name) + parent.Status.Ready = true + // conditions.MarkTrue(scope.Config, bootstrapv1.DataSecretAvailableCondition) + return nil +} + +func (r *KopsConfigReconciler) buildBootstrapData(ctx context.Context) ([]byte, error) { + // tf := &TemplateFunctions{ + // KopsModelContext: *modelContext, + // cloud: cloud, + // } + // TODO: Make dynamic + clusterName := "clusterapi.k8s.local" + clusterStoreBasePath := "gs://kops-state-justinsb-root-20220725" + + wellKnownAddresses := model.WellKnownAddresses{} + wellKnownAddresses[wellknownservices.KopsController] = append(wellKnownAddresses[wellknownservices.KopsController], "10.0.16.2") + wellKnownAddresses[wellknownservices.KubeAPIServer] = append(wellKnownAddresses[wellknownservices.KubeAPIServer], "10.0.16.2") + + vfsContext := vfs.NewVFSContext() + basePath, err := vfsContext.BuildVfsPath(clusterStoreBasePath) + if err != nil { + return nil, fmt.Errorf("parsing vfs base path: %w", err) + } + + // cluster := &kops.Cluster{} + // cluster.Spec.KubernetesVersion = "1.28.3" + // cluster.Spec.KubeAPIServer = &kops.KubeAPIServerConfig{} + + vfsClientset := vfsclientset.NewVFSClientset(vfsContext, basePath) + cluster, err := vfsClientset.GetCluster(ctx, clusterName) + if err != nil { + return nil, fmt.Errorf("getting cluster %q: %w", clusterName, err) + } + + if cluster.Spec.KubeAPIServer == nil { + cluster.Spec.KubeAPIServer = &kops.KubeAPIServerConfig{} + } + + ig := &kops.InstanceGroup{} + ig.Spec.Role = kops.InstanceGroupRoleNode + + getAssets := false + assetBuilder := assets.NewAssetBuilder(vfsContext, cluster.Spec.Assets, cluster.Spec.KubernetesVersion, getAssets) + + encryptionConfigSecretHash := "" + // if fi.ValueOf(c.Cluster.Spec.EncryptionConfig) { + // secret, err := secretStore.FindSecret("encryptionconfig") + // if err != nil { + // return fmt.Errorf("could not load encryptionconfig secret: %v", err) + // } + // if secret == nil { + // fmt.Println("") + // fmt.Println("You have encryptionConfig enabled, but no encryptionconfig secret has been set.") + // fmt.Println("See `kops create secret encryptionconfig -h` and https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/") + // return fmt.Errorf("could not find encryptionconfig secret") + // } + // hashBytes := sha256.Sum256(secret.Data) + // encryptionConfigSecretHash = base64.URLEncoding.EncodeToString(hashBytes[:]) + // } + + nodeUpAssets := make(map[architectures.Architecture]*mirrors.MirroredAsset) + for _, arch := range architectures.GetSupported() { + + asset, err := cloudup.NodeUpAsset(assetBuilder, arch) + if err != nil { + return nil, err + } + nodeUpAssets[arch] = asset + } + + assets := make(map[architectures.Architecture][]*mirrors.MirroredAsset) + configBuilder, err := cloudup.NewNodeUpConfigBuilder(cluster, assetBuilder, assets, encryptionConfigSecretHash) + if err != nil { + return nil, err + } + + // bootstrapScript := &model.BootstrapScript{ + // // KopsModelContext: modelContext, + // Lifecycle: fi.LifecycleSync, + // // NodeUpConfigBuilder: configBuilder, + // // NodeUpAssets: c.NodeUpAssets, + // } + + keysets := make(map[string]*fi.Keyset) + + keystore, err := vfsClientset.KeyStore(cluster) + if err != nil { + return nil, err + } + + for _, keyName := range []string{"kubernetes-ca"} { + keyset, err := keystore.FindKeyset(ctx, keyName) + if err != nil { + return nil, fmt.Errorf("getting keyset %q: %w", keyName, err) + } + + if keyset == nil { + return nil, fmt.Errorf("failed to find keyset %q", keyName) + } + + keysets[keyName] = keyset + } + + _, bootConfig, err := configBuilder.BuildConfig(ig, wellKnownAddresses, keysets) + if err != nil { + return nil, err + } + + // configData, err := utils.YamlMarshal(config) + // if err != nil { + // return nil, fmt.Errorf("error converting nodeup config to yaml: %v", err) + // } + // sum256 := sha256.Sum256(configData) + // bootConfig.NodeupConfigHash = base64.StdEncoding.EncodeToString(sum256[:]) + // b.nodeupConfig.Resource = fi.NewBytesResource(configData) + + var nodeupScript resources.NodeUpScript + nodeupScript.NodeUpAssets = nodeUpAssets + nodeupScript.BootConfig = bootConfig + + { + nodeupScript.EnvironmentVariables = func() (string, error) { + env := make(map[string]string) + + // env, err := b.buildEnvironmentVariables() + // if err != nil { + // return "", err + // } + + // Sort keys to have a stable sequence of "export xx=xxx"" statements + var keys []string + for k := range env { + keys = append(keys, k) + } + sort.Strings(keys) + + var b bytes.Buffer + for _, k := range keys { + b.WriteString(fmt.Sprintf("export %s=%s\n", k, env[k])) + } + return b.String(), nil + } + + nodeupScript.ProxyEnv = func() (string, error) { + return "", nil + // return b.createProxyEnv(cluster.Spec.Networking.EgressProxy) + } + } + + // TODO: nodeupScript.CompressUserData = fi.ValueOf(b.ig.Spec.CompressUserData) + + // By setting some sysctls early, we avoid broken configurations that prevent nodeup download. + // See https://github.com/kubernetes/kops/issues/10206 for details. + // TODO: nodeupScript.SetSysctls = setSysctls() + + nodeupScript.CloudProvider = string(cluster.Spec.GetCloudProvider()) + + nodeupScriptResource, err := nodeupScript.Build() + if err != nil { + return nil, err + } + + b, err := fi.ResourceAsBytes(nodeupScriptResource) + if err != nil { + return nil, err + } + + return b, nil +} diff --git a/clusterapi/bootstrap/kops/api/v1beta1/doc.go b/clusterapi/bootstrap/kops/api/v1beta1/doc.go new file mode 100644 index 0000000000..560adbec47 --- /dev/null +++ b/clusterapi/bootstrap/kops/api/v1beta1/doc.go @@ -0,0 +1,17 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 diff --git a/clusterapi/bootstrap/kops/api/v1beta1/groupversion_info.go b/clusterapi/bootstrap/kops/api/v1beta1/groupversion_info.go new file mode 100644 index 0000000000..01a85651c5 --- /dev/null +++ b/clusterapi/bootstrap/kops/api/v1beta1/groupversion_info.go @@ -0,0 +1,36 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1beta1 contains API Schema definitions for the kops v1beta1 API group +// +kubebuilder:object:generate=true +// +groupName=bootstrap.cluster.x-k8s.io +package v1beta1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects. + GroupVersion = schema.GroupVersion{Group: "bootstrap.cluster.x-k8s.io", Version: "v1beta1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme. + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/clusterapi/bootstrap/kops/api/v1beta1/kopsconfig_types.go b/clusterapi/bootstrap/kops/api/v1beta1/kopsconfig_types.go new file mode 100644 index 0000000000..0c3bc64c0d --- /dev/null +++ b/clusterapi/bootstrap/kops/api/v1beta1/kopsconfig_types.go @@ -0,0 +1,82 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// KopsConfigSpec defines the desired state of KopsConfig. +// Either ClusterConfiguration and InitConfiguration should be defined or the JoinConfiguration should be defined. +type KopsConfigSpec struct { +} + +// KopsConfigStatus defines the observed state of KopsConfig. +type KopsConfigStatus struct { + // Ready indicates the BootstrapData field is ready to be consumed + // +optional + Ready bool `json:"ready"` + + // DataSecretName is the name of the secret that stores the bootstrap data script. + // +optional + DataSecretName *string `json:"dataSecretName,omitempty"` + + // // FailureReason will be set on non-retryable errors + // // +optional + // FailureReason string `json:"failureReason,omitempty"` + + // // FailureMessage will be set on non-retryable errors + // // +optional + // FailureMessage string `json:"failureMessage,omitempty"` + + // // ObservedGeneration is the latest generation observed by the controller. + // // +optional + // ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // // Conditions defines current service state of the KopsConfig. + // // +optional + // Conditions clusterv1.Conditions `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=kopsconfigs,scope=Namespaced,categories=cluster-api +// +kubebuilder:storageversion +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Cluster",type="string",JSONPath=".metadata.labels['cluster\\.x-k8s\\.io/cluster-name']",description="Cluster" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of KopsConfig" + +// KopsConfig is the Schema for the kopsconfigs API. +type KopsConfig struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec KopsConfigSpec `json:"spec,omitempty"` + Status KopsConfigStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// KopsConfigList contains a list of KopsConfig. +type KopsConfigList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []KopsConfig `json:"items"` +} + +func init() { + SchemeBuilder.Register(&KopsConfig{}, &KopsConfigList{}) +} diff --git a/clusterapi/bootstrap/kops/api/v1beta1/kopsconfigtemplate_types.go b/clusterapi/bootstrap/kops/api/v1beta1/kopsconfigtemplate_types.go new file mode 100644 index 0000000000..f9aae71b0b --- /dev/null +++ b/clusterapi/bootstrap/kops/api/v1beta1/kopsconfigtemplate_types.go @@ -0,0 +1,64 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + clusterv1 "k8s.io/kops/clusterapi/snapshot/cluster-api/api/v1beta1" +) + +// KopsConfigTemplateSpec defines the desired state of KopsConfigTemplate. +type KopsConfigTemplateSpec struct { + Template KopsConfigTemplateResource `json:"template"` +} + +// KopsConfigTemplateResource defines the Template structure. +type KopsConfigTemplateResource struct { + // Standard object's metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // +optional + ObjectMeta clusterv1.ObjectMeta `json:"metadata,omitempty"` + + Spec KopsConfigSpec `json:"spec,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=kopsconfigtemplates,scope=Namespaced,categories=cluster-api +// +kubebuilder:storageversion +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of KopsConfigTemplate" + +// KopsConfigTemplate is the Schema for the kopsconfigtemplates API. +type KopsConfigTemplate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec KopsConfigTemplateSpec `json:"spec,omitempty"` +} + +// +kubebuilder:object:root=true + +// KopsConfigTemplateList contains a list of KopsConfigTemplate. +type KopsConfigTemplateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []KopsConfigTemplate `json:"items"` +} + +func init() { + SchemeBuilder.Register(&KopsConfigTemplate{}, &KopsConfigTemplateList{}) +} diff --git a/clusterapi/config/kustomization.yaml b/clusterapi/config/kustomization.yaml new file mode 100644 index 0000000000..32430fc516 --- /dev/null +++ b/clusterapi/config/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +commonLabels: + cluster.x-k8s.io/v1beta1: v1beta1 + +resources: +- crds/bootstrap.cluster.x-k8s.io_kopsconfigs.yaml +- crds/bootstrap.cluster.x-k8s.io_kopsconfigtemplates.yaml +- crds/controlplane.cluster.x-k8s.io_kopscontrolplanes.yaml +- crds/controlplane.cluster.x-k8s.io_kopscontrolplanetemplates.yaml diff --git a/clusterapi/controlplane/kops/api/v1beta1/doc.go b/clusterapi/controlplane/kops/api/v1beta1/doc.go new file mode 100644 index 0000000000..560adbec47 --- /dev/null +++ b/clusterapi/controlplane/kops/api/v1beta1/doc.go @@ -0,0 +1,17 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 diff --git a/clusterapi/controlplane/kops/api/v1beta1/groupversion_info.go b/clusterapi/controlplane/kops/api/v1beta1/groupversion_info.go new file mode 100644 index 0000000000..234ad14978 --- /dev/null +++ b/clusterapi/controlplane/kops/api/v1beta1/groupversion_info.go @@ -0,0 +1,36 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1beta1 contains API Schema definitions for the kops v1beta1 API group +// +kubebuilder:object:generate=true +// +groupName=controlplane.cluster.x-k8s.io +package v1beta1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects. + GroupVersion = schema.GroupVersion{Group: "controlplane.cluster.x-k8s.io", Version: "v1beta1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme. + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/clusterapi/controlplane/kops/api/v1beta1/kopscontrolplane_types.go b/clusterapi/controlplane/kops/api/v1beta1/kopscontrolplane_types.go new file mode 100644 index 0000000000..5799346cac --- /dev/null +++ b/clusterapi/controlplane/kops/api/v1beta1/kopscontrolplane_types.go @@ -0,0 +1,81 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + clusterv1 "k8s.io/kops/clusterapi/snapshot/cluster-api/api/v1beta1" +) + +// RolloutStrategyType defines the rollout strategies for a KopsControlPlane. +type RolloutStrategyType string + +// KopsControlPlaneSpec defines the desired state of KopsControlPlane. +type KopsControlPlaneSpec struct { +} + +// KopsControlPlaneMachineTemplate defines the template for Machines +// in a KopsControlPlane object. +type KopsControlPlaneMachineTemplate struct { + // Standard object's metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // +optional + ObjectMeta clusterv1.ObjectMeta `json:"metadata,omitempty"` +} + +// KopsControlPlaneStatus defines the observed state of KopsControlPlane. +type KopsControlPlaneStatus struct { +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=kopscontrolplanes,shortName=kcp,scope=Namespaced,categories=cluster-api +// +kubebuilder:storageversion +// +kubebuilder:subresource:status +// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.selector +// +kubebuilder:printcolumn:name="Cluster",type="string",JSONPath=".metadata.labels['cluster\\.x-k8s\\.io/cluster-name']",description="Cluster" +// +kubebuilder:printcolumn:name="Initialized",type=boolean,JSONPath=".status.initialized",description="This denotes whether or not the control plane has the uploaded kops-config configmap" +// +kubebuilder:printcolumn:name="API Server Available",type=boolean,JSONPath=".status.ready",description="KopsControlPlane API Server is ready to receive requests" +// +kubebuilder:printcolumn:name="Desired",type=integer,JSONPath=".spec.replicas",description="Total number of machines desired by this control plane",priority=10 +// +kubebuilder:printcolumn:name="Replicas",type=integer,JSONPath=".status.replicas",description="Total number of non-terminated machines targeted by this control plane" +// +kubebuilder:printcolumn:name="Ready",type=integer,JSONPath=".status.readyReplicas",description="Total number of fully running and ready control plane machines" +// +kubebuilder:printcolumn:name="Updated",type=integer,JSONPath=".status.updatedReplicas",description="Total number of non-terminated machines targeted by this control plane that have the desired template spec" +// +kubebuilder:printcolumn:name="Unavailable",type=integer,JSONPath=".status.unavailableReplicas",description="Total number of unavailable machines targeted by this control plane" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of KopsControlPlane" +// +kubebuilder:printcolumn:name="Version",type=string,JSONPath=".spec.version",description="Kubernetes version associated with this control plane" + +// KopsControlPlane is the Schema for the KopsControlPlane API. +type KopsControlPlane struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec KopsControlPlaneSpec `json:"spec,omitempty"` + Status KopsControlPlaneStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// KopsControlPlaneList contains a list of KopsControlPlane. +type KopsControlPlaneList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []KopsControlPlane `json:"items"` +} + +func init() { + SchemeBuilder.Register(&KopsControlPlane{}, &KopsControlPlaneList{}) +} diff --git a/clusterapi/controlplane/kops/api/v1beta1/kopscontrolplanetemplate_types.go b/clusterapi/controlplane/kops/api/v1beta1/kopscontrolplanetemplate_types.go new file mode 100644 index 0000000000..ac8a4c4575 --- /dev/null +++ b/clusterapi/controlplane/kops/api/v1beta1/kopscontrolplanetemplate_types.go @@ -0,0 +1,86 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + clusterv1 "k8s.io/kops/clusterapi/snapshot/cluster-api/api/v1beta1" + // bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kops/api/v1beta1" +) + +// KopsControlPlaneTemplateSpec defines the desired state of KopsControlPlaneTemplate. +type KopsControlPlaneTemplateSpec struct { + Template KopsControlPlaneTemplateResource `json:"template"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=kopscontrolplanetemplates,scope=Namespaced,categories=cluster-api +// +kubebuilder:storageversion +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of KopsControlPlaneTemplate" + +// KopsControlPlaneTemplate is the Schema for the kopscontrolplanetemplates API. +type KopsControlPlaneTemplate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec KopsControlPlaneTemplateSpec `json:"spec,omitempty"` +} + +// +kubebuilder:object:root=true + +// KopsControlPlaneTemplateList contains a list of KopsControlPlaneTemplate. +type KopsControlPlaneTemplateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []KopsControlPlaneTemplate `json:"items"` +} + +func init() { + SchemeBuilder.Register(&KopsControlPlaneTemplate{}, &KopsControlPlaneTemplateList{}) +} + +// KopsControlPlaneTemplateResource describes the data needed to create a KopsControlPlane from a template. +type KopsControlPlaneTemplateResource struct { + // Standard object's metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // +optional + ObjectMeta clusterv1.ObjectMeta `json:"metadata,omitempty"` + + Spec KopsControlPlaneTemplateResourceSpec `json:"spec"` +} + +// KopsControlPlaneTemplateResourceSpec defines the desired state of KopsControlPlane. +// NOTE: KopsControlPlaneTemplateResourceSpec is similar to KopsControlPlaneSpec but +// omits Replicas and Version fields. These fields do not make sense on the KopsControlPlaneTemplate, +// because they are calculated by the Cluster topology reconciler during reconciliation and thus cannot +// be configured on the KopsControlPlaneTemplate. +type KopsControlPlaneTemplateResourceSpec struct { +} + +// KopsControlPlaneTemplateMachineTemplate defines the template for Machines +// in a KopsControlPlaneTemplate object. +// NOTE: KopsControlPlaneTemplateMachineTemplate is similar to KopsControlPlaneMachineTemplate but +// omits ObjectMeta and InfrastructureRef fields. These fields do not make sense on the KopsControlPlaneTemplate, +// because they are calculated by the Cluster topology reconciler during reconciliation and thus cannot +// be configured on the KopsControlPlaneTemplate. +type KopsControlPlaneTemplateMachineTemplate struct { + // Standard object's metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // +optional + ObjectMeta clusterv1.ObjectMeta `json:"metadata,omitempty"` +} diff --git a/clusterapi/examples/manifest.yaml b/clusterapi/examples/manifest.yaml new file mode 100644 index 0000000000..c2e5a76ab3 --- /dev/null +++ b/clusterapi/examples/manifest.yaml @@ -0,0 +1,128 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" +spec: + #clusterNetwork: + # pods: + # cidrBlocks: ["192.168.0.0/16"] + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: GCPCluster + name: "${CLUSTER_NAME}" + controlPlaneRef: + kind: KopsControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: GCPCluster +metadata: + name: "${CLUSTER_NAME}" +spec: + project: "${GCP_PROJECT}" + region: "${GCP_REGION}" + network: + name: "${GCP_NETWORK_NAME}" +# --- +# kind: KubeadmControlPlane +# apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +# metadata: +# name: "${CLUSTER_NAME}-control-plane" +# spec: +# replicas: ${CONTROL_PLANE_MACHINE_COUNT} +# machineTemplate: +# infrastructureRef: +# kind: GCPMachineTemplate +# apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +# name: "${CLUSTER_NAME}-control-plane" +# kubeadmConfigSpec: +# initConfiguration: +# nodeRegistration: +# name: '{{ ds.meta_data.local_hostname.split(".")[0] }}' +# kubeletExtraArgs: +# cloud-provider: gce +# clusterConfiguration: +# apiServer: +# timeoutForControlPlane: 20m +# extraArgs: +# cloud-provider: gce +# controllerManager: +# extraArgs: +# cloud-provider: gce +# allocate-node-cidrs: "false" +# joinConfiguration: +# nodeRegistration: +# name: '{{ ds.meta_data.local_hostname.split(".")[0] }}' +# kubeletExtraArgs: +# cloud-provider: gce +# version: "${KUBERNETES_VERSION}" +# --- +# kind: GCPMachineTemplate +# apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +# metadata: +# name: "${CLUSTER_NAME}-control-plane" +# spec: +# template: +# spec: +# instanceType: "${GCP_CONTROL_PLANE_MACHINE_TYPE}" +# image: "${IMAGE_ID}" +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + failureDomain: "${GCP_ZONE}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KopsConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: GCPMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: GCPMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + instanceType: "${GCP_NODE_MACHINE_TYPE}" + image: "${IMAGE_ID}" + subnet: "${GCP_SUBNET}" + additionalNetworkTags: + - clusterapi-k8s-local-k8s-io-role-node + publicIP: true + additionalMetadata: + - key: kops-k8s-io-instance-group-name + value: nodes-us-east4-a + - key: cluster-name + value: clusterapi.k8s.local + + +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KopsConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: {} + #joinConfiguration: + # nodeRegistration: + # name: '{{ ds.meta_data.local_hostname.split(".")[0] }}' + # kubeletExtraArgs: + # cloud-provider: gce diff --git a/clusterapi/gen.go b/clusterapi/gen.go new file mode 100644 index 0000000000..e69ee0cfc6 --- /dev/null +++ b/clusterapi/gen.go @@ -0,0 +1,21 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +//go:generate go run sigs.k8s.io/controller-tools/cmd/controller-gen@v0.14.0 output:dir=config/crds crd:crdVersions=v1 paths=./bootstrap/kops/api/...;./controlplane/kops/api/... + +//go:generate go run sigs.k8s.io/controller-tools/cmd/controller-gen@v0.14.0 object paths=./snapshot/cluster-api/...;./bootstrap/kops/api/...;./controlplane/kops/api/... diff --git a/clusterapi/main.go b/clusterapi/main.go new file mode 100644 index 0000000000..80b2d88d42 --- /dev/null +++ b/clusterapi/main.go @@ -0,0 +1,113 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "flag" + "fmt" + "os" + + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + "k8s.io/klog/v2" + "k8s.io/klog/v2/klogr" + "k8s.io/kops/clusterapi/bootstrap/controllers" + bootstrapapi "k8s.io/kops/clusterapi/bootstrap/kops/api/v1beta1" + controlplaneapi "k8s.io/kops/clusterapi/controlplane/kops/api/v1beta1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/metrics/server" + // +kubebuilder:scaffold:imports +) + +var ( + scheme = runtime.NewScheme() +) + +func init() { + // +kubebuilder:scaffold:scheme +} + +func main() { + ctx := context.Background() + if err := run(ctx); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} + +func run(ctx context.Context) error { + klog.InitFlags(nil) + + // Disable metrics by default (avoid port conflicts, also risky because we are host network) + metricsAddress := ":0" + + flag.Parse() + + ctrl.SetLogger(klogr.New()) + + if err := buildScheme(); err != nil { + return fmt.Errorf("error building scheme: %w", err) + } + + kubeConfig := ctrl.GetConfigOrDie() + options := ctrl.Options{ + Scheme: scheme, + // MetricsBindAddress: metricsAddress, + // LeaderElection: true, + // LeaderElectionID: "kops-clusterapi-leader", + } + options.Metrics = server.Options{ + BindAddress: metricsAddress, + } + mgr, err := ctrl.NewManager(kubeConfig, options) + + if err != nil { + return fmt.Errorf("error starting manager: %w", err) + } + + if err := controllers.NewKopsConfigReconciler(mgr); err != nil { + return fmt.Errorf("error creating controller: %w", err) + } + + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + return fmt.Errorf("error running manager: %w", err) + } + return nil +} + +func buildScheme() error { + if err := corev1.AddToScheme(scheme); err != nil { + return fmt.Errorf("error registering corev1: %v", err) + } + + if err := bootstrapapi.AddToScheme(scheme); err != nil { + return fmt.Errorf("error registering api: %w", err) + } + + if err := controlplaneapi.AddToScheme(scheme); err != nil { + return fmt.Errorf("error registering api: %w", err) + } + + // Needed so that the leader-election system can post events + if err := coordinationv1.AddToScheme(scheme); err != nil { + return fmt.Errorf("error registering coordinationv1: %v", err) + } + return nil +} diff --git a/pkg/nodeidentity/interfaces.go b/pkg/nodeidentity/interfaces.go index 65d9bd8f8c..4236ff9978 100644 --- a/pkg/nodeidentity/interfaces.go +++ b/pkg/nodeidentity/interfaces.go @@ -36,7 +36,8 @@ type LegacyIdentifier interface { } type LegacyInfo struct { - InstanceID string - InstanceGroup string + InstanceID string + InstanceGroup string + // TODO: Remove InstanceLifecycle string } diff --git a/upup/pkg/fi/nodeup/command.go b/upup/pkg/fi/nodeup/command.go index 4a98c7c86c..d44d1e622a 100644 --- a/upup/pkg/fi/nodeup/command.go +++ b/upup/pkg/fi/nodeup/command.go @@ -153,8 +153,8 @@ func (c *NodeUpCommand) Run(out io.Writer) error { return fmt.Errorf("no instance group defined in nodeup config") } - if want := bootConfig.NodeupConfigHash; want != "" { - if got := base64.StdEncoding.EncodeToString(nodeupConfigHash[:]); got != want { + if bootConfig.NodeupConfigHash != "" { + if want, got := bootConfig.NodeupConfigHash, base64.StdEncoding.EncodeToString(nodeupConfigHash[:]); got != want { return fmt.Errorf("nodeup config hash mismatch (was %q, expected %q)", got, want) } }