Merge pull request #14460 from hakman/hetzner_bootstrap_kops-controller

hetzner: Use kops-controller for node bootstrap

cmd/kops-controller/main.go

@@ -41,6 +41,7 @@ import (
 	nodeidentityos "k8s.io/kops/pkg/nodeidentity/openstack"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	"k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmverifier"
+	"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/yaml"
@@ -119,6 +120,12 @@ func main() {
 				setupLog.Error(err, "unable to create verifier")
 				os.Exit(1)
 			}
+		} else if opt.Server.Provider.Hetzner != nil {
+			verifier, err = hetzner.NewHetznerVerifier(opt.Server.Provider.Hetzner)
+			if err != nil {
+				setupLog.Error(err, "unable to create verifier")
+				os.Exit(1)
+			}
 		} else {
 			klog.Fatalf("server cloud provider config not provided")
 		}

cmd/kops-controller/pkg/config/options.go

@@ -19,6 +19,7 @@ package config
 import (
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	gcetpm "k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm"
+	"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
 )
 
 type Options struct {
@@ -61,8 +62,9 @@ type ServerOptions struct {
 }
 
 type ServerProviderOptions struct {
-	AWS *awsup.AWSVerifierOptions  `json:"aws,omitempty"`
-	GCE *gcetpm.TPMVerifierOptions `json:"gce,omitempty"`
+	AWS     *awsup.AWSVerifierOptions       `json:"aws,omitempty"`
+	GCE     *gcetpm.TPMVerifierOptions      `json:"gce,omitempty"`
+	Hetzner *hetzner.HetznerVerifierOptions `json:"hetzner,omitempty"`
 }
 
 // DiscoveryOptions configures our support for discovery, particularly gossip DNS (i.e. k8s.local)

cmd/kops/integration_test.go

@@ -1467,7 +1467,6 @@ func (i *integrationTest) runTestTerraformHetzner(t *testing.T) {
 		"aws_s3_object_"+i.clusterName+"-addons-kops-controller.addons.k8s.io-k8s-1.16_content",
 		"aws_s3_object_"+i.clusterName+"-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content",
 		"aws_s3_object_"+i.clusterName+"-addons-limit-range.addons.k8s.io_content",
-		"aws_s3_object_"+i.clusterName+"-addons-rbac.addons.k8s.io-k8s-1.8_content",
 		"hcloud_server_master-fsn1_user_data",
 		"hcloud_server_nodes-fsn1_user_data",
 	)

nodeup/pkg/model/bootstrap_client.go

@@ -28,6 +28,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	"k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmsigner"
+	"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
 )
 
@@ -50,6 +51,8 @@ func (b BootstrapClientBuilder) Build(c *fi.ModelBuilderContext) error {
 		authenticator, err = gcetpmsigner.NewTPMAuthenticator()
 		// We don't use the custom resolver here in gossip mode (though we could);
 		// instead we use this as a check that protokube has now started.
+	case kops.CloudProviderHetzner:
+		authenticator, err = hetzner.NewHetznerAuthenticator()
 
 	default:
 		return fmt.Errorf("unsupported cloud provider for authenticator %q", b.CloudProvider)

pkg/apis/kops/model/features.go

@@ -27,6 +27,8 @@ func UseKopsControllerForNodeBootstrap(cluster *kops.Cluster) bool {
 		return true
 	case kops.CloudProviderGCE:
 		return cluster.IsKubernetesGTE("1.22")
+	case kops.CloudProviderHetzner:
+		return true
 	default:
 		return false
 	}

tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_minimal.example.com-addons-bootstrap_content

@@ -6,7 +6,7 @@ spec:
   addons:
   - id: k8s-1.16
     manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: 70c5a1ff20acaea9d84940b827cae1e35538ac7a8f2b17f8d48f02a105963baa
+    manifestHash: 7ee4cd99b996adcdee8611a23f5b27b88f50176e64c4639295bad04c3a865553
     name: kops-controller.addons.k8s.io
     needsRollingUpdate: control-plane
     selector:
@@ -19,13 +19,6 @@ spec:
     selector:
       k8s-addon: coredns.addons.k8s.io
     version: 9.99.0
-  - id: k8s-1.8
-    manifest: rbac.addons.k8s.io/k8s-1.8.yaml
-    manifestHash: f81bd7c57bc1902ca342635d7ad7d01b82dfeaff01a1192b076e66907d87871e
-    name: rbac.addons.k8s.io
-    selector:
-      k8s-addon: rbac.addons.k8s.io
-    version: 9.99.0
   - id: k8s-1.9
     manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml
     manifestHash: 01c120e887bd98d82ef57983ad58a0b22bc85efb48108092a24c4b82e4c9ea81

tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_minimal.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content

@@ -1,7 +1,7 @@
 apiVersion: v1
 data:
   config.yaml: |
-    {"cloud":"hetzner","configBase":"memfs://tests/minimal.example.com"}
+    {"cloud":"hetzner","configBase":"memfs://tests/minimal.example.com","server":{"Listen":":3988","provider":{"hetzner":{}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"]}}
 kind: ConfigMap
 metadata:
   creationTimestamp: null
@@ -32,6 +32,8 @@ spec:
       k8s-app: kops-controller
   template:
     metadata:
+      annotations:
+        dns.alpha.kubernetes.io/internal: kops-controller.internal.minimal.example.com
       creationTimestamp: null
       labels:
         k8s-addon: kops-controller.addons.k8s.io

tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_minimal.example.com-addons-rbac.addons.k8s.io-k8s-1.8_content

@@ -1,19 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  creationTimestamp: null
-  labels:
-    addon.kops.k8s.io/name: rbac.addons.k8s.io
-    addonmanager.kubernetes.io/mode: Reconcile
-    app.kubernetes.io/managed-by: kops
-    k8s-addon: rbac.addons.k8s.io
-    kubernetes.io/cluster-service: "true"
-  name: kubelet-cluster-admin
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:node
-subjects:
-- apiGroup: rbac.authorization.k8s.io
-  kind: User
-  name: kubelet

tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_nodeupconfig-nodes-fsn1_content

@@ -38,8 +38,6 @@ Hooks:
 - null
 - null
 KeypairIDs:
-  kube-proxy: "6986354184403674830529235586"
-  kubelet: "6986354184404014133128804066"
   kubernetes-ca: "6982820025135291416230495506"
 KubeletConfig:
   anonymousAuth: false

tests/integration/update_cluster/minimal_hetzner/data/hcloud_server_nodes-fsn1_user_data

@@ -162,7 +162,7 @@ CloudProvider: hetzner
 ConfigBase: memfs://tests/minimal.example.com
 InstanceGroupName: nodes-fsn1
 InstanceGroupRole: Node
-NodeupConfigHash: PWu9ALIVbND2iz1ciF+yjlTyJJFqCSux2yGwXKoPjUw=
+NodeupConfigHash: 2sOyCik6Vx/z8LXTIydiop+WbTq7ndCUfhQFvGQ/Ki0=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/minimal_hetzner/kubernetes.tf

@@ -131,14 +131,6 @@ resource "aws_s3_object" "minimal-example-com-addons-limit-range-addons-k8s-io"
   server_side_encryption = "AES256"
 }
 
-resource "aws_s3_object" "minimal-example-com-addons-rbac-addons-k8s-io-k8s-1-8" {
-  bucket                 = "testingBucket"
-  content                = file("${path.module}/data/aws_s3_object_minimal.example.com-addons-rbac.addons.k8s.io-k8s-1.8_content")
-  key                    = "tests/minimal.example.com/addons/rbac.addons.k8s.io/k8s-1.8.yaml"
-  provider               = aws.files
-  server_side_encryption = "AES256"
-}
-
 resource "aws_s3_object" "nodeupconfig-master-fsn1" {
   bucket                 = "testingBucket"
   content                = file("${path.module}/data/aws_s3_object_nodeupconfig-master-fsn1_content")

upup/pkg/fi/cloudup/hetzner/authenticator.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hetzner
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/hetznercloud/hcloud-go/hcloud/metadata"
+	"k8s.io/kops/pkg/bootstrap"
+)
+
+const HetznerAuthenticationTokenPrefix = "x-hetzner-id "
+
+type hetznerAuthenticator struct {
+}
+
+var _ bootstrap.Authenticator = &hetznerAuthenticator{}
+
+func NewHetznerAuthenticator() (bootstrap.Authenticator, error) {
+	return &hetznerAuthenticator{}, nil
+}
+
+func (h hetznerAuthenticator) CreateToken(body []byte) (string, error) {
+	serverID, err := metadata.NewClient().InstanceID()
+	if err != nil {
+		return "", fmt.Errorf("failed to retrieve server ID: %w", err)
+	}
+
+	return HetznerAuthenticationTokenPrefix + strconv.Itoa(serverID), nil
+}

upup/pkg/fi/cloudup/hetzner/verifier.go

@@ -0,0 +1,94 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hetzner
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/hetznercloud/hcloud-go/hcloud"
+	"k8s.io/kops/pkg/bootstrap"
+)
+
+type HetznerVerifierOptions struct {
+}
+
+type hetznerVerifier struct {
+	opt    HetznerVerifierOptions
+	client *hcloud.Client
+}
+
+var _ bootstrap.Verifier = &hetznerVerifier{}
+
+func NewHetznerVerifier(opt *HetznerVerifierOptions) (bootstrap.Verifier, error) {
+	hcloudToken := os.Getenv("HCLOUD_TOKEN")
+	if hcloudToken == "" {
+		return nil, fmt.Errorf("%s is required", "HCLOUD_TOKEN")
+	}
+
+	opts := []hcloud.ClientOption{
+		hcloud.WithToken(hcloudToken),
+	}
+	hcloudClient := hcloud.NewClient(opts...)
+
+	return &hetznerVerifier{
+		opt:    *opt,
+		client: hcloudClient,
+	}, nil
+}
+
+func (h hetznerVerifier) VerifyToken(ctx context.Context, token string, body []byte, useInstanceIDForNodeName bool) (*bootstrap.VerifyResult, error) {
+	if !strings.HasPrefix(token, HetznerAuthenticationTokenPrefix) {
+		return nil, fmt.Errorf("incorrect authorization type")
+	}
+	token = strings.TrimPrefix(token, HetznerAuthenticationTokenPrefix)
+
+	serverID, err := strconv.Atoi(token)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert server ID %q to int: %w", token, err)
+	}
+	server, _, err := h.client.Server.GetByID(ctx, serverID)
+	if err != nil || server == nil {
+		return nil, fmt.Errorf("failed to get info for server %q: %w", token, err)
+	}
+
+	var addrs []string
+	if server.PublicNet.IPv4.IP != nil {
+		addrs = append(addrs, server.PublicNet.IPv4.IP.String())
+	}
+	for _, network := range server.PrivateNet {
+		if network.IP != nil {
+			addrs = append(addrs, network.IP.String())
+		}
+	}
+
+	result := &bootstrap.VerifyResult{
+		NodeName:         server.Name,
+		CertificateNames: addrs,
+	}
+
+	for key, value := range server.Labels {
+		if key == TagKubernetesInstanceGroup {
+			result.InstanceGroupName = value
+		}
+	}
+
+	return result, nil
+}

upup/pkg/fi/cloudup/template_functions.go

@@ -64,6 +64,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	"k8s.io/kops/upup/pkg/fi/cloudup/gce"
 	gcetpm "k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm"
+	"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
 	"k8s.io/kops/util/pkg/env"
 )
 
@@ -655,6 +656,10 @@ func (tf *TemplateFunctions) KopsControllerConfig() (string, error) {
 				Region:      tf.Region,
 				MaxTimeSkew: 300,
 			}
+
+		case kops.CloudProviderHetzner:
+			config.Server.Provider.Hetzner = &hetzner.HetznerVerifierOptions{}
+
 		default:
 			return "", fmt.Errorf("unsupported cloud provider %s", cluster.Spec.GetCloudProvider())
 		}

upup/pkg/fi/nodeup/command.go

@@ -52,6 +52,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	"k8s.io/kops/upup/pkg/fi/cloudup/gce/gcediscovery"
 	"k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmsigner"
+	"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
 	"k8s.io/kops/upup/pkg/fi/nodeup/cloudinit"
 	"k8s.io/kops/upup/pkg/fi/nodeup/local"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
@@ -757,6 +758,12 @@ func getNodeConfigFromServer(ctx context.Context, bootConfig *nodeup.BootConfig,
 			return nil, err
 		}
 		resolver = discovery
+	case api.CloudProviderHetzner:
+		a, err := hetzner.NewHetznerAuthenticator()
+		if err != nil {
+			return nil, err
+		}
+		authenticator = a
 	default:
 		return nil, fmt.Errorf("unsupported cloud provider for node configuration %s", bootConfig.CloudProvider)
 	}