move openstack cloud config to k8s secrets

nodeup/pkg/model/cloudconfig.go

@@ -19,11 +19,11 @@ package model
 import (
 	"encoding/json"
 	"fmt"
-	"os"
 	"strings"
 
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
 )
 
@@ -123,95 +123,9 @@ func (b *CloudConfigBuilder) build(c *fi.NodeupModelBuilderContext, inTree bool)
 		if osc == nil {
 			break
 		}
-		// Support mapping of older keystone API
-		tenantName := os.Getenv("OS_TENANT_NAME")
-		if tenantName == "" {
-			tenantName = os.Getenv("OS_PROJECT_NAME")
-		}
-		tenantID := os.Getenv("OS_TENANT_ID")
-		if tenantID == "" {
-			tenantID = os.Getenv("OS_PROJECT_ID")
-		}
-		lines = append(lines,
-			fmt.Sprintf("auth-url=\"%s\"", os.Getenv("OS_AUTH_URL")),
-			fmt.Sprintf("username=\"%s\"", os.Getenv("OS_USERNAME")),
-			fmt.Sprintf("password=\"%s\"", os.Getenv("OS_PASSWORD")),
-			fmt.Sprintf("region=\"%s\"", os.Getenv("OS_REGION_NAME")),
-			fmt.Sprintf("tenant-id=\"%s\"", tenantID),
-			fmt.Sprintf("tenant-name=\"%s\"", tenantName),
-			fmt.Sprintf("domain-name=\"%s\"", os.Getenv("OS_DOMAIN_NAME")),
-			fmt.Sprintf("domain-id=\"%s\"", os.Getenv("OS_DOMAIN_ID")),
-		)
-		if b.Cluster.Spec.ExternalCloudControllerManager != nil {
-			lines = append(lines,
-				fmt.Sprintf("application-credential-id=\"%s\"", os.Getenv("OS_APPLICATION_CREDENTIAL_ID")),
-				fmt.Sprintf("application-credential-secret=\"%s\"", os.Getenv("OS_APPLICATION_CREDENTIAL_SECRET")),
-			)
-		}
 
-		lines = append(lines,
+		lines = append(lines, openstack.MakeCloudConfig(b.Cluster.Spec)...)
-			"",
-		)
 
-		if lb := osc.Loadbalancer; lb != nil {
-			ingressHostnameSuffix := "nip.io"
-			if fi.ValueOf(lb.IngressHostnameSuffix) != "" {
-				ingressHostnameSuffix = fi.ValueOf(lb.IngressHostnameSuffix)
-			}
-
-			lines = append(lines,
-				"[LoadBalancer]",
-				fmt.Sprintf("floating-network-id=%s", fi.ValueOf(lb.FloatingNetworkID)),
-				fmt.Sprintf("lb-method=%s", fi.ValueOf(lb.Method)),
-				fmt.Sprintf("lb-provider=%s", fi.ValueOf(lb.Provider)),
-				fmt.Sprintf("use-octavia=%t", fi.ValueOf(lb.UseOctavia)),
-				fmt.Sprintf("manage-security-groups=%t", fi.ValueOf(lb.ManageSecGroups)),
-				fmt.Sprintf("enable-ingress-hostname=%t", fi.ValueOf(lb.EnableIngressHostname)),
-				fmt.Sprintf("ingress-hostname-suffix=%s", ingressHostnameSuffix),
-				"",
-			)
-
-			if monitor := osc.Monitor; monitor != nil {
-				lines = append(lines,
-					"create-monitor=yes",
-					fmt.Sprintf("monitor-delay=%s", fi.ValueOf(monitor.Delay)),
-					fmt.Sprintf("monitor-timeout=%s", fi.ValueOf(monitor.Timeout)),
-					fmt.Sprintf("monitor-max-retries=%d", fi.ValueOf(monitor.MaxRetries)),
-					"",
-				)
-			}
-		}
-
-		if bs := osc.BlockStorage; bs != nil {
-			// Block Storage Config
-			lines = append(lines,
-				"[BlockStorage]",
-				fmt.Sprintf("bs-version=%s", fi.ValueOf(bs.Version)),
-				fmt.Sprintf("ignore-volume-az=%t", fi.ValueOf(bs.IgnoreAZ)),
-				"")
-		}
-
-		if networking := osc.Network; networking != nil {
-			// Networking Config
-			// https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/openstack-cloud-controller-manager/using-openstack-cloud-controller-manager.md#networking
-			var networkingLines []string
-
-			if networking.IPv6SupportDisabled != nil {
-				networkingLines = append(networkingLines, fmt.Sprintf("ipv6-support-disabled=%t", fi.ValueOf(networking.IPv6SupportDisabled)))
-			}
-			for _, name := range networking.PublicNetworkNames {
-				networkingLines = append(networkingLines, fmt.Sprintf("public-network-name=%s", fi.ValueOf(name)))
-			}
-			for _, name := range networking.InternalNetworkNames {
-				networkingLines = append(networkingLines, fmt.Sprintf("internal-network-name=%s", fi.ValueOf(name)))
-			}
-
-			if len(networkingLines) > 0 {
-				lines = append(lines, "[Networking]")
-				lines = append(lines, networkingLines...)
-				lines = append(lines, "")
-			}
-		}
 	case "azure":
 		requireGlobal = false
 

upup/models/cloudup/resources/addons/openstack.addons.k8s.io/k8s-1.13.yaml.template

@@ -1,5 +1,14 @@
 ---
 apiVersion: v1
+kind: Secret
+metadata:
+  name: openstack-project
+  namespace: kube-system
+stringData:
+  cloud.config: |
+{{ OPENSTACK_CONF | indent 4 }}
+---
+apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: cloud-controller-manager
@@ -221,7 +230,7 @@ spec:
           requests:
             cpu: {{ or .ExternalCloudControllerManager.CPURequest "200m" }}
         volumeMounts:
-        - mountPath: /etc/kubernetes/cloud.config
+        - mountPath: /etc/kubernetes
           name: cloudconfig
           readOnly: true
 {{ if .UseHostCertificates }}
@@ -230,8 +239,8 @@ spec:
           readOnly: true
 {{ end }}
       volumes:
-      - hostPath:
+      - secret:
-          path: /etc/kubernetes/cloud.config
+          secretName: openstack-project
         name: cloudconfig
 {{ if .UseHostCertificates }}
       - hostPath:

upup/models/cloudup/resources/addons/storage-openstack.addons.k8s.io/k8s-1.16.yaml.template

@@ -350,15 +350,14 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /csi
-            - mountPath: /etc/kubernetes/cloud.config
+            - mountPath: /etc/kubernetes
               name: cloudconfig
               readOnly: true
       volumes:
         - name: socket-dir
           emptyDir: {}
-        - hostPath:
+        - secret:
-            path: /etc/kubernetes/cloud.config
+            secretName: openstack-project
-            type: ""
           name: cloudconfig
 
 ---
@@ -493,7 +492,7 @@ spec:
             - name: pods-probe-dir
               mountPath: /dev
               mountPropagation: "HostToContainer"
-            - mountPath: /etc/kubernetes/cloud.config
+            - mountPath: /etc/kubernetes
               name: cloudconfig
               readOnly: true
       volumes:
@@ -513,9 +512,8 @@ spec:
           hostPath:
             path: /dev
             type: Directory
-        - hostPath:
+        - secret:
-            path: /etc/kubernetes/cloud.config
+            secretName: openstack-project
-            type: ""
           name: cloudconfig
 ---
 apiVersion: storage.k8s.io/v1

upup/pkg/fi/cloudup/openstack/cloud.go

@@ -20,12 +20,13 @@ import (
 	"crypto/tls"
 	"fmt"
 	"net/http"
+	"os"
 	"strings"
 	"time"
 
 	"github.com/blang/semver/v4"
 	"github.com/gophercloud/gophercloud"
-	os "github.com/gophercloud/gophercloud/openstack"
+	"github.com/gophercloud/gophercloud/openstack"
 	cinder "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes"
 	az "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/availabilityzones"
 	"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/keypairs"
@@ -338,7 +339,7 @@ func NewOpenstackCloud(cluster *kops.Cluster, uagent string) (OpenstackCloud, er
 		return nil, err
 	}
 
-	provider, err := os.NewClient(authOption.IdentityEndpoint)
+	provider, err := openstack.NewClient(authOption.IdentityEndpoint)
 	if err != nil {
 		return nil, fmt.Errorf("error building openstack provider client: %v", err)
 	}
@@ -358,7 +359,7 @@ func NewOpenstackCloud(cluster *kops.Cluster, uagent string) (OpenstackCloud, er
 
 	klog.V(2).Info("authenticating to keystone")
 
-	err = os.Authenticate(provider, authOption)
+	err = openstack.Authenticate(provider, authOption)
 	if err != nil {
 		return nil, fmt.Errorf("error building openstack authenticated client: %v", err)
 	}
@@ -375,7 +376,7 @@ func NewOpenstackCloud(cluster *kops.Cluster, uagent string) (OpenstackCloud, er
 }
 
 func buildClients(provider *gophercloud.ProviderClient, tags map[string]string, spec *kops.OpenstackSpec, config vfs.OpenstackConfig, region string, hasDNS bool) (OpenstackCloud, error) {
-	cinderClient, err := os.NewBlockStorageV3(provider, gophercloud.EndpointOpts{
+	cinderClient, err := openstack.NewBlockStorageV3(provider, gophercloud.EndpointOpts{
 		Type:   "volumev3",
 		Region: region,
 	})
@@ -383,7 +384,7 @@ func buildClients(provider *gophercloud.ProviderClient, tags map[string]string,
 		return nil, fmt.Errorf("error building cinder client: %w", err)
 	}
 
-	neutronClient, err := os.NewNetworkV2(provider, gophercloud.EndpointOpts{
+	neutronClient, err := openstack.NewNetworkV2(provider, gophercloud.EndpointOpts{
 		Type:   "network",
 		Region: region,
 	})
@@ -391,7 +392,7 @@ func buildClients(provider *gophercloud.ProviderClient, tags map[string]string,
 		return nil, fmt.Errorf("error building neutron client: %w", err)
 	}
 
-	novaClient, err := os.NewComputeV2(provider, gophercloud.EndpointOpts{
+	novaClient, err := openstack.NewComputeV2(provider, gophercloud.EndpointOpts{
 		Type:   "compute",
 		Region: region,
 	})
@@ -401,7 +402,7 @@ func buildClients(provider *gophercloud.ProviderClient, tags map[string]string,
 	// 2.47 is the minimum version where the compute API /server/details returns flavor names
 	novaClient.Microversion = "2.47"
 
-	glanceClient, err := os.NewImageServiceV2(provider, gophercloud.EndpointOpts{
+	glanceClient, err := openstack.NewImageServiceV2(provider, gophercloud.EndpointOpts{
 		Type:   "image",
 		Region: region,
 	})
@@ -417,7 +418,7 @@ func buildClients(provider *gophercloud.ProviderClient, tags map[string]string,
 			return nil, fmt.Errorf("failed to get service config: %w", err)
 		}
 
-		dnsClient, err = os.NewDNSV2(provider, endpointOpt)
+		dnsClient, err = openstack.NewDNSV2(provider, endpointOpt)
 		if err != nil {
 			return nil, fmt.Errorf("error building dns client: %w", err)
 		}
@@ -488,7 +489,7 @@ func buildLoadBalancerClient(c *openstackCloud, spec *kops.OpenstackSpec, provid
 	var lbClient *gophercloud.ServiceClient
 	if octavia {
 		klog.V(2).Infof("Openstack using Octavia lbaasv2 api")
-		client, err := os.NewLoadBalancerV2(provider, gophercloud.EndpointOpts{
+		client, err := openstack.NewLoadBalancerV2(provider, gophercloud.EndpointOpts{
 			Region: region,
 		})
 		if err != nil {
@@ -497,7 +498,7 @@ func buildLoadBalancerClient(c *openstackCloud, spec *kops.OpenstackSpec, provid
 		lbClient = client
 	} else {
 		klog.V(2).Infof("Openstack using deprecated lbaasv2 api")
-		client, err := os.NewNetworkV2(provider, gophercloud.EndpointOpts{
+		client, err := openstack.NewNetworkV2(provider, gophercloud.EndpointOpts{
 			Region: region,
 		})
 		if err != nil {
@@ -816,3 +817,104 @@ func isNotFound(err error) bool {
 
 	return false
 }
+
+func MakeCloudConfig(spec kops.ClusterSpec) []string {
+	var lines []string
+
+	osc := spec.CloudProvider.Openstack
+	if osc == nil {
+		return nil
+	}
+
+	// Support mapping of older keystone API
+	tenantName := os.Getenv("OS_TENANT_NAME")
+	if tenantName == "" {
+		tenantName = os.Getenv("OS_PROJECT_NAME")
+	}
+	tenantID := os.Getenv("OS_TENANT_ID")
+	if tenantID == "" {
+		tenantID = os.Getenv("OS_PROJECT_ID")
+	}
+	lines = append(lines,
+		fmt.Sprintf("auth-url=\"%s\"", os.Getenv("OS_AUTH_URL")),
+		fmt.Sprintf("username=\"%s\"", os.Getenv("OS_USERNAME")),
+		fmt.Sprintf("password=\"%s\"", os.Getenv("OS_PASSWORD")),
+		fmt.Sprintf("region=\"%s\"", os.Getenv("OS_REGION_NAME")),
+		fmt.Sprintf("tenant-id=\"%s\"", tenantID),
+		fmt.Sprintf("tenant-name=\"%s\"", tenantName),
+		fmt.Sprintf("domain-name=\"%s\"", os.Getenv("OS_DOMAIN_NAME")),
+		fmt.Sprintf("domain-id=\"%s\"", os.Getenv("OS_DOMAIN_ID")),
+	)
+	if spec.ExternalCloudControllerManager != nil {
+		lines = append(lines,
+			fmt.Sprintf("application-credential-id=\"%s\"", os.Getenv("OS_APPLICATION_CREDENTIAL_ID")),
+			fmt.Sprintf("application-credential-secret=\"%s\"", os.Getenv("OS_APPLICATION_CREDENTIAL_SECRET")),
+		)
+	}
+
+	lines = append(lines,
+		"",
+	)
+
+	if lb := osc.Loadbalancer; lb != nil {
+		ingressHostnameSuffix := "nip.io"
+		if fi.ValueOf(lb.IngressHostnameSuffix) != "" {
+			ingressHostnameSuffix = fi.ValueOf(lb.IngressHostnameSuffix)
+		}
+
+		lines = append(lines,
+			"[LoadBalancer]",
+			fmt.Sprintf("floating-network-id=%s", fi.ValueOf(lb.FloatingNetworkID)),
+			fmt.Sprintf("lb-method=%s", fi.ValueOf(lb.Method)),
+			fmt.Sprintf("lb-provider=%s", fi.ValueOf(lb.Provider)),
+			fmt.Sprintf("use-octavia=%t", fi.ValueOf(lb.UseOctavia)),
+			fmt.Sprintf("manage-security-groups=%t", fi.ValueOf(lb.ManageSecGroups)),
+			fmt.Sprintf("enable-ingress-hostname=%t", fi.ValueOf(lb.EnableIngressHostname)),
+			fmt.Sprintf("ingress-hostname-suffix=%s", ingressHostnameSuffix),
+			"",
+		)
+
+		if monitor := osc.Monitor; monitor != nil {
+			lines = append(lines,
+				"create-monitor=yes",
+				fmt.Sprintf("monitor-delay=%s", fi.ValueOf(monitor.Delay)),
+				fmt.Sprintf("monitor-timeout=%s", fi.ValueOf(monitor.Timeout)),
+				fmt.Sprintf("monitor-max-retries=%d", fi.ValueOf(monitor.MaxRetries)),
+				"",
+			)
+		}
+	}
+
+	if bs := osc.BlockStorage; bs != nil {
+		// Block Storage Config
+		lines = append(lines,
+			"[BlockStorage]",
+			fmt.Sprintf("bs-version=%s", fi.ValueOf(bs.Version)),
+			fmt.Sprintf("ignore-volume-az=%t", fi.ValueOf(bs.IgnoreAZ)),
+			"")
+	}
+
+	if networking := osc.Network; networking != nil {
+		// Networking Config
+		// https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/openstack-cloud-controller-manager/using-openstack-cloud-controller-manager.md#networking
+		var networkingLines []string
+
+		if networking.IPv6SupportDisabled != nil {
+			networkingLines = append(networkingLines, fmt.Sprintf("ipv6-support-disabled=%t", fi.ValueOf(networking.IPv6SupportDisabled)))
+		}
+		for _, name := range networking.PublicNetworkNames {
+			networkingLines = append(networkingLines, fmt.Sprintf("public-network-name=%s", fi.ValueOf(name)))
+		}
+		for _, name := range networking.InternalNetworkNames {
+			networkingLines = append(networkingLines, fmt.Sprintf("internal-network-name=%s", fi.ValueOf(name)))
+		}
+
+		if len(networkingLines) > 0 {
+			lines = append(lines, "[Networking]")
+			lines = append(lines, networkingLines...)
+			lines = append(lines, "")
+		}
+	}
+
+	return lines
+}

upup/pkg/fi/cloudup/template_functions.go

@@ -64,6 +64,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi/cloudup/gce"
 	gcetpm "k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm"
 	"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
+	"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
 	"k8s.io/kops/util/pkg/env"
 	"sigs.k8s.io/yaml"
 )
@@ -183,6 +184,11 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS
 		return cluster.Name
 	}
 
+	dest["OPENSTACK_CONF"] = func() string {
+		lines := openstack.MakeCloudConfig(cluster.Spec)
+		return "[global]\n" + strings.Join(lines, "\n") + "\n"
+	}
+
 	if featureflag.Spotinst.Enabled() {
 		if creds, err := spotinst.LoadCredentials(); err == nil {
 			dest["SpotinstToken"] = func() string { return creds.Token }