diff --git a/cmd/kops-controller/BUILD.bazel b/cmd/kops-controller/BUILD.bazel index 1e8663cfef..8007bf1f95 100644 --- a/cmd/kops-controller/BUILD.bazel +++ b/cmd/kops-controller/BUILD.bazel @@ -7,6 +7,7 @@ go_library( visibility = ["//visibility:private"], deps = [ "//cmd/kops-controller/controllers:go_default_library", + "//cmd/kops-controller/pkg/config:go_default_library", "//pkg/nodeidentity:go_default_library", "//pkg/nodeidentity/aws:go_default_library", "//pkg/nodeidentity/gce:go_default_library", @@ -18,6 +19,7 @@ go_library( "//vendor/k8s.io/klog/klogr:go_default_library", "//vendor/sigs.k8s.io/controller-runtime:go_default_library", "//vendor/sigs.k8s.io/controller-runtime/pkg/manager:go_default_library", + "//vendor/sigs.k8s.io/yaml:go_default_library", ], ) diff --git a/cmd/kops-controller/main.go b/cmd/kops-controller/main.go index bc082177e5..26f6953a6c 100644 --- a/cmd/kops-controller/main.go +++ b/cmd/kops-controller/main.go @@ -19,24 +19,24 @@ package main import ( "flag" "fmt" + "io/ioutil" "os" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "k8s.io/klog" - "k8s.io/kops/cmd/kops-controller/controllers" - ctrl "sigs.k8s.io/controller-runtime" - - // +kubebuilder:scaffold:imports - "k8s.io/klog/klogr" + "k8s.io/kops/cmd/kops-controller/controllers" + "k8s.io/kops/cmd/kops-controller/pkg/config" "k8s.io/kops/pkg/nodeidentity" nodeidentityaws "k8s.io/kops/pkg/nodeidentity/aws" nodeidentitygce "k8s.io/kops/pkg/nodeidentity/gce" nodeidentityos "k8s.io/kops/pkg/nodeidentity/openstack" - + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/yaml" + // +kubebuilder:scaffold:imports ) var ( @@ -49,26 +49,40 @@ func init() { // +kubebuilder:scaffold:scheme } -type Options struct { - Cloud string - ConfigPath string -} - func main() { klog.InitFlags(nil) - var metricsAddr string - var enableLeaderElection bool - flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") - flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, - "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") + // Disable metrics by default (avoid port conflicts, also risky because we are host network) + metricsAddress := ":0" + //flag.StringVar(&metricsAddr, "metrics-addr", metricsAddress, "The address the metric endpoint binds to.") + // Enable leader election + enableLeaderElection := false + //flag.BoolVar(&enableLeaderElection, "enable-leader-election", enableLeaderElection, + // "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") - var opt Options - flag.StringVar(&opt.Cloud, "cloud", opt.Cloud, "CloudProvider we are using (aws,gce,...)") - flag.StringVar(&opt.ConfigPath, "config", opt.ConfigPath, "Base location for cluster and instancegroup configurations") + configPath := "/etc/kubernetes/kops-controller/config.yaml" + flag.StringVar(&configPath, "conf", configPath, "Location of yaml configuration file") flag.Parse() + if configPath == "" { + klog.Fatalf("must specify --conf") + } + + var opt config.Options + opt.PopulateDefaults() + + { + b, err := ioutil.ReadFile(configPath) + if err != nil { + klog.Fatalf("failed to read configuration file %q: %v", configPath, err) + } + + if err := yaml.Unmarshal(b, &opt); err != nil { + klog.Fatalf("failed to parse configuration file %q: %v", configPath, err) + } + } + ctrl.SetLogger(klogr.New()) if err := buildScheme(); err != nil { @@ -78,7 +92,7 @@ func main() { mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, - MetricsBindAddress: metricsAddr, + MetricsBindAddress: metricsAddress, LeaderElection: enableLeaderElection, }) if err != nil { @@ -106,7 +120,7 @@ func buildScheme() error { return nil } -func addNodeController(mgr manager.Manager, opt *Options) error { +func addNodeController(mgr manager.Manager, opt *config.Options) error { var identifier nodeidentity.Identifier var err error switch opt.Cloud { @@ -134,11 +148,11 @@ func addNodeController(mgr manager.Manager, opt *Options) error { return fmt.Errorf("identifier for cloud %q not implemented", opt.Cloud) } - if opt.ConfigPath == "" { - return fmt.Errorf("must specify config-path") + if opt.ConfigBase == "" { + return fmt.Errorf("must specify configBase") } - nodeController, err := controllers.NewNodeReconciler(mgr, opt.ConfigPath, identifier) + nodeController, err := controllers.NewNodeReconciler(mgr, opt.ConfigBase, identifier) if err != nil { return err } diff --git a/cmd/kops-controller/pkg/config/BUILD.bazel b/cmd/kops-controller/pkg/config/BUILD.bazel new file mode 100644 index 0000000000..093d968260 --- /dev/null +++ b/cmd/kops-controller/pkg/config/BUILD.bazel @@ -0,0 +1,8 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["options.go"], + importpath = "k8s.io/kops/cmd/kops-controller/pkg/config", + visibility = ["//visibility:public"], +) diff --git a/cmd/kops-controller/pkg/config/options.go b/cmd/kops-controller/pkg/config/options.go new file mode 100644 index 0000000000..22a15914b1 --- /dev/null +++ b/cmd/kops-controller/pkg/config/options.go @@ -0,0 +1,25 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +type Options struct { + Cloud string `json:"cloud,omitempty"` + ConfigBase string `json:"configBase,omitempty"` +} + +func (o *Options) PopulateDefaults() { +} diff --git a/go.mod b/go.mod index 3c8190b492..61a723a7e7 100644 --- a/go.mod +++ b/go.mod @@ -136,4 +136,5 @@ require ( k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5 sigs.k8s.io/controller-runtime v0.2.2 sigs.k8s.io/controller-tools v0.2.2-0.20190919191502-76a25b63325a + sigs.k8s.io/yaml v1.1.0 ) diff --git a/hack/.packages b/hack/.packages index 71b8af917b..5ae4c8e9da 100644 --- a/hack/.packages +++ b/hack/.packages @@ -13,6 +13,7 @@ k8s.io/kops/cmd/kops k8s.io/kops/cmd/kops/util k8s.io/kops/cmd/kops-controller k8s.io/kops/cmd/kops-controller/controllers +k8s.io/kops/cmd/kops-controller/pkg/config k8s.io/kops/cmd/nodeup k8s.io/kops/dns-controller/cmd/dns-controller k8s.io/kops/dns-controller/pkg/dns diff --git a/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template index e3ce27b02b..12f16605dc 100644 --- a/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template +++ b/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template @@ -1,3 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kops-controller + namespace: kube-system + labels: + k8s-addon: kops-controller.addons.k8s.io +data: + config.yaml: | + {{ KopsControllerConfig }} + +--- + kind: Deployment apiVersion: apps/v1 metadata: @@ -32,12 +45,14 @@ spec: containers: - name: kops-controller image: kope/kops-controller:1.15.0-alpha.1 -{{ if .UseHostCertificates }} volumeMounts: +{{ if .UseHostCertificates }} - mountPath: /etc/ssl/certs name: etc-ssl-certs readOnly: true {{ end }} + - mountPath: /etc/kubernetes/kops-controller/ + name: kops-controller-config command: {{ range $arg := KopsControllerArgv }} - "{{ $arg }}" @@ -53,13 +68,16 @@ spec: requests: cpu: 50m memory: 50Mi -{{ if .UseHostCertificates }} volumes: +{{ if .UseHostCertificates }} - hostPath: path: /etc/ssl/certs type: DirectoryOrCreate name: etc-ssl-certs {{ end }} + - name: kops-controller-config + configMap: + name: kops-controller --- diff --git a/upup/pkg/fi/cloudup/BUILD.bazel b/upup/pkg/fi/cloudup/BUILD.bazel index 388297bad1..817ef3528a 100644 --- a/upup/pkg/fi/cloudup/BUILD.bazel +++ b/upup/pkg/fi/cloudup/BUILD.bazel @@ -25,6 +25,7 @@ go_library( deps = [ "//:go_default_library", "//channels/pkg/api:go_default_library", + "//cmd/kops-controller/pkg/config:go_default_library", "//dns-controller/pkg/dns:go_default_library", "//dnsprovider/pkg/dnsprovider:go_default_library", "//dnsprovider/pkg/dnsprovider/providers/aws/route53:go_default_library", diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index 67908a053c..2e3856245a 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -64,10 +64,12 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error { return fmt.Errorf("error reading manifest %s: %v", manifestPath, err) } - manifestBytes, err = b.assetBuilder.RemapManifest(manifestBytes) + remapped, err := b.assetBuilder.RemapManifest(manifestBytes) if err != nil { + klog.Infof("invalid manifest: %s", string(manifestBytes)) return fmt.Errorf("error remapping manifest %s: %v", manifestPath, err) } + manifestBytes = remapped // Trim whitespace manifestBytes = []byte(strings.TrimSpace(string(manifestBytes))) diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 25e431011d..68fc88ce6a 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -36,6 +36,9 @@ import ( "text/template" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog" + kopscontrollerconfig "k8s.io/kops/cmd/kops-controller/pkg/config" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/dns" "k8s.io/kops/pkg/featureflag" @@ -44,9 +47,6 @@ import ( "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/gce" "k8s.io/kops/util/pkg/env" - - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/klog" ) // TemplateFunctions provides a collection of methods used throughout the templates @@ -92,6 +92,7 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS } dest["KopsControllerArgv"] = tf.KopsControllerArgv + dest["KopsControllerConfig"] = tf.KopsControllerConfig dest["DnsControllerArgv"] = tf.DnsControllerArgv dest["ExternalDnsArgv"] = tf.ExternalDnsArgv @@ -295,21 +296,34 @@ func (tf *TemplateFunctions) DnsControllerArgv() ([]string, error) { return argv, nil } +// KopsControllerConfig returns the yaml configuration for kops-controller +func (tf *TemplateFunctions) KopsControllerConfig() (string, error) { + config := &kopscontrollerconfig.Options{ + Cloud: tf.cluster.Spec.CloudProvider, + ConfigBase: tf.cluster.Spec.ConfigBase, + } + + // To avoid indentation problems, we marshal as json. json is a subset of yaml + b, err := json.Marshal(config) + if err != nil { + return "", fmt.Errorf("failed to serialize kops-controller config: %v", err) + } + + return string(b), nil +} + // KopsControllerArgv returns the args to kops-controller func (tf *TemplateFunctions) KopsControllerArgv() ([]string, error) { + var argv []string argv = append(argv, "/usr/bin/kops-controller") - argv = append(argv, "--cloud="+tf.cluster.Spec.CloudProvider) - argv = append(argv, "--config="+tf.cluster.Spec.ConfigBase) - - // Disable metrics (avoid port conflicts, also risky because we are host network) - argv = append(argv, "--metrics-addr=0") - - // Verbose, but not crazy logging + // Verbose, but not excessive logging argv = append(argv, "--v=2") + argv = append(argv, "--conf=/etc/kubernetes/kops-controller/config.yaml") + return argv, nil } diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml index 682622e7d8..cd2c5bb038 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml @@ -1,3 +1,16 @@ +apiVersion: v1 +data: + config.yaml: | + {"cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com"} +kind: ConfigMap +metadata: + labels: + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system + +--- + apiVersion: apps/v1 kind: Deployment metadata: @@ -24,16 +37,17 @@ spec: containers: - command: - /usr/bin/kops-controller - - --cloud=aws - - --config=memfs://clusters.example.com/minimal.example.com - - --metrics-addr=0 - --v=2 + - --conf=/etc/kubernetes/kops-controller/config.yaml image: kope/kops-controller:1.15.0-alpha.1 name: kops-controller resources: requests: cpu: 50m memory: 50Mi + volumeMounts: + - mountPath: /etc/kubernetes/kops-controller/ + name: kops-controller-config dnsPolicy: Default hostNetwork: true nodeSelector: @@ -42,6 +56,10 @@ spec: tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master + volumes: + - configMap: + name: kops-controller + name: kops-controller-config --- diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml index 4649fa7ed0..a563c254b7 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml @@ -7,7 +7,7 @@ spec: - id: k8s-1.16 kubernetesVersion: '>=1.16.0-alpha.0' manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 736a3efe35f5edf14a8b7bd6ad723935e12f2a4d + manifestHash: 2e308b2c4d21ed023e2418068669afd4a5eb0592 name: kops-controller.addons.k8s.io selector: k8s-addon: kops-controller.addons.k8s.io diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml index 682622e7d8..cd2c5bb038 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml @@ -1,3 +1,16 @@ +apiVersion: v1 +data: + config.yaml: | + {"cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com"} +kind: ConfigMap +metadata: + labels: + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system + +--- + apiVersion: apps/v1 kind: Deployment metadata: @@ -24,16 +37,17 @@ spec: containers: - command: - /usr/bin/kops-controller - - --cloud=aws - - --config=memfs://clusters.example.com/minimal.example.com - - --metrics-addr=0 - --v=2 + - --conf=/etc/kubernetes/kops-controller/config.yaml image: kope/kops-controller:1.15.0-alpha.1 name: kops-controller resources: requests: cpu: 50m memory: 50Mi + volumeMounts: + - mountPath: /etc/kubernetes/kops-controller/ + name: kops-controller-config dnsPolicy: Default hostNetwork: true nodeSelector: @@ -42,6 +56,10 @@ spec: tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master + volumes: + - configMap: + name: kops-controller + name: kops-controller-config --- diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml index ca5a998424..bb0aeb103e 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml @@ -7,7 +7,7 @@ spec: - id: k8s-1.16 kubernetesVersion: '>=1.16.0-alpha.0' manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 736a3efe35f5edf14a8b7bd6ad723935e12f2a4d + manifestHash: 2e308b2c4d21ed023e2418068669afd4a5eb0592 name: kops-controller.addons.k8s.io selector: k8s-addon: kops-controller.addons.k8s.io diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml index 462ab1cc96..235b12bc20 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml @@ -7,7 +7,7 @@ spec: - id: k8s-1.16 kubernetesVersion: '>=1.16.0-alpha.0' manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: 736a3efe35f5edf14a8b7bd6ad723935e12f2a4d + manifestHash: 2e308b2c4d21ed023e2418068669afd4a5eb0592 name: kops-controller.addons.k8s.io selector: k8s-addon: kops-controller.addons.k8s.io