kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Move kops-controller to use a yaml configuration file 

This is more in keeping with componentconfig, and lets us add more
options easily in future.
This commit is contained in:
Justin SB 2019-10-06 17:11:54 -04:00
parent 0c66a304c3
commit d3bad10aa9
No known key found for this signature in database
GPG Key ID: 8DEC5C8217494E37
15 changed files with 168 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/kops-controller/BUILD.bazel
View File

@ -7,6 +7,7 @@ go_library(
visibility = ["//visibility:private"],
deps = [
"//cmd/kops-controller/controllers:go_default_library",
"//cmd/kops-controller/pkg/config:go_default_library",
"//pkg/nodeidentity:go_default_library",
"//pkg/nodeidentity/aws:go_default_library",
"//pkg/nodeidentity/gce:go_default_library",
@ -18,6 +19,7 @@ go_library(
"//vendor/k8s.io/klog/klogr:go_default_library",
"//vendor/sigs.k8s.io/controller-runtime:go_default_library",
"//vendor/sigs.k8s.io/controller-runtime/pkg/manager:go_default_library",
"//vendor/sigs.k8s.io/yaml:go_default_library",
],
)

62
cmd/kops-controller/main.go
View File

@ -19,24 +19,24 @@ package main
import (
"flag"
"fmt"
"io/ioutil"
"os"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
"k8s.io/klog"
"k8s.io/kops/cmd/kops-controller/controllers"
ctrl "sigs.k8s.io/controller-runtime"
// +kubebuilder:scaffold:imports
"k8s.io/klog/klogr"
"k8s.io/kops/cmd/kops-controller/controllers"
"k8s.io/kops/cmd/kops-controller/pkg/config"
"k8s.io/kops/pkg/nodeidentity"
nodeidentityaws "k8s.io/kops/pkg/nodeidentity/aws"
nodeidentitygce "k8s.io/kops/pkg/nodeidentity/gce"
nodeidentityos "k8s.io/kops/pkg/nodeidentity/openstack"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/yaml"
// +kubebuilder:scaffold:imports
)
var (
@ -49,26 +49,40 @@ func init() {
// +kubebuilder:scaffold:scheme
}
type Options struct {
Cloud string
ConfigPath string
}
func main() {
klog.InitFlags(nil)
var metricsAddr string
var enableLeaderElection bool
flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
// Disable metrics by default (avoid port conflicts, also risky because we are host network)
metricsAddress := ":0"
//flag.StringVar(&metricsAddr, "metrics-addr", metricsAddress, "The address the metric endpoint binds to.")
// Enable leader election
enableLeaderElection := false
//flag.BoolVar(&enableLeaderElection, "enable-leader-election", enableLeaderElection,
// "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
var opt Options
flag.StringVar(&opt.Cloud, "cloud", opt.Cloud, "CloudProvider we are using (aws,gce,...)")
flag.StringVar(&opt.ConfigPath, "config", opt.ConfigPath, "Base location for cluster and instancegroup configurations")
configPath := "/etc/kubernetes/kops-controller/config.yaml"
flag.StringVar(&configPath, "conf", configPath, "Location of yaml configuration file")
flag.Parse()
if configPath == "" {
klog.Fatalf("must specify --conf")
}
var opt config.Options
opt.PopulateDefaults()
{
b, err := ioutil.ReadFile(configPath)
if err != nil {
klog.Fatalf("failed to read configuration file %q: %v", configPath, err)
}
if err := yaml.Unmarshal(b, &opt); err != nil {
klog.Fatalf("failed to parse configuration file %q: %v", configPath, err)
}
}
ctrl.SetLogger(klogr.New())
if err := buildScheme(); err != nil {
@ -78,7 +92,7 @@ func main() {
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
Scheme: scheme,
MetricsBindAddress: metricsAddr,
MetricsBindAddress: metricsAddress,
LeaderElection: enableLeaderElection,
})
if err != nil {
@ -106,7 +120,7 @@ func buildScheme() error {
return nil
}
func addNodeController(mgr manager.Manager, opt *Options) error {
func addNodeController(mgr manager.Manager, opt *config.Options) error {
var identifier nodeidentity.Identifier
var err error
switch opt.Cloud {
@ -134,11 +148,11 @@ func addNodeController(mgr manager.Manager, opt *Options) error {
return fmt.Errorf("identifier for cloud %q not implemented", opt.Cloud)
}
if opt.ConfigPath == "" {
return fmt.Errorf("must specify config-path")
if opt.ConfigBase == "" {
return fmt.Errorf("must specify configBase")
}
nodeController, err := controllers.NewNodeReconciler(mgr, opt.ConfigPath, identifier)
nodeController, err := controllers.NewNodeReconciler(mgr, opt.ConfigBase, identifier)
if err != nil {
return err
}

8
cmd/kops-controller/pkg/config/BUILD.bazel Normal file
View File

@ -0,0 +1,8 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = ["options.go"],
importpath = "k8s.io/kops/cmd/kops-controller/pkg/config",
visibility = ["//visibility:public"],
)

25
cmd/kops-controller/pkg/config/options.go Normal file
View File

@ -0,0 +1,25 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config
type Options struct {
Cloud string `json:"cloud,omitempty"`
ConfigBase string `json:"configBase,omitempty"`
}
func (o *Options) PopulateDefaults() {
}

1
go.mod
View File

@ -136,4 +136,5 @@ require (
k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5
sigs.k8s.io/controller-runtime v0.2.2
sigs.k8s.io/controller-tools v0.2.2-0.20190919191502-76a25b63325a
sigs.k8s.io/yaml v1.1.0
)

1
hack/.packages
View File

@ -13,6 +13,7 @@ k8s.io/kops/cmd/kops
k8s.io/kops/cmd/kops/util
k8s.io/kops/cmd/kops-controller
k8s.io/kops/cmd/kops-controller/controllers
k8s.io/kops/cmd/kops-controller/pkg/config
k8s.io/kops/cmd/nodeup
k8s.io/kops/dns-controller/cmd/dns-controller
k8s.io/kops/dns-controller/pkg/dns

22
upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template
View File

@ -1,3 +1,16 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: kops-controller
namespace: kube-system
labels:
k8s-addon: kops-controller.addons.k8s.io
data:
config.yaml: |
{{ KopsControllerConfig }}
---
kind: Deployment
apiVersion: apps/v1
metadata:
@ -32,12 +45,14 @@ spec:
containers:
- name: kops-controller
image: kope/kops-controller:1.15.0-alpha.1
{{ if .UseHostCertificates }}
volumeMounts:
{{ if .UseHostCertificates }}
- mountPath: /etc/ssl/certs
name: etc-ssl-certs
readOnly: true
{{ end }}
- mountPath: /etc/kubernetes/kops-controller/
name: kops-controller-config
command:
{{ range $arg := KopsControllerArgv }}
- "{{ $arg }}"
@ -53,13 +68,16 @@ spec:
requests:
cpu: 50m
memory: 50Mi
{{ if .UseHostCertificates }}
volumes:
{{ if .UseHostCertificates }}
- hostPath:
path: /etc/ssl/certs
type: DirectoryOrCreate
name: etc-ssl-certs
{{ end }}
- name: kops-controller-config
configMap:
name: kops-controller
---

1
upup/pkg/fi/cloudup/BUILD.bazel
View File

@ -25,6 +25,7 @@ go_library(
deps = [
"//:go_default_library",
"//channels/pkg/api:go_default_library",
"//cmd/kops-controller/pkg/config:go_default_library",
"//dns-controller/pkg/dns:go_default_library",
"//dnsprovider/pkg/dnsprovider:go_default_library",
"//dnsprovider/pkg/dnsprovider/providers/aws/route53:go_default_library",

4
upup/pkg/fi/cloudup/bootstrapchannelbuilder.go
View File

@ -64,10 +64,12 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error {
return fmt.Errorf("error reading manifest %s: %v", manifestPath, err)
}
manifestBytes, err = b.assetBuilder.RemapManifest(manifestBytes)
remapped, err := b.assetBuilder.RemapManifest(manifestBytes)
if err != nil {
klog.Infof("invalid manifest: %s", string(manifestBytes))
return fmt.Errorf("error remapping manifest %s: %v", manifestPath, err)
}
manifestBytes = remapped
// Trim whitespace
manifestBytes = []byte(strings.TrimSpace(string(manifestBytes)))

34
upup/pkg/fi/cloudup/template_functions.go
View File

@ -36,6 +36,9 @@ import (
"text/template"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog"
kopscontrollerconfig "k8s.io/kops/cmd/kops-controller/pkg/config"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/dns"
"k8s.io/kops/pkg/featureflag"
@ -44,9 +47,6 @@ import (
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/gce"
"k8s.io/kops/util/pkg/env"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog"
)
// TemplateFunctions provides a collection of methods used throughout the templates
@ -92,6 +92,7 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS
}
dest["KopsControllerArgv"] = tf.KopsControllerArgv
dest["KopsControllerConfig"] = tf.KopsControllerConfig
dest["DnsControllerArgv"] = tf.DnsControllerArgv
dest["ExternalDnsArgv"] = tf.ExternalDnsArgv
@ -295,21 +296,34 @@ func (tf *TemplateFunctions) DnsControllerArgv() ([]string, error) {
return argv, nil
}
// KopsControllerConfig returns the yaml configuration for kops-controller
func (tf *TemplateFunctions) KopsControllerConfig() (string, error) {
config := &kopscontrollerconfig.Options{
Cloud: tf.cluster.Spec.CloudProvider,
ConfigBase: tf.cluster.Spec.ConfigBase,
}
// To avoid indentation problems, we marshal as json. json is a subset of yaml
b, err := json.Marshal(config)
if err != nil {
return "", fmt.Errorf("failed to serialize kops-controller config: %v", err)
}
return string(b), nil
}
// KopsControllerArgv returns the args to kops-controller
func (tf *TemplateFunctions) KopsControllerArgv() ([]string, error) {
var argv []string
argv = append(argv, "/usr/bin/kops-controller")
argv = append(argv, "--cloud="+tf.cluster.Spec.CloudProvider)
argv = append(argv, "--config="+tf.cluster.Spec.ConfigBase)
// Disable metrics (avoid port conflicts, also risky because we are host network)
argv = append(argv, "--metrics-addr=0")
// Verbose, but not crazy logging
// Verbose, but not excessive logging
argv = append(argv, "--v=2")
argv = append(argv, "--conf=/etc/kubernetes/kops-controller/config.yaml")
return argv, nil
}

24
upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml
View File

@ -1,3 +1,16 @@
apiVersion: v1
data:
config.yaml: |
{"cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com"}
kind: ConfigMap
metadata:
labels:
k8s-addon: kops-controller.addons.k8s.io
name: kops-controller
namespace: kube-system
---
apiVersion: apps/v1
kind: Deployment
metadata:
@ -24,16 +37,17 @@ spec:
containers:
- command:
- /usr/bin/kops-controller
- --cloud=aws
- --config=memfs://clusters.example.com/minimal.example.com
- --metrics-addr=0
- --v=2
- --conf=/etc/kubernetes/kops-controller/config.yaml
image: kope/kops-controller:1.15.0-alpha.1
name: kops-controller
resources:
requests:
cpu: 50m
memory: 50Mi
volumeMounts:
- mountPath: /etc/kubernetes/kops-controller/
name: kops-controller-config
dnsPolicy: Default
hostNetwork: true
nodeSelector:
@ -42,6 +56,10 @@ spec:
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
volumes:
- configMap:
name: kops-controller
name: kops-controller-config
---

2
upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml
View File

@ -7,7 +7,7 @@ spec:
- id: k8s-1.16
kubernetesVersion: '>=1.16.0-alpha.0'
manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
manifestHash: 736a3efe35f5edf14a8b7bd6ad723935e12f2a4d
manifestHash: 2e308b2c4d21ed023e2418068669afd4a5eb0592
name: kops-controller.addons.k8s.io
selector:
k8s-addon: kops-controller.addons.k8s.io

24
upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml
View File

@ -1,3 +1,16 @@
apiVersion: v1
data:
config.yaml: |
{"cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com"}
kind: ConfigMap
metadata:
labels:
k8s-addon: kops-controller.addons.k8s.io
name: kops-controller
namespace: kube-system
---
apiVersion: apps/v1
kind: Deployment
metadata:
@ -24,16 +37,17 @@ spec:
containers:
- command:
- /usr/bin/kops-controller
- --cloud=aws
- --config=memfs://clusters.example.com/minimal.example.com
- --metrics-addr=0
- --v=2
- --conf=/etc/kubernetes/kops-controller/config.yaml
image: kope/kops-controller:1.15.0-alpha.1
name: kops-controller
resources:
requests:
cpu: 50m
memory: 50Mi
volumeMounts:
- mountPath: /etc/kubernetes/kops-controller/
name: kops-controller-config
dnsPolicy: Default
hostNetwork: true
nodeSelector:
@ -42,6 +56,10 @@ spec:
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
volumes:
- configMap:
name: kops-controller
name: kops-controller-config
---

2
upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml
View File

@ -7,7 +7,7 @@ spec:
- id: k8s-1.16
kubernetesVersion: '>=1.16.0-alpha.0'
manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
manifestHash: 736a3efe35f5edf14a8b7bd6ad723935e12f2a4d
manifestHash: 2e308b2c4d21ed023e2418068669afd4a5eb0592
name: kops-controller.addons.k8s.io
selector:
k8s-addon: kops-controller.addons.k8s.io

2
upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml
View File

@ -7,7 +7,7 @@ spec:
- id: k8s-1.16
kubernetesVersion: '>=1.16.0-alpha.0'
manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
manifestHash: 736a3efe35f5edf14a8b7bd6ad723935e12f2a4d
manifestHash: 2e308b2c4d21ed023e2418068669afd4a5eb0592
name: kops-controller.addons.k8s.io
selector:
k8s-addon: kops-controller.addons.k8s.io