diff --git a/nodeup/pkg/model/kubeapiserver.go b/nodeup/pkg/model/kubeapiserver.go index 33e7afa350..3b97630c52 100644 --- a/nodeup/pkg/model/kubeapiserver.go +++ b/nodeup/pkg/model/kubeapiserver.go @@ -106,6 +106,18 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { }, } + probeAction := &v1.HTTPGetAction{ + Host: "127.0.0.1", + Path: "/healthz", + Port: intstr.FromInt(8080), + } + if kubeAPIServer.InsecurePort != 0 { + probeAction.Port = intstr.FromInt(int(kubeAPIServer.InsecurePort)) + } else if kubeAPIServer.SecurePort != 0 { + probeAction.Port = intstr.FromInt(int(kubeAPIServer.SecurePort)) + probeAction.Scheme = v1.URISchemeHTTPS + } + container := &v1.Container{ Name: "kube-apiserver", Image: b.Cluster.Spec.KubeAPIServer.Image, @@ -117,11 +129,7 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { Command: redirectCommand, LivenessProbe: &v1.Probe{ Handler: v1.Handler{ - HTTPGet: &v1.HTTPGetAction{ - Host: "127.0.0.1", - Path: "/healthz", - Port: intstr.FromInt(8080), - }, + HTTPGet: probeAction, }, InitialDelaySeconds: 15, TimeoutSeconds: 15, diff --git a/nodeup/pkg/model/kubeapiserver_test.go b/nodeup/pkg/model/kubeapiserver_test.go new file mode 100644 index 0000000000..ee4a993216 --- /dev/null +++ b/nodeup/pkg/model/kubeapiserver_test.go @@ -0,0 +1,65 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/flagbuilder" + "testing" +) + +func Test_KubeAPIServer_BuildFlags(t *testing.T) { + grid := []struct { + config kops.KubeAPIServerConfig + expected string + }{ + { + kops.KubeAPIServerConfig{}, + "--insecure-port=0 --secure-port=0", + }, + { + kops.KubeAPIServerConfig{ + SecurePort: 443, + }, + "--insecure-port=0 --secure-port=443", + }, + { + kops.KubeAPIServerConfig{ + InsecurePort: 8080, + SecurePort: 443, + }, + "--insecure-port=8080 --secure-port=443", + }, + { + kops.KubeAPIServerConfig{ + InsecurePort: 8080, + }, + "--insecure-port=8080 --secure-port=0", + }, + } + + for _, g := range grid { + actual, err := flagbuilder.BuildFlags(&g.config) + if err != nil { + t.Errorf("error building flags for %v: %v", g.config, err) + continue + } + if actual != g.expected { + t.Errorf("flags did not match. actual=%q expected=%q", actual, g.expected) + } + } +} diff --git a/nodeup/pkg/model/kubectl.go b/nodeup/pkg/model/kubectl.go index 5a6e45ec53..b2ad252771 100644 --- a/nodeup/pkg/model/kubectl.go +++ b/nodeup/pkg/model/kubectl.go @@ -18,6 +18,7 @@ package model import ( "fmt" + "github.com/golang/glog" "k8s.io/kops/nodeup/pkg/distros" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" @@ -58,6 +59,37 @@ func (b *KubectlBuilder) Build(c *fi.ModelBuilderContext) error { c.AddTask(t) } + // Add kubeconfig + { + kubeconfig, err := b.buildPKIKubeconfig("kubecfg") + if err != nil { + return err + } + + t := &nodetasks.File{ + Path: "/var/lib/kubectl/kubeconfig", + Contents: fi.NewStringResource(kubeconfig), + Type: nodetasks.FileType_File, + Mode: s("0400"), + } + c.AddTask(t) + + switch b.Distribution { + case distros.DistributionJessie: + c.AddTask(&nodetasks.File{ + Path: "/home/admin/.kube/config", + Contents: fi.NewStringResource(kubeconfig), + Type: nodetasks.FileType_File, + Mode: s("0400"), + Owner: s("admin"), + Group: s("admin"), + }) + + default: + glog.Warningf("Unknown distro; won't write kubeconfig to homedir %s", b.Distribution) + } + } + return nil } diff --git a/nodeup/pkg/model/protokube.go b/nodeup/pkg/model/protokube.go index d71b5f2c5b..3b0c72574b 100644 --- a/nodeup/pkg/model/protokube.go +++ b/nodeup/pkg/model/protokube.go @@ -37,6 +37,20 @@ type ProtokubeBuilder struct { var _ fi.ModelBuilder = &ProtokubeBuilder{} func (b *ProtokubeBuilder) Build(c *fi.ModelBuilderContext) error { + if b.IsMaster { + kubeconfig, err := b.buildPKIKubeconfig("kops") + if err != nil { + return err + } + + c.AddTask(&nodetasks.File{ + Path: "/var/lib/kops/kubeconfig", + Contents: fi.NewStringResource(kubeconfig), + Type: nodetasks.FileType_File, + Mode: s("0400"), + }) + } + // TODO: Should we run _protokube on the nodes? service, err := b.buildSystemdService() if err != nil { @@ -59,9 +73,19 @@ func (b *ProtokubeBuilder) buildSystemdService() (*nodetasks.Service, error) { return nil, err } - protokubeCommand := "/usr/bin/docker run -v /:/rootfs/ -v /var/run/dbus:/var/run/dbus -v /run/systemd:/run/systemd --net=host --privileged " - protokubeCommand += b.ProtokubeImageName() + " /usr/bin/protokube " - protokubeCommand += protokubeFlagsArgs + dockerArgs := []string{ + "/usr/bin/docker", + "run", + "-v", "/:/rootfs/", + "-v", "/var/run/dbus:/var/run/dbus", + "-v", "/run/systemd:/run/systemd", + "--net=host", + "--privileged", + "--env", "KUBECONFIG=/rootfs/var/lib/kops/kubeconfig", + b.ProtokubeImageName(), + "/usr/bin/protokube", + } + protokubeCommand := strings.Join(dockerArgs, " ") + " " + protokubeFlagsArgs manifest := &systemd.Manifest{} manifest.Set("Unit", "Description", "Kubernetes Protokube Service") diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index 86ec1e6efa..228cb00ca4 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -388,10 +388,11 @@ type KubeAPIServerConfig struct { Image string `json:"image,omitempty"` - LogLevel int32 `json:"logLevel,omitempty" flag:"v"` + LogLevel int32 `json:"logLevel,omitempty" flag:"v" flag-empty:"0"` CloudProvider string `json:"cloudProvider,omitempty" flag:"cloud-provider"` SecurePort int32 `json:"securePort,omitempty" flag:"secure-port"` + InsecurePort int32 `json:"insecurePort,omitempty" flag:"insecure-port"` Address string `json:"address,omitempty" flag:"address"` EtcdServers []string `json:"etcdServers,omitempty" flag:"etcd-servers"` EtcdServersOverrides []string `json:"etcdServersOverrides,omitempty" flag:"etcd-servers-overrides"` diff --git a/pkg/apis/kops/v1alpha1/componentconfig.go b/pkg/apis/kops/v1alpha1/componentconfig.go index ccbaff0892..92ce9b084a 100644 --- a/pkg/apis/kops/v1alpha1/componentconfig.go +++ b/pkg/apis/kops/v1alpha1/componentconfig.go @@ -388,6 +388,7 @@ type KubeAPIServerConfig struct { CloudProvider string `json:"cloudProvider,omitempty" flag:"cloud-provider"` SecurePort int32 `json:"securePort,omitempty" flag:"secure-port"` + InsecurePort int32 `json:"insecurePort,omitempty" flag:"insecure-port"` Address string `json:"address,omitempty" flag:"address"` EtcdServers []string `json:"etcdServers,omitempty" flag:"etcd-servers"` EtcdServersOverrides []string `json:"etcdServersOverrides,omitempty" flag:"etcd-servers-overrides"` diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index a856a3fed6..cee3c6e6f8 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -1057,6 +1057,7 @@ func autoConvert_v1alpha1_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *Ku out.LogLevel = in.LogLevel out.CloudProvider = in.CloudProvider out.SecurePort = in.SecurePort + out.InsecurePort = in.InsecurePort out.Address = in.Address out.EtcdServers = in.EtcdServers out.EtcdServersOverrides = in.EtcdServersOverrides @@ -1100,6 +1101,7 @@ func autoConvert_kops_KubeAPIServerConfig_To_v1alpha1_KubeAPIServerConfig(in *ko out.LogLevel = in.LogLevel out.CloudProvider = in.CloudProvider out.SecurePort = in.SecurePort + out.InsecurePort = in.InsecurePort out.Address = in.Address out.EtcdServers = in.EtcdServers out.EtcdServersOverrides = in.EtcdServersOverrides diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go index 6eb1174645..b1bd7d6d2d 100644 --- a/pkg/apis/kops/v1alpha2/componentconfig.go +++ b/pkg/apis/kops/v1alpha2/componentconfig.go @@ -169,6 +169,7 @@ type KubeAPIServerConfig struct { CloudProvider string `json:"cloudProvider,omitempty" flag:"cloud-provider"` SecurePort int32 `json:"securePort,omitempty" flag:"secure-port"` + InsecurePort int32 `json:"insecurePort,omitempty" flag:"insecure-port"` Address string `json:"address,omitempty" flag:"address"` EtcdServers []string `json:"etcdServers,omitempty" flag:"etcd-servers"` EtcdServersOverrides []string `json:"etcdServersOverrides,omitempty" flag:"etcd-servers-overrides"` diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 3266cd3f73..1d293b0260 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -1155,6 +1155,7 @@ func autoConvert_v1alpha2_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *Ku out.LogLevel = in.LogLevel out.CloudProvider = in.CloudProvider out.SecurePort = in.SecurePort + out.InsecurePort = in.InsecurePort out.Address = in.Address out.EtcdServers = in.EtcdServers out.EtcdServersOverrides = in.EtcdServersOverrides @@ -1198,6 +1199,7 @@ func autoConvert_kops_KubeAPIServerConfig_To_v1alpha2_KubeAPIServerConfig(in *ko out.LogLevel = in.LogLevel out.CloudProvider = in.CloudProvider out.SecurePort = in.SecurePort + out.InsecurePort = in.InsecurePort out.Address = in.Address out.EtcdServers = in.EtcdServers out.EtcdServersOverrides = in.EtcdServersOverrides diff --git a/pkg/model/components/apiserver.go b/pkg/model/components/apiserver.go index 5d1d16d2a5..325ff63310 100644 --- a/pkg/model/components/apiserver.go +++ b/pkg/model/components/apiserver.go @@ -18,7 +18,6 @@ package components import ( "fmt" - "github.com/blang/semver" "k8s.io/client-go/pkg/api/v1" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi" @@ -27,7 +26,7 @@ import ( // KubeAPIServerOptionsBuilder adds options for the apiserver to the model type KubeAPIServerOptionsBuilder struct { - Context *OptionsContext + *OptionsContext } var _ loader.OptionsBuilder = &KubeAPIServerOptionsBuilder{} @@ -38,27 +37,23 @@ func (b *KubeAPIServerOptionsBuilder) BuildOptions(o interface{}) error { clusterSpec.KubeAPIServer = &kops.KubeAPIServerConfig{} } - // TODO: Set insecure-port=0 to turn it off + c := clusterSpec.KubeAPIServer - if clusterSpec.KubeAPIServer.APIServerCount == nil { + if c.APIServerCount == nil { count := b.buildAPIServerCount(clusterSpec) if count == 0 { return fmt.Errorf("no instance groups found") } - clusterSpec.KubeAPIServer.APIServerCount = fi.Int32(int32(count)) + c.APIServerCount = fi.Int32(int32(count)) } - if clusterSpec.KubeAPIServer.StorageBackend == nil { + if c.StorageBackend == nil { // For the moment, we continue to use etcd2 - clusterSpec.KubeAPIServer.StorageBackend = fi.String("etcd2") + c.StorageBackend = fi.String("etcd2") } - k8sVersion, err := KubernetesVersion(clusterSpec) - if err != nil { - return err - } - if clusterSpec.KubeAPIServer.KubeletPreferredAddressTypes == nil { - if k8sVersion.GTE(semver.MustParse("1.5.0")) { + if c.KubeletPreferredAddressTypes == nil { + if b.IsKubernetesGTE("1.5") { // Default precedence //options.KubeAPIServer.KubeletPreferredAddressTypes = []string { // string(api.NodeHostName), @@ -68,7 +63,7 @@ func (b *KubeAPIServerOptionsBuilder) BuildOptions(o interface{}) error { //} // We prioritize the internal IP above the hostname - clusterSpec.KubeAPIServer.KubeletPreferredAddressTypes = []string{ + c.KubeletPreferredAddressTypes = []string{ string(v1.NodeInternalIP), string(v1.NodeHostName), string(v1.NodeExternalIP), @@ -83,6 +78,15 @@ func (b *KubeAPIServerOptionsBuilder) BuildOptions(o interface{}) error { } } + c.SecurePort = 443 + + // We disable the insecure port from 1.6 onwards + if b.IsKubernetesGTE("1.6") { + c.InsecurePort = 0 + } else { + c.InsecurePort = 8080 + } + return nil } diff --git a/pkg/model/components/kubecontrollermanager.go b/pkg/model/components/kubecontrollermanager.go index 085c50bd48..7c3a563796 100644 --- a/pkg/model/components/kubecontrollermanager.go +++ b/pkg/model/components/kubecontrollermanager.go @@ -108,7 +108,13 @@ func (b *KubeControllerManagerOptionsBuilder) BuildOptions(o interface{}) error return fmt.Errorf("unknown cloud provider %q", clusterSpec.CloudProvider) } - kcm.Master = "127.0.0.1:8080" + if kcm.Master == "" { + if b.Context.IsKubernetesLT("1.6") { + // As of 1.6, we find the master using kubeconfig + kcm.Master = "127.0.0.1:8080" + } + } + kcm.LogLevel = 2 image, err := Image("kube-controller-manager", clusterSpec) diff --git a/pkg/model/pki.go b/pkg/model/pki.go index ba7ced6bec..e6a5778b9b 100644 --- a/pkg/model/pki.go +++ b/pkg/model/pki.go @@ -80,6 +80,16 @@ func (b *PKIModelBuilder) Build(c *fi.ModelBuilderContext) error { c.AddTask(t) } + { + // Keypair used by kops / protokube + t := &fitasks.Keypair{ + Name: fi.String("kops"), + Subject: "o=" + user.SystemPrivilegedGroup + ",cn=kops", + Type: "client", + } + c.AddTask(t) + } + { // TLS certificate used for apiserver diff --git a/upup/pkg/fi/cloudup/populate_cluster_spec.go b/upup/pkg/fi/cloudup/populate_cluster_spec.go index e3eb6b5c3c..0836a4d1da 100644 --- a/upup/pkg/fi/cloudup/populate_cluster_spec.go +++ b/upup/pkg/fi/cloudup/populate_cluster_spec.go @@ -278,7 +278,7 @@ func (c *populateClusterSpec) run() error { // Note: DefaultOptionsBuilder comes first codeModels = append(codeModels, &components.DefaultsOptionsBuilder{Context: optionsContext}) - codeModels = append(codeModels, &components.KubeAPIServerOptionsBuilder{Context: optionsContext}) + codeModels = append(codeModels, &components.KubeAPIServerOptionsBuilder{OptionsContext: optionsContext}) codeModels = append(codeModels, &components.DockerOptionsBuilder{Context: optionsContext}) codeModels = append(codeModels, &components.NetworkingOptionsBuilder{Context: optionsContext}) codeModels = append(codeModels, &components.KubeDnsOptionsBuilder{Context: optionsContext}) diff --git a/upup/pkg/fi/nodeup/nodetasks/file.go b/upup/pkg/fi/nodeup/nodetasks/file.go index 00bd1feb87..7d86ace34f 100644 --- a/upup/pkg/fi/nodeup/nodetasks/file.go +++ b/upup/pkg/fi/nodeup/nodetasks/file.go @@ -82,9 +82,11 @@ func (f *File) GetDependencies(tasks map[string]fi.Task) []fi.Task { if f.Owner != nil { ownerTask := tasks["user/"+*f.Owner] if ownerTask == nil { - glog.Fatalf("Unable to find task %q", "user/"+*f.Owner) + // The user might be a pre-existing user (e.g. admin) + glog.Warningf("Unable to find task %q", "user/"+*f.Owner) + } else { + deps = append(deps, ownerTask) } - deps = append(deps, ownerTask) } // Depend on disk mounts diff --git a/upup/pkg/fi/nodeup/template_functions.go b/upup/pkg/fi/nodeup/template_functions.go index a0e4d709e8..78fb4f3168 100644 --- a/upup/pkg/fi/nodeup/template_functions.go +++ b/upup/pkg/fi/nodeup/template_functions.go @@ -115,7 +115,6 @@ func (t *templateFunctions) populate(dest template.FuncMap) { dest["KubeControllerManager"] = func() *api.KubeControllerManagerConfig { return t.cluster.Spec.KubeControllerManager } - dest["KubeProxy"] = t.KubeProxyConfig dest["ClusterName"] = func() string { return t.cluster.ObjectMeta.Name @@ -159,19 +158,3 @@ func (t *templateFunctions) hasTag(tag string) bool { _, found := t.tags[tag] return found } - -// KubeProxyConfig builds the KubeProxyConfig configuration object -func (t *templateFunctions) KubeProxyConfig() *api.KubeProxyConfig { - config := &api.KubeProxyConfig{} - *config = *t.cluster.Spec.KubeProxy - - // As a special case, if this is the master, we point kube-proxy to the local IP - // This prevents a circular dependency where kube-proxy can't come up until DNS comes up, - // which would mean that DNS can't rely on API to come up - if t.isMaster() { - glog.Infof("kube-proxy running on the master; setting API endpoint to localhost") - config.Master = "http://127.0.0.1:8080" - } - - return config -}