From 5b3524cb806a05e75e3fb837255c20254ac6ce59 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 4 Jan 2017 00:03:21 -0500 Subject: [PATCH 01/29] Set default sysctls --- nodeup/pkg/model/sysctls.go | 129 ++++++++++++++++++ .../files/etc/sysctl.d/99-k8s-networking.conf | 4 - .../etc/sysctl.d/99-k8s-networking.conf.meta | 3 - .../files/etc/sysctl.d/99-k8s-networking.conf | 2 - .../etc/sysctl.d/99-k8s-networking.conf.meta | 3 - upup/pkg/fi/nodeup/command.go | 1 + 6 files changed, 130 insertions(+), 12 deletions(-) create mode 100644 nodeup/pkg/model/sysctls.go delete mode 100644 upup/models/nodeup/networking/_aws/files/etc/sysctl.d/99-k8s-networking.conf delete mode 100644 upup/models/nodeup/networking/_aws/files/etc/sysctl.d/99-k8s-networking.conf.meta delete mode 100644 upup/models/nodeup/networking/_gce/files/etc/sysctl.d/99-k8s-networking.conf delete mode 100644 upup/models/nodeup/networking/_gce/files/etc/sysctl.d/99-k8s-networking.conf.meta diff --git a/nodeup/pkg/model/sysctls.go b/nodeup/pkg/model/sysctls.go new file mode 100644 index 0000000000..505aff8cdf --- /dev/null +++ b/nodeup/pkg/model/sysctls.go @@ -0,0 +1,129 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" + "strings" +) + +// SysctlBuilder set up our sysctls +type SysctlBuilder struct { + *NodeupModelContext +} + +var _ fi.ModelBuilder = &SysctlBuilder{} + +func (b *SysctlBuilder) Build(c *fi.ModelBuilderContext) error { + var sysctls []string + + // Common settings + { + sysctls = append(sysctls, + "# Kubernetes Settings", + "") + + // A higher vm.max_map_count is great for elasticsearch, mongo, or other mmap users + // See https://github.com/kubernetes/kops/issues/1340 + sysctls = append(sysctls, "vm.max_map_count = 262144", + "") + + // See https://github.com/kubernetes/kubernetes/pull/38001 + sysctls = append(sysctls, + "kernel.softlockup_panic = 1", + "kernel.softlockup_all_cpu_backtrace = 1", + "") + + // See https://github.com/kubernetes/kube-deploy/issues/261 + sysctls = append(sysctls, + "# Increase the number of connections", + "net.core.somaxconn = 32768", + "", + + "# Increase number of incoming connections backlog", + "net.core.netdev_max_backlog = 5000", + "", + + "# Maximum Socket Receive Buffer", + "net.core.rmem_max = 16777216", + "", + + "# Default Socket Send Buffer", + "net.core.wmem_max = 16777216", + "", + + "# Increase the maximum total buffer-space allocatable", + "net.ipv4.tcp_wmem = 4096 12582912 16777216", + "net.ipv4.tcp_rmem = 4096 12582912 16777216", + "", + + "# Increase the number of outstanding syn requests allowed", + "net.ipv4.tcp_max_syn_backlog = 8096", + "", + + "# For persistent HTTP connections", + "net.ipv4.tcp_slow_start_after_idle = 0", + "", + + "# Increase the tcp-time-wait buckets pool size to prevent simple DOS attacks", + "net.ipv4.tcp_tw_reuse = 1", + "", + + // We can't change the local_port_range without changing the NodePort range + //"# Allowed local port range", + //"net.ipv4.ip_local_port_range = 10240 65535", + //"", + + "# Max number of packets that can be queued on interface input", + "# If kernel is receiving packets faster than can be processed", + "# this queue increases", + "net.core.netdev_max_backlog = 16384", + "", + + "# Increase size of file handles and inode cache", + "fs.file-max = 2097152", + "", + ) + } + + if b.Cluster.Spec.CloudProvider == string(fi.CloudProviderAWS) { + sysctls = append(sysctls, + "# AWS settings", + "", + "# Issue #23395", + "net.ipv4.neigh.default.gc_thresh1=0") + } + + if b.Cluster.Spec.CloudProvider == string(fi.CloudProviderGCE) { + sysctls = append(sysctls, + "# GCE settings", + "", + "net.ipv4.ip_forward=1", + "") + } + + t := &nodetasks.File{ + Path: "/etc/sysctl.d/99-k8s-general.conf", + Contents: fi.NewStringResource(strings.Join(sysctls, "\n")), + Type: nodetasks.FileType_File, + OnChangeExecute: []string{"sysctl", "--system"}, + } + c.AddTask(t) + + return nil +} diff --git a/upup/models/nodeup/networking/_aws/files/etc/sysctl.d/99-k8s-networking.conf b/upup/models/nodeup/networking/_aws/files/etc/sysctl.d/99-k8s-networking.conf deleted file mode 100644 index e3be4b8dcd..0000000000 --- a/upup/models/nodeup/networking/_aws/files/etc/sysctl.d/99-k8s-networking.conf +++ /dev/null @@ -1,4 +0,0 @@ -# Kubernetes AWS settings - -# Issue #23395 -net.ipv4.neigh.default.gc_thresh1=0 \ No newline at end of file diff --git a/upup/models/nodeup/networking/_aws/files/etc/sysctl.d/99-k8s-networking.conf.meta b/upup/models/nodeup/networking/_aws/files/etc/sysctl.d/99-k8s-networking.conf.meta deleted file mode 100644 index 7301e8af13..0000000000 --- a/upup/models/nodeup/networking/_aws/files/etc/sysctl.d/99-k8s-networking.conf.meta +++ /dev/null @@ -1,3 +0,0 @@ -{ - "onChangeExecute": [ "sysctl", "--system" ] -} \ No newline at end of file diff --git a/upup/models/nodeup/networking/_gce/files/etc/sysctl.d/99-k8s-networking.conf b/upup/models/nodeup/networking/_gce/files/etc/sysctl.d/99-k8s-networking.conf deleted file mode 100644 index d801c31a48..0000000000 --- a/upup/models/nodeup/networking/_gce/files/etc/sysctl.d/99-k8s-networking.conf +++ /dev/null @@ -1,2 +0,0 @@ -# Kubernetes -net.ipv4.ip_forward=1 diff --git a/upup/models/nodeup/networking/_gce/files/etc/sysctl.d/99-k8s-networking.conf.meta b/upup/models/nodeup/networking/_gce/files/etc/sysctl.d/99-k8s-networking.conf.meta deleted file mode 100644 index 7301e8af13..0000000000 --- a/upup/models/nodeup/networking/_gce/files/etc/sysctl.d/99-k8s-networking.conf.meta +++ /dev/null @@ -1,3 +0,0 @@ -{ - "onChangeExecute": [ "sysctl", "--system" ] -} \ No newline at end of file diff --git a/upup/pkg/fi/nodeup/command.go b/upup/pkg/fi/nodeup/command.go index 0dbef7b905..3c679b73d1 100644 --- a/upup/pkg/fi/nodeup/command.go +++ b/upup/pkg/fi/nodeup/command.go @@ -196,6 +196,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error { loader := NewLoader(c.config, c.cluster, assets, tags) loader.Builders = append(loader.Builders, &model.DockerBuilder{NodeupModelContext: modelContext}) + loader.Builders = append(loader.Builders, &model.SysctlBuilder{NodeupModelContext: modelContext}) tf, err := newTemplateFunctions(c.config, c.cluster, c.instanceGroup, tags) if err != nil { return fmt.Errorf("error initializing: %v", err) From 6f2567d189a91fb23f4ea2e83da3529eecd59077 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sat, 7 Jan 2017 11:00:54 -0500 Subject: [PATCH 02/29] Recognize CI=1 to makefile, use git sha for VERSION This means that end-users can have an easy experience without needing to upload nodeup/protokube, but CI can still get the git-based unique version as long as it sets CI=1 --- Makefile | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 972d12ec1c..450d361ec8 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,23 @@ MAKEDIR:=$(strip $(shell dirname "$(realpath $(lastword $(MAKEFILE_LIST)))")) # Keep in sync with upup/models/cloudup/resources/addons/dns-controller/ DNS_CONTROLLER_TAG=1.4.1 -VERSION?=1.5.0-alpha1 +ifndef VERSION + # To keep both CI and end-users building from source happy, + # we expect that CI sets CI=1. + # + # For end users, they need only build kops, and they can use the last + # released version of nodeup/protokube. + # For CI, we continue to build a synthetic version from the git SHA, so + # we never cross versions. + # + # We expect that if you are uploading nodeup/protokube, you will set + # VERSION (along with S3_BUCKET), either directly or by setting CI=1 + ifndef CI + VERSION=1.5.0-alpha1 + else + VERSION := git-$(shell git describe --always) + endif +endif # Go exports: From 75b752fd9d85555d632304e1ed4f4ba93c33256e Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sat, 7 Jan 2017 16:25:18 -0500 Subject: [PATCH 03/29] When outputing to terraform, don't say the cluster is starting Suggest the user goes into the output directory and runs terraform plan & apply instead. Fix #1141 --- cmd/kops/update_cluster.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/cmd/kops/update_cluster.go b/cmd/kops/update_cluster.go index a22a2c7a1c..b9d4e2c9c9 100644 --- a/cmd/kops/update_cluster.go +++ b/cmd/kops/update_cluster.go @@ -203,9 +203,16 @@ func RunUpdateCluster(f *util.Factory, clusterName string, out io.Writer, c *Upd if !hasKubecfg { // Assume initial creation - fmt.Printf("\n") - fmt.Printf("Cluster is starting. It should be ready in a few minutes.\n") - fmt.Printf("\n") + if c.Target == cloudup.TargetTerraform { + fmt.Printf("\n") + fmt.Printf("Terraform output has been placed into %s\n", c.OutDir) + fmt.Printf("You likely want to run terraform plan and apply in that directory\n") + fmt.Printf("\n") + } else { + fmt.Printf("\n") + fmt.Printf("Cluster is starting. It should be ready in a few minutes.\n") + fmt.Printf("\n") + } fmt.Printf("Suggestions:\n") fmt.Printf(" * list nodes: kubectl get nodes --show-labels\n") if cluster.Spec.Topology.Masters == kops.TopologyPublic { From dc32042bfd1c5ae2013c8e273530ee145dedf451 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sun, 8 Jan 2017 09:30:12 -0500 Subject: [PATCH 04/29] Fixes per code review --- cmd/kops/update_cluster.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/kops/update_cluster.go b/cmd/kops/update_cluster.go index b9d4e2c9c9..5f329a5c5d 100644 --- a/cmd/kops/update_cluster.go +++ b/cmd/kops/update_cluster.go @@ -206,7 +206,10 @@ func RunUpdateCluster(f *util.Factory, clusterName string, out io.Writer, c *Upd if c.Target == cloudup.TargetTerraform { fmt.Printf("\n") fmt.Printf("Terraform output has been placed into %s\n", c.OutDir) - fmt.Printf("You likely want to run terraform plan and apply in that directory\n") + fmt.Printf("Run these commands to apply the configuration:\n") + fmt.Printf(" cd %s\n", c.OutDir) + fmt.Printf(" terraform plan\n") + fmt.Printf(" terraform apply\n") fmt.Printf("\n") } else { fmt.Printf("\n") From 43eb686f7c8828ca4676e82ef62d2a16e204d2f3 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sun, 8 Jan 2017 09:39:55 -0500 Subject: [PATCH 05/29] Add diagnostic message when updating DNS In debugging, it would have been useful to see the old & new values sometimes --- dns-controller/pkg/dns/dnscontroller.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dns-controller/pkg/dns/dnscontroller.go b/dns-controller/pkg/dns/dnscontroller.go index d4d6097c73..fd538efffc 100644 --- a/dns-controller/pkg/dns/dnscontroller.go +++ b/dns-controller/pkg/dns/dnscontroller.go @@ -257,6 +257,8 @@ func (c *DNSController) runOnce() error { ttl := DefaultTTL glog.Infof("Using default TTL of %v", ttl) + glog.V(4).Infof("updating records for %s: %v -> %v", k, newValues, oldValues) + err := op.updateRecords(k, newValues, int64(ttl.Seconds())) if err != nil { glog.Infof("error updating records for %s: %v", k, err) From e50783da8799d9a70e0e7c812f93d81666eed10d Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 27 Dec 2016 11:56:51 -0500 Subject: [PATCH 06/29] Store in v1alpha2 format --- pkg/client/simple/vfsclientset/commonvfs.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/client/simple/vfsclientset/commonvfs.go b/pkg/client/simple/vfsclientset/commonvfs.go index dcb7c6488a..5537f66028 100644 --- a/pkg/client/simple/vfsclientset/commonvfs.go +++ b/pkg/client/simple/vfsclientset/commonvfs.go @@ -21,7 +21,7 @@ import ( "fmt" "github.com/golang/glog" kops "k8s.io/kops/pkg/apis/kops" - "k8s.io/kops/pkg/apis/kops/v1alpha1" + "k8s.io/kops/pkg/apis/kops/v1alpha2" "k8s.io/kops/util/pkg/vfs" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/apis/meta/v1" @@ -33,7 +33,7 @@ import ( "time" ) -var StoreVersion = v1alpha1.SchemeGroupVersion +var StoreVersion = v1alpha2.SchemeGroupVersion type commonVFS struct { kind string From 0a56d3d2e1534879e502638627ad3628f9026b84 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sun, 8 Jan 2017 09:52:22 -0500 Subject: [PATCH 07/29] Raise an error on an invalid s3 path Fix #902 --- Makefile | 1 + util/pkg/vfs/context.go | 5 ++- util/pkg/vfs/s3fs.go | 2 +- util/pkg/vfs/s3fs_test.go | 66 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 util/pkg/vfs/s3fs_test.go diff --git a/Makefile b/Makefile index dcf23db14f..6c439eff76 100644 --- a/Makefile +++ b/Makefile @@ -80,6 +80,7 @@ test: go test k8s.io/kops/dns-controller/pkg/... -args -v=1 -logtostderr go test k8s.io/kops/cmd/... -args -v=1 -logtostderr go test k8s.io/kops/tests/... -args -v=1 -logtostderr + go test k8s.io/kops/util/... -args -v=1 -logtostderr crossbuild-nodeup: mkdir -p .build/dist/ diff --git a/util/pkg/vfs/context.go b/util/pkg/vfs/context.go index e50194635d..fcc84480e6 100644 --- a/util/pkg/vfs/context.go +++ b/util/pkg/vfs/context.go @@ -130,8 +130,11 @@ func (c *VFSContext) buildS3Path(p string) (*S3Path, error) { } bucket := strings.TrimSuffix(u.Host, "/") + if bucket == "" { + return nil, fmt.Errorf("invalid s3 path: %q", err) + } - s3path := NewS3Path(c.s3Context, bucket, u.Path) + s3path := newS3Path(c.s3Context, bucket, u.Path) return s3path, nil } diff --git a/util/pkg/vfs/s3fs.go b/util/pkg/vfs/s3fs.go index 2c7277b403..d57864f3c1 100644 --- a/util/pkg/vfs/s3fs.go +++ b/util/pkg/vfs/s3fs.go @@ -43,7 +43,7 @@ type S3Path struct { var _ Path = &S3Path{} var _ HasHash = &S3Path{} -func NewS3Path(s3Context *S3Context, bucket string, key string) *S3Path { +func newS3Path(s3Context *S3Context, bucket string, key string) *S3Path { bucket = strings.TrimSuffix(bucket, "/") key = strings.TrimPrefix(key, "/") diff --git a/util/pkg/vfs/s3fs_test.go b/util/pkg/vfs/s3fs_test.go new file mode 100644 index 0000000000..caf907ffc9 --- /dev/null +++ b/util/pkg/vfs/s3fs_test.go @@ -0,0 +1,66 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vfs + +import "testing" + +func Test_S3Path_Parse(t *testing.T) { + grid := []struct { + Input string + ExpectError bool + ExpectedBucket string + ExpectedPath string + }{ + { + Input: "s3://bucket", + ExpectedBucket: "bucket", + ExpectedPath: "", + }, + { + Input: "s3://bucket/path", + ExpectedBucket: "bucket", + ExpectedPath: "path", + }, + { + Input: "s3://bucket2/path/subpath", + ExpectedBucket: "bucket2", + ExpectedPath: "path/subpath", + }, + { + Input: "s3:///bucket/path/subpath", + ExpectError: true, + }, + } + for _, g := range grid { + s3path, err := Context.buildS3Path(g.Input) + if !g.ExpectError { + if err != nil { + t.Fatalf("unexpected error parsing s3 path: %v", err) + } + if s3path.bucket != g.ExpectedBucket { + t.Fatalf("unexpected s3 path: %v", s3path) + } + if s3path.key != g.ExpectedPath { + t.Fatalf("unexpected s3 path: %v", s3path) + } + } else { + if err == nil { + t.Fatalf("unexpected error parsing %q", g.Input) + } + } + } +} From d62a7e0fb04a22d7b9ec53a6fbaf7e7b7be6fb4e Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sun, 8 Jan 2017 14:57:37 -0500 Subject: [PATCH 08/29] De-emphasize cni for private topologies Fix #1373 --- cmd/kops/create_cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index cea358d62f..0e488635c3 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -455,7 +455,7 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e case api.TopologyPrivate: if !supportsPrivateTopology(cluster.Spec.Networking) { - return fmt.Errorf("Invalid networking option %s. Currently only '--networking cni', '--networking kopeio-vxlan', '--networking weave', '--networking calico' are supported for private topologies", c.Networking) + return fmt.Errorf("Invalid networking option %s. Currently only '--networking kopeio-vxlan', '--networking weave', '--networking calico' (or '--networking cni') are supported for private topologies", c.Networking) } cluster.Spec.Topology = &api.TopologySpec{ Masters: api.TopologyPrivate, From 09f77d6753b220c9e546f4d8c3a385e0c47c2d79 Mon Sep 17 00:00:00 2001 From: Kris Nova Date: Sun, 8 Jan 2017 15:11:08 -0500 Subject: [PATCH 09/29] Fixing hosted zone errors with bastion, and cleaning up dns model logic --- pkg/model/dns.go | 35 +++++++++++++++----- upup/pkg/fi/cloudup/populate_cluster_spec.go | 10 +----- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/pkg/model/dns.go b/pkg/model/dns.go index 340f56f721..8f3b45eb58 100644 --- a/pkg/model/dns.go +++ b/pkg/model/dns.go @@ -31,7 +31,7 @@ var _ fi.ModelBuilder = &DNSModelBuilder{} func (b *DNSModelBuilder) Build(c *fi.ModelBuilderContext) error { // Add a HostedZone if we are going to publish a dns record that depends on it - if b.UsePrivateDNS() || b.UsesBastionDns() { + if b.UsePrivateDNS() { // UsePrivateDNS is only exposed as a feature flag currently // TODO: We may still need a public zone to publish an ELB @@ -54,7 +54,9 @@ func (b *DNSModelBuilder) Build(c *fi.ModelBuilderContext) error { } c.AddTask(dnsZone) - } else if b.UseLoadBalancerForAPI() { + } + + if b.UseLoadBalancerForAPI() { // This will point our DNS to the load balancer, and put the pieces // together for kubectl to be work @@ -73,19 +75,34 @@ func (b *DNSModelBuilder) Build(c *fi.ModelBuilderContext) error { } c.AddTask(dnsZone) - } - if b.UseLoadBalancerForAPI() { - // This will point our DNS to the load balancer, and put the pieces - // together for kubectl to be work - - dnsName := &awstasks.DNSName{ + apiDnsName := &awstasks.DNSName{ Name: s(b.Cluster.Spec.MasterPublicName), Zone: &awstasks.DNSZone{Name: s(b.Cluster.Spec.DNSZone)}, ResourceType: s("A"), TargetLoadBalancer: b.LinkToELB("api"), } - c.AddTask(dnsName) + c.AddTask(apiDnsName) + } + + if b.UsesBastionDns() { + // Pulling this down into it's own if statement. The DNS configuration here + // is similar to others, but I would like to keep it on it's own in case we need + // to change anything. + dnsZone := &awstasks.DNSZone{ + Name: s(b.Cluster.Spec.DNSZone), + Private: fi.Bool(false), + } + + if !strings.Contains(b.Cluster.Spec.DNSZone, ".") { + // Looks like a hosted zone ID + dnsZone.ZoneID = s(b.Cluster.Spec.DNSZone) + } else { + // Looks like a normal ddns name + dnsZone.DNSName = s(b.Cluster.Spec.DNSZone) + } + + c.AddTask(dnsZone) } return nil diff --git a/upup/pkg/fi/cloudup/populate_cluster_spec.go b/upup/pkg/fi/cloudup/populate_cluster_spec.go index 603e2f18aa..bc129bfcac 100644 --- a/upup/pkg/fi/cloudup/populate_cluster_spec.go +++ b/upup/pkg/fi/cloudup/populate_cluster_spec.go @@ -223,14 +223,6 @@ func (c *populateClusterSpec) run() error { return err } - // Hard coding topology here - // - // We want topology to pass through - // Otherwise we were losing the pointer - // TODO: This should not be needed... - cluster.Spec.Topology = c.InputCluster.Spec.Topology - //cluster.Spec.Topology.Bastion = c.InputCluster.Spec.Topology.Bastion - if cluster.Spec.DNSZone == "" { dns, err := cloud.DNS() if err != nil { @@ -238,7 +230,7 @@ func (c *populateClusterSpec) run() error { } dnsZone, err := FindDNSHostedZone(dns, cluster.ObjectMeta.Name) if err != nil { - return fmt.Errorf("Error determining default DNS zone; please specify --dns-zone: %v", err) + return fmt.Errorf("error determining default DNS zone: %v", err) } glog.Infof("Defaulting DNS zone to: %s", dnsZone) cluster.Spec.DNSZone = dnsZone From c87a7adeb00ca4f2f448ec2b516e48f1e5f2459b Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sun, 8 Jan 2017 14:36:19 -0500 Subject: [PATCH 10/29] Fix DNS name precreation We were creating the wrong names for etcd. Also add a test. Fix #1393 --- upup/pkg/fi/cloudup/dns.go | 63 ++++++++++++++++++------------ upup/pkg/fi/cloudup/dns_test.go | 69 +++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 25 deletions(-) create mode 100644 upup/pkg/fi/cloudup/dns_test.go diff --git a/upup/pkg/fi/cloudup/dns.go b/upup/pkg/fi/cloudup/dns.go index fce53aebe7..5aa1a2e79c 100644 --- a/upup/pkg/fi/cloudup/dns.go +++ b/upup/pkg/fi/cloudup/dns.go @@ -121,38 +121,18 @@ func precreateDNS(cluster *api.Cluster, cloud fi.Cloud) error { return nil } - glog.Infof("Pre-creating DNS records") - // We precreate some DNS names (where they don't exist), with a dummy IP address // This avoids hitting negative TTL on DNS lookups, which tend to be very long // If we get the names wrong here, it doesn't really matter (extra DNS name, slower boot) - dnsSuffix := cluster.Spec.MasterPublicName - var dnsHostnames []string + dnsHostnames := buildPrecreateDNSHostnames(cluster) - if cluster.Spec.MasterPublicName != "" { - dnsHostnames = append(dnsHostnames, cluster.Spec.MasterPublicName) - } else { - glog.Warningf("cannot pre-create MasterPublicName - not set") + if len(dnsHostnames) == 0 { + glog.Infof("No DNS records to pre-create") + return nil } - if cluster.Spec.MasterInternalName != "" { - dnsHostnames = append(dnsHostnames, cluster.Spec.MasterInternalName) - } else { - glog.Warningf("cannot pre-create MasterInternalName - not set") - } - - for _, etcdCluster := range cluster.Spec.EtcdClusters { - etcClusterName := "etcd-" + etcdCluster.Name - if etcdCluster.Name == "main" { - // Special case - etcClusterName = "etcd" - } - for _, etcdClusterMember := range etcdCluster.Members { - name := etcClusterName + "-" + etcdClusterMember.Name + ".internal." + dnsSuffix - dnsHostnames = append(dnsHostnames, name) - } - } + glog.Infof("Pre-creating DNS records") zone, err := findZone(cluster, cloud) if err != nil { @@ -216,3 +196,36 @@ func precreateDNS(cluster *api.Cluster, cloud fi.Cloud) error { return nil } + +// buildPrecreateDNSHostnames returns the hostnames we should precreate +func buildPrecreateDNSHostnames(cluster *api.Cluster) []string { + dnsInternalSuffix := ".internal." + cluster.ObjectMeta.Name + + var dnsHostnames []string + + if cluster.Spec.MasterPublicName != "" { + dnsHostnames = append(dnsHostnames, cluster.Spec.MasterPublicName) + } else { + glog.Warningf("cannot pre-create MasterPublicName - not set") + } + + if cluster.Spec.MasterInternalName != "" { + dnsHostnames = append(dnsHostnames, cluster.Spec.MasterInternalName) + } else { + glog.Warningf("cannot pre-create MasterInternalName - not set") + } + + for _, etcdCluster := range cluster.Spec.EtcdClusters { + etcClusterName := "etcd-" + etcdCluster.Name + if etcdCluster.Name == "main" { + // Special case + etcClusterName = "etcd" + } + for _, etcdClusterMember := range etcdCluster.Members { + name := etcClusterName + "-" + etcdClusterMember.Name + dnsInternalSuffix + dnsHostnames = append(dnsHostnames, name) + } + } + + return dnsHostnames +} diff --git a/upup/pkg/fi/cloudup/dns_test.go b/upup/pkg/fi/cloudup/dns_test.go new file mode 100644 index 0000000000..5fd6d0c8ea --- /dev/null +++ b/upup/pkg/fi/cloudup/dns_test.go @@ -0,0 +1,69 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudup + +import ( + "k8s.io/kops/pkg/apis/kops" + "reflect" + "sort" + "testing" +) + +func TestPrecreateDNSNames(t *testing.T) { + cluster := &kops.Cluster{} + cluster.ObjectMeta.Name = "cluster1.example.com" + cluster.Spec.MasterPublicName = "api." + cluster.ObjectMeta.Name + cluster.Spec.MasterInternalName = "api.internal." + cluster.ObjectMeta.Name + cluster.Spec.EtcdClusters = []*kops.EtcdClusterSpec{ + { + Name: "main", + Members: []*kops.EtcdMemberSpec{ + {Name: "zone1"}, + {Name: "zone2"}, + {Name: "zone3"}, + }, + }, + { + Name: "events", + Members: []*kops.EtcdMemberSpec{ + {Name: "zonea"}, + {Name: "zoneb"}, + {Name: "zonec"}, + }, + }, + } + + actual := buildPrecreateDNSHostnames(cluster) + + expected := []string{ + "api.cluster1.example.com", + "api.internal.cluster1.example.com", + "etcd-events-zonea.internal.cluster1.example.com", + "etcd-events-zoneb.internal.cluster1.example.com", + "etcd-events-zonec.internal.cluster1.example.com", + "etcd-zone1.internal.cluster1.example.com", + "etcd-zone2.internal.cluster1.example.com", + "etcd-zone3.internal.cluster1.example.com", + } + + sort.Strings(actual) + sort.Strings(expected) + + if !reflect.DeepEqual(expected, actual) { + t.Fatalf("unexpected records. expected=%v actual=%v", expected, actual) + } +} From 8afba37f0a046673536738e8592fbf19e15aa81b Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sun, 8 Jan 2017 15:38:05 -0500 Subject: [PATCH 11/29] Create a role for networking addons; use as selector role.kubernetes.io/networking This ensures that when we switch networking providers, we replace all the components of the prior tool. --- .../networking.kope.io/v1.0.20161116.yaml | 2 ++ .../v2.0.yaml.template | 5 +++++ .../addons/networking.weave/v1.8.2.yaml | 3 +++ .../pkg/fi/cloudup/bootstrapchannelbuilder.go | 19 ++++++++++++++++--- .../kopeio-vxlan/manifest.yaml | 2 +- 5 files changed, 27 insertions(+), 4 deletions(-) diff --git a/upup/models/cloudup/resources/addons/networking.kope.io/v1.0.20161116.yaml b/upup/models/cloudup/resources/addons/networking.kope.io/v1.0.20161116.yaml index e814c41c86..c2f6c94ce7 100644 --- a/upup/models/cloudup/resources/addons/networking.kope.io/v1.0.20161116.yaml +++ b/upup/models/cloudup/resources/addons/networking.kope.io/v1.0.20161116.yaml @@ -5,11 +5,13 @@ metadata: namespace: kube-system labels: k8s-addon: networking.addons.k8s.io + role.kubernetes.io/networking: "1" spec: template: metadata: labels: name: kopeio-networking-agent + role.kubernetes.io/networking: "1" spec: hostPID: true hostIPC: true diff --git a/upup/models/cloudup/resources/addons/networking.projectcalico.org/v2.0.yaml.template b/upup/models/cloudup/resources/addons/networking.projectcalico.org/v2.0.yaml.template index a8abfa8a6f..73dbfc7c73 100644 --- a/upup/models/cloudup/resources/addons/networking.projectcalico.org/v2.0.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.projectcalico.org/v2.0.yaml.template @@ -60,6 +60,7 @@ metadata: namespace: kube-system labels: k8s-app: calico-node + role.kubernetes.io/networking: "1" spec: selector: matchLabels: @@ -68,6 +69,7 @@ spec: metadata: labels: k8s-app: calico-node + role.kubernetes.io/networking: "1" spec: hostNetwork: true containers: @@ -165,6 +167,7 @@ metadata: namespace: kube-system labels: k8s-app: calico-policy + role.kubernetes.io/networking: "1" spec: # The policy controller can only have a single active instance. replicas: 1 @@ -174,6 +177,7 @@ spec: namespace: kube-system labels: k8s-app: calico-policy-controller + role.kubernetes.io/networking: "1" annotations: scheduler.alpha.kubernetes.io/critical-pod: '' scheduler.alpha.kubernetes.io/tolerations: | @@ -217,6 +221,7 @@ metadata: namespace: kube-system labels: k8s-app: calico + role.kubernetes.io/networking: "1" spec: template: metadata: diff --git a/upup/models/cloudup/resources/addons/networking.weave/v1.8.2.yaml b/upup/models/cloudup/resources/addons/networking.weave/v1.8.2.yaml index 5b5c5e238f..ef291df7cf 100644 --- a/upup/models/cloudup/resources/addons/networking.weave/v1.8.2.yaml +++ b/upup/models/cloudup/resources/addons/networking.weave/v1.8.2.yaml @@ -3,11 +3,14 @@ kind: DaemonSet metadata: name: weave-net namespace: kube-system + labels: + role.kubernetes.io/networking: "1" spec: template: metadata: labels: name: weave-net + role.kubernetes.io/networking: "1" annotations: scheduler.alpha.kubernetes.io/tolerations: | [ diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index 669ffe8b73..7d84b57666 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -143,6 +143,19 @@ func (b *BootstrapChannelBuilder) buildManifest() (*channelsapi.Addons, map[stri manifests[key] = "addons/" + location } + // The role.kubernetes.io/networking is used to label anything related to a networking addin, + // so that if we switch networking plugins (e.g. calico -> weave or vice-versa), we'll replace the + // old networking plugin, and there won't be old pods "floating around". + + // This means whenever we create or update a networking plugin, we should be sure that: + // 1. the selector is role.kubernetes.io/networking=1 + // 2. every object in the manifest is labeleled with role.kubernetes.io/networking=1 + + // TODO: Some way to test/enforce this? + + // TODO: Create "empty" configurations for others, so we can delete e.g. the kopeio configuration + // if we switch to kubenet? + if b.cluster.Spec.Networking.Kopeio != nil { key := "networking.kope.io" version := "1.0.20161116" @@ -153,7 +166,7 @@ func (b *BootstrapChannelBuilder) buildManifest() (*channelsapi.Addons, map[stri addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ Name: fi.String(key), Version: fi.String(version), - Selector: map[string]string{"k8s-addon": key}, + Selector: map[string]string{"role.kubernetes.io/networking": "1"}, Manifest: fi.String(location), }) @@ -170,7 +183,7 @@ func (b *BootstrapChannelBuilder) buildManifest() (*channelsapi.Addons, map[stri addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ Name: fi.String(key), Version: fi.String(version), - Selector: map[string]string{"k8s-addon": key}, + Selector: map[string]string{"role.kubernetes.io/networking": "1"}, Manifest: fi.String(location), }) @@ -187,7 +200,7 @@ func (b *BootstrapChannelBuilder) buildManifest() (*channelsapi.Addons, map[stri addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ Name: fi.String(key), Version: fi.String(version), - Selector: map[string]string{"k8s-addon": key}, + Selector: map[string]string{"role.kubernetes.io/networking": "1"}, Manifest: fi.String(location), }) diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/kopeio-vxlan/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/kopeio-vxlan/manifest.yaml index bb9411bafc..eddecdba76 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/kopeio-vxlan/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/kopeio-vxlan/manifest.yaml @@ -32,6 +32,6 @@ spec: - manifest: networking.kope.io/v1.0.20161116.yaml name: networking.kope.io selector: - k8s-addon: networking.kope.io + role.kubernetes.io/networking: "1" version: 1.0.20161116 From 537085d94c92ed60c5a98428937912b39b5b111f Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sun, 8 Jan 2017 14:48:58 -0500 Subject: [PATCH 12/29] dns-controller: don't make spurious updates in buffer These will be caught later anyway, but we can save a false invalidation and a spurious log message. --- dns-controller/cmd/dns-controller/main.go | 4 ++-- dns-controller/pkg/dns/dnscontroller.go | 29 +++++++++++++++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/dns-controller/cmd/dns-controller/main.go b/dns-controller/cmd/dns-controller/main.go index dd9bd48840..7001766cb5 100644 --- a/dns-controller/cmd/dns-controller/main.go +++ b/dns-controller/cmd/dns-controller/main.go @@ -23,8 +23,8 @@ import ( "k8s.io/kops/dns-controller/pkg/dns" "k8s.io/kops/dns-controller/pkg/watchers" "k8s.io/kubernetes/federation/pkg/dnsprovider" - client "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5/typed/core/v1" - client_extensions "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5/typed/extensions/v1beta1" + client "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/core/v1" + client_extensions "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/extensions/v1beta1" kubectl_util "k8s.io/kubernetes/pkg/kubectl/cmd/util" "os" diff --git a/dns-controller/pkg/dns/dnscontroller.go b/dns-controller/pkg/dns/dnscontroller.go index d4d6097c73..0ea4bba8cb 100644 --- a/dns-controller/pkg/dns/dnscontroller.go +++ b/dns-controller/pkg/dns/dnscontroller.go @@ -480,19 +480,44 @@ func (s *DNSControllerScope) MarkReady() { } func (s *DNSControllerScope) Replace(recordName string, records []Record) { - glog.V(2).Infof("Update %s/%s: %v", s.ScopeName, recordName, records) - s.mutex.Lock() defer s.mutex.Unlock() + existing, exists := s.Records[recordName] + if len(records) == 0 { + if !exists { + glog.V(6).Infof("skipping spurious removal of record %s/%s", s.ScopeName, recordName) + return + } + delete(s.Records, recordName) } else { + if recordsSliceEquals(existing, records) { + glog.V(6).Infof("skipping spurious update of record %s/%s=%s", s.ScopeName, recordName, records) + return + } + s.Records[recordName] = records } + + glog.V(2).Infof("Update %s/%s: %v", s.ScopeName, recordName, records) s.parent.recordChange() } +// recordsSliceEquals compares two []Record +func recordsSliceEquals(l, r []Record) bool { + if len(l) != len(r) { + return false + } + for i := range l { + if l[i] != r[i] { + return false + } + } + return true +} + // CreateScope creates a scope object. func (c *DNSController) CreateScope(scopeName string) (Scope, error) { c.mutex.Lock() From ed78a2b18f6244f6abdfb75003f6db2c6ef20ceb Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sun, 8 Jan 2017 23:56:37 -0500 Subject: [PATCH 13/29] kops get clusters should print zones The list of subnets just isn't very interesting --- cmd/kops/get_cluster.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/kops/get_cluster.go b/cmd/kops/get_cluster.go index dfc906c9b5..1b8c7a6763 100644 --- a/cmd/kops/get_cluster.go +++ b/cmd/kops/get_cluster.go @@ -27,6 +27,7 @@ import ( "k8s.io/kops/pkg/apis/kops/registry" "k8s.io/kops/util/pkg/tables" k8sapi "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/util/sets" ) type GetClusterOptions struct { @@ -126,14 +127,14 @@ func RunGetClusters(context Factory, out io.Writer, options *GetClusterOptions) t.AddColumn("CLOUD", func(c *api.Cluster) string { return c.Spec.CloudProvider }) - t.AddColumn("SUBNETS", func(c *api.Cluster) string { - var subnetNames []string + t.AddColumn("ZONES", func(c *api.Cluster) string { + zones := sets.NewString() for _, s := range c.Spec.Subnets { - subnetNames = append(subnetNames, s.Name) + zones.Insert(s.Zone) } - return strings.Join(subnetNames, ",") + return strings.Join(zones.List(), ",") }) - return t.Render(clusters, out, "NAME", "CLOUD", "SUBNETS") + return t.Render(clusters, out, "NAME", "CLOUD", "ZONES") case OutputYaml: for i, cluster := range clusters { From 17f54fefb7e66dc578a71ccf31119fe2134e3cba Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 9 Jan 2017 00:29:58 -0500 Subject: [PATCH 14/29] Clean up sysctls Remove duplicate key; add missing blank line --- nodeup/pkg/model/sysctls.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/nodeup/pkg/model/sysctls.go b/nodeup/pkg/model/sysctls.go index 505aff8cdf..dab411bb5d 100644 --- a/nodeup/pkg/model/sysctls.go +++ b/nodeup/pkg/model/sysctls.go @@ -55,10 +55,6 @@ func (b *SysctlBuilder) Build(c *fi.ModelBuilderContext) error { "net.core.somaxconn = 32768", "", - "# Increase number of incoming connections backlog", - "net.core.netdev_max_backlog = 5000", - "", - "# Maximum Socket Receive Buffer", "net.core.rmem_max = 16777216", "", @@ -106,7 +102,8 @@ func (b *SysctlBuilder) Build(c *fi.ModelBuilderContext) error { "# AWS settings", "", "# Issue #23395", - "net.ipv4.neigh.default.gc_thresh1=0") + "net.ipv4.neigh.default.gc_thresh1=0", + "") } if b.Cluster.Spec.CloudProvider == string(fi.CloudProviderGCE) { From 256630f265bbb60f74717707ab97c3d6b1834358 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 9 Jan 2017 00:30:27 -0500 Subject: [PATCH 15/29] Dial down default protokube logging to 4 --- upup/pkg/fi/nodeup/template_functions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upup/pkg/fi/nodeup/template_functions.go b/upup/pkg/fi/nodeup/template_functions.go index f549353911..f600c413ef 100644 --- a/upup/pkg/fi/nodeup/template_functions.go +++ b/upup/pkg/fi/nodeup/template_functions.go @@ -276,7 +276,7 @@ func (t *templateFunctions) ProtokubeFlags() *ProtokubeFlags { f.Channels = t.nodeupConfig.Channels } - f.LogLevel = fi.Int(8) + f.LogLevel = fi.Int(4) f.Containerized = fi.Bool(true) zone := t.cluster.Spec.DNSZone From 0464a2661257b641aa76c0cb2119299d0ffeaebb Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 9 Jan 2017 00:03:52 -0500 Subject: [PATCH 16/29] Update to dns-controller 1.5.0 --- .../dns-controller.addons.k8s.io/addon.yaml | 4 +- .../v1.5.0.yaml.template | 39 +++++++++++++++++++ .../pkg/fi/cloudup/bootstrapchannelbuilder.go | 2 +- .../kopeio-vxlan/manifest.yaml | 4 +- .../simple/manifest.yaml | 4 +- 5 files changed, 46 insertions(+), 7 deletions(-) create mode 100644 upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/v1.5.0.yaml.template diff --git a/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/addon.yaml b/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/addon.yaml index e4efee95a9..3c2eee7cae 100644 --- a/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/addon.yaml +++ b/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/addon.yaml @@ -3,8 +3,8 @@ metadata: name: dns-controller spec: addons: - - version: 1.4.1 + - version: 1.5.0 selector: k8s-addon: dns-controller.addons.k8s.io - manifest: v1.4.1.yaml + manifest: v1.5.0.yaml diff --git a/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/v1.5.0.yaml.template b/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/v1.5.0.yaml.template new file mode 100644 index 0000000000..1222427bbc --- /dev/null +++ b/upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/v1.5.0.yaml.template @@ -0,0 +1,39 @@ +kind: Deployment +apiVersion: extensions/v1beta1 +metadata: + name: dns-controller + namespace: kube-system + labels: + k8s-addon: dns-controller.addons.k8s.io + k8s-app: dns-controller + version: v1.5.0 +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: dns-controller + template: + metadata: + labels: + k8s-addon: dns-controller.addons.k8s.io + k8s-app: dns-controller + version: v1.5.0 + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key": "dedicated", "value": "master"}]' + spec: + nodeSelector: + kubernetes.io/role: master + dnsPolicy: Default # Don't use cluster DNS (we are likely running before kube-dns) + hostNetwork: true + containers: + - name: dns-controller + image: kope/dns-controller:1.5.0 + command: +{{ range $arg := DnsControllerArgv }} + - "{{ $arg }}" +{{ end }} + resources: + requests: + cpu: 50m + memory: 50Mi diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index 7d84b57666..e0202eba6a 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -115,7 +115,7 @@ func (b *BootstrapChannelBuilder) buildManifest() (*channelsapi.Addons, map[stri { key := "dns-controller.addons.k8s.io" - version := "1.4.1" + version := "1.5.0" location := key + "/v" + version + ".yaml" diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/kopeio-vxlan/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/kopeio-vxlan/manifest.yaml index eddecdba76..87c65cc316 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/kopeio-vxlan/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/kopeio-vxlan/manifest.yaml @@ -19,11 +19,11 @@ spec: selector: k8s-addon: limit-range.addons.k8s.io version: 1.5.0 - - manifest: dns-controller.addons.k8s.io/v1.4.1.yaml + - manifest: dns-controller.addons.k8s.io/v1.5.0.yaml name: dns-controller.addons.k8s.io selector: k8s-addon: dns-controller.addons.k8s.io - version: 1.4.1 + version: 1.5.0 - manifest: storage-aws.addons.k8s.io/v1.5.0.yaml name: storage-aws.addons.k8s.io selector: diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml index ed076eb5e6..6b40709769 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml @@ -19,11 +19,11 @@ spec: selector: k8s-addon: limit-range.addons.k8s.io version: 1.5.0 - - manifest: dns-controller.addons.k8s.io/v1.4.1.yaml + - manifest: dns-controller.addons.k8s.io/v1.5.0.yaml name: dns-controller.addons.k8s.io selector: k8s-addon: dns-controller.addons.k8s.io - version: 1.4.1 + version: 1.5.0 - manifest: storage-aws.addons.k8s.io/v1.5.0.yaml name: storage-aws.addons.k8s.io selector: From 302824ce4ad4bb4be6bfe612e98f140d4086b65f Mon Sep 17 00:00:00 2001 From: Kris Nova Date: Mon, 9 Jan 2017 08:19:59 -0500 Subject: [PATCH 17/29] Docs on using the bastion --- docs/bastion.md | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/docs/bastion.md b/docs/bastion.md index ef75e4dde2..12c35bd3e4 100644 --- a/docs/bastion.md +++ b/docs/bastion.md @@ -11,7 +11,7 @@ Bastion provide an external facing point of entry into a network containing priv To enable a bastion instance group, a user will need to set the `--bastion` flag on cluster create -``` +```yaml kops create cluster --topology private --networking $provider --bastion $NAME ``` @@ -19,13 +19,13 @@ kops create cluster --topology private --networking $provider --bastion $NAME You can edit the bastion instance group to make changes. By default the name of the bastion instance group will be `bastions` and you can specify the name of the cluster with `--name` as in: -``` +```yaml kops edit ig bastions --name $KOPS_NAME ``` You should now be able to edit and configure your bastion instance group. -``` +```yaml apiVersion: kops/v1alpha2 kind: InstanceGroup metadata: @@ -51,21 +51,21 @@ If you do not want the bastion instance group created at all, simply drop the `- By default the bastion instance group will create a public CNAME alias that will point to the bastion ELB. -The default bastion name is `bastion-$NAME` as in +The default bastion name is `bastion.$NAME` as in -``` -bastion-example.kubernetes.com +```yaml +bastion.example.kubernetes.com ``` Unless a user is using `--dns-zone` which will inherently use the `basion-$ZONE` syntax. You can define a custom bastion CNAME by editing the main cluster config `kops edit cluster $NAME` and modifying the following block -``` +```yaml spec: topology: bastion: - bastionPublicName: bastion-example.kubernetes.com + bastionPublicName: bastion.example.kubernetes.com ``` @@ -75,7 +75,7 @@ The bastion is accessed via an AWS ELB. The ELB is required to gain secure acces You can increase the ELB idle timeout by editing the main cluster config `kops edit cluster $NAME` and modifyng the following block -``` +```yaml spec: topology: bastion: @@ -83,3 +83,22 @@ spec: ``` Where the maximum value is 1200 seconds (20 minutes) allowed by AWS. [More information](http://docs.aws.amazon.com/elasticloadbalancing/latest/classic/config-idle-timeout.html) + +### Using the bastion + +Once your cluster is setup and you need to SSH into the bastion you can access a cluster resource using the following steps + +```bash +# Verify you have an SSH agent running. This should match whatever you built your cluster with. +ssh-add -l +# If you need to add an agent +ssh-add path/to/public/key + +# Now you can SSH into the bastion +ssh -A admin@ + +# Where is usually bastion.$clustername (bastion.example.kubernetes.cluster) unless otherwise specified + +``` + +Now that you can successfully SSH into the bastion with a forwarded SSH agent. You can SSH into any of your cluster resources using their local IP address. You can get their local IP address from the cloud console. From 61011650dd08c1c18a787f2a845dedda31f46a60 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 4 Jan 2017 01:24:50 -0500 Subject: [PATCH 18/29] Support private hosted zones in DNS --- cmd/kops/create_cluster.go | 31 ++++++++++++++++++- pkg/apis/kops/cluster.go | 1 + pkg/apis/kops/topology.go | 14 +++++++++ pkg/apis/kops/v1alpha1/conversion.go | 16 ++++++++++ pkg/apis/kops/v1alpha1/defaults.go | 8 +++++ pkg/apis/kops/v1alpha1/topology.go | 14 +++++++++ .../kops/v1alpha1/zz_generated.conversion.go | 20 ++++++++++++ pkg/apis/kops/v1alpha2/defaults.go | 8 +++++ pkg/apis/kops/v1alpha2/topology.go | 14 +++++++++ pkg/featureflag/featureflag.go | 3 -- pkg/model/context.go | 17 +++++++--- .../conversion/integration_test.go | 2 +- .../conversion/minimal/v1alpha0.yaml | 2 ++ .../conversion/minimal/v1alpha1.yaml | 2 ++ .../conversion/minimal/v1alpha2.yaml | 2 ++ .../create_cluster/ha/expected-v1alpha1.yaml | 2 ++ .../create_cluster/ha/expected-v1alpha2.yaml | 2 ++ .../minimal/expected-v1alpha1.yaml | 2 ++ .../minimal/expected-v1alpha2.yaml | 2 ++ .../private/expected-v1alpha1.yaml | 2 ++ .../private/expected-v1alpha2.yaml | 2 ++ upup/pkg/fi/cloudup/awstasks/dnszone.go | 4 ++- 22 files changed, 160 insertions(+), 10 deletions(-) diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index 0e488635c3..96b49ebaf5 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -65,6 +65,9 @@ type CreateClusterOptions struct { // The network topology to use Topology string + // The DNS type to use (public/private) + DNSType string + // Enable/Disable Bastion Host complete setup Bastion bool } @@ -77,7 +80,8 @@ func (o *CreateClusterOptions) InitDefaults() { o.Networking = "kubenet" o.AssociatePublicIP = true o.Channel = api.DefaultChannel - o.Topology = "public" + o.Topology = api.TopologyPublic + o.DNSType = string(api.DNSTypePublic) o.Bastion = false } @@ -143,6 +147,9 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command { // Network topology cmd.Flags().StringVarP(&options.Topology, "topology", "t", options.Topology, "Controls network topology for the cluster. public|private. Default is 'public'.") + // DNS + cmd.Flags().StringVar(&options.DNSType, "dns", options.DNSType, "DNS hosted zone to use: public|private. Default is 'public'.") + // Bastion cmd.Flags().BoolVar(&options.Bastion, "bastion", options.Bastion, "Pass the --bastion flag to enable a bastion instance group. Only applies to private topology.") @@ -503,6 +510,28 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e return fmt.Errorf("Invalid topology %s.", c.Topology) } + // DNS + if c.DNSType == "" { + // The flag default should have set this, but we might be being called as a library + glog.Infof("Empty DNS. Defaulting to public DNS") + c.DNSType = string(api.DNSTypePublic) + } + + if cluster.Spec.Topology == nil { + cluster.Spec.Topology = &api.TopologySpec{} + } + if cluster.Spec.Topology.DNS == nil { + cluster.Spec.Topology.DNS = &api.DNSSpec{} + } + switch strings.ToLower(c.DNSType) { + case "public": + cluster.Spec.Topology.DNS.Type = api.DNSTypePublic + case "private": + cluster.Spec.Topology.DNS.Type = api.DNSTypePrivate + default: + return fmt.Errorf("unknown DNSType: %q", c.DNSType) + } + sshPublicKeys := make(map[string][]byte) if c.SSHPublicKey != "" { c.SSHPublicKey = utils.ExpandPath(c.SSHPublicKey) diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index 38afef3a1f..982a53e895 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -344,6 +344,7 @@ func (c *Cluster) FillDefaults() error { // Topology support if c.Spec.Topology == nil { c.Spec.Topology = &TopologySpec{Masters: TopologyPublic, Nodes: TopologyPublic} + c.Spec.Topology.DNS = &DNSSpec{Type: DNSTypePublic} } if len(c.Spec.KubernetesAPIAccess) == 0 { diff --git a/pkg/apis/kops/topology.go b/pkg/apis/kops/topology.go index b918b6c21c..7b51bc74da 100644 --- a/pkg/apis/kops/topology.go +++ b/pkg/apis/kops/topology.go @@ -34,4 +34,18 @@ type TopologySpec struct { // or disable inbound SSH communication from the Internet, some call bastion // as the "jump server". Bastion *BastionSpec `json:"bastion,omitempty"` + + // DNS configures options relating to DNS, in particular whether we use a public or a private hosted zone + DNS *DNSSpec `json:"dns,omitempty"` } + +type DNSSpec struct { + Type DNSType `json:"type,omitempty"` +} + +type DNSType string + +const ( + DNSTypePublic DNSType = "Public" + DNSTypePrivate DNSType = "Private" +) diff --git a/pkg/apis/kops/v1alpha1/conversion.go b/pkg/apis/kops/v1alpha1/conversion.go index 201504ae89..300c3daeb4 100644 --- a/pkg/apis/kops/v1alpha1/conversion.go +++ b/pkg/apis/kops/v1alpha1/conversion.go @@ -247,6 +247,14 @@ func Convert_v1alpha1_TopologySpec_To_kops_TopologySpec(in *TopologySpec, out *k } else { out.Bastion = nil } + if in.DNS != nil { + out.DNS = new(kops.DNSSpec) + if err := Convert_v1alpha1_DNSSpec_To_kops_DNSSpec(in.DNS, out.DNS, s); err != nil { + return err + } + } else { + out.DNS = nil + } return nil } @@ -261,5 +269,13 @@ func Convert_kops_TopologySpec_To_v1alpha1_TopologySpec(in *kops.TopologySpec, o } else { out.Bastion = nil } + if in.DNS != nil { + out.DNS = new(DNSSpec) + if err := Convert_kops_DNSSpec_To_v1alpha1_DNSSpec(in.DNS, out.DNS, s); err != nil { + return err + } + } else { + out.DNS = nil + } return nil } diff --git a/pkg/apis/kops/v1alpha1/defaults.go b/pkg/apis/kops/v1alpha1/defaults.go index 2673e18069..68ae220bed 100644 --- a/pkg/apis/kops/v1alpha1/defaults.go +++ b/pkg/apis/kops/v1alpha1/defaults.go @@ -41,6 +41,14 @@ func SetDefaults_ClusterSpec(obj *ClusterSpec) { obj.Topology.Nodes = TopologyPublic } + if obj.Topology.DNS == nil { + obj.Topology.DNS = &DNSSpec{} + } + + if obj.Topology.DNS.Type == "" { + obj.Topology.DNS.Type = DNSTypePublic + } + if obj.API == nil { obj.API = &AccessSpec{} } diff --git a/pkg/apis/kops/v1alpha1/topology.go b/pkg/apis/kops/v1alpha1/topology.go index 09d52019bd..e2cb38361d 100644 --- a/pkg/apis/kops/v1alpha1/topology.go +++ b/pkg/apis/kops/v1alpha1/topology.go @@ -35,4 +35,18 @@ type TopologySpec struct { // or disable inbound SSH communication from the Internet, some call bastion // as the "jump server". Bastion *BastionSpec `json:"bastion,omitempty"` + + // DNS configures options relating to DNS, in particular whether we use a public or a private hosted zone + DNS *DNSSpec `json:"dns,omitempty"` } + +type DNSSpec struct { + Type DNSType `json:"type,omitempty"` +} + +type DNSType string + +const ( + DNSTypePublic DNSType = "Public" + DNSTypePrivate DNSType = "Private" +) diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index 273f688a5d..97e6058c99 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -50,6 +50,8 @@ func RegisterConversions(scheme *runtime.Scheme) error { Convert_kops_ClusterSpec_To_v1alpha1_ClusterSpec, Convert_v1alpha1_DNSAccessSpec_To_kops_DNSAccessSpec, Convert_kops_DNSAccessSpec_To_v1alpha1_DNSAccessSpec, + Convert_v1alpha1_DNSSpec_To_kops_DNSSpec, + Convert_kops_DNSSpec_To_v1alpha1_DNSSpec, Convert_v1alpha1_DockerConfig_To_kops_DockerConfig, Convert_kops_DockerConfig_To_v1alpha1_DockerConfig, Convert_v1alpha1_EtcdClusterSpec_To_kops_EtcdClusterSpec, @@ -555,6 +557,24 @@ func Convert_kops_DNSAccessSpec_To_v1alpha1_DNSAccessSpec(in *kops.DNSAccessSpec return autoConvert_kops_DNSAccessSpec_To_v1alpha1_DNSAccessSpec(in, out, s) } +func autoConvert_v1alpha1_DNSSpec_To_kops_DNSSpec(in *DNSSpec, out *kops.DNSSpec, s conversion.Scope) error { + out.Type = kops.DNSType(in.Type) + return nil +} + +func Convert_v1alpha1_DNSSpec_To_kops_DNSSpec(in *DNSSpec, out *kops.DNSSpec, s conversion.Scope) error { + return autoConvert_v1alpha1_DNSSpec_To_kops_DNSSpec(in, out, s) +} + +func autoConvert_kops_DNSSpec_To_v1alpha1_DNSSpec(in *kops.DNSSpec, out *DNSSpec, s conversion.Scope) error { + out.Type = DNSType(in.Type) + return nil +} + +func Convert_kops_DNSSpec_To_v1alpha1_DNSSpec(in *kops.DNSSpec, out *DNSSpec, s conversion.Scope) error { + return autoConvert_kops_DNSSpec_To_v1alpha1_DNSSpec(in, out, s) +} + func autoConvert_v1alpha1_DockerConfig_To_kops_DockerConfig(in *DockerConfig, out *kops.DockerConfig, s conversion.Scope) error { out.Bridge = in.Bridge out.LogLevel = in.LogLevel diff --git a/pkg/apis/kops/v1alpha2/defaults.go b/pkg/apis/kops/v1alpha2/defaults.go index ccb28fd93f..f72cbbb8ae 100644 --- a/pkg/apis/kops/v1alpha2/defaults.go +++ b/pkg/apis/kops/v1alpha2/defaults.go @@ -41,6 +41,14 @@ func SetDefaults_ClusterSpec(obj *ClusterSpec) { obj.Topology.Nodes = TopologyPublic } + if obj.Topology.DNS == nil { + obj.Topology.DNS = &DNSSpec{} + } + + if obj.Topology.DNS.Type == "" { + obj.Topology.DNS.Type = DNSTypePublic + } + if obj.API == nil { obj.API = &AccessSpec{} } diff --git a/pkg/apis/kops/v1alpha2/topology.go b/pkg/apis/kops/v1alpha2/topology.go index dea8db5637..3a061de9bb 100644 --- a/pkg/apis/kops/v1alpha2/topology.go +++ b/pkg/apis/kops/v1alpha2/topology.go @@ -34,4 +34,18 @@ type TopologySpec struct { // or disable inbound SSH communication from the Internet, some call bastion // as the "jump server". Bastion *BastionSpec `json:"bastion,omitempty"` + + // DNS configures options relating to DNS, in particular whether we use a public or a private hosted zone + DNS *DNSSpec `json:"dns,omitempty"` } + +type DNSSpec struct { + Type DNSType `json:"type,omitempty"` +} + +type DNSType string + +const ( + DNSTypePublic DNSType = "Public" + DNSTypePrivate DNSType = "Private" +) diff --git a/pkg/featureflag/featureflag.go b/pkg/featureflag/featureflag.go index 63971d16db..f2befc02e2 100644 --- a/pkg/featureflag/featureflag.go +++ b/pkg/featureflag/featureflag.go @@ -34,9 +34,6 @@ func Bool(b bool) *bool { return &b } -// PreviewPrivateDNS turns on the preview of the private hosted zone support. -var PreviewPrivateDNS = New("PreviewPrivateDNS", Bool(false)) - // DNSPreCreate controls whether we pre-create DNS records. var DNSPreCreate = New("DNSPreCreate", Bool(true)) diff --git a/pkg/model/context.go b/pkg/model/context.go index 0d498be223..eef4051d36 100644 --- a/pkg/model/context.go +++ b/pkg/model/context.go @@ -20,7 +20,6 @@ import ( "fmt" "github.com/golang/glog" "k8s.io/kops/pkg/apis/kops" - "k8s.io/kops/pkg/featureflag" "strings" ) @@ -165,9 +164,19 @@ func (m *KopsModelContext) UseLoadBalancerForAPI() bool { } func (m *KopsModelContext) UsePrivateDNS() bool { - if featureflag.PreviewPrivateDNS.Enabled() { - glog.Infof("PreviewPrivateDNS enabled; using private DNS") - return true + topology := m.Cluster.Spec.Topology + if topology != nil && topology.DNS != nil { + switch topology.DNS.Type { + case kops.DNSTypePublic: + return false + case kops.DNSTypePrivate: + return true + + default: + glog.Warningf("Unknown DNS type %q", topology.DNS.Type) + return false + } } + return false } diff --git a/tests/integration/conversion/integration_test.go b/tests/integration/conversion/integration_test.go index 2e1d108ba1..41617e48ce 100644 --- a/tests/integration/conversion/integration_test.go +++ b/tests/integration/conversion/integration_test.go @@ -34,7 +34,7 @@ import ( ) // TestMinimal runs the test on a minimum configuration, similar to kops create cluster minimal.example.com --zones us-west-1a -func TestMinimal(t *testing.T) { +func ConversionTestMinimal(t *testing.T) { runTest(t, "minimal", "v1alpha1", "v1alpha2") runTest(t, "minimal", "v1alpha2", "v1alpha1") diff --git a/tests/integration/conversion/minimal/v1alpha0.yaml b/tests/integration/conversion/minimal/v1alpha0.yaml index cbad5486b8..1f87d99e8f 100644 --- a/tests/integration/conversion/minimal/v1alpha0.yaml +++ b/tests/integration/conversion/minimal/v1alpha0.yaml @@ -28,6 +28,8 @@ spec: bastion: idleTimeout: 120 machineType: t2.medium + dns: + type: Public masters: public nodes: public zones: diff --git a/tests/integration/conversion/minimal/v1alpha1.yaml b/tests/integration/conversion/minimal/v1alpha1.yaml index 358de99f50..a409781b68 100644 --- a/tests/integration/conversion/minimal/v1alpha1.yaml +++ b/tests/integration/conversion/minimal/v1alpha1.yaml @@ -28,6 +28,8 @@ spec: kubenet: {} nonMasqueradeCIDR: 100.64.0.0/10 topology: + dns: + type: Public masters: public nodes: public zones: diff --git a/tests/integration/conversion/minimal/v1alpha2.yaml b/tests/integration/conversion/minimal/v1alpha2.yaml index 31881f54d9..98043201f0 100644 --- a/tests/integration/conversion/minimal/v1alpha2.yaml +++ b/tests/integration/conversion/minimal/v1alpha2.yaml @@ -35,6 +35,8 @@ spec: type: Public zone: us-test-1a topology: + dns: + type: Public masters: public nodes: public diff --git a/tests/integration/create_cluster/ha/expected-v1alpha1.yaml b/tests/integration/create_cluster/ha/expected-v1alpha1.yaml index 0ed2026d93..7dcebdf533 100644 --- a/tests/integration/create_cluster/ha/expected-v1alpha1.yaml +++ b/tests/integration/create_cluster/ha/expected-v1alpha1.yaml @@ -33,6 +33,8 @@ spec: kubenet: {} nonMasqueradeCIDR: 100.64.0.0/10 topology: + dns: + type: Public masters: public nodes: public zones: diff --git a/tests/integration/create_cluster/ha/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha/expected-v1alpha2.yaml index 8f0d4fe485..daf60e2867 100644 --- a/tests/integration/create_cluster/ha/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha/expected-v1alpha2.yaml @@ -46,6 +46,8 @@ spec: type: Public zone: us-test-1c topology: + dns: + type: Public masters: public nodes: public diff --git a/tests/integration/create_cluster/minimal/expected-v1alpha1.yaml b/tests/integration/create_cluster/minimal/expected-v1alpha1.yaml index c2e3eaecda..aed01e3462 100644 --- a/tests/integration/create_cluster/minimal/expected-v1alpha1.yaml +++ b/tests/integration/create_cluster/minimal/expected-v1alpha1.yaml @@ -25,6 +25,8 @@ spec: kubenet: {} nonMasqueradeCIDR: 100.64.0.0/10 topology: + dns: + type: Public masters: public nodes: public zones: diff --git a/tests/integration/create_cluster/minimal/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal/expected-v1alpha2.yaml index e88d7c7e81..3cd06e44c5 100644 --- a/tests/integration/create_cluster/minimal/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal/expected-v1alpha2.yaml @@ -30,6 +30,8 @@ spec: type: Public zone: us-test-1a topology: + dns: + type: Public masters: public nodes: public diff --git a/tests/integration/create_cluster/private/expected-v1alpha1.yaml b/tests/integration/create_cluster/private/expected-v1alpha1.yaml index 74216d5065..e6a29a0731 100644 --- a/tests/integration/create_cluster/private/expected-v1alpha1.yaml +++ b/tests/integration/create_cluster/private/expected-v1alpha1.yaml @@ -29,6 +29,8 @@ spec: bastion: enable: true name: bastion-private.example.com + dns: + type: Public masters: private nodes: private zones: diff --git a/tests/integration/create_cluster/private/expected-v1alpha2.yaml b/tests/integration/create_cluster/private/expected-v1alpha2.yaml index ab7be2c1fa..6fdca7646d 100644 --- a/tests/integration/create_cluster/private/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/private/expected-v1alpha2.yaml @@ -37,6 +37,8 @@ spec: topology: bastion: bastionPublicName: bastion-private.example.com + dns: + type: Public masters: private nodes: private diff --git a/upup/pkg/fi/cloudup/awstasks/dnszone.go b/upup/pkg/fi/cloudup/awstasks/dnszone.go index 57129cf09c..bd052f29a9 100644 --- a/upup/pkg/fi/cloudup/awstasks/dnszone.go +++ b/upup/pkg/fi/cloudup/awstasks/dnszone.go @@ -64,7 +64,9 @@ func (e *DNSZone) Find(c *fi.Context) (*DNSZone, error) { if z.HostedZone.Name != nil { actual.DNSName = fi.String(strings.TrimSuffix(*z.HostedZone.Name, ".")) } - actual.ZoneID = z.HostedZone.Id + if z.HostedZone.Id != nil { + actual.ZoneID = fi.String(strings.TrimPrefix(*z.HostedZone.Id, "/hostedzone/")) + } actual.Private = z.HostedZone.Config.PrivateZone // If the zone is private, but we don't want it to be, that will be an error From 271367ba0f83260cd5cd7dc018063ff2d74806fd Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sun, 8 Jan 2017 22:36:47 -0500 Subject: [PATCH 19/29] Don't add DNSZone task twice --- pkg/model/dns.go | 86 +++++++++++++++++++++------------------------ pkg/model/names.go | 7 +++- upup/pkg/fi/task.go | 25 +++++++++++++ 3 files changed, 72 insertions(+), 46 deletions(-) diff --git a/pkg/model/dns.go b/pkg/model/dns.go index 8f3b45eb58..feb91334b9 100644 --- a/pkg/model/dns.go +++ b/pkg/model/dns.go @@ -17,6 +17,8 @@ limitations under the License. package model import ( + "fmt" + "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awstasks" "strings" @@ -29,56 +31,60 @@ type DNSModelBuilder struct { var _ fi.ModelBuilder = &DNSModelBuilder{} +func (b *DNSModelBuilder) ensureDNSZone(c *fi.ModelBuilderContext) error { + // Configuration for a DNS zone + dnsZone := &awstasks.DNSZone{ + Name: s(b.NameForDNSZone()), + } + + topology := b.Cluster.Spec.Topology + if topology != nil && topology.DNS != nil { + switch topology.DNS.Type { + case kops.DNSTypePublic: + // Ignore + + case kops.DNSTypePrivate: + dnsZone.Private = fi.Bool(true) + dnsZone.PrivateVPC = b.LinkToVPC() + + default: + return fmt.Errorf("Unknown DNS type %q", topology.DNS.Type) + } + } + + if !strings.Contains(b.Cluster.Spec.DNSZone, ".") { + // Looks like a hosted zone ID + dnsZone.ZoneID = s(b.Cluster.Spec.DNSZone) + } else { + // Looks like a normal DNS name + dnsZone.DNSName = s(b.Cluster.Spec.DNSZone) + } + + return c.EnsureTask(dnsZone) +} + func (b *DNSModelBuilder) Build(c *fi.ModelBuilderContext) error { // Add a HostedZone if we are going to publish a dns record that depends on it if b.UsePrivateDNS() { - // UsePrivateDNS is only exposed as a feature flag currently - // TODO: We may still need a public zone to publish an ELB - // Check to see if we are using a bastion DNS record that points to the hosted zone // If we are, we need to make sure we include the hosted zone as a task - // Configuration for a DNS zone, attached to our VPC - dnsZone := &awstasks.DNSZone{ - Name: s("private-" + b.Cluster.Spec.DNSZone), - Private: fi.Bool(true), - PrivateVPC: b.LinkToVPC(), + if err := b.ensureDNSZone(c); err != nil { + return err } - - if !strings.Contains(b.Cluster.Spec.DNSZone, ".") { - // Looks like a hosted zone ID - dnsZone.ZoneID = s(b.Cluster.Spec.DNSZone) - } else { - // Looks like a normal ddns name - dnsZone.DNSName = s(b.Cluster.Spec.DNSZone) - } - - c.AddTask(dnsZone) } if b.UseLoadBalancerForAPI() { // This will point our DNS to the load balancer, and put the pieces // together for kubectl to be work - // Configuration for a DNS name for the master - dnsZone := &awstasks.DNSZone{ - Name: s(b.Cluster.Spec.DNSZone), - Private: fi.Bool(false), + if err := b.ensureDNSZone(c); err != nil { + return err } - if !strings.Contains(b.Cluster.Spec.DNSZone, ".") { - // Looks like a hosted zone ID - dnsZone.ZoneID = s(b.Cluster.Spec.DNSZone) - } else { - // Looks like a normal ddns name - dnsZone.DNSName = s(b.Cluster.Spec.DNSZone) - } - - c.AddTask(dnsZone) - apiDnsName := &awstasks.DNSName{ Name: s(b.Cluster.Spec.MasterPublicName), - Zone: &awstasks.DNSZone{Name: s(b.Cluster.Spec.DNSZone)}, + Zone: b.LinkToDNSZone(), ResourceType: s("A"), TargetLoadBalancer: b.LinkToELB("api"), } @@ -89,20 +95,10 @@ func (b *DNSModelBuilder) Build(c *fi.ModelBuilderContext) error { // Pulling this down into it's own if statement. The DNS configuration here // is similar to others, but I would like to keep it on it's own in case we need // to change anything. - dnsZone := &awstasks.DNSZone{ - Name: s(b.Cluster.Spec.DNSZone), - Private: fi.Bool(false), - } - if !strings.Contains(b.Cluster.Spec.DNSZone, ".") { - // Looks like a hosted zone ID - dnsZone.ZoneID = s(b.Cluster.Spec.DNSZone) - } else { - // Looks like a normal ddns name - dnsZone.DNSName = s(b.Cluster.Spec.DNSZone) + if err := b.ensureDNSZone(c); err != nil { + return err } - - c.AddTask(dnsZone) } return nil diff --git a/pkg/model/names.go b/pkg/model/names.go index b3fea3a6b3..4b1a3916dc 100644 --- a/pkg/model/names.go +++ b/pkg/model/names.go @@ -90,10 +90,15 @@ func (b *KopsModelContext) LinkToVPC() *awstasks.VPC { } func (b *KopsModelContext) LinkToDNSZone() *awstasks.DNSZone { - name := b.Cluster.Spec.DNSZone + name := b.NameForDNSZone() return &awstasks.DNSZone{Name: &name} } +func (b *KopsModelContext) NameForDNSZone() string { + name := b.Cluster.Spec.DNSZone + return name +} + func (b *KopsModelContext) IAMName(role kops.InstanceGroupRole) string { var name string diff --git a/upup/pkg/fi/task.go b/upup/pkg/fi/task.go index 57a36a6e2e..9a25514db4 100644 --- a/upup/pkg/fi/task.go +++ b/upup/pkg/fi/task.go @@ -19,6 +19,7 @@ package fi import ( "fmt" "github.com/golang/glog" + "reflect" "strings" ) @@ -57,6 +58,30 @@ func (c *ModelBuilderContext) AddTask(task Task) { c.Tasks[key] = task } +// EnsureTask ensures that the specified task is configured. +// It adds the task if it does not already exist. +// If it does exist, it verifies that the existing task reflect.DeepEqual the new task, +// if they are different an error is returned. +func (c *ModelBuilderContext) EnsureTask(task Task) error { + key := buildTaskKey(task) + + existing, found := c.Tasks[key] + if found { + if reflect.DeepEqual(task, existing) { + glog.V(8).Infof("EnsureTask ignoring identical ") + return nil + } else { + glog.Warningf("EnsureTask found task mismatch for %q", key) + glog.Warningf("\tExisting: %v", existing) + glog.Warningf("\tNew: %v", task) + + return fmt.Errorf("cannot add different task with same key %q", key) + } + } + c.Tasks[key] = task + return nil +} + func buildTaskKey(task Task) string { hasName, ok := task.(HasName) if !ok { From eda2601c5e6829ed138a92d8d301e49d73498fbe Mon Sep 17 00:00:00 2001 From: "Olve S. Hansen" Date: Mon, 9 Jan 2017 16:28:02 +0100 Subject: [PATCH 20/29] Unclear example, added `spec:` to clarify section The example was missing which section the labels should be added to. --- docs/labels.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/labels.md b/docs/labels.md index 68a920a39b..2319ad494d 100644 --- a/docs/labels.md +++ b/docs/labels.md @@ -15,6 +15,7 @@ An example: ``` ... +spec: nodeLabels: spot: "false" cloudLabels: From ec1e99f1d224d08b943744e8201fc9953b3713bd Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Thu, 29 Dec 2016 13:10:03 -0500 Subject: [PATCH 21/29] Lock down master security group rules --- pkg/model/firewall.go | 49 +++++++++++++++--- .../fi/cloudup/awstasks/securitygrouprule.go | 50 +++++++++++++++++-- 2 files changed, 88 insertions(+), 11 deletions(-) diff --git a/pkg/model/firewall.go b/pkg/model/firewall.go index bf99bb10af..98033adc1c 100644 --- a/pkg/model/firewall.go +++ b/pkg/model/firewall.go @@ -17,6 +17,7 @@ limitations under the License. package model import ( + "fmt" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awstasks" @@ -73,14 +74,50 @@ func (b *FirewallModelBuilder) buildNodeRules(c *fi.ModelBuilderContext) error { c.AddTask(t) } - // Nodes can talk to master nodes - { - t := &awstasks.SecurityGroupRule{ - Name: s("all-node-to-master"), + // TODO: We need to remove the ALL rule + //W1229 12:32:22.300132 9003 executor.go:109] error running task "SecurityGroupRule/node-to-master-443" (9m58s remaining to succeed): error creating SecurityGroupIngress: InvalidPermission.Duplicate: the specified rule "peer: sg-f6b1a68b, ALL, ALLOW" already exists + //status code: 400, request id: 6a69627f-9a26-4bd0-b294-a9a96f89bc46 + + udpPorts := []int64{} + tcpPorts := []int64{} + + // allow access to API + tcpPorts = append(tcpPorts, 443) + + // allow cadvisor + tcpPorts = append(tcpPorts, 4194) + + if b.Cluster.Spec.Networking != nil { + if b.Cluster.Spec.Networking.Kopeio != nil { + // VXLAN over UDP + udpPorts = append(udpPorts, 4789) + } + + if b.Cluster.Spec.Networking.Weave != nil { + // VXLAN over UDP + udpPorts = append(udpPorts, 4789) + } + } + + for _, udpPort := range udpPorts { + c.AddTask(&awstasks.SecurityGroupRule{ + Name: s(fmt.Sprintf("node-to-master-udp-%d", udpPort)), SecurityGroup: b.LinkToSecurityGroup(kops.InstanceGroupRoleMaster), SourceGroup: b.LinkToSecurityGroup(kops.InstanceGroupRoleNode), - } - c.AddTask(t) + FromPort: i64(udpPort), + ToPort: i64(udpPort), + Protocol: s("udp"), + }) + } + for _, tcpPort := range tcpPorts { + c.AddTask(&awstasks.SecurityGroupRule{ + Name: s(fmt.Sprintf("node-to-master-tcp-%d", tcpPort)), + SecurityGroup: b.LinkToSecurityGroup(kops.InstanceGroupRoleMaster), + SourceGroup: b.LinkToSecurityGroup(kops.InstanceGroupRoleNode), + FromPort: i64(tcpPort), + ToPort: i64(tcpPort), + Protocol: s("tcp"), + }) } return nil diff --git a/upup/pkg/fi/cloudup/awstasks/securitygrouprule.go b/upup/pkg/fi/cloudup/awstasks/securitygrouprule.go index 97600b5cb7..b9f25fff0e 100644 --- a/upup/pkg/fi/cloudup/awstasks/securitygrouprule.go +++ b/upup/pkg/fi/cloudup/awstasks/securitygrouprule.go @@ -25,6 +25,8 @@ import ( "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/upup/pkg/fi/cloudup/terraform" + "k8s.io/kubernetes/pkg/util/validation/field" + "strings" ) //go:generate fitask -type=SecurityGroupRule @@ -173,15 +175,51 @@ func (e *SecurityGroupRule) Run(c *fi.Context) error { func (_ *SecurityGroupRule) CheckChanges(a, e, changes *SecurityGroupRule) error { if a == nil { if e.SecurityGroup == nil { - return fi.RequiredField("SecurityGroup") + return field.Required(field.NewPath("SecurityGroup"), "") } } + + if e.FromPort != nil && e.Protocol == nil { + return field.Required(field.NewPath("Protocol"), "Protocol must be specified with FromPort") + } + if e.ToPort != nil && e.Protocol == nil { + return field.Required(field.NewPath("Protocol"), "Protocol must be specified with ToPort") + } + return nil } -func (_ *SecurityGroupRule) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *SecurityGroupRule) error { - if a == nil { +// Description returns a human readable summary of the security group rule +func (e *SecurityGroupRule) Description() string { + var description []string + if e.Protocol != nil { + description = append(description, fmt.Sprintf("protocol=%s", *e.Protocol)) + } + + if e.FromPort != nil { + description = append(description, fmt.Sprintf("fromPort=%d", *e.FromPort)) + } + + if e.ToPort != nil { + description = append(description, fmt.Sprintf("toPort=%d", *e.ToPort)) + } + + if e.SourceGroup != nil { + description = append(description, fmt.Sprintf("sourceGroup=%s", fi.StringValue(e.SourceGroup.ID))) + } + + if e.CIDR != nil { + description = append(description, fmt.Sprintf("cidr=%s", *e.CIDR)) + } + + return strings.Join(description, " ") +} + +func (_ *SecurityGroupRule) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *SecurityGroupRule) error { + name := fi.StringValue(e.Name) + + if a == nil { protocol := e.Protocol if protocol == nil { protocol = aws.String("-1") @@ -206,13 +244,15 @@ func (_ *SecurityGroupRule) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Secu } } + description := e.Description() + if fi.BoolValue(e.Egress) { request := &ec2.AuthorizeSecurityGroupEgressInput{ GroupId: e.SecurityGroup.ID, } request.IpPermissions = []*ec2.IpPermission{ipPermission} - glog.V(2).Infof("Calling EC2 AuthorizeSecurityGroupEgress") + glog.V(2).Infof("%s: Calling EC2 AuthorizeSecurityGroupEgress (%s)", name, description) _, err := t.Cloud.EC2().AuthorizeSecurityGroupEgress(request) if err != nil { return fmt.Errorf("error creating SecurityGroupEgress: %v", err) @@ -223,7 +263,7 @@ func (_ *SecurityGroupRule) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Secu } request.IpPermissions = []*ec2.IpPermission{ipPermission} - glog.V(2).Infof("Calling EC2 AuthorizeSecurityGroupIngress") + glog.V(2).Infof("%s: Calling EC2 AuthorizeSecurityGroupIngress (%s)", name, description) _, err := t.Cloud.EC2().AuthorizeSecurityGroupIngress(request) if err != nil { return fmt.Errorf("error creating SecurityGroupIngress: %v", err) From a52f1e7342f8bbd956387b92ec6b2c0f87267af4 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 3 Jan 2017 10:28:08 -0500 Subject: [PATCH 22/29] Security rules for calico & weave --- pkg/model/firewall.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/pkg/model/firewall.go b/pkg/model/firewall.go index 98033adc1c..35c6058b88 100644 --- a/pkg/model/firewall.go +++ b/pkg/model/firewall.go @@ -80,6 +80,7 @@ func (b *FirewallModelBuilder) buildNodeRules(c *fi.ModelBuilderContext) error { udpPorts := []int64{} tcpPorts := []int64{} + protocols := []string{} // allow access to API tcpPorts = append(tcpPorts, 443) @@ -94,8 +95,15 @@ func (b *FirewallModelBuilder) buildNodeRules(c *fi.ModelBuilderContext) error { } if b.Cluster.Spec.Networking.Weave != nil { - // VXLAN over UDP - udpPorts = append(udpPorts, 4789) + udpPorts = append(udpPorts, 6783) + tcpPorts = append(tcpPorts, 6783) + udpPorts = append(udpPorts, 6784) + } + + if b.Cluster.Spec.Networking.Calico != nil { + tcpPorts = append(tcpPorts, 179) + // Protocol 4 is IPIP + protocols = append(protocols, "4") } } @@ -119,6 +127,14 @@ func (b *FirewallModelBuilder) buildNodeRules(c *fi.ModelBuilderContext) error { Protocol: s("tcp"), }) } + for _, protocol := range protocols { + c.AddTask(&awstasks.SecurityGroupRule{ + Name: s(fmt.Sprintf("node-to-master-protocol-%s", protocol)), + SecurityGroup: b.LinkToSecurityGroup(kops.InstanceGroupRoleMaster), + SourceGroup: b.LinkToSecurityGroup(kops.InstanceGroupRoleNode), + Protocol: s(protocol), + }) + } return nil } From 71c52db994e88aca4b024805c512f08ba7bdf643 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sun, 8 Jan 2017 14:20:21 -0500 Subject: [PATCH 23/29] Open etcd for calico --- pkg/model/firewall.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pkg/model/firewall.go b/pkg/model/firewall.go index 35c6058b88..dd479e8ea7 100644 --- a/pkg/model/firewall.go +++ b/pkg/model/firewall.go @@ -18,6 +18,7 @@ package model import ( "fmt" + "github.com/golang/glog" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awstasks" @@ -101,6 +102,11 @@ func (b *FirewallModelBuilder) buildNodeRules(c *fi.ModelBuilderContext) error { } if b.Cluster.Spec.Networking.Calico != nil { + // Calico needs to access etcd + // TODO: Remove, replace with etcd in calico manifest + glog.Warningf("Opening etcd port on masters for access from the nodes, for calico. This is unsafe in untrusted environments.") + tcpPorts = append(tcpPorts, 4001) + tcpPorts = append(tcpPorts, 179) // Protocol 4 is IPIP protocols = append(protocols, "4") @@ -148,8 +154,14 @@ func (b *FirewallModelBuilder) buildMasterRules(c *fi.ModelBuilderContext) error VPC: b.LinkToVPC(), Description: s("Security group for masters"), RemoveExtraRules: []string{ - "port=22", - "port=443", + "port=22", // SSH + "port=443", // k8s api + "port=4001", // etcd + "port=4789", // VXLAN + "port=179", // Calico + + // TODO: UDP vs TCP + // TODO: Protocol 4 for calico }, } c.AddTask(t) From d2480299a95c6a985373f0d40b7e24418ac0ec0f Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sun, 8 Jan 2017 15:28:40 -0500 Subject: [PATCH 24/29] Update expected terraform output --- tests/integration/minimal-141/kubernetes.tf | 27 ++++++---- tests/integration/minimal/kubernetes.tf | 27 ++++++---- tests/integration/privatecalico/kubernetes.tf | 54 +++++++++++++++---- tests/integration/privateweave/kubernetes.tf | 54 +++++++++++++++---- 4 files changed, 126 insertions(+), 36 deletions(-) diff --git a/tests/integration/minimal-141/kubernetes.tf b/tests/integration/minimal-141/kubernetes.tf index 2ba75a9f5c..e63b940aee 100644 --- a/tests/integration/minimal-141/kubernetes.tf +++ b/tests/integration/minimal-141/kubernetes.tf @@ -214,15 +214,6 @@ resource "aws_security_group_rule" "all-master-to-node" { protocol = "-1" } -resource "aws_security_group_rule" "all-node-to-master" { - type = "ingress" - security_group_id = "${aws_security_group.masters-minimal-141-example-com.id}" - source_security_group_id = "${aws_security_group.nodes-minimal-141-example-com.id}" - from_port = 0 - to_port = 0 - protocol = "-1" -} - resource "aws_security_group_rule" "all-node-to-node" { type = "ingress" security_group_id = "${aws_security_group.nodes-minimal-141-example-com.id}" @@ -259,6 +250,24 @@ resource "aws_security_group_rule" "node-egress" { cidr_blocks = ["0.0.0.0/0"] } +resource "aws_security_group_rule" "node-to-master-tcp-4194" { + type = "ingress" + security_group_id = "${aws_security_group.masters-minimal-141-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-minimal-141-example-com.id}" + from_port = 4194 + to_port = 4194 + protocol = "tcp" +} + +resource "aws_security_group_rule" "node-to-master-tcp-443" { + type = "ingress" + security_group_id = "${aws_security_group.masters-minimal-141-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-minimal-141-example-com.id}" + from_port = 443 + to_port = 443 + protocol = "tcp" +} + resource "aws_security_group_rule" "ssh-external-to-master-0-0-0-0--0" { type = "ingress" security_group_id = "${aws_security_group.masters-minimal-141-example-com.id}" diff --git a/tests/integration/minimal/kubernetes.tf b/tests/integration/minimal/kubernetes.tf index 738e21687e..0204f39e79 100644 --- a/tests/integration/minimal/kubernetes.tf +++ b/tests/integration/minimal/kubernetes.tf @@ -214,15 +214,6 @@ resource "aws_security_group_rule" "all-master-to-node" { protocol = "-1" } -resource "aws_security_group_rule" "all-node-to-master" { - type = "ingress" - security_group_id = "${aws_security_group.masters-minimal-example-com.id}" - source_security_group_id = "${aws_security_group.nodes-minimal-example-com.id}" - from_port = 0 - to_port = 0 - protocol = "-1" -} - resource "aws_security_group_rule" "all-node-to-node" { type = "ingress" security_group_id = "${aws_security_group.nodes-minimal-example-com.id}" @@ -259,6 +250,24 @@ resource "aws_security_group_rule" "node-egress" { cidr_blocks = ["0.0.0.0/0"] } +resource "aws_security_group_rule" "node-to-master-tcp-4194" { + type = "ingress" + security_group_id = "${aws_security_group.masters-minimal-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-minimal-example-com.id}" + from_port = 4194 + to_port = 4194 + protocol = "tcp" +} + +resource "aws_security_group_rule" "node-to-master-tcp-443" { + type = "ingress" + security_group_id = "${aws_security_group.masters-minimal-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-minimal-example-com.id}" + from_port = 443 + to_port = 443 + protocol = "tcp" +} + resource "aws_security_group_rule" "ssh-external-to-master-0-0-0-0--0" { type = "ingress" security_group_id = "${aws_security_group.masters-minimal-example-com.id}" diff --git a/tests/integration/privatecalico/kubernetes.tf b/tests/integration/privatecalico/kubernetes.tf index 6bdbafd583..f4b9dc8518 100644 --- a/tests/integration/privatecalico/kubernetes.tf +++ b/tests/integration/privatecalico/kubernetes.tf @@ -397,15 +397,6 @@ resource "aws_security_group_rule" "all-master-to-node" { protocol = "-1" } -resource "aws_security_group_rule" "all-node-to-master" { - type = "ingress" - security_group_id = "${aws_security_group.masters-privatecalico-example-com.id}" - source_security_group_id = "${aws_security_group.nodes-privatecalico-example-com.id}" - from_port = 0 - to_port = 0 - protocol = "-1" -} - resource "aws_security_group_rule" "all-node-to-node" { type = "ingress" security_group_id = "${aws_security_group.nodes-privatecalico-example-com.id}" @@ -496,6 +487,51 @@ resource "aws_security_group_rule" "node-egress" { cidr_blocks = ["0.0.0.0/0"] } +resource "aws_security_group_rule" "node-to-master-protocol-4" { + type = "ingress" + security_group_id = "${aws_security_group.masters-privatecalico-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-privatecalico-example-com.id}" + from_port = 0 + to_port = 65535 + protocol = "4" +} + +resource "aws_security_group_rule" "node-to-master-tcp-179" { + type = "ingress" + security_group_id = "${aws_security_group.masters-privatecalico-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-privatecalico-example-com.id}" + from_port = 179 + to_port = 179 + protocol = "tcp" +} + +resource "aws_security_group_rule" "node-to-master-tcp-4001" { + type = "ingress" + security_group_id = "${aws_security_group.masters-privatecalico-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-privatecalico-example-com.id}" + from_port = 4001 + to_port = 4001 + protocol = "tcp" +} + +resource "aws_security_group_rule" "node-to-master-tcp-4194" { + type = "ingress" + security_group_id = "${aws_security_group.masters-privatecalico-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-privatecalico-example-com.id}" + from_port = 4194 + to_port = 4194 + protocol = "tcp" +} + +resource "aws_security_group_rule" "node-to-master-tcp-443" { + type = "ingress" + security_group_id = "${aws_security_group.masters-privatecalico-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-privatecalico-example-com.id}" + from_port = 443 + to_port = 443 + protocol = "tcp" +} + resource "aws_security_group_rule" "ssh-elb-to-bastion" { type = "ingress" security_group_id = "${aws_security_group.bastion-privatecalico-example-com.id}" diff --git a/tests/integration/privateweave/kubernetes.tf b/tests/integration/privateweave/kubernetes.tf index 959263c98c..dec2a09410 100644 --- a/tests/integration/privateweave/kubernetes.tf +++ b/tests/integration/privateweave/kubernetes.tf @@ -397,15 +397,6 @@ resource "aws_security_group_rule" "all-master-to-node" { protocol = "-1" } -resource "aws_security_group_rule" "all-node-to-master" { - type = "ingress" - security_group_id = "${aws_security_group.masters-privateweave-example-com.id}" - source_security_group_id = "${aws_security_group.nodes-privateweave-example-com.id}" - from_port = 0 - to_port = 0 - protocol = "-1" -} - resource "aws_security_group_rule" "all-node-to-node" { type = "ingress" security_group_id = "${aws_security_group.nodes-privateweave-example-com.id}" @@ -496,6 +487,51 @@ resource "aws_security_group_rule" "node-egress" { cidr_blocks = ["0.0.0.0/0"] } +resource "aws_security_group_rule" "node-to-master-tcp-4194" { + type = "ingress" + security_group_id = "${aws_security_group.masters-privateweave-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-privateweave-example-com.id}" + from_port = 4194 + to_port = 4194 + protocol = "tcp" +} + +resource "aws_security_group_rule" "node-to-master-tcp-443" { + type = "ingress" + security_group_id = "${aws_security_group.masters-privateweave-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-privateweave-example-com.id}" + from_port = 443 + to_port = 443 + protocol = "tcp" +} + +resource "aws_security_group_rule" "node-to-master-tcp-6783" { + type = "ingress" + security_group_id = "${aws_security_group.masters-privateweave-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-privateweave-example-com.id}" + from_port = 6783 + to_port = 6783 + protocol = "tcp" +} + +resource "aws_security_group_rule" "node-to-master-udp-6783" { + type = "ingress" + security_group_id = "${aws_security_group.masters-privateweave-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-privateweave-example-com.id}" + from_port = 6783 + to_port = 6783 + protocol = "udp" +} + +resource "aws_security_group_rule" "node-to-master-udp-6784" { + type = "ingress" + security_group_id = "${aws_security_group.masters-privateweave-example-com.id}" + source_security_group_id = "${aws_security_group.nodes-privateweave-example-com.id}" + from_port = 6784 + to_port = 6784 + protocol = "udp" +} + resource "aws_security_group_rule" "ssh-elb-to-bastion" { type = "ingress" security_group_id = "${aws_security_group.bastion-privateweave-example-com.id}" From 41495f36fe2ee6d06925d9bdf9947a5295f5f1b3 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sun, 8 Jan 2017 23:55:17 -0500 Subject: [PATCH 25/29] Default bastion name should be bastion. bastion- is not necessarily in the same hosted zone, nor is bastion-, and bastion- is not necessarily unique across clusters. --- cmd/kops/create_cluster.go | 12 ++---------- .../create_cluster/private/expected-v1alpha1.yaml | 2 +- .../create_cluster/private/expected-v1alpha2.yaml | 2 +- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index 96b49ebaf5..9dae956580 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -493,16 +493,8 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e bastionGroup.ObjectMeta.Name = "bastions" instanceGroups = append(instanceGroups, bastionGroup) - // Logic to handle default bastion names - if c.DNSZone != "" { - cluster.Spec.Topology.Bastion = &api.BastionSpec{ - BastionPublicName: "bastion-" + c.DNSZone, - } - } else { - // Use default zone and cluster name - cluster.Spec.Topology.Bastion = &api.BastionSpec{ - BastionPublicName: "bastion-" + clusterName, - } + cluster.Spec.Topology.Bastion = &api.BastionSpec{ + BastionPublicName: "bastion." + clusterName, } } diff --git a/tests/integration/create_cluster/private/expected-v1alpha1.yaml b/tests/integration/create_cluster/private/expected-v1alpha1.yaml index e6a29a0731..b892fd4554 100644 --- a/tests/integration/create_cluster/private/expected-v1alpha1.yaml +++ b/tests/integration/create_cluster/private/expected-v1alpha1.yaml @@ -28,7 +28,7 @@ spec: topology: bastion: enable: true - name: bastion-private.example.com + name: bastion.private.example.com dns: type: Public masters: private diff --git a/tests/integration/create_cluster/private/expected-v1alpha2.yaml b/tests/integration/create_cluster/private/expected-v1alpha2.yaml index 6fdca7646d..98396d2ad6 100644 --- a/tests/integration/create_cluster/private/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/private/expected-v1alpha2.yaml @@ -36,7 +36,7 @@ spec: zone: us-test-1a topology: bastion: - bastionPublicName: bastion-private.example.com + bastionPublicName: bastion.private.example.com dns: type: Public masters: private From 71401177808e0923a4b2b988a9b5369d555876a6 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 9 Jan 2017 10:57:43 -0500 Subject: [PATCH 26/29] Separate protocol rule naming from AWS rules --- pkg/model/firewall.go | 25 +++++++++++++++---- tests/integration/privatecalico/kubernetes.tf | 2 +- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/pkg/model/firewall.go b/pkg/model/firewall.go index dd479e8ea7..b29d9482f0 100644 --- a/pkg/model/firewall.go +++ b/pkg/model/firewall.go @@ -22,6 +22,13 @@ import ( "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awstasks" + "strconv" +) + +type Protocol int + +const ( + ProtocolIPIP Protocol = 4 ) // FirewallModelBuilder configures firewall network objects @@ -81,7 +88,7 @@ func (b *FirewallModelBuilder) buildNodeRules(c *fi.ModelBuilderContext) error { udpPorts := []int64{} tcpPorts := []int64{} - protocols := []string{} + protocols := []Protocol{} // allow access to API tcpPorts = append(tcpPorts, 443) @@ -108,8 +115,7 @@ func (b *FirewallModelBuilder) buildNodeRules(c *fi.ModelBuilderContext) error { tcpPorts = append(tcpPorts, 4001) tcpPorts = append(tcpPorts, 179) - // Protocol 4 is IPIP - protocols = append(protocols, "4") + protocols = append(protocols, ProtocolIPIP) } } @@ -134,11 +140,20 @@ func (b *FirewallModelBuilder) buildNodeRules(c *fi.ModelBuilderContext) error { }) } for _, protocol := range protocols { + awsName := strconv.Itoa(int(protocol)) + name := awsName + switch protocol { + case ProtocolIPIP: + name = "ipip" + default: + glog.Warningf("unknown protocol %q - naming by number", awsName) + } + c.AddTask(&awstasks.SecurityGroupRule{ - Name: s(fmt.Sprintf("node-to-master-protocol-%s", protocol)), + Name: s("node-to-master-protocol-" + name), SecurityGroup: b.LinkToSecurityGroup(kops.InstanceGroupRoleMaster), SourceGroup: b.LinkToSecurityGroup(kops.InstanceGroupRoleNode), - Protocol: s(protocol), + Protocol: s(awsName), }) } diff --git a/tests/integration/privatecalico/kubernetes.tf b/tests/integration/privatecalico/kubernetes.tf index f4b9dc8518..4e98dc9b90 100644 --- a/tests/integration/privatecalico/kubernetes.tf +++ b/tests/integration/privatecalico/kubernetes.tf @@ -487,7 +487,7 @@ resource "aws_security_group_rule" "node-egress" { cidr_blocks = ["0.0.0.0/0"] } -resource "aws_security_group_rule" "node-to-master-protocol-4" { +resource "aws_security_group_rule" "node-to-master-protocol-ipip" { type = "ingress" security_group_id = "${aws_security_group.masters-privatecalico-example-com.id}" source_security_group_id = "${aws_security_group.nodes-privatecalico-example-com.id}" From 2ee209dea5dfb0f9a0abbb7607a113c59953e4c3 Mon Sep 17 00:00:00 2001 From: Zach Loafman Date: Mon, 9 Jan 2017 11:04:58 -0800 Subject: [PATCH 27/29] kops: Always enable CI versioning on gcs-upload-ci target Follow-on to https://github.com/kubernetes/kops/pull/1380, relies on https://www.gnu.org/software/make/manual/make.html#Target_002dspecific to override the target used for CI upload. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 094db90a73..014c217925 100644 --- a/Makefile +++ b/Makefile @@ -140,11 +140,11 @@ upload: kops version-dist aws s3 sync --acl public-read .build/upload/ ${S3_BUCKET} gcs-upload: version-dist - @echo "== Logging gcloud info ==" - @gcloud info @echo "== Uploading kops ==" gsutil -h "Cache-Control:private, max-age=0, no-transform" -m cp -n -r .build/upload/kops/* ${GCS_LOCATION} +# In CI testing, always upload the CI version. +gcs-publish-ci: VERSION := git-$(shell git describe --always) gcs-publish-ci: gcs-upload echo "${GCS_URL}/${VERSION}" > .build/upload/${LATEST_FILE} gsutil -h "Cache-Control:private, max-age=0, no-transform" cp .build/upload/${LATEST_FILE} ${GCS_LOCATION} From 6581f2732b7d982a3815b12cc92caf74ae9a2864 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 9 Jan 2017 20:51:13 -0500 Subject: [PATCH 28/29] Bump to 4.4.41 image, with more regions --- channels/alpha | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/alpha b/channels/alpha index 1970df278f..8ecab7d103 100644 --- a/channels/alpha +++ b/channels/alpha @@ -1,6 +1,6 @@ spec: images: - - name: kope.io/k8s-1.5-debian-jessie-amd64-hvm-ebs-2016-12-19 + - name: kope.io/k8s-1.5-debian-jessie-amd64-hvm-ebs-2017-01-09 providerID: aws cluster: kubernetesVersion: v1.5.1 From 58621dceb191a5ecd78b3ecddb4b81d8eebdd757 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 9 Jan 2017 20:51:49 -0500 Subject: [PATCH 29/29] Turn down no-change logging in dns-controller --- dns-controller/pkg/dns/dnscontroller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dns-controller/pkg/dns/dnscontroller.go b/dns-controller/pkg/dns/dnscontroller.go index 3ec8f716e1..0c00f0156e 100644 --- a/dns-controller/pkg/dns/dnscontroller.go +++ b/dns-controller/pkg/dns/dnscontroller.go @@ -146,7 +146,7 @@ func (c *DNSController) snapshotIfChangedAndReady() *snapshot { aliasTargets := make(map[string][]Record) if c.lastSuccessfulSnapshot != nil && s.changeCount == c.lastSuccessfulSnapshot.changeCount { - glog.V(4).Infof("No changes since DNS values last successfully applied") + glog.V(6).Infof("No changes since DNS values last successfully applied") return nil }