Change default port for memberlist from 3997

We had a port collision on 3997; change the default memberlist ports to avoid the collision (we haven't shipped a release with this in it). Also create a go file so that we can use constants to keep track of our port numbers, rather than magic values.
2019-10-13 22:43:33 -04:00 · 2019-10-13 22:43:33 -04:00 · b6862103bb
parent 32a6c08b8e
commit b6862103bb
14 changed files with 124 additions and 39 deletions
--- a/dns-controller/cmd/dns-controller/BUILD.bazel
+++ b/dns-controller/cmd/dns-controller/BUILD.bazel
@ -15,6 +15,7 @@ go_library(
        "//dnsprovider/pkg/dnsprovider/providers/coredns:go_default_library",
        "//dnsprovider/pkg/dnsprovider/providers/google/clouddns:go_default_library",
        "//pkg/resources/digitalocean/dns:go_default_library",
+        "//pkg/wellknownports:go_default_library",
        "//protokube/pkg/gossip:go_default_library",
        "//protokube/pkg/gossip/dns:go_default_library",
        "//protokube/pkg/gossip/dns/provider:go_default_library",
--- a/dns-controller/cmd/dns-controller/main.go
+++ b/dns-controller/cmd/dns-controller/main.go
@ -41,6 +41,7 @@ import (
 	k8scoredns "k8s.io/kops/dnsprovider/pkg/dnsprovider/providers/coredns"
 	_ "k8s.io/kops/dnsprovider/pkg/dnsprovider/providers/google/clouddns"
 	_ "k8s.io/kops/pkg/resources/digitalocean/dns"
+	"k8s.io/kops/pkg/wellknownports"
 	"k8s.io/kops/protokube/pkg/gossip"
 	gossipdns "k8s.io/kops/protokube/pkg/gossip/dns"
 	gossipdnsprovider "k8s.io/kops/protokube/pkg/gossip/dns/provider"
@ -70,10 +71,10 @@ func main() {
 	flags.StringSliceVarP(&zones, "zone", "z", []string{}, "Configure permitted zones and their mappings")
 	flags.StringVar(&dnsProviderID, "dns", "aws-route53", "DNS provider we should use (aws-route53, google-clouddns, digitalocean, coredns, gossip)")
 	flag.StringVar(&gossipProtocol, "gossip-protocol", "mesh", "mesh/memberlist")
-	flags.StringVar(&gossipListen, "gossip-listen", "0.0.0.0:3998", "The address on which to listen if gossip is enabled")
+	flags.StringVar(&gossipListen, "gossip-listen", fmt.Sprintf("0.0.0.0:%d", wellknownports.DNSControllerGossipWeaveMesh), "The address on which to listen if gossip is enabled")
 	flags.StringVar(&gossipSecret, "gossip-secret", gossipSecret, "Secret to use to secure gossip")
 	flag.StringVar(&gossipProtocolSecondary, "gossip-protocol-secondary", "", "mesh/memberlist")
-	flag.StringVar(&gossipListenSecondary, "gossip-listen-secondary", "0.0.0.0:4000", "address:port on which to bind for gossip")
+	flag.StringVar(&gossipListenSecondary, "gossip-listen-secondary", fmt.Sprintf("0.0.0.0:%d", wellknownports.DNSControllerGossipMemberlist), "address:port on which to bind for gossip")
 	flags.StringVar(&gossipSecretSecondary, "gossip-secret-secondary", gossipSecret, "Secret to use to secure gossip")
 	flags.StringSliceVar(&gossipSeedsSecondary, "gossip-seed-secondary", gossipSeedsSecondary, "If set, will enable gossip zones and seed using the provided addresses")
 	flags.StringVar(&watchNamespace, "watch-namespace", "", "Limits the functionality for pods, services and ingress to specific namespace, by default all")
--- a/docs/development/ports.md
+++ b/docs/development/ports.md
@ -3,15 +3,24 @@
 This document includes the port used by system components,
 so we can avoid port collisions.

-| Port | Description                            |
-|------|----------------------------------------|
-| 22   | SSH                                    |
-| 443  | Kubernetes API                         |
-| 179  | Calico                                 |
-| 2380 | etcd main peering                      |
-| 2381 | etcd events peering                    |
-| 3998 | dns gossip - protokube                 |
-| 3999 | dns gossip - dns-controller            |
-| 4001 | etcd main client                       |
-| 4002 | etcd events client                     |
-| 4789 | VXLAN                                  |
+See also pkg/wellknownports/wellknownports.go
+
+
+| Port | Description                              |
+|------|------------------------------------------|
+| 22   | SSH                                      |
+| 443  | Kubernetes API                           |
+| 179  | Calico                                   |
+| 2380 | etcd main peering                        |
+| 2381 | etcd events peering                      |
+| 3992 | dns gossip - protokube - memberlist      |
+| 3993 | dns gossip - dns-controller - memberlist |
+| 3994 | etcd-manager - main - quarantined        |
+| 3995 | etcd-manager - events - quarantined      |
+| 3996 | etcd-manager - main - grpc               |
+| 3997 | etcd-manager - events - grpc             |
+| 3998 | dns gossip - protokube - weave mesh      |
+| 3999 | dns gossip - dns-controller - weave mesh |
+| 4001 | etcd main client                         |
+| 4002 | etcd events client                       |
+| 4789 | VXLAN                                    |
--- a/hack/.packages
+++ b/hack/.packages
@ -140,6 +140,7 @@ k8s.io/kops/pkg/util/subnet
 k8s.io/kops/pkg/util/templater
 k8s.io/kops/pkg/validation
 k8s.io/kops/pkg/values
+k8s.io/kops/pkg/wellknownports
 k8s.io/kops/protokube/cmd/protokube
 k8s.io/kops/protokube/pkg/etcd
 k8s.io/kops/protokube/pkg/gossip
--- a/pkg/model/components/etcdmanager/BUILD.bazel
+++ b/pkg/model/components/etcdmanager/BUILD.bazel
@ -19,6 +19,7 @@ go_library(
        "//pkg/model:go_default_library",
        "//pkg/model/components:go_default_library",
        "//pkg/urls:go_default_library",
+        "//pkg/wellknownports:go_default_library",
        "//upup/pkg/fi:go_default_library",
        "//upup/pkg/fi/cloudup/awsup:go_default_library",
        "//upup/pkg/fi/cloudup/do:go_default_library",
--- a/pkg/model/components/etcdmanager/model.go
+++ b/pkg/model/components/etcdmanager/model.go
@ -37,6 +37,7 @@ import (
 	"k8s.io/kops/pkg/k8scodecs"
 	"k8s.io/kops/pkg/kubemanifest"
 	"k8s.io/kops/pkg/model"
+	"k8s.io/kops/pkg/wellknownports"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
 	"k8s.io/kops/upup/pkg/fi/cloudup/do"
@ -286,9 +287,9 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster *kops.EtcdClusterSpec) (*v1.Po
 	pod.Labels["k8s-app"] = pod.Name

 	// TODO: Use a socket file for the quarantine port
-	quarantinedClientPort := 3994
+	quarantinedClientPort := wellknownports.EtcdMainQuarantinedClientPort

-	grpcPort := 3996
+	grpcPort := wellknownports.EtcdMainGRPC

 	// The dns suffix logic mirrors the existing logic, so we should be compatible with existing clusters
 	// (etcd makes it difficult to change peer urls, treating it as a cluster event, for reasons unknown)
@ -310,8 +311,8 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster *kops.EtcdClusterSpec) (*v1.Po
 	case "events":
 		clientPort = 4002
 		peerPort = 2381
-		grpcPort = 3997
-		quarantinedClientPort = 3995
+		grpcPort = wellknownports.EtcdEventsGRPC
+		quarantinedClientPort = wellknownports.EtcdEventsQuarantinedClientPort

 	default:
 		return nil, fmt.Errorf("unknown etcd cluster key %q", etcdCluster.Name)
--- a/pkg/model/openstackmodel/BUILD.bazel
+++ b/pkg/model/openstackmodel/BUILD.bazel
@ -16,6 +16,7 @@ go_library(
        "//pkg/apis/kops:go_default_library",
        "//pkg/dns:go_default_library",
        "//pkg/model:go_default_library",
+        "//pkg/wellknownports:go_default_library",
        "//upup/pkg/fi:go_default_library",
        "//upup/pkg/fi/cloudup/openstack:go_default_library",
        "//upup/pkg/fi/cloudup/openstacktasks:go_default_library",
--- a/pkg/model/openstackmodel/firewall.go
+++ b/pkg/model/openstackmodel/firewall.go
@ -22,9 +22,9 @@ import (
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/openstacktasks"

-	//TODO: Replace with klog
 	"k8s.io/klog"
 	"k8s.io/kops/pkg/dns"
+	"k8s.io/kops/pkg/wellknownports"
 )

 const (
@ -438,18 +438,20 @@ func (b *FirewallModelBuilder) addProtokubeRules(c *fi.ModelBuilderContext, sgMa
 		nodeName := b.SecurityGroupName(kops.InstanceGroupRoleNode)
 		masterSG := sgMap[masterName]
 		nodeSG := sgMap[nodeName]
-		protokubeRule := &openstacktasks.SecurityGroupRule{
-			Lifecycle:    b.Lifecycle,
-			Direction:    s(string(rules.DirIngress)),
-			Protocol:     s(string(rules.ProtocolTCP)),
-			EtherType:    s(string(rules.EtherType4)),
-			PortRangeMin: i(3994),
-			PortRangeMax: i(3999),
+		for _, portRange := range wellknownports.DNSGossipPortRanges() {
+			protokubeRule := &openstacktasks.SecurityGroupRule{
+				Lifecycle:    b.Lifecycle,
+				Direction:    s(string(rules.DirIngress)),
+				Protocol:     s(string(rules.ProtocolTCP)),
+				EtherType:    s(string(rules.EtherType4)),
+				PortRangeMin: i(portRange.Min),
+				PortRangeMax: i(portRange.Max),
+			}
+			addDirectionalGroupRule(c, masterSG, nodeSG, protokubeRule)
+			addDirectionalGroupRule(c, nodeSG, masterSG, protokubeRule)
+			addDirectionalGroupRule(c, masterSG, masterSG, protokubeRule)
+			addDirectionalGroupRule(c, nodeSG, nodeSG, protokubeRule)
 		}
-		addDirectionalGroupRule(c, masterSG, nodeSG, protokubeRule)
-		addDirectionalGroupRule(c, nodeSG, masterSG, protokubeRule)
-		addDirectionalGroupRule(c, masterSG, masterSG, protokubeRule)
-		addDirectionalGroupRule(c, nodeSG, nodeSG, protokubeRule)
 	}
 	return nil
 }
--- a/pkg/wellknownports/BUILD.bazel
+++ b/pkg/wellknownports/BUILD.bazel
@ -0,0 +1,8 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["wellknownports.go"],
+    importpath = "k8s.io/kops/pkg/wellknownports",
+    visibility = ["//visibility:public"],
+)
--- a/pkg/wellknownports/wellknownports.go
+++ b/pkg/wellknownports/wellknownports.go
@ -0,0 +1,56 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package wellknownports
+
+const (
+	// EtcdMainQuarantinedClientPort is the port used by etcd when quarantined, for the main etcd
+	EtcdMainQuarantinedClientPort = 3994
+
+	// EtcdEventsQuarantinedClientPort is the port used by etcd when quarantined, for the events etcd
+	EtcdEventsQuarantinedClientPort = 3995
+
+	// EtcdMainGRPC is the GRPC port used by etcd-manager, for the main etcd
+	EtcdMainGRPC = 3996
+
+	// EtcdEventsGRPC is the GRPC port used by etcd-manager, for the events etcd
+	EtcdEventsGRPC = 3997
+
+	// DNSControllerGossipWeaveMesh is the port where dns-controller listens for the weave-mesh backend gossip
+	DNSControllerGossipWeaveMesh = 3998
+
+	// ProtokubeGossipWeaveMesh is the port where protokube listens for the weave-mesh-backed gossip
+	ProtokubeGossipWeaveMesh = 3999
+
+	// ProtokubeGossipMemberlist is the port where protokube listens for the memberlist-backed gossip
+	ProtokubeGossipMemberlist = 4000
+
+	// DNSControllerGossipMemberlist is the port where dns-controller listens for the memberlist-backed gossip
+	DNSControllerGossipMemberlist = 3993
+)
+
+type PortRange struct {
+	Min int
+	Max int
+}
+
+func DNSGossipPortRanges() []PortRange {
+	return []PortRange{
+		// 3993 is used by dns-controller, which is less important, so we might be able to drop it
+		{Min: 3993, Max: 3993},
+		{Min: 3998, Max: 4000},
+	}
+}
--- a/protokube/cmd/protokube/BUILD.bazel
+++ b/protokube/cmd/protokube/BUILD.bazel
@ -14,6 +14,7 @@ go_library(
        "//dnsprovider/pkg/dnsprovider/providers/aws/route53:go_default_library",
        "//dnsprovider/pkg/dnsprovider/providers/coredns:go_default_library",
        "//dnsprovider/pkg/dnsprovider/providers/google/clouddns:go_default_library",
+        "//pkg/wellknownports:go_default_library",
        "//protokube/pkg/gossip:go_default_library",
        "//protokube/pkg/gossip/dns:go_default_library",
        "//protokube/pkg/gossip/memberlist:go_default_library",
--- a/protokube/cmd/protokube/main.go
+++ b/protokube/cmd/protokube/main.go
@ -26,8 +26,11 @@ import (
 	"path"
 	"strings"

+	"github.com/spf13/pflag"
+	"k8s.io/klog"
 	"k8s.io/kops/dns-controller/pkg/dns"
 	"k8s.io/kops/dnsprovider/pkg/dnsprovider"
+	"k8s.io/kops/pkg/wellknownports"
 	"k8s.io/kops/protokube/pkg/gossip"
 	gossipdns "k8s.io/kops/protokube/pkg/gossip/dns"
 	_ "k8s.io/kops/protokube/pkg/gossip/memberlist"
@ -35,8 +38,6 @@ import (
 	"k8s.io/kops/protokube/pkg/protokube"

 	// Load DNS plugins
-	"github.com/spf13/pflag"
-	"k8s.io/klog"
 	_ "k8s.io/kops/dnsprovider/pkg/dnsprovider/providers/aws/route53"
 	k8scoredns "k8s.io/kops/dnsprovider/pkg/dnsprovider/providers/coredns"
 	_ "k8s.io/kops/dnsprovider/pkg/dnsprovider/providers/google/clouddns"
@ -80,10 +81,10 @@ func run() error {
 	flags.IntVar(&dnsUpdateInterval, "dns-update-interval", 5, "Configure interval at which to update DNS records.")
 	flag.StringVar(&flagChannels, "channels", flagChannels, "channels to install")
 	flag.StringVar(&gossipProtocol, "gossip-protocol", "mesh", "mesh/memberlist")
-	flag.StringVar(&gossipListen, "gossip-listen", "0.0.0.0:3999", "address:port on which to bind for gossip")
+	flag.StringVar(&gossipListen, "gossip-listen", fmt.Sprintf("0.0.0.0:%d", wellknownports.ProtokubeGossipWeaveMesh), "address:port on which to bind for gossip")
 	flags.StringVar(&gossipSecret, "gossip-secret", gossipSecret, "Secret to use to secure gossip")
 	flag.StringVar(&gossipProtocolSecondary, "gossip-protocol-secondary", "memberlist", "mesh/memberlist")
-	flag.StringVar(&gossipListenSecondary, "gossip-listen-secondary", "0.0.0.0:4000", "address:port on which to bind for gossip")
+	flag.StringVar(&gossipListenSecondary, "gossip-listen-secondary", fmt.Sprintf("0.0.0.0:%d", wellknownports.ProtokubeGossipMemberlist), "address:port on which to bind for gossip")
 	flags.StringVar(&gossipSecretSecondary, "gossip-secret-secondary", gossipSecret, "Secret to use to secure gossip")
 	flag.StringVar(&peerCA, "peer-ca", peerCA, "Path to a file containing the peer ca in PEM format")
 	flag.StringVar(&peerCert, "peer-cert", peerCert, "Path to a file containing the peer certificate")
--- a/upup/pkg/fi/cloudup/BUILD.bazel
+++ b/upup/pkg/fi/cloudup/BUILD.bazel
@ -56,6 +56,7 @@ go_library(
        "//pkg/resources/spotinst:go_default_library",
        "//pkg/templates:go_default_library",
        "//pkg/util/subnet:go_default_library",
+        "//pkg/wellknownports:go_default_library",
        "//upup/models:go_default_library",
        "//upup/pkg/fi:go_default_library",
        "//upup/pkg/fi/assettasks:go_default_library",
--- a/upup/pkg/fi/cloudup/template_functions.go
+++ b/upup/pkg/fi/cloudup/template_functions.go
@ -44,6 +44,7 @@ import (
 	"k8s.io/kops/pkg/featureflag"
 	"k8s.io/kops/pkg/model"
 	"k8s.io/kops/pkg/resources/spotinst"
+	"k8s.io/kops/pkg/wellknownports"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/gce"
 	"k8s.io/kops/util/pkg/env"
@ -229,7 +230,7 @@ func (tf *TemplateFunctions) DnsControllerArgv() ([]string, error) {
 			if tf.cluster.Spec.DNSControllerGossipConfig.Seed != nil {
 				argv = append(argv, "--gossip-seed="+*tf.cluster.Spec.DNSControllerGossipConfig.Seed)
 			} else {
-				argv = append(argv, "--gossip-seed=127.0.0.1:3999")
+				argv = append(argv, fmt.Sprintf("--gossip-seed=127.0.0.1:%d", wellknownports.ProtokubeGossipWeaveMesh))
 			}

 			if tf.cluster.Spec.DNSControllerGossipConfig.Secondary != nil {
@ -246,16 +247,16 @@ func (tf *TemplateFunctions) DnsControllerArgv() ([]string, error) {
 				if tf.cluster.Spec.DNSControllerGossipConfig.Secondary.Seed != nil {
 					argv = append(argv, "--gossip-seed-secondary="+*tf.cluster.Spec.DNSControllerGossipConfig.Secondary.Seed)
 				} else {
-					argv = append(argv, "--gossip-seed-secondary=127.0.0.1:4000")
+					argv = append(argv, fmt.Sprintf("--gossip-seed-secondary=127.0.0.1:%d", wellknownports.ProtokubeGossipMemberlist))
 				}
 			}
 		} else {
 			// Default to primary mesh and secondary memberlist
-			argv = append(argv, "--gossip-seed=127.0.0.1:3999")
+			argv = append(argv, fmt.Sprintf("--gossip-seed=127.0.0.1:%d", wellknownports.ProtokubeGossipWeaveMesh))

 			argv = append(argv, "--gossip-protocol-secondary=memberlist")
-			argv = append(argv, "--gossip-listen-secondary=0.0.0.0:3997")
-			argv = append(argv, "--gossip-seed-secondary=127.0.0.1:4000")
+			argv = append(argv, fmt.Sprintf("--gossip-listen-secondary=0.0.0.0:%d", wellknownports.DNSControllerGossipMemberlist))
+			argv = append(argv, fmt.Sprintf("--gossip-seed-secondary=127.0.0.1:%d", wellknownports.ProtokubeGossipMemberlist))
 		}
 	} else {
 		switch kops.CloudProviderID(tf.cluster.Spec.CloudProvider) {