Merge pull request #9175 from mtl-wgtwo/calico-ip-detection-method

Enable configuration of the calico IP_AUTODETECTION_METHOD  and IP6_AUTODETECTION_METHOD

go.mod

@@ -97,6 +97,7 @@ require (
 	github.com/zclconf/go-cty v1.3.1
 	go.uber.org/zap v1.10.0
 	golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975
+	golang.org/x/net v0.0.0-20200202094626-16171245cfb2
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9
 	golang.org/x/tools v0.0.0-20191203134012-c197fd4bf371

k8s/crds/kops.k8s.io_clusters.yaml

@@ -2668,6 +2668,22 @@ spec:
                           binary Felix uses Default: Auto (other options: Legacy,
                           NFT)'
                         type: string
+                      ipv4AutoDetectionMethod:
+                        description: 'IPv4AutoDetectionMethod configures how Calico
+                          chooses the IP address used to route between nodes.  This
+                          should be set when the host has multiple interfaces and
+                          it is important to select the interface used. Options: "first-found"
+                          (default), "can-reach=DESTINATION", "interface=INTERFACE-REGEX",
+                          or "skip-interface=INTERFACE-REGEX"'
+                        type: string
+                      ipv6AutoDetectionMethod:
+                        description: 'IPv6AutoDetectionMethod configures how Calico
+                          chooses the IP address used to route between nodes.  This
+                          should be set when the host has multiple interfaces and
+                          it is important to select the interface used. Options: "first-found"
+                          (default), "can-reach=DESTINATION", "interface=INTERFACE-REGEX",
+                          or "skip-interface=INTERFACE-REGEX"'
+                        type: string
                       logSeverityScreen:
                         description: 'LogSeverityScreen lets us set the desired log
                           level. (Default: info)'

pkg/apis/kops/networking.go

@@ -118,6 +118,18 @@ type CalicoNetworkingSpec struct {
 	PrometheusProcessMetricsEnabled bool `json:"prometheusProcessMetricsEnabled,omitempty"`
 	// MajorVersion is the version of Calico to use
 	MajorVersion string `json:"majorVersion,omitempty"`
+	// IPv4AutoDetectionMethod configures how Calico chooses the IP address used to route
+	// between nodes.  This should be set when the host has multiple interfaces
+	// and it is important to select the interface used.
+	// Options: "first-found" (default), "can-reach=DESTINATION",
+	// "interface=INTERFACE-REGEX", or "skip-interface=INTERFACE-REGEX"
+	IPv4AutoDetectionMethod string `json:"ipv4AutoDetectionMethod,omitempty"`
+	// IPv6AutoDetectionMethod configures how Calico chooses the IP address used to route
+	// between nodes.  This should be set when the host has multiple interfaces
+	// and it is important to select the interface used.
+	// Options: "first-found" (default), "can-reach=DESTINATION",
+	// "interface=INTERFACE-REGEX", or "skip-interface=INTERFACE-REGEX"
+	IPv6AutoDetectionMethod string `json:"ipv6AutoDetectionMethod,omitempty"`
 	// IptablesBackend controls which variant of iptables binary Felix uses
 	// Default: Auto (other options: Legacy, NFT)
 	IptablesBackend string `json:"iptablesBackend,omitempty"`

pkg/apis/kops/v1alpha2/networking.go

@@ -123,6 +123,18 @@ type CalicoNetworkingSpec struct {
 	IptablesBackend string `json:"iptablesBackend,omitempty"`
 	// IPIPMode is mode for CALICO_IPV4POOL_IPIP
 	IPIPMode string `json:"ipipMode,omitempty"`
+	// IPv4AutoDetectionMethod configures how Calico chooses the IP address used to route
+	// between nodes.  This should be set when the host has multiple interfaces
+	// and it is important to select the interface used.
+	// Options: "first-found" (default), "can-reach=DESTINATION",
+	// "interface=INTERFACE-REGEX", or "skip-interface=INTERFACE-REGEX"
+	IPv4AutoDetectionMethod string `json:"ipv4AutoDetectionMethod,omitempty"`
+	// IPv6AutoDetectionMethod configures how Calico chooses the IP address used to route
+	// between nodes.  This should be set when the host has multiple interfaces
+	// and it is important to select the interface used.
+	// Options: "first-found" (default), "can-reach=DESTINATION",
+	// "interface=INTERFACE-REGEX", or "skip-interface=INTERFACE-REGEX"
+	IPv6AutoDetectionMethod string `json:"ipv6AutoDetectionMethod,omitempty"`
 	// TyphaPrometheusMetricsEnabled enables Prometheus metrics collection from Typha
 	// (default: false)
 	TyphaPrometheusMetricsEnabled bool `json:"typhaPrometheusMetricsEnabled,omitempty"`

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -1292,6 +1292,8 @@ func autoConvert_v1alpha2_CalicoNetworkingSpec_To_kops_CalicoNetworkingSpec(in *
 	out.MajorVersion = in.MajorVersion
 	out.IptablesBackend = in.IptablesBackend
 	out.IPIPMode = in.IPIPMode
+	out.IPv4AutoDetectionMethod = in.IPv4AutoDetectionMethod
+	out.IPv6AutoDetectionMethod = in.IPv6AutoDetectionMethod
 	out.TyphaPrometheusMetricsEnabled = in.TyphaPrometheusMetricsEnabled
 	out.TyphaPrometheusMetricsPort = in.TyphaPrometheusMetricsPort
 	out.TyphaReplicas = in.TyphaReplicas
@@ -1313,6 +1315,8 @@ func autoConvert_kops_CalicoNetworkingSpec_To_v1alpha2_CalicoNetworkingSpec(in *
 	out.PrometheusGoMetricsEnabled = in.PrometheusGoMetricsEnabled
 	out.PrometheusProcessMetricsEnabled = in.PrometheusProcessMetricsEnabled
 	out.MajorVersion = in.MajorVersion
+	out.IPv4AutoDetectionMethod = in.IPv4AutoDetectionMethod
+	out.IPv6AutoDetectionMethod = in.IPv6AutoDetectionMethod
 	out.IptablesBackend = in.IptablesBackend
 	out.IPIPMode = in.IPIPMode
 	out.TyphaPrometheusMetricsEnabled = in.TyphaPrometheusMetricsEnabled

pkg/apis/kops/validation/BUILD.bazel

@@ -24,6 +24,8 @@ go_library(
         "//upup/pkg/fi/cloudup/awsup:go_default_library",
         "//vendor/github.com/aws/aws-sdk-go/aws/arn:go_default_library",
         "//vendor/github.com/blang/semver:go_default_library",
+        "//vendor/golang.org/x/net/ipv4:go_default_library",
+        "//vendor/golang.org/x/net/ipv6:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/validation:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",

pkg/apis/kops/validation/validation.go

@@ -17,17 +17,22 @@ limitations under the License.
 package validation
 
 import (
+	"errors"
 	"fmt"
 	"net"
+	"regexp"
 	"strings"
 
 	"github.com/blang/semver"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/kops/upup/pkg/fi"
 
 	"k8s.io/apimachinery/pkg/api/validation"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/sets"
+	utilvalidation "k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/model/components"
@@ -699,9 +704,76 @@ func validateNetworkingCalico(v *kops.CalicoNetworkingSpec, e *kops.EtcdClusterS
 		allErrs = append(allErrs, IsValidValue(fldPath.Child("iptablesBackend"), &v.IptablesBackend, valid)...)
 	}
 
+	if v.IPv4AutoDetectionMethod != "" {
+		allErrs = append(allErrs, validateCalicoAutoDetectionMethod(fldPath.Child("ipv4AutoDetectionMethod"), v.IPv4AutoDetectionMethod, ipv4.Version)...)
+	}
+
+	if v.IPv6AutoDetectionMethod != "" {
+		allErrs = append(allErrs, validateCalicoAutoDetectionMethod(fldPath.Child("ipv6AutoDetectionMethod"), v.IPv6AutoDetectionMethod, ipv6.Version)...)
+	}
+
 	return allErrs
 }
 
+func validateCalicoAutoDetectionMethod(fldPath *field.Path, runtime string, version int) field.ErrorList {
+	validationError := field.ErrorList{}
+
+	// validation code is based on the checks in calico/node startup code
+	// valid formats are "first-found", "can-reach=DEST", or
+	// "(skip-)interface=<COMMA-SEPARATED LIST OF INTERFACES>"
+	//
+	// We won't do deep validation of the values in this check, since they can
+	// be actual interface names or regexes
+	method := strings.Split(runtime, "=")
+	if len(method) == 0 {
+		return field.ErrorList{field.Invalid(fldPath, runtime, "missing autodetection method")}
+	}
+	if len(method) > 2 {
+		return field.ErrorList{field.Invalid(fldPath, runtime, "malformed autodetection method")}
+	}
+
+	// 'method' should contain something like "[interface eth0,en.*]" or "[first-found]"
+	switch method[0] {
+	case "first-found":
+		return nil
+	case "can-reach":
+		destStr := method[1]
+		if version == ipv4.Version {
+			return utilvalidation.IsValidIPv4Address(fldPath, destStr)
+		} else if version == ipv6.Version {
+			return utilvalidation.IsValidIPv6Address(fldPath, destStr)
+		}
+
+		return field.ErrorList{field.InternalError(fldPath, errors.New("IP version is incorrect"))}
+	case "interface":
+		ifRegexes := regexp.MustCompile(`\s*,\s*`).Split(method[1], -1)
+		if len(ifRegexes) == 0 || ifRegexes[0] == "" {
+			validationError = append(validationError, field.Invalid(fldPath, runtime, "'interface=' must be followed by a comma separated list of interface regular expressions"))
+		}
+		for _, r := range ifRegexes {
+			_, e := regexp.Compile(r)
+			if e != nil {
+				validationError = append(validationError, field.Invalid(fldPath, runtime, fmt.Sprintf("regexp %s does not compile: %s", r, e.Error())))
+			}
+		}
+		return validationError
+	case "skip-interface":
+		ifRegexes := regexp.MustCompile(`\s*,\s*`).Split(method[1], -1)
+		if len(ifRegexes) == 0 || ifRegexes[0] == "" {
+			validationError = append(validationError, field.Invalid(fldPath, runtime, "'skip-interface=' must be followed by a comma separated list of interface regular expressions"))
+		}
+		for _, r := range ifRegexes {
+			_, e := regexp.Compile(r)
+			if e != nil {
+				validationError = append(validationError, field.Invalid(fldPath, runtime, fmt.Sprintf("regexp %s does not compile: %s", r, e.Error())))
+			}
+		}
+		return validationError
+	default:
+		return field.ErrorList{field.Invalid(fldPath, runtime, "unsupported autodetection method")}
+	}
+}
+
 func validateContainerRuntime(runtime *string, fldPath *field.Path) field.ErrorList {
 	valid := []string{"containerd", "docker"}
 

pkg/apis/kops/validation/validation_test.go

@@ -408,6 +408,108 @@ func Test_Validate_Calico(t *testing.T) {
 			},
 			ExpectedErrors: []string{"Forbidden::calico.majorVersion"},
 		},
+		{
+			Input: caliInput{
+				Calico: &kops.CalicoNetworkingSpec{
+					IPv4AutoDetectionMethod: "first-found",
+				},
+				Etcd: &kops.EtcdClusterSpec{},
+			},
+		},
+		{
+			Input: caliInput{
+				Calico: &kops.CalicoNetworkingSpec{
+					IPv6AutoDetectionMethod: "first-found",
+				},
+				Etcd: &kops.EtcdClusterSpec{},
+			},
+		},
+		{
+			Input: caliInput{
+				Calico: &kops.CalicoNetworkingSpec{
+					IPv4AutoDetectionMethod: "can-reach=8.8.8.8",
+				},
+				Etcd: &kops.EtcdClusterSpec{},
+			},
+		},
+		{
+			Input: caliInput{
+				Calico: &kops.CalicoNetworkingSpec{
+					IPv6AutoDetectionMethod: "can-reach=2001:4860:4860::8888",
+				},
+				Etcd: &kops.EtcdClusterSpec{},
+			},
+		},
+		{
+			Input: caliInput{
+				Calico: &kops.CalicoNetworkingSpec{
+					IPv4AutoDetectionMethod: "bogus",
+				},
+				Etcd: &kops.EtcdClusterSpec{},
+			},
+			ExpectedErrors: []string{"Invalid value::calico.ipv4AutoDetectionMethod"},
+		},
+		{
+			Input: caliInput{
+				Calico: &kops.CalicoNetworkingSpec{
+					IPv6AutoDetectionMethod: "bogus",
+				},
+				Etcd: &kops.EtcdClusterSpec{},
+			},
+			ExpectedErrors: []string{"Invalid value::calico.ipv6AutoDetectionMethod"},
+		},
+		{
+			Input: caliInput{
+				Calico: &kops.CalicoNetworkingSpec{
+					IPv6AutoDetectionMethod: "interface=",
+				},
+				Etcd: &kops.EtcdClusterSpec{},
+			},
+			ExpectedErrors: []string{"Invalid value::calico.ipv6AutoDetectionMethod"},
+		},
+		{
+			Input: caliInput{
+				Calico: &kops.CalicoNetworkingSpec{
+					IPv4AutoDetectionMethod: "interface=en.*,eth0",
+				},
+				Etcd: &kops.EtcdClusterSpec{},
+			},
+		},
+		{
+			Input: caliInput{
+				Calico: &kops.CalicoNetworkingSpec{
+					IPv6AutoDetectionMethod: "skip-interface=en.*,eth0",
+				},
+				Etcd: &kops.EtcdClusterSpec{},
+			},
+		},
+		{
+			Input: caliInput{
+				Calico: &kops.CalicoNetworkingSpec{
+					IPv4AutoDetectionMethod: "interface=(,en1",
+				},
+				Etcd: &kops.EtcdClusterSpec{},
+			},
+			ExpectedErrors: []string{"Invalid value::calico.ipv4AutoDetectionMethod"},
+		},
+		{
+			Input: caliInput{
+				Calico: &kops.CalicoNetworkingSpec{
+					IPv4AutoDetectionMethod: "interface=foo=bar",
+				},
+				Etcd: &kops.EtcdClusterSpec{},
+			},
+			ExpectedErrors: []string{"Invalid value::calico.ipv4AutoDetectionMethod"},
+		},
+		{
+			Input: caliInput{
+				Calico: &kops.CalicoNetworkingSpec{
+					IPv4AutoDetectionMethod: "=en0,eth.*",
+				},
+				Etcd: &kops.EtcdClusterSpec{},
+			},
+			ExpectedErrors: []string{"Invalid value::calico.ipv4AutoDetectionMethod"},
+		},
 	}
 	for _, g := range grid {
 		errs := validateNetworkingCalico(g.Input.Calico, g.Input.Etcd, field.NewPath("calico"))

upup/models/bindata.go

@@ -7600,6 +7600,10 @@ spec:
             # Auto-detect the BGP IP address.
             - name: IP
               value: "autodetect"
+            - name: IP_AUTODETECTION_METHOD
+              value: "{{- or .Networking.Calico.IPv4AutoDetectionMethod "first-found" }}"
+            - name: IP6_AUTODETECTION_METHOD
+              value: "{{- or .Networking.Calico.IPv6AutoDetectionMethod "first-found" }}"
             # Enable IPIP
             - name: CALICO_IPV4POOL_IPIP
               value: "{{- if and (eq .CloudProvider "aws") (.Networking.Calico.CrossSubnet) -}}CrossSubnet{{- else -}} {{- or .Networking.Calico.IPIPMode "Always" -}} {{- end -}}"
@@ -8715,6 +8719,10 @@ spec:
             # Auto-detect the BGP IP address.
             - name: IP
               value: "autodetect"
+            - name: IP_AUTODETECTION_METHOD
+              value: "{{- or .Networking.Calico.IPv4AutoDetectionMethod "first-found" }}"
+            - name: IP6_AUTODETECTION_METHOD
+              value: "{{- or .Networking.Calico.IPv6AutoDetectionMethod "first-found" }}"
             # Enable IPIP
             - name: CALICO_IPV4POOL_IPIP
               value: "{{- if and (eq .CloudProvider "aws") (.Networking.Calico.CrossSubnet) -}}CrossSubnet{{- else -}} {{- or .Networking.Calico.IPIPMode "Always" -}} {{- end -}}"

upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.12.yaml.template

@@ -789,6 +789,10 @@ spec:
             # Auto-detect the BGP IP address.
             - name: IP
               value: "autodetect"
+            - name: IP_AUTODETECTION_METHOD
+              value: "{{- or .Networking.Calico.IPv4AutoDetectionMethod "first-found" }}"
+            - name: IP6_AUTODETECTION_METHOD
+              value: "{{- or .Networking.Calico.IPv6AutoDetectionMethod "first-found" }}"
             # Enable IPIP
             - name: CALICO_IPV4POOL_IPIP
               value: "{{- if and (eq .CloudProvider "aws") (.Networking.Calico.CrossSubnet) -}}CrossSubnet{{- else -}} {{- or .Networking.Calico.IPIPMode "Always" -}} {{- end -}}"

upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template

@@ -800,6 +800,10 @@ spec:
             # Auto-detect the BGP IP address.
             - name: IP
               value: "autodetect"
+            - name: IP_AUTODETECTION_METHOD
+              value: "{{- or .Networking.Calico.IPv4AutoDetectionMethod "first-found" }}"
+            - name: IP6_AUTODETECTION_METHOD
+              value: "{{- or .Networking.Calico.IPv6AutoDetectionMethod "first-found" }}"
             # Enable IPIP
             - name: CALICO_IPV4POOL_IPIP
               value: "{{- if and (eq .CloudProvider "aws") (.Networking.Calico.CrossSubnet) -}}CrossSubnet{{- else -}} {{- or .Networking.Calico.IPIPMode "Always" -}} {{- end -}}"

upup/pkg/fi/cloudup/bootstrapchannelbuilder.go

@@ -725,8 +725,8 @@ func (b *BootstrapChannelBuilder) buildAddons() *channelsapi.Addons {
 		versions := map[string]string{
 			"k8s-1.7":    "2.6.12-kops.1",
 			"k8s-1.7-v3": "3.8.0-kops.2",
-			"k8s-1.12":   "3.9.5-kops.3",
+			"k8s-1.12":   "3.9.5-kops.4",
-			"k8s-1.16":   "3.13.3-kops.2",
+			"k8s-1.16":   "3.13.3-kops.3",
 		}
 
 		{