From 0bf33140d81d3f109cce705be5e1d703f676671e Mon Sep 17 00:00:00 2001 From: Greg Barker Date: Fri, 26 Feb 2021 18:56:38 -0500 Subject: [PATCH] Private gke fixes (#1746) * private gke: nodeLocations, enablePrivateEndpoint, masterCidr, masterGlobalAcccess, remove invalid ipAllocationPolicy attributes, networkingMode VPC_NATIVE * update firewalls with master IP cidr to use the master-ip-cidr-block kpt setter * result of running `make generate-changed-only` --- distributions/gcp/Kptfile | 24 +++++++++++++++++++ .../v2/privateGKE/cluster-private-patch.yaml | 18 ++++++++------ distributions/gcp/v2/privateGKE/firewall.yaml | 6 ++--- go.sum | 1 + tests/go.mod | 3 ++- 5 files changed, 41 insertions(+), 11 deletions(-) diff --git a/distributions/gcp/Kptfile b/distributions/gcp/Kptfile index ba6482ee8..83c5600b9 100644 --- a/distributions/gcp/Kptfile +++ b/distributions/gcp/Kptfile @@ -850,3 +850,27 @@ openAPI: values: - marker: ${mgmt-project} ref: '#/definitions/io.k8s.cli.setters.mgmt-project' + io.k8s.cli.setters.node-locations: + type: array + x-k8s-cli: + setter: + name: node-locations + listValues: ["ZONE"] + io.k8s.cli.setters.enable-private-endpoint: + type: string + x-k8s-cli: + setter: + name: enable-private-endpoint + value: "false" + io.k8s.cli.setters.master-ip-cidr-block: + type: string + x-k8s-cli: + setter: + name: master-ip-cidr-block + value: 172.16.0.32/28 + io.k8s.cli.setters.master-global-access-enabled: + type: string + x-k8s-cli: + setter: + name: master-global-access-enabled + value: "true" diff --git a/distributions/gcp/v2/privateGKE/cluster-private-patch.yaml b/distributions/gcp/v2/privateGKE/cluster-private-patch.yaml index 3c05664d0..0c9252f12 100644 --- a/distributions/gcp/v2/privateGKE/cluster-private-patch.yaml +++ b/distributions/gcp/v2/privateGKE/cluster-private-patch.yaml @@ -5,22 +5,24 @@ metadata: clusterName: "project-id/location/name" # {"$kpt-set":"asm-cluster-name"} name: name # {"$kpt-set":"name"} spec: + nodeLocations: # {"$kpt-set":"node-locations"} + - "ZONE" # https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1/projects.locations.clusters#Cluster.PrivateClusterConfig # This is the least secure config because it allows access to master from all public IPs. # For alternative options see the above link. privateClusterConfig: enablePrivateNodes: true # We set enablePrivateEndpoint to false because we want a publicly accessible endpoint. - enablePrivateEndpoint: false + enablePrivateEndpoint: false # {"$kpt-set":"enable-private-endpoint"} # Keep this in sync with the range specified in the allow-egress to master firewall rule. - masterIpv4CidrBlock: 172.16.0.32/28 + masterIpv4CidrBlock: 172.16.0.32/28 # {"$kpt-set":"master-ip-cidr-block"} + # private cluster master global accessiblity https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#cp-global-access + masterGlobalAccessConfig: + enabled: true # {"$kpt-set":"master-global-access-enabled"} # # TODO(https://github.com/kubeflow/gcp-blueprints/issues/32): Following options don't appear to be supported in CNRM; will private GKE work # without them? ipAllocationPolicy: - # Make the cluster VPC Native - useIpAliases: true - createSubnetwork: false # TODO(jlewi): https://github.com/kubeflow/gcp-blueprints/issues/32 the following fields # Automatic creation of the subnetwork and its secondary ranges doesn't seem to be possible # with CNRM. We have an explicit CNRM resource for the subnetwork which we reference @@ -30,8 +32,10 @@ spec: servicesSecondaryRangeName: services # TODO(jlewi): https://github.com/kubeflow/gcp-blueprints/issues/32 the following fields # don't seem to be included in CNRM 1.9.1 - #createSubnetwork: true - # Create the clsuter in the private network we created. + # createSubnetwork: true + # Make the cluster VPC Native + networkingMode: VPC_NATIVE + # Create the cluster in the private network we created. networkRef: name: name # {"$kpt-set":"name"} subnetworkRef: diff --git a/distributions/gcp/v2/privateGKE/firewall.yaml b/distributions/gcp/v2/privateGKE/firewall.yaml index eecbb5923..e2c41c861 100644 --- a/distributions/gcp/v2/privateGKE/firewall.yaml +++ b/distributions/gcp/v2/privateGKE/firewall.yaml @@ -92,7 +92,7 @@ spec: - "10250" destinationRanges: # Keep this in sync with the masterCidrBlock specified in cluster-private-patch.yaml - - 172.16.0.32/28 + - 172.16.0.32/28 # {"$kpt-set":"master-ip-cidr-block"} # TODO(jlewi): This was a bit of a hack to try to fix failing health checks during cluster # provisioning. I was seeing packets get blocket. #- 172.217.0.0/28 @@ -139,7 +139,7 @@ spec: - "15017" - "9443" sourceRanges: - - 172.16.0.32/28 + - 172.16.0.32/28 # {"$kpt-set":"master-ip-cidr-block"} direction: INGRESS networkRef: name: name # {"$kpt-set":"name"} @@ -160,7 +160,7 @@ spec: - "443" - "6443" sourceRanges: - - 172.16.0.32/28 + - 172.16.0.32/28 # {"$kpt-set":"master-ip-cidr-block"} direction: INGRESS networkRef: name: name # {"$kpt-set":"name"} diff --git a/go.sum b/go.sum index dfc2d31f8..621dc038a 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,7 @@ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb6 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/tests/go.mod b/tests/go.mod index 40759ce57..e09c9257c 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -3,6 +3,7 @@ module github.com/kubeflow/manifests/tests go 1.13 require ( + github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/emicklei/go-restful v2.15.0+incompatible // indirect github.com/evanphx/json-patch v4.9.0+incompatible // indirect github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 @@ -20,7 +21,7 @@ require ( github.com/mailru/easyjson v0.7.6 // indirect github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.1.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect golang.org/x/net v0.0.0-20201224014010-6772e930b67b // indirect golang.org/x/text v0.3.5 // indirect google.golang.org/protobuf v1.25.0 // indirect