Switch to using k8s.io/cloud-provider-gcp

dnsprovider/pkg/dnsprovider/providers/google/clouddns/clouddns.go

@@ -28,12 +28,12 @@ import (
 	dns "google.golang.org/api/dns/v1"
 	"google.golang.org/api/option"
 	gcfg "gopkg.in/gcfg.v1"
+	"k8s.io/cloud-provider-gcp/providers/gce"
 	"k8s.io/klog/v2"
 
 	"k8s.io/kops/dnsprovider/pkg/dnsprovider"
 	"k8s.io/kops/dnsprovider/pkg/dnsprovider/providers/google/clouddns/internal"
 	"k8s.io/kops/dnsprovider/pkg/dnsprovider/providers/google/clouddns/internal/stubs"
-	"k8s.io/legacy-cloud-providers/gce"
 )
 
 const (

go.mod

@@ -60,12 +60,12 @@ require (
 	k8s.io/cli-runtime v0.25.4
 	k8s.io/client-go v0.25.4
 	k8s.io/cloud-provider-aws v1.25.1
+	k8s.io/cloud-provider-gcp/providers v0.25.3
 	k8s.io/component-base v0.25.4
 	k8s.io/gengo v0.0.0-20221011193443-fad74ee6edd9
 	k8s.io/klog/v2 v2.80.1
 	k8s.io/kubectl v0.25.4
 	k8s.io/kubelet v0.25.4
-	k8s.io/legacy-cloud-providers v0.25.4
 	k8s.io/mount-utils v0.25.4
 	k8s.io/utils v0.0.0-20221108210102-8e77b1f39fe2
 	sigs.k8s.io/controller-runtime v0.13.1

go.sum

@@ -1353,6 +1353,8 @@ k8s.io/cloud-provider v0.25.4 h1:juIfru1jVX6BlDWcJ18sv6aWxMSMmcjWf2HNXUtnkiI=
 k8s.io/cloud-provider v0.25.4/go.mod h1:L018fvnYxHrJP93UNSu8ODZYd/HCukliBzzNsV4TqC0=
 k8s.io/cloud-provider-aws v1.25.1 h1:coux06A4qvG6/IDt5a2YBKl5o9j3YAzT6dArZDUawBA=
 k8s.io/cloud-provider-aws v1.25.1/go.mod h1:BizrTUsF5lW3esndEaE++FW8P0ENBdAiKptj0BQrTBo=
+k8s.io/cloud-provider-gcp/providers v0.25.3 h1:qamR0M6QMPGB32oDSWMIR9W8g+RR96QJkL5tq/GcOGg=
+k8s.io/cloud-provider-gcp/providers v0.25.3/go.mod h1:+xLjTJ6rgP+wzyBPW7RaqyL8J+HzTwQaL9NB83pKYjQ=
 k8s.io/component-base v0.25.4 h1:n1bjg9Yt+G1C0WnIDJmg2fo6wbEU1UGMRiQSjmj7hNQ=
 k8s.io/component-base v0.25.4/go.mod h1:nnZJU8OP13PJEm6/p5V2ztgX2oyteIaAGKGMYb2L2cY=
 k8s.io/csi-translation-lib v0.25.4 h1:y8+C2sVIkA9K3Q8yofMc8DJzHUyKdFXlHk563nR1Kgc=
@@ -1372,8 +1374,6 @@ k8s.io/kubectl v0.25.4 h1:O3OA1z4V1ZyvxCvScjq0pxAP7ABgznr8UvnVObgI6Dc=
 k8s.io/kubectl v0.25.4/go.mod h1:CKMrQ67Bn2YCP26tZStPQGq62zr9pvzEf65A0navm8k=
 k8s.io/kubelet v0.25.4 h1:24MmTTQGBHr08UkMYFC/RaLjuiMREM53HfRgJKWRquI=
 k8s.io/kubelet v0.25.4/go.mod h1:dWAxzvWR7B6LrSgE+6H6Dc7bOzNOzm+O+W6zLic9daA=
-k8s.io/legacy-cloud-providers v0.25.4 h1:mbYPwS81rinU1DKtbc42nXXxQ7Lu7yYz6UirTPz23Mw=
-k8s.io/legacy-cloud-providers v0.25.4/go.mod h1:20tayVpZTfvZnxH2vz21SLQvHh7IhKcAE75hcnuIaDU=
 k8s.io/mount-utils v0.25.4 h1:+j1GBo6rH6sM1GvOI4jcu1IpjA5WssuwW7UEkQevaTU=
 k8s.io/mount-utils v0.25.4/go.mod h1:odpFnGwJfFjN3SRnjfGS0902ubcj/W6hDOrNDmSSINo=
 k8s.io/utils v0.0.0-20221108210102-8e77b1f39fe2 h1:GfD9OzL11kvZN5iArC6oTS7RTj7oJOIfnislxYlqTj8=

vendor/k8s.io/cloud-provider-gcp/providers/LICENSE

@@ -1,3 +1,4 @@
+
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
@@ -178,7 +179,7 @@
    APPENDIX: How to apply the Apache License to your work.
 
       To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
+      boilerplate notice, with the fields enclosed by brackets "[]"
       replaced with your own identifying information. (Don't include
       the brackets!)  The text should be enclosed in the appropriate
       comment syntax for the file format. We also recommend that a
@@ -186,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright {yyyy} {name of copyright owner}
+   Copyright [yyyy] [name of copyright owner]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

vendor/k8s.io/cloud-provider-gcp/providers/gce/BUILD

@@ -0,0 +1,151 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "gce",
+    srcs = [
+        "doc.go",
+        "gce.go",
+        "gce_address_manager.go",
+        "gce_addresses.go",
+        "gce_alpha.go",
+        "gce_annotations.go",
+        "gce_backendservice.go",
+        "gce_cert.go",
+        "gce_clusterid.go",
+        "gce_clusters.go",
+        "gce_disks.go",
+        "gce_fake.go",
+        "gce_firewall.go",
+        "gce_forwardingrule.go",
+        "gce_healthchecks.go",
+        "gce_instancegroup.go",
+        "gce_instances.go",
+        "gce_interfaces.go",
+        "gce_loadbalancer.go",
+        "gce_loadbalancer_external.go",
+        "gce_loadbalancer_internal.go",
+        "gce_loadbalancer_metrics.go",
+        "gce_loadbalancer_naming.go",
+        "gce_networkendpointgroup.go",
+        "gce_routes.go",
+        "gce_securitypolicy.go",
+        "gce_targetpool.go",
+        "gce_targetproxy.go",
+        "gce_tpu.go",
+        "gce_urlmap.go",
+        "gce_util.go",
+        "gce_zones.go",
+        "metrics.go",
+        "support.go",
+        "token_source.go",
+    ],
+    importpath = "k8s.io/cloud-provider-gcp/providers/gce",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/cloud.google.com/go/compute/metadata",
+        "//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud",
+        "//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/filter",
+        "//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta",
+        "//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/mock",
+        "//vendor/github.com/google/go-cmp/cmp",
+        "//vendor/golang.org/x/oauth2",
+        "//vendor/golang.org/x/oauth2/google",
+        "//vendor/google.golang.org/api/compute/v0.alpha:v0_alpha",
+        "//vendor/google.golang.org/api/compute/v0.beta:v0_beta",
+        "//vendor/google.golang.org/api/compute/v1:compute",
+        "//vendor/google.golang.org/api/container/v1:container",
+        "//vendor/google.golang.org/api/googleapi",
+        "//vendor/google.golang.org/api/option",
+        "//vendor/google.golang.org/api/tpu/v1:tpu",
+        "//vendor/gopkg.in/gcfg.v1:gcfg_v1",
+        "//vendor/k8s.io/api/core/v1:core",
+        "//vendor/k8s.io/apimachinery/pkg/api/resource",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:meta",
+        "//vendor/k8s.io/apimachinery/pkg/fields",
+        "//vendor/k8s.io/apimachinery/pkg/runtime",
+        "//vendor/k8s.io/apimachinery/pkg/types",
+        "//vendor/k8s.io/apimachinery/pkg/util/errors",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets",
+        "//vendor/k8s.io/apimachinery/pkg/util/version",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait",
+        "//vendor/k8s.io/apimachinery/pkg/watch",
+        "//vendor/k8s.io/client-go/informers",
+        "//vendor/k8s.io/client-go/kubernetes",
+        "//vendor/k8s.io/client-go/kubernetes/fake",
+        "//vendor/k8s.io/client-go/kubernetes/scheme",
+        "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:core",
+        "//vendor/k8s.io/client-go/pkg/version",
+        "//vendor/k8s.io/client-go/tools/cache",
+        "//vendor/k8s.io/client-go/tools/record",
+        "//vendor/k8s.io/client-go/util/flowcontrol",
+        "//vendor/k8s.io/cloud-provider",
+        "//vendor/k8s.io/cloud-provider/service/helpers",
+        "//vendor/k8s.io/cloud-provider/volume",
+        "//vendor/k8s.io/cloud-provider/volume/errors",
+        "//vendor/k8s.io/cloud-provider/volume/helpers",
+        "//vendor/k8s.io/component-base/metrics",
+        "//vendor/k8s.io/component-base/metrics/legacyregistry",
+        "//vendor/k8s.io/klog/v2:klog",
+        "//vendor/k8s.io/utils/net",
+    ],
+)
+
+go_test(
+    name = "gce_test",
+    srcs = [
+        "gce_address_manager_test.go",
+        "gce_annotations_test.go",
+        "gce_disks_test.go",
+        "gce_healthchecks_test.go",
+        "gce_instances_test.go",
+        "gce_loadbalancer_external_test.go",
+        "gce_loadbalancer_internal_test.go",
+        "gce_loadbalancer_metrics_test.go",
+        "gce_loadbalancer_test.go",
+        "gce_loadbalancer_utils_test.go",
+        "gce_test.go",
+        "gce_util_test.go",
+        "metrics_test.go",
+    ],
+    embed = [":gce"],
+    deps = [
+        "//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud",
+        "//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta",
+        "//vendor/github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/mock",
+        "//vendor/github.com/google/go-cmp/cmp",
+        "//vendor/github.com/stretchr/testify/assert",
+        "//vendor/github.com/stretchr/testify/require",
+        "//vendor/golang.org/x/oauth2/google",
+        "//vendor/google.golang.org/api/compute/v0.alpha:v0_alpha",
+        "//vendor/google.golang.org/api/compute/v0.beta:v0_beta",
+        "//vendor/google.golang.org/api/compute/v1:compute",
+        "//vendor/google.golang.org/api/googleapi",
+        "//vendor/k8s.io/api/core/v1:core",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:meta",
+        "//vendor/k8s.io/apimachinery/pkg/types",
+        "//vendor/k8s.io/apimachinery/pkg/util/intstr",
+        "//vendor/k8s.io/apimachinery/pkg/util/json",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets",
+        "//vendor/k8s.io/client-go/tools/record",
+        "//vendor/k8s.io/cloud-provider",
+        "//vendor/k8s.io/cloud-provider/service/helpers",
+        "//vendor/k8s.io/utils/net",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//providers/gce/gcpcredential:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/cloud-provider-gcp/providers/gce/OWNERS

@@ -0,0 +1,12 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+approvers:
+- saad-ali
+- jingxu97
+- bowei
+- freehan
+- mrhohn
+- cheftako
+reviewers:
+- cici37
+- jiahuif
+- jpbetz

vendor/k8s.io/cloud-provider-gcp/providers/gce/doc.go

@@ -16,4 +16,4 @@ limitations under the License.
 
 // Package gce is an implementation of Interface, LoadBalancer
 // and Instances for Google Compute Engine.
-package gce // import "k8s.io/legacy-cloud-providers/gce"
+package gce // import "k8s.io/cloud-provider-gcp/providers/gce"

vendor/k8s.io/cloud-provider-gcp/providers/gce/gce.go

@@ -94,6 +94,12 @@ var _ cloudprovider.Zones = (*Cloud)(nil)
 var _ cloudprovider.PVLabeler = (*Cloud)(nil)
 var _ cloudprovider.Clusters = (*Cloud)(nil)
 
+type StackType string
+
+const NetworkStackDualStack StackType = "IPV4_IPV6"
+const NetworkStackIPV4 StackType = "IPV4"
+const NetworkStackIPV6 StackType = "IPV6"
+
 // Cloud is an implementation of Interface, LoadBalancer and Instances for Google Compute Engine.
 type Cloud struct {
 	// ClusterID contains functionality for getting (and initializing) the ingress-uid. Call Cloud.Initialize()
@@ -167,9 +173,9 @@ type Cloud struct {
 	s *cloud.Service
 
 	metricsCollector loadbalancerMetricsCollector
-
+	// stackType indicates whether the cluster is a single stack IPv4, single
-	// the compute API endpoint with the `projects/` element.
+	// stack IPv6 or a dual stack cluster
-	projectsBasePath string
+	stackType StackType
 }
 
 // ConfigGlobal is the in memory representation of the gce.conf config data
@@ -184,6 +190,7 @@ type ConfigGlobal struct {
 	NetworkProjectID string `gcfg:"network-project-id"`
 	NetworkName      string `gcfg:"network-name"`
 	SubnetworkName   string `gcfg:"subnetwork-name"`
+	StackType        string `gcfg:"stack-type"`
 	// DEPRECATED: Do not rely on this value as it may be incorrect.
 	// SecondaryRangeName is the name of the secondary range to allocate IP
 	// aliases. The secondary range must be present on the subnetwork the
@@ -239,6 +246,7 @@ type CloudConfig struct {
 	TokenSource        oauth2.TokenSource
 	UseMetadataServer  bool
 	AlphaFeatureGate   *AlphaFeatureGate
+	StackType          string
 }
 
 func init() {
@@ -396,6 +404,10 @@ func generateCloudConfig(configFile *ConfigFile) (cloudConfig *CloudConfig, err
 		cloudConfig.SecondaryRangeName = configFile.Global.SecondaryRangeName
 	}
 
+	if configFile != nil {
+		cloudConfig.StackType = configFile.Global.StackType
+	}
+
 	return cloudConfig, err
 }
 
@@ -435,17 +447,21 @@ func CreateGCECloud(config *CloudConfig) (*Cloud, error) {
 	}
 	serviceAlpha.UserAgent = userAgent
 
+	// Expect override api endpoint to always be v1 api and follows the same pattern as prod.
+	// Generate alpha and beta api endpoints based on override v1 api endpoint.
+	// For example,
+	// staging API endpoint: https://www.googleapis.com/compute/staging_v1/
 	if config.APIEndpoint != "" {
-		if strings.HasSuffix(service.BasePath, "/projects/") {
-			service.BasePath = getProjectsBasePath(config.APIEndpoint)
-			serviceBeta.BasePath = getProjectsBasePath(strings.Replace(config.APIEndpoint, "v1", "beta", -1))
-			serviceAlpha.BasePath = getProjectsBasePath(strings.Replace(config.APIEndpoint, "v1", "alpha", -1))
-		} else {
 		service.BasePath = config.APIEndpoint
 		serviceBeta.BasePath = strings.Replace(config.APIEndpoint, "v1", "beta", -1)
 		serviceAlpha.BasePath = strings.Replace(config.APIEndpoint, "v1", "alpha", -1)
 	}
-	}
+
+	// Previously "projects/" was a part of BasePath, but recent changes in Google Cloud SDK removed it from there.
+	// To bring the old format back we update BasePath including "projects/" there again.
+	service.BasePath += "projects/"
+	serviceBeta.BasePath += "projects/"
+	serviceAlpha.BasePath += "projects/"
 
 	containerService, err := container.NewService(context.Background(), option.WithTokenSource(config.TokenSource))
 	if err != nil {
@@ -530,7 +546,7 @@ func CreateGCECloud(config *CloudConfig) (*Cloud, error) {
 		AlphaFeatureGate:         config.AlphaFeatureGate,
 		nodeZones:                map[string]sets.String{},
 		metricsCollector:         newLoadBalancerMetrics(),
-		projectsBasePath:         getProjectsBasePath(service.BasePath),
+		stackType:                StackType(config.StackType),
 	}
 
 	gce.manager = &gceServiceManager{gce}
@@ -800,18 +816,6 @@ func (g *Cloud) HasClusterID() bool {
 	return true
 }
 
-// getProjectsBasePath returns the compute API endpoint with the `projects/` element.
-// The suffix must be added when generating compute resource urls.
-func getProjectsBasePath(basePath string) string {
-	if !strings.HasSuffix(basePath, "/") {
-		basePath += "/"
-	}
-	if !strings.HasSuffix(basePath, "/projects/") {
-		basePath += "projects/"
-	}
-	return basePath
-}
-
 // Project IDs cannot have a digit for the first characeter. If the id contains a digit,
 // then it must be a project number.
 func isProjectNumber(idOrNumber string) bool {
@@ -955,7 +959,7 @@ func newOauthClient(tokenSource oauth2.TokenSource) (*http.Client, error) {
 func (manager *gceServiceManager) getProjectsAPIEndpoint() string {
 	projectsAPIEndpoint := gceComputeAPIEndpoint + "projects/"
 	if manager.gce.service != nil {
-		projectsAPIEndpoint = manager.gce.projectsBasePath
+		projectsAPIEndpoint = manager.gce.service.BasePath
 	}
 
 	return projectsAPIEndpoint

vendor/k8s.io/cloud-provider-gcp/providers/gce/gce_alpha.go

@@ -23,10 +23,6 @@ const (
 	// AlphaFeatureILBSubsets allows InternalLoadBalancer services to include a subset
 	// of cluster nodes as backends instead of all nodes.
 	AlphaFeatureILBSubsets = "ILBSubsets"
-
-	// AlphaFeatureSkipIGsManagement enabled L4 Regional Backend Services and
-	// disables instance group management in service controller
-	AlphaFeatureSkipIGsManagement = "SkipIGsManagement"
 )
 
 // AlphaFeatureGate contains a mapping of alpha features to whether they are enabled

vendor/k8s.io/cloud-provider-gcp/providers/gce/gce_annotations.go

@@ -74,13 +74,6 @@ const (
 
 	// NetworkTierAnnotationPremium is an annotation to indicate the Service is on the Premium network tier
 	NetworkTierAnnotationPremium = cloud.NetworkTierPremium
-
-	// RBSAnnotationKey is annotated on a Service object to indicate
-	// opt-in mode for RBS NetLB
-	RBSAnnotationKey = "cloud.google.com/l4-rbs"
-
-	// RBSEnabled is an annotation to indicate the Service is opt-in for RBS
-	RBSEnabled = "enabled"
 )
 
 // GetLoadBalancerAnnotationType returns the type of GCP load balancer which should be assembled.

vendor/k8s.io/cloud-provider-gcp/providers/gce/gce_fake.go

@@ -78,7 +78,6 @@ func NewFakeGCECloud(vals TestClusterValues) *Cloud {
 		ClusterID:        fakeClusterID(vals.ClusterID),
 		onXPN:            vals.OnXPN,
 		metricsCollector: newLoadBalancerMetrics(),
-		projectsBasePath: getProjectsBasePath(service.BasePath),
 	}
 	c := cloud.NewMockGCE(&gceProjectRouter{gce})
 	gce.c = c

vendor/k8s.io/cloud-provider-gcp/providers/gce/gce_firewall.go

@@ -66,12 +66,3 @@ func (g *Cloud) UpdateFirewall(f *compute.Firewall) error {
 	mc := newFirewallMetricContext("update")
 	return mc.Observe(g.c.Firewalls().Update(ctx, meta.GlobalKey(f.Name), f))
 }
-
-// PatchFirewall applies the given firewall as an update to an existing service.
-func (g *Cloud) PatchFirewall(f *compute.Firewall) error {
-	ctx, cancel := cloud.ContextWithCallTimeout()
-	defer cancel()
-
-	mc := newFirewallMetricContext("Patch")
-	return mc.Observe(g.c.Firewalls().Patch(ctx, meta.GlobalKey(f.Name), f))
-}

vendor/k8s.io/cloud-provider-gcp/providers/gce/gce_instancegroup.go

@@ -50,16 +50,6 @@ func (g *Cloud) DeleteInstanceGroup(name string, zone string) error {
 	return mc.Observe(g.c.InstanceGroups().Delete(ctx, meta.ZonalKey(name, zone)))
 }
 
-// FilterInstanceGroupsByName lists all InstanceGroups in the project and
-// zone that match the name regexp.
-func (g *Cloud) FilterInstanceGroupsByNamePrefix(namePrefix, zone string) ([]*compute.InstanceGroup, error) {
-	ctx, cancel := cloud.ContextWithCallTimeout()
-	defer cancel()
-	mc := newInstanceGroupMetricContext("filter", zone)
-	v, err := g.c.InstanceGroups().List(ctx, zone, filter.Regexp("name", namePrefix+".*"))
-	return v, mc.Observe(err)
-}
-
 // ListInstanceGroups lists all InstanceGroups in the project and
 // zone.
 func (g *Cloud) ListInstanceGroups(zone string) ([]*compute.InstanceGroup, error) {

vendor/k8s.io/cloud-provider-gcp/providers/gce/gce_instances.go

@@ -45,6 +45,7 @@ import (
 const (
 	defaultZone                   = ""
 	networkInterfaceIP            = "instance/network-interfaces/%s/ip"
+	networkInterfaceIPV6          = "instance/network-interfaces/%s/ipv6s"
 	networkInterfaceAccessConfigs = "instance/network-interfaces/%s/access-configs"
 	networkInterfaceExternalIP    = "instance/network-interfaces/%s/access-configs/%s/external-ip"
 )
@@ -80,7 +81,7 @@ func makeHostURL(projectsAPIEndpoint, projectID, zone, host string) string {
 // ToInstanceReferences returns instance references by links
 func (g *Cloud) ToInstanceReferences(zone string, instanceNames []string) (refs []*compute.InstanceReference) {
 	for _, ins := range instanceNames {
-		instanceLink := makeHostURL(g.projectsBasePath, g.projectID, zone, ins)
+		instanceLink := makeHostURL(g.service.BasePath, g.projectID, zone, ins)
 		refs = append(refs, &compute.InstanceReference{Instance: instanceLink})
 	}
 	return refs
@@ -117,6 +118,27 @@ func (g *Cloud) NodeAddresses(ctx context.Context, nodeName types.NodeName) ([]v
 				}
 				nodeAddresses = append(nodeAddresses, v1.NodeAddress{Type: v1.NodeInternalIP, Address: internalIP})
 
+				// Both internal and external IPv6 addresses are written to this array
+				ipv6s, err := metadata.Get(fmt.Sprintf(networkInterfaceIPV6, nic))
+				if err != nil || ipv6s == "" {
+					klog.Infof("no internal IPV6 addresses found for node %v: %v", nodeName, err)
+				} else {
+					ipv6Arr := strings.Split(ipv6s, "/\n")
+					var internalIPV6 string
+					for _, ip := range ipv6Arr {
+						if ip == "" {
+							continue
+						}
+						internalIPV6 = ip
+						break
+					}
+					if internalIPV6 != "" {
+						nodeAddresses = append(nodeAddresses, v1.NodeAddress{Type: v1.NodeInternalIP, Address: internalIPV6})
+					} else {
+						klog.Warningf("internal IPV6 range is empty for node %v", nodeName)
+					}
+				}
+
 				acs, err := metadata.Get(fmt.Sprintf(networkInterfaceAccessConfigs, nic))
 				if err != nil {
 					return nil, fmt.Errorf("couldn't get access configs: %v", err)
@@ -162,10 +184,10 @@ func (g *Cloud) NodeAddresses(ctx context.Context, nodeName types.NodeName) ([]v
 
 	instance, err := g.c.Instances().Get(timeoutCtx, meta.ZonalKey(canonicalizeInstanceName(instanceObj.Name), instanceObj.Zone))
 	if err != nil {
-		return []v1.NodeAddress{}, fmt.Errorf("error while querying for instance: %v", err)
+		return nil, fmt.Errorf("error while querying for instance: %v", err)
 	}
 
-	return nodeAddressesFromInstance(instance)
+	return g.nodeAddressesFromInstance(instance)
 }
 
 // NodeAddressesByProviderID will not be called from the node that is requesting this ID.
@@ -184,7 +206,7 @@ func (g *Cloud) NodeAddressesByProviderID(ctx context.Context, providerID string
 		return []v1.NodeAddress{}, fmt.Errorf("error while querying for providerID %q: %v", providerID, err)
 	}
 
-	return nodeAddressesFromInstance(instance)
+	return g.nodeAddressesFromInstance(instance)
 }
 
 // instanceByProviderID returns the cloudprovider instance of the node
@@ -216,22 +238,35 @@ func (g *Cloud) InstanceShutdown(ctx context.Context, node *v1.Node) (bool, erro
 	return false, cloudprovider.NotImplemented
 }
 
-func nodeAddressesFromInstance(instance *compute.Instance) ([]v1.NodeAddress, error) {
+func (g *Cloud) nodeAddressesFromInstance(instance *compute.Instance) ([]v1.NodeAddress, error) {
 	if len(instance.NetworkInterfaces) < 1 {
 		return nil, fmt.Errorf("could not find network interfaces for instanceID %q", instance.Id)
 	}
 	nodeAddresses := []v1.NodeAddress{}
-
 	for _, nic := range instance.NetworkInterfaces {
 		nodeAddresses = append(nodeAddresses, v1.NodeAddress{Type: v1.NodeInternalIP, Address: nic.NetworkIP})
 		for _, config := range nic.AccessConfigs {
 			nodeAddresses = append(nodeAddresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: config.NatIP})
 		}
+		ipv6Addr := getIPV6AddressFromInterface(nic)
+		if ipv6Addr != "" {
+			nodeAddresses = append(nodeAddresses, v1.NodeAddress{Type: v1.NodeInternalIP, Address: ipv6Addr})
+		}
 	}
 
 	return nodeAddresses, nil
 }
 
+func getIPV6AddressFromInterface(nic *compute.NetworkInterface) string {
+	ipv6Addr := nic.Ipv6Address
+	if ipv6Addr == "" && nic.Ipv6AccessType == "EXTERNAL" {
+		for _, r := range nic.Ipv6AccessConfigs {
+			ipv6Addr = r.ExternalIpv6
+		}
+	}
+	return ipv6Addr
+}
+
 // InstanceTypeByProviderID returns the cloudprovider instance type of the node
 // with the specified unique providerID This method will not be called from the
 // node that is requesting this ID. i.e. metadata service and other local
@@ -298,19 +333,23 @@ func (g *Cloud) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloudprov
 		return nil, err
 	}
 
+	var addresses []v1.NodeAddress
+	var instanceType string
 	instance, err := g.c.Instances().Get(timeoutCtx, meta.ZonalKey(canonicalizeInstanceName(name), zone))
 	if err != nil {
 		return nil, fmt.Errorf("error while querying for providerID %q: %v", providerID, err)
 	}
 
-	addresses, err := nodeAddressesFromInstance(instance)
+	addresses, err = g.nodeAddressesFromInstance(instance)
 	if err != nil {
 		return nil, err
 	}
 
+	instanceType = lastComponent(instance.MachineType)
+
 	return &cloudprovider.InstanceMetadata{
 		ProviderID:    providerID,
-		InstanceType:  lastComponent(instance.MachineType),
+		InstanceType:  instanceType,
 		NodeAddresses: addresses,
 		Zone:          zone,
 		Region:        region,
@@ -502,8 +541,8 @@ func (g *Cloud) AliasRangesByProviderID(providerID string) (cidrs []string, err
 		return nil, err
 	}
 
-	var res *computebeta.Instance
+	var res *compute.Instance
-	res, err = g.c.BetaInstances().Get(ctx, meta.ZonalKey(canonicalizeInstanceName(name), zone))
+	res, err = g.c.Instances().Get(ctx, meta.ZonalKey(canonicalizeInstanceName(name), zone))
 	if err != nil {
 		return
 	}
@@ -512,6 +551,15 @@ func (g *Cloud) AliasRangesByProviderID(providerID string) (cidrs []string, err
 		for _, r := range networkInterface.AliasIpRanges {
 			cidrs = append(cidrs, r.IpCidrRange)
 		}
+		ipv6Addr := getIPV6AddressFromInterface(networkInterface)
+		if ipv6Addr != "" {
+			// The podCIDR range is the first /112 subrange from the /96 assigned to
+			// the node
+			ipv6PodCIDR := fmt.Sprintf("%s/112", ipv6Addr)
+			cidrs = append(cidrs, ipv6PodCIDR)
+		} else {
+			klog.Infof("No IPv6 addresses found for %s", providerID)
+		}
 	}
 	return
 }

vendor/k8s.io/cloud-provider-gcp/providers/gce/gce_loadbalancer.go

@@ -23,19 +23,20 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"net"
 	"sort"
 	"strings"
 
-	v1 "k8s.io/api/core/v1"
+	"k8s.io/api/core/v1"
 	"k8s.io/klog/v2"
 
 	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
 	cloudprovider "k8s.io/cloud-provider"
-	netutils "k8s.io/utils/net"
+	utilnet "k8s.io/utils/net"
 )
 
 type cidrs struct {
-	ipn   netutils.IPNetSet
+	ipn   utilnet.IPNetSet
 	isSet bool
 }
 
@@ -47,12 +48,12 @@ var (
 func init() {
 	var err error
 	// L3/4 health checkers have client addresses within these known CIDRs.
-	l4LbSrcRngsFlag.ipn, err = netutils.ParseIPNets([]string{"130.211.0.0/22", "35.191.0.0/16", "209.85.152.0/22", "209.85.204.0/22"}...)
+	l4LbSrcRngsFlag.ipn, err = utilnet.ParseIPNets([]string{"130.211.0.0/22", "35.191.0.0/16", "209.85.152.0/22", "209.85.204.0/22"}...)
 	if err != nil {
 		panic("Incorrect default GCE L3/4 source ranges")
 	}
 	// L7 health checkers have client addresses within these known CIDRs.
-	l7lbSrcRngsFlag.ipn, err = netutils.ParseIPNets([]string{"130.211.0.0/22", "35.191.0.0/16"}...)
+	l7lbSrcRngsFlag.ipn, err = utilnet.ParseIPNets([]string{"130.211.0.0/22", "35.191.0.0/16"}...)
 	if err != nil {
 		panic("Incorrect default GCE L7 source ranges")
 	}
@@ -73,13 +74,13 @@ func (c *cidrs) Set(value string) error {
 	// On first Set(), clear the original defaults
 	if !c.isSet {
 		c.isSet = true
-		c.ipn = make(netutils.IPNetSet)
+		c.ipn = make(utilnet.IPNetSet)
 	} else {
 		return fmt.Errorf("GCE LB CIDRs have already been set")
 	}
 
 	for _, cidr := range strings.Split(value, ",") {
-		_, ipnet, err := netutils.ParseCIDRSloppy(cidr)
+		_, ipnet, err := net.ParseCIDR(cidr)
 		if err != nil {
 			return err
 		}

vendor/k8s.io/cloud-provider-gcp/providers/gce/gce_loadbalancer_external.go

@@ -23,20 +23,18 @@ import (
 	"context"
 	"fmt"
 	"net/http"
-	"reflect"
 	"strconv"
 	"strings"
 
 	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
-	v1 "k8s.io/api/core/v1"
+	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
-	cloudprovider "k8s.io/cloud-provider"
 	servicehelpers "k8s.io/cloud-provider/service/helpers"
 	utilnet "k8s.io/utils/net"
 
-	"google.golang.org/api/compute/v1"
+	compute "google.golang.org/api/compute/v1"
 	"k8s.io/klog/v2"
 )
 
@@ -53,11 +51,6 @@ const (
 // new load balancers and updating existing load balancers, recognizing when
 // each is needed.
 func (g *Cloud) ensureExternalLoadBalancer(clusterName string, clusterID string, apiService *v1.Service, existingFwdRule *compute.ForwardingRule, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) {
-	// Skip service handling if it uses Regional Backend Services and handled by other controllers
-	if usesL4RBS(apiService, existingFwdRule) {
-		return nil, cloudprovider.ImplementedElsewhere
-	}
-
 	if len(nodes) == 0 {
 		return nil, fmt.Errorf(errStrLbNoHosts)
 	}
@@ -89,12 +82,7 @@ func (g *Cloud) ensureExternalLoadBalancer(clusterName string, clusterID string,
 		return nil, err
 	}
 	klog.V(4).Infof("ensureExternalLoadBalancer(%s): Desired network tier %q.", lbRefStr, netTier)
-	// TODO: distinguish between unspecified and specified network tiers annotation properly in forwardingrule creation
-	// Only delete ForwardingRule when network tier annotation is specified, otherwise leave it only to avoid wrongful
-	// deletion against user intention when network tier annotation is not specified.
-	if _, ok := apiService.Annotations[NetworkTierAnnotationKey]; ok {
 	g.deleteWrongNetworkTieredResources(loadBalancerName, lbRefStr, netTier)
-	}
 
 	// Check if the forwarding rule exists, and if so, what its IP is.
 	fwdRuleExists, fwdRuleNeedsUpdate, fwdRuleIP, err := g.forwardingRuleNeedsUpdate(loadBalancerName, g.region, requestedIP, ports)
@@ -194,13 +182,13 @@ func (g *Cloud) ensureExternalLoadBalancer(clusterName string, clusterID string,
 		// without needing to be deleted and recreated.
 		if firewallExists {
 			klog.Infof("ensureExternalLoadBalancer(%s): Updating firewall.", lbRefStr)
-			if err := g.updateFirewall(apiService, MakeFirewallName(loadBalancerName), desc, ipAddressToUse, sourceRanges, ports, hosts); err != nil {
+			if err := g.updateFirewall(apiService, MakeFirewallName(loadBalancerName), desc, sourceRanges, ports, hosts); err != nil {
 				return nil, err
 			}
 			klog.Infof("ensureExternalLoadBalancer(%s): Updated firewall.", lbRefStr)
 		} else {
 			klog.Infof("ensureExternalLoadBalancer(%s): Creating firewall.", lbRefStr)
-			if err := g.createFirewall(apiService, MakeFirewallName(loadBalancerName), desc, ipAddressToUse, sourceRanges, ports, hosts); err != nil {
+			if err := g.createFirewall(apiService, MakeFirewallName(loadBalancerName), desc, sourceRanges, ports, hosts); err != nil {
 				return nil, err
 			}
 			klog.Infof("ensureExternalLoadBalancer(%s): Created firewall.", lbRefStr)
@@ -292,11 +280,6 @@ func (g *Cloud) ensureExternalLoadBalancer(clusterName string, clusterID string,
 
 // updateExternalLoadBalancer is the external implementation of LoadBalancer.UpdateLoadBalancer.
 func (g *Cloud) updateExternalLoadBalancer(clusterName string, service *v1.Service, nodes []*v1.Node) error {
-	// Skip service update if it uses Regional Backend Services and handled by other controllers
-	if usesL4RBS(service, nil) {
-		return cloudprovider.ImplementedElsewhere
-	}
-
 	hosts, err := g.getInstancesByNames(nodeNames(nodes))
 	if err != nil {
 		return err
@@ -308,11 +291,6 @@ func (g *Cloud) updateExternalLoadBalancer(clusterName string, service *v1.Servi
 
 // ensureExternalLoadBalancerDeleted is the external implementation of LoadBalancer.EnsureLoadBalancerDeleted
 func (g *Cloud) ensureExternalLoadBalancerDeleted(clusterName, clusterID string, service *v1.Service) error {
-	// Skip service deletion if it uses Regional Backend Services and handled by other controllers
-	if usesL4RBS(service, nil) {
-		return cloudprovider.ImplementedElsewhere
-	}
-
 	loadBalancerName := g.GetLoadBalancerName(context.TODO(), clusterName, service)
 	serviceName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name}
 	lbRefStr := fmt.Sprintf("%v(%v)", loadBalancerName, serviceName)
@@ -470,7 +448,7 @@ func verifyUserRequestedIP(s CloudAddressService, region, requestedIP, fwdRuleIP
 		netTier := cloud.NetworkTierGCEValueToType(netTierStr)
 		if netTier != desiredNetTier {
 			klog.Errorf("verifyUserRequestedIP: requested static IP %q (name: %s) for LB %s has network tier %s, need %s.", requestedIP, existingAddress.Name, lbRef, netTier, desiredNetTier)
-			return false, fmt.Errorf("requested IP %q belongs to the %s network tier; expected %s", requestedIP, netTier, desiredNetTier)
+			return false, fmt.Errorf("requrested IP %q belongs to the %s network tier; expected %s", requestedIP, netTier, desiredNetTier)
 		}
 		klog.V(4).Infof("verifyUserRequestedIP: the requested static IP %q (name: %s, tier: %s) for LB %s exists.", requestedIP, existingAddress.Name, netTier, lbRef)
 		return true, nil
@@ -655,7 +633,7 @@ func (g *Cloud) updateTargetPool(loadBalancerName string, hosts []*gceInstance)
 }
 
 func (g *Cloud) targetPoolURL(name string) string {
-	return g.projectsBasePath + strings.Join([]string{g.projectID, "regions", g.region, "targetPools", name}, "/")
+	return g.service.BasePath + strings.Join([]string{g.projectID, "regions", g.region, "targetPools", name}, "/")
 }
 
 func makeHTTPHealthCheck(name, path string, port int32) *compute.HttpHealthCheck {
@@ -903,13 +881,6 @@ func (g *Cloud) firewallNeedsUpdate(name, serviceName, ipAddress string, ports [
 	if !sourceRanges.Equal(actualSourceRanges) {
 		return true, true, nil
 	}
-
-	destinationRanges := []string{ipAddress}
-
-	if !reflect.DeepEqual(destinationRanges, fw.DestinationRanges) {
-		return true, true, nil
-	}
-
 	return true, false, nil
 }
 
@@ -929,7 +900,7 @@ func (g *Cloud) ensureHTTPHealthCheckFirewall(svc *v1.Service, serviceName, ipAd
 			return fmt.Errorf("error getting firewall for health checks: %v", err)
 		}
 		klog.Infof("Creating firewall %v for health checks.", fwName)
-		if err := g.createFirewall(svc, fwName, desc, ipAddress, sourceRanges, ports, hosts); err != nil {
+		if err := g.createFirewall(svc, fwName, desc, sourceRanges, ports, hosts); err != nil {
 			return err
 		}
 		klog.Infof("Created firewall %v for health checks.", fwName)
@@ -942,7 +913,7 @@ func (g *Cloud) ensureHTTPHealthCheckFirewall(svc *v1.Service, serviceName, ipAd
 		!equalStringSets(fw.Allowed[0].Ports, []string{strconv.Itoa(int(ports[0].Port))}) ||
 		!equalStringSets(fw.SourceRanges, sourceRanges.StringSlice()) {
 		klog.Warningf("Firewall %v exists but parameters have drifted - updating...", fwName)
-		if err := g.updateFirewall(svc, fwName, desc, ipAddress, sourceRanges, ports, hosts); err != nil {
+		if err := g.updateFirewall(svc, fwName, desc, sourceRanges, ports, hosts); err != nil {
 			klog.Warningf("Failed to reconcile firewall %v parameters.", fwName)
 			return err
 		}
@@ -978,8 +949,8 @@ func createForwardingRule(s CloudForwardingRuleService, name, serviceName, regio
 	return nil
 }
 
-func (g *Cloud) createFirewall(svc *v1.Service, name, desc, destinationIP string, sourceRanges utilnet.IPNetSet, ports []v1.ServicePort, hosts []*gceInstance) error {
+func (g *Cloud) createFirewall(svc *v1.Service, name, desc string, sourceRanges utilnet.IPNetSet, ports []v1.ServicePort, hosts []*gceInstance) error {
-	firewall, err := g.firewallObject(name, desc, destinationIP, sourceRanges, ports, hosts)
+	firewall, err := g.firewallObject(name, desc, sourceRanges, ports, hosts)
 	if err != nil {
 		return err
 	}
@@ -996,13 +967,13 @@ func (g *Cloud) createFirewall(svc *v1.Service, name, desc, destinationIP string
 	return nil
 }
 
-func (g *Cloud) updateFirewall(svc *v1.Service, name, desc, destinationIP string, sourceRanges utilnet.IPNetSet, ports []v1.ServicePort, hosts []*gceInstance) error {
+func (g *Cloud) updateFirewall(svc *v1.Service, name, desc string, sourceRanges utilnet.IPNetSet, ports []v1.ServicePort, hosts []*gceInstance) error {
-	firewall, err := g.firewallObject(name, desc, destinationIP, sourceRanges, ports, hosts)
+	firewall, err := g.firewallObject(name, desc, sourceRanges, ports, hosts)
 	if err != nil {
 		return err
 	}
 
-	if err = g.PatchFirewall(firewall); err != nil {
+	if err = g.UpdateFirewall(firewall); err != nil {
 		if isHTTPErrorCode(err, http.StatusConflict) {
 			return nil
 		} else if isForbidden(err) && g.OnXPN() {
@@ -1015,9 +986,7 @@ func (g *Cloud) updateFirewall(svc *v1.Service, name, desc, destinationIP string
 	return nil
 }
 
-func (g *Cloud) firewallObject(name, desc, destinationIP string, sourceRanges utilnet.IPNetSet, ports []v1.ServicePort, hosts []*gceInstance) (*compute.Firewall, error) {
+func (g *Cloud) firewallObject(name, desc string, sourceRanges utilnet.IPNetSet, ports []v1.ServicePort, hosts []*gceInstance) (*compute.Firewall, error) {
-	// destinationIP can be empty string "" and this means that it is not set.
-	// GCE considers empty destinationRanges as "all" for ingress firewall-rules.
 	// Concatenate service ports into port ranges. This help to workaround the gce firewall limitation where only
 	// 100 ports or port ranges can be used in a firewall rule.
 	_, portRanges, _ := getPortsAndProtocol(ports)
@@ -1050,9 +1019,6 @@ func (g *Cloud) firewallObject(name, desc, destinationIP string, sourceRanges ut
 			},
 		},
 	}
-	if destinationIP != "" {
-		firewall.DestinationRanges = []string{destinationIP}
-	}
 	return firewall, nil
 }
 

vendor/k8s.io/cloud-provider-gcp/providers/gce/gce_loadbalancer_internal.go

@@ -31,7 +31,7 @@ import (
 	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
 	"github.com/google/go-cmp/cmp"
 	compute "google.golang.org/api/compute/v1"
-	v1 "k8s.io/api/core/v1"
+	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	cloudprovider "k8s.io/cloud-provider"
@@ -53,21 +53,19 @@ const (
 )
 
 func (g *Cloud) ensureInternalLoadBalancer(clusterName, clusterID string, svc *v1.Service, existingFwdRule *compute.ForwardingRule, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) {
-	if existingFwdRule == nil && !hasFinalizer(svc, ILBFinalizerV1) {
+	if g.AlphaFeatureGate.Enabled(AlphaFeatureILBSubsets) && existingFwdRule == nil {
-		// Neither the forwarding rule nor the V1 finalizer exists. This is most likely a new service.
-		if g.AlphaFeatureGate.Enabled(AlphaFeatureILBSubsets) {
 		// When ILBSubsets is enabled, new ILB services will not be processed here.
-			// Services that have existing GCE resources created by this controller or the v1 finalizer
+		// Services that have existing GCE resources created by this controller will continue to update.
-			// will continue to update.
+		g.eventRecorder.Eventf(svc, v1.EventTypeNormal, "SkippingEnsureInternalLoadBalancer",
-			klog.V(2).Infof("Skipped ensureInternalLoadBalancer for service %s/%s, since %s feature is enabled.", svc.Namespace, svc.Name, AlphaFeatureILBSubsets)
+			"Skipped ensureInternalLoadBalancer since %s feature is enabled.", AlphaFeatureILBSubsets)
 		return nil, cloudprovider.ImplementedElsewhere
 	}
 	if hasFinalizer(svc, ILBFinalizerV2) {
-			// No V1 resources present - Another controller is handling the resources for this service.
+		// Another controller is handling the resources for this service.
-			klog.V(2).Infof("Skipped ensureInternalLoadBalancer for service %s/%s, as service contains %q finalizer.", svc.Namespace, svc.Name, ILBFinalizerV2)
+		g.eventRecorder.Eventf(svc, v1.EventTypeNormal, "SkippingEnsureInternalLoadBalancer",
+			"Skipped ensureInternalLoadBalancer as service contains '%s' finalizer.", ILBFinalizerV2)
 		return nil, cloudprovider.ImplementedElsewhere
 	}
-	}
 
 	nm := types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}
 
@@ -166,6 +164,11 @@ func (g *Cloud) ensureInternalLoadBalancer(clusterName, clusterID string, svc *v
 		}()
 	}
 
+	// Ensure firewall rules if necessary
+	if err = g.ensureInternalFirewalls(loadBalancerName, ipToUse, clusterID, nm, svc, strconv.Itoa(int(hcPort)), sharedHealthCheck, nodes); err != nil {
+		return nil, err
+	}
+
 	fwdRuleDescription := &forwardingRuleDescription{ServiceName: nm.String()}
 	fwdRuleDescriptionString, err := fwdRuleDescription.marshal()
 	if err != nil {
@@ -197,10 +200,8 @@ func (g *Cloud) ensureInternalLoadBalancer(clusterName, clusterID string, svc *v
 		// Delete existing forwarding rule before making changes to the backend service. For example - changing protocol
 		// of backend service without first deleting forwarding rule will throw an error since the linked forwarding
 		// rule would show the old protocol.
-		if klogV := klog.V(2); klogV.Enabled() {
 		frDiff := cmp.Diff(existingFwdRule, newFwdRule)
-			klogV.Infof("ensureInternalLoadBalancer(%v): forwarding rule changed - Existing - %+v\n, New - %+v\n, Diff(-existing, +new) - %s\n. Deleting existing forwarding rule.", loadBalancerName, existingFwdRule, newFwdRule, frDiff)
+		klog.V(2).Infof("ensureInternalLoadBalancer(%v): forwarding rule changed - Existing - %+v\n, New - %+v\n, Diff(-existing, +new) - %s\n. Deleting existing forwarding rule.", loadBalancerName, existingFwdRule, newFwdRule, frDiff)
-		}
 		if err = ignoreNotFound(g.DeleteRegionForwardingRule(loadBalancerName, g.region)); err != nil {
 			return nil, err
 		}
@@ -220,23 +221,17 @@ func (g *Cloud) ensureInternalLoadBalancer(clusterName, clusterID string, svc *v
 		}
 	}
 
+	// Delete the previous internal load balancer resources if necessary
+	if existingBackendService != nil {
+		g.clearPreviousInternalResources(svc, loadBalancerName, existingBackendService, backendServiceName, hcName)
+	}
+
 	// Get the most recent forwarding rule for the address.
 	updatedFwdRule, err := g.GetRegionForwardingRule(loadBalancerName, g.region)
 	if err != nil {
 		return nil, err
 	}
 
-	ipToUse = updatedFwdRule.IPAddress
-	// Ensure firewall rules if necessary
-	if err = g.ensureInternalFirewalls(loadBalancerName, ipToUse, clusterID, nm, svc, strconv.Itoa(int(hcPort)), sharedHealthCheck, nodes); err != nil {
-		return nil, err
-	}
-
-	// Delete the previous internal load balancer resources if necessary
-	if existingBackendService != nil {
-		g.clearPreviousInternalResources(svc, loadBalancerName, existingBackendService, backendServiceName, hcName)
-	}
-
 	serviceState.InSuccess = true
 	if options.AllowGlobalAccess {
 		serviceState.EnabledGlobalAccess = true
@@ -280,8 +275,7 @@ func (g *Cloud) clearPreviousInternalResources(svc *v1.Service, loadBalancerName
 // updateInternalLoadBalancer is called when the list of nodes has changed. Therefore, only the instance groups
 // and possibly the backend service need to be updated.
 func (g *Cloud) updateInternalLoadBalancer(clusterName, clusterID string, svc *v1.Service, nodes []*v1.Node) error {
-	if g.AlphaFeatureGate.Enabled(AlphaFeatureILBSubsets) && !hasFinalizer(svc, ILBFinalizerV1) {
+	if g.AlphaFeatureGate.Enabled(AlphaFeatureILBSubsets) {
-		klog.V(2).Infof("Skipped updateInternalLoadBalancer for service %s/%s since it does not contain %q finalizer.", svc.Namespace, svc.Name, ILBFinalizerV1)
 		return cloudprovider.ImplementedElsewhere
 	}
 	g.sharedResourceLock.Lock()
@@ -417,7 +411,7 @@ func (g *Cloud) teardownInternalHealthCheckAndFirewall(svc *v1.Service, hcName s
 	return nil
 }
 
-func (g *Cloud) ensureInternalFirewall(svc *v1.Service, fwName, fwDesc, destinationIP string, sourceRanges []string, portRanges []string, protocol v1.Protocol, nodes []*v1.Node, legacyFwName string) error {
+func (g *Cloud) ensureInternalFirewall(svc *v1.Service, fwName, fwDesc string, sourceRanges []string, portRanges []string, protocol v1.Protocol, nodes []*v1.Node, legacyFwName string) error {
 	klog.V(2).Infof("ensureInternalFirewall(%v): checking existing firewall", fwName)
 	targetTags, err := g.GetNodeTags(nodeNames(nodes))
 	if err != nil {
@@ -466,10 +460,6 @@ func (g *Cloud) ensureInternalFirewall(svc *v1.Service, fwName, fwDesc, destinat
 		},
 	}
 
-	if destinationIP != "" {
-		expectedFirewall.DestinationRanges = []string{destinationIP}
-	}
-
 	if existingFirewall == nil {
 		klog.V(2).Infof("ensureInternalFirewall(%v): creating firewall", fwName)
 		err = g.CreateFirewall(expectedFirewall)
@@ -486,7 +476,7 @@ func (g *Cloud) ensureInternalFirewall(svc *v1.Service, fwName, fwDesc, destinat
 	}
 
 	klog.V(2).Infof("ensureInternalFirewall(%v): updating firewall", fwName)
-	err = g.PatchFirewall(expectedFirewall)
+	err = g.UpdateFirewall(expectedFirewall)
 	if err != nil && isForbidden(err) && g.OnXPN() {
 		klog.V(2).Infof("ensureInternalFirewall(%v): do not have permission to update firewall rule (on XPN). Raising event.", fwName)
 		g.raiseFirewallChangeNeededEvent(svc, FirewallToGCloudUpdateCmd(expectedFirewall, g.NetworkProjectID()))
@@ -503,7 +493,7 @@ func (g *Cloud) ensureInternalFirewalls(loadBalancerName, ipAddress, clusterID s
 	if err != nil {
 		return err
 	}
-	err = g.ensureInternalFirewall(svc, MakeFirewallName(loadBalancerName), fwDesc, ipAddress, sourceRanges.StringSlice(), portRanges, protocol, nodes, loadBalancerName)
+	err = g.ensureInternalFirewall(svc, MakeFirewallName(loadBalancerName), fwDesc, sourceRanges.StringSlice(), portRanges, protocol, nodes, loadBalancerName)
 	if err != nil {
 		return err
 	}
@@ -511,7 +501,7 @@ func (g *Cloud) ensureInternalFirewalls(loadBalancerName, ipAddress, clusterID s
 	// Second firewall is for health checking nodes / services
 	fwHCName := makeHealthCheckFirewallName(loadBalancerName, clusterID, sharedHealthCheck)
 	hcSrcRanges := L4LoadBalancerSrcRanges()
-	return g.ensureInternalFirewall(svc, fwHCName, "", "", hcSrcRanges, []string{healthCheckPort}, v1.ProtocolTCP, nodes, "")
+	return g.ensureInternalFirewall(svc, fwHCName, "", hcSrcRanges, []string{healthCheckPort}, v1.ProtocolTCP, nodes, "")
 }
 
 func (g *Cloud) ensureInternalHealthCheck(name string, svcName types.NamespacedName, shared bool, path string, port int32) (*compute.HealthCheck, error) {
@@ -630,22 +620,12 @@ func (g *Cloud) ensureInternalInstanceGroups(name string, nodes []*v1.Node) ([]s
 	klog.V(2).Infof("ensureInternalInstanceGroups(%v): %d nodes over %d zones in region %v", name, len(nodes), len(zonedNodes), g.region)
 	var igLinks []string
 	for zone, nodes := range zonedNodes {
-		if g.AlphaFeatureGate.Enabled(AlphaFeatureSkipIGsManagement) {
-			igs, err := g.FilterInstanceGroupsByNamePrefix(name, zone)
-			if err != nil {
-				return nil, err
-			}
-			for _, ig := range igs {
-				igLinks = append(igLinks, ig.SelfLink)
-			}
-		} else {
 		igLink, err := g.ensureInternalInstanceGroup(name, zone, nodes)
 		if err != nil {
-				return nil, err
+			return []string{}, err
 		}
 		igLinks = append(igLinks, igLink)
 	}
-	}
 
 	return igLinks, nil
 }
@@ -657,15 +637,12 @@ func (g *Cloud) ensureInternalInstanceGroupsDeleted(name string) error {
 		return err
 	}
 
-	// Skip Instance Group deletion if IG management was moved out of k/k code
-	if !g.AlphaFeatureGate.Enabled(AlphaFeatureSkipIGsManagement) {
 	klog.V(2).Infof("ensureInternalInstanceGroupsDeleted(%v): attempting delete instance group in all %d zones", name, len(zones))
 	for _, z := range zones {
 		if err := g.DeleteInstanceGroup(name, z.Name); err != nil && !isNotFoundOrInUse(err) {
 			return err
 		}
 	}
-	}
 	return nil
 }
 
@@ -776,7 +753,6 @@ func firewallRuleEqual(a, b *compute.Firewall) bool {
 		a.Allowed[0].IPProtocol == b.Allowed[0].IPProtocol &&
 		equalStringSets(a.Allowed[0].Ports, b.Allowed[0].Ports) &&
 		equalStringSets(a.SourceRanges, b.SourceRanges) &&
-		equalStringSets(a.DestinationRanges, b.DestinationRanges) &&
 		equalStringSets(a.TargetTags, b.TargetTags)
 }
 
@@ -908,7 +884,7 @@ func getPortRanges(ports []int) (ranges []string) {
 }
 
 func (g *Cloud) getBackendServiceLink(name string) string {
-	return g.projectsBasePath + strings.Join([]string{g.projectID, "regions", g.region, "backendServices", name}, "/")
+	return g.service.BasePath + strings.Join([]string{g.projectID, "regions", g.region, "backendServices", name}, "/")
 }
 
 func getNameFromLink(link string) string {

vendor/k8s.io/cloud-provider-gcp/providers/gce/gce_util.go

@@ -31,24 +31,20 @@ import (
 	"sync"
 
 	"cloud.google.com/go/compute/metadata"
+
 	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
 	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
 	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/mock"
+
 	compute "google.golang.org/api/compute/v1"
 	"google.golang.org/api/googleapi"
 
-	v1 "k8s.io/api/core/v1"
+	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/kubernetes/fake"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 	servicehelper "k8s.io/cloud-provider/service/helpers"
-	netutils "k8s.io/utils/net"
-)
-
-const (
-	// NetLBFinalizerV2 is the finalizer used by newer controllers that manage L4 External LoadBalancer services.
-	NetLBFinalizerV2 = "gke.networking.io/l4-netlb-v2"
 )
 
 func fakeGCECloud(vals TestClusterValues) (*Cloud, error) {
@@ -78,7 +74,6 @@ func fakeGCECloud(vals TestClusterValues) (*Cloud, error) {
 	mockGCE.MockRegionBackendServices.UpdateHook = mock.UpdateRegionBackendServiceHook
 	mockGCE.MockHealthChecks.UpdateHook = mock.UpdateHealthCheckHook
 	mockGCE.MockFirewalls.UpdateHook = mock.UpdateFirewallHook
-	mockGCE.MockFirewalls.PatchHook = mock.UpdateFirewallHook
 
 	keyGA := meta.GlobalKey("key-ga")
 	mockGCE.MockZones.Objects[*keyGA] = &cloud.MockZonesObj{
@@ -126,7 +121,7 @@ type gceInstance struct {
 
 var (
 	autoSubnetIPRange = &net.IPNet{
-		IP:   netutils.ParseIPSloppy("10.128.0.0"),
+		IP:   net.ParseIP("10.128.0.0"),
 		Mask: net.CIDRMask(9, 32),
 	}
 )
@@ -311,7 +306,7 @@ func lastIPInRange(cidr *net.IPNet) net.IP {
 func subnetsInCIDR(subnets []*compute.Subnetwork, cidr *net.IPNet) ([]*compute.Subnetwork, error) {
 	var res []*compute.Subnetwork
 	for _, subnet := range subnets {
-		_, subnetRange, err := netutils.ParseCIDRSloppy(subnet.IpCidrRange)
+		_, subnetRange, err := net.ParseCIDR(subnet.IpCidrRange)
 		if err != nil {
 			return nil, fmt.Errorf("unable to parse CIDR %q for subnet %q: %v", subnet.IpCidrRange, subnet.Name, err)
 		}
@@ -394,23 +389,3 @@ func removeString(slice []string, s string) []string {
 	}
 	return newSlice
 }
-
-// usesL4RBS checks if service uses Regional Backend Service as a Backend.
-// Such services implemented in other controllers and
-// should not be handled by Service Controller.
-func usesL4RBS(service *v1.Service, forwardingRule *compute.ForwardingRule) bool {
-	// Detect RBS by annotation
-	if val, ok := service.Annotations[RBSAnnotationKey]; ok && val == RBSEnabled {
-		return true
-	}
-	// Detect RBS by finalizer
-	if hasFinalizer(service, NetLBFinalizerV2) {
-		return true
-	}
-	// Detect RBS by existing forwarding rule with Backend Service attached
-	if forwardingRule != nil && forwardingRule.BackendService != "" {
-		return true
-	}
-
-	return false
-}

vendor/k8s.io/cloud-provider-gcp/providers/gce/gce_zones.go

@@ -91,5 +91,5 @@ func (g *Cloud) ListZonesInRegion(region string) ([]*compute.Zone, error) {
 }
 
 func (g *Cloud) getRegionLink(region string) string {
-	return g.projectsBasePath + strings.Join([]string{g.projectID, "regions", region}, "/")
+	return g.service.BasePath + strings.Join([]string{g.projectID, "regions", region}, "/")
 }

vendor/k8s.io/cloud-provider-gcp/providers/gce/token_source.go

@@ -44,7 +44,7 @@ const (
 
 /*
  * By default, all the following metrics are defined as falling under
- * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/1209-metrics-stability/kubernetes-control-plane-metrics-stability.md#stability-classes)
+ * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/1209-metrics-stability/20190404-kubernetes-control-plane-metrics-stability.md#stability-classes)
  *
  * Promoting the stability level of the metric is a responsibility of the component owner, since it
  * involves explicitly acknowledging support for the metric across multiple releases, in accordance with

vendor/k8s.io/legacy-cloud-providers/gce/OWNERS

@@ -1,10 +0,0 @@
-# See the OWNERS docs at https://go.k8s.io/owners
-# We are no longer accepting features into k8s.io/legacy-cloud-providers.
-# Any kind/feature PRs must be approved by SIG Cloud Provider going forward.
-emeritus_approvers:
-  - saad-ali
-  - jingxu97
-  - bowei
-  - freehan
-  - mrhohn
-  - cheftako

vendor/modules.txt

@@ -1670,6 +1670,9 @@ k8s.io/cloud-provider/volume/helpers
 # k8s.io/cloud-provider-aws v1.25.1
 ## explicit; go 1.19
 k8s.io/cloud-provider-aws/pkg/providers/v1
+# k8s.io/cloud-provider-gcp/providers v0.25.3
+## explicit; go 1.18
+k8s.io/cloud-provider-gcp/providers/gce
 # k8s.io/component-base v0.25.4
 ## explicit; go 1.19
 k8s.io/component-base/cli/flag
@@ -1737,9 +1740,6 @@ k8s.io/kubectl/pkg/validation
 # k8s.io/kubelet v0.25.4
 ## explicit; go 1.19
 k8s.io/kubelet/config/v1beta1
-# k8s.io/legacy-cloud-providers v0.25.4
-## explicit; go 1.19
-k8s.io/legacy-cloud-providers/gce
 # k8s.io/mount-utils v0.25.4
 ## explicit; go 1.19
 k8s.io/mount-utils