kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13201 from zetaab/removesa 

cleanup GCP Cluster Service Accounts
This commit is contained in:
Kubernetes Prow Robot 2022-02-23 04:24:19 -08:00 committed by GitHub
parent 70f436b787 67beb3fef5
commit 02dc9dd8b3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 267 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cloudmock/gce/BUILD.bazel generated
View File

@ -19,7 +19,6 @@ go_library(
"//upup/pkg/fi/cloudup/gce:go_default_library",
"//vendor/google.golang.org/api/cloudresourcemanager/v1:go_default_library",
"//vendor/google.golang.org/api/compute/v1:go_default_library",
"//vendor/google.golang.org/api/iam/v1:go_default_library",
"//vendor/google.golang.org/api/storage/v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/klog/v2:go_default_library",

7
cloudmock/gce/mock_gce_cloud.go
View File

@ -21,7 +21,6 @@ import (
"google.golang.org/api/cloudresourcemanager/v1"
compute "google.golang.org/api/compute/v1"
"google.golang.org/api/iam/v1"
"google.golang.org/api/storage/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/klog/v2"
@ -46,7 +45,7 @@ type MockGCECloud struct {
computeClient *mockcompute.MockClient
dnsClient *mockdns.MockClient
iamClient *iam.Service
iamClient *mockiam.MockClient
storageClient *storage.Service
cloudResourceManagerClient *cloudresourcemanager.Service
}
@ -60,7 +59,7 @@ func InstallMockGCECloud(region string, project string) *MockGCECloud {
region: region,
computeClient: mockcompute.NewMockClient(project),
dnsClient: mockdns.NewMockClient(),
iamClient: mockiam.New(project),
iamClient: mockiam.NewMockClient(project),
storageClient: mockstorage.New(),
cloudResourceManagerClient: mockcloudresourcemanager.New(),
}
@ -117,7 +116,7 @@ func (c *MockGCECloud) Storage() *storage.Service {
}
// IAM returns the IAM client
func (c *MockGCECloud) IAM() *iam.Service {
func (c *MockGCECloud) IAM() gce.IamClient {
return c.iamClient
}

5
cloudmock/gce/mockiam/BUILD.bazel generated
View File

@ -9,9 +9,8 @@ go_library(
importpath = "k8s.io/kops/cloudmock/gce/mockiam",
visibility = ["//visibility:public"],
deps = [
"//cloudmock/gce/gcphttp:go_default_library",
"//upup/pkg/fi/cloudup/gce:go_default_library",
"//vendor/google.golang.org/api/googleapi:go_default_library",
"//vendor/google.golang.org/api/iam/v1:go_default_library",
"//vendor/google.golang.org/api/option:go_default_library",
"//vendor/k8s.io/klog/v2:go_default_library",
],
)

71
cloudmock/gce/mockiam/api.go
View File

@ -1,5 +1,5 @@
/*
Copyright 2021 The Kubernetes Authors.
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -17,63 +17,30 @@ limitations under the License.
package mockiam
import (
"context"
"fmt"
"net/http"
"strings"
"google.golang.org/api/iam/v1"
option "google.golang.org/api/option"
"k8s.io/klog/v2"
"google.golang.org/api/googleapi"
"k8s.io/kops/upup/pkg/fi/cloudup/gce"
)
// mockIAMService represents a mocked IAM client.
type mockIAMService struct {
svc *iam.Service
serviceAccounts serviceAccounts
// MockClient represents a mocked IAM client.
type MockClient struct {
serviceAccounts *serviceAccountClient
}
// New creates a new mock IAM client.
func New(project string) *iam.Service {
ctx := context.Background()
var _ gce.IamClient = &MockClient{}
s := &mockIAMService{}
s.serviceAccounts.Init()
httpClient := &http.Client{Transport: s}
svc, err := iam.NewService(ctx, option.WithHTTPClient(httpClient))
if err != nil {
klog.Fatalf("failed to build mock iam service: %v", err)
// NewMockClient creates a new mock client.
func NewMockClient(project string) *MockClient {
return &MockClient{
serviceAccounts: newServiceAccounts(project),
}
s.svc = svc
return svc
}
func (s *mockIAMService) RoundTrip(request *http.Request) (*http.Response, error) {
url := request.URL
if url.Host != "iam.googleapis.com" {
return nil, fmt.Errorf("unexpected host in request %#v", request)
}
pathTokens := strings.Split(strings.TrimPrefix(url.Path, "/"), "/")
if len(pathTokens) >= 1 && pathTokens[0] == "v1" {
if len(pathTokens) >= 3 && pathTokens[1] == "projects" {
projectID := pathTokens[2]
if len(pathTokens) >= 5 && pathTokens[3] == "serviceAccounts" {
serviceAccount := pathTokens[4]
if len(pathTokens) == 5 && request.Method == "GET" {
return s.serviceAccounts.Get(projectID, serviceAccount, request)
}
}
if len(pathTokens) == 4 && pathTokens[3] == "serviceAccounts" && request.Method == "POST" {
return s.serviceAccounts.Create(projectID, request)
}
}
}
klog.Warningf("request: %s %s %#v", request.Method, request.URL, request)
return nil, fmt.Errorf("unhandled request %#v", request)
func (c *MockClient) ServiceAccounts() gce.ServiceAccountClient {
return c.serviceAccounts
}
func notFoundError() error {
return &googleapi.Error{
Code: 404,
}
}

121
cloudmock/gce/mockiam/serviceaccounts.go
View File

@ -1,5 +1,5 @@
/*
Copyright 2021 The Kubernetes Authors.
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -17,75 +17,76 @@ limitations under the License.
package mockiam
import (
"encoding/json"
"io"
"net/http"
"context"
"fmt"
"strings"
"sync"
"google.golang.org/api/iam/v1"
"k8s.io/kops/cloudmock/gce/gcphttp"
"k8s.io/kops/upup/pkg/fi/cloudup/gce"
)
// serviceAccounts manages the ServiceAccount resources.
type serviceAccounts struct {
mutex sync.Mutex
serviceAccountsByEmail map[string]*iam.ServiceAccount
type serviceAccountClient struct {
// serviceaccounts are keyed by name.
serviceaccounts map[string]*iam.ServiceAccount
project string
sync.Mutex
}
func (s *serviceAccounts) Init() {
s.serviceAccountsByEmail = make(map[string]*iam.ServiceAccount)
var _ gce.ServiceAccountClient = &serviceAccountClient{}
func newServiceAccounts(project string) *serviceAccountClient {
return &serviceAccountClient{
serviceaccounts: map[string]*iam.ServiceAccount{},
project: project,
}
}
func (s *serviceAccounts) Get(projectID string, serviceAccount string, request *http.Request) (*http.Response, error) {
s.mutex.Lock()
defer s.mutex.Unlock()
sa := s.serviceAccountsByEmail[serviceAccount]
if sa == nil {
return gcphttp.ErrorNotFound("Unknown service account")
func (s *serviceAccountClient) Get(ctx context.Context, name string) (*iam.ServiceAccount, error) {
s.Lock()
defer s.Unlock()
result, ok := s.serviceaccounts[name]
if !ok {
return nil, notFoundError()
}
return gcphttp.OKResponse(sa)
return result, nil
}
func (s *serviceAccounts) Create(projectID string, request *http.Request) (*http.Response, error) {
b, err := io.ReadAll(request.Body)
if err != nil {
return gcphttp.ErrorBadRequest("")
}
req := &iam.CreateServiceAccountRequest{}
if err := json.Unmarshal(b, &req); err != nil {
return gcphttp.ErrorBadRequest("")
}
if req.AccountId == "" {
return gcphttp.ErrorBadRequest("")
}
sa := &iam.ServiceAccount{
Email: req.AccountId + "@" + projectID + ".iam.gserviceaccount.com",
Description: req.ServiceAccount.Description,
DisplayName: req.ServiceAccount.DisplayName,
}
s.mutex.Lock()
defer s.mutex.Unlock()
existing := s.serviceAccountsByEmail[sa.Email]
if existing != nil {
// TODO: details
// "details": [
// {
// "@type": "type.googleapis.com/google.rpc.ResourceInfo",
// "resourceName": "projects/testproject/serviceAccounts/testaccount@testproject.iam.gserviceaccount.com"
// }
return gcphttp.ErrorAlreadyExists("Service account %s already exists within project projects/%s.", req.AccountId, projectID)
}
s.serviceAccountsByEmail[sa.Email] = sa
return gcphttp.OKResponse(sa)
func (s *serviceAccountClient) Update(ctx context.Context, name string, sa *iam.ServiceAccount) (*iam.ServiceAccount, error) {
s.Lock()
defer s.Unlock()
s.serviceaccounts[name] = sa
return s.serviceaccounts[name], nil
}
func (s *serviceAccountClient) Create(ctx context.Context, name string, req *iam.CreateServiceAccountRequest) (*iam.ServiceAccount, error) {
s.Lock()
defer s.Unlock()
fqn := fmt.Sprintf("%s/serviceAccounts/%s", name, req.ServiceAccount.Email)
req.ServiceAccount.Name = fqn
s.serviceaccounts[fqn] = req.ServiceAccount
return s.serviceaccounts[fqn], nil
}
func (s *serviceAccountClient) Delete(name string) (*iam.Empty, error) {
s.Lock()
defer s.Unlock()
fqn := "projects/" + s.project + "/serviceAccounts/" + name
if _, ok := s.serviceaccounts[fqn]; !ok {
return nil, nil
}
delete(s.serviceaccounts, fqn)
return nil, nil
}
func (s *serviceAccountClient) List(ctx context.Context, project string) ([]*iam.ServiceAccount, error) {
s.Lock()
defer s.Unlock()
var r []*iam.ServiceAccount
for k, v := range s.serviceaccounts {
if strings.Contains(k, project) {
r = append(r, v)
}
}
return r, nil
}

6
pkg/model/gcemodel/context.go
View File

@ -128,13 +128,13 @@ func (c *GCEModelContext) LinkToServiceAccount(ig *kops.InstanceGroup) *gcetasks
name := ""
switch role {
case kops.InstanceGroupRoleAPIServer, kops.InstanceGroupRoleMaster:
name = "control-plane"
name = gce.ControlPlane
case kops.InstanceGroupRoleBastion:
name = "bastion"
name = gce.Bastion
case kops.InstanceGroupRoleNode:
name = "node"
name = gce.Node
default:
klog.Fatalf("unknown role %q", role)

1
pkg/resources/gce/BUILD.bazel generated
View File

@ -17,6 +17,7 @@ go_library(
"//upup/pkg/fi/cloudup/gce:go_default_library",
"//vendor/google.golang.org/api/compute/v1:go_default_library",
"//vendor/google.golang.org/api/dns/v1:go_default_library",
"//vendor/google.golang.org/api/iam/v1:go_default_library",
"//vendor/k8s.io/klog/v2:go_default_library",
],
)

51
pkg/resources/gce/gce.go
View File

@ -23,6 +23,7 @@ import (
compute "google.golang.org/api/compute/v1"
clouddns "google.golang.org/api/dns/v1"
"google.golang.org/api/iam/v1"
"k8s.io/klog/v2"
"k8s.io/kops/pkg/dns"
"k8s.io/kops/pkg/resources"
@ -47,6 +48,7 @@ const (
typeSubnet = "Subnet"
typeRouter = "Router"
typeDNSRecord = "DNSRecord"
typeServiceAccount = "ServiceAccount"
)
// Maximum number of `-` separated tokens in a name
@ -106,6 +108,7 @@ func ListResourcesGCE(gceCloud gce.GCECloud, clusterName string, region string)
d.listSubnets,
d.listRouters,
d.listNetworks,
d.listServiceAccounts,
}
for _, fn := range listFunctions {
resourceTrackers, err := fn()
@ -986,6 +989,54 @@ func deleteRouter(cloud fi.Cloud, r *resources.Resource) error {
return c.WaitForOp(op)
}
func (d *clusterDiscoveryGCE) listServiceAccounts() ([]*resources.Resource, error) {
c := d.gceCloud
ctx := context.Background()
sas, err := c.IAM().ServiceAccounts().List(ctx, fmt.Sprintf("projects/%s", c.Project()))
if err != nil {
return nil, fmt.Errorf("error listing ServiceAccounts %w", err)
}
var resourceTrackers []*resources.Resource
for _, sa := range sas {
tokens := strings.Split(gce.LastComponent(sa.Name), "@")
if len(tokens) != 2 {
return nil, fmt.Errorf("Invalid service account email '%s'", gce.LastComponent(sa.Name))
}
accountID := tokens[0]
names := []string{gce.ControlPlane, gce.Bastion, gce.Node}
for _, name := range names {
generatedName, err := gce.ServiceAccountName(name, d.clusterName)
if err != nil {
return nil, err
}
if generatedName == accountID {
resourceTracker := &resources.Resource{
Name: gce.LastComponent(sa.Name),
ID: sa.Name,
Type: typeServiceAccount,
Deleter: deleteServiceAccount,
Obj: sa,
}
klog.V(4).Infof("found resource: %s", sa.Name)
resourceTrackers = append(resourceTrackers, resourceTracker)
break
}
}
}
return resourceTrackers, nil
}
func deleteServiceAccount(cloud fi.Cloud, r *resources.Resource) error {
c := cloud.(gce.GCECloud)
o := r.Obj.(*iam.ServiceAccount)
klog.V(2).Infof("deleting GCE ServiceAccount %s", o.Name)
_, err := c.IAM().ServiceAccounts().Delete(o.Name)
return err
}
func (d *clusterDiscoveryGCE) listNetworks() ([]*resources.Resource, error) {
// Templates are very accurate because of the metadata, so use those as the sanity check
templates, err := d.findInstanceTemplates()

1
upup/pkg/fi/cloudup/gce/BUILD.bazel generated
View File

@ -8,6 +8,7 @@ go_library(
"gce_apitarget.go",
"gce_cloud.go",
"gce_url.go",
"iam.go",
"instancegroups.go",
"labels.go",
"network.go",

28
upup/pkg/fi/cloudup/gce/gce_cloud.go
View File

@ -27,7 +27,6 @@ import (
"golang.org/x/oauth2/google"
"google.golang.org/api/cloudresourcemanager/v1"
compute "google.golang.org/api/compute/v1"
"google.golang.org/api/iam/v1"
oauth2 "google.golang.org/api/oauth2/v2"
"google.golang.org/api/storage/v1"
"k8s.io/klog/v2"
@ -42,7 +41,7 @@ type GCECloud interface {
fi.Cloud
Compute() ComputeClient
Storage() *storage.Service
IAM() *iam.Service
IAM() IamClient
CloudDNS() DNSClient
Project() string
WaitForOp(op *compute.Operation) error
@ -59,7 +58,7 @@ type GCECloud interface {
type gceCloudImplementation struct {
compute *computeClientImpl
storage *storage.Service
iam *iam.Service
iam *iamClientImpl
dns *dnsClientImpl
// cloudResourceManager is the client for the cloudresourcemanager API
@ -139,7 +138,7 @@ func NewGCECloud(region string, project string, labels map[string]string) (GCECl
}
c.storage = storageService
iamService, err := iam.NewService(ctx)
iamService, err := newIamClientImpl(ctx)
if err != nil {
return nil, fmt.Errorf("error building IAM API client: %v", err)
}
@ -202,7 +201,7 @@ func (c *gceCloudImplementation) Storage() *storage.Service {
}
// IAM returns the IAM client
func (c *gceCloudImplementation) IAM() *iam.Service {
func (c *gceCloudImplementation) IAM() IamClient {
return c.iam
}
@ -378,3 +377,22 @@ func (c *gceCloudImplementation) getTokenInfo(ctx context.Context) (*oauth2.Toke
return tokenInfo, nil
}
// SplitServiceAccountEmail splits service account email
func SplitServiceAccountEmail(email string) (string, string, error) {
accountID := ""
projectID := ""
tokens := strings.Split(email, "@")
if len(tokens) == 2 {
accountID = tokens[0]
if strings.HasSuffix(tokens[1], ".iam.gserviceaccount.com") {
projectID = strings.TrimSuffix(tokens[1], ".iam.gserviceaccount.com")
}
}
if accountID == "" || projectID == "" {
return "", "", fmt.Errorf("unexpected format for ServiceAccount email %q", email)
}
return accountID, projectID, nil
}

91
upup/pkg/fi/cloudup/gce/iam.go Normal file
View File

@ -0,0 +1,91 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package gce
import (
"context"
"fmt"
"google.golang.org/api/iam/v1"
)
type IamClient interface {
ServiceAccounts() ServiceAccountClient
}
type iamClientImpl struct {
srv *iam.Service
}
var _ IamClient = &iamClientImpl{}
func newIamClientImpl(ctx context.Context) (*iamClientImpl, error) {
srv, err := iam.NewService(ctx)
if err != nil {
return nil, fmt.Errorf("error building iam API client: %v", err)
}
return &iamClientImpl{
srv: srv,
}, nil
}
func (i *iamClientImpl) ServiceAccounts() ServiceAccountClient {
return &serviceAccountClientImpl{
srv: i.srv.Projects.ServiceAccounts,
}
}
type ServiceAccountClient interface {
Get(ctx context.Context, fqn string) (*iam.ServiceAccount, error)
Create(ctx context.Context, project string, req *iam.CreateServiceAccountRequest) (*iam.ServiceAccount, error)
Update(ctx context.Context, fqn string, sa *iam.ServiceAccount) (*iam.ServiceAccount, error)
Delete(saName string) (*iam.Empty, error)
List(ctx context.Context, project string) ([]*iam.ServiceAccount, error)
}
type serviceAccountClientImpl struct {
srv *iam.ProjectsServiceAccountsService
}
var _ ServiceAccountClient = &serviceAccountClientImpl{}
func (s *serviceAccountClientImpl) Create(ctx context.Context, project string, req *iam.CreateServiceAccountRequest) (*iam.ServiceAccount, error) {
return s.srv.Create(project, req).Context(ctx).Do()
}
func (s *serviceAccountClientImpl) Update(ctx context.Context, fqn string, sa *iam.ServiceAccount) (*iam.ServiceAccount, error) {
return s.srv.Update(fqn, sa).Context(ctx).Do()
}
func (s *serviceAccountClientImpl) Get(ctx context.Context, fqn string) (*iam.ServiceAccount, error) {
return s.srv.Get(fqn).Context(ctx).Do()
}
func (s *serviceAccountClientImpl) List(ctx context.Context, project string) ([]*iam.ServiceAccount, error) {
var sas []*iam.ServiceAccount
if err := s.srv.List(project).Pages(ctx, func(p *iam.ListServiceAccountsResponse) error {
sas = append(sas, p.Accounts...)
return nil
}); err != nil {
return nil, err
}
return sas, nil
}
func (s *serviceAccountClientImpl) Delete(saName string) (*iam.Empty, error) {
return s.srv.Delete(saName).Do()
}

3
upup/pkg/fi/cloudup/gce/labels.go
View File

@ -31,6 +31,9 @@ const (
GceLabelNameInstanceGroup = "k8s-io-instance-group"
GceLabelNameRolePrefix = "k8s-io-role-"
GceLabelNameEtcdClusterPrefix = "k8s-io-etcd-"
ControlPlane = "control-plane"
Bastion = "bastion"
Node = "node"
)
// EncodeGCELabel encodes a string into an RFC1035 compatible value, suitable for use as GCE label key or value

34
upup/pkg/fi/cloudup/gcetasks/serviceaccount.go
View File

@ -20,7 +20,6 @@ import (
"context"
"fmt"
"reflect"
"strings"
"google.golang.org/api/iam/v1"
"k8s.io/klog/v2"
@ -48,24 +47,6 @@ func (e *ServiceAccount) CompareWithID() *string {
return e.Email
}
func splitServiceAccountEmail(email string) (string, string, error) {
accountID := ""
projectID := ""
tokens := strings.Split(email, "@")
if len(tokens) == 2 {
accountID = tokens[0]
if strings.HasSuffix(tokens[1], ".iam.gserviceaccount.com") {
projectID = strings.TrimSuffix(tokens[1], ".iam.gserviceaccount.com")
}
}
if accountID == "" || projectID == "" {
return "", "", fmt.Errorf("unexpected format for ServiceAccount email %q", email)
}
return accountID, projectID, nil
}
func (e *ServiceAccount) Find(c *fi.Context) (*ServiceAccount, error) {
cloud := c.Cloud.(gce.GCECloud)
@ -78,12 +59,12 @@ func (e *ServiceAccount) Find(c *fi.Context) (*ServiceAccount, error) {
return e, nil
}
_, projectID, err := splitServiceAccountEmail(email)
_, projectID, err := gce.SplitServiceAccountEmail(email)
if err != nil {
return nil, err
}
fqn := "projects/" + projectID + "/serviceAccounts/" + email
sa, err := cloud.IAM().Projects.ServiceAccounts.Get(fqn).Context(ctx).Do()
sa, err := cloud.IAM().ServiceAccounts().Get(ctx, fqn)
if err != nil {
if gce.IsNotFound(err) {
return nil, nil
@ -132,7 +113,7 @@ func (_ *ServiceAccount) RenderGCE(t *gce.GCEAPITarget, a, e, changes *ServiceAc
}
}
accountID, projectID, err := splitServiceAccountEmail(email)
accountID, projectID, err := gce.SplitServiceAccountEmail(email)
if err != nil {
return err
}
@ -145,12 +126,13 @@ func (_ *ServiceAccount) RenderGCE(t *gce.GCEAPITarget, a, e, changes *ServiceAc
sa := &iam.CreateServiceAccountRequest{
AccountId: accountID,
ServiceAccount: &iam.ServiceAccount{
Email: email,
Description: fi.StringValue(e.Description),
DisplayName: fi.StringValue(e.DisplayName),
},
}
created, err := cloud.IAM().Projects.ServiceAccounts.Create("projects/"+projectID, sa).Context(ctx).Do()
created, err := cloud.IAM().ServiceAccounts().Create(ctx, "projects/"+projectID, sa)
if err != nil {
return fmt.Errorf("error creating ServiceAccount %q: %w", fqn, err)
}
@ -160,10 +142,12 @@ func (_ *ServiceAccount) RenderGCE(t *gce.GCEAPITarget, a, e, changes *ServiceAc
} else {
if changes.Description != nil || changes.DisplayName != nil {
sa := &iam.ServiceAccount{
Email: email,
Description: fi.StringValue(e.Description),
DisplayName: fi.StringValue(e.DisplayName),
}
_, err := cloud.IAM().Projects.ServiceAccounts.Update(fqn, sa).Context(ctx).Do()
_, err := cloud.IAM().ServiceAccounts().Update(ctx, fqn, sa)
if err != nil {
return fmt.Errorf("error creating ServiceAccount %q: %w", fqn, err)
}
@ -196,7 +180,7 @@ func (_ *ServiceAccount) RenderTerraform(t *terraform.TerraformTarget, a, e, cha
}
email := fi.StringValue(e.Email)
accountID, projectID, err := splitServiceAccountEmail(email)
accountID, projectID, err := gce.SplitServiceAccountEmail(email)
if err != nil {
return err
}