Adds a gce-service-account flag so you BYO service-account

Generated code and some cleanup

Not sure where that code went

Tests for service account

fixes case on gceserviceaccount

cmd/kops/create_cluster.go

@@ -161,6 +161,9 @@ type CreateClusterOptions struct {
 	// OpenstackLBOctavia is boolean value should we use octavia or old loadbalancer api
 	OpenstackLBOctavia bool
 
+	// GCEServiceAccount specifies the service account with which the GCE VM runs
+	GCEServiceAccount string
+
 	// ConfigBase is the location where we will store the configuration, it defaults to the state store
 	ConfigBase string
 
@@ -298,7 +301,6 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 	cmd.Flags().StringSliceVar(&options.Zones, "zones", options.Zones, "Zones in which to run the cluster")
 	cmd.Flags().StringSliceVar(&options.MasterZones, "master-zones", options.MasterZones, "Zones in which to run masters (must be an odd number)")
 
-	cmd.Flags().StringVar(&options.Project, "project", options.Project, "Project to use (must be set on GCE)")
 	cmd.Flags().StringVar(&options.KubernetesVersion, "kubernetes-version", options.KubernetesVersion, "Version of kubernetes to run (defaults to version in channel)")
 
 	cmd.Flags().StringVar(&options.ContainerRuntime, "container-runtime", options.ContainerRuntime, "Container runtime to use: containerd, docker")
@@ -373,6 +375,10 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 		cmd.Flags().StringSliceVar(&options.Overrides, "override", options.Overrides, "Directly configure values in the spec")
 	}
 
+	// GCE flags
+	cmd.Flags().StringVar(&options.Project, "project", options.Project, "Project to use (must be set on GCE)")
+	cmd.Flags().StringVar(&options.GCEServiceAccount, "gce-service-account", options.GCEServiceAccount, "Service account with which the GCE VM runs. Warning: if not set, VMs will run as default compute service account.")
+
 	if featureflag.VSphereCloudProvider.Enabled() {
 		// vSphere flags
 		cmd.Flags().StringVar(&options.VSphereServer, "vsphere-server", options.VSphereServer, "vsphere-server is required for vSphere. Set vCenter URL Ex: 10.192.10.30 or myvcenter.io (without https://)")
@@ -988,6 +994,13 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
 			}
 			cluster.Spec.Project = project
 		}
+		if c.GCEServiceAccount != "" {
+			klog.Infof("using GCE service account: %v", c.GCEServiceAccount)
+			cluster.Spec.GCEServiceAccount = fi.String(c.GCEServiceAccount)
+		} else {
+			klog.Warning("using GCE default service account")
+			cluster.Spec.GCEServiceAccount = fi.String("default")
+		}
 	}
 
 	if c.KubernetesVersion != "" {

docs/cli/kops_create_cluster.md

@@ -81,6 +81,7 @@ kops create cluster [flags]
       --dry-run                          If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest.
       --encrypt-etcd-storage             Generate key in aws kms and use it for encrypt etcd volumes
       --etcd-storage-type string         The default storage type for etc members
+      --gce-service-account string       Service account with which the GCE VM runs. Warning: if not set, VMs will run as default compute service account.
   -h, --help                             help for cluster
       --image string                     Image to use for all instances.
       --kubernetes-version string        Version of kubernetes to run (defaults to version in channel)

docs/releases/1.18-NOTES.md

@@ -12,6 +12,8 @@
 
 * New clusters in GCE are configured to run the [metadata-proxy](https://github.com/kubernetes/kubernetes/tree/master/cluster/addons/metadata-proxy) by default. The proxy runs as a DaemonSet and lands on nodes with the nodeLabel `cloud.google.com/metadata-proxy-ready: "true"`. If you want to enable metadata-proxy on an existing cluster/instance group, add that nodeLabel to your instancegroup specs (`kops edit ig ...`) and run `kops update cluster`. When the changes are applied, the proxy will roll out to those targeted nodes.
 
+* GCE has a new flag: `--gce-service-account`. This takes the email of an existing GCP service account and launches the instances with it. This setting applies to the whole cluster (ie: it is not currently designed to support Instance Groups with different service accounts). If you do not specify a service account during cluster creation, the default compute service account will be used which matches the prior behavior.
+
 # Breaking changes
 
 * Terraform users on AWS may need to rename some resources in their state file in order to prepare for Terraform 0.12 support. See Required Actions below.

k8s/crds/kops.k8s.io_clusters.yaml

@@ -738,6 +738,10 @@ spec:
                     type: array
                 type: object
               type: array
+            gceServiceAccount:
+              description: gceServiceAccount specifies the service account with which
+                the GCE VM runs
+              type: string
             gossipConfig:
               description: GossipConfig for the cluster assuming the use of gossip
                 DNS

pkg/apis/kops/cluster.go

@@ -191,6 +191,8 @@ type ClusterSpec struct {
 	SysctlParameters []string `json:"sysctlParameters,omitempty"`
 	// RollingUpdate defines the default rolling-update settings for instance groups
 	RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"`
+	// GCEServiceAccount specifies the service account with which the GCE VM runs
+	GCEServiceAccount string `json:"gceServiceAccount,omitempty"`
 }
 
 // NodeAuthorizationSpec is used to node authorization

pkg/apis/kops/v1alpha2/cluster.go

@@ -189,6 +189,8 @@ type ClusterSpec struct {
 	SysctlParameters []string `json:"sysctlParameters,omitempty"`
 	// RollingUpdate defines the default rolling-update settings for instance groups
 	RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"`
+	// GCEServiceAccount specifies the service account with which the GCE VM runs
+	GCEServiceAccount string `json:"gceServiceAccount,omitempty"`
 }
 
 // NodeAuthorizationSpec is used to node authorization

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -1995,6 +1995,7 @@ func autoConvert_v1alpha2_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out *
 	} else {
 		out.RollingUpdate = nil
 	}
+	out.GCEServiceAccount = in.GCEServiceAccount
 	return nil
 }
 
@@ -2318,6 +2319,7 @@ func autoConvert_kops_ClusterSpec_To_v1alpha2_ClusterSpec(in *kops.ClusterSpec,
 	} else {
 		out.RollingUpdate = nil
 	}
+	out.GCEServiceAccount = in.GCEServiceAccount
 	return nil
 }
 

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -821,6 +821,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 		*out = new(RollingUpdate)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.GCEServiceAccount != nil {
+		in, out := &in.GCEServiceAccount, &out.GCEServiceAccount
+		*out = new(string)
+		**out = **in
+	}
 	return
 }
 

pkg/apis/kops/zz_generated.deepcopy.go

@@ -921,6 +921,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 		*out = new(RollingUpdate)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.GCEServiceAccount != nil {
+		in, out := &in.GCEServiceAccount, &out.GCEServiceAccount
+		*out = new(string)
+		**out = **in
+	}
 	return
 }
 

pkg/model/gcemodel/autoscalinggroup.go

@@ -35,6 +35,7 @@ const (
 	DefaultVolumeType = "pd-standard"
 )
 
+// TODO: rework these parts to be more GCE native. ie: Managed Instance Groups > ASGs
 // AutoscalingGroupModelBuilder configures AutoscalingGroup objects
 type AutoscalingGroupModelBuilder struct {
 	*GCEModelContext
@@ -70,7 +71,6 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
 			}
 
 			namePrefix := gce.LimitedLengthName(name, gcetasks.InstanceTemplateNamePrefixMaxLength)
-
 			t := &gcetasks.InstanceTemplate{
 				Name:           s(name),
 				NamePrefix:     s(namePrefix),
@@ -89,7 +89,6 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
 					"monitoring",
 					"logging-write",
 				},
-
 				Metadata: map[string]*fi.ResourceHolder{
 					"startup-script": startupScript,
 					//"config": resources/config.yaml $nodeset.Name
@@ -121,6 +120,7 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
 			switch ig.Spec.Role {
 			case kops.InstanceGroupRoleMaster:
 				// Grant DNS permissions
+				// TODO: migrate to IAM permissions instead of oldschool scopes?
 				t.Scopes = append(t.Scopes, "https://www.googleapis.com/auth/ndev.clouddns.readwrite")
 				t.Tags = append(t.Tags, b.GCETagForRole(kops.InstanceGroupRoleMaster))
 
@@ -139,6 +139,7 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
 				t.CanIPForward = fi.Bool(true)
 			}
 
+			t.ServiceAccount = b.Cluster.Spec.GCEServiceAccount
 			//labels, err := b.CloudTagsForInstanceGroup(ig)
 			//if err != nil {
 			//	return fmt.Errorf("error building cloud tags: %v", err)

tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml

@@ -0,0 +1,153 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  name: ha-gce.example.com
+spec:
+  api:
+    dns: {}
+  authorization:
+    rbac: {}
+  channel: stable
+  cloudProvider: gce
+  configBase: memfs://tests/ha-gce.example.com
+  containerRuntime: docker
+  etcdClusters:
+  - cpuRequest: 200m
+    etcdMembers:
+    - instanceGroup: master-us-test1-a
+      name: a
+    - instanceGroup: master-us-test1-b
+      name: b
+    - instanceGroup: master-us-test1-c
+      name: c
+    memoryRequest: 100Mi
+    name: main
+  - cpuRequest: 100m
+    etcdMembers:
+    - instanceGroup: master-us-test1-a
+      name: a
+    - instanceGroup: master-us-test1-b
+      name: b
+    - instanceGroup: master-us-test1-c
+      name: c
+    memoryRequest: 100Mi
+    name: events
+  iam:
+    allowContainerRegistry: true
+    legacy: false
+  kubelet:
+    anonymousAuth: false
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  kubernetesVersion: v1.15.6-beta.1
+  masterPublicName: api.ha-gce.example.com
+  networking:
+    kubenet: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  project: testproject
+  sshAccess:
+  - 0.0.0.0/0
+  subnets:
+  - name: us-test1
+    region: us-test1
+    type: Public
+  topology:
+    dns:
+      type: Public
+    masters: public
+    nodes: public
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: ha-gce.example.com
+  name: master-us-test1-a
+spec:
+  image: cos-cloud/cos-stable-65-10323-99-0
+  machineType: n1-standard-1
+  maxSize: 1
+  minSize: 1
+  nodeLabels:
+    cloud.google.com/metadata-proxy-ready: "true"
+    kops.k8s.io/instancegroup: master-us-test1-a
+  role: Master
+  subnets:
+  - us-test1
+  zones:
+  - us-test1-a
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: ha-gce.example.com
+  name: master-us-test1-b
+spec:
+  image: cos-cloud/cos-stable-65-10323-99-0
+  machineType: n1-standard-1
+  maxSize: 1
+  minSize: 1
+  nodeLabels:
+    cloud.google.com/metadata-proxy-ready: "true"
+    kops.k8s.io/instancegroup: master-us-test1-b
+  role: Master
+  subnets:
+  - us-test1
+  zones:
+  - us-test1-b
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: ha-gce.example.com
+  name: master-us-test1-c
+spec:
+  image: cos-cloud/cos-stable-65-10323-99-0
+  machineType: n1-standard-1
+  maxSize: 1
+  minSize: 1
+  nodeLabels:
+    cloud.google.com/metadata-proxy-ready: "true"
+    kops.k8s.io/instancegroup: master-us-test1-c
+  role: Master
+  subnets:
+  - us-test1
+  zones:
+  - us-test1-c
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: ha-gce.example.com
+  name: nodes
+spec:
+  image: cos-cloud/cos-stable-65-10323-99-0
+  machineType: n1-standard-2
+  maxSize: 2
+  minSize: 2
+  nodeLabels:
+    cloud.google.com/metadata-proxy-ready: "true"
+    kops.k8s.io/instancegroup: nodes
+  role: Node
+  subnets:
+  - us-test1
+  zones:
+  - us-test1-a
+  - us-test1-b
+  - us-test1-c

tests/integration/create_cluster/gce_byo_sa/options.yaml

@@ -0,0 +1,9 @@
+ClusterName: ha-gce.example.com
+Zones:
+- us-test1-a
+MasterZones:
+- us-test1-a
+Cloud: gce
+KubernetesVersion: v1.15.6-beta.1
+Project: testproject
+GCEServiceAccount: test-account@testproject.iam.gserviceaccount.com

tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml

@@ -4,6 +4,7 @@ metadata:
   creationTimestamp: "2017-01-01T00:00:00Z"
   name: ha-gce.example.com
 spec:
+  GCEServiceAccount: default
   api:
     dns: {}
   authorization:

tests/integration/update_cluster/ha_gce/kubernetes.tf

@@ -397,10 +397,6 @@ resource "google_compute_instance_template" "master-us-test1-a-ha-gce-example-co
   can_ip_forward = true
   machine_type   = "n1-standard-1"
 
-  service_account = {
-    scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_write", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"]
-  }
-
   scheduling = {
     automatic_restart   = true
     on_host_maintenance = "MIGRATE"
@@ -438,10 +434,6 @@ resource "google_compute_instance_template" "master-us-test1-b-ha-gce-example-co
   can_ip_forward = true
   machine_type   = "n1-standard-1"
 
-  service_account = {
-    scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_write", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"]
-  }
-
   scheduling = {
     automatic_restart   = true
     on_host_maintenance = "MIGRATE"
@@ -479,10 +471,6 @@ resource "google_compute_instance_template" "master-us-test1-c-ha-gce-example-co
   can_ip_forward = true
   machine_type   = "n1-standard-1"
 
-  service_account = {
-    scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_write", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"]
-  }
-
   scheduling = {
     automatic_restart   = true
     on_host_maintenance = "MIGRATE"
@@ -520,10 +508,6 @@ resource "google_compute_instance_template" "nodes-ha-gce-example-com" {
   can_ip_forward = true
   machine_type   = "n1-standard-2"
 
-  service_account = {
-    scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_only"]
-  }
-
   scheduling = {
     automatic_restart   = true
     on_host_maintenance = "MIGRATE"

upup/pkg/fi/cloudup/gcetasks/instance.go

@@ -35,11 +35,12 @@ type Instance struct {
 	Name      *string
 	Lifecycle *fi.Lifecycle
 
-	Network     *Network
-	Tags        []string
-	Preemptible *bool
-	Image       *string
-	Disks       map[string]*Disk
+	Network        *Network
+	Tags           []string
+	Preemptible    *bool
+	Image          *string
+	Disks          map[string]*Disk
+	ServiceAccount *string
 
 	CanIPForward *bool
 	IPAddress    *Address
@@ -253,17 +254,18 @@ func (e *Instance) mapToGCE(project string, ipAddressResolver func(*Address) (*s
 	}
 
 	var serviceAccounts []*compute.ServiceAccount
-	if e.Scopes != nil {
-		var scopes []string
-		for _, s := range e.Scopes {
-			s = scopeToLongForm(s)
-
-			scopes = append(scopes, s)
+	if e.ServiceAccount != nil {
+		if e.Scopes != nil {
+			var scopes []string
+			for _, s := range e.Scopes {
+				s = scopeToLongForm(s)
+				scopes = append(scopes, s)
+			}
+			serviceAccounts = append(serviceAccounts, &compute.ServiceAccount{
+				Email:  fi.StringValue(e.ServiceAccount),
+				Scopes: scopes,
+			})
 		}
-		serviceAccounts = append(serviceAccounts, &compute.ServiceAccount{
-			Email:  "default",
-			Scopes: scopes,
-		})
 	}
 
 	var metadataItems []*compute.MetadataItems

upup/pkg/fi/cloudup/gcetasks/instancetemplate.go

@@ -58,7 +58,8 @@ type InstanceTemplate struct {
 	Subnet        *Subnet
 	AliasIPRanges map[string]string
 
-	Scopes []string
+	Scopes         []string
+	ServiceAccount *string
 
 	Metadata    map[string]*fi.ResourceHolder
 	MachineType *string
@@ -270,15 +271,16 @@ func (e *InstanceTemplate) mapToGCE(project string, region string) (*compute.Ins
 	networkInterfaces = append(networkInterfaces, ni)
 
 	var serviceAccounts []*compute.ServiceAccount
-	if e.Scopes != nil {
-		var scopes []string
-		for _, s := range e.Scopes {
-			s = scopeToLongForm(s)
-
-			scopes = append(scopes, s)
+	if e.ServiceAccount != nil {
+		scopes := make([]string, 0)
+		if e.Scopes != nil {
+			for _, s := range e.Scopes {
+				s = scopeToLongForm(s)
+				scopes = append(scopes, s)
+			}
 		}
 		serviceAccounts = append(serviceAccounts, &compute.ServiceAccount{
-			Email:  "default",
+			Email:  fi.StringValue(e.ServiceAccount),
 			Scopes: scopes,
 		})
 	}