From d6bec33e4e47eb9e6085d1a68d14e057d238c16c Mon Sep 17 00:00:00 2001 From: Luke Weber Date: Wed, 28 Nov 2018 11:08:20 -0800 Subject: [PATCH] Add EKSCluster worker node spec for cloudformation Signed-off-by: Luke Weber --- pkg/apis/aws/compute/v1alpha1/register.go | 9 +- pkg/apis/aws/compute/v1alpha1/types.go | 212 ++++++++++++++++-- pkg/apis/aws/compute/v1alpha1/types_test.go | 136 ++++++++++- .../compute/v1alpha1/zz_generated.deepcopy.go | 42 +++- 4 files changed, 366 insertions(+), 33 deletions(-) diff --git a/pkg/apis/aws/compute/v1alpha1/register.go b/pkg/apis/aws/compute/v1alpha1/register.go index 4f67fb2..d2dcfba 100644 --- a/pkg/apis/aws/compute/v1alpha1/register.go +++ b/pkg/apis/aws/compute/v1alpha1/register.go @@ -30,10 +30,11 @@ import ( ) const ( - Group = "compute.aws.crossplane.io" - Version = "v1alpha1" - APIVersion = Group + "/" + Version - EKSClusterKind = "eksclusters" + Group = "compute.aws.crossplane.io" + Version = "v1alpha1" + APIVersion = Group + "/" + Version + EKSClusterKind = "eksclusters" + EKSClusterKindAPIVersion = EKSClusterKind + "." + APIVersion ) var ( diff --git a/pkg/apis/aws/compute/v1alpha1/types.go b/pkg/apis/aws/compute/v1alpha1/types.go index 3de16a3..f1dd7ff 100644 --- a/pkg/apis/aws/compute/v1alpha1/types.go +++ b/pkg/apis/aws/compute/v1alpha1/types.go @@ -17,6 +17,10 @@ limitations under the License. package v1alpha1 import ( + "fmt" + "strconv" + "strings" + corev1alpha1 "github.com/crossplaneio/crossplane/pkg/apis/core/v1alpha1" "github.com/crossplaneio/crossplane/pkg/util" corev1 "k8s.io/api/core/v1" @@ -37,12 +41,42 @@ const ( // ClusterStatusFailed = "FAILED" ) +type EKSRegion string + +const ( + // EKSRegionUSWest2 - us-west-2 (Oregon) region for eks cluster + EKSRegionUSWest2 EKSRegion = "us-west-2" + // EKSRegionUSEast1 - us-east-1 (N. Virginia) region for eks cluster + EKSRegionUSEast1 EKSRegion = "us-east-1" + // EKSRegionUSEast2 - us-east-2 (Ohio) region for eks worker only + EKSRegionUSEast2 EKSRegion = "us-east-2" + // EKSRegionEUWest1 - eu-west-1 (Ireland) region for eks cluster + EKSRegionEUWest1 EKSRegion = "eu-west-1" +) + +var ( + workerNodeRegionAMI = map[EKSRegion]string{ + EKSRegionUSWest2: "ami-0f54a2f7d2e9c88b3", + EKSRegionUSEast1: "ami-0a0b913ef3249b655", + EKSRegionUSEast2: "ami-0958a76db2d150238", + EKSRegionEUWest1: "ami-00c3b2d35bddd4f5c", + } +) + type EKSClusterSpec struct { + // Configuration of this Spec is dependent on the readme as described here + // https://docs.aws.amazon.com/eks/latest/userguide/getting-started.html + + // Region for EKS Cluster + // +kubebuilder:validation:Enum=us-west-2,us-east-1,eu-west-1 + Region EKSRegion `json:"region"` + // RoleARN --role-arn // The Amazon Resource Name (ARN) of the IAM role that provides permis- // sions for Amazon EKS to make calls to other AWS API operations on // your behalf. For more information, see Amazon EKS Service IAM Role // in the * Amazon EKS User Guide * . + // TODO: we could simplify this to roleName. RoleARN string `json:"roleARN"` // ResourcesVPCConfig --resources-vpc-config (structure) @@ -54,12 +88,16 @@ type EKSClusterSpec struct { // specify up to 5 security groups, but we recommend that you use a // dedicated security group for your cluster control plane. // + // VpcID of EKS cluster + VpcID string `json:"vpcId"` + // SubnetIds // Syntax: // subnetIds=string,string, SubnetIds []string `json:"subnetIds"` + // SecurityGroupIds // Syntax: - // securityGroupsIds=string,string, - SecurityGroupsIds []string `json:"securityGroupIds"` + // securityGroupIds=string,string, + SecurityGroupIds []string `json:"securityGroupIds"` // ClientRequestToken // --client-request-token (string) @@ -68,7 +106,7 @@ type EKSClusterSpec struct { ClientRequestToken string `json:"clientRequestToken,omitempty"` // ClusterVersion --kubernetes-version (string) - // The desired Kubernetes version for your clustee. If you do not spec- + // The desired Kubernetes version for your cluster. If you do not spec- // ify a value here, the latest version available in Amazon EKS is // used. ClusterVersion string `json:"clusterVersion,omitempty"` @@ -88,16 +126,58 @@ type EKSClusterSpec struct { // the command inputs and returns a sample output JSON for that command. GenerateCLISkeleton string `json:"generateCLISkeleton,omitempty"` + // WorkerNodes configuration for cloudformation + WorkerNodes WorkerNodesSpec `json:"workerNodes"` + + // ConnectionSecretNameOverride set this override the generated name of Status.ConnectionSecretRef.Name + ConnectionSecretNameOverride string `json:"connectionSecretNameOverride,omitempty"` + // Kubernetes object references - ClaimRef *corev1.ObjectReference `json:"claimRef,omitempty"` - ClassRef *corev1.ObjectReference `json:"classRef,omitempty"` - ConnectionSecretRef *corev1.LocalObjectReference `json:"connectionSecretRef,omitempty"` - ProviderRef corev1.LocalObjectReference `json:"providerRef"` + ClaimRef *corev1.ObjectReference `json:"claimRef,omitempty"` + ClassRef *corev1.ObjectReference `json:"classRef,omitempty"` + ProviderRef corev1.LocalObjectReference `json:"providerRef"` // ReclaimPolicy identifies how to handle the cloud resource after the deletion of this type ReclaimPolicy corev1alpha1.ReclaimPolicy `json:"reclaimPolicy,omitempty"` } +//WorkerNodesSpec - Worker node spec used to define cloudformation template that provisions workers for cluster +type WorkerNodesSpec struct { + // KeyName The EC2 Key Pair to allow SSH access to the instances + KeyName string `json:"keyName"` + + // NodeImageId The EC2 Key Pair to allow SSH access to the instances + // defaults to region standard AMI + NodeImageID string `json:"nodeImageId,omitempty"` + + // NodeInstanceType EC2 instance type for the node instances + // +kubebuilder:validation:Enum=t2.small,t2.medium,t2.large,t2.xlarge,t2.2xlarge,t3.nano,t3.micro,t3.small,t3.medium,t3.large,t3.xlarge,t3.2xlarge,m3.medium,m3.large,m3.xlarge,m3.2xlarge,m4.large,m4.xlarge,m4.2xlarge,m4.4xlarge,m4.10xlarge,m5.large,m5.xlarge,m5.2xlarge,m5.4xlarge,m5.12xlarge,m5.24xlarge,c4.large,c4.xlarge,c4.2xlarge,c4.4xlarge,c4.8xlarge,c5.large,c5.xlarge,c5.2xlarge,c5.4xlarge,c5.9xlarge,c5.18xlarge,i3.large,i3.xlarge,i3.2xlarge,i3.4xlarge,i3.8xlarge,i3.16xlarge,r3.xlarge,r3.2xlarge,r3.4xlarge,r3.8xlarge,r4.large,r4.xlarge,r4.2xlarge,r4.4xlarge,r4.8xlarge,r4.16xlarge,x1.16xlarge,x1.32xlarge,p2.xlarge,p2.8xlarge,p2.16xlarge,p3.2xlarge,p3.8xlarge,p3.16xlarge,r5.large,r5.xlarge,r5.2xlarge,r5.4xlarge,r5.12xlarge,r5.24xlarge,r5d.large,r5d.xlarge,r5d.2xlarge,r5d.4xlarge,r5d.12xlarge,r5d.24xlarge,z1d.large,z1d.xlarge,z1d.2xlarge,z1d.3xlarge,z1d.6xlarge,z1d.12xlarge + NodeInstanceType string `json:"nodeInstanceType"` + + // NodeAutoScalingGroupMinSize Minimum size of Node Group ASG. + // default 1 + NodeAutoScalingGroupMinSize *int `json:"nodeAutoScalingGroupMinSize,omitempty"` + + // NodeAutoScalingGroupMaxSize Maximum size of Node Group ASG. + // Default: 3 + NodeAutoScalingGroupMaxSize *int `json:"nodeAutoScalingGroupMaxSize,omitempty"` + + // NodeVolumeSize Node volume size in GB + // Default: 20 + NodeVolumeSize *int `json:"nodeVolumeSize,omitempty"` + + // BootstrapArguments Arguments to pass to the bootstrap script. See files/bootstrap.sh in https://github.com/awslabs/amazon-eks-ami + // Default: "" + BootstrapArguments string `json:"bootstrapArguments,omitempty"` + + // NodeGroupName Unique identifier for the Node Group. + NodeGroupName string `json:"nodeGroupName,omitempty"` + + // ClusterControlPlaneSecurityGroup The security group of the cluster control plane. + ClusterControlPlaneSecurityGroup string `json:"clusterControlPlaneSecurityGroup,omitempty"` +} + +// EKSClusterStatus schema of the status of eks cluster type EKSClusterStatus struct { corev1alpha1.ConditionedStatus corev1alpha1.BindingStatusPhase @@ -108,6 +188,10 @@ type EKSClusterStatus struct { ClusterName string `json:"resourceName,omitempty"` // Endpoint for cluster Endpoint string `json:"endpoint,omitempty"` + // CloudFormationStackID Stack-id + CloudFormationStackID string `json:"cloudformationStackId,omitempty"` + + ConnectionSecretRef corev1.LocalObjectReference `json:"connectionSecretRef,omitempty"` } // +genclient @@ -134,12 +218,96 @@ type EKSClusterList struct { } // NewEKSClusterSpec from properties map -// TODO: will be using once abstract resource support is added func NewEKSClusterSpec(properties map[string]string) *EKSClusterSpec { spec := &EKSClusterSpec{ ReclaimPolicy: corev1alpha1.ReclaimRetain, } - // TODO: complete spec fields assignment + + val, ok := properties["region"] + if ok { + spec.Region = EKSRegion(val) + } + + val, ok = properties["roleARN"] + if ok { + spec.RoleARN = val + } + + val, ok = properties["vpcId"] + if ok { + spec.VpcID = val + } + + val, ok = properties["subnetIds"] + if ok { + spec.SubnetIds = append(spec.SubnetIds, strings.Split(val, ",")...) + } + + val, ok = properties["securityGroupIds"] + if ok { + spec.SecurityGroupIds = append(spec.SecurityGroupIds, strings.Split(val, ",")...) + } + + val, ok = properties["clusterVersion"] + if ok { + spec.ClusterVersion = val + } + + val, ok = properties["workerKeyName"] + if ok { + spec.WorkerNodes.KeyName = val + } + + val, ok = properties["workerNodeImageId"] + if ok { + spec.WorkerNodes.NodeImageID = val + } + + val, ok = properties["workerNodeInstanceType"] + if ok { + spec.WorkerNodes.NodeInstanceType = val + } + + val, ok = properties["workerNodeAutoScalingGroupMinSize"] + if ok { + if size, err := strconv.Atoi(val); err == nil { + spec.WorkerNodes.NodeAutoScalingGroupMinSize = &size + } + } + + val, ok = properties["workerNodeAutoScalingGroupMaxSize"] + if ok { + if size, err := strconv.Atoi(val); err == nil { + spec.WorkerNodes.NodeAutoScalingGroupMaxSize = &size + } + } + + val, ok = properties["workerNodeVolumeSize"] + if ok { + if size, err := strconv.Atoi(val); err == nil { + spec.WorkerNodes.NodeVolumeSize = &size + } + } + + val, ok = properties["workerBootstrapArguments"] + if ok { + spec.WorkerNodes.BootstrapArguments = val + } + + val, ok = properties["workerNodeGroupName"] + if ok { + spec.WorkerNodes.NodeGroupName = val + } + val, ok = properties["workerClusterControlPlaneSecurityGroup"] + if ok { + spec.WorkerNodes.ClusterControlPlaneSecurityGroup = val + } + + val, ok = properties["connectionSecretNameOverride"] + if ok { + spec.ConnectionSecretNameOverride = val + } + return spec } @@ -154,17 +322,19 @@ func (e *EKSCluster) ConnectionSecret() *corev1.Secret { } } -// ConnectionSecretName returns a secret name from the reference +// ConnectionSecretName returns the name of the connection secret func (e *EKSCluster) ConnectionSecretName() string { - if e.Spec.ConnectionSecretRef == nil { - e.Spec.ConnectionSecretRef = &corev1.LocalObjectReference{ - Name: e.Name, - } - } else if e.Spec.ConnectionSecretRef.Name == "" { - e.Spec.ConnectionSecretRef.Name = e.Name + if e.Status.ConnectionSecretRef.Name != "" { + return e.Status.ConnectionSecretRef.Name + } else if e.Spec.ConnectionSecretNameOverride != "" { + return e.Spec.ConnectionSecretNameOverride } + return e.Name +} - return e.Spec.ConnectionSecretRef.Name +// SetConnectionSecretReference sets a local object reference to this secret in Status.ConnectionSecretRef +func (e *EKSCluster) SetConnectionSecretReference(secret *corev1.Secret) { + e.Status.ConnectionSecretRef.Name = secret.Name } // Endpoint returns rds resource endpoint value saved in the status (could be empty) @@ -215,3 +385,11 @@ func (e *EKSCluster) SetBound(state bool) { e.Status.Phase = corev1alpha1.BindingStateUnbound } } + +// GetAMIByRegion returns the default ami id for a given EKS region +func GetRegionAMI(region EKSRegion) (string, error) { + if val, ok := workerNodeRegionAMI[region]; ok { + return val, nil + } + return "", fmt.Errorf("not a valid EKS region, %s", string(region)) +} diff --git a/pkg/apis/aws/compute/v1alpha1/types_test.go b/pkg/apis/aws/compute/v1alpha1/types_test.go index 97b99d5..4a4d10c 100644 --- a/pkg/apis/aws/compute/v1alpha1/types_test.go +++ b/pkg/apis/aws/compute/v1alpha1/types_test.go @@ -19,6 +19,8 @@ package v1alpha1 import ( "context" "log" + "strconv" + "strings" "testing" corev1alpha1 "github.com/crossplaneio/crossplane/pkg/apis/core/v1alpha1" @@ -59,15 +61,29 @@ func TestMain(m *testing.M) { } func TestEKSCluster(t *testing.T) { + autoscaleSize := 1 + volSize := 20 key := types.NamespacedName{Name: name, Namespace: namespace} created := &EKSCluster{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, Spec: EKSClusterSpec{ - ClusterVersion: "1.1.1", - RoleARN: "test-arn", - SubnetIds: []string{"one", "two"}, - SecurityGroupsIds: []string{"sg-1", "sg-2"}, - ReclaimPolicy: corev1alpha1.ReclaimRetain, + Region: "us-west-2", + ClusterVersion: "1.1.1", + RoleARN: "test-arn", + SubnetIds: []string{"one", "two"}, + SecurityGroupIds: []string{"sg-1", "sg-2"}, + WorkerNodes: WorkerNodesSpec{ + KeyName: "test-key-name", + NodeImageID: "ami-id-test", + NodeInstanceType: "t2.small", + NodeAutoScalingGroupMinSize: &autoscaleSize, + NodeAutoScalingGroupMaxSize: &autoscaleSize, + NodeVolumeSize: &volSize, + BootstrapArguments: "test-bootstrap", + NodeGroupName: "special-group-name", + ClusterControlPlaneSecurityGroup: "sg-cluster-sec-group", + }, + ReclaimPolicy: corev1alpha1.ReclaimRetain, }, } g := NewGomegaWithT(t) @@ -90,4 +106,114 @@ func TestEKSCluster(t *testing.T) { // Test Delete g.Expect(c.Delete(ctx, fetched)).NotTo(HaveOccurred()) g.Expect(c.Get(ctx, key, fetched)).To(HaveOccurred()) + + // Test create w/invalid region + badRegion := created.DeepCopy() + badRegion.Spec.Region = "bad-region" + g.Expect(c.Create(ctx, badRegion)).To(MatchError(ContainSubstring("spec.region in body should be one of [us-west-2 us-east-1 eu-west-1]"))) + + // Test create w/invalid instance type + badInstanceType := created.DeepCopy() + badInstanceType.Spec.WorkerNodes.NodeInstanceType = "xs-bad-type" + g.Expect(c.Create(ctx, badInstanceType)).To(MatchError(ContainSubstring("spec.workerNodes.nodeInstanceType in body should be one of [t2.small"))) +} + +func TestNewEKSClusterSpec(t *testing.T) { + g := NewGomegaWithT(t) + + m := make(map[string]string) + exp := &EKSClusterSpec{ReclaimPolicy: corev1alpha1.ReclaimRetain} + g.Expect(NewEKSClusterSpec(m)).To(Equal(exp)) + + val := "test-region" + m["region"] = val + exp.Region = EKSRegion(val) + g.Expect(NewEKSClusterSpec(m)).To(Equal(exp)) + + val = "test-arm" + m["roleARN"] = val + exp.RoleARN = val + g.Expect(NewEKSClusterSpec(m)).To(Equal(exp)) + + val = "test-vpc" + m["vpcId"] = val + exp.VpcID = val + g.Expect(NewEKSClusterSpec(m)).To(Equal(exp)) + + val = "test-subnet-1-id,test-subnet-2-id" + m["subnetIds"] = val + exp.SubnetIds = append(exp.SubnetIds, strings.Split(val, ",")...) + g.Expect(NewEKSClusterSpec(m)).To(Equal(exp)) + + val = "test-sg-1-id,test-sg-2-id" + m["securityGroupIds"] = val + exp.SecurityGroupIds = append(exp.SecurityGroupIds, strings.Split(val, ",")...) + g.Expect(NewEKSClusterSpec(m)).To(Equal(exp)) + + val = "1.10.1" + m["clusterVersion"] = val + exp.ClusterVersion = val + g.Expect(NewEKSClusterSpec(m)).To(Equal(exp)) + + val = "key-name-test" + m["workerKeyName"] = val + exp.WorkerNodes.KeyName = val + g.Expect(NewEKSClusterSpec(m)).To(Equal(exp)) + + val = "test-node-image-id" + m["workerNodeImageId"] = val + exp.WorkerNodes.NodeImageID = val + g.Expect(NewEKSClusterSpec(m)).To(Equal(exp)) + + val = "test-node-instance-type" + m["workerNodeInstanceType"] = val + exp.WorkerNodes.NodeInstanceType = val + g.Expect(NewEKSClusterSpec(m)).To(Equal(exp)) + + minSize := 5 + val = strconv.Itoa(minSize) + m["workerNodeAutoScalingGroupMinSize"] = "" + exp.WorkerNodes.NodeAutoScalingGroupMinSize = nil + g.Expect(NewEKSClusterSpec(m)).To(Equal(exp)) + m["workerNodeAutoScalingGroupMinSize"] = val + exp.WorkerNodes.NodeAutoScalingGroupMinSize = &minSize + g.Expect(NewEKSClusterSpec(m)).To(Equal(exp)) + + maxSize := 10 + val = strconv.Itoa(maxSize) + m["workerNodeAutoScalingGroupMaxSize"] = "" + exp.WorkerNodes.NodeAutoScalingGroupMaxSize = nil + g.Expect(NewEKSClusterSpec(m)).To(Equal(exp)) + m["workerNodeAutoScalingGroupMaxSize"] = val + exp.WorkerNodes.NodeAutoScalingGroupMaxSize = &maxSize + g.Expect(NewEKSClusterSpec(m)).To(Equal(exp)) + + volSize := 20 + val = strconv.Itoa(volSize) + m["workerNodeVolumeSize"] = "" + exp.WorkerNodes.NodeVolumeSize = nil + g.Expect(NewEKSClusterSpec(m)).To(Equal(exp)) + m["workerNodeVolumeSize"] = val + exp.WorkerNodes.NodeVolumeSize = &volSize + g.Expect(NewEKSClusterSpec(m)).To(Equal(exp)) + + val = "test-bootstrap-args" + m["workerBootstrapArguments"] = val + exp.WorkerNodes.BootstrapArguments = val + g.Expect(NewEKSClusterSpec(m)).To(Equal(exp)) + + val = "test-node-group-name" + m["workerNodeGroupName"] = val + exp.WorkerNodes.NodeGroupName = val + g.Expect(NewEKSClusterSpec(m)).To(Equal(exp)) + + val = "cp-security-group" + m["workerClusterControlPlaneSecurityGroup"] = val + exp.WorkerNodes.ClusterControlPlaneSecurityGroup = val + g.Expect(NewEKSClusterSpec(m)).To(Equal(exp)) + + val = "conn-secret-name-override-test" + m["connectionSecretNameOverride"] = val + exp.ConnectionSecretNameOverride = val + g.Expect(NewEKSClusterSpec(m)).To(Equal(exp)) } diff --git a/pkg/apis/aws/compute/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/aws/compute/v1alpha1/zz_generated.deepcopy.go index ec9afea..7d632df 100644 --- a/pkg/apis/aws/compute/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/aws/compute/v1alpha1/zz_generated.deepcopy.go @@ -93,11 +93,12 @@ func (in *EKSClusterSpec) DeepCopyInto(out *EKSClusterSpec) { *out = make([]string, len(*in)) copy(*out, *in) } - if in.SecurityGroupsIds != nil { - in, out := &in.SecurityGroupsIds, &out.SecurityGroupsIds + if in.SecurityGroupIds != nil { + in, out := &in.SecurityGroupIds, &out.SecurityGroupIds *out = make([]string, len(*in)) copy(*out, *in) } + in.WorkerNodes.DeepCopyInto(&out.WorkerNodes) if in.ClaimRef != nil { in, out := &in.ClaimRef, &out.ClaimRef *out = new(v1.ObjectReference) @@ -108,11 +109,6 @@ func (in *EKSClusterSpec) DeepCopyInto(out *EKSClusterSpec) { *out = new(v1.ObjectReference) **out = **in } - if in.ConnectionSecretRef != nil { - in, out := &in.ConnectionSecretRef, &out.ConnectionSecretRef - *out = new(v1.LocalObjectReference) - **out = **in - } out.ProviderRef = in.ProviderRef return } @@ -132,6 +128,7 @@ func (in *EKSClusterStatus) DeepCopyInto(out *EKSClusterStatus) { *out = *in in.ConditionedStatus.DeepCopyInto(&out.ConditionedStatus) out.BindingStatusPhase = in.BindingStatusPhase + out.ConnectionSecretRef = in.ConnectionSecretRef return } @@ -144,3 +141,34 @@ func (in *EKSClusterStatus) DeepCopy() *EKSClusterStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkerNodesSpec) DeepCopyInto(out *WorkerNodesSpec) { + *out = *in + if in.NodeAutoScalingGroupMinSize != nil { + in, out := &in.NodeAutoScalingGroupMinSize, &out.NodeAutoScalingGroupMinSize + *out = new(int) + **out = **in + } + if in.NodeAutoScalingGroupMaxSize != nil { + in, out := &in.NodeAutoScalingGroupMaxSize, &out.NodeAutoScalingGroupMaxSize + *out = new(int) + **out = **in + } + if in.NodeVolumeSize != nil { + in, out := &in.NodeVolumeSize, &out.NodeVolumeSize + *out = new(int) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkerNodesSpec. +func (in *WorkerNodesSpec) DeepCopy() *WorkerNodesSpec { + if in == nil { + return nil + } + out := new(WorkerNodesSpec) + in.DeepCopyInto(out) + return out +}