kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4677 from usabilla/external-load-balancers 

Add the ability to specify external loadbalancers for instancegroups
This commit is contained in:
Justin Santa Barbara 2018-07-19 21:54:00 -04:00 committed by GitHub
parent 967fe70e34 8256c3cc73
commit 630ea429ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 2649 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cmd/kops/integration_test.go
View File

@ -169,6 +169,11 @@ func TestPhaseNetwork(t *testing.T) {
runTestPhase(t, "lifecyclephases.example.com", "lifecycle_phases", "v1alpha2", true, 1, cloudup.PhaseNetwork)
}
func TestExternalLoadBalancer(t *testing.T) {
runTestAWS(t, "externallb.example.com", "externallb", "v1alpha2", false, 1, true)
runTestCloudformation(t, "externallb.example.com", "externallb", "v1alpha2", false, nil)
}
// TestPhaseIAM tests the output of tf for the iam phase
func TestPhaseIAM(t *testing.T) {
t.Skip("unable to test w/o allowing failed validation")

11
docs/development/api_updates.md
View File

@ -10,16 +10,7 @@ that directory directly (the unversioned API) and in the versioned subdirectorie
## Updating the generated API code
You will need a few tools from kubernetes (these will likely be moved to a shared repo soon):
```
go get k8s.io/kubernetes
cd ${GOPATH}/src/k8s.io/kubernetes
git checkout master
git pull
```
Then you can run `make apimachinery && make` to update the generated API machinery code (conversion functions). Note
To generate the API code simply run `make apimachinery && make` to update the generated API machinery code (conversion functions). Note
that `make apimachinery` (currently) only updates the autogenerated code; it does not trigger a rebuild, hence the
need for `&& make`.

31
docs/instance_groups.md
View File

@ -318,6 +318,37 @@ spec:
- AZRebalance
```
## Attaching existing Load Balancers to Instance Groups
Instance groups can be linked to up to 10 load balancers. When attached, any instance launched will
automatically register itself to the load balancer. For example, if you can create an instance group
dedicated to running an ingress controller exposed on a
[NodePort](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport), you can
manually create a load balancer and link it to the instance group. Traffic to the load balancer will now
automatically go to one of the nodes.
You can specify either `loadBalancerName` to link the instance group to an AWS Classic ELB or you can
specify `targetGroupARN` to link the instance group to a target group, which are used by Application
load balancers and Network load balancers.
```
# Example ingress nodes
apiVersion: kops/v1alpha2
kind: InstanceGroup
metadata:
labels:
kops.k8s.io/cluster: k8s.dev.local
name: ingress
spec:
machineType: m4.large
maxSize: 2
minSize: 2
role: Node
externalLoadBalancers:
- targetGroupARN: arn:aws:elasticloadbalancing:eu-west-1:123456789012:targetgroup/my-ingress-target-group/0123456789abcdef
- loadBalancerName: my-elb-classic-load-balancer
```
## Enabling Detailed-Monitoring on AWS instances
Detailed-Monitoring will cause the monitoring data to be available every 1 minute instead of every 5 minutes. [Enabling Detailed Monitoring](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html). In production environments you may want to consider to enable detailed monitoring for quicker troubleshooting.

10
pkg/apis/kops/instancegroup.go
View File

@ -116,6 +116,8 @@ type InstanceGroupSpec struct {
AdditionalUserData []UserData `json:"additionalUserData,omitempty"`
// SuspendProcesses disables the listed Scaling Policies
SuspendProcesses []string `json:"suspendProcesses,omitempty"`
// ExternalLoadBalancers define loadbalancers that should be attached to the instancegroup
ExternalLoadBalancers []LoadBalancer `json:"externalLoadBalancers,omitempty"`
// DetailedInstanceMonitoring defines if detailed-monitoring is enabled (AWS only)
DetailedInstanceMonitoring *bool `json:"detailedInstanceMonitoring,omitempty"`
// IAMProfileSpec defines the identity of the cloud group iam profile (AWS only).
@ -206,3 +208,11 @@ func (g *InstanceGroup) AddInstanceGroupNodeLabel() {
g.Spec.NodeLabels[NodeLabelInstanceGroup] = g.Name
}
}
// LoadBalancers defines a load balancer
type LoadBalancer struct {
// LoadBalancerName to associate with this instance group (AWS ELB)
LoadBalancerName *string `json:"loadBalancerName,omitempty"`
// TargetGroupARN to associate with this instance group (AWS ALB/NLB)
TargetGroupARN *string `json:"targetGroupArn,omitempty"`
}

10
pkg/apis/kops/v1alpha1/instancegroup.go
View File

@ -96,6 +96,8 @@ type InstanceGroupSpec struct {
Zones []string `json:"zones,omitempty"`
// SuspendProcesses disables the listed Scaling Policies
SuspendProcesses []string `json:"suspendProcesses,omitempty"`
// ExternalLoadBalancers define loadbalancers that should be attached to the instancegroup
ExternalLoadBalancers []LoadBalancer `json:"externalLoadBalancers,omitempty"`
// DetailedInstanceMonitoring defines if detailed-monitoring is enabled (AWS only)
DetailedInstanceMonitoring *bool `json:"detailedInstanceMonitoring,omitempty"`
// IAMProfileSpec defines the identity of the cloud group iam profile (AWS only).
@ -119,3 +121,11 @@ type UserData struct {
// Content is the user-data content
Content string `json:"content,omitempty"`
}
// LoadBalancers defines a load balancer
type LoadBalancer struct {
// LoadBalancerName to associate with this instance group (AWS ELB)
LoadBalancerName *string `json:"loadBalancerName,omitempty"`
// TargetGroupARN to associate with this instance group (AWS ALB/NLB)
TargetGroupARN *string `json:"targetGroupArn,omitempty"`
}

46
pkg/apis/kops/v1alpha1/zz_generated.conversion.go
View File

@ -132,6 +132,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
Convert_kops_KuberouterNetworkingSpec_To_v1alpha1_KuberouterNetworkingSpec,
Convert_v1alpha1_LeaderElectionConfiguration_To_kops_LeaderElectionConfiguration,
Convert_kops_LeaderElectionConfiguration_To_v1alpha1_LeaderElectionConfiguration,
Convert_v1alpha1_LoadBalancer_To_kops_LoadBalancer,
Convert_kops_LoadBalancer_To_v1alpha1_LoadBalancer,
Convert_v1alpha1_LoadBalancerAccessSpec_To_kops_LoadBalancerAccessSpec,
Convert_kops_LoadBalancerAccessSpec_To_v1alpha1_LoadBalancerAccessSpec,
Convert_v1alpha1_NetworkingSpec_To_kops_NetworkingSpec,
@ -2001,6 +2003,17 @@ func autoConvert_v1alpha1_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan
}
out.Zones = in.Zones
out.SuspendProcesses = in.SuspendProcesses
if in.ExternalLoadBalancers != nil {
in, out := &in.ExternalLoadBalancers, &out.ExternalLoadBalancers
*out = make([]kops.LoadBalancer, len(*in))
for i := range *in {
if err := Convert_v1alpha1_LoadBalancer_To_kops_LoadBalancer(&(*in)[i], &(*out)[i], s); err != nil {
return err
}
}
} else {
out.ExternalLoadBalancers = nil
}
out.DetailedInstanceMonitoring = in.DetailedInstanceMonitoring
if in.IAM != nil {
in, out := &in.IAM, &out.IAM
@ -2076,6 +2089,17 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha1_InstanceGroupSpec(in *kops.I
out.AdditionalUserData = nil
}
out.SuspendProcesses = in.SuspendProcesses
if in.ExternalLoadBalancers != nil {
in, out := &in.ExternalLoadBalancers, &out.ExternalLoadBalancers
*out = make([]LoadBalancer, len(*in))
for i := range *in {
if err := Convert_kops_LoadBalancer_To_v1alpha1_LoadBalancer(&(*in)[i], &(*out)[i], s); err != nil {
return err
}
}
} else {
out.ExternalLoadBalancers = nil
}
out.DetailedInstanceMonitoring = in.DetailedInstanceMonitoring
if in.IAM != nil {
in, out := &in.IAM, &out.IAM
@ -2683,6 +2707,28 @@ func Convert_kops_LeaderElectionConfiguration_To_v1alpha1_LeaderElectionConfigur
return autoConvert_kops_LeaderElectionConfiguration_To_v1alpha1_LeaderElectionConfiguration(in, out, s)
}
func autoConvert_v1alpha1_LoadBalancer_To_kops_LoadBalancer(in *LoadBalancer, out *kops.LoadBalancer, s conversion.Scope) error {
out.LoadBalancerName = in.LoadBalancerName
out.TargetGroupARN = in.TargetGroupARN
return nil
}
// Convert_v1alpha1_LoadBalancer_To_kops_LoadBalancer is an autogenerated conversion function.
func Convert_v1alpha1_LoadBalancer_To_kops_LoadBalancer(in *LoadBalancer, out *kops.LoadBalancer, s conversion.Scope) error {
return autoConvert_v1alpha1_LoadBalancer_To_kops_LoadBalancer(in, out, s)
}
func autoConvert_kops_LoadBalancer_To_v1alpha1_LoadBalancer(in *kops.LoadBalancer, out *LoadBalancer, s conversion.Scope) error {
out.LoadBalancerName = in.LoadBalancerName
out.TargetGroupARN = in.TargetGroupARN
return nil
}
// Convert_kops_LoadBalancer_To_v1alpha1_LoadBalancer is an autogenerated conversion function.
func Convert_kops_LoadBalancer_To_v1alpha1_LoadBalancer(in *kops.LoadBalancer, out *LoadBalancer, s conversion.Scope) error {
return autoConvert_kops_LoadBalancer_To_v1alpha1_LoadBalancer(in, out, s)
}
func autoConvert_v1alpha1_LoadBalancerAccessSpec_To_kops_LoadBalancerAccessSpec(in *LoadBalancerAccessSpec, out *kops.LoadBalancerAccessSpec, s conversion.Scope) error {
out.Type = kops.LoadBalancerType(in.Type)
out.IdleTimeoutSeconds = in.IdleTimeoutSeconds

41
pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go
View File

@ -1765,6 +1765,13 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.ExternalLoadBalancers != nil {
in, out := &in.ExternalLoadBalancers, &out.ExternalLoadBalancers
*out = make([]LoadBalancer, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.DetailedInstanceMonitoring != nil {
in, out := &in.DetailedInstanceMonitoring, &out.DetailedInstanceMonitoring
if *in == nil {
@ -2796,6 +2803,40 @@ func (in *LeaderElectionConfiguration) DeepCopy() *LeaderElectionConfiguration {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *LoadBalancer) DeepCopyInto(out *LoadBalancer) {
*out = *in
if in.LoadBalancerName != nil {
in, out := &in.LoadBalancerName, &out.LoadBalancerName
if *in == nil {
*out = nil
} else {
*out = new(string)
**out = **in
}
}
if in.TargetGroupARN != nil {
in, out := &in.TargetGroupARN, &out.TargetGroupARN
if *in == nil {
*out = nil
} else {
*out = new(string)
**out = **in
}
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancer.
func (in *LoadBalancer) DeepCopy() *LoadBalancer {
if in == nil {
return nil
}
out := new(LoadBalancer)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *LoadBalancerAccessSpec) DeepCopyInto(out *LoadBalancerAccessSpec) {
*out = *in

12
pkg/apis/kops/v1alpha2/instancegroup.go
View File

@ -55,7 +55,7 @@ var AllInstanceGroupRoles = []InstanceGroupRole{
InstanceGroupRoleBastion,
}
// InstanceGroupSpec is the specification for a instanceGroup
// InstanceGroupSpec is the specification for an instanceGroup
type InstanceGroupSpec struct {
// Type determines the role of instances in this group: masters or nodes
Role InstanceGroupRole `json:"role,omitempty"`
@ -105,6 +105,8 @@ type InstanceGroupSpec struct {
AdditionalUserData []UserData `json:"additionalUserData,omitempty"`
// SuspendProcesses disables the listed Scaling Policies
SuspendProcesses []string `json:"suspendProcesses,omitempty"`
// ExternalLoadBalancers define loadbalancers that should be attached to the instancegroup
ExternalLoadBalancers []LoadBalancer `json:"externalLoadBalancers,omitempty"`
// DetailedInstanceMonitoring defines if detailed-monitoring is enabled (AWS only)
DetailedInstanceMonitoring *bool `json:"detailedInstanceMonitoring,omitempty"`
// IAMProfileSpec defines the identity of the cloud group iam profile (AWS only).
@ -128,3 +130,11 @@ type IAMProfileSpec struct {
// for the iam instance profile
Profile *string `json:"profile,omitempty"`
}
// LoadBalancer defines a load balancer
type LoadBalancer struct {
// LoadBalancerName to associate with this instance group (AWS ELB)
LoadBalancerName *string `json:"loadBalancerName,omitempty"`
// TargetGroupARN to associate with this instance group (AWS ALB/NLB)
TargetGroupARN *string `json:"targetGroupArn,omitempty"`
}

46
pkg/apis/kops/v1alpha2/zz_generated.conversion.go
View File

@ -144,6 +144,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
Convert_kops_KuberouterNetworkingSpec_To_v1alpha2_KuberouterNetworkingSpec,
Convert_v1alpha2_LeaderElectionConfiguration_To_kops_LeaderElectionConfiguration,
Convert_kops_LeaderElectionConfiguration_To_v1alpha2_LeaderElectionConfiguration,
Convert_v1alpha2_LoadBalancer_To_kops_LoadBalancer,
Convert_kops_LoadBalancer_To_v1alpha2_LoadBalancer,
Convert_v1alpha2_LoadBalancerAccessSpec_To_kops_LoadBalancerAccessSpec,
Convert_kops_LoadBalancerAccessSpec_To_v1alpha2_LoadBalancerAccessSpec,
Convert_v1alpha2_NetworkingSpec_To_kops_NetworkingSpec,
@ -2113,6 +2115,17 @@ func autoConvert_v1alpha2_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *Instan
out.AdditionalUserData = nil
}
out.SuspendProcesses = in.SuspendProcesses
if in.ExternalLoadBalancers != nil {
in, out := &in.ExternalLoadBalancers, &out.ExternalLoadBalancers
*out = make([]kops.LoadBalancer, len(*in))
for i := range *in {
if err := Convert_v1alpha2_LoadBalancer_To_kops_LoadBalancer(&(*in)[i], &(*out)[i], s); err != nil {
return err
}
}
} else {
out.ExternalLoadBalancers = nil
}
out.DetailedInstanceMonitoring = in.DetailedInstanceMonitoring
if in.IAM != nil {
in, out := &in.IAM, &out.IAM
@ -2193,6 +2206,17 @@ func autoConvert_kops_InstanceGroupSpec_To_v1alpha2_InstanceGroupSpec(in *kops.I
out.AdditionalUserData = nil
}
out.SuspendProcesses = in.SuspendProcesses
if in.ExternalLoadBalancers != nil {
in, out := &in.ExternalLoadBalancers, &out.ExternalLoadBalancers
*out = make([]LoadBalancer, len(*in))
for i := range *in {
if err := Convert_kops_LoadBalancer_To_v1alpha2_LoadBalancer(&(*in)[i], &(*out)[i], s); err != nil {
return err
}
}
} else {
out.ExternalLoadBalancers = nil
}
out.DetailedInstanceMonitoring = in.DetailedInstanceMonitoring
if in.IAM != nil {
in, out := &in.IAM, &out.IAM
@ -2947,6 +2971,28 @@ func Convert_kops_LeaderElectionConfiguration_To_v1alpha2_LeaderElectionConfigur
return autoConvert_kops_LeaderElectionConfiguration_To_v1alpha2_LeaderElectionConfiguration(in, out, s)
}
func autoConvert_v1alpha2_LoadBalancer_To_kops_LoadBalancer(in *LoadBalancer, out *kops.LoadBalancer, s conversion.Scope) error {
out.LoadBalancerName = in.LoadBalancerName
out.TargetGroupARN = in.TargetGroupARN
return nil
}
// Convert_v1alpha2_LoadBalancer_To_kops_LoadBalancer is an autogenerated conversion function.
func Convert_v1alpha2_LoadBalancer_To_kops_LoadBalancer(in *LoadBalancer, out *kops.LoadBalancer, s conversion.Scope) error {
return autoConvert_v1alpha2_LoadBalancer_To_kops_LoadBalancer(in, out, s)
}
func autoConvert_kops_LoadBalancer_To_v1alpha2_LoadBalancer(in *kops.LoadBalancer, out *LoadBalancer, s conversion.Scope) error {
out.LoadBalancerName = in.LoadBalancerName
out.TargetGroupARN = in.TargetGroupARN
return nil
}
// Convert_kops_LoadBalancer_To_v1alpha2_LoadBalancer is an autogenerated conversion function.
func Convert_kops_LoadBalancer_To_v1alpha2_LoadBalancer(in *kops.LoadBalancer, out *LoadBalancer, s conversion.Scope) error {
return autoConvert_kops_LoadBalancer_To_v1alpha2_LoadBalancer(in, out, s)
}
func autoConvert_v1alpha2_LoadBalancerAccessSpec_To_kops_LoadBalancerAccessSpec(in *LoadBalancerAccessSpec, out *kops.LoadBalancerAccessSpec, s conversion.Scope) error {
out.Type = kops.LoadBalancerType(in.Type)
out.IdleTimeoutSeconds = in.IdleTimeoutSeconds

41
pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go
View File

@ -1737,6 +1737,13 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.ExternalLoadBalancers != nil {
in, out := &in.ExternalLoadBalancers, &out.ExternalLoadBalancers
*out = make([]LoadBalancer, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.DetailedInstanceMonitoring != nil {
in, out := &in.DetailedInstanceMonitoring, &out.DetailedInstanceMonitoring
if *in == nil {
@ -2877,6 +2884,40 @@ func (in *LeaderElectionConfiguration) DeepCopy() *LeaderElectionConfiguration {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *LoadBalancer) DeepCopyInto(out *LoadBalancer) {
*out = *in
if in.LoadBalancerName != nil {
in, out := &in.LoadBalancerName, &out.LoadBalancerName
if *in == nil {
*out = nil
} else {
*out = new(string)
**out = **in
}
}
if in.TargetGroupARN != nil {
in, out := &in.TargetGroupARN, &out.TargetGroupARN
if *in == nil {
*out = nil
} else {
*out = new(string)
**out = **in
}
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancer.
func (in *LoadBalancer) DeepCopy() *LoadBalancer {
if in == nil {
return nil
}
out := new(LoadBalancer)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *LoadBalancerAccessSpec) DeepCopyInto(out *LoadBalancerAccessSpec) {
*out = *in

41
pkg/apis/kops/zz_generated.deepcopy.go
View File

@ -1918,6 +1918,13 @@ func (in *InstanceGroupSpec) DeepCopyInto(out *InstanceGroupSpec) {
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.ExternalLoadBalancers != nil {
in, out := &in.ExternalLoadBalancers, &out.ExternalLoadBalancers
*out = make([]LoadBalancer, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.DetailedInstanceMonitoring != nil {
in, out := &in.DetailedInstanceMonitoring, &out.DetailedInstanceMonitoring
if *in == nil {
@ -3090,6 +3097,40 @@ func (in *LeaderElectionConfiguration) DeepCopy() *LeaderElectionConfiguration {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *LoadBalancer) DeepCopyInto(out *LoadBalancer) {
*out = *in
if in.LoadBalancerName != nil {
in, out := &in.LoadBalancerName, &out.LoadBalancerName
if *in == nil {
*out = nil
} else {
*out = new(string)
**out = **in
}
}
if in.TargetGroupARN != nil {
in, out := &in.TargetGroupARN, &out.TargetGroupARN
if *in == nil {
*out = nil
} else {
*out = new(string)
**out = **in
}
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancer.
func (in *LoadBalancer) DeepCopy() *LoadBalancer {
if in == nil {
return nil
}
out := new(LoadBalancer)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *LoadBalancerAccessSpec) DeepCopyInto(out *LoadBalancerAccessSpec) {
*out = *in

27
pkg/model/awsmodel/autoscalinggroup.go
View File

@ -238,6 +238,33 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
c.AddTask(t)
}
// External Load Balancer/TargetGroup Attachments
{
for _, lb := range ig.Spec.ExternalLoadBalancers {
if lb.LoadBalancerName != nil {
t := &awstasks.ExternalLoadBalancerAttachment{
Name: s("extlb-" + *lb.LoadBalancerName + "-" + ig.Name),
Lifecycle: b.Lifecycle,
LoadBalancerName: *lb.LoadBalancerName,
AutoscalingGroup: b.LinkToAutoscalingGroup(ig),
}
c.AddTask(t)
}
if lb.TargetGroupARN != nil {
t := &awstasks.ExternalTargetGroupAttachment{
Name: s("exttg-" + *lb.TargetGroupARN + "-" + ig.Name),
Lifecycle: b.Lifecycle,
TargetGroupARN: *lb.TargetGroupARN,
AutoscalingGroup: b.LinkToAutoscalingGroup(ig),
}
c.AddTask(t)
}
}
}
}
return nil

852
tests/integration/update_cluster/externallb/cloudformation.json Normal file
View File

@ -0,0 +1,852 @@
{
"Resources": {
"AWSAutoScalingAutoScalingGroupmasterustest1amastersexternallbexamplecom": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"Properties": {
"AutoScalingGroupName": "master-us-test-1a.masters.externallb.example.com",
"LaunchConfigurationName": {
"Ref": "AWSAutoScalingLaunchConfigurationmasterustest1amastersexternallbexamplecom"
},
"MaxSize": 1,
"MinSize": 1,
"VPCZoneIdentifier": [
{
"Ref": "AWSEC2Subnetustest1aexternallbexamplecom"
}
],
"Tags": [
{
"Key": "KubernetesCluster",
"Value": "externallb.example.com",
"PropagateAtLaunch": true
},
{
"Key": "Name",
"Value": "master-us-test-1a.masters.externallb.example.com",
"PropagateAtLaunch": true
},
{
"Key": "k8s.io/role/master",
"Value": "1",
"PropagateAtLaunch": true
}
],
"MetricsCollection": [
{
"Granularity": "1Minute",
"Metrics": [
"GroupDesiredCapacity",
"GroupInServiceInstances",
"GroupMaxSize",
"GroupMinSize",
"GroupPendingInstances",
"GroupStandbyInstances",
"GroupTerminatingInstances",
"GroupTotalInstances"
]
}
],
"LoadBalancerNames": [
"my-other-elb"
],
"TargetGroupARNs": [
"aws:arn:elasticloadbalancing:us-test-1a:123456789012:targetgroup/my-tg/0123456789abcdef"
]
}
},
"AWSAutoScalingAutoScalingGroupnodesexternallbexamplecom": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"Properties": {
"AutoScalingGroupName": "nodes.externallb.example.com",
"LaunchConfigurationName": {
"Ref": "AWSAutoScalingLaunchConfigurationnodesexternallbexamplecom"
},
"MaxSize": 2,
"MinSize": 2,
"VPCZoneIdentifier": [
{
"Ref": "AWSEC2Subnetustest1aexternallbexamplecom"
}
],
"Tags": [
{
"Key": "KubernetesCluster",
"Value": "externallb.example.com",
"PropagateAtLaunch": true
},
{
"Key": "Name",
"Value": "nodes.externallb.example.com",
"PropagateAtLaunch": true
},
{
"Key": "k8s.io/role/node",
"Value": "1",
"PropagateAtLaunch": true
}
],
"MetricsCollection": [
{
"Granularity": "1Minute",
"Metrics": [
"GroupDesiredCapacity",
"GroupInServiceInstances",
"GroupMaxSize",
"GroupMinSize",
"GroupPendingInstances",
"GroupStandbyInstances",
"GroupTerminatingInstances",
"GroupTotalInstances"
]
}
],
"LoadBalancerNames": [
"my-elb"
]
}
},
"AWSAutoScalingLaunchConfigurationmasterustest1amastersexternallbexamplecom": {
"Type": "AWS::AutoScaling::LaunchConfiguration",
"Properties": {
"AssociatePublicIpAddress": true,
"BlockDeviceMappings": [
{
"DeviceName": "/dev/xvda",
"Ebs": {
"VolumeType": "gp2",
"VolumeSize": 64,
"DeleteOnTermination": true
}
},
{
"DeviceName": "/dev/sdc",
"VirtualName": "ephemeral0"
}
],
"IamInstanceProfile": {
"Ref": "AWSIAMInstanceProfilemastersexternallbexamplecom"
},
"ImageId": "ami-12345678",
"InstanceType": "m3.medium",
"KeyName": "kubernetes.externallb.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57",
"SecurityGroups": [
{
"Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
}
],
"UserData": "extracted",
"InstanceMonitoring": false
}
},
"AWSAutoScalingLaunchConfigurationnodesexternallbexamplecom": {
"Type": "AWS::AutoScaling::LaunchConfiguration",
"Properties": {
"AssociatePublicIpAddress": true,
"BlockDeviceMappings": [
{
"DeviceName": "/dev/xvda",
"Ebs": {
"VolumeType": "gp2",
"VolumeSize": 128,
"DeleteOnTermination": true
}
}
],
"IamInstanceProfile": {
"Ref": "AWSIAMInstanceProfilenodesexternallbexamplecom"
},
"ImageId": "ami-12345678",
"InstanceType": "t2.medium",
"KeyName": "kubernetes.externallb.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57",
"SecurityGroups": [
{
"Ref": "AWSEC2SecurityGroupnodesexternallbexamplecom"
}
],
"UserData": "extracted",
"InstanceMonitoring": false
}
},
"AWSEC2DHCPOptionsexternallbexamplecom": {
"Type": "AWS::EC2::DHCPOptions",
"Properties": {
"DomainName": "us-test-1.compute.internal",
"DomainNameServers": [
"AmazonProvidedDNS"
],
"Tags": [
{
"Key": "KubernetesCluster",
"Value": "externallb.example.com"
},
{
"Key": "Name",
"Value": "externallb.example.com"
},
{
"Key": "kubernetes.io/cluster/externallb.example.com",
"Value": "owned"
}
]
}
},
"AWSEC2InternetGatewayexternallbexamplecom": {
"Type": "AWS::EC2::InternetGateway",
"Properties": {
"Tags": [
{
"Key": "KubernetesCluster",
"Value": "externallb.example.com"
},
{
"Key": "Name",
"Value": "externallb.example.com"
},
{
"Key": "kubernetes.io/cluster/externallb.example.com",
"Value": "owned"
}
]
}
},
"AWSEC2Route00000": {
"Type": "AWS::EC2::Route",
"Properties": {
"RouteTableId": {
"Ref": "AWSEC2RouteTableexternallbexamplecom"
},
"DestinationCidrBlock": "0.0.0.0/0",
"GatewayId": {
"Ref": "AWSEC2InternetGatewayexternallbexamplecom"
}
}
},
"AWSEC2RouteTableexternallbexamplecom": {
"Type": "AWS::EC2::RouteTable",
"Properties": {
"VpcId": {
"Ref": "AWSEC2VPCexternallbexamplecom"
},
"Tags": [
{
"Key": "KubernetesCluster",
"Value": "externallb.example.com"
},
{
"Key": "Name",
"Value": "externallb.example.com"
},
{
"Key": "kubernetes.io/cluster/externallb.example.com",
"Value": "owned"
},
{
"Key": "kubernetes.io/kops/role",
"Value": "public"
}
]
}
},
"AWSEC2SecurityGroupEgressmasteregress": {
"Type": "AWS::EC2::SecurityGroupEgress",
"Properties": {
"GroupId": {
"Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
},
"FromPort": 0,
"ToPort": 0,
"IpProtocol": "-1",
"CidrIp": "0.0.0.0/0"
}
},
"AWSEC2SecurityGroupEgressnodeegress": {
"Type": "AWS::EC2::SecurityGroupEgress",
"Properties": {
"GroupId": {
"Ref": "AWSEC2SecurityGroupnodesexternallbexamplecom"
},
"FromPort": 0,
"ToPort": 0,
"IpProtocol": "-1",
"CidrIp": "0.0.0.0/0"
}
},
"AWSEC2SecurityGroupIngressallmastertomaster": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"GroupId": {
"Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
},
"SourceSecurityGroupId": {
"Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
},
"FromPort": 0,
"ToPort": 0,
"IpProtocol": "-1"
}
},
"AWSEC2SecurityGroupIngressallmastertonode": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"GroupId": {
"Ref": "AWSEC2SecurityGroupnodesexternallbexamplecom"
},
"SourceSecurityGroupId": {
"Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
},
"FromPort": 0,
"ToPort": 0,
"IpProtocol": "-1"
}
},
"AWSEC2SecurityGroupIngressallnodetonode": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"GroupId": {
"Ref": "AWSEC2SecurityGroupnodesexternallbexamplecom"
},
"SourceSecurityGroupId": {
"Ref": "AWSEC2SecurityGroupnodesexternallbexamplecom"
},
"FromPort": 0,
"ToPort": 0,
"IpProtocol": "-1"
}
},
"AWSEC2SecurityGroupIngresshttpsexternaltomaster00000": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"GroupId": {
"Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
},
"FromPort": 443,
"ToPort": 443,
"IpProtocol": "tcp",
"CidrIp": "0.0.0.0/0"
}
},
"AWSEC2SecurityGroupIngressnodetomastertcp12379": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"GroupId": {
"Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
},
"SourceSecurityGroupId": {
"Ref": "AWSEC2SecurityGroupnodesexternallbexamplecom"
},
"FromPort": 1,
"ToPort": 2379,
"IpProtocol": "tcp"
}
},
"AWSEC2SecurityGroupIngressnodetomastertcp23824000": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"GroupId": {
"Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
},
"SourceSecurityGroupId": {
"Ref": "AWSEC2SecurityGroupnodesexternallbexamplecom"
},
"FromPort": 2382,
"ToPort": 4000,
"IpProtocol": "tcp"
}
},
"AWSEC2SecurityGroupIngressnodetomastertcp400365535": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"GroupId": {
"Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
},
"SourceSecurityGroupId": {
"Ref": "AWSEC2SecurityGroupnodesexternallbexamplecom"
},
"FromPort": 4003,
"ToPort": 65535,
"IpProtocol": "tcp"
}
},
"AWSEC2SecurityGroupIngressnodetomasterudp165535": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"GroupId": {
"Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
},
"SourceSecurityGroupId": {
"Ref": "AWSEC2SecurityGroupnodesexternallbexamplecom"
},
"FromPort": 1,
"ToPort": 65535,
"IpProtocol": "udp"
}
},
"AWSEC2SecurityGroupIngresssshexternaltomaster00000": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"GroupId": {
"Ref": "AWSEC2SecurityGroupmastersexternallbexamplecom"
},
"FromPort": 22,
"ToPort": 22,
"IpProtocol": "tcp",
"CidrIp": "0.0.0.0/0"
}
},
"AWSEC2SecurityGroupIngresssshexternaltonode00000": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"GroupId": {
"Ref": "AWSEC2SecurityGroupnodesexternallbexamplecom"
},
"FromPort": 22,
"ToPort": 22,
"IpProtocol": "tcp",
"CidrIp": "0.0.0.0/0"
}
},
"AWSEC2SecurityGroupmastersexternallbexamplecom": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"VpcId": {
"Ref": "AWSEC2VPCexternallbexamplecom"
},
"GroupDescription": "Security group for masters",
"Tags": [
{
"Key": "KubernetesCluster",
"Value": "externallb.example.com"
},
{
"Key": "Name",
"Value": "masters.externallb.example.com"
},
{
"Key": "kubernetes.io/cluster/externallb.example.com",
"Value": "owned"
}
]
}
},
"AWSEC2SecurityGroupnodesexternallbexamplecom": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"VpcId": {
"Ref": "AWSEC2VPCexternallbexamplecom"
},
"GroupDescription": "Security group for nodes",
"Tags": [
{
"Key": "KubernetesCluster",
"Value": "externallb.example.com"
},
{
"Key": "Name",
"Value": "nodes.externallb.example.com"
},
{
"Key": "kubernetes.io/cluster/externallb.example.com",
"Value": "owned"
}
]
}
},
"AWSEC2SubnetRouteTableAssociationustest1aexternallbexamplecom": {
"Type": "AWS::EC2::SubnetRouteTableAssociation",
"Properties": {
"SubnetId": {
"Ref": "AWSEC2Subnetustest1aexternallbexamplecom"
},
"RouteTableId": {
"Ref": "AWSEC2RouteTableexternallbexamplecom"
}
}
},
"AWSEC2Subnetustest1aexternallbexamplecom": {
"Type": "AWS::EC2::Subnet",
"Properties": {
"VpcId": {
"Ref": "AWSEC2VPCexternallbexamplecom"
},
"CidrBlock": "172.20.32.0/19",
"AvailabilityZone": "us-test-1a",
"Tags": [
{
"Key": "KubernetesCluster",
"Value": "externallb.example.com"
},
{
"Key": "Name",
"Value": "us-test-1a.externallb.example.com"
},
{
"Key": "SubnetType",
"Value": "Public"
},
{
"Key": "kubernetes.io/cluster/externallb.example.com",
"Value": "owned"
},
{
"Key": "kubernetes.io/role/elb",
"Value": "1"
}
]
}
},
"AWSEC2VPCDHCPOptionsAssociationexternallbexamplecom": {
"Type": "AWS::EC2::VPCDHCPOptionsAssociation",
"Properties": {
"VpcId": {
"Ref": "AWSEC2VPCexternallbexamplecom"
},
"DhcpOptionsId": {
"Ref": "AWSEC2DHCPOptionsexternallbexamplecom"
}
}
},
"AWSEC2VPCGatewayAttachmentexternallbexamplecom": {
"Type": "AWS::EC2::VPCGatewayAttachment",
"Properties": {
"VpcId": {
"Ref": "AWSEC2VPCexternallbexamplecom"
},
"InternetGatewayId": {
"Ref": "AWSEC2InternetGatewayexternallbexamplecom"
}
}
},
"AWSEC2VPCexternallbexamplecom": {
"Type": "AWS::EC2::VPC",
"Properties": {
"CidrBlock": "172.20.0.0/16",
"EnableDnsHostnames": true,
"EnableDnsSupport": true,
"Tags": [
{
"Key": "KubernetesCluster",
"Value": "externallb.example.com"
},
{
"Key": "Name",
"Value": "externallb.example.com"
},
{
"Key": "kubernetes.io/cluster/externallb.example.com",
"Value": "owned"
}
]
}
},
"AWSEC2Volumeustest1aetcdeventsexternallbexamplecom": {
"Type": "AWS::EC2::Volume",
"Properties": {
"AvailabilityZone": "us-test-1a",
"Size": 20,
"VolumeType": "gp2",
"Encrypted": false,
"Tags": [
{
"Key": "KubernetesCluster",
"Value": "externallb.example.com"
},
{
"Key": "Name",
"Value": "us-test-1a.etcd-events.externallb.example.com"
},
{
"Key": "k8s.io/etcd/events",
"Value": "us-test-1a/us-test-1a"
},
{
"Key": "k8s.io/role/master",
"Value": "1"
},
{
"Key": "kubernetes.io/cluster/externallb.example.com",
"Value": "owned"
}
]
}
},
"AWSEC2Volumeustest1aetcdmainexternallbexamplecom": {
"Type": "AWS::EC2::Volume",
"Properties": {
"AvailabilityZone": "us-test-1a",
"Size": 20,
"VolumeType": "gp2",
"Encrypted": false,
"Tags": [
{
"Key": "KubernetesCluster",
"Value": "externallb.example.com"
},
{
"Key": "Name",
"Value": "us-test-1a.etcd-main.externallb.example.com"
},
{
"Key": "k8s.io/etcd/main",
"Value": "us-test-1a/us-test-1a"
},
{
"Key": "k8s.io/role/master",
"Value": "1"
},
{
"Key": "kubernetes.io/cluster/externallb.example.com",
"Value": "owned"
}
]
}
},
"AWSIAMInstanceProfilemastersexternallbexamplecom": {
"Type": "AWS::IAM::InstanceProfile",
"Properties": {
"Roles": [
{
"Ref": "AWSIAMRolemastersexternallbexamplecom"
}
]
}
},
"AWSIAMInstanceProfilenodesexternallbexamplecom": {
"Type": "AWS::IAM::InstanceProfile",
"Properties": {
"Roles": [
{
"Ref": "AWSIAMRolenodesexternallbexamplecom"
}
]
}
},
"AWSIAMPolicymastersexternallbexamplecom": {
"Type": "AWS::IAM::Policy",
"Properties": {
"PolicyName": "masters.externallb.example.com",
"Roles": [
{
"Ref": "AWSIAMRolemastersexternallbexamplecom"
}
],
"PolicyDocument": {
"Statement": [
{
"Action": [
"ec2:*"
],
"Effect": "Allow",
"Resource": [
"*"
]
},
{
"Action": [
"autoscaling:DescribeAutoScalingGroups",
"autoscaling:DescribeAutoScalingInstances",
"autoscaling:DescribeLaunchConfigurations",
"autoscaling:DescribeTags",
"autoscaling:GetAsgForInstance",
"autoscaling:SetDesiredCapacity",
"autoscaling:TerminateInstanceInAutoScalingGroup",
"autoscaling:UpdateAutoScalingGroup"
],
"Effect": "Allow",
"Resource": [
"*"
]
},
{
"Action": [
"elasticloadbalancing:*"
],
"Effect": "Allow",
"Resource": [
"*"
]
},
{
"Action": [
"iam:ListServerCertificates",
"iam:GetServerCertificate"
],
"Effect": "Allow",
"Resource": [
"*"
]
},
{
"Action": [
"route53:ChangeResourceRecordSets",
"route53:ListResourceRecordSets",
"route53:GetHostedZone"
],
"Effect": "Allow",
"Resource": [
"arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO"
]
},
{
"Action": [
"route53:GetChange"
],
"Effect": "Allow",
"Resource": [
"arn:aws:route53:::change/*"
]
},
{
"Action": [
"route53:ListHostedZones"
],
"Effect": "Allow",
"Resource": [
"*"
]
},
{
"Action": [
"route53:ListHostedZones"
],
"Effect": "Allow",
"Resource": [
"*"
]
},
{
"Action": [
"ecr:GetAuthorizationToken",
"ecr:BatchCheckLayerAvailability",
"ecr:GetDownloadUrlForLayer",
"ecr:GetRepositoryPolicy",
"ecr:DescribeRepositories",
"ecr:ListImages",
"ecr:BatchGetImage"
],
"Effect": "Allow",
"Resource": [
"*"
]
}
],
"Version": "2012-10-17"
}
}
},
"AWSIAMPolicynodesexternallbexamplecom": {
"Type": "AWS::IAM::Policy",
"Properties": {
"PolicyName": "nodes.externallb.example.com",
"Roles": [
{
"Ref": "AWSIAMRolenodesexternallbexamplecom"
}
],
"PolicyDocument": {
"Statement": [
{
"Action": [
"ec2:DescribeInstances",
"ec2:DescribeRegions"
],
"Effect": "Allow",
"Resource": [
"*"
]
},
{
"Action": [
"route53:ChangeResourceRecordSets",
"route53:ListResourceRecordSets",
"route53:GetHostedZone"
],
"Effect": "Allow",
"Resource": [
"arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO"
]
},
{
"Action": [
"route53:GetChange"
],
"Effect": "Allow",
"Resource": [
"arn:aws:route53:::change/*"
]
},
{
"Action": [
"route53:ListHostedZones"
],
"Effect": "Allow",
"Resource": [
"*"
]
},
{
"Action": [
"route53:ListHostedZones"
],
"Effect": "Allow",
"Resource": [
"*"
]
},
{
"Action": [
"ecr:GetAuthorizationToken",
"ecr:BatchCheckLayerAvailability",
"ecr:GetDownloadUrlForLayer",
"ecr:GetRepositoryPolicy",
"ecr:DescribeRepositories",
"ecr:ListImages",
"ecr:BatchGetImage"
],
"Effect": "Allow",
"Resource": [
"*"
]
}
],
"Version": "2012-10-17"
}
}
},
"AWSIAMRolemastersexternallbexamplecom": {
"Type": "AWS::IAM::Role",
"Properties": {
"RoleName": "masters.externallb.example.com",
"AssumeRolePolicyDocument": {
"Statement": [
{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": {
"Service": "ec2.amazonaws.com"
}
}
],
"Version": "2012-10-17"
}
}
},
"AWSIAMRolenodesexternallbexamplecom": {
"Type": "AWS::IAM::Role",
"Properties": {
"RoleName": "nodes.externallb.example.com",
"AssumeRolePolicyDocument": {
"Statement": [
{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": {
"Service": "ec2.amazonaws.com"
}
}
],
"Version": "2012-10-17"
}
}
}
}
}

507
tests/integration/update_cluster/externallb/cloudformation.json.extracted.yaml Normal file
View File

@ -0,0 +1,507 @@
Resources.AWSAutoScalingLaunchConfigurationmasterustest1amastersexternallbexamplecom.Properties.UserData: |
#!/bin/bash
# Copyright 2016 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
NODEUP_URL=https://kubeupv2.s3.amazonaws.com/kops/1.8.1/linux/amd64/nodeup
NODEUP_HASH=bb41724c37d15ab7e039e06230e742b9b38d0808
export AWS_REGION=us-test-1
function ensure-install-dir() {
INSTALL_DIR="/var/cache/kubernetes-install"
# On ContainerOS, we install to /var/lib/toolbox install (because of noexec)
if [[ -d /var/lib/toolbox ]]; then
INSTALL_DIR="/var/lib/toolbox/kubernetes-install"
fi
mkdir -p ${INSTALL_DIR}
cd ${INSTALL_DIR}
}
# Retry a download until we get it. Takes a hash and a set of URLs.
#
# $1 is the sha1 of the URL. Can be "" if the sha1 is unknown.
# $2+ are the URLs to download.
download-or-bust() {
local -r hash="$1"
shift 1
urls=( $* )
while true; do
for url in "${urls[@]}"; do
local file="${url##*/}"
rm -f "${file}"
if [[ $(which curl) ]]; then
if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; then
echo "== Failed to curl ${url}. Retrying. =="
break
fi
elif [[ $(which wget ) ]]; then
if ! wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10 "${url}"; then
echo "== Failed to wget ${url}. Retrying. =="
break
fi
else
echo "== Could not find curl or wget. Retrying. =="
break
fi
if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
echo "== Hash validation of ${url} failed. Retrying. =="
else
if [[ -n "${hash}" ]]; then
echo "== Downloaded ${url} (SHA1 = ${hash}) =="
else
echo "== Downloaded ${url} =="
fi
return
fi
done
echo "All downloads failed; sleeping before retrying"
sleep 60
done
}
validate-hash() {
local -r file="$1"
local -r expected="$2"
local actual
actual=$(sha1sum ${file} | awk '{ print $1 }') || true
if [[ "${actual}" != "${expected}" ]]; then
echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} =="
return 1
fi
}
function split-commas() {
echo $1 | tr "," "\n"
}
function try-download-release() {
# TODO(zmerlynn): Now we REALLY have no excuse not to do the reboot
# optimization.
local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
local -r nodeup_filename="${nodeup_urls[0]##*/}"
if [[ -n "${NODEUP_HASH:-}" ]]; then
local -r nodeup_hash="${NODEUP_HASH}"
else
# TODO: Remove?
echo "Downloading sha1 (not found in env)"
download-or-bust "" "${nodeup_urls[@]/%/.sha1}"
local -r nodeup_hash=$(cat "${nodeup_filename}.sha1")
fi
echo "Downloading nodeup (${nodeup_urls[@]})"
download-or-bust "${nodeup_hash}" "${nodeup_urls[@]}"
chmod +x nodeup
}
function download-release() {
# In case of failure checking integrity of release, retry.
until try-download-release; do
sleep 15
echo "Couldn't download release. Retrying..."
done
echo "Running nodeup"
# We can't run in the foreground because of https://github.com/docker/docker/issues/23793
( cd ${INSTALL_DIR}; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/kube_env.yaml --v=8 )
}
####################################################################################
/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
echo "== nodeup node config starting =="
ensure-install-dir
cat > cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
cloudConfig: null
docker:
ipMasq: false
ipTables: false
logDriver: json-file
logLevel: warn
logOpt:
- max-size=10m
- max-file=5
storage: overlay,aufs
version: 1.13.1
encryptionConfig: null
etcdClusters:
events:
image: gcr.io/google_containers/etcd:2.2.1
version: 2.2.1
main:
image: gcr.io/google_containers/etcd:2.2.1
version: 2.2.1
kubeAPIServer:
address: 127.0.0.1
admissionControl:
- Initializers
- NamespaceLifecycle
- LimitRanger
- ServiceAccount
- PersistentVolumeLabel
- DefaultStorageClass
- DefaultTolerationSeconds
- NodeRestriction
- ResourceQuota
allowPrivileged: true
anonymousAuth: false
apiServerCount: 1
authorizationMode: AlwaysAllow
cloudProvider: aws
etcdServers:
- http://127.0.0.1:4001
etcdServersOverrides:
- /events#http://127.0.0.1:4002
image: gcr.io/google_containers/kube-apiserver:v1.8.0
insecurePort: 8080
kubeletPreferredAddressTypes:
- InternalIP
- Hostname
- ExternalIP
logLevel: 2
requestheaderAllowedNames:
- aggregator
requestheaderExtraHeaderPrefixes:
- X-Remote-Extra-
requestheaderGroupHeaders:
- X-Remote-Group
requestheaderUsernameHeaders:
- X-Remote-User
securePort: 443
serviceClusterIPRange: 100.64.0.0/13
storageBackend: etcd2
kubeControllerManager:
allocateNodeCIDRs: true
attachDetachReconcileSyncPeriod: 1m0s
cloudProvider: aws
clusterCIDR: 100.96.0.0/11
clusterName: externallb.example.com
configureCloudRoutes: true
image: gcr.io/google_containers/kube-controller-manager:v1.8.0
leaderElection:
leaderElect: true
logLevel: 2
useServiceAccountCredentials: true
kubeProxy:
clusterCIDR: 100.96.0.0/11
cpuRequest: 100m
hostnameOverride: '@aws'
image: gcr.io/google_containers/kube-proxy:v1.8.0
logLevel: 2
kubeScheduler:
image: gcr.io/google_containers/kube-scheduler:v1.8.0
leaderElection:
leaderElect: true
logLevel: 2
kubelet:
allowPrivileged: true
cgroupRoot: /
cloudProvider: aws
clusterDNS: 100.64.0.10
clusterDomain: cluster.local
enableDebuggingHandlers: true
evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
featureGates:
ExperimentalCriticalPodAnnotation: "true"
hostnameOverride: '@aws'
kubeconfigPath: /var/lib/kubelet/kubeconfig
logLevel: 2
networkPluginMTU: 9001
networkPluginName: kubenet
nonMasqueradeCIDR: 100.64.0.0/10
podInfraContainerImage: gcr.io/google_containers/pause-amd64:3.0
podManifestPath: /etc/kubernetes/manifests
requireKubeconfig: true
masterKubelet:
allowPrivileged: true
cgroupRoot: /
cloudProvider: aws
clusterDNS: 100.64.0.10
clusterDomain: cluster.local
enableDebuggingHandlers: true
evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
featureGates:
ExperimentalCriticalPodAnnotation: "true"
hostnameOverride: '@aws'
kubeconfigPath: /var/lib/kubelet/kubeconfig
logLevel: 2
networkPluginMTU: 9001
networkPluginName: kubenet
nonMasqueradeCIDR: 100.64.0.0/10
podInfraContainerImage: gcr.io/google_containers/pause-amd64:3.0
podManifestPath: /etc/kubernetes/manifests
registerSchedulable: false
requireKubeconfig: true
__EOF_CLUSTER_SPEC
cat > ig_spec.yaml << '__EOF_IG_SPEC'
kubelet: null
nodeLabels: null
suspendProcesses: null
taints: null
__EOF_IG_SPEC
cat > kube_env.yaml << '__EOF_KUBE_ENV'
Assets:
- 4c7b8aafe652ae107c9131754a2ad4e9641a025b@https://storage.googleapis.com/kubernetes-release/release/v1.8.0/bin/linux/amd64/kubelet
- 006fd43085e6ba2dc6b35b89af4d68cee3f689c9@https://storage.googleapis.com/kubernetes-release/release/v1.8.0/bin/linux/amd64/kubectl
- 1d9788b0f5420e1a219aad2cb8681823fc515e7c@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-0799f5732f2a11b329d9e3d51b9c8f2e3759f2ff.tar.gz
- 42b15a0a0a56531750bde3c7b08d0cf27c170c48@https://kubeupv2.s3.amazonaws.com/kops/1.8.1/linux/amd64/utils.tar.gz
ClusterName: externallb.example.com
ConfigBase: memfs://clusters.example.com/externallb.example.com
InstanceGroupName: master-us-test-1a
Tags:
- _automatic_upgrades
- _aws
- _kubernetes_master
channels:
- memfs://clusters.example.com/externallb.example.com/addons/bootstrap-channel.yaml
protokubeImage:
hash: 0b1f26208f8f6cc02468368706d0236670fec8a2
name: protokube:1.8.1
source: https://kubeupv2.s3.amazonaws.com/kops/1.8.1/images/protokube.tar.gz
__EOF_KUBE_ENV
download-release
echo "== nodeup node config done =="
Resources.AWSAutoScalingLaunchConfigurationnodesexternallbexamplecom.Properties.UserData: |
#!/bin/bash
# Copyright 2016 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
NODEUP_URL=https://kubeupv2.s3.amazonaws.com/kops/1.8.1/linux/amd64/nodeup
NODEUP_HASH=bb41724c37d15ab7e039e06230e742b9b38d0808
export AWS_REGION=us-test-1
function ensure-install-dir() {
INSTALL_DIR="/var/cache/kubernetes-install"
# On ContainerOS, we install to /var/lib/toolbox install (because of noexec)
if [[ -d /var/lib/toolbox ]]; then
INSTALL_DIR="/var/lib/toolbox/kubernetes-install"
fi
mkdir -p ${INSTALL_DIR}
cd ${INSTALL_DIR}
}
# Retry a download until we get it. Takes a hash and a set of URLs.
#
# $1 is the sha1 of the URL. Can be "" if the sha1 is unknown.
# $2+ are the URLs to download.
download-or-bust() {
local -r hash="$1"
shift 1
urls=( $* )
while true; do
for url in "${urls[@]}"; do
local file="${url##*/}"
rm -f "${file}"
if [[ $(which curl) ]]; then
if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; then
echo "== Failed to curl ${url}. Retrying. =="
break
fi
elif [[ $(which wget ) ]]; then
if ! wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10 "${url}"; then
echo "== Failed to wget ${url}. Retrying. =="
break
fi
else
echo "== Could not find curl or wget. Retrying. =="
break
fi
if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
echo "== Hash validation of ${url} failed. Retrying. =="
else
if [[ -n "${hash}" ]]; then
echo "== Downloaded ${url} (SHA1 = ${hash}) =="
else
echo "== Downloaded ${url} =="
fi
return
fi
done
echo "All downloads failed; sleeping before retrying"
sleep 60
done
}
validate-hash() {
local -r file="$1"
local -r expected="$2"
local actual
actual=$(sha1sum ${file} | awk '{ print $1 }') || true
if [[ "${actual}" != "${expected}" ]]; then
echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} =="
return 1
fi
}
function split-commas() {
echo $1 | tr "," "\n"
}
function try-download-release() {
# TODO(zmerlynn): Now we REALLY have no excuse not to do the reboot
# optimization.
local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
local -r nodeup_filename="${nodeup_urls[0]##*/}"
if [[ -n "${NODEUP_HASH:-}" ]]; then
local -r nodeup_hash="${NODEUP_HASH}"
else
# TODO: Remove?
echo "Downloading sha1 (not found in env)"
download-or-bust "" "${nodeup_urls[@]/%/.sha1}"
local -r nodeup_hash=$(cat "${nodeup_filename}.sha1")
fi
echo "Downloading nodeup (${nodeup_urls[@]})"
download-or-bust "${nodeup_hash}" "${nodeup_urls[@]}"
chmod +x nodeup
}
function download-release() {
# In case of failure checking integrity of release, retry.
until try-download-release; do
sleep 15
echo "Couldn't download release. Retrying..."
done
echo "Running nodeup"
# We can't run in the foreground because of https://github.com/docker/docker/issues/23793
( cd ${INSTALL_DIR}; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/kube_env.yaml --v=8 )
}
####################################################################################
/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
echo "== nodeup node config starting =="
ensure-install-dir
cat > cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
cloudConfig: null
docker:
ipMasq: false
ipTables: false
logDriver: json-file
logLevel: warn
logOpt:
- max-size=10m
- max-file=5
storage: overlay,aufs
version: 1.13.1
kubeProxy:
clusterCIDR: 100.96.0.0/11
cpuRequest: 100m
hostnameOverride: '@aws'
image: gcr.io/google_containers/kube-proxy:v1.8.0
logLevel: 2
kubelet:
allowPrivileged: true
cgroupRoot: /
cloudProvider: aws
clusterDNS: 100.64.0.10
clusterDomain: cluster.local
enableDebuggingHandlers: true
evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
featureGates:
ExperimentalCriticalPodAnnotation: "true"
hostnameOverride: '@aws'
kubeconfigPath: /var/lib/kubelet/kubeconfig
logLevel: 2
networkPluginMTU: 9001
networkPluginName: kubenet
nonMasqueradeCIDR: 100.64.0.0/10
podInfraContainerImage: gcr.io/google_containers/pause-amd64:3.0
podManifestPath: /etc/kubernetes/manifests
requireKubeconfig: true
__EOF_CLUSTER_SPEC
cat > ig_spec.yaml << '__EOF_IG_SPEC'
kubelet: null
nodeLabels: null
suspendProcesses: null
taints: null
__EOF_IG_SPEC
cat > kube_env.yaml << '__EOF_KUBE_ENV'
Assets:
- 4c7b8aafe652ae107c9131754a2ad4e9641a025b@https://storage.googleapis.com/kubernetes-release/release/v1.8.0/bin/linux/amd64/kubelet
- 006fd43085e6ba2dc6b35b89af4d68cee3f689c9@https://storage.googleapis.com/kubernetes-release/release/v1.8.0/bin/linux/amd64/kubectl
- 1d9788b0f5420e1a219aad2cb8681823fc515e7c@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-0799f5732f2a11b329d9e3d51b9c8f2e3759f2ff.tar.gz
- 42b15a0a0a56531750bde3c7b08d0cf27c170c48@https://kubeupv2.s3.amazonaws.com/kops/1.8.1/linux/amd64/utils.tar.gz
ClusterName: externallb.example.com
ConfigBase: memfs://clusters.example.com/externallb.example.com
InstanceGroupName: nodes
Tags:
- _automatic_upgrades
- _aws
channels:
- memfs://clusters.example.com/externallb.example.com/addons/bootstrap-channel.yaml
protokubeImage:
hash: 0b1f26208f8f6cc02468368706d0236670fec8a2
name: protokube:1.8.1
source: https://kubeupv2.s3.amazonaws.com/kops/1.8.1/images/protokube.tar.gz
__EOF_KUBE_ENV
download-release
echo "== nodeup node config done =="

1
tests/integration/update_cluster/externallb/id_rsa.pub Normal file
View File

@ -0,0 +1 @@
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==

82
tests/integration/update_cluster/externallb/in-v1alpha2.yaml Normal file
View File

@ -0,0 +1,82 @@
apiVersion: kops/v1alpha2
kind: Cluster
metadata:
creationTimestamp: "2018-03-20T16:00:27Z"
name: externallb.example.com
spec:
kubernetesApiAccess:
- 0.0.0.0/0
channel: stable
cloudProvider: aws
configBase: memfs://clusters.example.com/externallb.example.com
etcdClusters:
- etcdMembers:
- instanceGroup: master-us-test-1a
name: us-test-1a
name: main
- etcdMembers:
- instanceGroup: master-us-test-1a
name: us-test-1a
name: events
kubernetesVersion: v1.8.0
masterInternalName: api.internal.externallb.example.com
masterPublicName: api.externallb.example.com
networkCIDR: 172.20.0.0/16
networking:
kubenet: {}
nonMasqueradeCIDR: 100.64.0.0/10
sshAccess:
- 0.0.0.0/0
topology:
masters: public
nodes: public
subnets:
- cidr: 172.20.32.0/19
name: us-test-1a
type: Public
zone: us-test-1a
---
apiVersion: kops/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: "2016-12-10T22:42:28Z"
name: nodes
labels:
kops.k8s.io/cluster: externallb.example.com
spec:
associatePublicIp: true
image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21
machineType: t2.medium
maxSize: 2
minSize: 2
role: Node
subnets:
- us-test-1a
externalLoadBalancers:
- loadBalancerName: my-elb
---
apiVersion: kops/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: "2016-12-10T22:42:28Z"
name: master-us-test-1a
labels:
kops.k8s.io/cluster: externallb.example.com
spec:
associatePublicIp: true
image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21
machineType: m3.medium
maxSize: 1
minSize: 1
role: Master
subnets:
- us-test-1a
externalLoadBalancers:
- targetGroupArn: aws:arn:elasticloadbalancing:us-test-1a:123456789012:targetgroup/my-tg/0123456789abcdef
- loadBalancerName: my-other-elb

456
tests/integration/update_cluster/externallb/kubernetes.tf Normal file
View File

@ -0,0 +1,456 @@
output "cluster_name" {
value = "externallb.example.com"
}
output "master_security_group_ids" {
value = ["${aws_security_group.masters-externallb-example-com.id}"]
}
output "masters_role_arn" {
value = "${aws_iam_role.masters-externallb-example-com.arn}"
}
output "masters_role_name" {
value = "${aws_iam_role.masters-externallb-example-com.name}"
}
output "node_security_group_ids" {
value = ["${aws_security_group.nodes-externallb-example-com.id}"]
}
output "node_subnet_ids" {
value = ["${aws_subnet.us-test-1a-externallb-example-com.id}"]
}
output "nodes_role_arn" {
value = "${aws_iam_role.nodes-externallb-example-com.arn}"
}
output "nodes_role_name" {
value = "${aws_iam_role.nodes-externallb-example-com.name}"
}
output "region" {
value = "us-test-1"
}
output "route_table_public_id" {
value = "${aws_route_table.externallb-example-com.id}"
}
output "subnet_us-test-1a-public_id" {
value = "${aws_subnet.us-test-1a-externallb-example-com.id}"
}
output "vpc_cidr_block" {
value = "${aws_vpc.externallb-example-com.cidr_block}"
}
output "vpc_id" {
value = "${aws_vpc.externallb-example-com.id}"
}
provider "aws" {
region = "us-test-1"
}
resource "aws_autoscaling_attachment" "extlb-my-elb-nodes" {
elb = "my-elb"
autoscaling_group_name = "${aws_autoscaling_group.nodes-externallb-example-com.id}"
}
resource "aws_autoscaling_attachment" "extlb-my-other-elb-master-us-test-1a" {
elb = "my-other-elb"
autoscaling_group_name = "${aws_autoscaling_group.master-us-test-1a-masters-externallb-example-com.id}"
}
resource "aws_autoscaling_attachment" "exttg-aws:my-tg--0123456789abcdef-master-us-test-1a" {
alb_target_group_arn = "aws:arn:elasticloadbalancing:us-test-1a:123456789012:targetgroup/my-tg/0123456789abcdef"
autoscaling_group_name = "${aws_autoscaling_group.master-us-test-1a-masters-externallb-example-com.id}"
}
resource "aws_autoscaling_group" "master-us-test-1a-masters-externallb-example-com" {
name = "master-us-test-1a.masters.externallb.example.com"
launch_configuration = "${aws_launch_configuration.master-us-test-1a-masters-externallb-example-com.id}"
max_size = 1
min_size = 1
vpc_zone_identifier = ["${aws_subnet.us-test-1a-externallb-example-com.id}"]
tag = {
key = "KubernetesCluster"
value = "externallb.example.com"
propagate_at_launch = true
}
tag = {
key = "Name"
value = "master-us-test-1a.masters.externallb.example.com"
propagate_at_launch = true
}
tag = {
key = "k8s.io/role/master"
value = "1"
propagate_at_launch = true
}
metrics_granularity = "1Minute"
enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
}
resource "aws_autoscaling_group" "nodes-externallb-example-com" {
name = "nodes.externallb.example.com"
launch_configuration = "${aws_launch_configuration.nodes-externallb-example-com.id}"
max_size = 2
min_size = 2
vpc_zone_identifier = ["${aws_subnet.us-test-1a-externallb-example-com.id}"]
tag = {
key = "KubernetesCluster"
value = "externallb.example.com"
propagate_at_launch = true
}
tag = {
key = "Name"
value = "nodes.externallb.example.com"
propagate_at_launch = true
}
tag = {
key = "k8s.io/role/node"
value = "1"
propagate_at_launch = true
}
metrics_granularity = "1Minute"
enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"]
}
resource "aws_ebs_volume" "us-test-1a-etcd-events-externallb-example-com" {
availability_zone = "us-test-1a"
size = 20
type = "gp2"
encrypted = false
tags = {
KubernetesCluster = "externallb.example.com"
Name = "us-test-1a.etcd-events.externallb.example.com"
"k8s.io/etcd/events" = "us-test-1a/us-test-1a"
"k8s.io/role/master" = "1"
"kubernetes.io/cluster/externallb.example.com" = "owned"
}
}
resource "aws_ebs_volume" "us-test-1a-etcd-main-externallb-example-com" {
availability_zone = "us-test-1a"
size = 20
type = "gp2"
encrypted = false
tags = {
KubernetesCluster = "externallb.example.com"
Name = "us-test-1a.etcd-main.externallb.example.com"
"k8s.io/etcd/main" = "us-test-1a/us-test-1a"
"k8s.io/role/master" = "1"
"kubernetes.io/cluster/externallb.example.com" = "owned"
}
}
resource "aws_iam_instance_profile" "masters-externallb-example-com" {
name = "masters.externallb.example.com"
role = "${aws_iam_role.masters-externallb-example-com.name}"
}
resource "aws_iam_instance_profile" "nodes-externallb-example-com" {
name = "nodes.externallb.example.com"
role = "${aws_iam_role.nodes-externallb-example-com.name}"
}
resource "aws_iam_role" "masters-externallb-example-com" {
name = "masters.externallb.example.com"
assume_role_policy = "${file("${path.module}/data/aws_iam_role_masters.externallb.example.com_policy")}"
}
resource "aws_iam_role" "nodes-externallb-example-com" {
name = "nodes.externallb.example.com"
assume_role_policy = "${file("${path.module}/data/aws_iam_role_nodes.externallb.example.com_policy")}"
}
resource "aws_iam_role_policy" "masters-externallb-example-com" {
name = "masters.externallb.example.com"
role = "${aws_iam_role.masters-externallb-example-com.name}"
policy = "${file("${path.module}/data/aws_iam_role_policy_masters.externallb.example.com_policy")}"
}
resource "aws_iam_role_policy" "nodes-externallb-example-com" {
name = "nodes.externallb.example.com"
role = "${aws_iam_role.nodes-externallb-example-com.name}"
policy = "${file("${path.module}/data/aws_iam_role_policy_nodes.externallb.example.com_policy")}"
}
resource "aws_internet_gateway" "externallb-example-com" {
vpc_id = "${aws_vpc.externallb-example-com.id}"
tags = {
KubernetesCluster = "externallb.example.com"
Name = "externallb.example.com"
"kubernetes.io/cluster/externallb.example.com" = "owned"
}
}
resource "aws_key_pair" "kubernetes-externallb-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" {
key_name = "kubernetes.externallb.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57"
public_key = "${file("${path.module}/data/aws_key_pair_kubernetes.externallb.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key")}"
}
resource "aws_launch_configuration" "master-us-test-1a-masters-externallb-example-com" {
name_prefix = "master-us-test-1a.masters.externallb.example.com-"
image_id = "ami-12345678"
instance_type = "m3.medium"
key_name = "${aws_key_pair.kubernetes-externallb-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id}"
iam_instance_profile = "${aws_iam_instance_profile.masters-externallb-example-com.id}"
security_groups = ["${aws_security_group.masters-externallb-example-com.id}"]
associate_public_ip_address = true
user_data = "${file("${path.module}/data/aws_launch_configuration_master-us-test-1a.masters.externallb.example.com_user_data")}"
root_block_device = {
volume_type = "gp2"
volume_size = 64
delete_on_termination = true
}
ephemeral_block_device = {
device_name = "/dev/sdc"
virtual_name = "ephemeral0"
}
lifecycle = {
create_before_destroy = true
}
enable_monitoring = false
}
resource "aws_launch_configuration" "nodes-externallb-example-com" {
name_prefix = "nodes.externallb.example.com-"
image_id = "ami-12345678"
instance_type = "t2.medium"
key_name = "${aws_key_pair.kubernetes-externallb-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id}"
iam_instance_profile = "${aws_iam_instance_profile.nodes-externallb-example-com.id}"
security_groups = ["${aws_security_group.nodes-externallb-example-com.id}"]
associate_public_ip_address = true
user_data = "${file("${path.module}/data/aws_launch_configuration_nodes.externallb.example.com_user_data")}"
root_block_device = {
volume_type = "gp2"
volume_size = 128
delete_on_termination = true
}
lifecycle = {
create_before_destroy = true
}
enable_monitoring = false
}
resource "aws_route" "0-0-0-0--0" {
route_table_id = "${aws_route_table.externallb-example-com.id}"
destination_cidr_block = "0.0.0.0/0"
gateway_id = "${aws_internet_gateway.externallb-example-com.id}"
}
resource "aws_route_table" "externallb-example-com" {
vpc_id = "${aws_vpc.externallb-example-com.id}"
tags = {
KubernetesCluster = "externallb.example.com"
Name = "externallb.example.com"
"kubernetes.io/cluster/externallb.example.com" = "owned"
"kubernetes.io/kops/role" = "public"
}
}
resource "aws_route_table_association" "us-test-1a-externallb-example-com" {
subnet_id = "${aws_subnet.us-test-1a-externallb-example-com.id}"
route_table_id = "${aws_route_table.externallb-example-com.id}"
}
resource "aws_security_group" "masters-externallb-example-com" {
name = "masters.externallb.example.com"
vpc_id = "${aws_vpc.externallb-example-com.id}"
description = "Security group for masters"
tags = {
KubernetesCluster = "externallb.example.com"
Name = "masters.externallb.example.com"
"kubernetes.io/cluster/externallb.example.com" = "owned"
}
}
resource "aws_security_group" "nodes-externallb-example-com" {
name = "nodes.externallb.example.com"
vpc_id = "${aws_vpc.externallb-example-com.id}"
description = "Security group for nodes"
tags = {
KubernetesCluster = "externallb.example.com"
Name = "nodes.externallb.example.com"
"kubernetes.io/cluster/externallb.example.com" = "owned"
}
}
resource "aws_security_group_rule" "all-master-to-master" {
type = "ingress"
security_group_id = "${aws_security_group.masters-externallb-example-com.id}"
source_security_group_id = "${aws_security_group.masters-externallb-example-com.id}"
from_port = 0
to_port = 0
protocol = "-1"
}
resource "aws_security_group_rule" "all-master-to-node" {
type = "ingress"
security_group_id = "${aws_security_group.nodes-externallb-example-com.id}"
source_security_group_id = "${aws_security_group.masters-externallb-example-com.id}"
from_port = 0
to_port = 0
protocol = "-1"
}
resource "aws_security_group_rule" "all-node-to-node" {
type = "ingress"
security_group_id = "${aws_security_group.nodes-externallb-example-com.id}"
source_security_group_id = "${aws_security_group.nodes-externallb-example-com.id}"
from_port = 0
to_port = 0
protocol = "-1"
}
resource "aws_security_group_rule" "https-external-to-master-0-0-0-0--0" {
type = "ingress"
security_group_id = "${aws_security_group.masters-externallb-example-com.id}"
from_port = 443
to_port = 443
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
}
resource "aws_security_group_rule" "master-egress" {
type = "egress"
security_group_id = "${aws_security_group.masters-externallb-example-com.id}"
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
resource "aws_security_group_rule" "node-egress" {
type = "egress"
security_group_id = "${aws_security_group.nodes-externallb-example-com.id}"
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
resource "aws_security_group_rule" "node-to-master-tcp-1-2379" {
type = "ingress"
security_group_id = "${aws_security_group.masters-externallb-example-com.id}"
source_security_group_id = "${aws_security_group.nodes-externallb-example-com.id}"
from_port = 1
to_port = 2379
protocol = "tcp"
}
resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" {
type = "ingress"
security_group_id = "${aws_security_group.masters-externallb-example-com.id}"
source_security_group_id = "${aws_security_group.nodes-externallb-example-com.id}"
from_port = 2382
to_port = 4000
protocol = "tcp"
}
resource "aws_security_group_rule" "node-to-master-tcp-4003-65535" {
type = "ingress"
security_group_id = "${aws_security_group.masters-externallb-example-com.id}"
source_security_group_id = "${aws_security_group.nodes-externallb-example-com.id}"
from_port = 4003
to_port = 65535
protocol = "tcp"
}
resource "aws_security_group_rule" "node-to-master-udp-1-65535" {
type = "ingress"
security_group_id = "${aws_security_group.masters-externallb-example-com.id}"
source_security_group_id = "${aws_security_group.nodes-externallb-example-com.id}"
from_port = 1
to_port = 65535
protocol = "udp"
}
resource "aws_security_group_rule" "ssh-external-to-master-0-0-0-0--0" {
type = "ingress"
security_group_id = "${aws_security_group.masters-externallb-example-com.id}"
from_port = 22
to_port = 22
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
}
resource "aws_security_group_rule" "ssh-external-to-node-0-0-0-0--0" {
type = "ingress"
security_group_id = "${aws_security_group.nodes-externallb-example-com.id}"
from_port = 22
to_port = 22
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
}
resource "aws_subnet" "us-test-1a-externallb-example-com" {
vpc_id = "${aws_vpc.externallb-example-com.id}"
cidr_block = "172.20.32.0/19"
availability_zone = "us-test-1a"
tags = {
KubernetesCluster = "externallb.example.com"
Name = "us-test-1a.externallb.example.com"
SubnetType = "Public"
"kubernetes.io/cluster/externallb.example.com" = "owned"
"kubernetes.io/role/elb" = "1"
}
}
resource "aws_vpc" "externallb-example-com" {
cidr_block = "172.20.0.0/16"
enable_dns_hostnames = true
enable_dns_support = true
tags = {
KubernetesCluster = "externallb.example.com"
Name = "externallb.example.com"
"kubernetes.io/cluster/externallb.example.com" = "owned"
}
}
resource "aws_vpc_dhcp_options" "externallb-example-com" {
domain_name = "us-test-1.compute.internal"
domain_name_servers = ["AmazonProvidedDNS"]
tags = {
KubernetesCluster = "externallb.example.com"
Name = "externallb.example.com"
"kubernetes.io/cluster/externallb.example.com" = "owned"
}
}
resource "aws_vpc_dhcp_options_association" "externallb-example-com" {
vpc_id = "${aws_vpc.externallb-example-com.id}"
dhcp_options_id = "${aws_vpc_dhcp_options.externallb-example-com.id}"
}
terraform = {
required_version = ">= 0.9.3"
}

4
upup/pkg/fi/cloudup/awstasks/BUILD.bazel
View File

@ -18,6 +18,10 @@ go_library(
"ebsvolume_fitask.go",
"elastic_ip.go",
"elasticip_fitask.go",
"external_load_balancer_attachment.go",
"external_target_group_attachment.go",
"externalloadbalancerattachment_fitask.go",
"externaltargetgroupattachment_fitask.go",
"iaminstanceprofile.go",
"iaminstanceprofile_fitask.go",
"iaminstanceprofilerole.go",

1
upup/pkg/fi/cloudup/awstasks/autoscalinggroup.go
View File

@ -459,6 +459,7 @@ type cloudformationAutoscalingGroup struct {
MetricsCollection []*cloudformationASGMetricsCollection `json:"MetricsCollection,omitempty"`
LoadBalancerNames []*cloudformation.Literal `json:"LoadBalancerNames,omitempty"`
TargetGroupARNs []*cloudformation.Literal `json:"TargetGroupARNs,omitempty"`
}
func (_ *AutoscalingGroup) RenderCloudformation(t *cloudformation.CloudformationTarget, a, e, changes *AutoscalingGroup) error {

140
upup/pkg/fi/cloudup/awstasks/external_load_balancer_attachment.go Normal file
View File

@ -0,0 +1,140 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package awstasks
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/autoscaling"
"github.com/golang/glog"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
"k8s.io/kops/upup/pkg/fi/cloudup/cloudformation"
"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
)
//go:generate fitask -type=ExternalLoadBalancerAttachment
type ExternalLoadBalancerAttachment struct {
Name *string
Lifecycle *fi.Lifecycle
LoadBalancerName string
AutoscalingGroup *AutoscalingGroup
}
func (e *ExternalLoadBalancerAttachment) Find(c *fi.Context) (*ExternalLoadBalancerAttachment, error) {
cloud := c.Cloud.(awsup.AWSCloud)
if e.LoadBalancerName == "" {
return nil, fmt.Errorf("InstanceGroup did not have LoadBalancerNames set")
}
g, err := findAutoscalingGroup(cloud, *e.AutoscalingGroup.Name)
if err != nil {
return nil, err
}
if g == nil {
return nil, nil
}
for _, name := range g.LoadBalancerNames {
if aws.StringValue(name) != e.LoadBalancerName {
continue
}
actual := &ExternalLoadBalancerAttachment{}
actual.LoadBalancerName = e.LoadBalancerName
actual.AutoscalingGroup = e.AutoscalingGroup
// Prevent spurious changes
actual.Name = e.Name // ELB attachments don't have tags
actual.Lifecycle = e.Lifecycle
return actual, nil
}
return nil, nil
}
func (e *ExternalLoadBalancerAttachment) Run(c *fi.Context) error {
return fi.DefaultDeltaRunMethod(e, c)
}
func (s *ExternalLoadBalancerAttachment) CheckChanges(a, e, changes *ExternalLoadBalancerAttachment) error {
if a == nil {
if e.LoadBalancerName == "" {
return fi.RequiredField("LoadBalancerName")
}
if e.AutoscalingGroup == nil {
return fi.RequiredField("AutoscalingGroup")
}
}
return nil
}
func (_ *ExternalLoadBalancerAttachment) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *ExternalLoadBalancerAttachment) error {
if e.LoadBalancerName == "" {
return fi.RequiredField("LoadBalancerName")
}
request := &autoscaling.AttachLoadBalancersInput{}
request.AutoScalingGroupName = e.AutoscalingGroup.Name
request.LoadBalancerNames = aws.StringSlice([]string{e.LoadBalancerName})
glog.V(2).Infof("Attaching autoscaling group %q to ELB %q", fi.StringValue(e.AutoscalingGroup.Name), e.LoadBalancerName)
_, err := t.Cloud.Autoscaling().AttachLoadBalancers(request)
if err != nil {
return fmt.Errorf("error attaching autoscaling group to ELB: %v", err)
}
return nil
}
type terraformExternalLoadBalancerAttachment struct {
ELB *terraform.Literal `json:"elb"`
AutoscalingGroup *terraform.Literal `json:"autoscaling_group_name,omitempty"`
}
func (_ *ExternalLoadBalancerAttachment) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *ExternalLoadBalancerAttachment) error {
tf := &terraformExternalLoadBalancerAttachment{
ELB: terraform.LiteralFromStringValue(e.LoadBalancerName),
AutoscalingGroup: e.AutoscalingGroup.TerraformLink(),
}
return t.RenderResource("aws_autoscaling_attachment", *e.Name, tf)
}
func (e *ExternalLoadBalancerAttachment) TerraformLink() *terraform.Literal {
return terraform.LiteralProperty("aws_autoscaling_attachment", e.LoadBalancerName, "id")
}
func (_ *ExternalLoadBalancerAttachment) RenderCloudformation(t *cloudformation.CloudformationTarget, a, e, changes *ExternalLoadBalancerAttachment) error {
cfObj, ok := t.Find(e.AutoscalingGroup.CloudformationLink())
if !ok {
// topo-sort fail?
return fmt.Errorf("AutoScalingGroup not yet rendered")
}
cf, ok := cfObj.(*cloudformationAutoscalingGroup)
if !ok {
return fmt.Errorf("unexpected type for CF record: %T", cfObj)
}
cf.LoadBalancerNames = append(cf.LoadBalancerNames, cloudformation.LiteralString(e.LoadBalancerName))
return nil
}

146
upup/pkg/fi/cloudup/awstasks/external_target_group_attachment.go Normal file
View File

@ -0,0 +1,146 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package awstasks
import (
"fmt"
"regexp"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/autoscaling"
"github.com/golang/glog"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
"k8s.io/kops/upup/pkg/fi/cloudup/cloudformation"
"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
)
//go:generate fitask -type=ExternalTargetGroupAttachment
type ExternalTargetGroupAttachment struct {
Name *string
Lifecycle *fi.Lifecycle
TargetGroupARN string
AutoscalingGroup *AutoscalingGroup
}
func (e *ExternalTargetGroupAttachment) name() string {
re := regexp.MustCompile("arn:.*:targetgroup/")
return re.ReplaceAllString(*e.Name, "")
}
func (e *ExternalTargetGroupAttachment) Find(c *fi.Context) (*ExternalTargetGroupAttachment, error) {
cloud := c.Cloud.(awsup.AWSCloud)
if e.TargetGroupARN == "" {
return nil, fmt.Errorf("InstanceGroup did not have TargetGroupARNs set")
}
g, err := findAutoscalingGroup(cloud, *e.AutoscalingGroup.Name)
if err != nil {
return nil, err
}
if g == nil {
return nil, nil
}
for _, name := range g.TargetGroupARNs {
if aws.StringValue(name) != e.TargetGroupARN {
continue
}
actual := &ExternalTargetGroupAttachment{}
actual.TargetGroupARN = e.TargetGroupARN
actual.AutoscalingGroup = e.AutoscalingGroup
// Prevent spurious changes
actual.Name = e.Name // ELB attachments don't have tags
actual.Lifecycle = e.Lifecycle
return actual, nil
}
return nil, nil
}
func (e *ExternalTargetGroupAttachment) Run(c *fi.Context) error {
return fi.DefaultDeltaRunMethod(e, c)
}
func (s *ExternalTargetGroupAttachment) CheckChanges(a, e, changes *ExternalTargetGroupAttachment) error {
if a == nil {
if e.TargetGroupARN == "" {
return fi.RequiredField("TargetGroupARN")
}
if e.AutoscalingGroup == nil {
return fi.RequiredField("AutoscalingGroup")
}
}
return nil
}
func (_ *ExternalTargetGroupAttachment) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *ExternalTargetGroupAttachment) error {
if e.TargetGroupARN == "" {
return fi.RequiredField("TargetGroupARN")
}
request := &autoscaling.AttachLoadBalancerTargetGroupsInput{}
request.AutoScalingGroupName = e.AutoscalingGroup.Name
request.TargetGroupARNs = aws.StringSlice([]string{e.TargetGroupARN})
glog.V(2).Infof("Attaching autoscaling group %q to Target Group %q", fi.StringValue(e.AutoscalingGroup.Name), e.TargetGroupARN)
_, err := t.Cloud.Autoscaling().AttachLoadBalancerTargetGroups(request)
if err != nil {
return fmt.Errorf("error attaching autoscaling group to ELB: %v", err)
}
return nil
}
type terraformExternalTargetGroupAttachment struct {
TargetGroupARN *terraform.Literal `json:"alb_target_group_arn,omitempty"`
AutoscalingGroup *terraform.Literal `json:"autoscaling_group_name,omitempty"`
}
func (_ *ExternalTargetGroupAttachment) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *ExternalTargetGroupAttachment) error {
tf := &terraformExternalTargetGroupAttachment{
TargetGroupARN: terraform.LiteralFromStringValue(e.TargetGroupARN),
AutoscalingGroup: e.AutoscalingGroup.TerraformLink(),
}
return t.RenderResource("aws_autoscaling_attachment", e.name(), tf)
}
func (e *ExternalTargetGroupAttachment) TerraformLink() *terraform.Literal {
return terraform.LiteralProperty("aws_autoscaling_attachment", e.name(), "id")
}
func (_ *ExternalTargetGroupAttachment) RenderCloudformation(t *cloudformation.CloudformationTarget, a, e, changes *ExternalTargetGroupAttachment) error {
cfObj, ok := t.Find(e.AutoscalingGroup.CloudformationLink())
if !ok {
// topo-sort fail?
return fmt.Errorf("AutoScalingGroup not yet rendered")
}
cf, ok := cfObj.(*cloudformationAutoscalingGroup)
if !ok {
return fmt.Errorf("unexpected type for CF record: %T", cfObj)
}
cf.TargetGroupARNs = append(cf.TargetGroupARNs, cloudformation.LiteralString(e.TargetGroupARN))
return nil
}

75
upup/pkg/fi/cloudup/awstasks/externalloadbalancerattachment_fitask.go Normal file
View File

@ -0,0 +1,75 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Code generated by ""fitask" -type=ExternalLoadBalancerAttachment"; DO NOT EDIT
package awstasks
import (
"encoding/json"
"k8s.io/kops/upup/pkg/fi"
)
// ExternalLoadBalancerAttachment
// JSON marshalling boilerplate
type realExternalLoadBalancerAttachment ExternalLoadBalancerAttachment
// UnmarshalJSON implements conversion to JSON, supporitng an alternate specification of the object as a string
func (o *ExternalLoadBalancerAttachment) UnmarshalJSON(data []byte) error {
var jsonName string
if err := json.Unmarshal(data, &jsonName); err == nil {
o.Name = &jsonName
return nil
}
var r realExternalLoadBalancerAttachment
if err := json.Unmarshal(data, &r); err != nil {
return err
}
*o = ExternalLoadBalancerAttachment(r)
return nil
}
var _ fi.HasLifecycle = &ExternalLoadBalancerAttachment{}
// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle
func (o *ExternalLoadBalancerAttachment) GetLifecycle() *fi.Lifecycle {
return o.Lifecycle
}
// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle
func (o *ExternalLoadBalancerAttachment) SetLifecycle(lifecycle fi.Lifecycle) {
o.Lifecycle = &lifecycle
}
var _ fi.HasName = &ExternalLoadBalancerAttachment{}
// GetName returns the Name of the object, implementing fi.HasName
func (o *ExternalLoadBalancerAttachment) GetName() *string {
return o.Name
}
// SetName sets the Name of the object, implementing fi.SetName
func (o *ExternalLoadBalancerAttachment) SetName(name string) {
o.Name = &name
}
// String is the stringer function for the task, producing readable output using fi.TaskAsString
func (o *ExternalLoadBalancerAttachment) String() string {
return fi.TaskAsString(o)
}

75
upup/pkg/fi/cloudup/awstasks/externaltargetgroupattachment_fitask.go Normal file
View File

@ -0,0 +1,75 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Code generated by ""fitask" -type=ExternalTargetGroupAttachment"; DO NOT EDIT
package awstasks
import (
"encoding/json"
"k8s.io/kops/upup/pkg/fi"
)
// ExternalTargetGroupAttachment
// JSON marshalling boilerplate
type realExternalTargetGroupAttachment ExternalTargetGroupAttachment
// UnmarshalJSON implements conversion to JSON, supporitng an alternate specification of the object as a string
func (o *ExternalTargetGroupAttachment) UnmarshalJSON(data []byte) error {
var jsonName string
if err := json.Unmarshal(data, &jsonName); err == nil {
o.Name = &jsonName
return nil
}
var r realExternalTargetGroupAttachment
if err := json.Unmarshal(data, &r); err != nil {
return err
}
*o = ExternalTargetGroupAttachment(r)
return nil
}
var _ fi.HasLifecycle = &ExternalTargetGroupAttachment{}
// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle
func (o *ExternalTargetGroupAttachment) GetLifecycle() *fi.Lifecycle {
return o.Lifecycle
}
// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle
func (o *ExternalTargetGroupAttachment) SetLifecycle(lifecycle fi.Lifecycle) {
o.Lifecycle = &lifecycle
}
var _ fi.HasName = &ExternalTargetGroupAttachment{}
// GetName returns the Name of the object, implementing fi.HasName
func (o *ExternalTargetGroupAttachment) GetName() *string {
return o.Name
}
// SetName sets the Name of the object, implementing fi.SetName
func (o *ExternalTargetGroupAttachment) SetName(name string) {
o.Name = &name
}
// String is the stringer function for the task, producing readable output using fi.TaskAsString
func (o *ExternalTargetGroupAttachment) String() string {
return fi.TaskAsString(o)
}