kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #9348 from johngmyers/rollingupdate-disable 

Create separate field for disabling rolling updates
This commit is contained in:
Kubernetes Prow Robot 2020-07-02 09:08:47 -07:00 committed by GitHub
parent 4d20013ead 16daa3ecac
commit 219147e2f4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 139 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docs/operations/rolling-update.md
View File

@ -144,12 +144,14 @@ then creates any remaining surge instances.
#### Disabling rolling updates
Rolling updates may be disabled for an instance group by setting both `maxSurge` and `maxUnavailable`
to `0`.
Rolling updates may be partially disabled for an instance group by setting the `drainAndTerminate`
field to `false`.
```yaml
spec:
rollingUpdate:
maxSurge: 0
maxUnavailable: 0
drainAndTerminate: false
```
Nodes needing update will still be tainted. If `maxSurge` is nonzero, up to that many extra
nodes will still be created.

29
k8s/crds/kops.k8s.io_clusters.yaml
View File

@ -3363,6 +3363,10 @@ spec:
description: RollingUpdate defines the default rolling-update settings
for instance groups
properties:
drainAndTerminate:
description: DrainAndTerminate enables draining and terminating
nodes during rolling updates. Defaults to true.
type: boolean
maxSurge:
anyOf:
- type: integer
@ -3371,12 +3375,11 @@ spec:
can be created during the update. The value can be an absolute
number (for example 5) or a percentage of desired machines (for
example 10%). The absolute number is calculated from a percentage
by rounding up. A value of 0 for both this and MaxUnavailable
disables rolling updates. Has no effect on instance groups with
role "Master". Defaults to 1 on AWS, 0 otherwise. Example: when
this is set to 30%, the InstanceGroup can be scaled up immediately
when the rolling update starts, such that the total number of
old and new nodes do not exceed 130% of desired nodes.'
by rounding up. Has no effect on instance groups with role "Master".
Defaults to 1 on AWS, 0 otherwise. Example: when this is set
to 30%, the InstanceGroup can be scaled up immediately when
the rolling update starts, such that the total number of old
and new nodes do not exceed 130% of desired nodes.'
x-kubernetes-int-or-string: true
maxUnavailable:
anyOf:
@ -3386,13 +3389,13 @@ spec:
can be unavailable during the update. The value can be an absolute
number (for example 5) or a percentage of desired nodes (for
example 10%). The absolute number is calculated from a percentage
by rounding down. A value of 0 for both this and MaxSurge disables
rolling updates. Defaults to 1 if MaxSurge is 0, otherwise defaults
to 0. Example: when this is set to 30%, the InstanceGroup can
be scaled down to 70% of desired nodes immediately when the
rolling update starts. Once new nodes are ready, more old nodes
can be drained, ensuring that the total number of nodes available
at all times during the update is at least 70% of desired nodes.'
by rounding down. Defaults to 1 if MaxSurge is 0, otherwise
defaults to 0. Example: when this is set to 30%, the InstanceGroup
can be scaled down to 70% of desired nodes immediately when
the rolling update starts. Once new nodes are ready, more old
nodes can be drained, ensuring that the total number of nodes
available at all times during the update is at least 70% of
desired nodes.'
x-kubernetes-int-or-string: true
type: object
secretStore:

29
k8s/crds/kops.k8s.io_instancegroups.yaml
View File

@ -653,6 +653,10 @@ spec:
rollingUpdate:
description: RollingUpdate defines the rolling-update behavior
properties:
drainAndTerminate:
description: DrainAndTerminate enables draining and terminating
nodes during rolling updates. Defaults to true.
type: boolean
maxSurge:
anyOf:
- type: integer
@ -661,12 +665,11 @@ spec:
can be created during the update. The value can be an absolute
number (for example 5) or a percentage of desired machines (for
example 10%). The absolute number is calculated from a percentage
by rounding up. A value of 0 for both this and MaxUnavailable
disables rolling updates. Has no effect on instance groups with
role "Master". Defaults to 1 on AWS, 0 otherwise. Example: when
this is set to 30%, the InstanceGroup can be scaled up immediately
when the rolling update starts, such that the total number of
old and new nodes do not exceed 130% of desired nodes.'
by rounding up. Has no effect on instance groups with role "Master".
Defaults to 1 on AWS, 0 otherwise. Example: when this is set
to 30%, the InstanceGroup can be scaled up immediately when
the rolling update starts, such that the total number of old
and new nodes do not exceed 130% of desired nodes.'
x-kubernetes-int-or-string: true
maxUnavailable:
anyOf:
@ -676,13 +679,13 @@ spec:
can be unavailable during the update. The value can be an absolute
number (for example 5) or a percentage of desired nodes (for
example 10%). The absolute number is calculated from a percentage
by rounding down. A value of 0 for both this and MaxSurge disables
rolling updates. Defaults to 1 if MaxSurge is 0, otherwise defaults
to 0. Example: when this is set to 30%, the InstanceGroup can
be scaled down to 70% of desired nodes immediately when the
rolling update starts. Once new nodes are ready, more old nodes
can be drained, ensuring that the total number of nodes available
at all times during the update is at least 70% of desired nodes.'
by rounding down. Defaults to 1 if MaxSurge is 0, otherwise
defaults to 0. Example: when this is set to 30%, the InstanceGroup
can be scaled down to 70% of desired nodes immediately when
the rolling update starts. Once new nodes are ready, more old
nodes can be drained, ensuring that the total number of nodes
available at all times during the update is at least 70% of
desired nodes.'
x-kubernetes-int-or-string: true
type: object
rootVolumeDeleteOnTermination:

5
pkg/apis/kops/cluster.go
View File

@ -695,11 +695,13 @@ type DNSControllerGossipConfig struct {
}
type RollingUpdate struct {
// DrainAndTerminate enables draining and terminating nodes during rolling updates.
// Defaults to true.
DrainAndTerminate *bool `json:"drainAndTerminate,omitempty"`
// MaxUnavailable is the maximum number of nodes that can be unavailable during the update.
// The value can be an absolute number (for example 5) or a percentage of desired
// nodes (for example 10%).
// The absolute number is calculated from a percentage by rounding down.
// A value of 0 for both this and MaxSurge disables rolling updates.
// Defaults to 1 if MaxSurge is 0, otherwise defaults to 0.
// Example: when this is set to 30%, the InstanceGroup can be scaled
// down to 70% of desired nodes immediately when the rolling update
@ -713,7 +715,6 @@ type RollingUpdate struct {
// The value can be an absolute number (for example 5) or a percentage of
// desired machines (for example 10%).
// The absolute number is calculated from a percentage by rounding up.
// A value of 0 for both this and MaxUnavailable disables rolling updates.
// Has no effect on instance groups with role "Master".
// Defaults to 1 on AWS, 0 otherwise.
// Example: when this is set to 30%, the InstanceGroup can be scaled

5
pkg/apis/kops/v1alpha2/cluster.go
View File

@ -590,11 +590,13 @@ type DNSControllerGossipConfig struct {
}
type RollingUpdate struct {
// DrainAndTerminate enables draining and terminating nodes during rolling updates.
// Defaults to true.
DrainAndTerminate *bool `json:"drainAndTerminate,omitempty"`
// MaxUnavailable is the maximum number of nodes that can be unavailable during the update.
// The value can be an absolute number (for example 5) or a percentage of desired
// nodes (for example 10%).
// The absolute number is calculated from a percentage by rounding down.
// A value of 0 for both this and MaxSurge disables rolling updates.
// Defaults to 1 if MaxSurge is 0, otherwise defaults to 0.
// Example: when this is set to 30%, the InstanceGroup can be scaled
// down to 70% of desired nodes immediately when the rolling update
@ -608,7 +610,6 @@ type RollingUpdate struct {
// The value can be an absolute number (for example 5) or a percentage of
// desired machines (for example 10%).
// The absolute number is calculated from a percentage by rounding up.
// A value of 0 for both this and MaxUnavailable disables rolling updates.
// Has no effect on instance groups with role "Master".
// Defaults to 1 on AWS, 0 otherwise.
// Example: when this is set to 30%, the InstanceGroup can be scaled

2
pkg/apis/kops/v1alpha2/zz_generated.conversion.go
View File

@ -5149,6 +5149,7 @@ func Convert_kops_RBACAuthorizationSpec_To_v1alpha2_RBACAuthorizationSpec(in *ko
}
func autoConvert_v1alpha2_RollingUpdate_To_kops_RollingUpdate(in *RollingUpdate, out *kops.RollingUpdate, s conversion.Scope) error {
out.DrainAndTerminate = in.DrainAndTerminate
out.MaxUnavailable = in.MaxUnavailable
out.MaxSurge = in.MaxSurge
return nil
@ -5160,6 +5161,7 @@ func Convert_v1alpha2_RollingUpdate_To_kops_RollingUpdate(in *RollingUpdate, out
}
func autoConvert_kops_RollingUpdate_To_v1alpha2_RollingUpdate(in *kops.RollingUpdate, out *RollingUpdate, s conversion.Scope) error {
out.DrainAndTerminate = in.DrainAndTerminate
out.MaxUnavailable = in.MaxUnavailable
out.MaxSurge = in.MaxSurge
return nil

5
pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go
View File

@ -3486,6 +3486,11 @@ func (in *RBACAuthorizationSpec) DeepCopy() *RBACAuthorizationSpec {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *RollingUpdate) DeepCopyInto(out *RollingUpdate) {
*out = *in
if in.DrainAndTerminate != nil {
in, out := &in.DrainAndTerminate, &out.DrainAndTerminate
*out = new(bool)
**out = **in
}
if in.MaxUnavailable != nil {
in, out := &in.MaxUnavailable, &out.MaxUnavailable
*out = new(intstr.IntOrString)

7
pkg/apis/kops/validation/validation.go
View File

@ -1040,8 +1040,10 @@ func validateDockerConfig(config *kops.DockerConfig, fldPath *field.Path) field.
func validateRollingUpdate(rollingUpdate *kops.RollingUpdate, fldpath *field.Path, onMasterInstanceGroup bool) field.ErrorList {
allErrs := field.ErrorList{}
var err error
unavailable := 1
if rollingUpdate.MaxUnavailable != nil {
unavailable, err := intstr.GetValueFromIntOrPercent(rollingUpdate.MaxUnavailable, 1, false)
unavailable, err = intstr.GetValueFromIntOrPercent(rollingUpdate.MaxUnavailable, 1, false)
if err != nil {
allErrs = append(allErrs, field.Invalid(fldpath.Child("maxUnavailable"), rollingUpdate.MaxUnavailable,
fmt.Sprintf("Unable to parse: %v", err)))
@ -1061,6 +1063,9 @@ func validateRollingUpdate(rollingUpdate *kops.RollingUpdate, fldpath *field.Pat
} else if surge < 0 {
allErrs = append(allErrs, field.Invalid(fldpath.Child("maxSurge"), rollingUpdate.MaxSurge, "Cannot be negative"))
}
if unavailable == 0 && surge == 0 {
allErrs = append(allErrs, field.Forbidden(fldpath.Child("maxSurge"), "Cannot be zero if maxUnavailable is zero"))
}
}
return allErrs

7
pkg/apis/kops/validation/validation_test.go
View File

@ -749,6 +749,13 @@ func Test_Validate_RollingUpdate(t *testing.T) {
OnMasterIG: true,
ExpectedErrors: []string{"Forbidden::testField.maxSurge"},
},
{
Input: kops.RollingUpdate{
MaxUnavailable: intStr(intstr.FromInt(0)),
MaxSurge: intStr(intstr.FromInt(0)),
},
ExpectedErrors: []string{"Forbidden::testField.maxSurge"},
},
}
for _, g := range grid {
errs := validateRollingUpdate(&g.Input, field.NewPath("testField"), g.OnMasterIG)

5
pkg/apis/kops/zz_generated.deepcopy.go
View File

@ -3700,6 +3700,11 @@ func (in *RBACAuthorizationSpec) DeepCopy() *RBACAuthorizationSpec {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *RollingUpdate) DeepCopyInto(out *RollingUpdate) {
*out = *in
if in.DrainAndTerminate != nil {
in, out := &in.DrainAndTerminate, &out.DrainAndTerminate
*out = new(bool)
**out = **in
}
if in.MaxUnavailable != nil {
in, out := &in.MaxUnavailable, &out.MaxUnavailable
*out = new(intstr.IntOrString)

1
pkg/instancegroups/BUILD.bazel
View File

@ -42,6 +42,7 @@ go_test(
"//pkg/apis/kops:go_default_library",
"//pkg/cloudinstances:go_default_library",
"//pkg/validation:go_default_library",
"//upup/pkg/fi:go_default_library",
"//upup/pkg/fi/cloudup/awsup:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/service/autoscaling:go_default_library",

10
pkg/instancegroups/instancegroups.go
View File

@ -108,11 +108,6 @@ func (c *RollingUpdateCluster) rollingUpdateInstanceGroup(ctx context.Context, c
}
maxConcurrency := maxSurge + settings.MaxUnavailable.IntValue()
if maxConcurrency == 0 {
klog.Infof("Rolling updates for InstanceGroup %s are disabled", group.InstanceGroup.Name)
return nil
}
if group.InstanceGroup.Spec.Role == api.InstanceGroupRoleMaster && maxSurge != 0 {
// Masters are incapable of surging because they rely on registering themselves through
// the local apiserver. That apiserver depends on the local etcd, which relies on being
@ -157,6 +152,11 @@ func (c *RollingUpdateCluster) rollingUpdateInstanceGroup(ctx context.Context, c
}
}
if !*settings.DrainAndTerminate {
klog.Infof("Rolling updates for InstanceGroup %s are disabled", group.InstanceGroup.Name)
return nil
}
terminateChan := make(chan error, maxConcurrency)
for uIdx, u := range update {

55
pkg/instancegroups/rollingupdate_test.go
View File

@ -40,6 +40,7 @@ import (
kopsapi "k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/cloudinstances"
"k8s.io/kops/pkg/validation"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
)
@ -800,9 +801,8 @@ func TestRollingUpdateDisabled(t *testing.T) {
c, cloud, cluster := getTestSetup()
zero := intstr.FromInt(0)
cluster.Spec.RollingUpdate = &kopsapi.RollingUpdate{
MaxUnavailable: &zero,
DrainAndTerminate: fi.Bool(false),
}
groups := getGroupsAllNeedUpdate(c.K8sClient, cloud)
@ -821,9 +821,8 @@ func TestRollingUpdateDisabledCloudonly(t *testing.T) {
c, cloud, cluster := getTestSetup()
c.CloudOnly = true
zero := intstr.FromInt(0)
cluster.Spec.RollingUpdate = &kopsapi.RollingUpdate{
MaxUnavailable: &zero,
DrainAndTerminate: fi.Bool(false),
}
groups := getGroupsAllNeedUpdate(c.K8sClient, cloud)
@ -836,6 +835,54 @@ func TestRollingUpdateDisabledCloudonly(t *testing.T) {
assertGroupInstanceCount(t, cloud, "bastion-1", 1)
}
type disabledSurgeTest struct {
autoscalingiface.AutoScalingAPI
t *testing.T
mutex sync.Mutex
numDetached int
}
func (m *disabledSurgeTest) DetachInstances(input *autoscaling.DetachInstancesInput) (*autoscaling.DetachInstancesOutput, error) {
m.mutex.Lock()
defer m.mutex.Unlock()
for _, id := range input.InstanceIds {
assert.NotContains(m.t, *id, "master")
m.numDetached++
}
return &autoscaling.DetachInstancesOutput{}, nil
}
func TestRollingUpdateDisabledSurge(t *testing.T) {
ctx := context.Background()
c, cloud, cluster := getTestSetup()
disabledSurgeTest := &disabledSurgeTest{
AutoScalingAPI: cloud.MockAutoscaling,
t: t,
}
cloud.MockAutoscaling = disabledSurgeTest
cloud.MockEC2 = &ec2IgnoreTags{EC2API: cloud.MockEC2}
one := intstr.FromInt(1)
cluster.Spec.RollingUpdate = &kopsapi.RollingUpdate{
DrainAndTerminate: fi.Bool(false),
MaxSurge: &one,
}
groups := getGroupsAllNeedUpdate(c.K8sClient, cloud)
err := c.RollingUpdate(ctx, groups, cluster, &kopsapi.InstanceGroupList{})
assert.NoError(t, err, "rolling update")
assertGroupInstanceCount(t, cloud, "node-1", 3)
assertGroupInstanceCount(t, cloud, "node-2", 3)
assertGroupInstanceCount(t, cloud, "master-1", 2)
assertGroupInstanceCount(t, cloud, "bastion-1", 1)
assert.Equal(t, 3, disabledSurgeTest.numDetached)
}
// The concurrent update tests attempt to induce the following expected update sequence:
//
// (Only for surging "all need update" test, to verify the toe-dipping behavior)

8
pkg/instancegroups/settings.go
View File

@ -20,6 +20,7 @@ import (
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/featureflag"
"k8s.io/kops/upup/pkg/fi"
)
func resolveSettings(cluster *kops.Cluster, group *kops.InstanceGroup, numInstances int) kops.RollingUpdate {
@ -29,6 +30,9 @@ func resolveSettings(cluster *kops.Cluster, group *kops.InstanceGroup, numInstan
}
if def := cluster.Spec.RollingUpdate; def != nil {
if rollingUpdate.DrainAndTerminate == nil {
rollingUpdate.DrainAndTerminate = def.DrainAndTerminate
}
if rollingUpdate.MaxUnavailable == nil {
rollingUpdate.MaxUnavailable = def.MaxUnavailable
}
@ -37,6 +41,10 @@ func resolveSettings(cluster *kops.Cluster, group *kops.InstanceGroup, numInstan
}
}
if rollingUpdate.DrainAndTerminate == nil {
rollingUpdate.DrainAndTerminate = fi.Bool(true)
}
if rollingUpdate.MaxSurge == nil {
val := intstr.FromInt(0)
if kops.CloudProviderID(cluster.Spec.CloudProvider) == kops.CloudProviderAWS && !featureflag.Spotinst.Enabled() {

5
pkg/instancegroups/settings_test.go
View File

@ -32,6 +32,11 @@ func TestSettings(t *testing.T) {
defaultValue interface{}
nonDefaultValue interface{}
}{
{
name: "DrainAndTerminate",
defaultValue: true,
nonDefaultValue: false,
},
{
name: "MaxUnavailable",
defaultValue: intstr.FromInt(1),