diff --git a/channels/pkg/api/channel.go b/channels/pkg/api/channel.go index 32eb826643..41e1a6397f 100644 --- a/channels/pkg/api/channel.go +++ b/channels/pkg/api/channel.go @@ -62,6 +62,11 @@ type AddonSpec struct { // version of the software we are packaging. But we always want to reinstall when we // switch kubernetes versions. Id string `json:"id,omitempty"` + + // NeedsRollingUpdate determines if we should mark nodes as needing an update. + // Legal values are control-plane, workers, and all + // Empty value means no update needed + NeedsRollingUpdate string `json:"needsRollingUpdate,omitempty"` } func (a *Addons) Verify() error { diff --git a/channels/pkg/channels/addon.go b/channels/pkg/channels/addon.go index 327cf479ef..c3dada651f 100644 --- a/channels/pkg/channels/addon.go +++ b/channels/pkg/channels/addon.go @@ -18,13 +18,18 @@ package channels import ( "context" + "encoding/json" "fmt" "net/url" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" "k8s.io/kops/channels/pkg/api" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // Addon is a wrapper around a single version of an addon @@ -144,6 +149,13 @@ func (a *Addon) EnsureUpdated(ctx context.Context, k8sClient kubernetes.Interfac return nil, fmt.Errorf("error applying update from %q: %v", manifestURL, err) } + if a.Spec.NeedsRollingUpdate != "" { + err = a.AddNeedsUpdateLabel(ctx, k8sClient) + if err != nil { + return nil, fmt.Errorf("error adding needs-update label: %v", err) + } + } + channel := a.buildChannel() err = channel.SetInstalledVersion(ctx, k8sClient, a.ChannelVersion()) if err != nil { @@ -152,3 +164,36 @@ func (a *Addon) EnsureUpdated(ctx context.Context, k8sClient kubernetes.Interfac return required, nil } + +func (a *Addon) AddNeedsUpdateLabel(ctx context.Context, k8sClient kubernetes.Interface) error { + klog.Infof("addon %v wants to update %v nodes", a.Name, a.Spec.NeedsRollingUpdate) + selector := "" + switch a.Spec.NeedsRollingUpdate { + case "control-plane": + selector = "node-role.kubernetes.io/master=" + case "worker": + selector = "node-role.kubernetes.io/node=" + } + + annotationPatch := &annotationPatch{Metadata: annotationPatchMetadata{Annotations: map[string]string{ + "kops.k8s.io/needs-update": "", + }}} + annotationPatchJSON, err := json.Marshal(annotationPatch) + if err != nil { + return err + } + + nodeInterface := k8sClient.CoreV1().Nodes() + nodes, err := nodeInterface.List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return err + } + for _, node := range nodes.Items { + _, err = nodeInterface.Patch(ctx, node.Name, types.StrategicMergePatchType, annotationPatchJSON, metav1.PatchOptions{}) + + if err != nil { + return err + } + } + return nil +} diff --git a/cmd/kops/get_instances.go b/cmd/kops/get_instances.go index 1e7234c154..92b0553af6 100644 --- a/cmd/kops/get_instances.go +++ b/cmd/kops/get_instances.go @@ -126,6 +126,7 @@ func RunGetInstances(ctx context.Context, f *util.Factory, out io.Writer, option var cloudInstances []*cloudinstances.CloudInstance cloudGroups, err := cloud.GetCloudGroups(cluster, instanceGroups, false, nodeList.Items) + if err != nil { return err } @@ -133,6 +134,7 @@ func RunGetInstances(ctx context.Context, f *util.Factory, out io.Writer, option for _, cg := range cloudGroups { cloudInstances = append(cloudInstances, cg.Ready...) cloudInstances = append(cloudInstances, cg.NeedUpdate...) + cg.AdjustNeedUpdate() } switch options.output { diff --git a/cmd/kops/rollingupdatecluster.go b/cmd/kops/rollingupdatecluster.go index 567bf85013..84aa3ce237 100644 --- a/cmd/kops/rollingupdatecluster.go +++ b/cmd/kops/rollingupdatecluster.go @@ -350,7 +350,7 @@ func RunRollingUpdateCluster(ctx context.Context, f *util.Factory, out io.Writer ValidateSuccessDuration: 10 * time.Second, } - err = d.AdjustNeedUpdate(groups, list) + err = d.AdjustNeedUpdate(groups) if err != nil { return err } diff --git a/pkg/cloudinstances/cloud_instance_group.go b/pkg/cloudinstances/cloud_instance_group.go index 341d8faa63..4f60c30ad4 100644 --- a/pkg/cloudinstances/cloud_instance_group.go +++ b/pkg/cloudinstances/cloud_instance_group.go @@ -75,6 +75,29 @@ func (c *CloudInstanceGroup) Status() string { return "NeedsUpdate" } +func (group *CloudInstanceGroup) AdjustNeedUpdate() { + + if group.Ready != nil { + var newReady []*CloudInstance + for _, member := range group.Ready { + makeNotReady := false + if member.Node != nil && member.Node.Annotations != nil { + if _, ok := member.Node.Annotations["kops.k8s.io/needs-update"]; ok { + makeNotReady = true + } + } + + if makeNotReady { + group.NeedUpdate = append(group.NeedUpdate, member) + member.Status = CloudInstanceStatusNeedsUpdate + } else { + newReady = append(newReady, member) + } + } + group.Ready = newReady + } +} + // GetNodeMap returns a list of nodes keyed by their external id func GetNodeMap(nodes []v1.Node, cluster *kopsapi.Cluster) map[string]*v1.Node { nodeMap := make(map[string]*v1.Node) diff --git a/pkg/instancegroups/rollingupdate.go b/pkg/instancegroups/rollingupdate.go index 70a6366ad6..01a4f0b45d 100644 --- a/pkg/instancegroups/rollingupdate.go +++ b/pkg/instancegroups/rollingupdate.go @@ -80,27 +80,9 @@ type RollingUpdateCluster struct { } // AdjustNeedUpdate adjusts the set of instances that need updating, using factors outside those known by the cloud implementation -func (c *RollingUpdateCluster) AdjustNeedUpdate(groups map[string]*cloudinstances.CloudInstanceGroup, instanceGroups *api.InstanceGroupList) error { +func (*RollingUpdateCluster) AdjustNeedUpdate(groups map[string]*cloudinstances.CloudInstanceGroup) error { for _, group := range groups { - if group.Ready != nil { - var newReady []*cloudinstances.CloudInstance - for _, member := range group.Ready { - makeNotReady := false - if member.Node != nil && member.Node.Annotations != nil { - if _, ok := member.Node.Annotations["kops.k8s.io/needs-update"]; ok { - makeNotReady = true - } - } - - if makeNotReady { - group.NeedUpdate = append(group.NeedUpdate, member) - member.Status = cloudinstances.CloudInstanceStatusNeedsUpdate - } else { - newReady = append(newReady, member) - } - } - group.Ready = newReady - } + group.AdjustNeedUpdate() } return nil } diff --git a/pkg/instancegroups/rollingupdate_test.go b/pkg/instancegroups/rollingupdate_test.go index 441ec51a53..9cfa4edbff 100644 --- a/pkg/instancegroups/rollingupdate_test.go +++ b/pkg/instancegroups/rollingupdate_test.go @@ -736,7 +736,7 @@ func TestAddAnnotatedNodesToNeedsUpdate(t *testing.T) { addNeedsUpdateAnnotation(groups["node-2"], "node-2a") addNeedsUpdateAnnotation(groups["master-1"], "master-1b") - err := c.AdjustNeedUpdate(groups, &kopsapi.InstanceGroupList{}) + err := c.AdjustNeedUpdate(groups) assert.NoError(t, err, "AddAnnotatedNodesToGroups") assertGroupNeedUpdate(t, groups, "node-1", "node-1a", "node-1b") @@ -759,7 +759,7 @@ func TestAddAnnotatedNodesToNeedsUpdateCloudonly(t *testing.T) { c.CloudOnly = true c.ClusterValidator = &assertNotCalledClusterValidator{T: t} - err := c.AdjustNeedUpdate(groups, &kopsapi.InstanceGroupList{}) + err := c.AdjustNeedUpdate(groups) assert.NoError(t, err, "AddAnnotatedNodesToGroups") assertGroupNeedUpdate(t, groups, "node-1", "node-1a", "node-1b") @@ -776,7 +776,7 @@ func TestAddAnnotatedNodesToNeedsUpdateNodesMissing(t *testing.T) { groups["node-1"].Ready[0].Node = nil groups["node-1"].NeedUpdate[0].Node = nil - err := c.AdjustNeedUpdate(groups, &kopsapi.InstanceGroupList{}) + err := c.AdjustNeedUpdate(groups) assert.NoError(t, err, "AddAnnotatedNodesToGroups") } diff --git a/upup/models/bindata.go b/upup/models/bindata.go index 54b6a79832..3e698ceffd 100644 --- a/upup/models/bindata.go +++ b/upup/models/bindata.go @@ -3889,9 +3889,7 @@ spec: secretName: cilium-ipsec-keys {{ end }} updateStrategy: - rollingUpdate: - maxUnavailable: 2 - type: RollingUpdate + type: OnDelete --- apiVersion: apps/v1 kind: Deployment diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.8.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.8.yaml.template index 03da0b1a99..24276fb492 100644 --- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.8.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.8.yaml.template @@ -685,9 +685,7 @@ spec: secretName: cilium-ipsec-keys {{ end }} updateStrategy: - rollingUpdate: - maxUnavailable: 2 - type: RollingUpdate + type: OnDelete --- apiVersion: apps/v1 kind: Deployment diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index 3aed99761f..faefc6cf32 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -877,11 +877,12 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann location := key + "/" + id + "-v1.8.yaml" addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ - Name: fi.String(key), - Version: fi.String(version), - Selector: networkingSelector, - Manifest: fi.String(location), - Id: id, + Name: fi.String(key), + Version: fi.String(version), + Selector: networkingSelector, + Manifest: fi.String(location), + Id: id, + NeedsRollingUpdate: "all", }) } } diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml index 84b53f6a47..dd841f1004 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml @@ -70,8 +70,9 @@ spec: version: 1.15.0 - id: k8s-1.12 manifest: networking.cilium.io/k8s-1.12-v1.8.yaml - manifestHash: d3ec6a179bbf7de2fdc6d34190cc31819ece539b + manifestHash: 67853b52a456f1b701ada61ecab28c8c1f615183 name: networking.cilium.io + needsRollingUpdate: all selector: role.kubernetes.io/networking: "1" version: 1.8.0-kops.1