openyurtio
/
kruise
mirror of https://github.com/openkruise/kruise.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor sidecarset control and extended upgrade strategy 

Signed-off-by: liheng.zms <liheng.zms@alibaba-inc.com>
This commit is contained in:
liheng.zms 2021-01-05 17:17:23 +08:00 committed by Siyu Wang
parent 3ebd9e3f68
commit c2bcef7ad2
62 changed files with 7014 additions and 1408 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/ci.yaml vendored
View File

@ -145,4 +145,4 @@ jobs:
- name: Run E2E Tests
run: |
export KUBECONFIG=/home/runner/.kube/config
go test ./test/e2e/ -timeout 30m -v
go test ./test/e2e/ -timeout 90m -v

41
apis/apps/v1alpha1/cloneset_types.go
View File

@ -17,8 +17,6 @@ limitations under the License.
package v1alpha1
import (
"fmt"
appspub "github.com/openkruise/kruise/apis/apps/pub"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -116,7 +114,7 @@ type CloneSetUpdateStrategy struct {
// This will avoid pods with the same key-value to be updated in one batch.
// - Note that pods will be scattered after priority sort. So, although priority strategy and scatter strategy can be applied together, we suggest to use either one of them.
// - If scatterStrategy is used, we suggest to just use one term. Otherwise, the update order can be hard to understand.
ScatterStrategy CloneSetUpdateScatterStrategy `json:"scatterStrategy,omitempty"`
ScatterStrategy UpdateScatterStrategy `json:"scatterStrategy,omitempty"`
// InPlaceUpdateStrategy contains strategies for in-place update.
InPlaceUpdateStrategy *appspub.InPlaceUpdateStrategy `json:"inPlaceUpdateStrategy,omitempty"`
}
@ -138,43 +136,6 @@ const (
InPlaceOnlyCloneSetUpdateStrategyType CloneSetUpdateStrategyType = "InPlaceOnly"
)
// CloneSetUpdateScatterStrategy defines a map for label key-value. Pods matches the key-value will be scattered when update.
//
// Example1: [{"Key": "labelA", "Value": "AAA"}]
// It means all pods with label labelA=AAA will be scattered when update.
//
// Example2: [{"Key": "labelA", "Value": "AAA"}, {"Key": "labelB", "Value": "BBB"}]
// Controller will calculate the two sums of pods with labelA=AAA and with labelB=BBB,
// pods with the label that has bigger amount will be scattered first, then pods with the other label will be scattered.
type CloneSetUpdateScatterStrategy []CloneSetUpdateScatterTerm
type CloneSetUpdateScatterTerm struct {
Key string `json:"key"`
Value string `json:"value"`
}
// FieldsValidation checks invalid fields in CloneSetUpdateScatterStrategy.
func (strategy CloneSetUpdateScatterStrategy) FieldsValidation() error {
if len(strategy) == 0 {
return nil
}
m := make(map[string]struct{}, len(strategy))
for _, term := range strategy {
if term.Key == "" {
return fmt.Errorf("key should not be empty")
}
id := term.Key + ":" + term.Value
if _, ok := m[id]; !ok {
m[id] = struct{}{}
} else {
return fmt.Errorf("duplicated key=%v value=%v", term.Key, term.Value)
}
}
return nil
}
// CloneSetStatus defines the observed state of CloneSet
type CloneSetStatus struct {
// ObservedGeneration is the most recent generation observed for this CloneSet. It corresponds to the

31
apis/apps/v1alpha1/defaults.go
View File

@ -34,18 +34,35 @@ func SetDefaultsSidecarSet(obj *SidecarSet) {
}
for i := range obj.Spec.Containers {
setSidecarDefaultContainer(&obj.Spec.Containers[i])
setDefaultSidecarContainer(&obj.Spec.Containers[i])
}
}
func setSidecarSetUpdateStratety(strategy *SidecarSetUpdateStrategy) {
if strategy.RollingUpdate == nil {
rollingUpdate := RollingUpdateSidecarSet{}
strategy.RollingUpdate = &rollingUpdate
func setDefaultSidecarContainer(sidecarContainer *SidecarContainer) {
if sidecarContainer.PodInjectPolicy == "" {
sidecarContainer.PodInjectPolicy = BeforeAppContainerType
}
if strategy.RollingUpdate.MaxUnavailable == nil {
if sidecarContainer.UpgradeStrategy.UpgradeType == "" {
sidecarContainer.UpgradeStrategy.UpgradeType = SidecarContainerColdUpgrade
}
if sidecarContainer.ShareVolumePolicy.Type == "" {
sidecarContainer.ShareVolumePolicy.Type = ShareVolumePolicyDisabled
}
setSidecarDefaultContainer(sidecarContainer)
}
func setSidecarSetUpdateStratety(strategy *SidecarSetUpdateStrategy) {
if strategy.Type == "" {
strategy.Type = NotUpdateSidecarSetStrategyType
}
if strategy.MaxUnavailable == nil {
maxUnavailable := intstr.FromInt(1)
strategy.RollingUpdate.MaxUnavailable = &maxUnavailable
strategy.MaxUnavailable = &maxUnavailable
}
if strategy.Partition == nil {
partition := intstr.FromInt(0)
strategy.Partition = &partition
}
}

56
apis/apps/v1alpha1/scatter_strategy.go Normal file
View File

@ -0,0 +1,56 @@
/*
Copyright 2020 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1alpha1
import "fmt"
// UpdateScatterStrategy defines a map for label key-value. Pods matches the key-value will be scattered when update.
//
// Example1: [{"Key": "labelA", "Value": "AAA"}]
// It means all pods with label labelA=AAA will be scattered when update.
//
// Example2: [{"Key": "labelA", "Value": "AAA"}, {"Key": "labelB", "Value": "BBB"}]
// Controller will calculate the two sums of pods with labelA=AAA and with labelB=BBB,
// pods with the label that has bigger amount will be scattered first, then pods with the other label will be scattered.
type UpdateScatterStrategy []UpdateScatterTerm
type UpdateScatterTerm struct {
Key string `json:"key"`
Value string `json:"value"`
}
// FieldsValidation checks invalid fields in UpdateScatterStrategy.
func (strategy UpdateScatterStrategy) FieldsValidation() error {
if len(strategy) == 0 {
return nil
}
m := make(map[string]struct{}, len(strategy))
for _, term := range strategy {
if term.Key == "" {
return fmt.Errorf("key should not be empty")
}
id := term.Key + ":" + term.Value
if _, ok := m[id]; !ok {
m[id] = struct{}{}
} else {
return fmt.Errorf("duplicated key=%v value=%v", term.Key, term.Value)
}
}
return nil
}

103
apis/apps/v1alpha1/sidecarset_types.go
View File

@ -27,6 +27,10 @@ type SidecarSetSpec struct {
// selector is a label query over pods that should be injected
Selector *metav1.LabelSelector `json:"selector,omitempty"`
// Namespace sidecarSet will only match the pods in the namespace
// otherwise, match pods in all namespaces(in cluster)
Namespace string `json:"namespace,omitempty"`
// Containers is the list of init containers to be injected into the selected pod
// We will inject those containers by their name in ascending order
// We only inject init containers when a new pod is created, it does not apply to any existing pod
@ -38,9 +42,6 @@ type SidecarSetSpec struct {
// List of volumes that can be mounted by sidecar containers
Volumes []corev1.Volume `json:"volumes,omitempty"`
// Paused indicates that the sidecarset is paused and will not be processed by the sidecarset controller.
Paused bool `json:"paused,omitempty"`
// The sidecarset strategy to use to replace existing pods with new ones.
Strategy SidecarSetUpdateStrategy `json:"strategy,omitempty"`
}
@ -48,19 +49,102 @@ type SidecarSetSpec struct {
// SidecarContainer defines the container of Sidecar
type SidecarContainer struct {
corev1.Container `json:",inline"`
// The rules that injected SidecarContainer into Pod.spec.containers,
// not takes effect in initContainers
// If BeforeAppContainer, the SidecarContainer will be injected in front of the pod.spec.containers
// otherwise it will be injected into the back.
// default BeforeAppContainerType
PodInjectPolicy PodInjectPolicyType `json:"podInjectPolicy,omitempty"`
// sidecarContainer upgrade strategy, include: ColdUpgrade, HotUpgrade
UpgradeStrategy SidecarContainerUpgradeStrategy `json:"upgradeStrategy,omitempty"`
// If ShareVolumePolicy is enabled, the sidecar container will share the other container's VolumeMounts
// in the pod(don't contains the injected sidecar container).
ShareVolumePolicy ShareVolumePolicy `json:"shareVolumePolicy,omitempty"`
// TransferEnv will transfer env info from other container
// SourceContainerName is pod.spec.container[x].name; EnvName is pod.spec.container[x].Env.name
TransferEnv []TransferEnvVar `json:"transferEnv,omitempty"`
}
type ShareVolumePolicy struct {
Type ShareVolumePolicyType `json:"type,omitempty"`
}
type PodInjectPolicyType string
const (
BeforeAppContainerType PodInjectPolicyType = "BeforeAppContainer"
AfterAppContainerType PodInjectPolicyType = "AfterAppContainer"
)
type ShareVolumePolicyType string
const (
ShareVolumePolicyEnabled ShareVolumePolicyType = "enabled"
ShareVolumePolicyDisabled ShareVolumePolicyType = "disabled"
)
type TransferEnvVar struct {
SourceContainerName string `json:"sourceContainerName,omitempty"`
EnvName string `json:"envName,omitempty"`
}
type SidecarContainerUpgradeType string
const (
SidecarContainerColdUpgrade SidecarContainerUpgradeType = "ColdUpgrade"
)
type SidecarContainerUpgradeStrategy struct {
UpgradeType SidecarContainerUpgradeType `json:"upgradeType,omitempty"`
}
// SidecarSetUpdateStrategy indicates the strategy that the SidecarSet
// controller will use to perform updates. It includes any additional parameters
// necessary to perform the update for the indicated strategy.
type SidecarSetUpdateStrategy struct {
RollingUpdate *RollingUpdateSidecarSet `json:"rollingUpdate,omitempty"`
// Type is NotUpdate, the SidecarSet don't update the injected pods,
// it will only inject sidecar container into the newly created pods.
// Type is RollingUpdate, the SidecarSet will update the injected pods to the latest version on RollingUpdate Strategy
Type SidecarSetUpdateStrategyType `json:"type,omitempty"`
// Paused indicates that the SidecarSet is paused to update the injected pods,
// but it don't affect the webhook inject sidecar container into the newly created pods.
// default is false
Paused bool `json:"paused,omitempty"`
// If selector is not nil, this upgrade will only update the selected pods.
Selector *metav1.LabelSelector `json:"selector,omitempty"`
// Partition is the desired number of pods in old revisions. It means when partition
// is set during pods updating, (replicas - partition) number of pods will be updated.
// Default value is 0.
Partition *intstr.IntOrString `json:"partition,omitempty"`
// The maximum number of SidecarSet pods that can be unavailable during the
// update. Value can be an absolute number (ex: 5) or a percentage of total
// number of SidecarSet pods at the start of the update (ex: 10%). Absolute
// number is calculated from percentage by rounding up.
// This cannot be 0.
// Default value is 1.
MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
// ScatterStrategy defines the scatter rules to make pods been scattered when update.
// This will avoid pods with the same key-value to be updated in one batch.
// - Note that pods will be scattered after priority sort. So, although priority strategy and scatter strategy can be applied together, we suggest to use either one of them.
// - If scatterStrategy is used, we suggest to just use one term. Otherwise, the update order can be hard to understand.
ScatterStrategy UpdateScatterStrategy `json:"scatterStrategy,omitempty"`
}
// RollingUpdateSidecarSet is used to communicate parameter
type RollingUpdateSidecarSet struct {
MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
}
type SidecarSetUpdateStrategyType string
const (
NotUpdateSidecarSetStrategyType SidecarSetUpdateStrategyType = "NotUpdate"
RollingUpdateSidecarSetStrategyType SidecarSetUpdateStrategyType = "RollingUpdate"
)
// SidecarSetStatus defines the observed state of SidecarSet
type SidecarSetStatus struct {
@ -76,6 +160,9 @@ type SidecarSetStatus struct {
// readyPods is the number of matched Pods that have a ready condition
ReadyPods int32 `json:"readyPods"`
// updatedReadyPods is the number of matched pods that updated and ready
UpdatedReadyPods int32 `json:"updatedReadyPods,omitempty"`
}
// +genclient

163
apis/apps/v1alpha1/zz_generated.deepcopy.go
View File

@ -479,40 +479,6 @@ func (in *CloneSetTemplateSpec) DeepCopy() *CloneSetTemplateSpec {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in CloneSetUpdateScatterStrategy) DeepCopyInto(out *CloneSetUpdateScatterStrategy) {
{
in := &in
*out = make(CloneSetUpdateScatterStrategy, len(*in))
copy(*out, *in)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloneSetUpdateScatterStrategy.
func (in CloneSetUpdateScatterStrategy) DeepCopy() CloneSetUpdateScatterStrategy {
if in == nil {
return nil
}
out := new(CloneSetUpdateScatterStrategy)
in.DeepCopyInto(out)
return *out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *CloneSetUpdateScatterTerm) DeepCopyInto(out *CloneSetUpdateScatterTerm) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloneSetUpdateScatterTerm.
func (in *CloneSetUpdateScatterTerm) DeepCopy() *CloneSetUpdateScatterTerm {
if in == nil {
return nil
}
out := new(CloneSetUpdateScatterTerm)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *CloneSetUpdateStrategy) DeepCopyInto(out *CloneSetUpdateStrategy) {
*out = *in
@ -538,7 +504,7 @@ func (in *CloneSetUpdateStrategy) DeepCopyInto(out *CloneSetUpdateStrategy) {
}
if in.ScatterStrategy != nil {
in, out := &in.ScatterStrategy, &out.ScatterStrategy
*out = make(CloneSetUpdateScatterStrategy, len(*in))
*out = make(UpdateScatterStrategy, len(*in))
copy(*out, *in)
}
if in.InPlaceUpdateStrategy != nil {
@ -1301,26 +1267,6 @@ func (in *RollingUpdateDaemonSet) DeepCopy() *RollingUpdateDaemonSet {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *RollingUpdateSidecarSet) DeepCopyInto(out *RollingUpdateSidecarSet) {
*out = *in
if in.MaxUnavailable != nil {
in, out := &in.MaxUnavailable, &out.MaxUnavailable
*out = new(intstr.IntOrString)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RollingUpdateSidecarSet.
func (in *RollingUpdateSidecarSet) DeepCopy() *RollingUpdateSidecarSet {
if in == nil {
return nil
}
out := new(RollingUpdateSidecarSet)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *RollingUpdateStatefulSetStrategy) DeepCopyInto(out *RollingUpdateStatefulSetStrategy) {
*out = *in
@ -1361,10 +1307,32 @@ func (in *RollingUpdateStatefulSetStrategy) DeepCopy() *RollingUpdateStatefulSet
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ShareVolumePolicy) DeepCopyInto(out *ShareVolumePolicy) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShareVolumePolicy.
func (in *ShareVolumePolicy) DeepCopy() *ShareVolumePolicy {
if in == nil {
return nil
}
out := new(ShareVolumePolicy)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SidecarContainer) DeepCopyInto(out *SidecarContainer) {
*out = *in
in.Container.DeepCopyInto(&out.Container)
out.UpgradeStrategy = in.UpgradeStrategy
out.ShareVolumePolicy = in.ShareVolumePolicy
if in.TransferEnv != nil {
in, out := &in.TransferEnv, &out.TransferEnv
*out = make([]TransferEnvVar, len(*in))
copy(*out, *in)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SidecarContainer.
@ -1377,6 +1345,21 @@ func (in *SidecarContainer) DeepCopy() *SidecarContainer {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SidecarContainerUpgradeStrategy) DeepCopyInto(out *SidecarContainerUpgradeStrategy) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SidecarContainerUpgradeStrategy.
func (in *SidecarContainerUpgradeStrategy) DeepCopy() *SidecarContainerUpgradeStrategy {
if in == nil {
return nil
}
out := new(SidecarContainerUpgradeStrategy)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SidecarSet) DeepCopyInto(out *SidecarSet) {
*out = *in
@ -1496,11 +1479,26 @@ func (in *SidecarSetStatus) DeepCopy() *SidecarSetStatus {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SidecarSetUpdateStrategy) DeepCopyInto(out *SidecarSetUpdateStrategy) {
*out = *in
if in.RollingUpdate != nil {
in, out := &in.RollingUpdate, &out.RollingUpdate
*out = new(RollingUpdateSidecarSet)
if in.Selector != nil {
in, out := &in.Selector, &out.Selector
*out = new(metav1.LabelSelector)
(*in).DeepCopyInto(*out)
}
if in.Partition != nil {
in, out := &in.Partition, &out.Partition
*out = new(intstr.IntOrString)
**out = **in
}
if in.MaxUnavailable != nil {
in, out := &in.MaxUnavailable, &out.MaxUnavailable
*out = new(intstr.IntOrString)
**out = **in
}
if in.ScatterStrategy != nil {
in, out := &in.ScatterStrategy, &out.ScatterStrategy
*out = make(UpdateScatterStrategy, len(*in))
copy(*out, *in)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SidecarSetUpdateStrategy.
@ -1776,6 +1774,21 @@ func (in *Topology) DeepCopy() *Topology {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TransferEnvVar) DeepCopyInto(out *TransferEnvVar) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TransferEnvVar.
func (in *TransferEnvVar) DeepCopy() *TransferEnvVar {
if in == nil {
return nil
}
out := new(TransferEnvVar)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *UnitedDeployment) DeepCopyInto(out *UnitedDeployment) {
*out = *in
@ -1963,6 +1976,40 @@ func (in *UnorderedUpdateStrategy) DeepCopy() *UnorderedUpdateStrategy {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in UpdateScatterStrategy) DeepCopyInto(out *UpdateScatterStrategy) {
{
in := &in
*out = make(UpdateScatterStrategy, len(*in))
copy(*out, *in)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpdateScatterStrategy.
func (in UpdateScatterStrategy) DeepCopy() UpdateScatterStrategy {
if in == nil {
return nil
}
out := new(UpdateScatterStrategy)
in.DeepCopyInto(out)
return *out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *UpdateScatterTerm) DeepCopyInto(out *UpdateScatterTerm) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpdateScatterTerm.
func (in *UpdateScatterTerm) DeepCopy() *UpdateScatterTerm {
if in == nil {
return nil
}
out := new(UpdateScatterTerm)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *UpdateStatus) DeepCopyInto(out *UpdateStatus) {
*out = *in

183
config/crd/bases/apps.kruise.io_sidecarsets.yaml
View File

@ -61,6 +61,40 @@ spec:
into the selected pod
items:
description: SidecarContainer defines the container of Sidecar
properties:
podInjectPolicy:
description: The rules that injected SidecarContainer into Pod.spec.containers,
not takes effect in initContainers If BeforeAppContainer, the
SidecarContainer will be injected in front of the pod.spec.containers
otherwise it will be injected into the back. default BeforeAppContainerType
type: string
shareVolumePolicy:
description: If ShareVolumePolicy is enabled, the sidecar container
will share the other container's VolumeMounts in the pod(don't
contains the injected sidecar container).
properties:
type:
type: string
type: object
transferEnv:
description: TransferEnv will transfer env info from other container
SourceContainerName is pod.spec.container[x].name; EnvName is
pod.spec.container[x].Env.name
items:
properties:
envName:
type: string
sourceContainerName:
type: string
type: object
type: array
upgradeStrategy:
description: 'sidecarContainer upgrade strategy, include: ColdUpgrade,
HotUpgrade'
properties:
upgradeType:
type: string
type: object
type: object
x-kubernetes-preserve-unknown-fields: true
type: array
@ -71,13 +105,47 @@ spec:
created, it does not apply to any existing pod
items:
description: SidecarContainer defines the container of Sidecar
properties:
podInjectPolicy:
description: The rules that injected SidecarContainer into Pod.spec.containers,
not takes effect in initContainers If BeforeAppContainer, the
SidecarContainer will be injected in front of the pod.spec.containers
otherwise it will be injected into the back. default BeforeAppContainerType
type: string
shareVolumePolicy:
description: If ShareVolumePolicy is enabled, the sidecar container
will share the other container's VolumeMounts in the pod(don't
contains the injected sidecar container).
properties:
type:
type: string
type: object
transferEnv:
description: TransferEnv will transfer env info from other container
SourceContainerName is pod.spec.container[x].name; EnvName is
pod.spec.container[x].Env.name
items:
properties:
envName:
type: string
sourceContainerName:
type: string
type: object
type: array
upgradeStrategy:
description: 'sidecarContainer upgrade strategy, include: ColdUpgrade,
HotUpgrade'
properties:
upgradeType:
type: string
type: object
type: object
x-kubernetes-preserve-unknown-fields: true
type: array
paused:
description: Paused indicates that the sidecarset is paused and will
not be processed by the sidecarset controller.
type: boolean
namespace:
description: Namespace sidecarSet will only match the pods in the namespace
otherwise, match pods in all namespaces(in cluster)
type: string
selector:
description: selector is a label query over pods that should be injected
properties:
@ -125,15 +193,103 @@ spec:
description: The sidecarset strategy to use to replace existing pods
with new ones.
properties:
rollingUpdate:
description: RollingUpdateSidecarSet is used to communicate parameter
maxUnavailable:
anyOf:
- type: integer
- type: string
description: 'The maximum number of SidecarSet pods that can be
unavailable during the update. Value can be an absolute number
(ex: 5) or a percentage of total number of SidecarSet pods at
the start of the update (ex: 10%). Absolute number is calculated
from percentage by rounding up. This cannot be 0. Default value
is 1.'
x-kubernetes-int-or-string: true
partition:
anyOf:
- type: integer
- type: string
description: Partition is the desired number of pods in old revisions.
It means when partition is set during pods updating, (replicas
- partition) number of pods will be updated. Default value is
0.
x-kubernetes-int-or-string: true
paused:
description: Paused indicates that the SidecarSet is paused to update
the injected pods, but it don't affect the webhook inject sidecar
container into the newly created pods. default is false
type: boolean
scatterStrategy:
description: ScatterStrategy defines the scatter rules to make pods
been scattered when update. This will avoid pods with the same
key-value to be updated in one batch. - Note that pods will be
scattered after priority sort. So, although priority strategy
and scatter strategy can be applied together, we suggest to use
either one of them. - If scatterStrategy is used, we suggest to
just use one term. Otherwise, the update order can be hard to
understand.
items:
properties:
key:
type: string
value:
type: string
required:
- key
- value
type: object
type: array
selector:
description: If selector is not nil, this upgrade will only update
the selected pods.
properties:
maxUnavailable:
anyOf:
- type: integer
- type: string
x-kubernetes-int-or-string: true
matchExpressions:
description: matchExpressions is a list of label selector requirements.
The requirements are ANDed.
items:
description: A label selector requirement is a selector that
contains values, a key, and an operator that relates the
key and values.
properties:
key:
description: key is the label key that the selector applies
to.
type: string
operator:
description: operator represents a key's relationship
to a set of values. Valid operators are In, NotIn, Exists
and DoesNotExist.
type: string
values:
description: values is an array of string values. If the
operator is In or NotIn, the values array must be non-empty.
If the operator is Exists or DoesNotExist, the values
array must be empty. This array is replaced during a
strategic merge patch.
items:
type: string
type: array
required:
- key
- operator
type: object
type: array
matchLabels:
additionalProperties:
type: string
description: matchLabels is a map of {key,value} pairs. A single
{key,value} in the matchLabels map is equivalent to an element
of matchExpressions, whose key field is "key", the operator
is "In", and the values array contains only "value". The requirements
are ANDed.
type: object
type: object
type:
description: Type is NotUpdate, the SidecarSet don't update the
injected pods, it will only inject sidecar container into the
newly created pods. Type is RollingUpdate, the SidecarSet will
update the injected pods to the latest version on RollingUpdate
Strategy
type: string
type: object
volumes:
description: List of volumes that can be mounted by sidecar containers
@ -168,6 +324,11 @@ spec:
with the latest SidecarSet's containers
format: int32
type: integer
updatedReadyPods:
description: updatedReadyPods is the number of matched pods that updated
and ready
format: int32
type: integer
required:
- matchedPods
- readyPods

2
config/crd/kustomization.yaml
View File

@ -18,7 +18,7 @@ patchesStrategicMerge:
# patches here are for enabling the conversion webhook for each CRD
#- patches/webhook_in_clonesets.yaml
#- patches/webhook_in_broadcastjobs.yaml
#- patches/webhook_in_sidecarsets.yaml
- patches/webhook_in_sidecarsets.yaml
- patches/webhook_in_statefulsets.yaml
#- patches/webhook_in_uniteddeployments.yaml
#- patches/webhook_in_daemonsets.yaml

11
config/crd/patches/webhook_in_sidecarsets.yaml
View File

@ -5,13 +5,4 @@ kind: CustomResourceDefinition
metadata:
name: sidecarsets.apps.kruise.io
spec:
conversion:
strategy: Webhook
webhookClientConfig:
# this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank,
# but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager)
caBundle: Cg==
service:
namespace: system
name: webhook-service
path: /convert
preserveUnknownFields: false

70
config/webhook/manifests.yaml
View File

@ -6,41 +6,6 @@ metadata:
creationTimestamp: null
name: mutating-webhook-configuration
webhooks:
- clientConfig:
caBundle: Cg==
service:
name: webhook-service
namespace: system
path: /mutate-apps-kruise-io-v1alpha1-sidecarset
failurePolicy: Fail
name: msidecarset.kb.io
rules:
- apiGroups:
- apps.kruise.io
apiVersions:
- v1alpha1
operations:
- CREATE
- UPDATE
resources:
- sidecarsets
- clientConfig:
caBundle: Cg==
service:
name: webhook-service
namespace: system
path: /mutate-pod
failurePolicy: Fail
name: mpod.kb.io
rules:
- apiGroups:
- ""
apiVersions:
- v1
operations:
- CREATE
resources:
- pods
- clientConfig:
caBundle: Cg==
service:
@ -113,6 +78,41 @@ webhooks:
- UPDATE
resources:
- daemonsets
- clientConfig:
caBundle: Cg==
service:
name: webhook-service
namespace: system
path: /mutate-pod
failurePolicy: Fail
name: mpod.kb.io
rules:
- apiGroups:
- ""
apiVersions:
- v1
operations:
- CREATE
resources:
- pods
- clientConfig:
caBundle: Cg==
service:
name: webhook-service
namespace: system
path: /mutate-apps-kruise-io-v1alpha1-sidecarset
failurePolicy: Fail
name: msidecarset.kb.io
rules:
- apiGroups:
- apps.kruise.io
apiVersions:
- v1alpha1
operations:
- CREATE
- UPDATE
resources:
- sidecarsets
- clientConfig:
caBundle: Cg==
service:

2
go.mod
View File

@ -5,6 +5,7 @@ go 1.13
require (
github.com/appscode/jsonpatch v1.0.1
github.com/codegangsta/negroni v1.0.0
github.com/docker/distribution v2.7.1+incompatible
github.com/gorilla/mux v1.7.3
github.com/onsi/ginkgo v1.12.1
github.com/onsi/gomega v1.10.1
@ -50,6 +51,7 @@ replace (
k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.16.6
k8s.io/metrics => k8s.io/metrics v0.16.6
k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.16.6
github.com/docker/distribution => github.com/docker/distribution v2.7.2-0.20200708230840-70e0022e42fd+incompatible
)
replace github.com/prometheus/client_golang => github.com/prometheus/client_golang v0.9.2

2
go.sum
View File

@ -99,6 +99,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/distribution v2.7.2-0.20200708230840-70e0022e42fd+incompatible h1:CapxIe8ZVnS8n7c/rSdMeCfccBch6N08ecuyn/FqiVY=
github.com/docker/distribution v2.7.2-0.20200708230840-70e0022e42fd+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=

53
pkg/control/sidecarcontrol/api.go Normal file
View File

@ -0,0 +1,53 @@
/*
Copyright 2020 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sidecarcontrol
import (
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/sets"
)
type SidecarControl interface {
//common
// get sidecarset
GetSidecarset() *appsv1alpha1.SidecarSet
// inject
// whether need inject the volumeMount into container
IsNeedInjectVolumeMount(volumeMount v1.VolumeMount) bool
// when update pod, judge whether inject sidecar container into pod
NeedInjectOnUpdatedPod(pod, oldPod *v1.Pod, sidecarContainer *appsv1alpha1.SidecarContainer, injectedEnvs []v1.EnvVar,
injectedMounts []v1.VolumeMount) (needInject bool, existSidecars []*appsv1alpha1.SidecarContainer, existVolumes []v1.Volume)
// update
// pod whether consistent and ready
IsPodConsistentAndReady(pod *v1.Pod) bool
// update pod sidecar container to sidecarSet latest version
UpdateSidecarContainerToLatest(containerInSidecarSet, containerInPod v1.Container) v1.Container
// update pod information in upgrade
UpdatePodAnnotationsInUpgrade(changedContainers []string, pod *v1.Pod)
// IsPodUpdatedConsistent indicates whether pod.spec and pod.status are consistent after updating the sidecar containers
IsPodUpdatedConsistent(pod *v1.Pod, ignoreContainers sets.String) bool
// Is sidecarset can upgrade pods
IsSidecarSetCanUpgrade(pod *v1.Pod) bool
}
func New(cs *appsv1alpha1.SidecarSet) SidecarControl {
return &commonControl{SidecarSet: cs}
}

212
pkg/control/sidecarcontrol/sidecarset_control.go Normal file
View File

@ -0,0 +1,212 @@
/*
Copyright 2020 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sidecarcontrol
import (
"encoding/json"
"fmt"
"github.com/openkruise/kruise/apis/apps/pub"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/util"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog"
)
type commonControl struct {
*appsv1alpha1.SidecarSet
}
func (c *commonControl) GetSidecarset() *appsv1alpha1.SidecarSet {
return c.SidecarSet
}
func (c *commonControl) UpdateSidecarContainerToLatest(containerInSidecarSet, containerInPod v1.Container) v1.Container {
containerInPod.Image = containerInSidecarSet.Image
return containerInPod
}
func (c *commonControl) IsNeedInjectVolumeMount(volumeMount v1.VolumeMount) bool {
return true
}
func (c *commonControl) NeedInjectOnUpdatedPod(pod, oldPod *v1.Pod, sidecarContainer *appsv1alpha1.SidecarContainer,
injectedEnvs []v1.EnvVar, injectedMounts []v1.VolumeMount) (needInject bool, existSidecars []*appsv1alpha1.SidecarContainer, existVolumes []v1.Volume) {
return false, nil, nil
}
func (c *commonControl) IsPodConsistentAndReady(pod *v1.Pod) bool {
// If pod is consistent
if !c.IsPodUpdatedConsistent(pod, nil) {
return false
}
// 1. pod.Status.Phase == v1.PodRunning
// 2. pod.condition PodReady == true
return util.IsRunningAndReady(pod)
}
func (c *commonControl) UpdatePodAnnotationsInUpgrade(changedContainers []string, pod *v1.Pod) {
sidecarSet := c.GetSidecarset()
// 1. sidecar hash
updatePodSidecarSetHash(pod, sidecarSet)
// 3. record the ImageID, before update pod sidecar container
// if it is changed, indicates the update is complete.
//format: sidecarset.name -> appsv1alpha1.InPlaceUpdateState
sidecarUpdateStates := make(map[string]*pub.InPlaceUpdateState)
if stateStr, _ := pod.Annotations[SidecarsetInplaceUpdateStateKey]; len(stateStr) > 0 {
if err := json.Unmarshal([]byte(stateStr), &sidecarUpdateStates); err != nil {
klog.Errorf("parse pod(%s.%s) annotations[%s] value(%s) failed: %s",
pod.Namespace, pod.Name, SidecarsetInplaceUpdateStateKey, stateStr, err.Error())
}
}
inPlaceUpdateState, ok := sidecarUpdateStates[sidecarSet.Name]
if !ok {
inPlaceUpdateState = &pub.InPlaceUpdateState{
Revision: GetSidecarSetRevision(sidecarSet),
UpdateTimestamp: metav1.Now(),
}
}
// format: container.name -> pod.status.containers[container.name].ImageID
if inPlaceUpdateState.LastContainerStatuses == nil {
inPlaceUpdateState.LastContainerStatuses = make(map[string]pub.InPlaceUpdateContainerStatus)
}
cStatus := make(map[string]string, len(pod.Status.ContainerStatuses))
for i := range pod.Status.ContainerStatuses {
c := &pod.Status.ContainerStatuses[i]
cStatus[c.Name] = c.ImageID
}
for _, cName := range changedContainers {
updateStatus := pub.InPlaceUpdateContainerStatus{
ImageID: cStatus[cName],
}
//record status.ImageId before update pods in store
inPlaceUpdateState.LastContainerStatuses[cName] = updateStatus
}
//record sidecar container status information in pod's annotations
sidecarUpdateStates[sidecarSet.Name] = inPlaceUpdateState
by, _ := json.Marshal(sidecarUpdateStates)
pod.Annotations[SidecarsetInplaceUpdateStateKey] = string(by)
return
}
// only check sidecar container is consistent
func (c *commonControl) IsPodUpdatedConsistent(pod *v1.Pod, ignoreContainers sets.String) bool {
sidecarset := c.GetSidecarset()
sidecarContainers := GetSidecarContainersInPod(sidecarset)
allDigestImage := true
for _, container := range pod.Spec.Containers {
// only check whether sidecar container is consistent
if !sidecarContainers.Has(container.Name) {
continue
}
if ignoreContainers.Has(container.Name) {
continue
}
//whether image is digest format,
//for example: docker.io/busybox@sha256:a9286defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d
if !util.IsImageDigest(container.Image) {
allDigestImage = false
continue
}
if !util.IsPodContainerDigestEqual(sets.NewString(container.Name), pod) {
return false
}
}
// If all spec.container[x].image is digest format, only check digest imageId
if allDigestImage {
return true
}
// check container InpalceUpdate status
if err := isSidecarContainerUpdateCompleted(pod, sidecarset.Name, sidecarContainers); err != nil {
klog.V(5).Infof(err.Error())
return false
}
return true
}
// k8s only allow modify pod.spec.container[x].image,
// only when annotations[SidecarSetHashWithoutImageAnnotation] is the same, sidecarSet can upgrade pods
func (c *commonControl) IsSidecarSetCanUpgrade(pod *v1.Pod) bool {
sidecarSet := c.GetSidecarset()
return GetPodSidecarSetWithoutImageRevision(sidecarSet.Name, pod) == GetSidecarSetWithoutImageRevision(sidecarSet)
}
// isContainerInplaceUpdateCompleted checks whether imageID in container status has been changed since in-place update.
// If the imageID in containerStatuses has not been changed, we assume that kubelet has not updated containers in Pod.
func isSidecarContainerUpdateCompleted(pod *v1.Pod, sidecarSetName string, containers sets.String) error {
if len(pod.Spec.Containers) != len(pod.Status.ContainerStatuses) {
return fmt.Errorf("pod(%s.%s) is inconsistent in ContainerStatuses", pod.Namespace, pod.Name)
}
//format: sidecarset.name -> appsv1alpha1.InPlaceUpdateState
sidecarUpdateStates := make(map[string]*pub.InPlaceUpdateState)
// when the pod annotation not found, indicates the pod only injected sidecar container, and never inplace update
// then always think it update complete
if stateStr, ok := pod.Annotations[SidecarsetInplaceUpdateStateKey]; !ok {
klog.V(5).Infof("pod(%s.%s) annotations[%s] not found", pod.Namespace, pod.Name, SidecarsetInplaceUpdateStateKey)
return nil
// this won't happen in practice, unless someone manually edit pod annotations
} else if err := json.Unmarshal([]byte(stateStr), &sidecarUpdateStates); err != nil {
return fmt.Errorf("parse pod(%s.%s) annotations[%s] value(%s) failed: %s",
pod.Namespace, pod.Name, SidecarsetInplaceUpdateStateKey, stateStr, err.Error())
}
// when the sidecarset InPlaceUpdateState not found, indicates the pod only injected sidecar container, and never inplace update
// then always think it update complete
inPlaceUpdateState, ok := sidecarUpdateStates[sidecarSetName]
if !ok {
klog.V(5).Infof("parse pod(%s.%s) annotations[%s] sidecarset(%s) Not Found",
pod.Namespace, pod.Name, SidecarsetInplaceUpdateStateKey, sidecarSetName)
return nil
}
containerImages := make(map[string]string, len(pod.Spec.Containers))
for i := range pod.Spec.Containers {
c := &pod.Spec.Containers[i]
containerImages[c.Name] = c.Image
}
for _, cs := range pod.Status.ContainerStatuses {
// only check containers set
if !containers.Has(cs.Name) {
continue
}
if oldStatus, ok := inPlaceUpdateState.LastContainerStatuses[cs.Name]; ok {
// we assume that users should not update workload template with new image
// which actually has the same imageID as the old image
if oldStatus.ImageID == cs.ImageID && containerImages[cs.Name] != cs.Image {
return fmt.Errorf("container %s imageID not changed", cs.Name)
}
}
// If sidecar container status.ImageID changed, or this oldStatus ImageID don't exist
// indicate the sidecar container update is complete
}
return nil
}

296
pkg/control/sidecarcontrol/util.go Normal file
View File

@ -0,0 +1,296 @@
/*
Copyright 2020 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sidecarcontrol
import (
"encoding/json"
"fmt"
"regexp"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/util"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog"
kubecontroller "k8s.io/kubernetes/pkg/controller"
)
const (
// SidecarSetHashAnnotation represents the key of a sidecarSet hash
SidecarSetHashAnnotation = "kruise.io/sidecarset-hash"
// SidecarSetHashWithoutImageAnnotation represents the key of a sidecarset hash without images of sidecar
SidecarSetHashWithoutImageAnnotation = "kruise.io/sidecarset-hash-without-image"
// SidecarSetListAnnotation represent sidecarset list that injected pods
SidecarSetListAnnotation = "kruise.io/sidecarset-injected-list"
// SidecarEnvKey specifies the environment variable which record a container as injected
SidecarEnvKey = "IS_INJECTED"
// SidecarsetInplaceUpdateStateKey records the state of inplace-update.
// The value of annotation is SidecarsetInplaceUpdateStateKey.
SidecarsetInplaceUpdateStateKey string = "kruise.io/sidecarset-inplace-update-state"
)
var (
// SidecarIgnoredNamespaces specifies the namespaces where Pods won't get injected
SidecarIgnoredNamespaces = []string{"kube-system", "kube-public"}
// SubPathExprEnvReg format: $(ODD_NAME)、$(POD_NAME)...
SubPathExprEnvReg, _ = regexp.Compile(`\$\(([-._a-zA-Z][-._a-zA-Z0-9]*)\)`)
)
type SidecarSetUpgradeSpec struct {
UpdateTimestamp metav1.Time `json:"updateTimestamp"`
SidecarSetHash string `json:"hash"`
SidecarSetName string `json:"sidecarSetName"`
}
// PodMatchSidecarSet determines if pod match Selector of sidecar.
func PodMatchedSidecarSet(pod *corev1.Pod, sidecarSet appsv1alpha1.SidecarSet) (bool, error) {
//If matchedNamespace is not empty, sidecarSet will only match the pods in the namespace
if sidecarSet.Spec.Namespace != "" && sidecarSet.Spec.Namespace != pod.Namespace {
return false, nil
}
// if selector not matched, then continue
selector, err := metav1.LabelSelectorAsSelector(sidecarSet.Spec.Selector)
if err != nil {
return false, err
}
if !selector.Empty() && selector.Matches(labels.Set(pod.Labels)) {
return true, nil
}
return false, nil
}
// IsActivePod determines the pod whether need be injected and updated
func IsActivePod(pod *corev1.Pod) bool {
for _, namespace := range SidecarIgnoredNamespaces {
if pod.Namespace == namespace {
return false
}
}
if pod.ObjectMeta.GetDeletionTimestamp() != nil {
return false
}
return true
}
func GetSidecarSetRevision(sidecarSet *appsv1alpha1.SidecarSet) string {
return sidecarSet.Annotations[SidecarSetHashAnnotation]
}
func GetSidecarSetWithoutImageRevision(sidecarSet *appsv1alpha1.SidecarSet) string {
return sidecarSet.Annotations[SidecarSetHashWithoutImageAnnotation]
}
func GetPodSidecarSetRevision(sidecarSetName string, pod metav1.Object) string {
annotations := pod.GetAnnotations()
hashKey := SidecarSetHashAnnotation
if annotations[hashKey] == "" {
return ""
}
sidecarSetHash := make(map[string]SidecarSetUpgradeSpec)
if err := json.Unmarshal([]byte(annotations[hashKey]), &sidecarSetHash); err != nil {
klog.Warningf("parse pod(%s.%s) annotations[%s] value(%s) failed: %s", pod.GetNamespace(), pod.GetName(), hashKey,
annotations[hashKey], err.Error())
// to be compatible with older sidecarSet hash struct, map[string]string
olderSidecarSetHash := make(map[string]string)
if err = json.Unmarshal([]byte(annotations[hashKey]), &olderSidecarSetHash); err != nil {
return ""
}
for k, v := range olderSidecarSetHash {
sidecarSetHash[k] = SidecarSetUpgradeSpec{
SidecarSetHash: v,
}
}
}
if upgradeSpec, ok := sidecarSetHash[sidecarSetName]; ok {
return upgradeSpec.SidecarSetHash
}
klog.Warningf("parse pod(%s.%s) annotations[%s] sidecarSet(%s) Not Found", pod.GetNamespace(), pod.GetName(), hashKey, sidecarSetName)
return ""
}
func GetPodSidecarSetWithoutImageRevision(sidecarSetName string, pod metav1.Object) string {
annotations := pod.GetAnnotations()
hashKey := SidecarSetHashWithoutImageAnnotation
if annotations[hashKey] == "" {
return ""
}
sidecarSetHash := make(map[string]SidecarSetUpgradeSpec)
if err := json.Unmarshal([]byte(annotations[hashKey]), &sidecarSetHash); err != nil {
klog.Errorf("parse pod(%s.%s) annotations[%s] value(%s) failed: %s", pod.GetNamespace(), pod.GetName(), hashKey,
annotations[hashKey], err.Error())
// to be compatible with older sidecarSet hash struct, map[string]string
olderSidecarSetHash := make(map[string]string)
if err = json.Unmarshal([]byte(annotations[hashKey]), &olderSidecarSetHash); err != nil {
return ""
}
for k, v := range olderSidecarSetHash {
sidecarSetHash[k] = SidecarSetUpgradeSpec{
SidecarSetHash: v,
}
}
}
if upgradeSpec, ok := sidecarSetHash[sidecarSetName]; ok {
return upgradeSpec.SidecarSetHash
}
klog.Warningf("parse pod(%s.%s) annotations[%s] sidecarSet(%s) Not Found", pod.GetNamespace(), pod.GetName(), hashKey, sidecarSetName)
return ""
}
// whether this pod has been updated based on the latest sidecarSet
func IsPodSidecarUpdated(sidecarSet *appsv1alpha1.SidecarSet, pod *corev1.Pod) bool {
return GetSidecarSetRevision(sidecarSet) == GetPodSidecarSetRevision(sidecarSet.Name, pod)
}
func updatePodSidecarSetHash(pod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSet) {
hashKey := SidecarSetHashAnnotation
sidecarSetHash := make(map[string]SidecarSetUpgradeSpec)
if err := json.Unmarshal([]byte(pod.Annotations[hashKey]), &sidecarSetHash); err != nil {
klog.Errorf("unmarshal pod(%s.%s) annotations[%s] failed: %s", pod.Namespace, pod.Name, hashKey, err.Error())
// to be compatible with older sidecarSet hash struct, map[string]string
olderSidecarSetHash := make(map[string]string)
if err = json.Unmarshal([]byte(pod.Annotations[hashKey]), &olderSidecarSetHash); err == nil {
for k, v := range olderSidecarSetHash {
sidecarSetHash[k] = SidecarSetUpgradeSpec{
SidecarSetHash: v,
UpdateTimestamp: metav1.Now(),
SidecarSetName: sidecarSet.Name,
}
}
}
withoutImageHash := make(map[string]SidecarSetUpgradeSpec)
if err = json.Unmarshal([]byte(pod.Annotations[SidecarSetHashWithoutImageAnnotation]), &olderSidecarSetHash); err == nil {
for k, v := range olderSidecarSetHash {
withoutImageHash[k] = SidecarSetUpgradeSpec{
SidecarSetHash: v,
UpdateTimestamp: metav1.Now(),
SidecarSetName: sidecarSet.Name,
}
}
newWithoutImageHash, _ := json.Marshal(withoutImageHash)
pod.Annotations[SidecarSetHashWithoutImageAnnotation] = string(newWithoutImageHash)
} else {
}
// compatible done
}
sidecarSetHash[sidecarSet.Name] = SidecarSetUpgradeSpec{
UpdateTimestamp: metav1.Now(),
SidecarSetHash: GetSidecarSetRevision(sidecarSet),
SidecarSetName: sidecarSet.Name,
}
newHash, _ := json.Marshal(sidecarSetHash)
pod.Annotations[hashKey] = string(newHash)
}
func GetSidecarContainersInPod(sidecarSet *appsv1alpha1.SidecarSet) sets.String {
names := sets.NewString()
for _, sidecarContainer := range sidecarSet.Spec.Containers {
names.Insert(sidecarContainer.Name)
}
return names
}
func GetPodsSortFunc(pods []*corev1.Pod, waitUpdateIndexes []int) func(i, j int) bool {
// not-ready < ready, unscheduled < scheduled, and pending < running
return func(i, j int) bool {
return kubecontroller.ActivePods(pods).Less(waitUpdateIndexes[i], waitUpdateIndexes[j])
}
}
func IsInjectedSidecarContainerInPod(container *corev1.Container) bool {
if util.GetContainerEnvValue(container, SidecarEnvKey) == "true" {
return true
}
return false
}
func IsSharePodVolumeMounts(container *appsv1alpha1.SidecarContainer) bool {
return container.ShareVolumePolicy.Type == appsv1alpha1.ShareVolumePolicyEnabled
}
func GetInjectedVolumeMountsAndEnvs(control SidecarControl, sidecarContainer *appsv1alpha1.SidecarContainer, pod *corev1.Pod) ([]corev1.VolumeMount, []corev1.EnvVar) {
if !IsSharePodVolumeMounts(sidecarContainer) {
return nil, nil
}
// injected volumeMounts
var injectedMounts []corev1.VolumeMount
// injected EnvVar
var injectedEnvs []corev1.EnvVar
for _, appContainer := range pod.Spec.Containers {
// ignore the injected sidecar container
if IsInjectedSidecarContainerInPod(&appContainer) {
continue
}
for _, volumeMount := range appContainer.VolumeMounts {
if !control.IsNeedInjectVolumeMount(volumeMount) {
continue
}
injectedMounts = append(injectedMounts, volumeMount)
//If volumeMounts.SubPathExpr contains expansions, copy environment
//for example: SubPathExpr=foo/$(ODD_NAME)/$(POD_NAME), we need copy environment ODD_NAME、POD_NAME
//envs = [$(ODD_NAME) $(POD_NAME)]
envs := SubPathExprEnvReg.FindAllString(volumeMount.SubPathExpr, -1)
for _, env := range envs {
// $(ODD_NAME) -> ODD_NAME
envName := env[2 : len(env)-1]
// get envVar in container
eVar := util.GetContainerEnvVar(&appContainer, envName)
if eVar == nil {
klog.Warningf("pod(%s.%s) container(%s) get env(%s) is nil", pod.Namespace, pod.Name, appContainer.Name, envName)
continue
}
injectedEnvs = append(injectedEnvs, *eVar)
}
}
}
return injectedMounts, injectedEnvs
}
func GetSidecarTransferEnvs(sidecarContainer *appsv1alpha1.SidecarContainer, pod *corev1.Pod) (injectedEnvs []corev1.EnvVar) {
// pre-process envs in pod, format: container.name/env.name -> container.env
envsInPod := make(map[string]corev1.EnvVar)
for _, container := range pod.Spec.Containers {
for _, env := range container.Env {
key := fmt.Sprintf("%v/%v", container.Name, env.Name)
envsInPod[key] = env
}
}
for _, tEnv := range sidecarContainer.TransferEnv {
key := fmt.Sprintf("%v/%v", tEnv.SourceContainerName, tEnv.EnvName)
env, ok := envsInPod[key]
if !ok {
klog.Warningf("there is no env %v in container %v", tEnv.EnvName, tEnv.SourceContainerName)
continue
}
injectedEnvs = append(injectedEnvs, env)
}
return
}

215
pkg/control/sidecarcontrol/util_test.go Normal file
View File

@ -0,0 +1,215 @@
/*
Copyright 2020 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sidecarcontrol
import (
"encoding/json"
"testing"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
var (
podDemo = &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{},
Name: "test-pod-1",
Namespace: "default",
Labels: map[string]string{"app": "nginx"},
ResourceVersion: "495711227",
},
}
sidecarSetDemo = &appsv1alpha1.SidecarSet{
ObjectMeta: metav1.ObjectMeta{
Generation: 123,
Annotations: map[string]string{
SidecarSetHashAnnotation: "bbb",
SidecarSetHashWithoutImageAnnotation: "without-image-aaa",
},
Name: "test-sidecarset",
Labels: map[string]string{},
},
}
)
func TestGetPodSidecarSetRevision(t *testing.T) {
cases := []struct {
name string
getPod func() *corev1.Pod
//sidecarContainer -> sidecarSet.Revision
exceptRevision string
exceptWithoutImageRevision string
}{
{
name: "normal sidecarSet revision",
getPod: func() *corev1.Pod {
pod := podDemo.DeepCopy()
pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset":{"hash":"aaa"}}`
pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset":{"hash":"without-image-aaa"}}`
return pod
},
exceptRevision: "aaa",
exceptWithoutImageRevision: "without-image-aaa",
},
{
name: "older sidecarSet revision",
getPod: func() *corev1.Pod {
pod := podDemo.DeepCopy()
pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset": "aaa"}`
pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset": "without-image-aaa"}`
return pod
},
exceptRevision: "aaa",
exceptWithoutImageRevision: "without-image-aaa",
},
{
name: "failed sidecarSet revision",
getPod: func() *corev1.Pod {
pod := podDemo.DeepCopy()
pod.Annotations[SidecarSetHashAnnotation] = "failed-sidecarset-hash"
pod.Annotations[SidecarSetHashWithoutImageAnnotation] = "failed-sidecarset-hash"
return pod
},
exceptRevision: "",
exceptWithoutImageRevision: "",
},
}
for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) {
revison := GetPodSidecarSetRevision("test-sidecarset", cs.getPod())
if cs.exceptRevision != revison {
t.Fatalf("except sidecar container test-sidecarset revison %s, but get %s", cs.exceptRevision, revison)
}
withoutRevison := GetPodSidecarSetWithoutImageRevision("test-sidecarset", cs.getPod())
if cs.exceptWithoutImageRevision != withoutRevison {
t.Fatalf("except sidecar container test-sidecarset WithoutImageRevision %s, but get %s", cs.exceptWithoutImageRevision, withoutRevison)
}
})
}
}
func TestUpdatePodSidecarSetHash(t *testing.T) {
cases := []struct {
name string
getPod func() *corev1.Pod
getSidecarSet func() *appsv1alpha1.SidecarSet
exceptRevision map[string]SidecarSetUpgradeSpec
exceptWithoutImageRevision map[string]SidecarSetUpgradeSpec
}{
{
name: "normal sidecarSet revision",
getPod: func() *corev1.Pod {
pod := podDemo.DeepCopy()
pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset":{"hash":"aaa"}}`
pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset":{"hash":"without-image-aaa"}}`
return pod
},
getSidecarSet: func() *appsv1alpha1.SidecarSet {
return sidecarSetDemo.DeepCopy()
},
exceptRevision: map[string]SidecarSetUpgradeSpec{
"test-sidecarset": {
SidecarSetHash: "bbb",
},
},
exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{
"test-sidecarset": {
SidecarSetHash: "without-image-aaa",
},
},
},
{
name: "older sidecarSet revision",
getPod: func() *corev1.Pod {
pod := podDemo.DeepCopy()
pod.Annotations[SidecarSetHashAnnotation] = `{"test-sidecarset": "aaa"}`
pod.Annotations[SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset": "without-image-aaa"}`
return pod
},
getSidecarSet: func() *appsv1alpha1.SidecarSet {
return sidecarSetDemo.DeepCopy()
},
exceptRevision: map[string]SidecarSetUpgradeSpec{
"test-sidecarset": {
SidecarSetHash: "bbb",
},
},
exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{
"test-sidecarset": {
SidecarSetHash: "without-image-aaa",
},
},
},
{
name: "failed sidecarSet revision",
getPod: func() *corev1.Pod {
pod := podDemo.DeepCopy()
pod.Annotations[SidecarSetHashAnnotation] = "failed-sidecarset-hash"
pod.Annotations[SidecarSetHashWithoutImageAnnotation] = "failed-sidecarset-hash"
return pod
},
getSidecarSet: func() *appsv1alpha1.SidecarSet {
return sidecarSetDemo.DeepCopy()
},
exceptRevision: map[string]SidecarSetUpgradeSpec{
"test-sidecarset": {
SidecarSetHash: "bbb",
},
},
exceptWithoutImageRevision: map[string]SidecarSetUpgradeSpec{},
},
}
for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) {
podInput := cs.getPod()
sidecarSetInput := cs.getSidecarSet()
updatePodSidecarSetHash(podInput, sidecarSetInput)
// sidecarSet hash
sidecarSetHash := make(map[string]SidecarSetUpgradeSpec)
err := json.Unmarshal([]byte(podInput.Annotations[SidecarSetHashAnnotation]), &sidecarSetHash)
if err != nil {
t.Fatalf("parse pod sidecarSet hash failed: %s", err.Error())
}
for k, o := range sidecarSetHash {
eo := cs.exceptRevision[k]
if o.SidecarSetHash != eo.SidecarSetHash {
t.Fatalf("except sidecar container %s revision %s, but get revision %s", k, eo.SidecarSetHash, o.SidecarSetHash)
}
}
if len(cs.exceptWithoutImageRevision) == 0 {
return
}
// without image sidecarSet hash
sidecarSetHash = make(map[string]SidecarSetUpgradeSpec)
err = json.Unmarshal([]byte(podInput.Annotations[SidecarSetHashWithoutImageAnnotation]), &sidecarSetHash)
if err != nil {
t.Fatalf("parse pod sidecarSet hash failed: %s", err.Error())
}
for k, o := range sidecarSetHash {
eo := cs.exceptWithoutImageRevision[k]
if o.SidecarSetHash != eo.SidecarSetHash {
t.Fatalf("except sidecar container %s revision %s, but get revision %s", k, eo.SidecarSetHash, o.SidecarSetHash)
}
}
})
}
}

4
pkg/controller/cloneset/cloneset_status.go
View File

@ -90,10 +90,10 @@ func (r *realStatusUpdater) calculateStatus(cs *appsv1alpha1.CloneSet, newStatus
if coreControl.IsPodUpdateReady(pod, cs.Spec.MinReadySeconds) {
newStatus.AvailableReplicas++
}
if clonesetutils.GetPodRevision(pod) == newStatus.UpdateRevision {
if clonesetutils.GetPodRevision("", pod) == newStatus.UpdateRevision {
newStatus.UpdatedReplicas++
}
if clonesetutils.GetPodRevision(pod) == newStatus.UpdateRevision && coreControl.IsPodUpdateReady(pod, 0) {
if clonesetutils.GetPodRevision("", pod) == newStatus.UpdateRevision && coreControl.IsPodUpdateReady(pod, 0) {
newStatus.UpdatedReadyReplicas++
}
}

4
pkg/controller/cloneset/update/cloneset_update.go
View File

@ -96,7 +96,7 @@ func (c *realControl) Manage(cs *appsv1alpha1.CloneSet,
continue
}
if clonesetutils.GetPodRevision(pods[i]) != updateRevision.Name {
if clonesetutils.GetPodRevision("", pods[i]) != updateRevision.Name {
switch lifecycle.GetPodLifecycleState(pods[i]) {
case appspub.LifecycleStatePreparingDelete, appspub.LifecycleStateUpdated:
klog.V(3).Infof("CloneSet %s/%s find pod %s in state %s, so skip to update it",
@ -259,7 +259,7 @@ func (c *realControl) updatePod(cs *appsv1alpha1.CloneSet, coreControl clonesetc
cs.Spec.UpdateStrategy.Type == appsv1alpha1.InPlaceOnlyCloneSetUpdateStrategyType {
var oldRevision *apps.ControllerRevision
for _, r := range revisions {
if r.Name == clonesetutils.GetPodRevision(pod) {
if r.Name == clonesetutils.GetPodRevision("", pod) {
oldRevision = r
break
}

6
pkg/controller/cloneset/utils/cloneset_utils.go
View File

@ -67,7 +67,7 @@ func GetActivePods(reader client.Reader, opts *client.ListOptions) ([]*v1.Pod, e
}
// GetPodRevision returns revision hash of this pod.
func GetPodRevision(pod metav1.Object) string {
func GetPodRevision(controllerKey string, pod metav1.Object) string {
return pod.GetLabels()[apps.ControllerRevisionHashLabelKey]
}
@ -75,7 +75,7 @@ func GetPodRevision(pod metav1.Object) string {
func GetPodsRevisions(pods []*v1.Pod) sets.String {
revisions := sets.NewString()
for _, p := range pods {
revisions.Insert(GetPodRevision(p))
revisions.Insert(GetPodRevision("", p))
}
return revisions
}
@ -104,7 +104,7 @@ func IsRunningAndAvailable(pod *v1.Pod, minReadySeconds int32) bool {
// SplitPodsByRevision returns Pods matched and unmatched the given revision
func SplitPodsByRevision(pods []*v1.Pod, rev string) (matched, unmatched []*v1.Pod) {
for _, p := range pods {
if GetPodRevision(p) == rev {
if GetPodRevision("", p) == rev {
matched = append(matched, p)
} else {
unmatched = append(unmatched, p)

2
pkg/controller/daemonset/daemonset_util.go
View File

@ -315,7 +315,7 @@ func storeDaemonSetStatus(dsClient kubeClient.Client, ds *appsv1alpha1.DaemonSet
}
// GetPodRevision returns revision hash of this pod.
func GetPodRevision(pod metav1.Object) string {
func GetPodRevision(controllerKey string, pod metav1.Object) string {
return pod.GetLabels()[apps.ControllerRevisionHashLabelKey]
}

2
pkg/controller/daemonset/daemonset_util_test.go
View File

@ -257,7 +257,7 @@ func TestGetPodRevision(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := GetPodRevision(tt.args.pod); got != tt.want {
if got := GetPodRevision("", tt.args.pod); got != tt.want {
t.Errorf("GetPodRevision() = %v, want %v", got, tt.want)
}
})

96
pkg/controller/sidecarset/sidecarset_controller.go
View File

@ -21,14 +21,15 @@ import (
"flag"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/control/sidecarcontrol"
"github.com/openkruise/kruise/pkg/util/expectations"
"github.com/openkruise/kruise/pkg/util/gate"
"github.com/openkruise/kruise/pkg/util/ratelimiter"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/klog"
controllerutil "k8s.io/kubernetes/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/handler"
@ -61,7 +62,13 @@ func Add(mgr manager.Manager) error {
// newReconciler returns a new reconcile.Reconciler
func newReconciler(mgr manager.Manager) reconcile.Reconciler {
return &ReconcileSidecarSet{Client: mgr.GetClient(), scheme: mgr.GetScheme()}
expectations := expectations.NewUpdateExpectations(sidecarcontrol.GetPodSidecarSetRevision)
recorder := mgr.GetEventRecorderFor("sidecarset-controller")
return &ReconcileSidecarSet{
Client: mgr.GetClient(),
scheme: mgr.GetScheme(),
processor: NewSidecarSetProcessor(mgr.GetClient(), expectations, recorder),
}
}
// add adds a new Controller to mgr with r as the reconcile.Reconciler
@ -93,7 +100,9 @@ var _ reconcile.Reconciler = &ReconcileSidecarSet{}
// ReconcileSidecarSet reconciles a SidecarSet object
type ReconcileSidecarSet struct {
client.Client
scheme *runtime.Scheme
scheme *runtime.Scheme
updateExpectations expectations.UpdateExpectations
processor *Processor
}
// +kubebuilder:rbac:groups=apps.kruise.io,resources=sidecarsets,verbs=get;list;watch;create;update;patch;delete
@ -116,82 +125,5 @@ func (r *ReconcileSidecarSet) Reconcile(request reconcile.Request) (reconcile.Re
}
klog.V(3).Infof("begin to process sidecarset %v", sidecarSet.Name)
selector, err := metav1.LabelSelectorAsSelector(sidecarSet.Spec.Selector)
if err != nil {
return reconcile.Result{}, err
}
matchedPods := &corev1.PodList{}
if err := r.List(context.TODO(), matchedPods, &client.ListOptions{LabelSelector: selector}); err != nil {
return reconcile.Result{}, err
}
// ignore inactive pods and pods are created before sidecarset creates
var filteredPods []*corev1.Pod
for i := range matchedPods.Items {
pod := &matchedPods.Items[i]
podCreateBeforeSidecarSet, err := isPodCreatedBeforeSidecarSet(sidecarSet, pod)
if err != nil {
return reconcile.Result{}, err
}
if controllerutil.IsPodActive(pod) && !isIgnoredPod(pod) && !podCreateBeforeSidecarSet {
filteredPods = append(filteredPods, pod)
}
}
status, err := calculateStatus(sidecarSet, filteredPods)
if err != nil {
return reconcile.Result{}, err
}
err = r.updateSidecarSetStatus(sidecarSet, status)
if err != nil {
return reconcile.Result{}, err
}
// update procedure:
// 1. check if sidecarset paused, if so, then quit
// 2. check if fields other than image in sidecarset had changed, if so, then quit
// 3. check unavailable pod number, if > 0, then quit(maxUnavailable=1)
// 4. find out pods need update
// 5. update one pod(maxUnavailable=1)
if sidecarSet.Spec.Paused {
klog.V(3).Infof("sidecarset %v is paused, skip update", sidecarSet.Name)
return reconcile.Result{}, nil
}
if len(filteredPods) == 0 {
return reconcile.Result{}, nil
}
otherFieldsChanged, err := otherFieldsInSidecarChanged(sidecarSet, filteredPods[0])
if err != nil {
return reconcile.Result{}, err
}
if otherFieldsChanged {
klog.V(3).Infof("fields other than image in sidecarset %v had changed, skip update", sidecarSet.Name)
return reconcile.Result{}, nil
}
unavailableNum, err := getUnavailableNumber(sidecarSet, filteredPods)
if err != nil {
return reconcile.Result{}, err
}
maxUnavailableNum := getMaxUnavailable(sidecarSet)
if unavailableNum >= maxUnavailableNum {
klog.V(3).Infof("current unavailable pod number: %v(max: %v), skip update", unavailableNum, maxUnavailableNum)
return reconcile.Result{}, nil
}
var podsNeedUpdate []*corev1.Pod
for _, pod := range filteredPods {
isUpdated, err := isPodSidecarUpdated(sidecarSet, pod)
if err != nil {
return reconcile.Result{}, err
}
if !isUpdated {
podsNeedUpdate = append(podsNeedUpdate, pod)
}
}
updateNum := maxUnavailableNum - unavailableNum
return reconcile.Result{}, r.updateSidecarImageAndHash(sidecarSet, podsNeedUpdate, updateNum)
return r.processor.UpdateSidecarSet(sidecarSet)
}

229
pkg/controller/sidecarset/sidecarset_controller_test.go
View File

@ -4,42 +4,39 @@ import (
"context"
"testing"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/control/sidecarcontrol"
"github.com/openkruise/kruise/pkg/util/expectations"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/webhook/sidecarset/mutating"
)
type HandlePod func(pod []*corev1.Pod)
var (
scheme *runtime.Scheme
partition = intstr.FromInt(0)
maxUnavailable = intstr.FromInt(1)
sidecarSetDemo = &appsv1alpha1.SidecarSet{
ObjectMeta: metav1.ObjectMeta{
Generation: 123,
Annotations: map[string]string{
mutating.SidecarSetHashAnnotation: "ccc",
mutating.SidecarSetHashWithoutImageAnnotation: "bbb",
sidecarcontrol.SidecarSetHashAnnotation: "bbb",
},
Name: "test-sidecarset",
Name: "test-sidecarset",
Labels: map[string]string{},
},
Spec: appsv1alpha1.SidecarSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": "nginx"},
},
Strategy: appsv1alpha1.SidecarSetUpdateStrategy{
RollingUpdate: &appsv1alpha1.RollingUpdateSidecarSet{
MaxUnavailable: &maxUnavailable,
},
},
Containers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{
@ -48,28 +45,59 @@ var (
},
},
},
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": "nginx"},
},
Strategy: appsv1alpha1.SidecarSetUpdateStrategy{
Type: appsv1alpha1.RollingUpdateSidecarSetStrategyType,
Partition: &partition,
MaxUnavailable: &maxUnavailable,
},
},
}
podDemo = &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
mutating.SidecarSetHashAnnotation: `{"test-sidecarset":"aaa"}`,
mutating.SidecarSetHashWithoutImageAnnotation: `{"test-sidecarset":"bbb"}`,
sidecarcontrol.SidecarSetHashAnnotation: `{"test-sidecarset":{"hash":"aaa"}}`,
},
Name: "test-pod",
Namespace: "default",
Labels: map[string]string{"app": "nginx"},
Name: "test-pod-1",
Namespace: "default",
Labels: map[string]string{"app": "nginx"},
ResourceVersion: "495711227",
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "nginx",
Image: "nginx:1.15.1",
Env: []corev1.EnvVar{
{
Name: "nginx-env",
Value: "value-1",
},
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "nginx-volume",
MountPath: "/data/nginx",
},
},
},
{
Name: "test-sidecar",
Image: "test-image:v1",
Env: []corev1.EnvVar{
{
Name: "IS_INJECTED",
Value: "true",
},
},
},
},
Volumes: []corev1.Volume{
{
Name: "nginx-volume",
},
},
},
@ -81,6 +109,20 @@ var (
Status: corev1.ConditionTrue,
},
},
ContainerStatuses: []corev1.ContainerStatus{
{
Name: "nginx",
Image: "nginx:1.15.1",
ImageID: "docker-pullable://nginx@sha256:a9286defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d",
Ready: true,
},
{
Name: "test-sidecar",
Image: "test-image:v1",
ImageID: "docker-pullable://test-image@sha256:a9286defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d",
Ready: true,
},
},
},
}
)
@ -101,6 +143,15 @@ func getLatestPod(client client.Client, pod *corev1.Pod) (*corev1.Pod, error) {
return newPod, err
}
func getLatestSidecarSet(client client.Client, sidecarset *appsv1alpha1.SidecarSet) (*appsv1alpha1.SidecarSet, error) {
newSidecarSet := &appsv1alpha1.SidecarSet{}
Key := types.NamespacedName{
Name: sidecarset.Name,
}
err := client.Get(context.TODO(), Key, newSidecarSet)
return newSidecarSet, err
}
func isSidecarImageUpdated(pod *corev1.Pod, containerName, containerImage string) bool {
for _, container := range pod.Spec.Containers {
if container.Name == containerName {
@ -110,8 +161,13 @@ func isSidecarImageUpdated(pod *corev1.Pod, containerName, containerImage string
return false
}
func TestUpdateWhenEverythingIsFine(t *testing.T) {
func TestUpdateWhenUseNotUpdateStrategy(t *testing.T) {
sidecarSetInput := sidecarSetDemo.DeepCopy()
testUpdateWhenUseNotUpdateStrategy(t, sidecarSetInput)
}
func testUpdateWhenUseNotUpdateStrategy(t *testing.T, sidecarSetInput *appsv1alpha1.SidecarSet) {
sidecarSetInput.Spec.Strategy.Type = appsv1alpha1.NotUpdateSidecarSetStrategyType
podInput := podDemo.DeepCopy()
request := reconcile.Request{
NamespacedName: types.NamespacedName{
@ -121,7 +177,8 @@ func TestUpdateWhenEverythingIsFine(t *testing.T) {
}
fakeClient := fake.NewFakeClientWithScheme(scheme, sidecarSetInput, podInput)
reconciler := ReconcileSidecarSet{Client: fakeClient}
reconciler := ReconcileSidecarSet{
Client: fakeClient, updateExpectations: expectations.NewUpdateExpectations(sidecarcontrol.GetPodSidecarSetRevision)}
if _, err := reconciler.Reconcile(request); err != nil {
t.Errorf("reconcile failed, err: %v", err)
}
@ -130,14 +187,18 @@ func TestUpdateWhenEverythingIsFine(t *testing.T) {
if err != nil {
t.Errorf("get latest pod failed, err: %v", err)
}
if !isSidecarImageUpdated(podOutput, "test-sidecar", "test-image:v2") {
t.Errorf("should update sidecar")
if isSidecarImageUpdated(podOutput, "test-sidecar", "test-image:v2") {
t.Errorf("shouldn't update sidecar because sidecarset use not update strategy")
}
}
func TestUpdateWhenSidecarSetPaused(t *testing.T) {
sidecarSetInput := sidecarSetDemo.DeepCopy()
sidecarSetInput.Spec.Paused = true
testUpdateWhenSidecarSetPaused(t, sidecarSetInput)
}
func testUpdateWhenSidecarSetPaused(t *testing.T, sidecarSetInput *appsv1alpha1.SidecarSet) {
sidecarSetInput.Spec.Strategy.Paused = true
podInput := podDemo.DeepCopy()
request := reconcile.Request{
NamespacedName: types.NamespacedName{
@ -147,7 +208,12 @@ func TestUpdateWhenSidecarSetPaused(t *testing.T) {
}
fakeClient := fake.NewFakeClientWithScheme(scheme, sidecarSetInput, podInput)
reconciler := ReconcileSidecarSet{Client: fakeClient}
exps := expectations.NewUpdateExpectations(sidecarcontrol.GetPodSidecarSetRevision)
reconciler := ReconcileSidecarSet{
Client: fakeClient,
updateExpectations: exps,
processor: NewSidecarSetProcessor(fakeClient, exps, record.NewFakeRecorder(10)),
}
if _, err := reconciler.Reconcile(request); err != nil {
t.Errorf("reconcile failed, err: %v", err)
}
@ -161,36 +227,12 @@ func TestUpdateWhenSidecarSetPaused(t *testing.T) {
}
}
func TestUpdateWhenOtherFieldsChanged(t *testing.T) {
func TestUpdateWhenMaxUnavailableNotZero(t *testing.T) {
sidecarSetInput := sidecarSetDemo.DeepCopy()
sidecarSetInput.Annotations[mutating.SidecarSetHashAnnotation] = "ccc"
sidecarSetInput.Annotations[mutating.SidecarSetHashWithoutImageAnnotation] = "ddd"
podInput := podDemo.DeepCopy()
request := reconcile.Request{
NamespacedName: types.NamespacedName{
Namespace: sidecarSetInput.Namespace,
Name: sidecarSetInput.Name,
},
}
fakeClient := fake.NewFakeClientWithScheme(scheme, sidecarSetInput, podInput)
reconciler := ReconcileSidecarSet{Client: fakeClient}
if _, err := reconciler.Reconcile(request); err != nil {
t.Errorf("reconcile failed, err: %v", err)
}
podOutput, err := getLatestPod(fakeClient, podInput)
if err != nil {
t.Errorf("get latest pod failed, err: %v", err)
}
if isSidecarImageUpdated(podOutput, "test-sidecar", "test-image:v2") {
t.Errorf("shouldn't update sidecar because other fields in sidecarset had changed")
}
testUpdateWhenMaxUnavailableNotZero(t, sidecarSetInput)
}
func TestUpdateWhenExceedsMaxUnavailable(t *testing.T) {
sidecarSetInput := sidecarSetDemo.DeepCopy()
updateCache.reset(sidecarSetInput)
func testUpdateWhenMaxUnavailableNotZero(t *testing.T, sidecarSetInput *appsv1alpha1.SidecarSet) {
podInput := podDemo.DeepCopy()
podInput.Status.Phase = corev1.PodPending
request := reconcile.Request{
@ -201,7 +243,48 @@ func TestUpdateWhenExceedsMaxUnavailable(t *testing.T) {
}
fakeClient := fake.NewFakeClientWithScheme(scheme, sidecarSetInput, podInput)
reconciler := ReconcileSidecarSet{Client: fakeClient}
exps := expectations.NewUpdateExpectations(sidecarcontrol.GetPodSidecarSetRevision)
reconciler := ReconcileSidecarSet{
Client: fakeClient,
updateExpectations: exps,
processor: NewSidecarSetProcessor(fakeClient, exps, record.NewFakeRecorder(10)),
}
if _, err := reconciler.Reconcile(request); err != nil {
t.Errorf("reconcile failed, err: %v", err)
}
podOutput, err := getLatestPod(fakeClient, podInput)
if err != nil {
t.Errorf("get latest pod failed, err: %v", err)
}
if !isSidecarImageUpdated(podOutput, "test-sidecar", "test-image:v2") {
t.Errorf("should update sidecar with unavailable number not zero")
}
}
func TestUpdateWhenPartitionFinished(t *testing.T) {
sidecarSetInput := sidecarSetDemo.DeepCopy()
testUpdateWhenPartitionFinished(t, sidecarSetInput)
}
func testUpdateWhenPartitionFinished(t *testing.T, sidecarSetInput *appsv1alpha1.SidecarSet) {
newPartition := intstr.FromInt(1)
sidecarSetInput.Spec.Strategy.Partition = &newPartition
podInput := podDemo.DeepCopy()
request := reconcile.Request{
NamespacedName: types.NamespacedName{
Namespace: sidecarSetInput.Namespace,
Name: sidecarSetInput.Name,
},
}
fakeClient := fake.NewFakeClientWithScheme(scheme, sidecarSetInput, podInput)
exps := expectations.NewUpdateExpectations(sidecarcontrol.GetPodSidecarSetRevision)
reconciler := ReconcileSidecarSet{
Client: fakeClient,
updateExpectations: exps,
processor: NewSidecarSetProcessor(fakeClient, exps, record.NewFakeRecorder(10)),
}
if _, err := reconciler.Reconcile(request); err != nil {
t.Errorf("reconcile failed, err: %v", err)
}
@ -211,6 +294,42 @@ func TestUpdateWhenExceedsMaxUnavailable(t *testing.T) {
t.Errorf("get latest pod failed, err: %v", err)
}
if isSidecarImageUpdated(podOutput, "test-sidecar", "test-image:v2") {
t.Errorf("shouldn't update sidecar because exceeds unavailable number")
t.Errorf("shouldn't update sidecar because partition is 1")
}
}
func TestRemoveSidecarSet(t *testing.T) {
sidecarSetInput := sidecarSetDemo.DeepCopy()
testRemoveSidecarSet(t, sidecarSetInput)
}
func testRemoveSidecarSet(t *testing.T, sidecarSetInput *appsv1alpha1.SidecarSet) {
podInput := podDemo.DeepCopy()
hashKey := sidecarcontrol.SidecarSetHashAnnotation
podInput.Annotations[hashKey] = `{"test-sidecarset":{"hash":"bbb"}}`
request := reconcile.Request{
NamespacedName: types.NamespacedName{
Namespace: sidecarSetInput.Namespace,
Name: sidecarSetInput.Name,
},
}
fakeClient := fake.NewFakeClientWithScheme(scheme, sidecarSetInput, podInput)
exps := expectations.NewUpdateExpectations(sidecarcontrol.GetPodSidecarSetRevision)
reconciler := ReconcileSidecarSet{
Client: fakeClient,
updateExpectations: exps,
processor: NewSidecarSetProcessor(fakeClient, exps, record.NewFakeRecorder(10)),
}
if _, err := reconciler.Reconcile(request); err != nil {
t.Errorf("reconcile failed, err: %v", err)
}
podOutput, err := getLatestPod(fakeClient, podInput)
if err != nil {
t.Errorf("get latest pod failed, err: %v", err)
}
if podOutput.Annotations[hashKey] == "" {
t.Errorf("should remove sidecarset info")
}
}

129
pkg/controller/sidecarset/sidecarset_pod_event_handler.go
View File

@ -2,23 +2,23 @@ package sidecarset
import (
"context"
"reflect"
"strings"
"time"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/control/sidecarcontrol"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/util/workqueue"
"k8s.io/klog"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/webhook/pod/mutating"
)
var _ handler.EventHandler = &enqueueRequestForPod{}
@ -50,7 +50,7 @@ func (p *enqueueRequestForPod) addPod(q workqueue.RateLimitingInterface, obj run
return
}
sidecarSets, err := p.getPodSidecarSets(pod)
sidecarSets, err := p.getPodMatchedSidecarSets(pod)
if err != nil {
klog.Errorf("unable to get sidecarSets related with pod %s/%s, err: %v", pod.Namespace, pod.Name, err)
return
@ -79,74 +79,112 @@ func (p *enqueueRequestForPod) updatePod(q workqueue.RateLimitingInterface, old,
return
}
podChanged := isPodChanged(oldPod, newPod)
labelChanged := false
if !reflect.DeepEqual(newPod.Labels, oldPod.Labels) {
labelChanged = true
}
if !podChanged && !labelChanged {
//labels changed, and reconcile union sidecarSets
/*if !reflect.DeepEqual(newPod.Labels, oldPod.Labels) {
sidecarSets,err := p.getUnionSidecarSets(oldPod, newPod)
if err!=nil {
klog.Errorf("unable to get sidecarSets of pod %s/%s, err: %v", newPod.Namespace, newPod.Name, err)
return
}
for name := range sidecarSets {
q.Add(reconcile.Request{
NamespacedName: types.NamespacedName{
Name: name,
},
})
}
return
}
}*/
sidecarSets, err := p.getPodSidecarSetMemberships(newPod)
matchedSidecarSets, err := p.getPodMatchedSidecarSets(newPod)
if err != nil {
klog.Errorf("unable to get sidecarSets of pod %s/%s, err: %v", newPod.Namespace, newPod.Name, err)
return
}
if labelChanged {
oldSidecarSets, err := p.getPodSidecarSetMemberships(oldPod)
if err != nil {
klog.Errorf("unable to get sidecarSets of pod %s/%s, err: %v", oldPod.Namespace, oldPod.Name, err)
return
for _, sidecarSet := range matchedSidecarSets {
var isChanged bool
var enqueueDelayTime time.Duration
//check whether pod status is changed
if isChanged = isPodStatusChanged(oldPod, newPod); !isChanged {
//check whether pod consistent is changed
isChanged, enqueueDelayTime = isPodConsistentChanged(oldPod, newPod, sidecarSet)
}
if isChanged {
q.AddAfter(reconcile.Request{
NamespacedName: types.NamespacedName{
Name: sidecarSet.Name,
},
}, enqueueDelayTime)
}
sidecarSets = sidecarSets.Difference(oldSidecarSets).Union(oldSidecarSets.Difference(sidecarSets))
}
for name := range sidecarSets {
q.Add(reconcile.Request{
NamespacedName: types.NamespacedName{
Name: name,
},
})
}
/*func (p *enqueueRequestForPod) getUnionSidecarSets(oldPod, newPod *corev1.Pod) (sets.String, error) {
//if labels changed, then union older,new sidecarSets
sidecarSets, err := p.getPodSidecarSetMemberships(newPod)
if err != nil {
return nil, err
}
oldSidecarSets, err := p.getPodSidecarSetMemberships(oldPod)
if err != nil {
return nil, err
}
sidecarSets = sidecarSets.Difference(oldSidecarSets).Union(oldSidecarSets.Difference(sidecarSets))
return sidecarSets, nil
}
func (p *enqueueRequestForPod) getPodSidecarSetMemberships(pod *corev1.Pod) (sets.String, error) {
set := sets.String{}
sidecarSets, err := p.getPodSidecarSets(pod)
sidecarSets, err := p.getPodMatchedSidecarSets(pod)
if err != nil {
return set, err
}
for _, sidecarSet := range sidecarSets {
set.Insert(sidecarSet.Name)
}
return set, nil
}
}*/
func (p *enqueueRequestForPod) getPodMatchedSidecarSets(pod *corev1.Pod) ([]*appsv1alpha1.SidecarSet, error) {
sidecarSetNames, ok := pod.Annotations[sidecarcontrol.SidecarSetListAnnotation]
var matchedSidecarSets []*appsv1alpha1.SidecarSet
if ok && len(sidecarSetNames) > 0 {
for _, sidecarSetName := range strings.Split(sidecarSetNames, ",") {
sidecarSet := new(appsv1alpha1.SidecarSet)
if err := p.client.Get(context.TODO(), types.NamespacedName{
Name: sidecarSetName,
}, sidecarSet); err != nil {
if errors.IsNotFound(err) {
klog.Errorf("sidecarSet %v not fount", sidecarSetName)
continue
}
return nil, err
}
matchedSidecarSets = append(matchedSidecarSets, sidecarSet)
}
return matchedSidecarSets, nil
}
func (p *enqueueRequestForPod) getPodSidecarSets(pod *corev1.Pod) ([]appsv1alpha1.SidecarSet, error) {
sidecarSets := appsv1alpha1.SidecarSetList{}
if err := p.client.List(context.TODO(), &sidecarSets); err != nil {
return nil, err
}
var matchedSidecarSets []appsv1alpha1.SidecarSet
for _, sidecarSet := range sidecarSets.Items {
matched, err := mutating.PodMatchSidecarSet(pod, sidecarSet)
matched, err := sidecarcontrol.PodMatchedSidecarSet(pod, sidecarSet)
if err != nil {
return nil, err
}
if matched {
matchedSidecarSets = append(matchedSidecarSets, sidecarSet)
matchedSidecarSets = append(matchedSidecarSets, &sidecarSet)
}
}
return matchedSidecarSets, nil
}
func isPodChanged(oldPod, newPod *corev1.Pod) bool {
func isPodStatusChanged(oldPod, newPod *corev1.Pod) bool {
// If the pod's deletion timestamp is set, remove endpoint from ready address.
if newPod.DeletionTimestamp != oldPod.DeletionTimestamp {
return true
@ -160,9 +198,16 @@ func isPodChanged(oldPod, newPod *corev1.Pod) bool {
return true
}
if !isPodImageConsistent(oldPod) && isPodImageConsistent(newPod) {
return true
}
return false
}
func isPodConsistentChanged(oldPod, newPod *corev1.Pod, sidecarSet *appsv1alpha1.SidecarSet) (bool, time.Duration) {
control := sidecarcontrol.New(sidecarSet)
var enqueueDelayTime time.Duration
if !control.IsPodUpdatedConsistent(oldPod, nil) && control.IsPodUpdatedConsistent(newPod, nil) {
enqueueDelayTime = 5 * time.Second
return true, enqueueDelayTime
}
return false, enqueueDelayTime
}

177
pkg/controller/sidecarset/sidecarset_pod_event_handler_test.go Normal file
View File

@ -0,0 +1,177 @@
/*
Copyright 2020 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sidecarset
import (
"context"
"fmt"
"testing"
"time"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/control/sidecarcontrol"
corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/util/workqueue"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/event"
)
func TestPodEventHandler(t *testing.T) {
fakeClient := fake.NewFakeClientWithScheme(scheme)
handler := enqueueRequestForPod{client: fakeClient}
err := fakeClient.Create(context.TODO(), sidecarSetDemo)
if nil != err {
t.Fatalf("unexpected create sidecarSet %s failed: %v", sidecarSetDemo.Name, err)
}
// create
createQ := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
createEvt := event.CreateEvent{
Object: podDemo,
}
handler.Create(createEvt, createQ)
if createQ.Len() != 1 {
t.Errorf("unexpected create event handle queue size, expected 1 actual %d", createQ.Len())
}
// update with pod status changed and reconcile
newPod := podDemo.DeepCopy()
newPod.ResourceVersion = fmt.Sprintf("%d", time.Now().Unix())
readyCondition := podutil.GetPodReadyCondition(newPod.Status)
readyCondition.Status = corev1.ConditionFalse
updateQ := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
updateEvent := event.UpdateEvent{
ObjectOld: podDemo,
ObjectNew: newPod,
}
handler.Update(updateEvent, updateQ)
if updateQ.Len() != 1 {
t.Errorf("unexpected update event handle queue size, expected 1 actual %d", updateQ.Len())
}
// update with pod spec changed and no reconcile
newPod = podDemo.DeepCopy()
newPod.ResourceVersion = fmt.Sprintf("%d", time.Now().Unix())
newPod.Spec.Containers[0].Image = "nginx:latest"
updateQ = workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
updateEvent = event.UpdateEvent{
ObjectOld: podDemo,
ObjectNew: newPod,
}
handler.Update(updateEvent, updateQ)
if updateQ.Len() != 0 {
t.Errorf("unexpected update event handle queue size, expected 0 actual %d", updateQ.Len())
}
// delete
deleteQ := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
deleteEvt := event.DeleteEvent{
Object: podDemo,
}
handler.Delete(deleteEvt, deleteQ)
if deleteQ.Len() != 1 {
t.Errorf("unexpected delete event handle queue size, expected 1 actual %d", deleteQ.Len())
}
}
func TestGetPodMatchedSidecarSets(t *testing.T) {
cases := []struct {
name string
getPod func() *corev1.Pod
getSidecarSets func() []*appsv1alpha1.SidecarSet
exceptSidecarSetCount int
}{
{
name: "pod matched single sidecarSet",
getPod: func() *corev1.Pod {
pod := podDemo.DeepCopy()
pod.Annotations[sidecarcontrol.SidecarSetListAnnotation] = "test-sidecarset-2"
return pod
},
getSidecarSets: func() []*appsv1alpha1.SidecarSet {
sidecar1 := sidecarSetDemo.DeepCopy()
sidecar1.Name = "test-sidecarset-1"
sidecar2 := sidecarSetDemo.DeepCopy()
sidecar2.Name = "test-sidecarset-2"
sidecar3 := sidecarSetDemo.DeepCopy()
sidecar3.Name = "test-sidecarset-3"
return []*appsv1alpha1.SidecarSet{sidecar1, sidecar2, sidecar3}
},
exceptSidecarSetCount: 1,
},
{
name: "pod matched two sidecarSets",
getPod: func() *corev1.Pod {
pod := podDemo.DeepCopy()
pod.Annotations[sidecarcontrol.SidecarSetListAnnotation] = "test-sidecarset-1,test-sidecarset-3"
return pod
},
getSidecarSets: func() []*appsv1alpha1.SidecarSet {
sidecar1 := sidecarSetDemo.DeepCopy()
sidecar1.Name = "test-sidecarset-1"
sidecar2 := sidecarSetDemo.DeepCopy()
sidecar2.Name = "test-sidecarset-2"
sidecar3 := sidecarSetDemo.DeepCopy()
sidecar3.Name = "test-sidecarset-3"
return []*appsv1alpha1.SidecarSet{sidecar1, sidecar2, sidecar3}
},
exceptSidecarSetCount: 2,
},
{
name: "pod matched no sidecarSets",
getPod: func() *corev1.Pod {
pod := podDemo.DeepCopy()
pod.Annotations[sidecarcontrol.SidecarSetListAnnotation] = "test-sidecarset-4"
return pod
},
getSidecarSets: func() []*appsv1alpha1.SidecarSet {
sidecar1 := sidecarSetDemo.DeepCopy()
sidecar1.Name = "test-sidecarset-1"
sidecar2 := sidecarSetDemo.DeepCopy()
sidecar2.Name = "test-sidecarset-2"
sidecar3 := sidecarSetDemo.DeepCopy()
sidecar3.Name = "test-sidecarset-3"
return []*appsv1alpha1.SidecarSet{sidecar1, sidecar2, sidecar3}
},
exceptSidecarSetCount: 0,
},
}
for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) {
pod := cs.getPod()
fakeClient := fake.NewFakeClientWithScheme(scheme, pod)
sidecarSets := cs.getSidecarSets()
for _, sidecarSet := range sidecarSets {
fakeClient.Create(context.TODO(), sidecarSet)
}
e := enqueueRequestForPod{fakeClient}
matched, err := e.getPodMatchedSidecarSets(pod)
if err != nil {
t.Fatalf("getPodMatchedSidecarSets failed: %s", err.Error())
}
if len(matched) != cs.exceptSidecarSetCount {
t.Fatalf("except matched sidecarSet(count=%d), but get count=%d", cs.exceptSidecarSetCount, len(matched))
}
})
}
}

381
pkg/controller/sidecarset/sidecarset_processor.go Normal file
View File

@ -0,0 +1,381 @@
/*
Copyright 2020 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sidecarset
import (
"context"
"fmt"
"strings"
"time"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/control/sidecarcontrol"
"github.com/openkruise/kruise/pkg/util"
"github.com/openkruise/kruise/pkg/util/expectations"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/retry"
"k8s.io/klog"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
)
type Processor struct {
Client client.Client
recorder record.EventRecorder
updateExpectations expectations.UpdateExpectations
}
func NewSidecarSetProcessor(cli client.Client, expectations expectations.UpdateExpectations, rec record.EventRecorder) *Processor {
return &Processor{
Client: cli,
updateExpectations: expectations,
recorder: rec,
}
}
func (p *Processor) UpdateSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (reconcile.Result, error) {
// SidecarSet upgrade strategy type is NotUpdate
if !isSidecarSetNotUpdate(sidecarSet) {
return reconcile.Result{}, nil
}
control := sidecarcontrol.New(sidecarSet)
// 1. get matching pods with the sidecarSet
pods, err := p.getMatchingPods(sidecarSet)
if err != nil {
klog.Errorf("sidecarSet get matching pods error, err: %v, name: %s", err, sidecarSet.Name)
return reconcile.Result{}, err
}
// 2. calculate SidecarSet status based on pods
status := calculateStatus(control, pods)
//update sidecarSet status in store
if err := p.updateSidecarSetStatus(sidecarSet, status); err != nil {
return reconcile.Result{}, err
}
// in case of informer cache latency
for _, pod := range pods {
p.updateExpectations.ObserveUpdated(sidecarSet.Name, sidecarcontrol.GetSidecarSetRevision(sidecarSet), pod)
}
allUpdated, _, inflightPods := p.updateExpectations.SatisfiedExpectations(sidecarSet.Name, sidecarcontrol.GetSidecarSetRevision(sidecarSet))
if !allUpdated {
klog.V(3).Infof("sidecarset %s matched pods has some update in flight: %v, will sync later", sidecarSet.Name, inflightPods)
return reconcile.Result{RequeueAfter: time.Second}, nil
}
// 4. sidecarset already updates all matched pods, then return
if isSidecarSetUpdateFinish(status) {
klog.V(3).Infof("sidecarSet update pod finished, name: %s", sidecarSet.Name)
return reconcile.Result{}, nil
}
// 5. Paused indicates that the SidecarSet is paused to update matched pods
if sidecarSet.Spec.Strategy.Paused {
klog.V(3).Infof("sidecarSet is paused, name: %s", sidecarSet.Name)
return reconcile.Result{}, nil
}
// 6. In kubernetes cluster, when inplace update pod, only fields such as image can be updated for the container.
// It is to determine whether there are other fields that have been modified for pod.
var canUpgradePods []*corev1.Pod
for index := range pods {
if control.IsSidecarSetCanUpgrade(pods[index]) {
canUpgradePods = append(canUpgradePods, pods[index])
}
}
if len(canUpgradePods) == 0 {
klog.V(3).Infof("sidecarSet(%s) will not upgrade pods", sidecarSet.Name)
return reconcile.Result{}, nil
}
// 7. upgrade pod sidecar
if err := p.updatePods(control, canUpgradePods); err != nil {
return reconcile.Result{}, err
}
return reconcile.Result{}, nil
}
func (p *Processor) updatePods(control sidecarcontrol.SidecarControl, pods []*corev1.Pod) error {
sidecarset := control.GetSidecarset()
// compute next updated pods based on the sidecarset upgrade strategy
upgradePods := NewStrategy().GetNextUpgradePods(control, pods)
if len(upgradePods) == 0 {
klog.V(3).Infof("sidecarSet next update is nil, skip this round, name: %s", sidecarset.Name)
return nil
}
// upgrade pod sidecar
for _, pod := range upgradePods {
if err := p.updatePodSidecarAndHash(control, pod); err != nil {
err := fmt.Errorf("updatePodSidecarAndHash error, s:%s, pod:%s, err:%v", sidecarset.Name, pod.Name, err)
return err
}
p.updateExpectations.ExpectUpdated(sidecarset.Name, sidecarcontrol.GetSidecarSetRevision(sidecarset), pod)
}
// mark upgrade pods list
podNames := make([]string, 0, len(upgradePods))
for _, pod := range upgradePods {
podNames = append(podNames, pod.Name)
}
klog.V(3).Infof("sidecarSet inject pod step, name: %s, pods: %v", sidecarset.Name, podNames)
return nil
}
func (p *Processor) updatePodSidecarAndHash(control sidecarcontrol.SidecarControl, pod *corev1.Pod) error {
podClone := pod.DeepCopy()
err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
// update pod sidecar container
updatePodSidecarContainer(control, podClone)
// older pod don't have SidecarSetListAnnotation
// which is to improve the performance of the sidecarSet controller
sidecarSetNames, ok := podClone.Annotations[sidecarcontrol.SidecarSetListAnnotation]
if !ok || len(sidecarSetNames) == 0 {
podClone.Annotations[sidecarcontrol.SidecarSetListAnnotation] = p.listMatchedSidecarSets(podClone)
}
//update pod in store
updateErr := p.Client.Update(context.TODO(), podClone)
if updateErr == nil {
return nil
}
key := types.NamespacedName{
Namespace: podClone.Namespace,
Name: podClone.Name,
}
if err := p.Client.Get(context.TODO(), key, podClone); err != nil {
klog.Errorf("error getting updated pod %s from client", control.GetSidecarset().Name)
}
return updateErr
})
return err
}
func (p *Processor) listMatchedSidecarSets(pod *corev1.Pod) string {
sidecarSetList := &appsv1alpha1.SidecarSetList{}
if err := p.Client.List(context.TODO(), sidecarSetList); err != nil {
klog.Errorf("List SidecarSets failed: %s", err.Error())
return ""
}
//matched SidecarSet.Name list
sidecarSetNames := make([]string, 0)
for _, sidecarSet := range sidecarSetList.Items {
if matched, _ := sidecarcontrol.PodMatchedSidecarSet(pod, sidecarSet); matched {
sidecarSetNames = append(sidecarSetNames, sidecarSet.Name)
}
}
return strings.Join(sidecarSetNames, ",")
}
func (p *Processor) updateSidecarSetStatus(sidecarSet *appsv1alpha1.SidecarSet, status *appsv1alpha1.SidecarSetStatus) error {
if !inconsistentStatus(sidecarSet, status) {
return nil
}
sidecarSetClone := sidecarSet.DeepCopy()
if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
sidecarSetClone.Status = *status
sidecarSetClone.Status.ObservedGeneration = sidecarSetClone.Generation
updateErr := p.Client.Status().Update(context.TODO(), sidecarSetClone)
if updateErr == nil {
return nil
}
key := types.NamespacedName{
Name: sidecarSetClone.Name,
}
if err := p.Client.Get(context.TODO(), key, sidecarSetClone); err != nil {
klog.Errorf("error getting updated sidecarset %s from client", sidecarSetClone.Name)
}
return updateErr
}); err != nil {
return err
}
klog.V(3).Infof("sidecarSet update status success, name: %s", sidecarSet.Name)
return nil
}
// If you need update the pod object, you must DeepCopy it
func (p *Processor) getMatchingPods(s *appsv1alpha1.SidecarSet) ([]*corev1.Pod, error) {
// get more faster selector
selector, err := util.GetFastLabelSelector(s.Spec.Selector)
if err != nil {
return nil, err
}
// If sidecarSet.Spec.Namespace is empty, then select in cluster
scopedNamespaces := []string{s.Spec.Namespace}
selectedPods, err := p.getSelectedPods(scopedNamespaces, selector)
if err != nil {
return nil, err
}
// filter out pods that don't require updated, include the following:
// 1. Deletion pod
// 2. ignore namespace: "kube-system", "kube-public"
// 3. never be injected sidecar container
var filteredPods []*corev1.Pod
for _, pod := range selectedPods {
if sidecarcontrol.IsActivePod(pod) && isPodInjectedSidecar(s, pod) {
filteredPods = append(filteredPods, pod)
}
}
return filteredPods, nil
}
// get selected pods(DisableDeepCopy:true, indicates must be deep copy before update pod objection)
func (p *Processor) getSelectedPods(namespaces []string, selector labels.Selector) (relatedPods []*corev1.Pod, err error) {
// DisableDeepCopy:true, indicates must be deep copy before update pod objection
listOpts := &client.ListOptions{LabelSelector: selector}
for _, ns := range namespaces {
allPods := &corev1.PodList{}
listOpts.Namespace = ns
if listErr := p.Client.List(context.TODO(), allPods, listOpts); listErr != nil {
err = fmt.Errorf("sidecarSet list pods by ns error, ns[%s], err:%v", ns, listErr)
return
}
for i := range allPods.Items {
relatedPods = append(relatedPods, &allPods.Items[i])
}
}
return
}
// calculate SidecarSet status
// MatchedPods: all matched pods number
// UpdatedPods: updated pods number
// ReadyPods: ready pods number
// UpdatedReadyPods: updated and ready pods number
// UnavailablePods: MatchedPods - UpdatedReadyPods
func calculateStatus(control sidecarcontrol.SidecarControl, pods []*corev1.Pod) *appsv1alpha1.SidecarSetStatus {
sidecarset := control.GetSidecarset()
var matchedPods, updatedPods, readyPods, updatedAndReady int32
matchedPods = int32(len(pods))
for _, pod := range pods {
updated := sidecarcontrol.IsPodSidecarUpdated(sidecarset, pod)
if updated {
updatedPods++
}
if control.IsPodConsistentAndReady(pod) {
readyPods++
if updated {
updatedAndReady++
}
}
}
return &appsv1alpha1.SidecarSetStatus{
ObservedGeneration: sidecarset.Generation,
MatchedPods: matchedPods,
UpdatedPods: updatedPods,
ReadyPods: readyPods,
UpdatedReadyPods: updatedAndReady,
}
}
// whether this pod has been injected sidecar container based on the sidecarSet
func isPodInjectedSidecar(sidecarSet *appsv1alpha1.SidecarSet, pod *corev1.Pod) bool {
// if pod annotations contain sidecarset hash, then indicates the pod has been injected in sidecar container
return sidecarcontrol.GetPodSidecarSetRevision(sidecarSet.Name, pod) != ""
}
func isSidecarSetNotUpdate(s *appsv1alpha1.SidecarSet) bool {
if s.Spec.Strategy.Type == appsv1alpha1.NotUpdateSidecarSetStrategyType || s.Spec.Strategy.Type == "" {
klog.V(3).Infof("sidecarSet spreading RollingUpdate config type, name: %s, type: %s", s.Name, s.Spec.Strategy.Type)
return false
}
return true
}
func updateColdUpgradeContainerInPod(sidecarContainer *appsv1alpha1.SidecarContainer, control sidecarcontrol.SidecarControl, pod *corev1.Pod) (changedContainer string) {
var containerToUpgrade corev1.Container
for _, containerInPod := range pod.Spec.Containers {
if containerInPod.Name == sidecarContainer.Name {
containerToUpgrade = containerInPod
break
}
}
beforeContainerSpec := util.DumpJSON(containerToUpgrade)
newContainer := control.UpdateSidecarContainerToLatest(sidecarContainer.Container, containerToUpgrade)
afterContainerSpec := util.DumpJSON(newContainer)
// pod.container definition changed, then update container spec in pod
if beforeContainerSpec != afterContainerSpec {
klog.V(3).Infof("try to update container %v/%v/%v, before: %v, after: %v",
pod.Namespace, pod.Name, newContainer.Name, beforeContainerSpec, afterContainerSpec)
updateContainerInPod(newContainer, pod)
changedContainer = newContainer.Name
}
return
}
func updateContainerInPod(container corev1.Container, pod *corev1.Pod) {
for i := range pod.Spec.Containers {
if pod.Spec.Containers[i].Name == container.Name {
pod.Spec.Containers[i] = container
return
}
}
}
func updatePodSidecarContainer(control sidecarcontrol.SidecarControl, pod *corev1.Pod) {
sidecarset := control.GetSidecarset()
var changedContainers []string
for _, sidecarContainer := range sidecarset.Spec.Containers {
//sidecarContainer := &sidecarset.Spec.Containers[i]
// volumeMounts that injected into sidecar container
// when volumeMounts SubPathExpr contains expansions, then need copy container EnvVars(injectEnvs)
injectedMounts, injectedEnvs := sidecarcontrol.GetInjectedVolumeMountsAndEnvs(control, &sidecarContainer, pod)
// merge VolumeMounts from sidecar.VolumeMounts and shared VolumeMounts
sidecarContainer.VolumeMounts = util.MergeVolumeMounts(sidecarContainer.VolumeMounts, injectedMounts)
// get injected env & mounts explicitly so that can be compared with old ones in pod
transferEnvs := sidecarcontrol.GetSidecarTransferEnvs(&sidecarContainer, pod)
// append volumeMounts SubPathExpr environments
transferEnvs = util.MergeEnvVar(transferEnvs, injectedEnvs)
// merged Env from sidecar.Env and transfer envs
sidecarContainer.Env = util.MergeEnvVar(sidecarContainer.Env, transferEnvs)
var changedContainer string
changedContainer = updateColdUpgradeContainerInPod(&sidecarContainer, control, pod)
if changedContainer != "" {
changedContainers = append(changedContainers, changedContainer)
}
}
// update pod information in upgrade
control.UpdatePodAnnotationsInUpgrade(changedContainers, pod)
return
}
func inconsistentStatus(sidecarSet *appsv1alpha1.SidecarSet, status *appsv1alpha1.SidecarSetStatus) bool {
return status.ObservedGeneration > sidecarSet.Status.ObservedGeneration ||
status.MatchedPods != sidecarSet.Status.MatchedPods ||
status.UpdatedPods != sidecarSet.Status.UpdatedPods ||
status.ReadyPods != sidecarSet.Status.ReadyPods ||
status.UpdatedReadyPods != sidecarSet.Status.UpdatedReadyPods
}

299
pkg/controller/sidecarset/sidecarset_processor_test.go Normal file
View File

@ -0,0 +1,299 @@
/*
Copyright 2020 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sidecarset
import (
"context"
"fmt"
"testing"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/control/sidecarcontrol"
"github.com/openkruise/kruise/pkg/util"
"github.com/openkruise/kruise/pkg/util/expectations"
corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)
var (
testImageV1ImageID = "docker-pullable://test-image@sha256:a9286defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d"
testImageV2ImageID = "docker-pullable://test-image@sha256:f7988fb6c02e0ce69257d9bd9cf37ae20a60f1df7563c3a2a6abe24160306b8d"
)
func TestUpdateColdUpgradeSidecar(t *testing.T) {
sidecarSetInput := sidecarSetDemo.DeepCopy()
podInput := podDemo.DeepCopy()
podInput.Spec.Containers[1].Env = []corev1.EnvVar{
{
Name: "nginx-env",
Value: "nginx-value",
},
}
podInput.Spec.Containers[1].VolumeMounts = []corev1.VolumeMount{
{
MountPath: "/data/nginx",
},
}
handlers := map[string]HandlePod{
"pod test-pod-1 is upgrading": func(pods []*corev1.Pod) {
cStatus := &pods[0].Status.ContainerStatuses[1]
cStatus.Image = "test-image:v2"
cStatus.ImageID = testImageV2ImageID
},
"pod test-pod-2 is upgrading": func(pods []*corev1.Pod) {
cStatus := &pods[1].Status.ContainerStatuses[1]
cStatus.Image = "test-image:v2"
cStatus.ImageID = testImageV2ImageID
},
}
testUpdateColdUpgradeSidecar(t, podInput, sidecarSetInput, handlers)
}
func testUpdateColdUpgradeSidecar(t *testing.T, podDemo *corev1.Pod, sidecarSetInput *appsv1alpha1.SidecarSet, handlers map[string]HandlePod) {
podInput1 := podDemo.DeepCopy()
podInput2 := podDemo.DeepCopy()
podInput2.Name = "test-pod-2"
cases := []struct {
name string
getPods func() []*corev1.Pod
getSidecarset func() *appsv1alpha1.SidecarSet
// pod.name -> infos []string{Image, Env, volumeMounts}
expectedInfo map[*corev1.Pod][]string
// MatchedPods, UpdatedPods, ReadyPods, AvailablePods, UnavailablePods
expectedStatus []int32
}{
{
name: "sidecarset update pod test-pod-1",
getPods: func() []*corev1.Pod {
pods := []*corev1.Pod{
podInput1.DeepCopy(), podInput2.DeepCopy(),
}
return pods
},
getSidecarset: func() *appsv1alpha1.SidecarSet {
return sidecarSetInput.DeepCopy()
},
expectedInfo: map[*corev1.Pod][]string{
podInput1: {"test-image:v2", "nginx-env", "/data/nginx", "test-sidecarset"},
podInput2: {"test-image:v1"},
},
expectedStatus: []int32{2, 0, 2, 0},
},
{
name: "pod test-pod-1 is upgrading",
getPods: func() []*corev1.Pod {
pods := []*corev1.Pod{
podInput1.DeepCopy(), podInput2.DeepCopy(),
}
return pods
},
getSidecarset: func() *appsv1alpha1.SidecarSet {
return sidecarSetInput.DeepCopy()
},
expectedInfo: map[*corev1.Pod][]string{
podInput1: {"test-image:v2", "nginx-env", "/data/nginx", "test-sidecarset"},
podInput2: {"test-image:v1"},
},
expectedStatus: []int32{2, 1, 1, 0},
},
{
name: "pod test-pod-1 upgrade complete, and start update pod test-pod-2",
getPods: func() []*corev1.Pod {
pod1 := podInput1.DeepCopy()
pods := []*corev1.Pod{
pod1, podInput2.DeepCopy(),
}
return pods
},
getSidecarset: func() *appsv1alpha1.SidecarSet {
return sidecarSetInput.DeepCopy()
},
expectedInfo: map[*corev1.Pod][]string{
podInput1: {"test-image:v2", "nginx-env", "/data/nginx", "test-sidecarset"},
podInput2: {"test-image:v2", "nginx-env", "/data/nginx", "test-sidecarset"},
},
expectedStatus: []int32{2, 1, 2, 1},
},
{
name: "pod test-pod-2 is upgrading",
getPods: func() []*corev1.Pod {
pods := []*corev1.Pod{
podInput1.DeepCopy(), podInput2.DeepCopy(),
}
return pods
},
getSidecarset: func() *appsv1alpha1.SidecarSet {
return sidecarSetInput.DeepCopy()
},
expectedInfo: map[*corev1.Pod][]string{
podInput1: {"test-image:v2", "nginx-env", "/data/nginx", "test-sidecarset"},
podInput2: {"test-image:v2", "nginx-env", "/data/nginx", "test-sidecarset"},
},
expectedStatus: []int32{2, 2, 1, 1},
},
{
name: "pod test-pod-2 upgrade complete",
getPods: func() []*corev1.Pod {
pod2 := podInput2.DeepCopy()
pods := []*corev1.Pod{
podInput1.DeepCopy(), pod2,
}
return pods
},
getSidecarset: func() *appsv1alpha1.SidecarSet {
return sidecarSetInput.DeepCopy()
},
expectedInfo: map[*corev1.Pod][]string{
podInput1: {"test-image:v2", "nginx-env", "/data/nginx", "test-sidecarset"},
podInput2: {"test-image:v2", "nginx-env", "/data/nginx", "test-sidecarset"},
},
expectedStatus: []int32{2, 2, 2, 2},
},
}
exps := expectations.NewUpdateExpectations(sidecarcontrol.GetPodSidecarSetRevision)
for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) {
pods := cs.getPods()
sidecarset := cs.getSidecarset()
fakeClient := fake.NewFakeClientWithScheme(scheme, sidecarset, pods[0], pods[1])
processor := NewSidecarSetProcessor(fakeClient, exps, record.NewFakeRecorder(10))
_, err := processor.UpdateSidecarSet(sidecarset)
if err != nil {
t.Errorf("processor update sidecarset failed: %s", err.Error())
}
for pod, infos := range cs.expectedInfo {
podOutput, err := getLatestPod(fakeClient, pod)
if err != nil {
t.Errorf("get latest pod(%s) failed: %s", pod.Name, err.Error())
}
sidecarContainer := &podOutput.Spec.Containers[1]
if infos[0] != sidecarContainer.Image {
t.Fatalf("expect pod(%s) container(%s) image(%s), but get image(%s)", pod.Name, sidecarContainer.Name, infos[0], sidecarContainer.Image)
}
if len(infos) >= 2 && util.GetContainerEnvVar(sidecarContainer, infos[1]) == nil {
t.Fatalf("expect pod(%s) container(%s) env(%s), but get nil", pod.Name, sidecarContainer.Name, infos[1])
}
if len(infos) >= 3 && util.GetContainerVolumeMount(sidecarContainer, infos[2]) == nil {
t.Fatalf("expect pod(%s) container(%s) volumeMounts(%s), but get nil", pod.Name, sidecarContainer.Name, infos[2])
}
if len(infos) >= 4 && podOutput.Annotations[sidecarcontrol.SidecarSetListAnnotation] != infos[3] {
t.Fatalf("expect pod(%s) annotations[%s]=%s, but get %s", pod.Name, sidecarcontrol.SidecarSetListAnnotation, infos[3], podOutput.Annotations[sidecarcontrol.SidecarSetListAnnotation])
}
if pod.Name == "test-pod-1" {
podInput1 = podOutput
} else {
podInput2 = podOutput
}
}
sidecarsetOutput, err := getLatestSidecarSet(fakeClient, sidecarset)
if err != nil {
t.Errorf("get latest sidecarset(%s) failed: %s", sidecarset.Name, err.Error())
}
sidecarSetInput = sidecarsetOutput
for k, v := range cs.expectedStatus {
var actualValue int32
switch k {
case 0:
actualValue = sidecarsetOutput.Status.MatchedPods
case 1:
actualValue = sidecarsetOutput.Status.UpdatedPods
case 2:
actualValue = sidecarsetOutput.Status.ReadyPods
case 3:
actualValue = sidecarsetOutput.Status.UpdatedReadyPods
default:
//
}
if v != actualValue {
t.Fatalf("except sidecarset status(%d:%d), but get value(%d)", k, v, actualValue)
}
}
//handle potInput
if handle, ok := handlers[cs.name]; ok {
handle([]*corev1.Pod{podInput1, podInput2})
}
})
}
}
func TestScopeNamespacePods(t *testing.T) {
sidecarSet := sidecarSetDemo.DeepCopy()
sidecarSet.Spec.Namespace = "test-ns"
fakeClient := fake.NewFakeClientWithScheme(scheme, sidecarSet)
for i := 0; i < 100; i++ {
pod := podDemo.DeepCopy()
pod.Name = fmt.Sprintf("%s-%d", pod.Name, i)
if i >= 50 {
pod.Namespace = "test-ns"
}
fakeClient.Create(context.TODO(), pod)
}
exps := expectations.NewUpdateExpectations(sidecarcontrol.GetPodSidecarSetRevision)
processor := NewSidecarSetProcessor(fakeClient, exps, record.NewFakeRecorder(10))
pods, err := processor.getMatchingPods(sidecarSet)
if err != nil {
t.Fatalf("getMatchingPods failed: %s", err.Error())
return
}
if len(pods) != 50 {
t.Fatalf("except matching pods count(%d), but get count(%d)", 50, len(pods))
}
}
func TestCanUpgradePods(t *testing.T) {
sidecarSet := factorySidecarSet()
fakeClient := fake.NewFakeClientWithScheme(scheme, sidecarSet)
pods := factoryPodsCommon(100, 0, sidecarSet)
exps := expectations.NewUpdateExpectations(sidecarcontrol.GetPodSidecarSetRevision)
for i := range pods {
if i < 50 {
pods[i].Annotations[sidecarcontrol.SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset":{"hash":"without-aaa"}}`
} else {
pods[i].Annotations[sidecarcontrol.SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset":{"hash":"without-bbb"}}`
}
fakeClient.Create(context.TODO(), pods[i])
}
processor := NewSidecarSetProcessor(fakeClient, exps, record.NewFakeRecorder(10))
_, err := processor.UpdateSidecarSet(sidecarSet)
if err != nil {
t.Errorf("processor update sidecarset failed: %s", err.Error())
}
for i := range pods {
pod := pods[i]
podOutput, err := getLatestPod(fakeClient, pod)
if err != nil {
t.Errorf("get latest pod(%s) failed: %s", pod.Name, err.Error())
}
if i < 50 {
if podOutput.Spec.Containers[1].Image != "test-image:v1" {
t.Fatalf("except pod(%d) image(test-image:v1), but get image(%s)", i, podOutput.Spec.Containers[1].Image)
}
} else {
if podOutput.Spec.Containers[1].Image != "test-image:v2" {
t.Fatalf("except pod(%d) image(test-image:v2), but get image(%s)", i, podOutput.Spec.Containers[1].Image)
}
}
}
}

191
pkg/controller/sidecarset/sidecarset_strategy.go Normal file
View File

@ -0,0 +1,191 @@
package sidecarset
import (
"math"
"sort"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/control/sidecarcontrol"
"github.com/openkruise/kruise/pkg/util/updatesort"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
intstrutil "k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/klog"
)
type Strategy interface {
// according to sidecarset's upgrade strategy, select the pods to be upgraded, include the following:
//1. select which pods can be upgrade, the following:
// * pod must be not updated for the latest sidecarSet
// * If selector is not nil, this upgrade will only update the selected pods.
//2. Sort Pods with default sequence
//3. sort waitUpdateIndexes based on the scatter rules
//4. calculate max count of pods can update with maxUnavailable
GetNextUpgradePods(control sidecarcontrol.SidecarControl, pods []*corev1.Pod) []*corev1.Pod
}
type spreadingStrategy struct{}
var (
globalSpreadingStrategy = &spreadingStrategy{}
)
func NewStrategy() Strategy {
return globalSpreadingStrategy
}
func (p *spreadingStrategy) GetNextUpgradePods(control sidecarcontrol.SidecarControl, pods []*corev1.Pod) (upgradePods []*corev1.Pod) {
sidecarset := control.GetSidecarset()
// wait to upgrade pod index
var waitUpgradedIndexes []int
strategy := sidecarset.Spec.Strategy
// If selector is not nil, check whether the pods is selected to upgrade
isSelected := func(pod *corev1.Pod) bool {
//when selector is nil, always return ture
if strategy.Selector == nil {
return true
}
// if selector failed, always return false
selector, err := metav1.LabelSelectorAsSelector(strategy.Selector)
if err != nil {
klog.Errorf("sidecarSet(%s) rolling selector error, err: %v", sidecarset.Name, err)
return false
}
//matched
if selector.Matches(labels.Set(pod.Labels)) {
return true
}
//Not matched, then return false
return false
}
//1. select which pods can be upgraded, the following:
// * pod must be not updated for the latest sidecarSet
// * If selector is not nil, this upgrade will only update the selected pods.
for index, pod := range pods {
isUpdated := sidecarcontrol.IsPodSidecarUpdated(sidecarset, pod)
if !isUpdated && isSelected(pod) {
waitUpgradedIndexes = append(waitUpgradedIndexes, index)
}
}
//2. Sort Pods with default sequence
// - Unassigned < assigned
// - PodPending < PodUnknown < PodRunning
// - Not ready < ready
// - Been ready for empty time < less time < more time
// - Pods with containers with higher restart counts < lower restart counts
// - Empty creation time pods < newer pods < older pods
sort.Slice(waitUpgradedIndexes, sidecarcontrol.GetPodsSortFunc(pods, waitUpgradedIndexes))
//3. sort waitUpdateIndexes based on the scatter rules
if strategy.ScatterStrategy != nil {
// convert regular terms to scatter terms
// for examples: labelA=* -> labelA=value1, labelA=value2...(labels in pod definition)
scatter := parseUpdateScatterTerms(strategy.ScatterStrategy, pods)
waitUpgradedIndexes = updatesort.NewScatterSorter(scatter).Sort(pods, waitUpgradedIndexes)
}
//4. calculate to be upgraded pods number for the time
needToUpgradeCount := calculateUpgradeCount(control, waitUpgradedIndexes, pods)
if needToUpgradeCount < len(waitUpgradedIndexes) {
waitUpgradedIndexes = waitUpgradedIndexes[:needToUpgradeCount]
}
//5. injectPods will be upgraded in the following process
for _, idx := range waitUpgradedIndexes {
upgradePods = append(upgradePods, pods[idx])
}
return
}
func calculateUpgradeCount(coreControl sidecarcontrol.SidecarControl, waitUpdateIndexes []int, pods []*corev1.Pod) int {
totalReplicas := len(pods)
sidecarSet := coreControl.GetSidecarset()
strategy := sidecarSet.Spec.Strategy
// default partition = 0, indicates all pods will been upgraded
var partition int
if strategy.Partition != nil {
partition, _ = intstrutil.GetValueFromIntOrPercent(strategy.Partition, totalReplicas, false)
}
// indicates the partition pods will not be upgraded for the time
if len(waitUpdateIndexes)-partition <= 0 {
return 0
}
waitUpdateIndexes = waitUpdateIndexes[:(len(waitUpdateIndexes) - partition)]
// max unavailable pods number
maxUnavailable := math.MaxInt32
if strategy.MaxUnavailable != nil {
maxUnavailable, _ = intstrutil.GetValueFromIntOrPercent(strategy.MaxUnavailable, totalReplicas, false)
}
var upgradeAndNotReadyCount int
for _, pod := range pods {
if sidecarcontrol.IsPodSidecarUpdated(sidecarSet, pod) && !coreControl.IsPodConsistentAndReady(pod) {
upgradeAndNotReadyCount++
}
}
var needUpgradeCount int
for _, i := range waitUpdateIndexes {
// If pod is not ready, then not included in the calculation of maxUnavailable
if !coreControl.IsPodConsistentAndReady(pods[i]) {
needUpgradeCount++
continue
}
if upgradeAndNotReadyCount >= maxUnavailable {
break
}
upgradeAndNotReadyCount++
needUpgradeCount++
}
return needUpgradeCount
}
func parseUpdateScatterTerms(scatter appsv1alpha1.UpdateScatterStrategy, pods []*corev1.Pod) appsv1alpha1.UpdateScatterStrategy {
newScatter := appsv1alpha1.UpdateScatterStrategy{}
for _, term := range scatter {
if term.Value != "*" {
newScatter = insertUpdateScatterTerm(newScatter, term)
continue
}
// convert regular terms to scatter terms
// examples: labelA=* -> labelA=value1, labelA=value2...
newTerms := matchScatterTerms(pods, term.Key)
for _, obj := range newTerms {
newScatter = insertUpdateScatterTerm(newScatter, obj)
}
}
return newScatter
}
func insertUpdateScatterTerm(scatter appsv1alpha1.UpdateScatterStrategy, term appsv1alpha1.UpdateScatterTerm) appsv1alpha1.UpdateScatterStrategy {
for _, obj := range scatter {
//if term already exist, return
if term.Key == obj.Key && term.Value == obj.Value {
return scatter
}
}
scatter = append(scatter, term)
return scatter
}
// convert regular terms to scatter terms
func matchScatterTerms(pods []*corev1.Pod, regularLabel string) []appsv1alpha1.UpdateScatterTerm {
var terms []appsv1alpha1.UpdateScatterTerm
for _, pod := range pods {
labelValue, ok := pod.Labels[regularLabel]
if !ok {
continue
}
terms = append(terms, appsv1alpha1.UpdateScatterTerm{
Key: regularLabel,
Value: labelValue,
})
}
return terms
}

535
pkg/controller/sidecarset/sidecarset_strategy_test.go Normal file
View File

@ -0,0 +1,535 @@
/*
Copyright 2020 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sidecarset
import (
"encoding/json"
"fmt"
"math/rand"
"reflect"
"testing"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/control/sidecarcontrol"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
)
type FactorySidecarSet func() *appsv1alpha1.SidecarSet
type FactoryPods func(int, int, int) []*corev1.Pod
func factoryPodsCommon(count, upgraded int, sidecarSet *appsv1alpha1.SidecarSet) []*corev1.Pod {
control := sidecarcontrol.New(sidecarSet)
pods := make([]*corev1.Pod, 0, count)
for i := 0; i < count; i++ {
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
sidecarcontrol.SidecarSetHashAnnotation: `{"test-sidecarset":{"hash":"aaa"}}`,
},
Name: fmt.Sprintf("pod-%d", i),
Labels: map[string]string{
"app": "sidecar",
},
CreationTimestamp: metav1.Now(),
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "nginx",
Image: "nginx:1.15.1",
},
{
Name: "test-sidecar",
Image: "test-image:v1",
},
},
},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
Conditions: []corev1.PodCondition{
{
Type: corev1.PodReady,
Status: corev1.ConditionTrue,
},
},
ContainerStatuses: []corev1.ContainerStatus{
{
Name: "nginx",
Image: "nginx:1.15.1",
ImageID: "docker-pullable://nginx@sha256:a9286defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d",
Ready: true,
},
{
Name: "test-sidecar",
Image: "test-image:v1",
ImageID: testImageV1ImageID,
Ready: true,
},
},
},
}
pods = append(pods, pod)
}
for i := 0; i < upgraded; i++ {
pods[i].Spec.Containers[1].Image = "test-image:v2"
control.UpdatePodAnnotationsInUpgrade([]string{"test-sidecar"}, pods[i])
}
return pods
}
func factoryPods(count, upgraded, upgradedAndReady int) []*corev1.Pod {
sidecarSet := factorySidecarSet()
pods := factoryPodsCommon(count, upgraded, sidecarSet)
for i := 0; i < upgradedAndReady; i++ {
pods[i].Status.ContainerStatuses[1].Image = "test-image:v2"
pods[i].Status.ContainerStatuses[1].ImageID = testImageV2ImageID
}
return pods
}
func factorySidecarSet() *appsv1alpha1.SidecarSet {
sidecarSet := &appsv1alpha1.SidecarSet{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
sidecarcontrol.SidecarSetHashAnnotation: "bbb",
sidecarcontrol.SidecarSetHashWithoutImageAnnotation: "without-bbb",
},
Name: "test-sidecarset",
Labels: map[string]string{},
},
Spec: appsv1alpha1.SidecarSetSpec{
Containers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{
Name: "test-sidecar",
Image: "test-image:v2",
},
},
},
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": "sidecar"},
},
Strategy: appsv1alpha1.SidecarSetUpdateStrategy{
Type: appsv1alpha1.RollingUpdateSidecarSetStrategyType,
},
},
}
return sidecarSet
}
func TestGetNextUpgradePods(t *testing.T) {
testGetNextUpgradePods(t, factoryPods, factorySidecarSet)
}
func testGetNextUpgradePods(t *testing.T, factoryPods FactoryPods, factorySidecar FactorySidecarSet) {
cases := []struct {
name string
getPods func() []*corev1.Pod
getSidecarset func() *appsv1alpha1.SidecarSet
exceptNeedUpgradeCount int
}{
{
name: "only maxUnavailable(int=10), and pods(count=100, upgraded=30, upgradedAndReady=26)",
getPods: func() []*corev1.Pod {
pods := factoryPods(100, 30, 26)
return Random(pods)
},
getSidecarset: func() *appsv1alpha1.SidecarSet {
sidecarSet := factorySidecar()
sidecarSet.Spec.Strategy.MaxUnavailable = &intstr.IntOrString{
Type: intstr.Int,
IntVal: 10,
}
return sidecarSet
},
exceptNeedUpgradeCount: 6,
},
{
name: "only maxUnavailable(string=10%), and pods(count=1000, upgraded=300, upgradedAndReady=260)",
getPods: func() []*corev1.Pod {
pods := factoryPods(1000, 300, 260)
return Random(pods)
},
getSidecarset: func() *appsv1alpha1.SidecarSet {
sidecarSet := factorySidecar()
sidecarSet.Spec.Strategy.MaxUnavailable = &intstr.IntOrString{
Type: intstr.String,
StrVal: "10%",
}
return sidecarSet
},
exceptNeedUpgradeCount: 60,
},
{
name: "only maxUnavailable(string=5%), and pods(count=1000, upgraded=300, upgradedAndReady=250)",
getPods: func() []*corev1.Pod {
pods := factoryPods(1000, 300, 250)
return Random(pods)
},
getSidecarset: func() *appsv1alpha1.SidecarSet {
sidecarSet := factorySidecar()
sidecarSet.Spec.Strategy.MaxUnavailable = &intstr.IntOrString{
Type: intstr.String,
StrVal: "5%",
}
return sidecarSet
},
exceptNeedUpgradeCount: 0,
},
{
name: "only maxUnavailable(int=100), and pods(count=100, upgraded=30, upgradedAndReady=27)",
getPods: func() []*corev1.Pod {
pods := factoryPods(100, 30, 27)
return Random(pods)
},
getSidecarset: func() *appsv1alpha1.SidecarSet {
sidecarSet := factorySidecar()
sidecarSet.Spec.Strategy.MaxUnavailable = &intstr.IntOrString{
Type: intstr.Int,
IntVal: 100,
}
return sidecarSet
},
exceptNeedUpgradeCount: 70,
},
{
name: "partition(int=180) maxUnavailable(int=100), and pods(count=1000, upgraded=800, upgradedAndReady=760)",
getPods: func() []*corev1.Pod {
pods := factoryPods(1000, 800, 760)
return Random(pods)
},
getSidecarset: func() *appsv1alpha1.SidecarSet {
sidecarSet := factorySidecar()
sidecarSet.Spec.Strategy.MaxUnavailable = &intstr.IntOrString{
Type: intstr.Int,
IntVal: 100,
}
sidecarSet.Spec.Strategy.Partition = &intstr.IntOrString{
Type: intstr.Int,
IntVal: 180,
}
return sidecarSet
},
exceptNeedUpgradeCount: 20,
},
{
name: "partition(int=100) maxUnavailable(int=100), and pods(count=1000, upgraded=800, upgradedAndReady=760)",
getPods: func() []*corev1.Pod {
pods := factoryPods(1000, 800, 760)
return Random(pods)
},
getSidecarset: func() *appsv1alpha1.SidecarSet {
sidecarSet := factorySidecar()
sidecarSet.Spec.Strategy.MaxUnavailable = &intstr.IntOrString{
Type: intstr.Int,
IntVal: 100,
}
sidecarSet.Spec.Strategy.Partition = &intstr.IntOrString{
Type: intstr.Int,
IntVal: 100,
}
return sidecarSet
},
exceptNeedUpgradeCount: 60,
},
{
name: "partition(string=18%) maxUnavailable(int=100), and pods(count=1000, upgraded=800, upgradedAndReady=760)",
getPods: func() []*corev1.Pod {
pods := factoryPods(1000, 800, 760)
return Random(pods)
},
getSidecarset: func() *appsv1alpha1.SidecarSet {
sidecarSet := factorySidecar()
sidecarSet.Spec.Strategy.MaxUnavailable = &intstr.IntOrString{
Type: intstr.Int,
IntVal: 100,
}
sidecarSet.Spec.Strategy.Partition = &intstr.IntOrString{
Type: intstr.String,
StrVal: "18%",
}
return sidecarSet
},
exceptNeedUpgradeCount: 20,
},
{
name: "partition(string=10%) maxUnavailable(int=100), and pods(count=1000, upgraded=800, upgradedAndReady=760)",
getPods: func() []*corev1.Pod {
pods := factoryPods(1000, 800, 760)
return Random(pods)
},
getSidecarset: func() *appsv1alpha1.SidecarSet {
sidecarSet := factorySidecar()
sidecarSet.Spec.Strategy.MaxUnavailable = &intstr.IntOrString{
Type: intstr.Int,
IntVal: 100,
}
sidecarSet.Spec.Strategy.Partition = &intstr.IntOrString{
Type: intstr.String,
StrVal: "10%",
}
return sidecarSet
},
exceptNeedUpgradeCount: 60,
},
{
name: "selector(app=test, count=30) maxUnavailable(int=100), and pods(count=1000, upgraded=0, upgradedAndReady=0)",
getPods: func() []*corev1.Pod {
pods := factoryPods(1000, 0, 0)
for i := 0; i < 30; i++ {
pods[i].Labels["app"] = "test"
}
return Random(pods)
},
getSidecarset: func() *appsv1alpha1.SidecarSet {
sidecarSet := factorySidecar()
sidecarSet.Spec.Strategy.MaxUnavailable = &intstr.IntOrString{
Type: intstr.Int,
IntVal: 100,
}
sidecarSet.Spec.Strategy.Selector = &metav1.LabelSelector{
MatchLabels: map[string]string{"app": "test"},
}
return sidecarSet
},
exceptNeedUpgradeCount: 30,
},
}
strategy := NewStrategy()
for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) {
control := sidecarcontrol.New(cs.getSidecarset())
pods := cs.getPods()
injectedPods := strategy.GetNextUpgradePods(control, pods)
if cs.exceptNeedUpgradeCount != len(injectedPods) {
t.Fatalf("except NeedUpgradeCount(%d), but get value(%d)", cs.exceptNeedUpgradeCount, len(injectedPods))
}
})
}
}
func TestParseUpdateScatterTerms(t *testing.T) {
cases := []struct {
name string
getPods func() []*corev1.Pod
getScatterStrategy func() appsv1alpha1.UpdateScatterStrategy
exceptScatterStrategy func() appsv1alpha1.UpdateScatterStrategy
}{
{
name: "only scatter terms",
getPods: func() []*corev1.Pod {
pods := factoryPods(100, 0, 0)
return pods
},
getScatterStrategy: func() appsv1alpha1.UpdateScatterStrategy {
scatter := appsv1alpha1.UpdateScatterStrategy{
{
Key: "key-1",
Value: "value-1",
},
{
Key: "key-2",
Value: "value-2",
},
{
Key: "key-3",
Value: "value-3",
},
}
return scatter
},
exceptScatterStrategy: func() appsv1alpha1.UpdateScatterStrategy {
scatter := appsv1alpha1.UpdateScatterStrategy{
{
Key: "key-1",
Value: "value-1",
},
{
Key: "key-2",
Value: "value-2",
},
{
Key: "key-3",
Value: "value-3",
},
}
return scatter
},
},
{
name: "regular and scatter terms",
getPods: func() []*corev1.Pod {
pods := factoryPods(100, 0, 0)
pods[0].Labels["key-4"] = "value-4-0"
pods[1].Labels["key-4"] = "value-4-1"
pods[2].Labels["key-4"] = "value-4-2"
pods[3].Labels["key-4"] = "value-4"
pods[4].Labels["key-4"] = "value-4"
pods[5].Labels["key-4"] = "value-4"
return pods
},
getScatterStrategy: func() appsv1alpha1.UpdateScatterStrategy {
scatter := appsv1alpha1.UpdateScatterStrategy{
{
Key: "key-1",
Value: "value-1",
},
{
Key: "key-2",
Value: "value-2",
},
{
Key: "key-3",
Value: "value-3",
},
{
Key: "key-4",
Value: "*",
},
}
return scatter
},
exceptScatterStrategy: func() appsv1alpha1.UpdateScatterStrategy {
scatter := appsv1alpha1.UpdateScatterStrategy{
{
Key: "key-1",
Value: "value-1",
},
{
Key: "key-2",
Value: "value-2",
},
{
Key: "key-3",
Value: "value-3",
},
{
Key: "key-4",
Value: "value-4-0",
},
{
Key: "key-4",
Value: "value-4-1",
},
{
Key: "key-4",
Value: "value-4-2",
},
{
Key: "key-4",
Value: "value-4",
},
}
return scatter
},
},
}
for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) {
pods := cs.getPods()
scatter := cs.getScatterStrategy()
exceptScatter := cs.exceptScatterStrategy()
newScatter := parseUpdateScatterTerms(scatter, pods)
if !reflect.DeepEqual(newScatter, exceptScatter) {
except, _ := json.Marshal(exceptScatter)
new, _ := json.Marshal(newScatter)
t.Fatalf("except scatter(%s), but get scatter(%s)", string(except), string(new))
}
})
}
}
func Random(pods []*corev1.Pod) []*corev1.Pod {
for i := len(pods) - 1; i > 0; i-- {
num := rand.Intn(i + 1)
pods[i], pods[num] = pods[num], pods[i]
}
return pods
}
func TestSortNextUpgradePods(t *testing.T) {
testSortNextUpgradePods(t, factoryPods, factorySidecarSet)
}
func testSortNextUpgradePods(t *testing.T, factoryPods FactoryPods, factorySidecar FactorySidecarSet) {
cases := []struct {
name string
getPods func() []*corev1.Pod
getSidecarset func() *appsv1alpha1.SidecarSet
exceptNextUpgradePods []string
}{
{
name: "sort by pod.CreationTimestamp, maxUnavailable(int=10) and pods(count=20, upgraded=10, upgradedAndReady=5)",
getPods: func() []*corev1.Pod {
pods := factoryPods(20, 10, 5)
return Random(pods)
},
getSidecarset: func() *appsv1alpha1.SidecarSet {
sidecarSet := factorySidecar()
sidecarSet.Spec.Strategy.MaxUnavailable = &intstr.IntOrString{
Type: intstr.Int,
IntVal: 10,
}
return sidecarSet
},
exceptNextUpgradePods: []string{"pod-19", "pod-18", "pod-17", "pod-16", "pod-15"},
},
{
name: "not ready priority, maxUnavailable(int=10) and pods(count=20, upgraded=10, upgradedAndReady=5)",
getPods: func() []*corev1.Pod {
pods := factoryPods(20, 10, 5)
podutil.GetPodReadyCondition(pods[10].Status).Status = corev1.ConditionFalse
podutil.GetPodReadyCondition(pods[13].Status).Status = corev1.ConditionFalse
return Random(pods)
},
getSidecarset: func() *appsv1alpha1.SidecarSet {
sidecarSet := factorySidecar()
sidecarSet.Spec.Strategy.MaxUnavailable = &intstr.IntOrString{
Type: intstr.Int,
IntVal: 10,
}
return sidecarSet
},
exceptNextUpgradePods: []string{"pod-13", "pod-10", "pod-19", "pod-18", "pod-17", "pod-16", "pod-15"},
},
}
strategy := NewStrategy()
for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) {
control := sidecarcontrol.New(cs.getSidecarset())
pods := cs.getPods()
injectedPods := strategy.GetNextUpgradePods(control, pods)
if len(cs.exceptNextUpgradePods) != len(injectedPods) {
t.Fatalf("except NeedUpgradeCount(%d), but get value(%d)", len(cs.exceptNextUpgradePods), len(injectedPods))
}
for i, name := range cs.exceptNextUpgradePods {
if injectedPods[i].Name != name {
t.Fatalf("except NextUpgradePods[%d:%s], but get pods[%s]", i, name, injectedPods[i])
}
}
})
}
}

313
pkg/controller/sidecarset/sidecarset_utils.go
View File

@ -1,318 +1,9 @@
package sidecarset
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
intstrutil "k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/util/retry"
"k8s.io/klog"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
podmutating "github.com/openkruise/kruise/pkg/webhook/pod/mutating"
sidecarsetmutating "github.com/openkruise/kruise/pkg/webhook/sidecarset/mutating"
)
var (
updateCache = &updatedPodCache{
podSidecarUpdated: make(map[string]string),
}
)
func isIgnoredPod(pod *corev1.Pod) bool {
for _, namespace := range podmutating.SidecarIgnoredNamespaces {
if pod.Namespace == namespace {
return true
}
}
return false
}
func calculateStatus(sidecarSet *appsv1alpha1.SidecarSet, pods []*corev1.Pod) (*appsv1alpha1.SidecarSetStatus, error) {
var matchedPods, updatedPods, readyPods int32
matchedPods = int32(len(pods))
for _, pod := range pods {
updated, err := isPodSidecarUpdated(sidecarSet, pod)
if err != nil {
return nil, err
}
if updated {
updatedPods++
}
if isRunningAndReady(pod) {
readyPods++
}
}
return &appsv1alpha1.SidecarSetStatus{
ObservedGeneration: sidecarSet.Generation,
MatchedPods: matchedPods,
UpdatedPods: updatedPods,
ReadyPods: readyPods,
}, nil
}
func isPodSidecarUpdated(sidecarSet *appsv1alpha1.SidecarSet, pod *corev1.Pod) (bool, error) {
hashKey := sidecarsetmutating.SidecarSetHashAnnotation
if pod.Annotations[hashKey] == "" {
return false, nil
}
sidecarSetHash := make(map[string]string)
if err := json.Unmarshal([]byte(pod.Annotations[hashKey]), &sidecarSetHash); err != nil {
return false, err
}
return sidecarSetHash[sidecarSet.Name] == sidecarSet.Annotations[hashKey], nil
}
func isRunningAndReady(pod *corev1.Pod) bool {
return pod.Status.Phase == corev1.PodRunning && podutil.IsPodReady(pod)
}
func (r *ReconcileSidecarSet) updateSidecarSetStatus(sidecarSet *appsv1alpha1.SidecarSet, status *appsv1alpha1.SidecarSetStatus) error {
if !inconsistentStatus(sidecarSet, status) {
return nil
}
sidecarSetClone := sidecarSet.DeepCopy()
err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
sidecarSetClone.Status = *status
updateErr := r.Status().Update(context.TODO(), sidecarSetClone)
if updateErr == nil {
return nil
}
key := types.NamespacedName{
Name: sidecarSetClone.Name,
}
if err := r.Get(context.TODO(), key, sidecarSetClone); err != nil {
klog.Errorf("error getting updated sidecarset %s from client", sidecarSetClone.Name)
}
return updateErr
})
return err
}
func inconsistentStatus(sidecarSet *appsv1alpha1.SidecarSet, status *appsv1alpha1.SidecarSetStatus) bool {
return status.ObservedGeneration > sidecarSet.Status.ObservedGeneration ||
status.MatchedPods != sidecarSet.Status.MatchedPods ||
status.UpdatedPods != sidecarSet.Status.UpdatedPods ||
status.ReadyPods != sidecarSet.Status.ReadyPods
}
// add this cache to avoid be influenced by informer cache latency when controller try to count maxUnavailable
type updatedPodCache struct {
lock sync.RWMutex
// key is in the format: sidecarset name/pod namespace/pod name
// value is sidecarset hash
podSidecarUpdated map[string]string
}
func (u *updatedPodCache) set(key, hash string) {
u.lock.Lock()
u.podSidecarUpdated[key] = hash
u.lock.Unlock()
}
func (u *updatedPodCache) delete(key string) {
u.lock.Lock()
delete(u.podSidecarUpdated, key)
u.lock.Unlock()
}
// return true means: sidecar of pod is updated
func (u *updatedPodCache) isSidecarUpdated(sidecarSet *appsv1alpha1.SidecarSet, pod *corev1.Pod) bool {
key := fmt.Sprintf("%v/%v/%v", sidecarSet.Name, pod.Namespace, pod.Name)
u.lock.RLock()
hash := u.podSidecarUpdated[key]
u.lock.RUnlock()
if hash == sidecarSet.Annotations[sidecarsetmutating.SidecarSetHashAnnotation] {
return true
}
// if sidecarset changed, clean all stale cache related with this sidecarset
if hash != "" {
u.reset(sidecarSet)
}
return false
}
func (u *updatedPodCache) reset(sidecarSet *appsv1alpha1.SidecarSet) {
prefix := fmt.Sprintf("%v/", sidecarSet.Name)
u.lock.Lock()
for key := range u.podSidecarUpdated {
if strings.HasPrefix(key, prefix) {
delete(u.podSidecarUpdated, key)
}
}
u.lock.Unlock()
}
// available definition:
// 1. image in pod.spec and pod.status is exactly the same
// 2. pod is ready
func getUnavailableNumber(sidecarSet *appsv1alpha1.SidecarSet, pods []*corev1.Pod) (int, error) {
var unavailableNum int
for _, pod := range pods {
// in case of informer cache latency
key := fmt.Sprintf("%v/%v/%v", sidecarSet.Name, pod.Namespace, pod.Name)
podInCacheUpdated := updateCache.isSidecarUpdated(sidecarSet, pod)
podInInformerUpdated, err := isPodSidecarUpdated(sidecarSet, pod)
if err != nil {
return 0, err
}
if podInCacheUpdated && !podInInformerUpdated {
unavailableNum++
continue
}
if podInCacheUpdated && podInInformerUpdated {
updateCache.delete(key)
}
if !isPodImageConsistent(pod) {
unavailableNum++
continue
}
if !isRunningAndReady(pod) {
unavailableNum++
}
}
return unavailableNum, nil
}
func isPodCreatedBeforeSidecarSet(sidecarSet *appsv1alpha1.SidecarSet, pod *corev1.Pod) (bool, error) {
hashKey := sidecarsetmutating.SidecarSetHashAnnotation
if pod.Annotations[hashKey] == "" {
return true, nil
}
sidecarSetHash := make(map[string]string)
if err := json.Unmarshal([]byte(pod.Annotations[hashKey]), &sidecarSetHash); err != nil {
return false, err
}
if _, ok := sidecarSetHash[sidecarSet.Name]; !ok {
return true, nil
}
return false, nil
}
// check if fields other than sidecar image had changed
func otherFieldsInSidecarChanged(sidecarSet *appsv1alpha1.SidecarSet, pod *corev1.Pod) (bool, error) {
hashKey := sidecarsetmutating.SidecarSetHashWithoutImageAnnotation
if pod.Annotations[hashKey] == "" {
return false, nil
}
sidecarSetHash := make(map[string]string)
if err := json.Unmarshal([]byte(pod.Annotations[hashKey]), &sidecarSetHash); err != nil {
return false, err
}
return sidecarSetHash[sidecarSet.Name] != sidecarSet.Annotations[hashKey], nil
}
func isPodImageConsistent(pod *corev1.Pod) bool {
containerSpecImage := make(map[string]string, len(pod.Spec.Containers))
for _, container := range pod.Spec.Containers {
containerSpecImage[container.Name] = container.Image
}
for _, containerStatus := range pod.Status.ContainerStatuses {
if containerSpecImage[containerStatus.Name] != containerStatus.Image {
return false
}
}
return true
}
func (r *ReconcileSidecarSet) updateSidecarImageAndHash(sidecarSet *appsv1alpha1.SidecarSet, pods []*corev1.Pod, updateNum int) error {
if len(pods) < updateNum {
updateNum = len(pods)
}
for i := 0; i < updateNum; i++ {
klog.V(3).Infof("try to update sidecar of %v/%v", pods[i].Namespace, pods[i].Name)
if err := r.updatePodSidecarAndHash(sidecarSet, pods[i]); err != nil {
return err
}
updateCache.set(
fmt.Sprintf("%v/%v/%v", sidecarSet.Name, pods[i].Namespace, pods[i].Name),
sidecarSet.Annotations[sidecarsetmutating.SidecarSetHashAnnotation])
}
return nil
}
func (r *ReconcileSidecarSet) updatePodSidecarAndHash(sidecarSet *appsv1alpha1.SidecarSet, pod *corev1.Pod) error {
podClone := pod.DeepCopy()
err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
// update sidecar image
updatePodSidecar(sidecarSet, podClone)
// update hash
hashKey := sidecarsetmutating.SidecarSetHashAnnotation
sidecarSetHash := make(map[string]string)
if err := json.Unmarshal([]byte(podClone.Annotations[hashKey]), &sidecarSetHash); err != nil {
return err
}
sidecarSetHash[sidecarSet.Name] = sidecarSet.Annotations[hashKey]
newHash, err := json.Marshal(sidecarSetHash)
if err != nil {
return err
}
podClone.Annotations[hashKey] = string(newHash)
updateErr := r.Update(context.TODO(), podClone)
if updateErr == nil {
return nil
}
key := types.NamespacedName{
Namespace: podClone.Namespace,
Name: podClone.Name,
}
if err := r.Get(context.TODO(), key, podClone); err != nil {
klog.Errorf("error getting updated pod %s from client", sidecarSet.Name)
}
return updateErr
})
return err
}
func updatePodSidecar(sidecarSet *appsv1alpha1.SidecarSet, pod *corev1.Pod) {
sidecarImage := make(map[string]string, len(sidecarSet.Spec.Containers))
for _, container := range sidecarSet.Spec.Containers {
sidecarImage[container.Name] = container.Image
}
for i := range pod.Spec.Containers {
container := &pod.Spec.Containers[i]
if image, ok := sidecarImage[container.Name]; ok {
container.Image = image
}
}
}
func getMaxUnavailable(sidecarSet *appsv1alpha1.SidecarSet) int {
// Error caught by validation
unavailable, _ := intstrutil.GetValueFromIntOrPercent(
intstrutil.ValueOrDefault(sidecarSet.Spec.Strategy.RollingUpdate.MaxUnavailable, intstrutil.FromInt(0)),
int(sidecarSet.Status.MatchedPods),
false)
return unavailable
func isSidecarSetUpdateFinish(status *appsv1alpha1.SidecarSetStatus) bool {
return status.UpdatedPods >= status.MatchedPods
}

2
pkg/controller/statefulset/statefulset_controller.go
View File

@ -65,7 +65,7 @@ var (
controllerKind = appsv1beta1.SchemeGroupVersion.WithKind("StatefulSet")
concurrentReconciles = 3
updateExpectations = expectations.NewUpdateExpectations(func(o metav1.Object) string {
updateExpectations = expectations.NewUpdateExpectations(func(controllerKey string, o metav1.Object) string {
p := o.(*v1.Pod)
return getPodRevision(p)
})

6
pkg/util/expectations/update_expectations.go
View File

@ -34,7 +34,7 @@ type UpdateExpectations interface {
}
// NewUpdateExpectations returns a common UpdateExpectations.
func NewUpdateExpectations(getRevision func(metav1.Object) string) UpdateExpectations {
func NewUpdateExpectations(getRevision func(string, metav1.Object) string) UpdateExpectations {
return &realUpdateExpectations{
controllerCache: make(map[string]*realControllerUpdateExpectations),
getRevision: getRevision,
@ -46,7 +46,7 @@ type realUpdateExpectations struct {
// key: parent key, workload namespace/name
controllerCache map[string]*realControllerUpdateExpectations
// how to get pod revision
getRevision func(metav1.Object) string
getRevision func(string, metav1.Object) string
}
type realControllerUpdateExpectations struct {
@ -82,7 +82,7 @@ func (r *realUpdateExpectations) ObserveUpdated(controllerKey, revision string,
return
}
if expectations.revision == revision && expectations.objsUpdated.Has(getKey(obj)) && r.getRevision(obj) == revision {
if expectations.revision == revision && expectations.objsUpdated.Has(getKey(obj)) && r.getRevision(controllerKey, obj) == revision {
expectations.objsUpdated.Delete(getKey(obj))
}

2
pkg/util/expectations/update_expectations_test.go
View File

@ -35,7 +35,7 @@ func TestUpdate(t *testing.T) {
},
},
}
c := NewUpdateExpectations(func(p metav1.Object) string { return p.GetLabels()["revision"] })
c := NewUpdateExpectations(func(controllerKey string, p metav1.Object) string { return p.GetLabels()["revision"] })
// no pod in cache
if satisfied, _, _ := c.SatisfiedExpectations(controllerKey, revisions[0]); !satisfied {

69
pkg/util/json_test.go Normal file
View File

@ -0,0 +1,69 @@
/*
Copyright 2020 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"testing"
v1 "k8s.io/api/core/v1"
)
func TestDumpJson(t *testing.T) {
object := &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "nginx",
Image: "nginx:1.15.1",
Env: []v1.EnvVar{
{
Name: "nginx-env",
Value: "value-1",
},
},
VolumeMounts: []v1.VolumeMount{
{
Name: "nginx-volume",
MountPath: "/data/nginx",
},
},
},
{
Name: "test-sidecar",
Image: "test-image:v1",
Env: []v1.EnvVar{
{
Name: "IS_INJECTED",
Value: "true",
},
},
},
},
Volumes: []v1.Volume{
{
Name: "nginx-volume",
},
},
},
}
except := `{"metadata":{"creationTimestamp":null},"spec":{"volumes":[{"name":"nginx-volume"}],"containers":[{"name":"nginx","image":"nginx:1.15.1","env":[{"name":"nginx-env","value":"value-1"}],"resources":{},"volumeMounts":[{"name":"nginx-volume","mountPath":"/data/nginx"}]},{"name":"test-sidecar","image":"test-image:v1","env":[{"name":"IS_INJECTED","value":"true"}],"resources":{}}]},"status":{}}`
if except != DumpJSON(object) {
t.Errorf("expect %v but got %v", except, DumpJSON(object))
}
}

169
pkg/util/pods.go
View File

@ -17,8 +17,11 @@ limitations under the License.
package util
import (
"strings"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/sets"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
)
// GetPodNames returns names of the given Pods array
@ -49,3 +52,169 @@ func MergePods(pods1, pods2 []*v1.Pod) []*v1.Pod {
}
return ret
}
func MergeVolumeMounts(original, additional []v1.VolumeMount) []v1.VolumeMount {
mountpoints := sets.NewString()
for _, mount := range original {
mountpoints.Insert(mount.MountPath)
}
for _, mount := range additional {
if mountpoints.Has(mount.MountPath) {
continue
}
original = append(original, mount)
mountpoints.Insert(mount.MountPath)
}
return original
}
func MergeEnvVar(original []v1.EnvVar, additional []v1.EnvVar) []v1.EnvVar {
exists := sets.NewString()
for _, env := range original {
exists.Insert(env.Name)
}
for _, env := range additional {
if exists.Has(env.Name) {
continue
}
original = append(original, env)
exists.Insert(env.Name)
}
return original
}
func MergeVolumes(original []v1.Volume, additional []v1.Volume) []v1.Volume {
exists := sets.NewString()
for _, volume := range original {
exists.Insert(volume.Name)
}
for _, volume := range additional {
if exists.Has(volume.Name) {
continue
}
original = append(original, volume)
exists.Insert(volume.Name)
}
return original
}
func GetContainerEnvVar(container *v1.Container, key string) *v1.EnvVar {
if container == nil {
return nil
}
for i, e := range container.Env {
if e.Name == key {
return &container.Env[i]
}
}
return nil
}
func GetContainerEnvValue(container *v1.Container, key string) string {
if container == nil {
return ""
}
for i, e := range container.Env {
if e.Name == key {
return container.Env[i].Value
}
}
return ""
}
func GetContainerVolumeMount(container *v1.Container, key string) *v1.VolumeMount {
if container == nil {
return nil
}
for i, m := range container.VolumeMounts {
if m.MountPath == key {
return &container.VolumeMounts[i]
}
}
return nil
}
func GetContainer(name string, pod *v1.Pod) *v1.Container {
if pod == nil {
return nil
}
for i := range pod.Spec.InitContainers {
v := &pod.Spec.InitContainers[i]
if v.Name == name {
return v
}
}
for i := range pod.Spec.Containers {
v := &pod.Spec.Containers[i]
if v.Name == name {
return v
}
}
return nil
}
func GetPodVolume(pod *v1.Pod, volumeName string) *v1.Volume {
for idx, v := range pod.Spec.Volumes {
if v.Name == volumeName {
return &pod.Spec.Volumes[idx]
}
}
return nil
}
func IsRunningAndReady(pod *v1.Pod) bool {
return pod.Status.Phase == v1.PodRunning && podutil.IsPodReady(pod)
}
func IsPodContainerDigestEqual(containers sets.String, pod *v1.Pod) bool {
cStatus := make(map[string]string, len(pod.Status.ContainerStatuses))
for i := range pod.Status.ContainerStatuses {
c := &pod.Status.ContainerStatuses[i]
//ImageID format: docker-pullable://busybox@sha256:a9286defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d
imageID := c.ImageID
if strings.Contains(imageID, "://") {
imageID = strings.Split(imageID, "://")[1]
}
cStatus[c.Name] = imageID
}
for _, container := range pod.Spec.Containers {
if !containers.Has(container.Name) {
continue
}
// image must be digest format
if !IsImageDigest(container.Image) {
return false
}
imageID, ok := cStatus[container.Name]
if !ok {
return false
}
if !IsContainerImageEqual(container.Image, imageID) {
return false
}
}
return true
}
func MergeVolumeMountsInContainer(origin *v1.Container, other v1.Container) {
mountExist := make(map[string]bool)
for _, volume := range origin.VolumeMounts {
mountExist[volume.MountPath] = true
}
for _, volume := range other.VolumeMounts {
if mountExist[volume.MountPath] {
continue
}
origin.VolumeMounts = append(origin.VolumeMounts, volume)
}
}

104
pkg/util/pods_test.go Normal file
View File

@ -0,0 +1,104 @@
/*
Copyright 2020 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"testing"
v1 "k8s.io/api/core/v1"
)
func TestMergeVolumeMounts(t *testing.T) {
original := []v1.VolumeMount{
{
MountPath: "/origin-1",
},
{
MountPath: "/share",
},
}
additional := []v1.VolumeMount{
{
MountPath: "/addition-1",
},
{
MountPath: "/share",
},
}
volumeMounts := MergeVolumeMounts(original, additional)
excepts := []string{"/origin-1", "/share", "/addition-1"}
for i, except := range excepts {
if volumeMounts[i].MountPath != except {
t.Fatalf("except VolumeMount(%s), but get %s", except, volumeMounts[i].MountPath)
}
}
}
func TestMergeEnvVars(t *testing.T) {
original := []v1.EnvVar{
{
Name: "origin-1",
},
{
Name: "share",
},
}
additional := []v1.EnvVar{
{
Name: "addition-1",
},
{
Name: "share",
},
}
envVars := MergeEnvVar(original, additional)
excepts := []string{"origin-1", "share", "addition-1"}
for i, except := range excepts {
if envVars[i].Name != except {
t.Fatalf("except EnvVar(%s), but get %s", except, envVars[i].Name)
}
}
}
func TestMergeVolumes(t *testing.T) {
original := []v1.Volume{
{
Name: "origin-1",
},
{
Name: "share",
},
}
additional := []v1.Volume{
{
Name: "addition-1",
},
{
Name: "share",
},
}
volumes := MergeVolumes(original, additional)
excepts := []string{"origin-1", "share", "addition-1"}
for i, except := range excepts {
if volumes[i].Name != except {
t.Fatalf("except EnvVar(%s), but get %s", except, volumes[i].Name)
}
}
}

126
pkg/util/selector.go Normal file
View File

@ -0,0 +1,126 @@
/*
Copyright 2020 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/kubernetes/pkg/util/slice"
)
// whether selector overlaps, the criteria:
// if exist one same key has different value and not overlap, then it is judged non-overlap, for examples:
// * a=b and a=c
// * a in [b,c] and a not in [b,c...]
// * a not in [b] and a not exist
// * a=b,c=d,e=f and a=x,c=d,e=f
// then others is overlap：
// * a=b and c=d
func IsSelectorOverlapping(selector1, selector2 *metav1.LabelSelector) bool {
return !(isDisjoint(selector1, selector2) || isDisjoint(selector2, selector1))
}
func isDisjoint(selector1, selector2 *metav1.LabelSelector) bool {
// label -> values
// a=b convert to a -> [b]
// a in [b,c] convert to a -> [b,c]
// a exist convert to a -> [ALL]
matchedLabels1 := make(map[string][]string)
for key, value := range selector1.MatchLabels {
matchedLabels1[key] = []string{value}
}
for _, req := range selector1.MatchExpressions {
switch req.Operator {
case metav1.LabelSelectorOpIn:
for _, value := range req.Values {
matchedLabels1[req.Key] = append(matchedLabels1[req.Key], value)
}
case metav1.LabelSelectorOpExists:
matchedLabels1[req.Key] = []string{"ALL"}
}
}
for key, value := range selector2.MatchLabels {
values, ok := matchedLabels1[key]
if ok {
if !slice.ContainsString(values, "ALL", nil) && !slice.ContainsString(values, value, nil) {
return true
}
}
}
for _, req := range selector2.MatchExpressions {
values, ok := matchedLabels1[req.Key]
switch req.Operator {
case metav1.LabelSelectorOpIn:
if ok && !slice.ContainsString(values, "ALL", nil) && !sliceOverlaps(values, req.Values) {
return true
}
case metav1.LabelSelectorOpNotIn:
if ok && sliceContains(req.Values, values) {
return true
}
case metav1.LabelSelectorOpExists:
if !ok {
return true
}
case metav1.LabelSelectorOpDoesNotExist:
if ok {
return true
}
}
}
return false
}
func sliceOverlaps(a, b []string) bool {
keyExist := make(map[string]bool, len(a))
for _, key := range a {
keyExist[key] = true
}
for _, key := range b {
if keyExist[key] {
return true
}
}
return false
}
// a contains b
func sliceContains(a, b []string) bool {
keyExist := make(map[string]bool, len(a))
for _, key := range a {
keyExist[key] = true
}
for _, key := range b {
if !keyExist[key] {
return false
}
}
return true
}
func GetFastLabelSelector(ps *metav1.LabelSelector) (labels.Selector, error) {
var selector labels.Selector
if len(ps.MatchExpressions) == 0 && len(ps.MatchLabels) != 0 {
selector = labels.SelectorFromValidatedSet(ps.MatchLabels)
return selector, nil
}
return metav1.LabelSelectorAsSelector(ps)
}

175
pkg/util/selector_test.go Normal file
View File

@ -0,0 +1,175 @@
/*
Copyright 2020 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"testing"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
type TestCase struct {
Input [2]metav1.LabelSelector
Output bool
}
func TestSelectorConflict(t *testing.T) {
testCases := []TestCase{
{
Input: [2]metav1.LabelSelector{
{
MatchLabels: map[string]string{"a": "h"},
},
{
MatchLabels: map[string]string{"a": "h"},
},
},
Output: true,
},
{
Input: [2]metav1.LabelSelector{
{
MatchLabels: map[string]string{"a": "h"},
},
{
MatchLabels: map[string]string{"a": "i"},
},
},
Output: false,
},
{
Input: [2]metav1.LabelSelector{
{
MatchLabels: map[string]string{"a": "h"},
},
{
MatchLabels: map[string]string{"b": "i"},
},
},
Output: true,
},
{
Input: [2]metav1.LabelSelector{
{
MatchLabels: map[string]string{
"a": "h",
"b": "i",
"c": "j",
},
},
{
MatchLabels: map[string]string{
"a": "h",
"b": "x",
"c": "j",
},
},
},
Output: false,
},
{
Input: [2]metav1.LabelSelector{
{
MatchLabels: map[string]string{"a": "h"},
},
{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "a",
Operator: metav1.LabelSelectorOpIn,
Values: []string{"h", "i", "j"},
},
},
},
},
Output: true,
},
{
Input: [2]metav1.LabelSelector{
{
MatchLabels: map[string]string{"a": "h"},
},
{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "a",
Operator: metav1.LabelSelectorOpIn,
Values: []string{"i", "j"},
},
},
},
},
Output: false,
},
{
Input: [2]metav1.LabelSelector{
{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "a",
Operator: metav1.LabelSelectorOpNotIn,
Values: []string{"h", "i"},
},
},
},
{
MatchLabels: map[string]string{"a": "h"},
},
},
Output: false,
},
{
Input: [2]metav1.LabelSelector{
{
MatchLabels: map[string]string{"a": "h"},
},
{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "a",
Operator: metav1.LabelSelectorOpExists,
},
},
},
},
Output: true,
},
{
Input: [2]metav1.LabelSelector{
{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "a",
Operator: metav1.LabelSelectorOpDoesNotExist,
},
},
},
{
MatchLabels: map[string]string{"a": "h"},
},
},
Output: false,
},
}
for i, testCase := range testCases {
output := IsSelectorOverlapping(&testCase.Input[0], &testCase.Input[1])
if output != testCase.Output {
t.Errorf("%v: expect %v but got %v", i, testCase.Output, output)
}
}
}

55
pkg/util/tools.go
View File

@ -20,7 +20,9 @@ package util
import (
"sync"
"github.com/docker/distribution/reference"
intstrutil "k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/klog"
"k8s.io/utils/integer"
)
@ -80,3 +82,56 @@ func CheckDuplicate(list []string) []string {
func GetIntOrStrPointer(i intstrutil.IntOrString) *intstrutil.IntOrString {
return &i
}
// parse container images,
// 1. docker.io/busybox@sha256:a9286defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d
// repo=docker.io/busybox, tag="", digest=sha256:a9286defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d
// 2. docker.io/busybox:latest
// repo=docker.io/busybox, tag=latest, digest=""
func ParseImage(image string) (repo, tag, digest string, err error) {
refer, err := reference.Parse(image)
if err != nil {
return "", "", "", err
}
if named, ok := refer.(reference.Named); ok {
repo = named.Name()
}
if tagged, ok := refer.(reference.Tagged); ok {
tag = tagged.Tag()
}
if digested, ok := refer.(reference.Digested); ok {
digest = digested.Digest().String()
}
return
}
//whether image is digest format,
//for example: docker.io/busybox@sha256:a9286defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d
func IsImageDigest(image string) bool {
_, _, digest, _ := ParseImage(image)
return digest != ""
}
// 1. image1, image2 are digest image, compare repo+digest
// 2. image1, image2 are normal image, compare repo+tag
// 3. image1, image2 are digest+normal image, don't support compare it, return false
func IsContainerImageEqual(image1, image2 string) bool {
repo1, tag1, digest1, err := ParseImage(image1)
if err != nil {
klog.Errorf("parse image %s failed: %s", image1, err.Error())
return false
}
repo2, tag2, digest2, err := ParseImage(image2)
if err != nil {
klog.Errorf("parse image %s failed: %s", image2, err.Error())
return false
}
if IsImageDigest(image1) && IsImageDigest(image2) {
return repo1 == repo2 && digest1 == digest2
}
return repo1 == repo2 && tag1 == tag2
}

62
pkg/util/tools_test.go
View File

@ -92,3 +92,65 @@ func TestSlowStartBatch(t *testing.T) {
}
}
}
func TestIsContainerImageEqual(t *testing.T) {
cases := []struct {
name string
images [2]string
equal bool
exceptions map[string][3]string
}{
{
name: "image tag and equal",
images: [2]string{"docker.io/busybox:v1", "docker.io/busybox:v1"},
equal: true,
exceptions: map[string][3]string{
"docker.io/busybox:v1": {"docker.io/busybox", "v1", ""},
},
},
{
name: "image tag and not equal",
images: [2]string{"docker.io/busybox:v1", "docker.io/busybox:v2"},
equal: false,
exceptions: map[string][3]string{
"docker.io/busybox:v1": {"docker.io/busybox", "v1", ""},
"docker.io/busybox:v2": {"docker.io/busybox", "v2", ""},
},
},
{
name: "image digest and equal",
images: [2]string{"docker.io/busybox@sha256:a9286defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d", "docker.io/busybox@sha256:a9286defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d"},
equal: true,
exceptions: map[string][3]string{
"docker.io/busybox@sha256:a9286defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d": {"docker.io/busybox", "", "sha256:a9286defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d"},
},
},
{
name: "image digest and not equal",
images: [2]string{"docker.io/busybox@sha256:a9286defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d", "docker.io/busybox@sha256:a2d86defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d"},
equal: false,
exceptions: map[string][3]string{
"docker.io/busybox@sha256:a9286defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d": {"docker.io/busybox", "", "sha256:a9286defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d"},
"docker.io/busybox@sha256:a2d86defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d": {"docker.io/busybox", "", "sha256:a2d86defaba7b3a519d585ba0e37d0b2cbee74ebfe590960b0b1d6a5e97d1e1d"},
},
},
}
for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) {
if cs.equal != IsContainerImageEqual(cs.images[0], cs.images[1]) {
t.Fatalf("except %t, but get %t", cs.equal, IsContainerImageEqual(cs.images[0], cs.images[1]))
}
for image, excepts := range cs.exceptions {
repo, tag, digest, err := ParseImage(image)
if err != nil {
t.Errorf("ParseImage %s failed: %s", image, err.Error())
}
if repo != excepts[0] || tag != excepts[1] || digest != excepts[2] {
t.Fatalf("except repo %s tag %s digest %s, but get %s, %s, %s",
excepts[0], excepts[1], excepts[2], repo, tag, digest)
}
}
})
}
}

14
pkg/util/updatesort/scatter_sort.go
View File

@ -25,10 +25,10 @@ import (
)
type scatterSort struct {
strategy appsv1alpha1.CloneSetUpdateScatterStrategy
strategy appsv1alpha1.UpdateScatterStrategy
}
func NewScatterSorter(s appsv1alpha1.CloneSetUpdateScatterStrategy) Sorter {
func NewScatterSorter(s appsv1alpha1.UpdateScatterStrategy) Sorter {
return &scatterSort{strategy: s}
}
@ -46,22 +46,22 @@ func (ss *scatterSort) Sort(pods []*v1.Pod, indexes []int) []int {
}
// getScatterTerms returns all scatter terms in current sorting. It will sort all terms by sum of pods matched.
func (ss *scatterSort) getScatterTerms(pods []*v1.Pod, indexes []int) []appsv1alpha1.CloneSetUpdateScatterTerm {
func (ss *scatterSort) getScatterTerms(pods []*v1.Pod, indexes []int) []appsv1alpha1.UpdateScatterTerm {
if len(ss.strategy) == 1 {
return ss.strategy
}
var termSlice []appsv1alpha1.CloneSetUpdateScatterTerm
var termSlice []appsv1alpha1.UpdateScatterTerm
ruleCounter := map[string]int{}
termID := func(term appsv1alpha1.CloneSetUpdateScatterTerm) string {
termID := func(term appsv1alpha1.UpdateScatterTerm) string {
return term.Key + ":" + term.Value
}
for _, term := range ss.strategy {
for _, idx := range indexes {
if val, ok := pods[idx].Labels[term.Key]; ok && val == term.Value {
newTerm := appsv1alpha1.CloneSetUpdateScatterTerm{Key: term.Key, Value: val}
newTerm := appsv1alpha1.UpdateScatterTerm{Key: term.Key, Value: val}
id := termID(newTerm)
if count, ok := ruleCounter[id]; !ok {
termSlice = append(termSlice, newTerm)
@ -86,7 +86,7 @@ func (ss *scatterSort) getScatterTerms(pods []*v1.Pod, indexes []int) []appsv1al
}
// scatterPodsByRule scatters pods by given rule term.
func (ss *scatterSort) scatterPodsByRule(term appsv1alpha1.CloneSetUpdateScatterTerm, pods []*v1.Pod, indexes []int) (ret []int) {
func (ss *scatterSort) scatterPodsByRule(term appsv1alpha1.UpdateScatterTerm, pods []*v1.Pod, indexes []int) (ret []int) {
// 1. counts the total number of matched and unmatched pods; find matched and unmatched pods in indexes waiting to update
var matchedIndexes, unmatchedIndexes []int

76
pkg/util/updatesort/scatter_sort_test.go
View File

@ -31,38 +31,38 @@ func TestGenerateRules(t *testing.T) {
testCases := []struct {
desc string
podLabels []map[string]string
scatterStrategy appsv1alpha1.CloneSetUpdateScatterStrategy
expectedResult []appsv1alpha1.CloneSetUpdateScatterTerm
scatterStrategy appsv1alpha1.UpdateScatterStrategy
expectedResult []appsv1alpha1.UpdateScatterTerm
}{
{
desc: "one pod one label",
podLabels: []map[string]string{{}, {}, {}, {}, {"labelA": "AAA"}, {"labelA": "AAA"}},
scatterStrategy: []appsv1alpha1.CloneSetUpdateScatterTerm{{Key: "labelA", Value: "AAA"}},
expectedResult: []appsv1alpha1.CloneSetUpdateScatterTerm{{Key: "labelA", Value: "AAA"}},
scatterStrategy: []appsv1alpha1.UpdateScatterTerm{{Key: "labelA", Value: "AAA"}},
expectedResult: []appsv1alpha1.UpdateScatterTerm{{Key: "labelA", Value: "AAA"}},
},
{
desc: "same pods a label",
podLabels: []map[string]string{{}, {}, {"labelB": "B"}, {"labelB": "BBB"}, {"labelA": "AAA"}, {"labelA": "AAA"}},
scatterStrategy: []appsv1alpha1.CloneSetUpdateScatterTerm{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
expectedResult: []appsv1alpha1.CloneSetUpdateScatterTerm{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
scatterStrategy: []appsv1alpha1.UpdateScatterTerm{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
expectedResult: []appsv1alpha1.UpdateScatterTerm{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
},
//{
// desc: "test regular label",
// podLabels: []map[string]string{{"mode": "AAA"}, {"mode": "AAA"}, {"mode": "AAA"}, {"mode": "BBB"}, {"mode": "BBB"}, {}, {}, {"mode": "CCC"}, {"mode": "CCC"}, {"mode": "CCC"}, {"mode": "CCC"}},
// scatterStrategy: []appsv1alpha1.CloneSetUpdateScatterTerm{{Key: "mode", Value: "*"}},
// expectedResult: []appsv1alpha1.CloneSetUpdateScatterTerm{{Key: "mode", Value: "CCC"}, {Key: "mode", Value: "AAA"}, {Key: "mode", Value: "BBB"}},
// scatterStrategy: []appsv1alpha1.UpdateScatterTerm{{Key: "mode", Value: "*"}},
// expectedResult: []appsv1alpha1.UpdateScatterTerm{{Key: "mode", Value: "CCC"}, {Key: "mode", Value: "AAA"}, {Key: "mode", Value: "BBB"}},
//},
//{
// desc: "test regular label + other label",
// podLabels: []map[string]string{{"mode": "AAA"}, {"mode": "AAA", "labelB": "BBB"}, {"mode": "AAA"}, {"mode": "BBB"}, {"mode": "BBB"}, {}, {}, {"mode": "CCC"}, {"mode": "CCC"}, {"mode": "CCC", "labelB": "BBB"}, {"mode": "CCC"}},
// scatterStrategy: []appsv1alpha1.CloneSetUpdateScatterTerm{{Key: "mode", Value: "*"}, {Key: "labelB", Value: "BBB"}},
// expectedResult: []appsv1alpha1.CloneSetUpdateScatterTerm{{Key: "mode", Value: "CCC"}, {Key: "mode", Value: "AAA"}, {Key: "mode", Value: "BBB"}, {Key: "labelB", Value: "BBB"}},
// scatterStrategy: []appsv1alpha1.UpdateScatterTerm{{Key: "mode", Value: "*"}, {Key: "labelB", Value: "BBB"}},
// expectedResult: []appsv1alpha1.UpdateScatterTerm{{Key: "mode", Value: "CCC"}, {Key: "mode", Value: "AAA"}, {Key: "mode", Value: "BBB"}, {Key: "labelB", Value: "BBB"}},
//},
//{
// desc: "test more regular labels",
// podLabels: []map[string]string{{"mode": "AAA"}, {"mode": "AAA", "labelB": "BBB"}, {"mode": "AAA"}, {"mode": "BBB"}, {"mode": "BBB"}, {"env": "AAA"}, {}, {"env": "AAA", "mode": "CCC"}, {"mode": "CCC"}, {"mode": "CCC", "labelB": "BBB", "env": "AAA"}, {"mode": "CCC"}, {"env": "CCC"}, {"env": "CCC"}},
// scatterStrategy: []appsv1alpha1.CloneSetUpdateScatterTerm{{Key: "mode", Value: "*"}, {Key: "env", Value: "*"}},
// expectedResult: []appsv1alpha1.CloneSetUpdateScatterTerm{{Key: "mode", Value: "CCC"}, {Key: "mode", Value: "AAA"}, {Key: "env", Value: "AAA"}, {Key: "mode", Value: "BBB"}, {Key: "env", Value: "CCC"}},
// scatterStrategy: []appsv1alpha1.UpdateScatterTerm{{Key: "mode", Value: "*"}, {Key: "env", Value: "*"}},
// expectedResult: []appsv1alpha1.UpdateScatterTerm{{Key: "mode", Value: "CCC"}, {Key: "mode", Value: "AAA"}, {Key: "env", Value: "AAA"}, {Key: "mode", Value: "BBB"}, {Key: "env", Value: "CCC"}},
//},
}
@ -92,7 +92,7 @@ func TestGenerateRules(t *testing.T) {
}
func TestScatterPodsByRule(t *testing.T) {
strategyTerm := appsv1alpha1.CloneSetUpdateScatterTerm{Key: "labelA", Value: "AAA"}
strategyTerm := appsv1alpha1.UpdateScatterTerm{Key: "labelA", Value: "AAA"}
testCases := []struct {
desc string
podLabels []string
@ -184,7 +184,7 @@ func TestScatterPodsByRule(t *testing.T) {
indexes = append(indexes, i)
}
ss := &scatterSort{strategy: []appsv1alpha1.CloneSetUpdateScatterTerm{strategyTerm}}
ss := &scatterSort{strategy: []appsv1alpha1.UpdateScatterTerm{strategyTerm}}
gotIndexes := ss.scatterPodsByRule(strategyTerm, pods, indexes)
// compare
@ -198,145 +198,145 @@ func TestSort(t *testing.T) {
testCases := []struct {
desc string
podLabels []map[string]string
scatterStrategy appsv1alpha1.CloneSetUpdateScatterStrategy
scatterStrategy appsv1alpha1.UpdateScatterStrategy
expectedIndexes []int
}{
{
desc: "a scattered pod + a ordinary pod",
podLabels: []map[string]string{{"labelA": "AAA", "labelB": "BBB"}},
scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "labelA", Value: "AAA"}},
scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "labelA", Value: "AAA"}},
expectedIndexes: []int{0},
},
{
desc: "all ordinary pods",
podLabels: []map[string]string{{}, {}, {}, {}, {}, {}},
scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{},
scatterStrategy: appsv1alpha1.UpdateScatterStrategy{},
expectedIndexes: []int{0, 1, 2, 3, 4, 5},
},
{
desc: "one pod one label",
podLabels: []map[string]string{{}, {}, {}, {}, {"labelA": "AAA"}, {"labelA": "AAA"}},
scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "labelA", Value: "AAA"}},
scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "labelA", Value: "AAA"}},
expectedIndexes: []int{4, 0, 1, 2, 3, 5},
},
{
desc: "one pod more labels",
podLabels: []map[string]string{{}, {}, {}, {}, {"sdfb": "eee"}, {"dsf": "same"}, {"labelB": "BBB"}, {"labelA": "AAA"}, {"labelA": "AAA"}, {"labelA": "AAA", "labelB": "BBB"}},
scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "labelB", Value: "BBB"}},
scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "labelB", Value: "BBB"}},
expectedIndexes: []int{6, 0, 1, 2, 3, 4, 5, 7, 8, 9},
},
{
desc: "2 dimensions + one pod one label",
podLabels: []map[string]string{{"labelB": "BBB"}, {"labelB": "BBB"}, {}, {}, {"labelA": "AAA"}, {"labelA": "AAA"}},
scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
expectedIndexes: []int{0, 4, 2, 3, 5, 1},
},
{
desc: "2 dimensions + one pod more labels",
podLabels: []map[string]string{{}, {}, {}, {"labelB": "BBB"}, {"labelA": "AAA", "labelB": "BBB"}, {"labelA": "AAA"}},
scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
expectedIndexes: []int{4, 0, 1, 2, 5, 3},
},
{
desc: "2 dimensions + same label + sequence: A + B",
podLabels: []map[string]string{{}, {}, {}, {}, {"sdfb": "eee"}, {"dsf": "same"}, {"labelB": "BBB"}, {"labelA": "AAA"}, {"labelA": "AAA"}, {"labelB": "BBB"}},
scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "labelB", Value: "BBB"}, {Key: "labelA", Value: "AAA"}},
scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "labelB", Value: "BBB"}, {Key: "labelA", Value: "AAA"}},
expectedIndexes: []int{7, 6, 0, 1, 2, 3, 4, 5, 9, 8},
},
{
desc: "2 dimensions + same label + sequence: B + A",
podLabels: []map[string]string{{}, {}, {}, {}, {"sdfb": "eee"}, {"dsf": "same"}, {"labelB": "BBB"}, {"labelA": "AAA"}, {"labelA": "AAA"}, {"labelB": "BBB"}},
scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
expectedIndexes: []int{6, 7, 0, 1, 2, 3, 4, 5, 8, 9},
},
{
desc: "2 dimensions + same pod same label + even scatter pods + even ordinary pods",
podLabels: []map[string]string{{}, {}, {}, {}, {"sdfb": "eee"}, {"dsf": "same"}, {"labelA": "AAA", "labelB": "BBB"}, {}, {}, {"labelB": "BBB", "labelA": "AAA"}},
scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
expectedIndexes: []int{6, 0, 1, 2, 3, 4, 5, 7, 8, 9},
},
{
desc: "2 dimensions + same pod same label + even scatter pods + odd ordinary pods",
podLabels: []map[string]string{{}, {}, {}, {"sdfb": "eee"}, {"dsf": "same"}, {"labelA": "AAA", "labelB": "BBB"}, {}, {}, {"labelB": "BBB", "labelA": "AAA"}},
scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
expectedIndexes: []int{5, 0, 1, 2, 3, 4, 6, 7, 8},
},
{
desc: "2 dimensions + same pod same label + odd scatter pods + odd ordinary pods",
podLabels: []map[string]string{{}, {}, {}, {"labelA": "AAA"}, {"sdfb": "eee"}, {"dsf": "same"}, {"labelA": "AAA", "labelB": "BBB"}, {}, {}, {}},
scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
expectedIndexes: []int{3, 0, 1, 2, 4, 5, 7, 6},
},
{
desc: "2 dimensions + same pod same label + odd scatter pods + odd ordinary pods",
podLabels: []map[string]string{{}, {}, {}, {"labelA": "AAA"}, {}, {}, {"sdfb": "eee"}, {"dsf": "same"}, {"labelA": "AAA", "labelB": "BBB"}, {}, {}, {"labelB": "BBB", "labelA": "AAA"}},
scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
expectedIndexes: []int{0, 1, 2, 4, 3, 5, 6, 7, 9, 10, 8},
},
{
desc: "2 dimensions + same pod same label + even scatter pods + even ordinary pods + scatter pods more than ordinary pods",
podLabels: []map[string]string{{}, {}, {}, {"labelA": "AAA"}, {"sdfb": "eee", "labelA": "AAA"}, {"dsf": "same"}, {"labelA": "AAA", "labelB": "BBB"}, {}, {}, {"labelB": "BBB", "labelA": "AAA"}},
scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
expectedIndexes: []int{6, 3, 0, 1, 4, 2, 5, 7, 8, 9},
},
{
desc: "2 dimensions + same pod same label + even scatter pods + odd ordinary pods + scatter pods more than ordinary pods",
podLabels: []map[string]string{{}, {"labelA": "AAA"}, {"labelA": "AAA"}, {"sdfb": "eee"}, {"dsf": "same"}, {"labelA": "AAA", "labelB": "BBB"}, {}, {}, {"labelB": "BBB", "labelA": "AAA"}},
scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
expectedIndexes: []int{5, 1, 0, 3, 2, 4, 6, 7, 8},
},
{
desc: "2 dimensions + same pod same label + odd scatter pods + odd ordinary pods + scatter pods more than ordinary pods",
podLabels: []map[string]string{{"labelA": "AAA"}, {}, {}, {"labelA": "AAA"}, {"sdfb": "eee", "labelA": "AAA"}, {"dsf": "same"}, {"labelA": "AAA", "labelB": "BBB"}, {}, {}, {"labelB": "BBB", "labelA": "AAA"}},
scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
expectedIndexes: []int{1, 0, 2, 3, 5, 4, 7, 6},
},
{
desc: "2 dimensions + same pod same label + odd scatter pods + odd ordinary pods + scatter pods more than ordinary pods",
podLabels: []map[string]string{{}, {}, {"labelA": "AAA"}, {"labelA": "AAA"}, {}, {"labelA": "AAA"}, {"sdfb": "eee"}, {"dsf": "same"}, {"labelA": "AAA", "labelB": "BBB"}, {}, {}, {"labelB": "BBB", "labelA": "AAA"}},
scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}},
expectedIndexes: []int{0, 2, 1, 4, 3, 6, 7, 5, 9, 10, 8},
},
{
desc: "3 dimensions + one pod one label",
podLabels: []map[string]string{{}, {"labelA": "AAA"}, {"labelA": "AAA"}, {"labelB": "BBB"}, {"labelB": "BBB"}, {"labelC": "CCC"}, {"labelC": "CCC"}, {"labelC": "CCC"}, {"labelC": "CCC"}, {"labelC": "CCC"}},
scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}, {Key: "labelC", Value: "CCC"}},
scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}, {Key: "labelC", Value: "CCC"}},
expectedIndexes: []int{3, 1, 5, 0, 6, 7, 8, 9, 2, 4},
},
{
desc: "3 dimensions + one pod more label",
podLabels: []map[string]string{{}, {"labelA": "AAA"}, {"labelA": "AAA", "labelC": "CCC"}, {"labelB": "BBB", "labelC": "CCC"}, {"labelB": "BBB"}, {}, {}, {"labelC": "CCC"}, {"labelC": "CCC"}, {"labelC": "CCC"}},
scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}, {Key: "labelC", Value: "CCC"}},
scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}, {Key: "labelC", Value: "CCC"}},
expectedIndexes: []int{3, 2, 0, 7, 8, 5, 6, 9, 1, 4},
},
//{
// desc: "test regular label",
// podLabels: []map[string]string{{"mode": "AAA"}, {"mode": "AAA"}, {"mode": "AAA"}, {"mode": "BBB"}, {"mode": "BBB"}, {}, {}, {"mode": "CCC"}, {"mode": "CCC"}, {"mode": "CCC"}, {"mode": "CCC"}},
// scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "mode", Value: "*"}},
// scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "mode", Value: "*"}},
// expectedIndexes: []int{3, 0, 7, 8, 9, 1, 5, 6, 10, 2, 4},
//},
//{
// desc: "test regular label + other label",
// podLabels: []map[string]string{{"mode": "AAA"}, {"mode": "AAA", "labelB": "BBB"}, {"mode": "AAA"}, {"mode": "BBB"}, {"mode": "BBB"}, {}, {}, {"mode": "CCC"}, {"mode": "CCC"}, {"mode": "CCC", "labelB": "BBB"}, {"mode": "CCC"}},
// scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "mode", Value: "*"}, {Key: "labelB", Value: "BBB"}},
// scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "mode", Value: "*"}, {Key: "labelB", Value: "BBB"}},
// expectedIndexes: []int{9, 3, 0, 7, 8, 5, 6, 10, 2, 4, 1},
//},
{
desc: "continuous sort 1",
podLabels: []map[string]string{{}, {}, {}, {}, {"labelA": "AAA"}, {"labelA": "AAA"}, {"labelA": "AAA"}},
scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "labelA", Value: "AAA"}},
scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "labelA", Value: "AAA"}},
expectedIndexes: []int{0, 1, 4, 2, 3, 5},
},
{
desc: "continuous sort 2",
podLabels: []map[string]string{{"labelA": "AAA"}, {"labelA": "AAA"}, {}, {}, {}, {}, {}, {}, {}, {"labelA": "AAA"}},
scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "labelA", Value: "AAA"}},
scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "labelA", Value: "AAA"}},
expectedIndexes: []int{2, 0, 3, 4, 5, 6, 1},
},
{
desc: "reserveOrdinals nil in slice",
podLabels: []map[string]string{{}, {"labelA": "AAA"}, {"labelA": "AAA", "labelC": "CCC"}, {"labelB": "BBB", "labelC": "CCC"}, {"labelB": "BBB"}, nil, {}, {}, {"labelC": "CCC"}, {"labelC": "CCC"}, {"labelC": "CCC"}},
scatterStrategy: appsv1alpha1.CloneSetUpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}, {Key: "labelC", Value: "CCC"}},
scatterStrategy: appsv1alpha1.UpdateScatterStrategy{{Key: "labelA", Value: "AAA"}, {Key: "labelB", Value: "BBB"}, {Key: "labelC", Value: "CCC"}},
expectedIndexes: []int{3, 2, 0, 8, 9, 6, 7, 10, 1, 4},
},
}

217
pkg/webhook/pod/mutating/pod_create_handler.go
View File

@ -1,217 +0,0 @@
/*
Copyright 2019 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mutating
import (
"context"
"encoding/json"
"net/http"
"sort"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/util"
"github.com/openkruise/kruise/pkg/webhook/sidecarset/mutating"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)
var (
// SidecarIgnoredNamespaces specifies the namespaces where Pods won't get injected
SidecarIgnoredNamespaces = []string{"kube-system", "kube-public"}
// SidecarEnvKey specifies the environment variable which marks a container as injected
SidecarEnvKey = "IS_INJECTED"
)
// PodCreateHandler handles Pod
type PodCreateHandler struct {
// To use the client, you need to do the following:
// - uncomment it
// - import sigs.k8s.io/controller-runtime/pkg/client
// - uncomment the InjectClient method at the bottom of this file.
Client client.Client
// Decoder decodes objects
Decoder *admission.Decoder
}
func (h *PodCreateHandler) mutatingPodFn(ctx context.Context, obj *corev1.Pod) error {
return h.sidecarsetMutatingPod(ctx, obj)
}
func (h *PodCreateHandler) sidecarsetMutatingPod(ctx context.Context, pod *corev1.Pod) error {
for _, namespace := range SidecarIgnoredNamespaces {
if pod.Namespace == namespace {
return nil
}
}
klog.V(3).Infof("[sidecar inject] begin to process %s/%s", pod.Namespace, pod.Name)
sidecarSets := &appsv1alpha1.SidecarSetList{}
if err := h.Client.List(ctx, sidecarSets); err != nil {
return err
}
var sidecarInitContainers, sidecarContainers []corev1.Container
var sidecarVolumes []corev1.Volume
sidecarSetHash := make(map[string]string)
sidecarSetHashWithoutImage := make(map[string]string)
matchNothing := true
for _, sidecarSet := range sidecarSets.Items {
needInject, err := PodMatchSidecarSet(pod, sidecarSet)
if err != nil {
return err
}
if !needInject {
continue
}
matchNothing = false
sidecarSetHash[sidecarSet.Name] = sidecarSet.Annotations[mutating.SidecarSetHashAnnotation]
sidecarSetHashWithoutImage[sidecarSet.Name] = sidecarSet.Annotations[mutating.SidecarSetHashWithoutImageAnnotation]
for i := range sidecarSet.Spec.InitContainers {
initContainer := &sidecarSet.Spec.InitContainers[i]
// add "Injected" env to the init container
initContainer.Env = append(initContainer.Env, corev1.EnvVar{Name: SidecarEnvKey, Value: "true"})
sidecarInitContainers = append(sidecarInitContainers, initContainer.Container)
}
for i := range sidecarSet.Spec.Containers {
sidecarContainer := &sidecarSet.Spec.Containers[i]
// add the "Injected" env to the sidecar container
sidecarContainer.Env = append(sidecarContainer.Env, corev1.EnvVar{Name: SidecarEnvKey, Value: "true"})
sidecarContainers = append(sidecarContainers, sidecarContainer.Container)
}
sidecarVolumes = append(sidecarVolumes, sidecarSet.Spec.Volumes...)
}
if matchNothing {
return nil
}
klog.V(4).Infof("[sidecar inject] before mutating: %v", util.DumpJSON(pod))
// apply sidecar set info into pod
// 1. inject init containers, sort by their name, after the original init containers
sort.SliceStable(sidecarInitContainers, func(i, j int) bool {
return sidecarInitContainers[i].Name < sidecarInitContainers[j].Name
})
pod.Spec.InitContainers = append(pod.Spec.InitContainers, sidecarInitContainers...)
// 2. inject containers
pod.Spec.Containers = append(pod.Spec.Containers, sidecarContainers...)
// 3. inject volumes
pod.Spec.Volumes = mergeVolumes(pod.Spec.Volumes, sidecarVolumes)
// 4. apply annotations
if pod.Annotations == nil {
pod.Annotations = make(map[string]string)
}
if len(sidecarSetHash) != 0 {
encodedStr, err := json.Marshal(sidecarSetHash)
if err != nil {
return err
}
pod.Annotations[mutating.SidecarSetHashAnnotation] = string(encodedStr)
encodedStr, err = json.Marshal(sidecarSetHashWithoutImage)
if err != nil {
return err
}
pod.Annotations[mutating.SidecarSetHashWithoutImageAnnotation] = string(encodedStr)
}
klog.V(4).Infof("[sidecar inject] after mutating: %v", util.DumpJSON(pod))
return nil
}
// PodMatchSidecarSet determines if pod match Selector of sidecar.
func PodMatchSidecarSet(pod *corev1.Pod, sidecarSet appsv1alpha1.SidecarSet) (bool, error) {
selector, err := metav1.LabelSelectorAsSelector(sidecarSet.Spec.Selector)
if err != nil {
return false, err
}
if !selector.Empty() && selector.Matches(labels.Set(pod.Labels)) {
return true, nil
}
return false, nil
}
func mergeVolumes(original []corev1.Volume, additional []corev1.Volume) []corev1.Volume {
exists := sets.NewString()
for _, volume := range original {
exists.Insert(volume.Name)
}
for _, volume := range additional {
if exists.Has(volume.Name) {
continue
}
original = append(original, volume)
exists.Insert(volume.Name)
}
return original
}
var _ admission.Handler = &PodCreateHandler{}
// Handle handles admission requests.
func (h *PodCreateHandler) Handle(ctx context.Context, req admission.Request) admission.Response {
obj := &corev1.Pod{}
err := h.Decoder.Decode(req, obj)
if err != nil {
return admission.Errored(http.StatusBadRequest, err)
}
err = h.mutatingPodFn(ctx, obj)
if err != nil {
return admission.Errored(http.StatusInternalServerError, err)
}
marshalled, err := json.Marshal(obj)
if err != nil {
return admission.Errored(http.StatusInternalServerError, err)
}
return admission.PatchResponseFromRaw(req.AdmissionRequest.Object.Raw, marshalled)
}
var _ inject.Client = &PodCreateHandler{}
// InjectClient injects the client into the PodCreateHandler
func (h *PodCreateHandler) InjectClient(c client.Client) error {
h.Client = c
return nil
}
var _ admission.DecoderInjector = &PodCreateHandler{}
// InjectDecoder injects the decoder into the PodCreateHandler
func (h *PodCreateHandler) InjectDecoder(d *admission.Decoder) error {
h.Decoder = d
return nil
}

257
pkg/webhook/pod/mutating/pod_create_handler_test.go
View File

@ -1,257 +0,0 @@
package mutating
import (
"context"
"os"
"path/filepath"
"reflect"
"testing"
"github.com/openkruise/kruise/pkg/util"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/scheme"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/envtest"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
"github.com/openkruise/kruise/apis"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/webhook/sidecarset/mutating"
)
func TestMain(m *testing.M) {
t := &envtest.Environment{
CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")},
}
apis.AddToScheme(scheme.Scheme)
code := m.Run()
t.Stop()
os.Exit(code)
}
func TestSidecarSetMutatePod(t *testing.T) {
sidecarSet1 := &appsv1alpha1.SidecarSet{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
mutating.SidecarSetHashAnnotation: "c4k2dbb95d",
mutating.SidecarSetHashWithoutImageAnnotation: "26c9ct5hfb",
},
Name: "sidecarset1",
Generation: 123,
},
Spec: appsv1alpha1.SidecarSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": "nginx"},
},
InitContainers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{
Name: "init1",
Image: "init-image1",
},
},
{
Container: corev1.Container{
Name: "init3",
Image: "init-image3",
},
},
},
Containers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{
Name: "sidecar1",
Image: "sidecar-image1",
},
},
},
Volumes: []corev1.Volume{
{
Name: "volume1",
},
},
},
}
sidecarSet2 := &appsv1alpha1.SidecarSet{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
mutating.SidecarSetHashAnnotation: "gm967682cm",
mutating.SidecarSetHashWithoutImageAnnotation: "h8c6gb5d2b",
},
Name: "sidecarset2",
Generation: 456,
},
Spec: appsv1alpha1.SidecarSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": "nginx"},
},
InitContainers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{
Name: "init2",
Image: "init-image2",
},
},
},
Containers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{
Name: "sidecar2",
Image: "sidecar-image2",
},
},
},
Volumes: []corev1.Volume{
{
Name: "volume2",
},
},
},
}
pod1 := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod",
Namespace: "default",
Labels: map[string]string{"app": "nginx"},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "nginx",
Image: "nginx:1.15.1",
},
},
Volumes: []corev1.Volume{
{
Name: "nginx-volume",
},
},
},
}
pod2 := pod1.DeepCopy()
pod2.Labels = map[string]string{}
client := fake.NewFakeClient(sidecarSet1, sidecarSet2)
decoder, _ := admission.NewDecoder(scheme.Scheme)
podHandler := &PodCreateHandler{Decoder: decoder, Client: client}
expectedMutatedPod2 := pod2.DeepCopy()
_ = podHandler.mutatingPodFn(context.TODO(), pod1)
_ = podHandler.mutatingPodFn(context.TODO(), pod2)
if len(pod1.Spec.InitContainers) != 3 {
t.Errorf("expect 3 init containers, but got %v", len(pod1.Spec.InitContainers))
}
if pod1.Spec.InitContainers[0].Name != "init1" {
t.Errorf("expect first init container `init1`, but got %s", pod1.Spec.InitContainers[0].Name)
}
if pod1.Spec.InitContainers[1].Name != "init2" {
t.Errorf("expect first init container `init2`, but got %s", pod1.Spec.InitContainers[0].Name)
}
if pod1.Spec.InitContainers[2].Name != "init3" {
t.Errorf("expect first init container `init3`, but got %s", pod1.Spec.InitContainers[0].Name)
}
if len(pod1.Spec.Containers) != 3 {
t.Errorf("expect 3 containers, but got %v", len(pod1.Spec.Containers))
}
if len(pod1.Spec.Volumes) != 3 {
t.Errorf("expect 3 volumes, but got %v", len(pod1.Spec.Volumes))
}
if !isMarkedSidecar(pod1.Spec.Containers[1]) || !isMarkedSidecar(pod1.Spec.Containers[2]) {
t.Errorf("expect env injected, but got nothing")
}
hashKey1 := mutating.SidecarSetHashAnnotation
hashKey2 := mutating.SidecarSetHashWithoutImageAnnotation
expectedAnnotation1 := `{"sidecarset1":"c4k2dbb95d","sidecarset2":"gm967682cm"}`
expectedAnnotation2 := `{"sidecarset1":"26c9ct5hfb","sidecarset2":"h8c6gb5d2b"}`
if pod1.Annotations[hashKey1] != expectedAnnotation1 {
t.Errorf("expect annotation %v but got %v", expectedAnnotation1, pod1.Annotations[hashKey1])
}
if pod1.Annotations[hashKey2] != expectedAnnotation2 {
t.Errorf("expect annotation %v but got %v", expectedAnnotation2, pod1.Annotations[hashKey2])
}
// nothing changed
if !reflect.DeepEqual(pod2, expectedMutatedPod2) {
t.Errorf("\nexpected mutated pod:\n%+v,\nbut got %+v\n", expectedMutatedPod2, pod2)
}
}
func isMarkedSidecar(container corev1.Container) bool {
for _, env := range container.Env {
if env.Name == SidecarEnvKey && env.Value == "true" {
return true
}
}
return false
}
func TestMergeVolumes(t *testing.T) {
original := []corev1.Volume{
{
Name: "vol01",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
{
Name: "vol02",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{},
},
},
}
additional := []corev1.Volume{
{
Name: "vol02",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
{
Name: "vol03",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{},
},
},
{
Name: "vol03",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
}
expected := []corev1.Volume{
{
Name: "vol01",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
{
Name: "vol02",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{},
},
},
{
Name: "vol03",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{},
},
},
}
got := mergeVolumes(original, additional)
if !reflect.DeepEqual(got, expected) {
t.Fatalf("expected %v, got %v", util.DumpJSON(expected), util.DumpJSON(got))
}
}

98
pkg/webhook/pod/mutating/pod_create_update_handler.go Normal file
View File

@ -0,0 +1,98 @@
/*
Copyright 2019 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mutating
import (
"context"
"encoding/json"
"net/http"
admissionv1beta1 "k8s.io/api/admission/v1beta1"
corev1 "k8s.io/api/core/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)
// PodCreateHandler handles Pod
type PodCreateHandler struct {
// To use the client, you need to do the following:
// - uncomment it
// - import sigs.k8s.io/controller-runtime/pkg/client
// - uncomment the InjectClient method at the bottom of this file.
Client client.Client
// Decoder decodes objects
Decoder *admission.Decoder
}
func (h *PodCreateHandler) mutatingPodFn(ctx context.Context, obj *corev1.Pod, oldPod *corev1.Pod) error {
return h.sidecarsetMutatingPod(ctx, obj, oldPod)
}
var _ admission.Handler = &PodCreateHandler{}
// Handle handles admission requests.
func (h *PodCreateHandler) Handle(ctx context.Context, req admission.Request) admission.Response {
obj := &corev1.Pod{}
err := h.Decoder.Decode(req, obj)
if err != nil {
return admission.Errored(http.StatusBadRequest, err)
}
// when pod.namespace is empty, using req.namespace
if obj.Namespace == "" {
obj.Namespace = req.Namespace
}
var oldPod *corev1.Pod
//when Operation is update, decode older object
if req.AdmissionRequest.Operation == admissionv1beta1.Update {
oldPod = new(corev1.Pod)
if err := h.Decoder.Decode(
admission.Request{AdmissionRequest: admissionv1beta1.AdmissionRequest{Object: req.AdmissionRequest.OldObject}},
oldPod); err != nil {
return admission.Errored(http.StatusBadRequest, err)
}
}
err = h.mutatingPodFn(ctx, obj, oldPod)
if err != nil {
return admission.Errored(http.StatusInternalServerError, err)
}
marshalled, err := json.Marshal(obj)
if err != nil {
return admission.Errored(http.StatusInternalServerError, err)
}
return admission.PatchResponseFromRaw(req.AdmissionRequest.Object.Raw, marshalled)
}
var _ inject.Client = &PodCreateHandler{}
// InjectClient injects the client into the PodCreateHandler
func (h *PodCreateHandler) InjectClient(c client.Client) error {
h.Client = c
return nil
}
var _ admission.DecoderInjector = &PodCreateHandler{}
// InjectDecoder injects the decoder into the PodCreateHandler
func (h *PodCreateHandler) InjectDecoder(d *admission.Decoder) error {
h.Decoder = d
return nil
}

247
pkg/webhook/pod/mutating/sidecarset.go Normal file
View File

@ -0,0 +1,247 @@
/*
Copyright 2020 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mutating
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/control/sidecarcontrol"
"github.com/openkruise/kruise/pkg/util"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/klog"
"sigs.k8s.io/controller-runtime/pkg/client"
)
// mutate pod based on SidecarSet Object
func (h *PodCreateHandler) sidecarsetMutatingPod(ctx context.Context, pod *corev1.Pod, oldPod *corev1.Pod) error {
if !sidecarcontrol.IsActivePod(pod) {
return nil
}
klog.V(3).Infof("[sidecar inject] begin to process %s/%s", pod.Namespace, pod.Name)
// DisableDeepCopy:true, indicates must be deep copy before update sidecarSet objection
listOpts := &client.ListOptions{}
sidecarsetList := &appsv1alpha1.SidecarSetList{}
if err := h.Client.List(ctx, sidecarsetList, listOpts); err != nil {
return err
}
isUpdated := false
//when oldPod is not empty, indicates it is update event
if oldPod != nil {
isUpdated = true
} else {
oldPod = &corev1.Pod{}
}
//build sidecar containers, sidecar initContainers, sidecar volumes, annotations to inject into pod object
sidecarContainers, sidecarInitContainers, volumesInSidecar, injectedAnnotations,
err := buildSidecars(isUpdated, pod, oldPod, sidecarsetList)
if err != nil {
return err
} else if len(sidecarContainers) == 0 && len(sidecarInitContainers) == 0 {
return nil
}
klog.V(3).Infof("[sidecar inject] begin inject sidecarContainers(%v) sidecarInitContainers(%v) volumes(%s)"+
"annotations(%v) into pod(%s.%s)", sidecarContainers, sidecarInitContainers, volumesInSidecar, injectedAnnotations,
pod.Namespace, pod.Name)
klog.V(4).Infof("[sidecar inject] before mutating: %v", util.DumpJSON(pod))
// apply sidecar set info into pod
// 1. inject init containers, sort by their name, after the original init containers
sort.SliceStable(sidecarInitContainers, func(i, j int) bool {
return sidecarInitContainers[i].Name < sidecarInitContainers[j].Name
})
for _, initContainer := range sidecarInitContainers {
pod.Spec.InitContainers = append(pod.Spec.InitContainers, initContainer.Container)
}
// 2. inject containers
pod.Spec.Containers = mergeSidecarContainers(pod.Spec.Containers, sidecarContainers)
// 3. inject volumes
pod.Spec.Volumes = util.MergeVolumes(pod.Spec.Volumes, volumesInSidecar)
// 4. apply annotations
if pod.Annotations == nil {
pod.Annotations = make(map[string]string)
}
for k, v := range injectedAnnotations {
pod.Annotations[k] = v
}
klog.V(4).Infof("[sidecar inject] after mutating: %v", util.DumpJSON(pod))
return nil
}
func mergeSidecarContainers(origins []corev1.Container, injected []*appsv1alpha1.SidecarContainer) []corev1.Container {
//format: pod.spec.containers[index].name -> index(the index of container in pod)
containersInPod := make(map[string]int)
for index, container := range origins {
containersInPod[container.Name] = index
}
var beforeAppContainers []corev1.Container
var afterAppContainers []corev1.Container
for _, sidecar := range injected {
//sidecar container already exist in pod
//keep the order of pod's original containers unchanged
if index, ok := containersInPod[sidecar.Name]; ok {
origins[index] = sidecar.Container
continue
}
switch sidecar.PodInjectPolicy {
case appsv1alpha1.BeforeAppContainerType:
beforeAppContainers = append(beforeAppContainers, sidecar.Container)
case appsv1alpha1.AfterAppContainerType:
afterAppContainers = append(afterAppContainers, sidecar.Container)
default:
beforeAppContainers = append(beforeAppContainers, sidecar.Container)
}
}
origins = append(beforeAppContainers, origins...)
origins = append(origins, afterAppContainers...)
return origins
}
func buildSidecars(isUpdated bool, pod *corev1.Pod, oldPod *corev1.Pod, sidecarsetList *appsv1alpha1.SidecarSetList) (
sidecarContainers, sidecarInitContainers []*appsv1alpha1.SidecarContainer,
volumesInSidecars []corev1.Volume, injectedAnnotations map[string]string, err error) {
// injected into pod
injectedAnnotations = make(map[string]string)
// format: sidecarset.name -> sidecarset hash
sidecarSetHash := make(map[string]sidecarcontrol.SidecarSetUpgradeSpec)
// format: sidecarset.name -> sidecarset hash(without image)
sidecarSetHashWithoutImage := make(map[string]sidecarcontrol.SidecarSetUpgradeSpec)
// parse sidecar hash in pod annotations
if oldHashStr := pod.Annotations[sidecarcontrol.SidecarSetHashAnnotation]; len(oldHashStr) > 0 {
if err := json.Unmarshal([]byte(oldHashStr), &sidecarSetHash); err != nil {
return nil, nil, nil, nil,
fmt.Errorf("invalid %s value %v, unmarshal failed: %v", sidecarcontrol.SidecarSetHashAnnotation, oldHashStr, err)
}
}
if oldHashStr := pod.Annotations[sidecarcontrol.SidecarSetHashWithoutImageAnnotation]; len(oldHashStr) > 0 {
if err := json.Unmarshal([]byte(oldHashStr), &sidecarSetHashWithoutImage); err != nil {
return nil, nil, nil, nil,
fmt.Errorf("invalid %s value %v, unmarshal failed: %v", sidecarcontrol.SidecarSetHashWithoutImageAnnotation, oldHashStr, err)
}
}
//matched SidecarSet.Name list
sidecarSetNames := make([]string, 0)
for _, innerSidecarSet := range sidecarsetList.Items {
if matched, err := sidecarcontrol.PodMatchedSidecarSet(pod, innerSidecarSet); err != nil {
return nil, nil, nil, nil, err
// sidecarset don't select this pod, then continue
} else if !matched {
continue
}
sidecarSet := innerSidecarSet.DeepCopy()
control := sidecarcontrol.New(sidecarSet)
//process sidecarset inject pods container
sidecarSetNames = append(sidecarSetNames, sidecarSet.Name)
// pre-process volumes only in sidecar
volumesMap := getVolumesMapInSidecarSet(sidecarSet)
// process sidecarset hash
sidecarSetHash[sidecarSet.Name] = sidecarcontrol.SidecarSetUpgradeSpec{
UpdateTimestamp: metav1.Now(),
SidecarSetHash: sidecarcontrol.GetSidecarSetRevision(sidecarSet),
SidecarSetName: sidecarSet.Name,
}
sidecarSetHashWithoutImage[sidecarSet.Name] = sidecarcontrol.SidecarSetUpgradeSpec{
UpdateTimestamp: metav1.Now(),
SidecarSetHash: sidecarcontrol.GetSidecarSetWithoutImageRevision(sidecarSet),
SidecarSetName: sidecarSet.Name,
}
//process initContainers
//only when created pod, inject initContainer
if !isUpdated {
for i := range sidecarSet.Spec.InitContainers {
initContainer := &sidecarSet.Spec.InitContainers[i]
//add "IS_INJECTED" env in initContainer's envs
initContainer.Env = append(initContainer.Env, corev1.EnvVar{Name: sidecarcontrol.SidecarEnvKey, Value: "true"})
sidecarInitContainers = append(sidecarInitContainers, initContainer)
}
}
//process containers
for i := range sidecarSet.Spec.Containers {
sidecarContainer := &sidecarSet.Spec.Containers[i]
// volumeMounts that injected into sidecar container
// when volumeMounts SubPathExpr contains expansions, then need copy container EnvVars(injectEnvs)
injectedMounts, injectedEnvs := sidecarcontrol.GetInjectedVolumeMountsAndEnvs(control, sidecarContainer, pod)
// get injected env & mounts explicitly so that can be compared with old ones in pod
transferEnvs := sidecarcontrol.GetSidecarTransferEnvs(sidecarContainer, pod)
// append volumeMounts SubPathExpr environments
transferEnvs = util.MergeEnvVar(transferEnvs, injectedEnvs)
klog.Infof("try to inject sidecar %v@%v/%v, with injected envs: %v, volumeMounts: %v",
sidecarContainer.Name, pod.Namespace, pod.Name, transferEnvs, injectedMounts)
//when update pod object
if isUpdated {
// judge whether inject sidecar container into pod
needInject, existSidecars, existVolumes := control.NeedInjectOnUpdatedPod(pod, oldPod, sidecarContainer, transferEnvs, injectedMounts)
if !needInject {
sidecarContainers = append(sidecarContainers, existSidecars...)
volumesInSidecars = append(volumesInSidecars, existVolumes...)
continue
}
klog.Infof("upgrade or insert sidecar container %v during upgrade in pod %v/%v",
sidecarContainer.Name, pod.Namespace, pod.Name)
//when created pod object, need inject sidecar container into pod
} else {
klog.Infof("inject new sidecar container %v during creation in pod %v/%v",
sidecarContainer.Name, pod.Namespace, pod.Name)
}
// insert volume that sidecar container used
for _, mount := range sidecarContainer.VolumeMounts {
volumesInSidecars = append(volumesInSidecars, *volumesMap[mount.Name])
}
// merge VolumeMounts from sidecar.VolumeMounts and shared VolumeMounts
sidecarContainer.VolumeMounts = util.MergeVolumeMounts(sidecarContainer.VolumeMounts, injectedMounts)
// add the "Injected" env to the sidecar container
sidecarContainer.Env = append(sidecarContainer.Env, corev1.EnvVar{Name: sidecarcontrol.SidecarEnvKey, Value: "true"})
// merged Env from sidecar.Env and transfer envs
sidecarContainer.Env = util.MergeEnvVar(sidecarContainer.Env, transferEnvs)
sidecarContainers = append(sidecarContainers, sidecarContainer)
}
}
// store sidecarset hash in pod annotations
by, _ := json.Marshal(sidecarSetHash)
injectedAnnotations[sidecarcontrol.SidecarSetHashAnnotation] = string(by)
by, _ = json.Marshal(sidecarSetHashWithoutImage)
injectedAnnotations[sidecarcontrol.SidecarSetHashWithoutImageAnnotation] = string(by)
// store matched sidecarset list in pod annotations
injectedAnnotations[sidecarcontrol.SidecarSetListAnnotation] = strings.Join(sidecarSetNames, ",")
return sidecarContainers, sidecarInitContainers, volumesInSidecars, injectedAnnotations, nil
}
func getVolumesMapInSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) map[string]*corev1.Volume {
volumesMap := make(map[string]*corev1.Volume)
for idx, volume := range sidecarSet.Spec.Volumes {
volumesMap[volume.Name] = &sidecarSet.Spec.Volumes[idx]
}
return volumesMap
}

802
pkg/webhook/pod/mutating/sidecarset_test.go Normal file
View File

@ -0,0 +1,802 @@
/*
Copyright 2020 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mutating
import (
"context"
"os"
"path/filepath"
"testing"
"github.com/openkruise/kruise/apis"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/control/sidecarcontrol"
"github.com/openkruise/kruise/pkg/util"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/scheme"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/envtest"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)
const (
defaultNs = "default"
)
func TestMain(m *testing.M) {
t := &envtest.Environment{
CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")},
}
apis.AddToScheme(scheme.Scheme)
code := m.Run()
t.Stop()
os.Exit(code)
}
var (
sidecarSet1 = &appsv1alpha1.SidecarSet{
ObjectMeta: metav1.ObjectMeta{
Name: "sidecarset1",
Annotations: map[string]string{
sidecarcontrol.SidecarSetHashAnnotation: "c4k2dbb95d",
},
Labels: map[string]string{},
},
Spec: appsv1alpha1.SidecarSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": "suxing-test",
},
},
InitContainers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{
Name: "init-2",
Image: "busybox:1.0.0",
},
},
{
Container: corev1.Container{
Name: "init-1",
Image: "busybox:1.0.0",
},
},
},
Containers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{
Name: "dns-f",
Image: "dns-f-image:1.0",
},
PodInjectPolicy: appsv1alpha1.BeforeAppContainerType,
ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{
Type: appsv1alpha1.ShareVolumePolicyDisabled,
},
},
{
Container: corev1.Container{
Name: "log-agent",
Image: "log-agent-image:1.0",
},
PodInjectPolicy: appsv1alpha1.AfterAppContainerType,
ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{
Type: appsv1alpha1.ShareVolumePolicyDisabled,
},
},
},
},
}
sidecarsetWithTransferEnv = &appsv1alpha1.SidecarSet{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
sidecarcontrol.SidecarSetHashAnnotation: "c4k2dbb95d",
},
Name: "sidecarset2",
Labels: map[string]string{},
},
Spec: appsv1alpha1.SidecarSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": "suxing-test",
},
},
Containers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{
Name: "dns-f",
Image: "dns-f-image:1.0",
VolumeMounts: []corev1.VolumeMount{
{Name: "volume-1"},
},
},
PodInjectPolicy: appsv1alpha1.BeforeAppContainerType,
ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{
Type: appsv1alpha1.ShareVolumePolicyDisabled,
},
TransferEnv: []appsv1alpha1.TransferEnvVar{
{
SourceContainerName: "nginx",
EnvName: "hello2",
},
},
},
},
Volumes: []corev1.Volume{
{Name: "volume-1"},
{Name: "volume-2"},
},
},
}
sidecarSet3 = &appsv1alpha1.SidecarSet{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
sidecarcontrol.SidecarSetHashAnnotation: "gm967682cm",
},
Name: "sidecarset3",
Labels: map[string]string{},
},
Spec: appsv1alpha1.SidecarSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": "suxing-test",
},
},
Containers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{
Name: "dns-f",
Image: "dns-f-image:1.0",
VolumeMounts: []corev1.VolumeMount{
{
Name: "volume-1",
MountPath: "/a/b/c",
},
{
Name: "volume-2",
MountPath: "/d/e/f",
},
},
},
PodInjectPolicy: appsv1alpha1.AfterAppContainerType,
ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{
Type: appsv1alpha1.ShareVolumePolicyEnabled,
},
},
{
Container: corev1.Container{
Name: "log-agent",
Image: "log-agent-image:1.0",
},
PodInjectPolicy: appsv1alpha1.AfterAppContainerType,
ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{
Type: appsv1alpha1.ShareVolumePolicyDisabled,
},
},
},
Volumes: []corev1.Volume{
{Name: "volume-1"},
{Name: "volume-2"},
},
},
}
sidecarSetWithStaragent = &appsv1alpha1.SidecarSet{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
sidecarcontrol.SidecarSetHashAnnotation: "gm967682cm",
},
Name: "sidecarset3",
Labels: map[string]string{},
},
Spec: appsv1alpha1.SidecarSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": "suxing-test",
},
},
Containers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{
Name: "dns-f",
Image: "dns-f-image:1.0",
VolumeMounts: []corev1.VolumeMount{
{
Name: "volume-1",
MountPath: "/a/b/c",
},
{
Name: "volume-2",
MountPath: "/d/e/f",
},
{
Name: "volume-staragent",
MountPath: "/staragent",
},
},
},
PodInjectPolicy: appsv1alpha1.AfterAppContainerType,
ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{
Type: appsv1alpha1.ShareVolumePolicyEnabled,
},
},
{
Container: corev1.Container{
Name: "staragent",
Image: "staragent-image:1.0",
},
PodInjectPolicy: appsv1alpha1.AfterAppContainerType,
ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{
Type: appsv1alpha1.ShareVolumePolicyEnabled,
},
},
{
Container: corev1.Container{
Name: "log-agent",
Image: "log-agent-image:1.0",
},
PodInjectPolicy: appsv1alpha1.AfterAppContainerType,
ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{
Type: appsv1alpha1.ShareVolumePolicyDisabled,
},
},
},
Volumes: []corev1.Volume{
{Name: "volume-1"},
{Name: "volume-2"},
{Name: "volume-staragent"},
},
},
}
pod1 = &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod",
Namespace: defaultNs,
Labels: map[string]string{"app": "suxing-test"},
},
Spec: corev1.PodSpec{
InitContainers: []corev1.Container{
{
Name: "init-0",
Image: "busybox:1.0.0",
},
},
Containers: []corev1.Container{
{
Name: "nginx",
Image: "nginx:1.15.1",
Env: []corev1.EnvVar{
{
Name: "hello1",
Value: "world1",
},
{
Name: "hello2",
Value: "world2",
},
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "volume-a",
MountPath: "/a/b",
},
{
Name: "volume-b",
MountPath: "/e/f",
},
},
},
},
Volumes: []corev1.Volume{
{Name: "volume-a"},
{Name: "volume-b"},
},
},
}
podWithStaragent = &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod",
Namespace: defaultNs,
Labels: map[string]string{"app": "suxing-test"},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "nginx",
Image: "nginx:1.15.1",
Env: []corev1.EnvVar{
{
Name: "hello1",
Value: "world1",
},
{
Name: "hello2",
Value: "world2",
},
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "volume-a",
MountPath: "/a/b",
},
{
Name: "volume-b",
MountPath: "/e/f",
},
{
Name: "volume-staragent",
MountPath: "/staragent",
},
},
},
},
Volumes: []corev1.Volume{
{Name: "volume-a"},
{Name: "volume-b"},
{Name: "volume-staragent"},
},
},
}
)
func TestPodHasNoMatchedSidecarSet(t *testing.T) {
sidecarSetIn := sidecarSet1.DeepCopy()
testPodHasNoMatchedSidecarSet(t, sidecarSetIn)
}
func testPodHasNoMatchedSidecarSet(t *testing.T, sidecarSetIn *appsv1alpha1.SidecarSet) {
podIn := pod1.DeepCopy()
podIn.Labels["app"] = "doesnt-match"
podOut := podIn.DeepCopy()
decoder, _ := admission.NewDecoder(scheme.Scheme)
client := fake.NewFakeClient(sidecarSetIn)
podHandler := &PodCreateHandler{Decoder: decoder, Client: client}
_ = podHandler.sidecarsetMutatingPod(context.Background(), podOut, nil)
if len(podOut.Spec.Containers) != len(podIn.Spec.Containers) {
t.Fatalf("expect %v containers but got %v", len(podIn.Spec.Containers), len(podOut.Spec.Containers))
}
}
func TestSidecarSetPodInjectPolicy(t *testing.T) {
sidecarSetIn := sidecarSet1.DeepCopy()
testSidecarSetPodInjectPolicy(t, sidecarSetIn)
}
func testSidecarSetPodInjectPolicy(t *testing.T, sidecarSetIn *appsv1alpha1.SidecarSet) {
podIn := pod1.DeepCopy()
decoder, _ := admission.NewDecoder(scheme.Scheme)
client := fake.NewFakeClient(sidecarSetIn)
podOut := podIn.DeepCopy()
podHandler := &PodCreateHandler{Decoder: decoder, Client: client}
err := podHandler.sidecarsetMutatingPod(context.Background(), podOut, nil)
if err != nil {
t.Fatalf("inject sidecar into pod failed, err: %v", err)
}
expectLen := len(podIn.Spec.Containers) + len(sidecarSetIn.Spec.Containers)
if len(podOut.Spec.Containers) != expectLen {
t.Fatalf("expect %v containers but got %v", expectLen, len(podOut.Spec.Containers))
}
for i, container := range podOut.Spec.Containers {
switch i {
case 0:
if container.Name != "dns-f" {
t.Fatalf("expect dns-f but got %v", container.Name)
}
case 1:
if container.Name != "nginx" {
t.Fatalf("expect nginx but got %v", container.Name)
}
case 2:
if container.Name != "log-agent" {
t.Fatalf("expect log-agent but got %v", container.Name)
}
}
}
expectInitLen := len(podIn.Spec.InitContainers) + len(sidecarSetIn.Spec.InitContainers)
if len(podOut.Spec.InitContainers) != expectInitLen {
t.Fatalf("expect %v initContainers but got %v", expectLen, len(podOut.Spec.InitContainers))
}
for i, container := range podOut.Spec.InitContainers {
//injected container must be contain env "IS_INJECTED"
if i > 0 {
exist := false
for _, env := range container.Env {
if env.Name == sidecarcontrol.SidecarEnvKey {
exist = true
break
}
}
if !exist {
t.Fatalf("Injected initContainer %v don't contain env(%v)", container.Name, sidecarcontrol.SidecarEnvKey)
}
}
switch i {
case 0:
if container.Name != "init-0" {
t.Fatalf("expect dns-f but got %v", container.Name)
}
case 1:
if container.Name != "init-1" {
t.Fatalf("expect nginx but got %v", container.Name)
}
case 2:
if container.Name != "init-2" {
t.Fatalf("expect log-agent but got %v", container.Name)
}
}
}
}
func TestSidecarVolumesAppend(t *testing.T) {
sidecarSetIn := sidecarsetWithTransferEnv.DeepCopy()
testSidecarVolumesAppend(t, sidecarSetIn)
}
func testSidecarVolumesAppend(t *testing.T, sidecarSetIn *appsv1alpha1.SidecarSet) {
podIn := pod1.DeepCopy()
decoder, _ := admission.NewDecoder(scheme.Scheme)
client := fake.NewFakeClient(sidecarSetIn)
podOut := podIn.DeepCopy()
podHandler := &PodCreateHandler{Decoder: decoder, Client: client}
err := podHandler.sidecarsetMutatingPod(context.Background(), podOut, nil)
if err != nil {
t.Fatalf("inject sidecar into pod failed, err: %v", err)
}
expectLen := len(podIn.Spec.Volumes) + 1
if len(podOut.Spec.Volumes) != expectLen {
t.Fatalf("expect %v volumes but got %v", expectLen, len(podOut.Spec.Volumes))
}
for i, volume := range podOut.Spec.Volumes {
switch i {
case 0:
if volume.Name != "volume-a" {
t.Fatalf("expect volume-a but got %v", volume.Name)
}
case 1:
if volume.Name != "volume-b" {
t.Fatalf("expect volume-b but got %v", volume.Name)
}
case 2:
if volume.Name != "volume-1" {
t.Fatalf("expect volume-1 but got %v", volume.Name)
}
}
}
}
func TestPodVolumeMountsAppend(t *testing.T) {
sidecarSetIn := sidecarSetWithStaragent.DeepCopy()
// /a/b/c, /d/e/f, /staragent
sidecarSetIn.Spec.Containers = sidecarSetIn.Spec.Containers[:1]
testPodVolumeMountsAppend(t, sidecarSetIn)
}
func testPodVolumeMountsAppend(t *testing.T, sidecarSetIn *appsv1alpha1.SidecarSet) {
// /a/b、/e/f
podIn := podWithStaragent.DeepCopy()
cases := []struct {
name string
getPod func() *corev1.Pod
getSidecarSets func() *appsv1alpha1.SidecarSet
exceptVolumeMounts []string
exceptEnvs []string
}{
{
name: "append normal volumeMounts",
getPod: func() *corev1.Pod {
return podIn.DeepCopy()
},
getSidecarSets: func() *appsv1alpha1.SidecarSet {
return sidecarSetIn.DeepCopy()
},
exceptVolumeMounts: []string{"/a/b", "/e/f", "/a/b/c", "/d/e/f", "/staragent"},
},
{
name: "append volumeMounts SubPathExpr, volumes with expanded subpath",
getPod: func() *corev1.Pod {
podOut := podIn.DeepCopy()
podOut.Spec.Containers[0].VolumeMounts = append(podOut.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
Name: "volume-expansion",
MountPath: "/e/expansion",
SubPathExpr: "foo/$(POD_NAME)/$(OD_NAME)/conf",
})
podOut.Spec.Containers[0].Env = append(podOut.Spec.Containers[0].Env, corev1.EnvVar{
Name: "POD_NAME",
Value: "bar",
})
podOut.Spec.Containers[0].Env = append(podOut.Spec.Containers[0].Env, corev1.EnvVar{
Name: "OD_NAME",
Value: "od_name",
})
return podOut
},
getSidecarSets: func() *appsv1alpha1.SidecarSet {
return sidecarSetIn.DeepCopy()
},
exceptVolumeMounts: []string{"/a/b", "/e/f", "/a/b/c", "/d/e/f", "/staragent", "/e/expansion"},
exceptEnvs: []string{"POD_NAME", "OD_NAME"},
},
{
name: "append volumeMounts SubPathExpr, subpath with no expansion",
getPod: func() *corev1.Pod {
podOut := podIn.DeepCopy()
podOut.Spec.Containers[0].VolumeMounts = append(podOut.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
Name: "volume-expansion",
MountPath: "/e/expansion",
SubPathExpr: "foo",
})
return podOut
},
getSidecarSets: func() *appsv1alpha1.SidecarSet {
return sidecarSetIn.DeepCopy()
},
exceptVolumeMounts: []string{"/a/b", "/e/f", "/a/b/c", "/d/e/f", "/staragent", "/e/expansion"},
},
{
name: "append volumeMounts SubPathExpr, volumes expanded with empty subpath",
getPod: func() *corev1.Pod {
podOut := podIn.DeepCopy()
podOut.Spec.Containers[0].VolumeMounts = append(podOut.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
Name: "volume-expansion",
MountPath: "/e/expansion",
SubPathExpr: "",
})
return podOut
},
getSidecarSets: func() *appsv1alpha1.SidecarSet {
return sidecarSetIn.DeepCopy()
},
exceptVolumeMounts: []string{"/a/b", "/e/f", "/a/b/c", "/d/e/f", "/staragent", "/e/expansion"},
},
}
for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) {
podIn := cs.getPod()
decoder, _ := admission.NewDecoder(scheme.Scheme)
client := fake.NewFakeClient(cs.getSidecarSets())
podOut := podIn.DeepCopy()
podHandler := &PodCreateHandler{Decoder: decoder, Client: client}
err := podHandler.sidecarsetMutatingPod(context.Background(), podOut, nil)
if err != nil {
t.Fatalf("inject sidecar into pod failed, err: %v", err)
}
for _, mount := range cs.exceptVolumeMounts {
if util.GetContainerVolumeMount(&podOut.Spec.Containers[1], mount) == nil {
t.Fatalf("expect volume mounts %s but got nil", mount)
}
}
for _, env := range cs.exceptEnvs {
if util.GetContainerEnvVar(&podOut.Spec.Containers[1], env) == nil {
t.Fatalf("expect env %s but got nil", env)
}
}
})
}
}
func TestSidecarSetTransferEnv(t *testing.T) {
sidecarSetIn := sidecarsetWithTransferEnv.DeepCopy()
testSidecarSetTransferEnv(t, sidecarSetIn)
}
func testSidecarSetTransferEnv(t *testing.T, sidecarSetIn *appsv1alpha1.SidecarSet) {
podIn := pod1.DeepCopy()
decoder, _ := admission.NewDecoder(scheme.Scheme)
client := fake.NewFakeClient(sidecarSetIn)
podOut := podIn.DeepCopy()
podHandler := &PodCreateHandler{Decoder: decoder, Client: client}
err := podHandler.sidecarsetMutatingPod(context.Background(), podOut, nil)
if err != nil {
t.Fatalf("inject sidecar into pod failed, err: %v", err)
}
if len(podOut.Spec.Containers[0].Env) != 2 {
t.Fatalf("expect 2 envs but got %v", len(podOut.Spec.Containers[0].Env))
}
if podOut.Spec.Containers[0].Env[1].Value != "world2" {
t.Fatalf("expect env with value 'world2' but got %v", podOut.Spec.Containers[0].Env[1].Value)
}
}
func TestSidecarSetHashInject(t *testing.T) {
sidecarSetIn1 := sidecarSet1.DeepCopy()
testSidecarSetHashInject(t, sidecarSetIn1)
}
func testSidecarSetHashInject(t *testing.T, sidecarSetIn1 *appsv1alpha1.SidecarSet) {
podIn := pod1.DeepCopy()
sidecarSetIn1.Spec.Selector.MatchLabels["app"] = "doesnt-match"
sidecarSetIn2 := sidecarsetWithTransferEnv.DeepCopy()
sidecarSetIn3 := sidecarSet3.DeepCopy()
decoder, _ := admission.NewDecoder(scheme.Scheme)
client := fake.NewFakeClient(sidecarSetIn1, sidecarSetIn2, sidecarSetIn3)
podOut := podIn.DeepCopy()
podHandler := &PodCreateHandler{Decoder: decoder, Client: client}
err := podHandler.sidecarsetMutatingPod(context.Background(), podOut, nil)
if err != nil {
t.Fatalf("inject sidecar into pod failed, err: %v", err)
}
hashKey := sidecarcontrol.SidecarSetHashAnnotation
//expectedAnnotation := `{"sidecarset2":"c4k2dbb95d","sidecarset3":"gm967682cm"}`
expectedRevision := map[string]string{
"sidecarset2": "c4k2dbb95d",
"sidecarset3": "gm967682cm",
}
for k, v := range expectedRevision {
if sidecarcontrol.GetPodSidecarSetRevision(k, podOut) != v {
t.Errorf("except sidecarset(%s:%s), but get in pod annotations(%s)", k, v, podOut.Annotations[hashKey])
}
}
}
func TestSidecarSetNameInject(t *testing.T) {
sidecarSetIn1 := sidecarSet1.DeepCopy()
sidecarSetIn3 := sidecarSet3.DeepCopy()
testSidecarSetNameInject(t, sidecarSetIn1, sidecarSetIn3)
}
func testSidecarSetNameInject(t *testing.T, sidecarSetIn1, sidecarSetIn3 *appsv1alpha1.SidecarSet) {
podIn := pod1.DeepCopy()
decoder, _ := admission.NewDecoder(scheme.Scheme)
client := fake.NewFakeClient(sidecarSetIn1, sidecarSetIn3)
podOut := podIn.DeepCopy()
podHandler := &PodCreateHandler{Decoder: decoder, Client: client}
err := podHandler.sidecarsetMutatingPod(context.Background(), podOut, nil)
if err != nil {
t.Fatalf("inject sidecar into pod failed, err: %v", err)
}
sidecarSetListKey := sidecarcontrol.SidecarSetListAnnotation
expectedAnnotation := "sidecarset1,sidecarset3"
if podOut.Annotations[sidecarSetListKey] != expectedAnnotation {
t.Errorf("expect annotation %v but got %v", expectedAnnotation, podOut.Annotations[sidecarSetListKey])
}
}
func TestMergeSidecarContainers(t *testing.T) {
podContainers := []corev1.Container{
{
Name: "sidecar-1",
},
{
Name: "app-container",
},
{
Name: "sidecar-2",
},
}
sidecarContainers := []*appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{
Name: "sidecar-1",
},
PodInjectPolicy: appsv1alpha1.AfterAppContainerType,
},
{
Container: corev1.Container{
Name: "sidecar-2",
},
PodInjectPolicy: appsv1alpha1.AfterAppContainerType,
},
{
Container: corev1.Container{
Name: "new-sidecar-1",
},
PodInjectPolicy: appsv1alpha1.BeforeAppContainerType,
},
}
cases := []struct {
name string
getOrigins func() []corev1.Container
getInjected func() []*appsv1alpha1.SidecarContainer
expectContainerLen int
expectedContainers []string
}{
{
name: "origins not sidecar, and inject new sidecar",
getOrigins: func() []corev1.Container {
return podContainers[1:2]
},
getInjected: func() []*appsv1alpha1.SidecarContainer {
return sidecarContainers
},
expectContainerLen: 4,
expectedContainers: []string{"new-sidecar-1", "app-container", "sidecar-1", "sidecar-2"},
},
{
name: "origins not sidecar, and inject new sidecar, only before app container",
getOrigins: func() []corev1.Container {
return podContainers[1:2]
},
getInjected: func() []*appsv1alpha1.SidecarContainer {
return sidecarContainers[2:]
},
expectContainerLen: 2,
expectedContainers: []string{"new-sidecar-1", "app-container"},
},
{
name: "origins not sidecar, and inject new sidecar, only after app container",
getOrigins: func() []corev1.Container {
return podContainers[1:2]
},
getInjected: func() []*appsv1alpha1.SidecarContainer {
return sidecarContainers[:2]
},
expectContainerLen: 3,
expectedContainers: []string{"app-container", "sidecar-1", "sidecar-2"},
},
{
name: "origin have sidecars, sidecar no new containers",
getOrigins: func() []corev1.Container {
return podContainers
},
getInjected: func() []*appsv1alpha1.SidecarContainer {
return sidecarContainers[:2]
},
expectContainerLen: 3,
expectedContainers: []string{"sidecar-1", "app-container", "sidecar-2"},
},
{
name: "origin have sidecars, sidecar have new containers",
getOrigins: func() []corev1.Container {
return podContainers
},
getInjected: func() []*appsv1alpha1.SidecarContainer {
return sidecarContainers
},
expectContainerLen: 4,
expectedContainers: []string{"new-sidecar-1", "sidecar-1", "app-container", "sidecar-2"},
},
}
for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) {
origins := cs.getOrigins()
injected := cs.getInjected()
finals := mergeSidecarContainers(origins, injected)
if len(finals) != cs.expectContainerLen {
t.Fatalf("expect %d containers but got %v", cs.expectContainerLen, len(finals))
}
for index, cName := range cs.expectedContainers {
if finals[index].Name != cName {
t.Fatalf("expect index(%d) container(%s) but got %s", index, cName, finals[index].Name)
}
}
})
}
}

10
pkg/webhook/sidecarset/mutating/hash.go
View File

@ -29,7 +29,7 @@ import (
// SidecarSetHash returns a hash of the SidecarSet.
// The Containers are taken into account.
func SidecarSetHash(sidecarSet *appsv1alpha1.SidecarSet) (string, error) {
encoded, err := encodeSidecarSet(sidecarSet, true)
encoded, err := encodeSidecarSet(sidecarSet)
if err != nil {
return "", err
}
@ -44,19 +44,17 @@ func SidecarSetHashWithoutImage(sidecarSet *appsv1alpha1.SidecarSet) (string, er
for i := range ss.Spec.Containers {
ss.Spec.Containers[i].Image = ""
}
encoded, err := encodeSidecarSet(ss, false)
encoded, err := encodeSidecarSet(ss)
if err != nil {
return "", err
}
return rand.SafeEncodeString(hash(encoded)), nil
}
func encodeSidecarSet(sidecarSet *appsv1alpha1.SidecarSet, includeInit bool) (string, error) {
func encodeSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (string, error) {
// json.Marshal sorts the keys in a stable order in the encoding
m := map[string]interface{}{"containers": sidecarSet.Spec.Containers}
if includeInit {
m["initContainers"] = sidecarSet.Spec.InitContainers
}
m["initContainers"] = sidecarSet.Spec.InitContainers
data, err := json.Marshal(m)
if err != nil {
return "", err

13
pkg/webhook/sidecarset/mutating/sidecarset_create_update_handler.go
View File

@ -22,19 +22,14 @@ import (
"net/http"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/control/sidecarcontrol"
"github.com/openkruise/kruise/pkg/util"
"k8s.io/api/admission/v1beta1"
"k8s.io/klog"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)
const (
// SidecarSetHashAnnotation represents the key of a sidecarset hash
SidecarSetHashAnnotation = "kruise.io/sidecarset-hash"
// SidecarSetHashWithoutImageAnnotation represents the key of a sidecarset hash without images of sidecar
SidecarSetHashWithoutImageAnnotation = "kruise.io/sidecarset-hash-without-image"
)
// SidecarSetCreateHandler handles SidecarSet
type SidecarSetCreateHandler struct {
// To use the client, you need to do the following:
@ -56,13 +51,13 @@ func setHashSidecarSet(sidecarset *appsv1alpha1.SidecarSet) error {
if err != nil {
return err
}
sidecarset.Annotations[SidecarSetHashAnnotation] = hash
sidecarset.Annotations[sidecarcontrol.SidecarSetHashAnnotation] = hash
hash, err = SidecarSetHashWithoutImage(sidecarset)
if err != nil {
return err
}
sidecarset.Annotations[SidecarSetHashWithoutImageAnnotation] = hash
sidecarset.Annotations[sidecarcontrol.SidecarSetHashWithoutImageAnnotation] = hash
return nil
}

137
pkg/webhook/sidecarset/mutating/sidecarset_mutating_test.go
View File

@ -1,113 +1,66 @@
package mutating
import (
"reflect"
"testing"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/intstr"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/control/sidecarcontrol"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
var (
sidecarSetDemo = &appsv1alpha1.SidecarSet{
func TestMutatingSidecarSetFn(t *testing.T) {
sidecarSet := &appsv1alpha1.SidecarSet{
ObjectMeta: metav1.ObjectMeta{
ResourceVersion: "123",
Name: "sidecarset-test",
},
Spec: appsv1alpha1.SidecarSetSpec{
InitContainers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{
Name: "test-init-containers",
Image: "test-init-image:latest",
},
},
},
Containers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{
Name: "test-sidecar",
Image: "test-image:v1",
Name: "dns-f",
Image: "dns:1.0",
},
},
},
},
}
)
func TestSidecarSetDefault(t *testing.T) {
sidecarSet := sidecarSetDemo.DeepCopy()
expectedOutputSidecarSet := sidecarSet.DeepCopy()
expectedOutputSidecarSet.Spec.Containers[0].TerminationMessagePath = corev1.TerminationMessagePathDefault
expectedOutputSidecarSet.Spec.Containers[0].TerminationMessagePolicy = corev1.TerminationMessageReadFile
expectedOutputSidecarSet.Spec.Containers[0].ImagePullPolicy = corev1.PullIfNotPresent
expectedOutputSidecarSet.Spec.InitContainers[0].TerminationMessagePath = corev1.TerminationMessagePathDefault
expectedOutputSidecarSet.Spec.InitContainers[0].TerminationMessagePolicy = corev1.TerminationMessageReadFile
expectedOutputSidecarSet.Spec.InitContainers[0].ImagePullPolicy = corev1.PullAlways
maxUnavailable := intstr.FromInt(1)
expectedOutputSidecarSet.Spec.Strategy = appsv1alpha1.SidecarSetUpdateStrategy{
RollingUpdate: &appsv1alpha1.RollingUpdateSidecarSet{
MaxUnavailable: &maxUnavailable,
},
}
appsv1alpha1.SetDefaultsSidecarSet(sidecarSet)
_ = setHashSidecarSet(sidecarSet)
if sidecarSet.Spec.Strategy.Type != appsv1alpha1.NotUpdateSidecarSetStrategyType {
t.Fatalf("update strategy not initialized")
}
if *sidecarSet.Spec.Strategy.Partition != intstr.FromInt(0) {
t.Fatalf("partition not initialized")
}
if *sidecarSet.Spec.Strategy.MaxUnavailable != intstr.FromInt(1) {
t.Fatalf("maxUnavailable not initialized")
}
for _, container := range sidecarSet.Spec.Containers {
if container.PodInjectPolicy != appsv1alpha1.BeforeAppContainerType {
t.Fatalf("container %v podInjectPolicy initialized incorrectly", container.Name)
}
if container.ShareVolumePolicy.Type != appsv1alpha1.ShareVolumePolicyDisabled {
t.Fatalf("container %v shareVolumePolicy initialized incorrectly", container.Name)
}
if sidecarSet.Spec.Containers[0].UpgradeStrategy.UpgradeType != appsv1alpha1.SidecarContainerColdUpgrade {
t.Fatalf("container %v upgradePolicy initialized incorrectly", container.Name)
}
if !reflect.DeepEqual(expectedOutputSidecarSet, sidecarSet) {
t.Errorf("\nexpect:\n%+v\nbut got:\n%+v", expectedOutputSidecarSet, sidecarSet)
}
}
func TestSidecarSetHash(t *testing.T) {
sidecarSet := sidecarSetDemo.DeepCopy()
expectedOutputSidecarSet := sidecarSet.DeepCopy()
if expectedOutputSidecarSet.Annotations == nil {
expectedOutputSidecarSet.Annotations = make(map[string]string)
}
expectedOutputSidecarSet.Annotations[SidecarSetHashAnnotation] = "8f92wdb9w96824dvw54566vx89wcxd6b75cd4ccxbv4zcvbd7fvfffw4v889dcz2"
expectedOutputSidecarSet.Annotations[SidecarSetHashWithoutImageAnnotation] = "cfd67dc8z844x4f7cd9f7b624x5ddxxd97wdwv45x48z49cx4942w5c8z84v2dzx"
if err := setHashSidecarSet(sidecarSet); err != nil {
t.Errorf("got error %v", err)
}
if !reflect.DeepEqual(expectedOutputSidecarSet, sidecarSet) {
t.Errorf("\nexpect:\n%+v\nbut got:\n%+v", expectedOutputSidecarSet, sidecarSet)
}
}
func TestSidecarSetHashWithoutImage(t *testing.T) {
hashDemo, err := SidecarSetHashWithoutImage(sidecarSetDemo)
if err != nil {
t.Errorf("SidecarSetHashWithoutImage got error %v", err)
}
// change the container image alone won't change the hashWithoutImage value
sidecarSet := sidecarSetDemo.DeepCopy()
sidecarSet.Spec.Containers[0].Image = "test-image:v2"
hashNew, err := SidecarSetHashWithoutImage(sidecarSet)
if err != nil {
t.Errorf("SidecarSetHashWithoutImage got error %v", err)
}
if hashNew != hashDemo {
t.Errorf("\nexpect:\n%+v\nbut got:\n%+v", hashDemo, hashNew)
}
// change any of the init container won't change the hashWithoutImage value
sidecarSet.Spec.InitContainers[0].Image = "test-init-image:v1"
sidecarSet.Spec.InitContainers[0].Command = []string{"sh", "-c", "ls -l"}
hashNew, err = SidecarSetHashWithoutImage(sidecarSet)
if err != nil {
t.Errorf("SidecarSetHashWithoutImage got error %v", err)
}
if hashNew != hashDemo {
t.Errorf("\nexpect:\n%+v\nbut got:\n%+v", hashDemo, hashNew)
}
// change the other part of container will change the hashWithoutImage value
sidecarSet.Spec.Containers[0].WorkingDir = "/tmp"
hashNew, err = SidecarSetHashWithoutImage(sidecarSet)
if err != nil {
t.Errorf("SidecarSetHashWithoutImage got error %v", err)
}
if hashNew == hashDemo {
t.Errorf("\ndon't expect:\n%+v\nbut got:\n%+v", hashDemo, hashNew)
if container.ImagePullPolicy != corev1.PullIfNotPresent {
t.Fatalf("container %v imagePullPolicy initialized incorrectly", container.Name)
}
if container.TerminationMessagePath != "/dev/termination-log" {
t.Fatalf("container %v terminationMessagePath initialized incorrectly", container.Name)
}
if container.TerminationMessagePolicy != corev1.TerminationMessageReadFile {
t.Fatalf("container %v terminationMessagePolicy initialized incorrectly", container.Name)
}
}
if sidecarSet.Annotations[sidecarcontrol.SidecarSetHashAnnotation] != "9w829wfc74c22465fv2z2dwf54x7c5wv6424f98dv7bcwx8444768wf6wfv4bdfc" {
t.Fatalf("sidecarset %v hash initialized incorrectly, got %v", sidecarSet.Name, sidecarSet.Annotations[sidecarcontrol.SidecarSetHashAnnotation])
}
}

246
pkg/webhook/sidecarset/validating/sidecarset_create_update_handler.go
View File

@ -20,22 +20,26 @@ import (
"context"
"fmt"
"net/http"
"reflect"
"regexp"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/util"
admissionv1beta1 "k8s.io/api/admission/v1beta1"
v1 "k8s.io/api/core/v1"
genericvalidation "k8s.io/apimachinery/pkg/api/validation"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
metavalidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation"
validationutil "k8s.io/apimachinery/pkg/util/validation"
"k8s.io/apimachinery/pkg/util/validation/field"
validationfield "k8s.io/apimachinery/pkg/util/validation/field"
appsvalidation "k8s.io/kubernetes/pkg/apis/apps/validation"
"k8s.io/kubernetes/pkg/apis/core"
corev1 "k8s.io/kubernetes/pkg/apis/core/v1"
corevalidation "k8s.io/kubernetes/pkg/apis/core/validation"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
)
const (
@ -54,23 +58,35 @@ type SidecarSetCreateUpdateHandler struct {
// - uncomment it
// - import sigs.k8s.io/controller-runtime/pkg/client
// - uncomment the InjectClient method at the bottom of this file.
// Client client.Client
Client client.Client
// Decoder decodes objects
Decoder *admission.Decoder
}
func (h *SidecarSetCreateUpdateHandler) validatingSidecarSetFn(ctx context.Context, obj *appsv1alpha1.SidecarSet) (bool, string, error) {
allErrs := validateSidecarSet(obj)
func (h *SidecarSetCreateUpdateHandler) validatingSidecarSetFn(ctx context.Context, obj *appsv1alpha1.SidecarSet, older *appsv1alpha1.SidecarSet) (bool, string, error) {
allErrs := h.validateSidecarSet(obj, older)
if len(allErrs) != 0 {
return false, "", allErrs.ToAggregate()
}
return true, "allowed to be admitted", nil
}
func validateSidecarSet(obj *appsv1alpha1.SidecarSet) field.ErrorList {
func (h *SidecarSetCreateUpdateHandler) validateSidecarSet(obj *appsv1alpha1.SidecarSet, older *appsv1alpha1.SidecarSet) field.ErrorList {
// validating ObjectMeta
allErrs := genericvalidation.ValidateObjectMeta(&obj.ObjectMeta, false, validateSidecarSetName, field.NewPath("metadata"))
// validating spec
allErrs = append(allErrs, validateSidecarSetSpec(obj, field.NewPath("spec"))...)
// when operation is update, older isn't empty, and validating whether old and new containers conflict
if older != nil {
allErrs = append(allErrs, validateSidecarContainerConflict(obj.Spec.Containers, older.Spec.Containers, field.NewPath("spec.containers"))...)
}
// iterate across all containers in other sidecarsets to avoid duplication of name
sidecarSets := &appsv1alpha1.SidecarSetList{}
if err := h.Client.List(context.TODO(), sidecarSets, &client.ListOptions{}); err != nil {
allErrs = append(allErrs, field.InternalError(field.NewPath(""), fmt.Errorf("query other sidecarsets failed, err: %v", err)))
}
allErrs = append(allErrs, validateSidecarConflict(sidecarSets, obj, field.NewPath("spec"))...)
return allErrs
}
@ -88,67 +104,87 @@ func validateSidecarSetSpec(obj *appsv1alpha1.SidecarSet, fldPath *field.Path) f
spec := &obj.Spec
allErrs := field.ErrorList{}
//validate spec selector
if spec.Selector == nil {
allErrs = append(allErrs, field.Required(fldPath.Child("selector"), "no selector defined for sidecarset"))
allErrs = append(allErrs, field.Required(fldPath.Child("selector"), "no selector defined for SidecarSet"))
} else {
allErrs = append(allErrs, metavalidation.ValidateLabelSelector(spec.Selector, fldPath.Child("selector"))...)
if len(spec.Selector.MatchLabels)+len(spec.Selector.MatchExpressions) == 0 {
allErrs = append(allErrs, field.Invalid(fldPath.Child("selector"), spec.Selector, "empty selector is not valid for sidecarset."))
}
allErrs = append(allErrs, validateSelector(spec.Selector, fldPath.Child("selector"))...)
}
allErrs = append(allErrs, validateSidecarSetStrategy(&spec.Strategy, fldPath.Child("strategy"))...)
//validating SidecarSetUpdateStrategy
allErrs = append(allErrs, validateSidecarSetUpdateStrategy(&spec.Strategy, fldPath.Child("strategy"))...)
//validating volumes
vols, vErrs := getCoreVolumes(spec.Volumes, fldPath.Child("volumes"))
allErrs = append(allErrs, vErrs...)
allErrs = append(allErrs, validateContainersForSidecarSet(spec.InitContainers, spec.Containers, vols, fldPath.Child("containers"))...)
return allErrs
}
func validateSidecarSetStrategy(strategy *appsv1alpha1.SidecarSetUpdateStrategy, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
if strategy.RollingUpdate == nil {
allErrs = append(allErrs, validationfield.Required(fldPath.Child("rollingUpdate"), ""))
//validating sidecar container
// if don't have any initContainers, containers
if len(spec.InitContainers) == 0 && len(spec.Containers) == 0 {
allErrs = append(allErrs, field.Required(fldPath.Root(), "no initContainer or container defined for SidecarSet"))
} else {
allErrs = append(allErrs, appsvalidation.ValidatePositiveIntOrPercent(*(strategy.RollingUpdate.MaxUnavailable), fldPath.Child("maxUnavailable"))...)
allErrs = append(allErrs, validateContainersForSidecarSet(spec.InitContainers, spec.Containers, vols, fldPath.Root())...)
}
return allErrs
}
func validateSelector(selector *metav1.LabelSelector, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
allErrs = append(allErrs, metavalidation.ValidateLabelSelector(selector, fldPath)...)
if len(selector.MatchLabels)+len(selector.MatchExpressions) == 0 {
allErrs = append(allErrs, field.Invalid(fldPath, selector, "empty selector is not valid for sidecarset."))
}
return allErrs
}
func getCoreVolumes(volumes []v1.Volume, fldPath *field.Path) ([]core.Volume, field.ErrorList) {
func validateSidecarSetUpdateStrategy(strategy *appsv1alpha1.SidecarSetUpdateStrategy, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
var coreVolumes []core.Volume
for _, volume := range volumes {
coreVolume := core.Volume{}
if err := corev1.Convert_v1_Volume_To_core_Volume(&volume, &coreVolume, nil); err != nil {
allErrs = append(allErrs, field.Invalid(fldPath.Root(), volume, fmt.Sprintf("Convert_v1_Volume_To_core_Volume failed: %v", err)))
return nil, allErrs
// if SidecarSet update strategy is RollingUpdate
if strategy.Type == appsv1alpha1.RollingUpdateSidecarSetStrategyType {
if strategy.Selector != nil {
allErrs = append(allErrs, validateSelector(strategy.Selector, fldPath.Child("selector"))...)
}
if strategy.Partition != nil {
allErrs = append(allErrs, appsvalidation.ValidatePositiveIntOrPercent(*(strategy.Partition), fldPath.Child("partition"))...)
}
if strategy.MaxUnavailable != nil {
allErrs = append(allErrs, appsvalidation.ValidatePositiveIntOrPercent(*(strategy.MaxUnavailable), fldPath.Child("maxUnavailable"))...)
}
if strategy.ScatterStrategy != nil {
if err := strategy.ScatterStrategy.FieldsValidation(); err != nil {
allErrs = append(allErrs, field.Required(fldPath.Child("scatterStrategy"), err.Error()))
}
}
coreVolumes = append(coreVolumes, coreVolume)
}
return coreVolumes, allErrs
return allErrs
}
func validateContainersForSidecarSet(
initContainers, containers []appsv1alpha1.SidecarContainer, coreVolumes []core.Volume, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
initContainers, containers []appsv1alpha1.SidecarContainer,
coreVolumes []core.Volume, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
//validating initContainer
var coreInitContainers []core.Container
for _, container := range initContainers {
coreContainer := core.Container{}
if err := corev1.Convert_v1_Container_To_core_Container(&container.Container, &coreContainer, nil); err != nil {
allErrs = append(allErrs, field.Invalid(fldPath.Root(), container.Container, fmt.Sprintf("Convert_v1_Container_To_core_Container failed: %v", err)))
allErrs = append(allErrs, field.Invalid(fldPath.Child("initContainer"), container.Container, fmt.Sprintf("Convert_v1_Container_To_core_Container failed: %v", err)))
return allErrs
}
coreInitContainers = append(coreInitContainers, coreContainer)
}
//validating container
var coreContainers []core.Container
for _, container := range containers {
if container.PodInjectPolicy != appsv1alpha1.BeforeAppContainerType && container.PodInjectPolicy != appsv1alpha1.AfterAppContainerType {
allErrs = append(allErrs, field.Invalid(fldPath.Child("container").Child("podInjectPolicy"), container.PodInjectPolicy, "unsupported pod inject policy"))
}
if container.ShareVolumePolicy.Type != appsv1alpha1.ShareVolumePolicyEnabled && container.ShareVolumePolicy.Type != appsv1alpha1.ShareVolumePolicyDisabled {
allErrs = append(allErrs, field.Invalid(fldPath.Child("container").Child("shareVolumePolicy"), container.ShareVolumePolicy, "unsupported share volume policy"))
}
coreContainer := core.Container{}
if err := corev1.Convert_v1_Container_To_core_Container(&container.Container, &coreContainer, nil); err != nil {
allErrs = append(allErrs, field.Invalid(fldPath.Root(), container.Container, fmt.Sprintf("Convert_v1_Container_To_core_Container failed: %v", err)))
allErrs = append(allErrs, field.Invalid(fldPath.Child("container"), container.Container, fmt.Sprintf("Convert_v1_Container_To_core_Container failed: %v", err)))
return allErrs
}
coreContainers = append(coreContainers, coreContainer)
@ -184,6 +220,111 @@ func validateContainersForSidecarSet(
return allErrs
}
func validateSidecarContainerConflict(newContainers, oldContainers []appsv1alpha1.SidecarContainer, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
oldStrategy := make(map[string]appsv1alpha1.SidecarContainerUpgradeType)
for _, container := range oldContainers {
oldStrategy[container.Name] = container.UpgradeStrategy.UpgradeType
}
for _, container := range newContainers {
if strategy, ok := oldStrategy[container.Name]; ok {
if strategy != "" && container.UpgradeStrategy.UpgradeType != strategy {
allErrs = append(allErrs, field.Invalid(fldPath.Child("upgradeStrategy").Child("upgradeType"),
container.Name, fmt.Sprintf("container %v upgradeType is immutable", container.Name)))
}
}
}
return allErrs
}
// validate the sidecarset spec.container.name, spec.initContainer.name, volume.name conflicts with others in cluster
func validateSidecarConflict(sidecarSets *appsv1alpha1.SidecarSetList, sidecarSet *appsv1alpha1.SidecarSet, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
// record initContainer, container, volume name of other sidecarsets in cluster
// container name -> sidecarset
containerInOthers := make(map[string]*appsv1alpha1.SidecarSet)
// volume name -> sidecarset
volumeInOthers := make(map[string]*appsv1alpha1.SidecarSet)
// init container name -> sidecarset
initContainerInOthers := make(map[string]*appsv1alpha1.SidecarSet)
for i := range sidecarSets.Items {
set := &sidecarSets.Items[i]
//ignore this sidecarset
if set.Name == sidecarSet.Name {
continue
}
for _, container := range set.Spec.InitContainers {
initContainerInOthers[container.Name] = set
}
for _, container := range set.Spec.Containers {
containerInOthers[container.Name] = set
}
for _, volume := range set.Spec.Volumes {
volumeInOthers[volume.Name] = set
}
}
// whether initContainers conflict
for _, container := range sidecarSet.Spec.InitContainers {
if other, ok := initContainerInOthers[container.Name]; ok {
//if the two sidecarset scope namespace is different, continue
if isSidecarSetNamespaceDiff(sidecarSet, other) {
continue
}
// if the two sidecarset will selector same pod, then judge conflict
if util.IsSelectorOverlapping(sidecarSet.Spec.Selector, other.Spec.Selector) {
allErrs = append(allErrs, field.Invalid(fldPath.Child("containers"), container.Name, fmt.Sprintf(
"container %v already exist in %v", container.Name, other.Name)))
}
}
}
// whether containers conflict
for _, container := range sidecarSet.Spec.Containers {
if other, ok := containerInOthers[container.Name]; ok {
// if the two sidecarset scope namespace is different, continue
if isSidecarSetNamespaceDiff(sidecarSet, other) {
continue
}
// if the two sidecarset will selector same pod, then judge conflict
if util.IsSelectorOverlapping(sidecarSet.Spec.Selector, other.Spec.Selector) {
allErrs = append(allErrs, field.Invalid(fldPath.Child("containers"), container.Name, fmt.Sprintf(
"container %v already exist in %v", container.Name, other.Name)))
}
}
}
// whether volumes conflict
for _, volume := range sidecarSet.Spec.Volumes {
if other, ok := volumeInOthers[volume.Name]; ok {
//if the two sidecarset scope namespace is different, continue
if isSidecarSetNamespaceDiff(sidecarSet, other) {
continue
}
// if the two sidecarset will selector same pod, then judge conflict
if util.IsSelectorOverlapping(sidecarSet.Spec.Selector, other.Spec.Selector) {
if !reflect.DeepEqual(&volume, getSidecarsetVolume(volume.Name, other)) {
allErrs = append(allErrs, field.Invalid(fldPath.Child("volumes"), volume.Name, fmt.Sprintf(
"volume %s is in conflict with sidecarset %s", volume.Name, other.Name)))
}
}
}
}
return allErrs
}
func getSidecarsetVolume(volumeName string, sidecarset *appsv1alpha1.SidecarSet) *v1.Volume {
for _, volume := range sidecarset.Spec.Volumes {
if volume.Name == volumeName {
return &volume
}
}
return nil
}
var _ admission.Handler = &SidecarSetCreateUpdateHandler{}
// Handle handles admission requests.
@ -194,21 +335,30 @@ func (h *SidecarSetCreateUpdateHandler) Handle(ctx context.Context, req admissio
if err != nil {
return admission.Errored(http.StatusBadRequest, err)
}
allowed, reason, err := h.validatingSidecarSetFn(ctx, obj)
var oldSidecarSet *appsv1alpha1.SidecarSet
//when Operation is update, decode older object
if req.AdmissionRequest.Operation == admissionv1beta1.Update {
oldSidecarSet = new(appsv1alpha1.SidecarSet)
if err := h.Decoder.Decode(
admission.Request{AdmissionRequest: admissionv1beta1.AdmissionRequest{Object: req.AdmissionRequest.OldObject}},
oldSidecarSet); err != nil {
return admission.Errored(http.StatusBadRequest, err)
}
}
allowed, reason, err := h.validatingSidecarSetFn(ctx, obj, oldSidecarSet)
if err != nil {
return admission.Errored(http.StatusInternalServerError, err)
}
return admission.ValidationResponse(allowed, reason)
}
//var _ inject.Client = &SidecarSetCreateUpdateHandler{}
//
//// InjectClient injects the client into the SidecarSetCreateUpdateHandler
//func (h *SidecarSetCreateUpdateHandler) InjectClient(c client.Client) error {
// h.Client = c
// return nil
//}
var _ inject.Client = &SidecarSetCreateUpdateHandler{}
// InjectClient injects the client into the SidecarSetCreateUpdateHandler
func (h *SidecarSetCreateUpdateHandler) InjectClient(c client.Client) error {
h.Client = c
return nil
}
var _ admission.DecoderInjector = &SidecarSetCreateUpdateHandler{}

475
pkg/webhook/sidecarset/validating/sidecarset_validating_test.go
View File

@ -4,16 +4,12 @@ import (
"fmt"
"testing"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
"github.com/openkruise/kruise/pkg/util"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
)
var (
maxUnavailable = intstr.FromInt(1)
wrongUnavailable = intstr.FromInt(-1)
"k8s.io/apimachinery/pkg/util/validation/field"
)
func TestValidateSidecarSet(t *testing.T) {
@ -22,12 +18,17 @@ func TestValidateSidecarSet(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{Name: "test-sidecarset"},
Spec: appsv1alpha1.SidecarSetSpec{
Strategy: appsv1alpha1.SidecarSetUpdateStrategy{
RollingUpdate: &appsv1alpha1.RollingUpdateSidecarSet{
MaxUnavailable: &maxUnavailable,
},
Type: appsv1alpha1.NotUpdateSidecarSetStrategyType,
},
Containers: []appsv1alpha1.SidecarContainer{
{
PodInjectPolicy: appsv1alpha1.BeforeAppContainerType,
ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{
Type: appsv1alpha1.ShareVolumePolicyDisabled,
},
UpgradeStrategy: appsv1alpha1.SidecarContainerUpgradeStrategy{
UpgradeType: appsv1alpha1.SidecarContainerColdUpgrade,
},
Container: corev1.Container{
Name: "test-sidecar",
Image: "test-image",
@ -38,28 +39,34 @@ func TestValidateSidecarSet(t *testing.T) {
},
},
},
"wrong-init-containers": {
"wrong-updateStrategy": {
ObjectMeta: metav1.ObjectMeta{Name: "test-sidecarset"},
Spec: appsv1alpha1.SidecarSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"a": "b"},
},
Strategy: appsv1alpha1.SidecarSetUpdateStrategy{
RollingUpdate: &appsv1alpha1.RollingUpdateSidecarSet{
MaxUnavailable: &maxUnavailable,
},
},
InitContainers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{
Name: "test-initcontainer",
Image: "test-initimage",
ImagePullPolicy: corev1.PullIfNotPresent,
Type: appsv1alpha1.RollingUpdateSidecarSetStrategyType,
ScatterStrategy: appsv1alpha1.UpdateScatterStrategy{
{
Key: "key-1",
Value: "value-1",
},
{
Key: "key-1",
Value: "value-1",
},
},
},
Containers: []appsv1alpha1.SidecarContainer{
{
PodInjectPolicy: appsv1alpha1.BeforeAppContainerType,
ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{
Type: appsv1alpha1.ShareVolumePolicyDisabled,
},
UpgradeStrategy: appsv1alpha1.SidecarContainerUpgradeStrategy{
UpgradeType: appsv1alpha1.SidecarContainerColdUpgrade,
},
Container: corev1.Container{
Name: "test-sidecar",
Image: "test-image",
@ -70,28 +77,37 @@ func TestValidateSidecarSet(t *testing.T) {
},
},
},
"wrong-init-containers-only": {
"wrong-initContainer": {
ObjectMeta: metav1.ObjectMeta{Name: "test-sidecarset"},
Spec: appsv1alpha1.SidecarSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"a": "b"},
},
Strategy: appsv1alpha1.SidecarSetUpdateStrategy{
RollingUpdate: &appsv1alpha1.RollingUpdateSidecarSet{
MaxUnavailable: &maxUnavailable,
},
Type: appsv1alpha1.NotUpdateSidecarSetStrategyType,
},
InitContainers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{
Name: "test-initcontainer",
Image: "test-initimage",
ImagePullPolicy: corev1.PullIfNotPresent,
Name: "test-sidecar",
ImagePullPolicy: corev1.PullIfNotPresent,
TerminationMessagePolicy: corev1.TerminationMessageReadFile,
},
},
},
},
},
"missing-container": {
ObjectMeta: metav1.ObjectMeta{Name: "test-sidecarset"},
Spec: appsv1alpha1.SidecarSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"a": "b"},
},
Strategy: appsv1alpha1.SidecarSetUpdateStrategy{
Type: appsv1alpha1.NotUpdateSidecarSetStrategyType,
},
},
},
"wrong-containers": {
ObjectMeta: metav1.ObjectMeta{Name: "test-sidecarset"},
Spec: appsv1alpha1.SidecarSetSpec{
@ -99,34 +115,15 @@ func TestValidateSidecarSet(t *testing.T) {
MatchLabels: map[string]string{"a": "b"},
},
Strategy: appsv1alpha1.SidecarSetUpdateStrategy{
RollingUpdate: &appsv1alpha1.RollingUpdateSidecarSet{
MaxUnavailable: &maxUnavailable,
},
Type: appsv1alpha1.NotUpdateSidecarSetStrategyType,
},
Containers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{
Name: "test-sidecar",
Image: "test-image",
ImagePullPolicy: corev1.PullIfNotPresent,
PodInjectPolicy: appsv1alpha1.BeforeAppContainerType,
UpgradeStrategy: appsv1alpha1.SidecarContainerUpgradeStrategy{
UpgradeType: appsv1alpha1.SidecarContainerColdUpgrade,
},
},
},
},
},
"wrong-rollingUpdate": {
ObjectMeta: metav1.ObjectMeta{Name: "test-sidecarset"},
Spec: appsv1alpha1.SidecarSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"a": "b"},
},
Strategy: appsv1alpha1.SidecarSetUpdateStrategy{
RollingUpdate: &appsv1alpha1.RollingUpdateSidecarSet{
MaxUnavailable: &wrongUnavailable,
},
},
Containers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{
Name: "test-sidecar",
Image: "test-image",
@ -144,12 +141,17 @@ func TestValidateSidecarSet(t *testing.T) {
MatchLabels: map[string]string{"a": "b"},
},
Strategy: appsv1alpha1.SidecarSetUpdateStrategy{
RollingUpdate: &appsv1alpha1.RollingUpdateSidecarSet{
MaxUnavailable: &maxUnavailable,
},
Type: appsv1alpha1.NotUpdateSidecarSetStrategyType,
},
Containers: []appsv1alpha1.SidecarContainer{
{
PodInjectPolicy: appsv1alpha1.BeforeAppContainerType,
ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{
Type: appsv1alpha1.ShareVolumePolicyDisabled,
},
UpgradeStrategy: appsv1alpha1.SidecarContainerUpgradeStrategy{
UpgradeType: appsv1alpha1.SidecarContainerColdUpgrade,
},
Container: corev1.Container{
Name: "test-sidecar",
Image: "test-image",
@ -168,11 +170,368 @@ func TestValidateSidecarSet(t *testing.T) {
}
for name, sidecarSet := range errorCases {
allErrs := validateSidecarSet(&sidecarSet)
allErrs := validateSidecarSetSpec(&sidecarSet, field.NewPath("spec"))
if len(allErrs) != 1 {
t.Errorf("%v: expect errors len 1, but got: %v", name, allErrs)
} else {
fmt.Println(allErrs)
fmt.Printf("%v: %v\n", name, allErrs)
}
}
}
func TestSidecarSetNameConflict(t *testing.T) {
sidecarsetList := &appsv1alpha1.SidecarSetList{
Items: []appsv1alpha1.SidecarSet{
{
ObjectMeta: metav1.ObjectMeta{Name: "sidecarset1"},
Spec: appsv1alpha1.SidecarSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"a": "b"},
},
Containers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{Name: "container-name"},
},
},
InitContainers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{Name: "init-name"},
},
},
Volumes: []corev1.Volume{
{
Name: "volume-name",
},
},
},
},
},
}
sidecarset := &appsv1alpha1.SidecarSet{
ObjectMeta: metav1.ObjectMeta{Name: "sidecarset2"},
Spec: appsv1alpha1.SidecarSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"a": "b"},
},
Containers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{Name: "container-name"},
},
},
InitContainers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{Name: "init-name"},
},
},
Volumes: []corev1.Volume{
{
Name: "volume-name",
},
},
},
}
allErrs := validateSidecarConflict(sidecarsetList, sidecarset, field.NewPath("spec.containers"))
if len(allErrs) != 2 {
t.Errorf("expect errors len 2, but got: %v", len(allErrs))
} else {
fmt.Println(allErrs)
}
}
type TestCase struct {
Input [2]metav1.LabelSelector
Output bool
}
func TestSelectorConflict(t *testing.T) {
testCases := []TestCase{
{
Input: [2]metav1.LabelSelector{
{
MatchLabels: map[string]string{"a": "h"},
},
{
MatchLabels: map[string]string{"a": "h"},
},
},
Output: true,
},
{
Input: [2]metav1.LabelSelector{
{
MatchLabels: map[string]string{"a": "h"},
},
{
MatchLabels: map[string]string{"a": "i"},
},
},
Output: false,
},
{
Input: [2]metav1.LabelSelector{
{
MatchLabels: map[string]string{"a": "h"},
},
{
MatchLabels: map[string]string{"b": "i"},
},
},
Output: true,
},
{
Input: [2]metav1.LabelSelector{
{
MatchLabels: map[string]string{
"a": "h",
"b": "i",
"c": "j",
},
},
{
MatchLabels: map[string]string{
"a": "h",
"b": "x",
"c": "j",
},
},
},
Output: false,
},
{
Input: [2]metav1.LabelSelector{
{
MatchLabels: map[string]string{"a": "h"},
},
{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "a",
Operator: metav1.LabelSelectorOpIn,
Values: []string{"h", "i", "j"},
},
},
},
},
Output: true,
},
{
Input: [2]metav1.LabelSelector{
{
MatchLabels: map[string]string{"a": "h"},
},
{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "a",
Operator: metav1.LabelSelectorOpIn,
Values: []string{"i", "j"},
},
},
},
},
Output: false,
},
{
Input: [2]metav1.LabelSelector{
{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "a",
Operator: metav1.LabelSelectorOpNotIn,
Values: []string{"h", "i"},
},
},
},
{
MatchLabels: map[string]string{"a": "h"},
},
},
Output: false,
},
{
Input: [2]metav1.LabelSelector{
{
MatchLabels: map[string]string{"a": "h"},
},
{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "a",
Operator: metav1.LabelSelectorOpExists,
},
},
},
},
Output: true,
},
{
Input: [2]metav1.LabelSelector{
{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "a",
Operator: metav1.LabelSelectorOpDoesNotExist,
},
},
},
{
MatchLabels: map[string]string{"a": "h"},
},
},
Output: false,
},
}
for i, testCase := range testCases {
output := util.IsSelectorOverlapping(&testCase.Input[0], &testCase.Input[1])
if output != testCase.Output {
t.Errorf("%v: expect %v but got %v", i, testCase.Output, output)
}
}
}
func TestSidecarSetVolumeConflict(t *testing.T) {
sidecarsetList := &appsv1alpha1.SidecarSetList{
Items: []appsv1alpha1.SidecarSet{
{
ObjectMeta: metav1.ObjectMeta{Name: "sidecarset1"},
Spec: appsv1alpha1.SidecarSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"a": "b"},
},
},
},
},
}
sidecarset := &appsv1alpha1.SidecarSet{
ObjectMeta: metav1.ObjectMeta{Name: "sidecarset2"},
Spec: appsv1alpha1.SidecarSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"a": "b"},
},
},
}
cases := []struct {
name string
getSidecarSet func() *appsv1alpha1.SidecarSet
getSidecarSetList func() *appsv1alpha1.SidecarSetList
expectErrLen int
}{
{
name: "sidecarset volume name different",
getSidecarSet: func() *appsv1alpha1.SidecarSet {
newSidecar := sidecarset.DeepCopy()
newSidecar.Spec.Volumes = []corev1.Volume{
{
Name: "volume-1",
},
{
Name: "volume-2",
},
}
return newSidecar
},
getSidecarSetList: func() *appsv1alpha1.SidecarSetList {
newSidecarList := sidecarsetList.DeepCopy()
newSidecarList.Items[0].Spec.Volumes = []corev1.Volume{
{
Name: "volume-3",
},
{
Name: "volume-4",
},
}
return newSidecarList
},
expectErrLen: 0,
},
{
name: "sidecarset volume name same and equal",
getSidecarSet: func() *appsv1alpha1.SidecarSet {
newSidecar := sidecarset.DeepCopy()
newSidecar.Spec.Volumes = []corev1.Volume{
{
Name: "volume-1",
},
{
Name: "volume-2",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/home/work",
},
},
},
}
return newSidecar
},
getSidecarSetList: func() *appsv1alpha1.SidecarSetList {
newSidecarList := sidecarsetList.DeepCopy()
newSidecarList.Items[0].Spec.Volumes = []corev1.Volume{
{
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/home/work",
},
},
Name: "volume-2",
},
{
Name: "volume-3",
},
}
return newSidecarList
},
expectErrLen: 0,
},
{
name: "sidecarset volume name same, but not equal",
getSidecarSet: func() *appsv1alpha1.SidecarSet {
newSidecar := sidecarset.DeepCopy()
newSidecar.Spec.Volumes = []corev1.Volume{
{
Name: "volume-1",
},
{
Name: "volume-2",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/home/work-1",
},
},
},
}
return newSidecar
},
getSidecarSetList: func() *appsv1alpha1.SidecarSetList {
newSidecarList := sidecarsetList.DeepCopy()
newSidecarList.Items[0].Spec.Volumes = []corev1.Volume{
{
Name: "volume-2",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: "/home/work-2",
},
},
},
{
Name: "volume-3",
},
}
return newSidecarList
},
expectErrLen: 1,
},
}
for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) {
sidecarset := cs.getSidecarSet()
sidecarsetList := cs.getSidecarSetList()
errs := validateSidecarConflict(sidecarsetList, sidecarset, field.NewPath("spec"))
if len(errs) != cs.expectErrLen {
t.Fatalf("except ErrLen(%d), but get errs(%d)", cs.expectErrLen, len(errs))
}
})
}
}

49
pkg/webhook/sidecarset/validating/utils.go Normal file
View File

@ -0,0 +1,49 @@
/*
Copyright 2020 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package validating
import (
"fmt"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/kubernetes/pkg/apis/core"
corev1 "k8s.io/kubernetes/pkg/apis/core/v1"
)
func getCoreVolumes(volumes []v1.Volume, fldPath *field.Path) ([]core.Volume, field.ErrorList) {
allErrs := field.ErrorList{}
var coreVolumes []core.Volume
for _, volume := range volumes {
coreVolume := core.Volume{}
if err := corev1.Convert_v1_Volume_To_core_Volume(&volume, &coreVolume, nil); err != nil {
allErrs = append(allErrs, field.Invalid(fldPath.Root(), volume, fmt.Sprintf("Convert_v1_Volume_To_core_Volume failed: %v", err)))
return nil, allErrs
}
coreVolumes = append(coreVolumes, coreVolume)
}
return coreVolumes, allErrs
}
func isSidecarSetNamespaceDiff(origin *appsv1alpha1.SidecarSet, other *appsv1alpha1.SidecarSet) bool {
originNamespace := origin.Spec.Namespace
otherNamespace := other.Spec.Namespace
return originNamespace != "" && otherNamespace != "" && originNamespace != otherNamespace
}

702
test/e2e/apps/sidecarset.go Normal file
View File

@ -0,0 +1,702 @@
/*
Copyright 2020 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package apps
import (
"fmt"
"time"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
kruiseclientset "github.com/openkruise/kruise/pkg/client/clientset/versioned"
"github.com/openkruise/kruise/pkg/util"
"github.com/openkruise/kruise/test/e2e/framework"
"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
apps "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
clientset "k8s.io/client-go/kubernetes"
utilpointer "k8s.io/utils/pointer"
)
var _ = SIGDescribe("sidecarset", func() {
f := framework.NewDefaultFramework("sidecarset")
var ns string
var c clientset.Interface
var kc kruiseclientset.Interface
var tester *framework.SidecarSetTester
ginkgo.BeforeEach(func() {
c = f.ClientSet
kc = f.KruiseClientSet
ns = f.Namespace.Name
tester = framework.NewSidecarSetTester(c, kc)
})
framework.KruiseDescribe("SidecarSet Injecting functionality [SidecarSetInject]", func() {
ginkgo.AfterEach(func() {
if ginkgo.CurrentGinkgoTestDescription().Failed {
framework.DumpDebugInfo(c, ns)
}
framework.Logf("Deleting all SidecarSet in cluster")
tester.DeleteSidecarSets()
tester.DeleteDeployments(ns)
})
ginkgo.It("pods don't have matched sidecarSet", func() {
// create sidecarSet
sidecarSet := tester.NewBaseSidecarSet()
// sidecarSet no matched pods
sidecarSet.Spec.Selector.MatchLabels["app"] = "nomatched"
ginkgo.By(fmt.Sprintf("Creating SidecarSet %s", sidecarSet.Name))
tester.CreateSidecarSet(sidecarSet)
// create deployment
deployment := tester.NewBaseDeployment(ns)
ginkgo.By(fmt.Sprintf("Creating Deployment(%s.%s)", deployment.Namespace, deployment.Name))
tester.CreateDeployment(deployment)
// get pods
pods, err := tester.GetSelectorPods(deployment.Namespace, deployment.Spec.Selector)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
gomega.Expect(pods).To(gomega.HaveLen(int(*deployment.Spec.Replicas)))
pod := pods[0]
gomega.Expect(pod.Spec.Containers).To(gomega.HaveLen(len(deployment.Spec.Template.Spec.Containers)))
ginkgo.By(fmt.Sprintf("test no matched sidecarSet done"))
})
ginkgo.It("sidecarSet inject pod sidecar container", func() {
// create sidecarSet
sidecarSet := tester.NewBaseSidecarSet()
ginkgo.By(fmt.Sprintf("Creating SidecarSet %s", sidecarSet.Name))
tester.CreateSidecarSet(sidecarSet)
// create deployment
deployment := tester.NewBaseDeployment(ns)
ginkgo.By(fmt.Sprintf("Creating Deployment(%s.%s)", deployment.Namespace, deployment.Name))
tester.CreateDeployment(deployment)
// get pods
pods, err := tester.GetSelectorPods(deployment.Namespace, deployment.Spec.Selector)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
pod := pods[0]
gomega.Expect(pod.Spec.Containers).To(gomega.HaveLen(len(deployment.Spec.Template.Spec.Containers) + len(sidecarSet.Spec.Containers)))
gomega.Expect(pod.Spec.InitContainers).To(gomega.HaveLen(len(deployment.Spec.Template.Spec.InitContainers) + len(sidecarSet.Spec.InitContainers)))
exceptContainers := []string{"nginx-sidecar", "main", "busybox-sidecar"}
for i, except := range exceptContainers {
gomega.Expect(except).To(gomega.Equal(pod.Spec.Containers[i].Name))
}
ginkgo.By(fmt.Sprintf("sidecarSet inject pod sidecar container done"))
})
ginkgo.It("sidecarSet inject pod sidecar container volumeMounts", func() {
// create sidecarSet
sidecarSet := tester.NewBaseSidecarSet()
// create deployment
deployment := tester.NewBaseDeployment(ns)
cases := []struct {
name string
getDeployment func() *apps.Deployment
getSidecarSets func() *appsv1alpha1.SidecarSet
exceptVolumeMounts []string
exceptEnvs []string
exceptVolumes []string
}{
{
name: "append normal volumeMounts",
getDeployment: func() *apps.Deployment {
deployIn := deployment.DeepCopy()
deployIn.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
{
Name: "main-volume",
MountPath: "/main-volume",
},
}
deployIn.Spec.Template.Spec.Volumes = []corev1.Volume{
{
Name: "main-volume",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
}
return deployIn
},
getSidecarSets: func() *appsv1alpha1.SidecarSet {
sidecarSetIn := sidecarSet.DeepCopy()
sidecarSetIn.Spec.Containers = sidecarSetIn.Spec.Containers[:1]
sidecarSetIn.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
{
Name: "nginx-volume",
MountPath: "/nginx-volume",
},
}
sidecarSetIn.Spec.Volumes = []corev1.Volume{
{
Name: "nginx-volume",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
}
return sidecarSetIn
},
exceptVolumeMounts: []string{"/main-volume", "/nginx-volume"},
exceptVolumes: []string{"main-volume", "nginx-volume"},
},
}
for _, cs := range cases {
ginkgo.By(cs.name)
sidecarSetIn := cs.getSidecarSets()
ginkgo.By(fmt.Sprintf("Creating SidecarSet %s", sidecarSetIn.Name))
tester.CreateSidecarSet(sidecarSetIn)
deploymentIn := cs.getDeployment()
ginkgo.By(fmt.Sprintf("Creating Deployment(%s.%s)", deploymentIn.Namespace, deploymentIn.Name))
tester.CreateDeployment(deploymentIn)
// get pods
pods, err := tester.GetSelectorPods(deploymentIn.Namespace, deploymentIn.Spec.Selector)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
// volume
for _, volume := range cs.exceptVolumes {
object := util.GetPodVolume(&pods[0], volume)
gomega.Expect(object).ShouldNot(gomega.BeNil())
}
// volumeMounts
sidecarContainer := &pods[0].Spec.Containers[0]
for _, volumeMount := range cs.exceptVolumeMounts {
object := util.GetContainerVolumeMount(sidecarContainer, volumeMount)
gomega.Expect(object).ShouldNot(gomega.BeNil())
}
// envs
for _, env := range cs.exceptEnvs {
object := util.GetContainerEnvVar(sidecarContainer, env)
gomega.Expect(object).ShouldNot(gomega.BeNil())
}
}
ginkgo.By(fmt.Sprintf("sidecarSet inject pod sidecar container volumeMounts done"))
})
ginkgo.It("sidecarSet inject pod sidecar container volumeMounts, SubPathExpr with expanded subpath", func() {
// create sidecarSet
sidecarSet := tester.NewBaseSidecarSet()
// create deployment
deployment := tester.NewBaseDeployment(ns)
cases := []struct {
name string
getDeployment func() *apps.Deployment
getSidecarSets func() *appsv1alpha1.SidecarSet
exceptVolumeMounts []string
exceptEnvs []string
exceptVolumes []string
}{
{
name: "append volumeMounts SubPathExpr, volumes with expanded subpath",
getDeployment: func() *apps.Deployment {
deployIn := deployment.DeepCopy()
deployIn.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
{
Name: "main-volume",
MountPath: "/main-volume",
SubPathExpr: "foo/$(POD_NAME)/$(OD_NAME)/conf",
},
}
deployIn.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{
{
Name: "POD_NAME",
Value: "bar",
},
{
Name: "OD_NAME",
Value: "od_name",
},
}
deployIn.Spec.Template.Spec.Volumes = []corev1.Volume{
{
Name: "main-volume",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
}
return deployIn
},
getSidecarSets: func() *appsv1alpha1.SidecarSet {
sidecarSetIn := sidecarSet.DeepCopy()
sidecarSetIn.Spec.Containers = sidecarSetIn.Spec.Containers[:1]
sidecarSetIn.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
{
Name: "nginx-volume",
MountPath: "/nginx-volume",
},
}
sidecarSetIn.Spec.Volumes = []corev1.Volume{
{
Name: "nginx-volume",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
}
return sidecarSetIn
},
exceptVolumeMounts: []string{"/main-volume", "/nginx-volume"},
exceptVolumes: []string{"main-volume", "nginx-volume"},
exceptEnvs: []string{"POD_NAME", "OD_NAME"},
},
}
for _, cs := range cases {
ginkgo.By(cs.name)
sidecarSetIn := cs.getSidecarSets()
ginkgo.By(fmt.Sprintf("Creating SidecarSet %s", sidecarSetIn.Name))
tester.CreateSidecarSet(sidecarSetIn)
deploymentIn := cs.getDeployment()
ginkgo.By(fmt.Sprintf("Creating Deployment(%s.%s)", deploymentIn.Namespace, deploymentIn.Name))
tester.CreateDeployment(deploymentIn)
// get pods
pods, err := tester.GetSelectorPods(deploymentIn.Namespace, deploymentIn.Spec.Selector)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
// volume
for _, volume := range cs.exceptVolumes {
object := util.GetPodVolume(&pods[0], volume)
gomega.Expect(object).ShouldNot(gomega.BeNil())
}
// volumeMounts
sidecarContainer := &pods[0].Spec.Containers[0]
for _, volumeMount := range cs.exceptVolumeMounts {
object := util.GetContainerVolumeMount(sidecarContainer, volumeMount)
gomega.Expect(object).ShouldNot(gomega.BeNil())
}
// envs
for _, env := range cs.exceptEnvs {
object := util.GetContainerEnvVar(sidecarContainer, env)
gomega.Expect(object).ShouldNot(gomega.BeNil())
}
}
ginkgo.By(fmt.Sprintf("sidecarSet inject pod sidecar container volumeMounts, SubPathExpr with expanded subpath done"))
})
ginkgo.It("sidecarSet inject pod sidecar container transfer Envs", func() {
// create sidecarSet
sidecarSetIn := tester.NewBaseSidecarSet()
sidecarSetIn.Spec.Containers = sidecarSetIn.Spec.Containers[:1]
sidecarSetIn.Spec.Containers[0].Env = []corev1.EnvVar{
{
Name: "OD_NAME",
Value: "sidecar_name",
},
{
Name: "SidecarName",
Value: "nginx-sidecar",
},
}
sidecarSetIn.Spec.Containers[0].TransferEnv = []appsv1alpha1.TransferEnvVar{
{
SourceContainerName: "main",
EnvName: "POD_NAME",
},
{
SourceContainerName: "main",
EnvName: "OD_NAME",
},
{
SourceContainerName: "main",
EnvName: "PROXY_IP",
},
}
ginkgo.By(fmt.Sprintf("Creating SidecarSet %s", sidecarSetIn.Name))
tester.CreateSidecarSet(sidecarSetIn)
// create deployment
deploymentIn := tester.NewBaseDeployment(ns)
deploymentIn.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{
{
Name: "POD_NAME",
Value: "bar",
},
{
Name: "OD_NAME",
Value: "od_name",
},
{
Name: "PROXY_IP",
Value: "127.0.0.1",
},
}
ginkgo.By(fmt.Sprintf("Creating Deployment(%s.%s)", deploymentIn.Namespace, deploymentIn.Name))
tester.CreateDeployment(deploymentIn)
// get pods
pods, err := tester.GetSelectorPods(deploymentIn.Namespace, deploymentIn.Spec.Selector)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
// except envs
exceptEnvs := map[string]string{
"POD_NAME": "bar",
"OD_NAME": "sidecar_name",
"PROXY_IP": "127.0.0.1",
"SidecarName": "nginx-sidecar",
}
sidecarContainer := &pods[0].Spec.Containers[0]
// envs
for key, value := range exceptEnvs {
object := util.GetContainerEnvValue(sidecarContainer, key)
gomega.Expect(object).To(gomega.Equal(value))
}
ginkgo.By(fmt.Sprintf("sidecarSet inject pod sidecar container transfer Envs done"))
})
})
framework.KruiseDescribe("SidecarSet Upgrade functionality [SidecarSeUpgrade]", func() {
ginkgo.AfterEach(func() {
if ginkgo.CurrentGinkgoTestDescription().Failed {
framework.DumpDebugInfo(c, ns)
}
framework.Logf("Deleting all SidecarSet in cluster")
tester.DeleteSidecarSets()
tester.DeleteDeployments(ns)
})
ginkgo.It("sidecarSet upgrade cold sidecar container image", func() {
// create sidecarSet
sidecarSetIn := tester.NewBaseSidecarSet()
sidecarSetIn.Spec.Strategy = appsv1alpha1.SidecarSetUpdateStrategy{
Type: appsv1alpha1.RollingUpdateSidecarSetStrategyType,
MaxUnavailable: &intstr.IntOrString{
Type: intstr.Int,
IntVal: 2,
},
}
sidecarSetIn.Spec.Containers = sidecarSetIn.Spec.Containers[:1]
ginkgo.By(fmt.Sprintf("Creating SidecarSet %s", sidecarSetIn.Name))
sidecarSetIn = tester.CreateSidecarSet(sidecarSetIn)
// create deployment
deploymentIn := tester.NewBaseDeployment(ns)
deploymentIn.Spec.Replicas = utilpointer.Int32Ptr(2)
ginkgo.By(fmt.Sprintf("Creating Deployment(%s.%s)", deploymentIn.Namespace, deploymentIn.Name))
tester.CreateDeployment(deploymentIn)
// update sidecarSet sidecar container
sidecarSetIn.Spec.Containers[0].Image = "busybox:latest"
tester.UpdateSidecarSet(sidecarSetIn)
time.Sleep(time.Second * 60)
except := &appsv1alpha1.SidecarSetStatus{
MatchedPods: 2,
UpdatedPods: 2,
UpdatedReadyPods: 2,
ReadyPods: 2,
}
tester.WaitForSidecarSetUpgradeComplete(sidecarSetIn, except)
// get pods
pods, err := tester.GetSelectorPods(deploymentIn.Namespace, deploymentIn.Spec.Selector)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
for _, pod := range pods {
sidecarContainer := pod.Spec.Containers[0]
gomega.Expect(sidecarContainer.Image).To(gomega.Equal("busybox:latest"))
}
ginkgo.By(fmt.Sprintf("sidecarSet upgrade cold sidecar container image done"))
})
ginkgo.It("sidecarSet upgrade cold sidecar container failed image, and only update one pod", func() {
// create sidecarSet
sidecarSetIn := tester.NewBaseSidecarSet()
sidecarSetIn.Spec.Strategy = appsv1alpha1.SidecarSetUpdateStrategy{
Type: appsv1alpha1.RollingUpdateSidecarSetStrategyType,
}
sidecarSetIn.Spec.Containers = sidecarSetIn.Spec.Containers[:1]
ginkgo.By(fmt.Sprintf("Creating SidecarSet %s", sidecarSetIn.Name))
sidecarSetIn = tester.CreateSidecarSet(sidecarSetIn)
// create deployment
deploymentIn := tester.NewBaseDeployment(ns)
deploymentIn.Spec.Replicas = utilpointer.Int32Ptr(2)
ginkgo.By(fmt.Sprintf("Creating Deployment(%s.%s)", deploymentIn.Namespace, deploymentIn.Name))
tester.CreateDeployment(deploymentIn)
// update sidecarSet sidecar container failed image
sidecarSetIn.Spec.Containers[0].Image = "busybox:failed"
tester.UpdateSidecarSet(sidecarSetIn)
time.Sleep(time.Second * 60)
except := &appsv1alpha1.SidecarSetStatus{
MatchedPods: 2,
UpdatedPods: 1,
UpdatedReadyPods: 0,
ReadyPods: 1,
}
tester.WaitForSidecarSetUpgradeComplete(sidecarSetIn, except)
// update sidecarSet sidecar container success image
sidecarSetIn.Spec.Containers[0].Image = "busybox:latest"
tester.UpdateSidecarSet(sidecarSetIn)
time.Sleep(time.Second * 60)
except = &appsv1alpha1.SidecarSetStatus{
MatchedPods: 2,
UpdatedPods: 2,
UpdatedReadyPods: 2,
ReadyPods: 2,
}
tester.WaitForSidecarSetUpgradeComplete(sidecarSetIn, except)
ginkgo.By(fmt.Sprintf("sidecarSet upgrade cold sidecar container failed image, and only update one pod done"))
})
ginkgo.It("sidecarSet upgrade cold sidecar container image, and paused", func() {
// create sidecarSet
sidecarSetIn := tester.NewBaseSidecarSet()
sidecarSetIn.Spec.Strategy = appsv1alpha1.SidecarSetUpdateStrategy{
Type: appsv1alpha1.RollingUpdateSidecarSetStrategyType,
}
sidecarSetIn.Spec.Containers = sidecarSetIn.Spec.Containers[:1]
ginkgo.By(fmt.Sprintf("Creating SidecarSet %s", sidecarSetIn.Name))
sidecarSetIn = tester.CreateSidecarSet(sidecarSetIn)
// create deployment
deploymentIn := tester.NewBaseDeployment(ns)
deploymentIn.Spec.Replicas = utilpointer.Int32Ptr(2)
ginkgo.By(fmt.Sprintf("Creating Deployment(%s.%s)", deploymentIn.Namespace, deploymentIn.Name))
tester.CreateDeployment(deploymentIn)
// update sidecarSet sidecar container
sidecarSetIn.Spec.Containers[0].Image = "busybox:latest"
tester.UpdateSidecarSet(sidecarSetIn)
time.Sleep(time.Second * 5)
// paused
sidecarSetIn.Spec.Strategy.Paused = true
tester.UpdateSidecarSet(sidecarSetIn)
except := &appsv1alpha1.SidecarSetStatus{
MatchedPods: 2,
UpdatedPods: 1,
UpdatedReadyPods: 1,
ReadyPods: 2,
}
tester.WaitForSidecarSetUpgradeComplete(sidecarSetIn, except)
// paused = false, continue update pods
sidecarSetIn.Spec.Strategy.Paused = false
tester.UpdateSidecarSet(sidecarSetIn)
except = &appsv1alpha1.SidecarSetStatus{
MatchedPods: 2,
UpdatedPods: 2,
UpdatedReadyPods: 2,
ReadyPods: 2,
}
tester.WaitForSidecarSetUpgradeComplete(sidecarSetIn, except)
ginkgo.By(fmt.Sprintf("sidecarSet upgrade cold sidecar container image, and paused done"))
})
ginkgo.It("sidecarSet upgrade cold sidecar container image, and selector", func() {
// create sidecarSet
sidecarSetIn := tester.NewBaseSidecarSet()
sidecarSetIn.Spec.Strategy = appsv1alpha1.SidecarSetUpdateStrategy{
Type: appsv1alpha1.RollingUpdateSidecarSetStrategyType,
}
sidecarSetIn.Spec.Containers = sidecarSetIn.Spec.Containers[:1]
ginkgo.By(fmt.Sprintf("Creating SidecarSet %s", sidecarSetIn.Name))
sidecarSetIn = tester.CreateSidecarSet(sidecarSetIn)
// create deployment
deploymentIn := tester.NewBaseDeployment(ns)
deploymentIn.Spec.Replicas = utilpointer.Int32Ptr(2)
ginkgo.By(fmt.Sprintf("Creating Deployment(%s.%s)", deploymentIn.Namespace, deploymentIn.Name))
tester.CreateDeployment(deploymentIn)
// update pod[0] labels[canary.release] = true
pods, err := tester.GetSelectorPods(deploymentIn.Namespace, deploymentIn.Spec.Selector)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
gomega.Expect(pods).To(gomega.HaveLen(int(*deploymentIn.Spec.Replicas)))
canaryPod := pods[0]
canaryPod.Labels["canary.release"] = "true"
tester.UpdatePod(&canaryPod)
time.Sleep(time.Second)
// update sidecarSet sidecar container
sidecarSetIn.Spec.Containers[0].Image = "busybox:latest"
// update sidecarSet selector
sidecarSetIn.Spec.Strategy.Selector = &metav1.LabelSelector{
MatchLabels: map[string]string{
"canary.release": "true",
},
}
tester.UpdateSidecarSet(sidecarSetIn)
time.Sleep(time.Second * 5)
tester.UpdateSidecarSet(sidecarSetIn)
except := &appsv1alpha1.SidecarSetStatus{
MatchedPods: 2,
UpdatedPods: 1,
UpdatedReadyPods: 1,
ReadyPods: 2,
}
time.Sleep(time.Minute)
tester.WaitForSidecarSetUpgradeComplete(sidecarSetIn, except)
// check pod image
pods, err = tester.GetSelectorPods(deploymentIn.Namespace, deploymentIn.Spec.Selector)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
gomega.Expect(pods).To(gomega.HaveLen(int(*deploymentIn.Spec.Replicas)))
for _, pod := range pods {
if _, ok := pod.Labels["canary.release"]; ok {
sidecarContainer := pod.Spec.Containers[0]
gomega.Expect(sidecarContainer.Image).To(gomega.Equal("busybox:latest"))
} else {
sidecarContainer := pod.Spec.Containers[0]
gomega.Expect(sidecarContainer.Image).To(gomega.Equal("nginx:latest"))
}
}
// update sidecarSet selector == nil, and update all pods
sidecarSetIn.Spec.Strategy.Selector = nil
tester.UpdateSidecarSet(sidecarSetIn)
time.Sleep(time.Second * 5)
tester.UpdateSidecarSet(sidecarSetIn)
except = &appsv1alpha1.SidecarSetStatus{
MatchedPods: 2,
UpdatedPods: 2,
UpdatedReadyPods: 2,
ReadyPods: 2,
}
time.Sleep(time.Minute)
tester.WaitForSidecarSetUpgradeComplete(sidecarSetIn, except)
ginkgo.By(fmt.Sprintf("sidecarSet upgrade cold sidecar container image, and selector done"))
})
ginkgo.It("sidecarSet upgrade cold sidecar container image, and partition", func() {
// create sidecarSet
sidecarSetIn := tester.NewBaseSidecarSet()
sidecarSetIn.Spec.Strategy = appsv1alpha1.SidecarSetUpdateStrategy{
Type: appsv1alpha1.RollingUpdateSidecarSetStrategyType,
}
sidecarSetIn.Spec.Containers = sidecarSetIn.Spec.Containers[:1]
ginkgo.By(fmt.Sprintf("Creating SidecarSet %s", sidecarSetIn.Name))
sidecarSetIn = tester.CreateSidecarSet(sidecarSetIn)
// create deployment
deploymentIn := tester.NewBaseDeployment(ns)
deploymentIn.Spec.Replicas = utilpointer.Int32Ptr(2)
ginkgo.By(fmt.Sprintf("Creating Deployment(%s.%s)", deploymentIn.Namespace, deploymentIn.Name))
tester.CreateDeployment(deploymentIn)
// update sidecarSet sidecar container
sidecarSetIn.Spec.Containers[0].Image = "busybox:latest"
// update sidecarSet selector
sidecarSetIn.Spec.Strategy.Partition = &intstr.IntOrString{
Type: intstr.String,
StrVal: "50%",
}
tester.UpdateSidecarSet(sidecarSetIn)
except := &appsv1alpha1.SidecarSetStatus{
MatchedPods: 2,
UpdatedPods: 1,
UpdatedReadyPods: 1,
ReadyPods: 2,
}
time.Sleep(time.Minute)
tester.WaitForSidecarSetUpgradeComplete(sidecarSetIn, except)
// update sidecarSet partition, update all pods
sidecarSetIn.Spec.Strategy.Partition = nil
tester.UpdateSidecarSet(sidecarSetIn)
except = &appsv1alpha1.SidecarSetStatus{
MatchedPods: 2,
UpdatedPods: 2,
UpdatedReadyPods: 2,
ReadyPods: 2,
}
time.Sleep(time.Minute)
tester.WaitForSidecarSetUpgradeComplete(sidecarSetIn, except)
ginkgo.By(fmt.Sprintf("sidecarSet upgrade cold sidecar container image, and partition done"))
})
ginkgo.It("sidecarSet upgrade cold sidecar container image, and maxUnavailable", func() {
// create sidecarSet
sidecarSetIn := tester.NewBaseSidecarSet()
sidecarSetIn.Spec.Strategy = appsv1alpha1.SidecarSetUpdateStrategy{
Type: appsv1alpha1.RollingUpdateSidecarSetStrategyType,
}
sidecarSetIn.Spec.Containers = sidecarSetIn.Spec.Containers[:1]
ginkgo.By(fmt.Sprintf("Creating SidecarSet %s", sidecarSetIn.Name))
sidecarSetIn = tester.CreateSidecarSet(sidecarSetIn)
// create deployment
deploymentIn := tester.NewBaseDeployment(ns)
deploymentIn.Spec.Replicas = utilpointer.Int32Ptr(4)
ginkgo.By(fmt.Sprintf("Creating Deployment(%s.%s)", deploymentIn.Namespace, deploymentIn.Name))
tester.CreateDeployment(deploymentIn)
// update sidecarSet sidecar container
sidecarSetIn.Spec.Containers[0].Image = "busybox:failed"
// update sidecarSet selector
sidecarSetIn.Spec.Strategy.MaxUnavailable = &intstr.IntOrString{
Type: intstr.String,
StrVal: "50%",
}
tester.UpdateSidecarSet(sidecarSetIn)
except := &appsv1alpha1.SidecarSetStatus{
MatchedPods: 4,
UpdatedPods: 2,
UpdatedReadyPods: 0,
ReadyPods: 2,
}
time.Sleep(time.Minute)
tester.WaitForSidecarSetUpgradeComplete(sidecarSetIn, except)
// update sidecarSet sidecar container
sidecarSetIn.Spec.Containers[0].Image = "busybox:latest"
tester.UpdateSidecarSet(sidecarSetIn)
except = &appsv1alpha1.SidecarSetStatus{
MatchedPods: 4,
UpdatedPods: 4,
UpdatedReadyPods: 4,
ReadyPods: 4,
}
time.Sleep(time.Minute)
tester.WaitForSidecarSetUpgradeComplete(sidecarSetIn, except)
ginkgo.By(fmt.Sprintf("sidecarSet upgrade cold sidecar container image, and maxUnavailable done"))
})
ginkgo.It("sidecarSet update init sidecar container, and don't upgrade", func() {
// create sidecarSet
sidecarSetIn := tester.NewBaseSidecarSet()
sidecarSetIn.Spec.Strategy = appsv1alpha1.SidecarSetUpdateStrategy{
Type: appsv1alpha1.RollingUpdateSidecarSetStrategyType,
}
sidecarSetIn.Spec.Containers = sidecarSetIn.Spec.Containers[:1]
ginkgo.By(fmt.Sprintf("Creating SidecarSet %s", sidecarSetIn.Name))
sidecarSetIn = tester.CreateSidecarSet(sidecarSetIn)
// create deployment
deploymentIn := tester.NewBaseDeployment(ns)
deploymentIn.Spec.Replicas = utilpointer.Int32Ptr(1)
ginkgo.By(fmt.Sprintf("Creating Deployment(%s.%s)", deploymentIn.Namespace, deploymentIn.Name))
tester.CreateDeployment(deploymentIn)
// update sidecarSet sidecar container
sidecarSetIn.Spec.InitContainers[0].Image = "busybox:failed"
tester.UpdateSidecarSet(sidecarSetIn)
except := &appsv1alpha1.SidecarSetStatus{
MatchedPods: 1,
UpdatedPods: 0,
UpdatedReadyPods: 0,
ReadyPods: 1,
}
time.Sleep(time.Minute)
tester.WaitForSidecarSetUpgradeComplete(sidecarSetIn, except)
ginkgo.By(fmt.Sprintf("sidecarSet upgrade init sidecar container, and don't upgrade done"))
})
})
})

316
test/e2e/framework/sidecarset_utils.go Normal file
View File

@ -0,0 +1,316 @@
/*
Copyright 2020 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package framework
import (
utilpointer "k8s.io/utils/pointer"
"time"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
kruiseclientset "github.com/openkruise/kruise/pkg/client/clientset/versioned"
"github.com/openkruise/kruise/pkg/util"
"github.com/onsi/gomega"
apps "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/util/retry"
)
type SidecarSetTester struct {
c clientset.Interface
kc kruiseclientset.Interface
}
func NewSidecarSetTester(c clientset.Interface, kc kruiseclientset.Interface) *SidecarSetTester {
return &SidecarSetTester{
c: c,
kc: kc,
}
}
func (s *SidecarSetTester) NewBaseSidecarSet() *appsv1alpha1.SidecarSet {
return &appsv1alpha1.SidecarSet{
TypeMeta: metav1.TypeMeta{
Kind: "SidecarSet",
APIVersion: "apps.kruise.io/v1alpha1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-sidecarset",
},
Spec: appsv1alpha1.SidecarSetSpec{
InitContainers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{
Name: "init-sidecar",
Command: []string{"/bin/sh", "-c", "sleep 1"},
Image: "busybox:latest",
},
},
},
Containers: []appsv1alpha1.SidecarContainer{
{
Container: corev1.Container{
Name: "nginx-sidecar",
Image: "nginx:latest",
Command: []string{"tail", "-f", "/dev/null"},
},
PodInjectPolicy: appsv1alpha1.BeforeAppContainerType,
ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{
Type: appsv1alpha1.ShareVolumePolicyEnabled,
},
},
{
Container: corev1.Container{
Name: "busybox-sidecar",
Image: "busybox:latest",
Command: []string{"/bin/sh", "-c", "sleep 10000000"},
},
PodInjectPolicy: appsv1alpha1.AfterAppContainerType,
},
},
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": "sidecarset"},
},
},
}
}
func (s *SidecarSetTester) NewBaseDeployment(namespace string) *apps.Deployment {
return &apps.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "deployment-test",
Namespace: namespace,
},
Spec: apps.DeploymentSpec{
Replicas: utilpointer.Int32Ptr(1),
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": "sidecarset",
},
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"app": "sidecarset",
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "main",
Image: "busybox:latest",
Command: []string{"/bin/sh", "-c", "sleep 10000000"},
},
},
},
},
},
}
}
func (s *SidecarSetTester) CreateSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) *appsv1alpha1.SidecarSet {
Logf("create sidecarSet(%s)", sidecarSet.Name)
_, err := s.kc.AppsV1alpha1().SidecarSets().Create(sidecarSet)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
s.WaitForSidecarSetCreated(sidecarSet)
sidecarSet, _ = s.kc.AppsV1alpha1().SidecarSets().Get(sidecarSet.Name, metav1.GetOptions{})
return sidecarSet
}
func (s *SidecarSetTester) UpdateSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) {
Logf("update sidecarSet(%s)", sidecarSet.Name)
sidecarSetClone := sidecarSet.DeepCopy()
err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
sidecarSetClone.Spec = sidecarSet.Spec
_, updateErr := s.kc.AppsV1alpha1().SidecarSets().Update(sidecarSetClone)
if updateErr == nil {
return nil
}
sidecarSetClone, _ = s.kc.AppsV1alpha1().SidecarSets().Get(sidecarSetClone.Name, metav1.GetOptions{})
return updateErr
})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
}
func (s *SidecarSetTester) UpdatePod(pod *corev1.Pod) {
Logf("update pod(%s.%s)", pod.Namespace, pod.Name)
podClone := pod.DeepCopy()
err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
podClone.ObjectMeta = pod.ObjectMeta
podClone.Spec = pod.Spec
_, updateErr := s.c.CoreV1().Pods(podClone.Namespace).Update(podClone)
if updateErr == nil {
return nil
}
podClone, _ = s.c.CoreV1().Pods(podClone.Namespace).Update(podClone)
return updateErr
})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
}
func (s *SidecarSetTester) WaitForSidecarSetUpgradeComplete(sidecarSet *appsv1alpha1.SidecarSet, exceptStatus *appsv1alpha1.SidecarSetStatus) {
pollErr := wait.PollImmediate(time.Second, time.Minute*5,
func() (bool, error) {
inner, err := s.kc.AppsV1alpha1().SidecarSets().Get(sidecarSet.Name, metav1.GetOptions{})
if err != nil {
return false, err
}
if inner.Status.MatchedPods == exceptStatus.MatchedPods &&
inner.Status.UpdatedPods == exceptStatus.UpdatedPods &&
inner.Status.UpdatedReadyPods == exceptStatus.UpdatedReadyPods &&
inner.Status.ReadyPods == exceptStatus.ReadyPods {
return true, nil
}
return false, nil
})
if pollErr != nil {
Failf("Failed waiting for sidecarSet to upgrade complete: %v", pollErr)
}
}
func (s *SidecarSetTester) CreateDeployment(deployment *apps.Deployment) {
Logf("create deployment(%s.%s)", deployment.Namespace, deployment.Name)
_, err := s.c.AppsV1().Deployments(deployment.Namespace).Create(deployment)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
s.WaitForDeploymentRunning(deployment)
}
func (s *SidecarSetTester) DeleteSidecarSets() {
sidecarSetList, err := s.kc.AppsV1alpha1().SidecarSets().List(metav1.ListOptions{})
if err != nil {
Logf("List sidecarSets failed: %s", err.Error())
return
}
for _, sidecarSet := range sidecarSetList.Items {
s.DeleteSidecarSet(&sidecarSet)
}
}
func (s *SidecarSetTester) DeleteSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) {
err := s.kc.AppsV1alpha1().SidecarSets().Delete(sidecarSet.Name, &metav1.DeleteOptions{})
if err != nil {
Logf("delete sidecarSet(%s) failed: %s", sidecarSet.Name, err.Error())
}
s.WaitForSidecarSetDeleted(sidecarSet)
}
func (s *SidecarSetTester) DeleteDeployments(namespace string) {
deploymentList, err := s.c.AppsV1().Deployments(namespace).List(metav1.ListOptions{})
if err != nil {
Logf("List Deployments failed: %s", err.Error())
return
}
for _, deployment := range deploymentList.Items {
s.DeleteDeployment(&deployment)
}
}
func (s *SidecarSetTester) DeleteDeployment(deployment *apps.Deployment) {
err := s.c.AppsV1().Deployments(deployment.Namespace).Delete(deployment.Name, &metav1.DeleteOptions{})
if err != nil {
Logf("delete deployment(%s.%s) failed: %s", deployment.Namespace, deployment.Name, err.Error())
return
}
s.WaitForDeploymentDeleted(deployment)
}
func (s *SidecarSetTester) WaitForSidecarSetCreated(sidecarSet *appsv1alpha1.SidecarSet) {
pollErr := wait.PollImmediate(time.Second, time.Minute,
func() (bool, error) {
_, err := s.kc.AppsV1alpha1().SidecarSets().Get(sidecarSet.Name, metav1.GetOptions{})
if err != nil {
return false, err
}
return true, nil
})
if pollErr != nil {
Failf("Failed waiting for sidecarSet to enter running: %v", pollErr)
}
}
func (s *SidecarSetTester) WaitForDeploymentRunning(deployment *apps.Deployment) {
pollErr := wait.PollImmediate(time.Second, time.Minute*5,
func() (bool, error) {
inner, err := s.c.AppsV1().Deployments(deployment.Namespace).Get(deployment.Name, metav1.GetOptions{})
if err != nil {
return false, nil
}
if *inner.Spec.Replicas == inner.Status.ReadyReplicas {
return true, nil
}
return false, nil
})
if pollErr != nil {
Failf("Failed waiting for deployment to enter running: %v", pollErr)
}
}
func (s *SidecarSetTester) WaitForDeploymentDeleted(deployment *apps.Deployment) {
pollErr := wait.PollImmediate(time.Second, time.Minute,
func() (bool, error) {
_, err := s.c.AppsV1().Deployments(deployment.Namespace).Get(deployment.Name, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
return true, nil
}
return false, err
}
return false, nil
})
if pollErr != nil {
Failf("Failed waiting for deployment to enter Deleted: %v", pollErr)
}
}
func (s *SidecarSetTester) WaitForSidecarSetDeleted(sidecarSet *appsv1alpha1.SidecarSet) {
pollErr := wait.PollImmediate(time.Second, time.Minute,
func() (bool, error) {
_, err := s.kc.AppsV1alpha1().SidecarSets().Get(sidecarSet.Name, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
return true, nil
}
return false, err
}
return false, nil
})
if pollErr != nil {
Failf("Failed waiting for SidecarSet to enter Deleted: %v", pollErr)
}
}
func (s *SidecarSetTester) GetSelectorPods(namespace string, selector *metav1.LabelSelector) ([]corev1.Pod, error) {
faster, err := util.GetFastLabelSelector(selector)
if err != nil {
return nil, err
}
podList, err := s.c.CoreV1().Pods(namespace).List(metav1.ListOptions{LabelSelector: faster.String()})
if err != nil {
return nil, err
}
return podList.Items, nil
}

1
test/e2e/manifest/manifest.go
View File

@ -19,6 +19,7 @@ package manifest
import (
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"

21
test/e2e/testing-manifests/sidecarset/deployment-base.yaml Normal file
View File

@ -0,0 +1,21 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: deployment-test
spec:
replicas: 1
selector:
matchLabels:
app: sidecarset
template:
metadata:
labels:
app: sidecarset
spec:
containers:
- image: busybox:latest
name: main
command:
- "/bin/sh"
- "-c"
- sleep 100000

22
test/e2e/testing-manifests/sidecarset/sidecarset-base.yaml Normal file
View File

@ -0,0 +1,22 @@
apiVersion: apps.kruise.io/v1alpha1
kind: SidecarSet
metadata:
name: test-sidecarset
spec:
selector:
matchLabels:
app: sidecarset
initContainers:
- image: busybox:latest
name: init-sidecar
command: ["/bin/sh", "-c", "sleep 1"]
containers:
- image: nginx:latest
command: [ "tail", "-f", "/dev/null" ]
name: nginx-sidecar
podInjectPolicy: BeforeAppContainer
shareVolumePolicy: enabled
- image: busybox:latest
name: busybox-sidecar
command: [ "/bin/sh", "-c", "sleep 10000000" ]
podInjectPolicy: AfterAppContainer

29
vendor/github.com/docker/distribution/reference/normalize.go generated vendored
View File

@ -56,6 +56,35 @@ func ParseNormalizedNamed(s string) (Named, error) {
return named, nil
}
// ParseDockerRef normalizes the image reference following the docker convention. This is added
// mainly for backward compatibility.
// The reference returned can only be either tagged or digested. For reference contains both tag
// and digest, the function returns digested reference, e.g. docker.io/library/busybox:latest@
// sha256:7cc4b5aefd1d0cadf8d97d4350462ba51c694ebca145b08d7d41b41acc8db5aa will be returned as
// docker.io/library/busybox@sha256:7cc4b5aefd1d0cadf8d97d4350462ba51c694ebca145b08d7d41b41acc8db5aa.
func ParseDockerRef(ref string) (Named, error) {
named, err := ParseNormalizedNamed(ref)
if err != nil {
return nil, err
}
if _, ok := named.(NamedTagged); ok {
if canonical, ok := named.(Canonical); ok {
// The reference is both tagged and digested, only
// return digested.
newNamed, err := WithName(canonical.Name())
if err != nil {
return nil, err
}
newCanonical, err := WithDigest(newNamed, canonical.Digest())
if err != nil {
return nil, err
}
return newCanonical, nil
}
}
return TagNameOnly(named), nil
}
// splitDockerDomain splits a repository name to domain and remotename string.
// If no valid domain is found, the default domain is used. Repository name
// needs to be already validated before.

32
vendor/k8s.io/kubernetes/pkg/util/slice/BUILD generated vendored Normal file
View File

@ -0,0 +1,32 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
name = "go_default_library",
srcs = ["slice.go"],
importpath = "k8s.io/kubernetes/pkg/util/slice",
)
go_test(
name = "go_default_test",
srcs = ["slice_test.go"],
embed = [":go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

75
vendor/k8s.io/kubernetes/pkg/util/slice/slice.go generated vendored Normal file
View File

@ -0,0 +1,75 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package slice provides utility methods for common operations on slices.
package slice
import (
"sort"
)
// CopyStrings copies the contents of the specified string slice
// into a new slice.
func CopyStrings(s []string) []string {
if s == nil {
return nil
}
c := make([]string, len(s))
copy(c, s)
return c
}
// SortStrings sorts the specified string slice in place. It returns the same
// slice that was provided in order to facilitate method chaining.
func SortStrings(s []string) []string {
sort.Strings(s)
return s
}
// ContainsString checks if a given slice of strings contains the provided string.
// If a modifier func is provided, it is called with the slice item before the comparation.
func ContainsString(slice []string, s string, modifier func(s string) string) bool {
for _, item := range slice {
if item == s {
return true
}
if modifier != nil && modifier(item) == s {
return true
}
}
return false
}
// RemoveString returns a newly created []string that contains all items from slice that
// are not equal to s and modifier(s) in case modifier func is provided.
func RemoveString(slice []string, s string, modifier func(s string) string) []string {
newSlice := make([]string, 0)
for _, item := range slice {
if item == s {
continue
}
if modifier != nil && modifier(item) == s {
continue
}
newSlice = append(newSlice, item)
}
if len(newSlice) == 0 {
// Sanitize for unit tests so we don't need to distinguish empty array
// and nil.
newSlice = nil
}
return newSlice
}

3
vendor/modules.txt vendored
View File

@ -34,7 +34,7 @@ github.com/coreos/go-systemd/journal
github.com/coreos/pkg/capnslog
# github.com/davecgh/go-spew v1.1.1
github.com/davecgh/go-spew/spew
# github.com/docker/distribution v2.7.1+incompatible
# github.com/docker/distribution v2.7.1+incompatible => github.com/docker/distribution v2.7.2-0.20200708230840-70e0022e42fd+incompatible
github.com/docker/distribution/digestset
github.com/docker/distribution/reference
# github.com/emicklei/go-restful v2.9.5+incompatible
@ -816,6 +816,7 @@ k8s.io/kubernetes/pkg/util/labels
k8s.io/kubernetes/pkg/util/mount
k8s.io/kubernetes/pkg/util/parsers
k8s.io/kubernetes/pkg/util/resizefs
k8s.io/kubernetes/pkg/util/slice
k8s.io/kubernetes/pkg/util/taints
k8s.io/kubernetes/pkg/volume
k8s.io/kubernetes/pkg/volume/util