mirror of https://github.com/kubernetes/kops.git
Merge pull request #11852 from hakman/hooks-containerd
Handle containerExec hooks when using containerd
This commit is contained in:
Kubernetes Prow Robot
GitHub
commit
0e4d766deb
|
|
@ -88,6 +88,7 @@ go_test(
|
|||
"containerd_test.go",
|
||||
"docker_test.go",
|
||||
"fakes_test.go",
|
||||
"hooks_test.go",
|
||||
"kops_controller_test.go",
|
||||
"kube_apiserver_test.go",
|
||||
"kube_controller_manager_test.go",
|
||||
|
|
|
|||
|
|
@ -36,11 +36,11 @@ func b(v bool) *bool {
|
|||
return fi.Bool(v)
|
||||
}
|
||||
|
||||
// buildDockerEnvironmentVars just converts a series of keypairs to docker environment variables switches
|
||||
func buildDockerEnvironmentVars(env map[string]string) []string {
|
||||
// buildContainerRuntimeEnvironmentVars just converts a series of keypairs to docker environment variables switches
|
||||
func buildContainerRuntimeEnvironmentVars(env map[string]string) []string {
|
||||
var list []string
|
||||
for k, v := range env {
|
||||
list = append(list, []string{"-e", fmt.Sprintf("%s=%s", k, v)}...)
|
||||
list = append(list, []string{"--env", fmt.Sprintf("%s=%s", k, v)}...)
|
||||
}
|
||||
|
||||
return list
|
||||
|
|
|
|||
|
|
@ -124,8 +124,17 @@ func (h *HookBuilder) buildSystemdService(name string, hook *kops.HookSpec) (*no
|
|||
case nil:
|
||||
unit.SetSection("Service", hook.Manifest)
|
||||
default:
|
||||
if err := h.buildDockerService(unit, hook); err != nil {
|
||||
return nil, err
|
||||
switch h.Cluster.Spec.ContainerRuntime {
|
||||
case "containerd":
|
||||
if err := h.buildContainerdService(unit, hook, name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case "docker":
|
||||
if err := h.buildDockerService(unit, hook); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown container runtime %q", h.Cluster.Spec.ContainerRuntime)
|
||||
}
|
||||
}
|
||||
definition = s(unit.Render())
|
||||
|
|
@ -141,6 +150,41 @@ func (h *HookBuilder) buildSystemdService(name string, hook *kops.HookSpec) (*no
|
|||
return service, nil
|
||||
}
|
||||
|
||||
// buildContainerdService is responsible for generating a containerd exec unit file
|
||||
func (h *HookBuilder) buildContainerdService(unit *systemd.Manifest, hook *kops.HookSpec, name string) error {
|
||||
containerdImage := hook.ExecContainer.Image
|
||||
if !strings.Contains(containerdImage, "/") {
|
||||
containerdImage = "docker.io/library/" + containerdImage
|
||||
}
|
||||
if !strings.Contains(containerdImage, ":") {
|
||||
containerdImage = containerdImage + ":latest"
|
||||
}
|
||||
|
||||
containerdArgs := []string{
|
||||
"/usr/bin/ctr", "--namespace", "k8s.io", "run", "--rm",
|
||||
"--mount", "type=bind,src=/,dst=/rootfs,options=rbind:rslave",
|
||||
"--mount", "type=bind,src=/var/run/dbus,dst=/var/run/dbus,options=rbind:rprivate",
|
||||
"--mount", "type=bind,src=/run/systemd,dst=/run/systemd,options=rbind:rprivate",
|
||||
"--net-host",
|
||||
"--privileged",
|
||||
}
|
||||
containerdArgs = append(containerdArgs, buildContainerRuntimeEnvironmentVars(hook.ExecContainer.Environment)...)
|
||||
containerdArgs = append(containerdArgs, containerdImage)
|
||||
containerdArgs = append(containerdArgs, name)
|
||||
containerdArgs = append(containerdArgs, hook.ExecContainer.Command...)
|
||||
|
||||
containerdRunCommand := systemd.EscapeCommand(containerdArgs)
|
||||
containerdPullCommand := systemd.EscapeCommand([]string{"/usr/bin/ctr", "--namespace", "k8s.io", "image", "pull", containerdImage})
|
||||
|
||||
unit.Set("Unit", "Requires", "containerd.service")
|
||||
unit.Set("Service", "ExecStartPre", containerdPullCommand)
|
||||
unit.Set("Service", "ExecStart", containerdRunCommand)
|
||||
unit.Set("Service", "Type", "oneshot")
|
||||
unit.Set("Install", "WantedBy", "multi-user.target")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildDockerService is responsible for generating a docker exec unit file
|
||||
func (h *HookBuilder) buildDockerService(unit *systemd.Manifest, hook *kops.HookSpec) error {
|
||||
dockerArgs := []string{
|
||||
|
|
@ -151,7 +195,7 @@ func (h *HookBuilder) buildDockerService(unit *systemd.Manifest, hook *kops.Hook
|
|||
"--net=host",
|
||||
"--privileged",
|
||||
}
|
||||
dockerArgs = append(dockerArgs, buildDockerEnvironmentVars(hook.ExecContainer.Environment)...)
|
||||
dockerArgs = append(dockerArgs, buildContainerRuntimeEnvironmentVars(hook.ExecContainer.Environment)...)
|
||||
dockerArgs = append(dockerArgs, hook.ExecContainer.Image)
|
||||
dockerArgs = append(dockerArgs, hook.ExecContainer.Command...)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,37 @@
|
|||
/*
|
||||
Copyright 2020 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"k8s.io/kops/upup/pkg/fi"
|
||||
)
|
||||
|
||||
func TestContainerdHooksBuilder(t *testing.T) {
|
||||
RunGoldenTest(t, "tests/golden/hooks-containerd-exec", "hooks", func(nodeupModelContext *NodeupModelContext, target *fi.ModelBuilderContext) error {
|
||||
builder := HookBuilder{NodeupModelContext: nodeupModelContext}
|
||||
return builder.Build(target)
|
||||
})
|
||||
}
|
||||
|
||||
func TestDockerHooksBuilder(t *testing.T) {
|
||||
RunGoldenTest(t, "tests/golden/hooks-docker-exec", "hooks", func(nodeupModelContext *NodeupModelContext, target *fi.ModelBuilderContext) error {
|
||||
builder := HookBuilder{NodeupModelContext: nodeupModelContext}
|
||||
return builder.Build(target)
|
||||
})
|
||||
}
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
apiVersion: kops.k8s.io/v1alpha2
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: minimal.example.com
|
||||
spec:
|
||||
kubernetesApiAccess:
|
||||
- 0.0.0.0/0
|
||||
channel: stable
|
||||
cloudProvider: aws
|
||||
configBase: memfs://clusters.example.com/minimal.example.com
|
||||
etcdClusters:
|
||||
- cpuRequest: 200m
|
||||
etcdMembers:
|
||||
- instanceGroup: master-us-test-1a
|
||||
name: us-test-1a
|
||||
memoryRequest: 100Mi
|
||||
name: main
|
||||
provider: Manager
|
||||
backups:
|
||||
backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-main
|
||||
- cpuRequest: 100m
|
||||
etcdMembers:
|
||||
- instanceGroup: master-us-test-1a
|
||||
name: us-test-1a
|
||||
memoryRequest: 100Mi
|
||||
name: events
|
||||
provider: Manager
|
||||
backups:
|
||||
backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-events
|
||||
iam: {}
|
||||
kubelet:
|
||||
anonymousAuth: false
|
||||
kubernetesVersion: v1.22.0
|
||||
masterInternalName: api.internal.minimal.example.com
|
||||
masterPublicName: api.minimal.example.com
|
||||
networkCIDR: 172.20.0.0/16
|
||||
networking:
|
||||
kubenet: {}
|
||||
nonMasqueradeCIDR: 100.64.0.0/10
|
||||
sshAccess:
|
||||
- 0.0.0.0/0
|
||||
topology:
|
||||
masters: public
|
||||
nodes: public
|
||||
subnets:
|
||||
- cidr: 172.20.32.0/19
|
||||
name: us-test-1a
|
||||
type: Public
|
||||
zone: us-test-1a
|
||||
hooks:
|
||||
- execContainer:
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- chroot /rootfs apt-get update && chroot /rootfs apt-get install -y ceph-common
|
||||
image: busybox
|
||||
|
||||
---
|
||||
|
||||
apiVersion: kops.k8s.io/v1alpha2
|
||||
kind: InstanceGroup
|
||||
metadata:
|
||||
name: master-us-test-1a
|
||||
labels:
|
||||
kops.k8s.io/cluster: minimal.example.com
|
||||
spec:
|
||||
associatePublicIp: true
|
||||
image: ami-1234
|
||||
machineType: m3.medium
|
||||
maxSize: 1
|
||||
minSize: 1
|
||||
role: Master
|
||||
subnets:
|
||||
- us-test-1a
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
Name: kops-hook-0.service
|
||||
definition: |
|
||||
[Unit]
|
||||
Description=Kops Hook kops-hook-0
|
||||
Requires=containerd.service
|
||||
|
||||
[Service]
|
||||
ExecStartPre=/usr/bin/ctr --namespace k8s.io image pull docker.io/library/busybox:latest
|
||||
ExecStart=/usr/bin/ctr --namespace k8s.io run --rm --mount type=bind,src=/,dst=/rootfs,options=rbind:rslave --mount type=bind,src=/var/run/dbus,dst=/var/run/dbus,options=rbind:rprivate --mount type=bind,src=/run/systemd,dst=/run/systemd,options=rbind:rprivate --net-host --privileged docker.io/library/busybox:latest kops-hook-0 sh -c "chroot /rootfs apt-get update && chroot /rootfs apt-get install -y ceph-common"
|
||||
Type=oneshot
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
enabled: true
|
||||
manageState: true
|
||||
running: true
|
||||
smartRestart: true
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
apiVersion: kops.k8s.io/v1alpha2
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: minimal.example.com
|
||||
spec:
|
||||
kubernetesApiAccess:
|
||||
- 0.0.0.0/0
|
||||
channel: stable
|
||||
cloudProvider: aws
|
||||
configBase: memfs://clusters.example.com/minimal.example.com
|
||||
etcdClusters:
|
||||
- cpuRequest: 200m
|
||||
etcdMembers:
|
||||
- instanceGroup: master-us-test-1a
|
||||
name: us-test-1a
|
||||
memoryRequest: 100Mi
|
||||
name: main
|
||||
provider: Manager
|
||||
backups:
|
||||
backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-main
|
||||
- cpuRequest: 100m
|
||||
etcdMembers:
|
||||
- instanceGroup: master-us-test-1a
|
||||
name: us-test-1a
|
||||
memoryRequest: 100Mi
|
||||
name: events
|
||||
provider: Manager
|
||||
backups:
|
||||
backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-events
|
||||
iam: {}
|
||||
kubelet:
|
||||
anonymousAuth: false
|
||||
kubernetesVersion: v1.22.0
|
||||
masterInternalName: api.internal.minimal.example.com
|
||||
masterPublicName: api.minimal.example.com
|
||||
networkCIDR: 172.20.0.0/16
|
||||
networking:
|
||||
kubenet: {}
|
||||
nonMasqueradeCIDR: 100.64.0.0/10
|
||||
sshAccess:
|
||||
- 0.0.0.0/0
|
||||
topology:
|
||||
masters: public
|
||||
nodes: public
|
||||
subnets:
|
||||
- cidr: 172.20.32.0/19
|
||||
name: us-test-1a
|
||||
type: Public
|
||||
zone: us-test-1a
|
||||
containerRuntime: docker
|
||||
hooks:
|
||||
- execContainer:
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- chroot /rootfs apt-get update && chroot /rootfs apt-get install -y ceph-common
|
||||
image: busybox
|
||||
|
||||
---
|
||||
|
||||
apiVersion: kops.k8s.io/v1alpha2
|
||||
kind: InstanceGroup
|
||||
metadata:
|
||||
name: master-us-test-1a
|
||||
labels:
|
||||
kops.k8s.io/cluster: minimal.example.com
|
||||
spec:
|
||||
associatePublicIp: true
|
||||
image: ami-1234
|
||||
machineType: m3.medium
|
||||
maxSize: 1
|
||||
minSize: 1
|
||||
role: Master
|
||||
subnets:
|
||||
- us-test-1a
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
Name: kops-hook-0.service
|
||||
definition: |
|
||||
[Unit]
|
||||
Description=Kops Hook kops-hook-0
|
||||
Requires=docker.service
|
||||
|
||||
[Service]
|
||||
ExecStartPre=/usr/bin/docker pull busybox
|
||||
ExecStart=/usr/bin/docker run -v /:/rootfs/ -v /var/run/dbus:/var/run/dbus -v /run/systemd:/run/systemd --net=host --privileged busybox sh -c "chroot /rootfs apt-get update && chroot /rootfs apt-get install -y ceph-common"
|
||||
Type=oneshot
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
enabled: true
|
||||
manageState: true
|
||||
running: true
|
||||
smartRestart: true
|
||||
Loading…
Reference in New Issue