diff --git a/nodeup/pkg/model/containerd.go b/nodeup/pkg/model/containerd.go index 62c27780aa..cf19009b19 100644 --- a/nodeup/pkg/model/containerd.go +++ b/nodeup/pkg/model/containerd.go @@ -48,21 +48,49 @@ func (b *ContainerdBuilder) Build(c *fi.ModelBuilderContext) error { return nil } + installContainerd := true + // @check: neither flatcar nor containeros need provision containerd.service, just the containerd daemon options switch b.Distribution { case distributions.DistributionFlatcar: klog.Infof("Detected Flatcar; won't install containerd") + installContainerd = false if b.Cluster.Spec.ContainerRuntime == "containerd" { b.buildSystemdServiceOverrideFlatcar(c) - b.buildConfigFile(c) } - return nil case distributions.DistributionContainerOS: klog.Infof("Detected ContainerOS; won't install containerd") + installContainerd = false b.buildSystemdServiceOverrideContainerOS(c) - return nil } + if b.Cluster.Spec.ContainerRuntime == "containerd" { + // Using containerd with Kubenet requires special configuration. + // This is a temporary backwards-compatible solution for kubenet users and will be deprecated when Kubenet is deprecated: + // https://github.com/containerd/containerd/blob/master/docs/cri/config.md#cni-config-template + if components.UsesKubenet(b.Cluster.Spec.Networking) { + b.buildCNIConfigTemplateFile(c) + if err := b.buildIPMasqueradeRules(c); err != nil { + return err + } + } + } + + // If there are containerd configuration overrides, apply them + b.buildOverrideConfigFile(c) + + if installContainerd { + if err := b.installContainerd(c); err != nil { + return err + } + } + + return nil +} + +// installContainerd installs the binaries and services to run containerd. +// We break it out because on immutable OSes we only configure containerd, we don't install it. +func (b *ContainerdBuilder) installContainerd(c *fi.ModelBuilderContext) error { // Add Apache2 license { t := &nodetasks.File{ @@ -73,9 +101,6 @@ func (b *ContainerdBuilder) Build(c *fi.ModelBuilderContext) error { c.AddTask(t) } - // Add config file - b.buildConfigFile(c) - // Add binaries from assets if b.Cluster.Spec.ContainerRuntime == "containerd" { f := b.Assets.FindMatches(regexp.MustCompile(`^(\./)?usr/local/(bin/containerd|bin/crictl|bin/ctr|sbin/runc)`)) @@ -97,17 +122,6 @@ func (b *ContainerdBuilder) Build(c *fi.ModelBuilderContext) error { // Add configuration file for easier use of crictl b.addCrictlConfig(c) - - // Using containerd with Kubenet requires special configuration. - // This is a temporary backwards-compatible solution for kubenet users and will be deprecated when Kubenet is deprecated: - // https://github.com/containerd/containerd/blob/master/docs/cri/config.md#cni-config-template - if components.UsesKubenet(b.Cluster.Spec.Networking) { - b.buildCNIConfigTemplateFile(c) - if err := b.buildIPMasqueradeRules(c); err != nil { - return err - } - } - } var containerRuntimeVersion string @@ -154,7 +168,7 @@ func (b *ContainerdBuilder) buildSystemdService(sv semver.Version) *nodetasks.Se manifest.Set("Service", "EnvironmentFile", "/etc/sysconfig/containerd") manifest.Set("Service", "EnvironmentFile", "/etc/environment") manifest.Set("Service", "ExecStartPre", "-/sbin/modprobe overlay") - manifest.Set("Service", "ExecStart", "/usr/bin/containerd -c /etc/containerd/config-kops.toml \"$CONTAINERD_OPTS\"") + manifest.Set("Service", "ExecStart", "/usr/bin/containerd -c "+b.containerdConfigFilePath()+" \"$CONTAINERD_OPTS\"") // notify the daemon's readiness to systemd if (b.Cluster.Spec.ContainerRuntime == "containerd" && sv.GTE(semver.MustParse("1.3.4"))) || sv.GTE(semver.MustParse("19.3.13")) { @@ -192,6 +206,18 @@ func (b *ContainerdBuilder) buildSystemdService(sv semver.Version) *nodetasks.Se return service } +// containerdConfigFilePath returns the path we use for the containerd config file +// We normally use a different path for clarity, but on some OSes we can't override the path. +// TODO: Should we just use config.toml everywhere? +func (b *ContainerdBuilder) containerdConfigFilePath() string { + switch b.Distribution { + case distributions.DistributionContainerOS: + return "/etc/containerd/config.toml" + default: + return "/etc/containerd/config-kops.toml" + } +} + // buildSystemdServiceOverrideContainerOS is responsible for overriding the containerd service for ContainerOS func (b *ContainerdBuilder) buildSystemdServiceOverrideContainerOS(c *fi.ModelBuilderContext) { lines := []string{ @@ -202,9 +228,10 @@ func (b *ContainerdBuilder) buildSystemdServiceOverrideContainerOS(c *fi.ModelBu contents := strings.Join(lines, "\n") c.AddTask(&nodetasks.File{ - Path: "/etc/systemd/system/containerd.service.d/10-kops.conf", - Contents: fi.NewStringResource(contents), - Type: nodetasks.FileType_File, + Path: "/etc/systemd/system/containerd.service.d/10-kops.conf", + Contents: fi.NewStringResource(contents), + Type: nodetasks.FileType_File, + AfterFiles: []string{b.containerdConfigFilePath()}, OnChangeExecute: [][]string{ {"systemctl", "daemon-reload"}, {"systemctl", "restart", "containerd.service"}, @@ -221,15 +248,16 @@ func (b *ContainerdBuilder) buildSystemdServiceOverrideContainerOS(c *fi.ModelBu func (b *ContainerdBuilder) buildSystemdServiceOverrideFlatcar(c *fi.ModelBuilderContext) { lines := []string{ "[Service]", - "Environment=CONTAINERD_CONFIG=/etc/containerd/config-kops.toml", + "Environment=CONTAINERD_CONFIG=" + b.containerdConfigFilePath(), "EnvironmentFile=/etc/environment", } contents := strings.Join(lines, "\n") c.AddTask(&nodetasks.File{ - Path: "/etc/systemd/system/containerd.service.d/10-kops.conf", - Contents: fi.NewStringResource(contents), - Type: nodetasks.FileType_File, + Path: "/etc/systemd/system/containerd.service.d/10-kops.conf", + Contents: fi.NewStringResource(contents), + Type: nodetasks.FileType_File, + AfterFiles: []string{b.containerdConfigFilePath()}, OnChangeExecute: [][]string{ {"systemctl", "daemon-reload"}, {"systemctl", "restart", "containerd.service"}, @@ -268,15 +296,15 @@ func (b *ContainerdBuilder) buildSysconfigFile(c *fi.ModelBuilderContext) error return nil } -// buildConfigFile is responsible for creating the containerd configuration file -func (b *ContainerdBuilder) buildConfigFile(c *fi.ModelBuilderContext) { +// buildOverrideConfigFile is responsible for creating the containerd configuration file +func (b *ContainerdBuilder) buildOverrideConfigFile(c *fi.ModelBuilderContext) { containerdConfigOverride := "" if b.Cluster.Spec.Containerd != nil { containerdConfigOverride = fi.StringValue(b.Cluster.Spec.Containerd.ConfigOverride) } c.AddTask(&nodetasks.File{ - Path: "/etc/containerd/config-kops.toml", + Path: b.containerdConfigFilePath(), Contents: fi.NewStringResource(containerdConfigOverride), Type: nodetasks.FileType_File, }) diff --git a/nodeup/pkg/model/tests/containerdbuilder/flatcar/tasks.yaml b/nodeup/pkg/model/tests/containerdbuilder/flatcar/tasks.yaml index 9b11f2b83c..030a5e9062 100644 --- a/nodeup/pkg/model/tests/containerdbuilder/flatcar/tasks.yaml +++ b/nodeup/pkg/model/tests/containerdbuilder/flatcar/tasks.yaml @@ -1,7 +1,31 @@ +contents: | + { + "cniVersion": "0.4.0", + "name": "k8s-pod-network", + "plugins": [ + { + "type": "ptp", + "ipam": { + "type": "host-local", + "ranges": [[{"subnet": "{{.PodCIDR}}"}]], + "routes": [{ "dst": "0.0.0.0/0" }] + } + }, + { + "type": "portmap", + "capabilities": {"portMappings": true} + } + ] + } +path: /etc/containerd/config-cni.template +type: file +--- contents: "" path: /etc/containerd/config-kops.toml type: file --- +afterFiles: +- /etc/containerd/config-kops.toml contents: |- [Service] Environment=CONTAINERD_CONFIG=/etc/containerd/config-kops.toml @@ -18,3 +42,34 @@ onChangeExecute: - '&' path: /etc/systemd/system/containerd.service.d/10-kops.conf type: file +--- +contents: | + #!/bin/bash + # Built by kOps - do not edit + + iptables -w -t nat -N IP-MASQ + iptables -w -t nat -A POSTROUTING -m comment --comment "ip-masq: ensure nat POSTROUTING directs all non-LOCAL destination traffic to our custom IP-MASQ chain" -m addrtype ! --dst-type LOCAL -j IP-MASQ + iptables -w -t nat -A IP-MASQ -d 100.64.0.0/10 -m comment --comment "ip-masq: pod cidr is not subject to MASQUERADE" -j RETURN + iptables -w -t nat -A IP-MASQ -m comment --comment "ip-masq: outbound traffic is subject to MASQUERADE (must be last in chain)" -j MASQUERADE +mode: "0755" +path: /opt/kops/bin/cni-iptables-setup +type: file +--- +Name: cni-iptables-setup.service +definition: | + [Unit] + Description=Configure iptables for kubernetes CNI + Documentation=https://github.com/kubernetes/kops + Before=network.target + + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=/opt/kops/bin/cni-iptables-setup + + [Install] + WantedBy=basic.target +enabled: true +manageState: true +running: true +smartRestart: true