Merge pull request #8269 from justinsb/ops_kops_writeable

Make /opt/kops writeable on COS

nodeup/pkg/model/directories.go

@@ -17,6 +17,8 @@ limitations under the License.
 package model
 
 import (
+	"path/filepath"
+
 	"k8s.io/kops/nodeup/pkg/distros"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
@@ -46,5 +48,70 @@ func (b *DirectoryBuilder) Build(c *fi.ModelBuilderContext) error {
 		})
 	}
 
+	// We try to put things into /opt/kops
+	// On some OSes though, /opt/ is not writeable, and we can't even create the mountpoint
+	if b.Distribution == distros.DistributionContainerOS {
+		src := "/mnt/stateful_partition/opt/"
+
+		c.AddTask(&nodetasks.File{
+			Path: src,
+			Type: nodetasks.FileType_Directory,
+			Mode: s("0755"),
+		})
+
+		// Rebuild things we are masking
+		c.AddTask(&nodetasks.File{
+			Path: filepath.Join(src, "google"),
+			Type: nodetasks.FileType_Directory,
+			Mode: s("0755"),
+		})
+		c.AddTask(&nodetasks.File{
+			Path: filepath.Join(src, "google", "crash-reporter"),
+			Type: nodetasks.FileType_Directory,
+			Mode: s("0755"),
+		})
+		c.AddTask(&nodetasks.File{
+			Path:     filepath.Join(src, "google", "crash-reporter", "filter"),
+			Type:     nodetasks.FileType_File,
+			Mode:     s("0755"),
+			Contents: fi.NewStringResource(cosCrashFilter),
+		})
+
+		// Precreate the directory that will be /opt/kops, so we can bind remount it
+		c.AddTask(&nodetasks.File{
+			Path: filepath.Join(src, "kops"),
+			Type: nodetasks.FileType_Directory,
+			Mode: s("0755"),
+		})
+		c.AddTask(&nodetasks.File{
+			Path: filepath.Join(src, "kops", "bin"),
+			Type: nodetasks.FileType_Directory,
+			Mode: s("0755"),
+		})
+
+		c.AddTask(&nodetasks.BindMount{
+			Source:     src,
+			Mountpoint: "/opt",
+			Options:    []string{"ro"},
+		})
+
+		c.AddTask(&nodetasks.BindMount{
+			Source:     filepath.Join(src, "kops", "bin"),
+			Mountpoint: "/opt/kops/bin",
+			Options:    []string{"exec", "nosuid", "nodev"},
+		})
+	}
+
 	return nil
 }
+
+// cosCrashFilter is used on COS to prevent userspace crash-reporting
+// This is the one thing we need from /opt
+const cosCrashFilter = `#!/bin/bash
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Do no collect any userland crash.
+exit 1
+`

pkg/model/resources/nodeup.go

@@ -54,6 +54,10 @@ NODEUP_HASH={{ NodeUpSourceHash }}
 
 function ensure-install-dir() {
   INSTALL_DIR="/opt/kops"
+  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+  if [[ -d /var/lib/toolbox ]]; then
+    INSTALL_DIR="/var/lib/toolbox/kops"
+  fi
   mkdir -p ${INSTALL_DIR}/bin
   mkdir -p ${INSTALL_DIR}/conf
   cd ${INSTALL_DIR}

pkg/model/tests/data/bootstrapscript_0.txt

@@ -43,6 +43,10 @@ systemctl daemon-reexec
 
 function ensure-install-dir() {
   INSTALL_DIR="/opt/kops"
+  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+  if [[ -d /var/lib/toolbox ]]; then
+    INSTALL_DIR="/var/lib/toolbox/kops"
+  fi
   mkdir -p ${INSTALL_DIR}/bin
   mkdir -p ${INSTALL_DIR}/conf
   cd ${INSTALL_DIR}

pkg/model/tests/data/bootstrapscript_1.txt

@@ -43,6 +43,10 @@ systemctl daemon-reexec
 
 function ensure-install-dir() {
   INSTALL_DIR="/opt/kops"
+  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+  if [[ -d /var/lib/toolbox ]]; then
+    INSTALL_DIR="/var/lib/toolbox/kops"
+  fi
   mkdir -p ${INSTALL_DIR}/bin
   mkdir -p ${INSTALL_DIR}/conf
   cd ${INSTALL_DIR}

pkg/model/tests/data/bootstrapscript_2.txt

@@ -43,6 +43,10 @@ systemctl daemon-reexec
 
 function ensure-install-dir() {
   INSTALL_DIR="/opt/kops"
+  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+  if [[ -d /var/lib/toolbox ]]; then
+    INSTALL_DIR="/var/lib/toolbox/kops"
+  fi
   mkdir -p ${INSTALL_DIR}/bin
   mkdir -p ${INSTALL_DIR}/conf
   cd ${INSTALL_DIR}

pkg/model/tests/data/bootstrapscript_3.txt

@@ -43,6 +43,10 @@ systemctl daemon-reexec
 
 function ensure-install-dir() {
   INSTALL_DIR="/opt/kops"
+  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+  if [[ -d /var/lib/toolbox ]]; then
+    INSTALL_DIR="/var/lib/toolbox/kops"
+  fi
   mkdir -p ${INSTALL_DIR}/bin
   mkdir -p ${INSTALL_DIR}/conf
   cd ${INSTALL_DIR}

pkg/model/tests/data/bootstrapscript_4.txt

@@ -43,6 +43,10 @@ systemctl daemon-reexec
 
 function ensure-install-dir() {
   INSTALL_DIR="/opt/kops"
+  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+  if [[ -d /var/lib/toolbox ]]; then
+    INSTALL_DIR="/var/lib/toolbox/kops"
+  fi
   mkdir -p ${INSTALL_DIR}/bin
   mkdir -p ${INSTALL_DIR}/conf
   cd ${INSTALL_DIR}

pkg/model/tests/data/bootstrapscript_5.txt

@@ -43,6 +43,10 @@ systemctl daemon-reexec
 
 function ensure-install-dir() {
   INSTALL_DIR="/opt/kops"
+  # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+  if [[ -d /var/lib/toolbox ]]; then
+    INSTALL_DIR="/var/lib/toolbox/kops"
+  fi
   mkdir -p ${INSTALL_DIR}/bin
   mkdir -p ${INSTALL_DIR}/conf
   cd ${INSTALL_DIR}

tests/integration/update_cluster/additional_cidr/cloudformation.json.extracted.yaml

@@ -28,6 +28,10 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1bmastersadditionalcidrex
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}
@@ -318,6 +322,10 @@ Resources.AWSAutoScalingLaunchConfigurationnodesadditionalcidrexamplecom.Propert
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}

tests/integration/update_cluster/additional_user-data/cloudformation.json.extracted.yaml

@@ -37,6 +37,10 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1amastersadditionaluserda
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}
@@ -347,6 +351,10 @@ Resources.AWSAutoScalingLaunchConfigurationnodesadditionaluserdataexamplecom.Pro
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}

tests/integration/update_cluster/complex/cloudformation.json.extracted.yaml

@@ -28,6 +28,10 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1amasterscomplexexampleco
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}
@@ -320,6 +324,10 @@ Resources.AWSAutoScalingLaunchConfigurationnodescomplexexamplecom.Properties.Use
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}

tests/integration/update_cluster/containerd-cloudformation/cloudformation.json.extracted.yaml

@@ -28,6 +28,10 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1amasterscontainerdexampl
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}
@@ -312,6 +316,10 @@ Resources.AWSAutoScalingLaunchConfigurationnodescontainerdexamplecom.Properties.
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}

tests/integration/update_cluster/existing_iam_cloudformation/cloudformation.json.extracted.yaml

@@ -28,6 +28,10 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1amastersminimalexampleco
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}
@@ -318,6 +322,10 @@ Resources.AWSAutoScalingLaunchConfigurationnodesminimalexamplecom.Properties.Use
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}

tests/integration/update_cluster/externallb/cloudformation.json.extracted.yaml

@@ -28,6 +28,10 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1amastersexternallbexampl
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}
@@ -318,6 +322,10 @@ Resources.AWSAutoScalingLaunchConfigurationnodesexternallbexamplecom.Properties.
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}

tests/integration/update_cluster/minimal-cloudformation/cloudformation.json.extracted.yaml

@@ -28,6 +28,10 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1amastersminimalexampleco
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}
@@ -318,6 +322,10 @@ Resources.AWSAutoScalingLaunchConfigurationnodesminimalexamplecom.Properties.Use
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}

tests/integration/update_cluster/mixed_instances/cloudformation.json.extracted.yaml

@@ -28,6 +28,10 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1amastersmixedinstancesex
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}
@@ -320,6 +324,10 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1bmastersmixedinstancesex
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}
@@ -612,6 +620,10 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1cmastersmixedinstancesex
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}
@@ -904,6 +916,10 @@ Resources.AWSEC2LaunchTemplatenodesmixedinstancesexamplecom.Properties.LaunchTem
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}

tests/integration/update_cluster/mixed_instances_spot/cloudformation.json.extracted.yaml

@@ -28,6 +28,10 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1amastersmixedinstancesex
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}
@@ -320,6 +324,10 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1bmastersmixedinstancesex
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}
@@ -612,6 +620,10 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1cmastersmixedinstancesex
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}
@@ -904,6 +916,10 @@ Resources.AWSEC2LaunchTemplatenodesmixedinstancesexamplecom.Properties.LaunchTem
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}

tests/integration/update_cluster/privatecalico/cloudformation.json.extracted.yaml

@@ -28,6 +28,10 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1amastersprivatecalicoexa
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}
@@ -316,6 +320,10 @@ Resources.AWSAutoScalingLaunchConfigurationnodesprivatecalicoexamplecom.Properti
 
   function ensure-install-dir() {
     INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
     mkdir -p ${INSTALL_DIR}/bin
     mkdir -p ${INSTALL_DIR}/conf
     cd ${INSTALL_DIR}

upup/pkg/fi/nodeup/nodetasks/bindmount.go

@@ -200,13 +200,13 @@ func (e *BindMount) execute(t Executor) error {
 	for _, option := range e.Options {
 		switch option {
 		case "ro":
-			simpleOptions = append(simpleOptions, "ro")
+			simpleOptions = append(simpleOptions, option)
 
 		case "rshared":
 			makeOptions = append(makeOptions, "--make-rshared")
 
-		case "exec":
+		case "exec", "noexec", "nosuid", "nodev":
-			remountOptions = append(remountOptions, "exec")
+			remountOptions = append(remountOptions, option)
 
 		default:
 			return fmt.Errorf("unknown option: %q", option)