diff --git a/pkg/model/bootstrapscript.go b/pkg/model/bootstrapscript.go index 4a9d29af79..03e5565f12 100644 --- a/pkg/model/bootstrapscript.go +++ b/pkg/model/bootstrapscript.go @@ -105,6 +105,11 @@ func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup, cs *kops.Cluste spec["masterKubelet"] = cs.MasterKubelet } + hooks := b.getRelevantHooks(cs.Hooks, ig.Spec.Role) + if len(hooks) > 0 { + spec["hooks"] = hooks + } + content, err := yaml.Marshal(spec) if err != nil { return "", fmt.Errorf("error converting cluster spec to yaml for inclusion within bootstrap script: %v", err) @@ -117,6 +122,10 @@ func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup, cs *kops.Cluste spec["kubelet"] = ig.Spec.Kubelet spec["nodeLabels"] = ig.Spec.NodeLabels spec["taints"] = ig.Spec.Taints + hooks := b.getRelevantHooks(ig.Spec.Hooks, ig.Spec.Role) + if len(hooks) > 0 { + spec["hooks"] = hooks + } content, err := yaml.Marshal(spec) if err != nil { @@ -133,6 +142,24 @@ func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup, cs *kops.Cluste return fi.WrapResource(templateResource), nil } +// getRelevantHooks returns a list of hooks to be applied to the instance group +func (b *BootstrapScript) getRelevantHooks(hooks []kops.HookSpec, role kops.InstanceGroupRole) []kops.HookSpec { + relevantHooks := []kops.HookSpec{} + for _, hook := range hooks { + if len(hook.Roles) == 0 { + relevantHooks = append(relevantHooks, hook) + continue + } + for _, hookRole := range hook.Roles { + if role == hookRole { + relevantHooks = append(relevantHooks, hook) + break + } + } + } + return relevantHooks +} + func (b *BootstrapScript) createProxyEnv(ps *kops.EgressProxySpec) string { var buffer bytes.Buffer diff --git a/pkg/model/bootstrapscript_test.go b/pkg/model/bootstrapscript_test.go index 46a61a5446..c9c85e383f 100644 --- a/pkg/model/bootstrapscript_test.go +++ b/pkg/model/bootstrapscript_test.go @@ -23,6 +23,7 @@ import ( "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/nodeup" + "k8s.io/kops/pkg/diff" ) func Test_ProxyFunc(t *testing.T) { @@ -57,20 +58,53 @@ func TestBootstrapUserData(t *testing.T) { cs := []struct { Role kops.InstanceGroupRole ExpectedFilePath string + HookSpecRoles []kops.InstanceGroupRole }{ { Role: "Master", ExpectedFilePath: "tests/data/bootstrapscript_0.txt", + HookSpecRoles: []kops.InstanceGroupRole{""}, + }, + { + Role: "Master", + ExpectedFilePath: "tests/data/bootstrapscript_0.txt", + HookSpecRoles: []kops.InstanceGroupRole{"Node"}, + }, + { + Role: "Master", + ExpectedFilePath: "tests/data/bootstrapscript_1.txt", + HookSpecRoles: []kops.InstanceGroupRole{"Master"}, + }, + { + Role: "Master", + ExpectedFilePath: "tests/data/bootstrapscript_2.txt", + HookSpecRoles: []kops.InstanceGroupRole{"Master", "Node"}, }, { Role: "Node", - ExpectedFilePath: "tests/data/bootstrapscript_1.txt", + ExpectedFilePath: "tests/data/bootstrapscript_3.txt", + HookSpecRoles: []kops.InstanceGroupRole{""}, + }, + { + Role: "Node", + ExpectedFilePath: "tests/data/bootstrapscript_4.txt", + HookSpecRoles: []kops.InstanceGroupRole{"Node"}, + }, + { + Role: "Node", + ExpectedFilePath: "tests/data/bootstrapscript_3.txt", + HookSpecRoles: []kops.InstanceGroupRole{"Master"}, + }, + { + Role: "Node", + ExpectedFilePath: "tests/data/bootstrapscript_5.txt", + HookSpecRoles: []kops.InstanceGroupRole{"Master", "Node"}, }, } for i, x := range cs { - spec := makeTestCluster().Spec - group := makeTestInstanceGroup(x.Role) + spec := makeTestCluster(x.HookSpecRoles).Spec + group := makeTestInstanceGroup(x.Role, x.HookSpecRoles) renderNodeUpConfig := func(ig *kops.InstanceGroup) (*nodeup.Config, error) { return &nodeup.Config{}, nil @@ -100,12 +134,14 @@ func TestBootstrapUserData(t *testing.T) { } if actual != string(expectedBytes) { - t.Errorf("case %d, expected: %s. got: %s", i, string(expectedBytes), actual) + diffString := diff.FormatDiff(string(expectedBytes), actual) + t.Errorf("case %d failed, actual output differed from expected.", i) + t.Logf("diff:\n%s\n", diffString) } } } -func makeTestCluster() *kops.Cluster { +func makeTestCluster(hookSpecRoles []kops.InstanceGroupRole) *kops.Cluster { return &kops.Cluster{ Spec: kops.ClusterSpec{ CloudProvider: "aws", @@ -159,11 +195,24 @@ func makeTestCluster() *kops.Cluster { Port: 80, }, }, + Hooks: []kops.HookSpec{ + { + ExecContainer: &kops.ExecContainerAction{ + Command: []string{ + "sh", + "-c", + "chroot /rootfs apt-get update && chroot /rootfs apt-get install -y ceph-common", + }, + Image: "busybox", + }, + Roles: hookSpecRoles, + }, + }, }, } } -func makeTestInstanceGroup(role kops.InstanceGroupRole) *kops.InstanceGroup { +func makeTestInstanceGroup(role kops.InstanceGroupRole, hookSpecRoles []kops.InstanceGroupRole) *kops.InstanceGroup { return &kops.InstanceGroup{ Spec: kops.InstanceGroupSpec{ Kubelet: &kops.KubeletConfigSpec{ @@ -178,6 +227,20 @@ func makeTestInstanceGroup(role kops.InstanceGroupRole) *kops.InstanceGroup { "key1=value1:NoSchedule", "key2=value2:NoExecute", }, + Hooks: []kops.HookSpec{ + { + Name: "disable-update-engine.service", + Before: []string{ + "update-engine.service", + "kubelet.service", + }, + Manifest: "Type=oneshot\nExecStart=/usr/bin/systemctl stop update-engine.service", + Roles: hookSpecRoles, + }, { + Name: "apply-to-all.service", + Manifest: "Type=oneshot\nExecStart=/usr/bin/systemctl start apply-to-all.service", + }, + }, }, } } diff --git a/pkg/model/tests/data/bootstrapscript_0.txt b/pkg/model/tests/data/bootstrapscript_0.txt index 3bc00be24c..3fca9be8ce 100644 --- a/pkg/model/tests/data/bootstrapscript_0.txt +++ b/pkg/model/tests/data/bootstrapscript_0.txt @@ -167,6 +167,11 @@ masterKubelet: __EOF_CLUSTER_SPEC cat > ig_spec.yaml << __EOF_IG_SPEC +hooks: +- manifest: |- + Type=oneshot + ExecStart=/usr/bin/systemctl start apply-to-all.service + name: apply-to-all.service kubelet: kubeconfigPath: /etc/kubernetes/igconfig.txt nodeLabels: diff --git a/pkg/model/tests/data/bootstrapscript_1.txt b/pkg/model/tests/data/bootstrapscript_1.txt index e81aabfe87..d42c571e56 100644 --- a/pkg/model/tests/data/bootstrapscript_1.txt +++ b/pkg/model/tests/data/bootstrapscript_1.txt @@ -149,16 +149,47 @@ cloudConfig: nodeTags: something docker: logLevel: INFO +hooks: +- execContainer: + command: + - sh + - -c + - chroot /rootfs apt-get update && chroot /rootfs apt-get install -y ceph-common + image: busybox + roles: + - Master +kubeAPIServer: + image: CoreOS +kubeControllerManager: + cloudProvider: aws kubeProxy: cpuRequest: 30m featureGates: AdvancedAuditing: "true" +kubeScheduler: + image: SomeImage kubelet: kubeconfigPath: /etc/kubernetes/config.txt +masterKubelet: + kubeconfigPath: /etc/kubernetes/config.cfg __EOF_CLUSTER_SPEC cat > ig_spec.yaml << __EOF_IG_SPEC +hooks: +- before: + - update-engine.service + - kubelet.service + manifest: |- + Type=oneshot + ExecStart=/usr/bin/systemctl stop update-engine.service + name: disable-update-engine.service + roles: + - Master +- manifest: |- + Type=oneshot + ExecStart=/usr/bin/systemctl start apply-to-all.service + name: apply-to-all.service kubelet: kubeconfigPath: /etc/kubernetes/igconfig.txt nodeLabels: diff --git a/pkg/model/tests/data/bootstrapscript_2.txt b/pkg/model/tests/data/bootstrapscript_2.txt new file mode 100644 index 0000000000..23f74981a7 --- /dev/null +++ b/pkg/model/tests/data/bootstrapscript_2.txt @@ -0,0 +1,212 @@ +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=NUSource +NODEUP_HASH=NUSHash + + + + +export http_proxy=http://example.com:80 +export https_proxy=${http_proxy} +export no_proxy= +export NO_PROXY=${no_proxy} +echo "export http_proxy=${http_proxy}" >> /etc/default/docker +echo "export https_proxy=${http_proxy}" >> /etc/default/docker +echo "export no_proxy=${no_proxy}" >> /etc/default/docker +echo "export NO_PROXY=${no_proxy}" >> /etc/default/docker +echo "export http_proxy=${http_proxy}" >> /etc/environment +echo "export https_proxy=${http_proxy}" >> /etc/environment +echo "export no_proxy=${no_proxy}" >> /etc/environment +echo "export NO_PROXY=${no_proxy}" >> /etc/environment +echo DefaultEnvironment=\"http_proxy=${http_proxy}\" \"https_proxy=${http_proxy}\"echo DefaultEnvironment=\"http_proxy=${http_proxy}\" \"https_proxy=${http_proxy}\" \"NO_PROXY=${no_proxy}\" \"no_proxy=${no_proxy}\" >> /etc/systemd/system.conf +source /etc/environment +systemctl daemon-reload +systemctl daemon-reexec +if [ -f /etc/lsb-release ] || [ -f /etc/debian_version ]; then + echo "Acquire::http::Proxy \"${http_proxy}\";" > /etc/apt/apt.conf.d/30proxy +elif [ -f /etc/redhat-release ]; then + echo "http_proxy=${http_proxy}" >> /etc/yum.conf +fi + + +function ensure-install-dir() { + INSTALL_DIR="/var/cache/kubernetes-install" + # On ContainerOS, we install to /var/lib/toolbox install (because of noexec) + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kubernetes-install" + fi + mkdir -p ${INSTALL_DIR} + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. Takes a hash and a set of URLs. +# +# $1 is the sha1 of the URL. Can be "" if the sha1 is unknown. +# $2+ are the URLs to download. +download-or-bust() { + local -r hash="$1" + shift 1 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + local file="${url##*/}" + rm -f "${file}" + if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; then + echo "== Failed to download ${url}. Retrying. ==" + elif [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha1sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + # TODO(zmerlynn): Now we REALLY have no excuse not to do the reboot + # optimization. + + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + local -r nodeup_filename="${nodeup_urls[0]##*/}" + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha1 (not found in env)" + download-or-bust "" "${nodeup_urls[@]/%/.sha1}" + local -r nodeup_hash=$(cat "${nodeup_filename}.sha1") + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + # In case of failure checking integrity of release, retry. + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > cluster_spec.yaml << __EOF_CLUSTER_SPEC +cloudConfig: + nodeTags: something +docker: + logLevel: INFO +hooks: +- execContainer: + command: + - sh + - -c + - chroot /rootfs apt-get update && chroot /rootfs apt-get install -y ceph-common + image: busybox + roles: + - Master + - Node +kubeAPIServer: + image: CoreOS +kubeControllerManager: + cloudProvider: aws +kubeProxy: + cpuRequest: 30m + featureGates: + AdvancedAuditing: "true" +kubeScheduler: + image: SomeImage +kubelet: + kubeconfigPath: /etc/kubernetes/config.txt +masterKubelet: + kubeconfigPath: /etc/kubernetes/config.cfg + +__EOF_CLUSTER_SPEC + +cat > ig_spec.yaml << __EOF_IG_SPEC +hooks: +- before: + - update-engine.service + - kubelet.service + manifest: |- + Type=oneshot + ExecStart=/usr/bin/systemctl stop update-engine.service + name: disable-update-engine.service + roles: + - Master + - Node +- manifest: |- + Type=oneshot + ExecStart=/usr/bin/systemctl start apply-to-all.service + name: apply-to-all.service +kubelet: + kubeconfigPath: /etc/kubernetes/igconfig.txt +nodeLabels: + label2: value2 + labelname: labelvalue +taints: +- key1=value1:NoSchedule +- key2=value2:NoExecute + +__EOF_IG_SPEC + +cat > kube_env.yaml << __EOF_KUBE_ENV +{} + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/pkg/model/tests/data/bootstrapscript_3.txt b/pkg/model/tests/data/bootstrapscript_3.txt new file mode 100644 index 0000000000..dda5adee86 --- /dev/null +++ b/pkg/model/tests/data/bootstrapscript_3.txt @@ -0,0 +1,184 @@ +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=NUSource +NODEUP_HASH=NUSHash + + + + +export http_proxy=http://example.com:80 +export https_proxy=${http_proxy} +export no_proxy= +export NO_PROXY=${no_proxy} +echo "export http_proxy=${http_proxy}" >> /etc/default/docker +echo "export https_proxy=${http_proxy}" >> /etc/default/docker +echo "export no_proxy=${no_proxy}" >> /etc/default/docker +echo "export NO_PROXY=${no_proxy}" >> /etc/default/docker +echo "export http_proxy=${http_proxy}" >> /etc/environment +echo "export https_proxy=${http_proxy}" >> /etc/environment +echo "export no_proxy=${no_proxy}" >> /etc/environment +echo "export NO_PROXY=${no_proxy}" >> /etc/environment +echo DefaultEnvironment=\"http_proxy=${http_proxy}\" \"https_proxy=${http_proxy}\"echo DefaultEnvironment=\"http_proxy=${http_proxy}\" \"https_proxy=${http_proxy}\" \"NO_PROXY=${no_proxy}\" \"no_proxy=${no_proxy}\" >> /etc/systemd/system.conf +source /etc/environment +systemctl daemon-reload +systemctl daemon-reexec +if [ -f /etc/lsb-release ] || [ -f /etc/debian_version ]; then + echo "Acquire::http::Proxy \"${http_proxy}\";" > /etc/apt/apt.conf.d/30proxy +elif [ -f /etc/redhat-release ]; then + echo "http_proxy=${http_proxy}" >> /etc/yum.conf +fi + + +function ensure-install-dir() { + INSTALL_DIR="/var/cache/kubernetes-install" + # On ContainerOS, we install to /var/lib/toolbox install (because of noexec) + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kubernetes-install" + fi + mkdir -p ${INSTALL_DIR} + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. Takes a hash and a set of URLs. +# +# $1 is the sha1 of the URL. Can be "" if the sha1 is unknown. +# $2+ are the URLs to download. +download-or-bust() { + local -r hash="$1" + shift 1 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + local file="${url##*/}" + rm -f "${file}" + if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; then + echo "== Failed to download ${url}. Retrying. ==" + elif [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha1sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + # TODO(zmerlynn): Now we REALLY have no excuse not to do the reboot + # optimization. + + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + local -r nodeup_filename="${nodeup_urls[0]##*/}" + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha1 (not found in env)" + download-or-bust "" "${nodeup_urls[@]/%/.sha1}" + local -r nodeup_hash=$(cat "${nodeup_filename}.sha1") + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + # In case of failure checking integrity of release, retry. + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > cluster_spec.yaml << __EOF_CLUSTER_SPEC +cloudConfig: + nodeTags: something +docker: + logLevel: INFO +kubeProxy: + cpuRequest: 30m + featureGates: + AdvancedAuditing: "true" +kubelet: + kubeconfigPath: /etc/kubernetes/config.txt + +__EOF_CLUSTER_SPEC + +cat > ig_spec.yaml << __EOF_IG_SPEC +hooks: +- manifest: |- + Type=oneshot + ExecStart=/usr/bin/systemctl start apply-to-all.service + name: apply-to-all.service +kubelet: + kubeconfigPath: /etc/kubernetes/igconfig.txt +nodeLabels: + label2: value2 + labelname: labelvalue +taints: +- key1=value1:NoSchedule +- key2=value2:NoExecute + +__EOF_IG_SPEC + +cat > kube_env.yaml << __EOF_KUBE_ENV +{} + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/pkg/model/tests/data/bootstrapscript_4.txt b/pkg/model/tests/data/bootstrapscript_4.txt new file mode 100644 index 0000000000..8be9a9125e --- /dev/null +++ b/pkg/model/tests/data/bootstrapscript_4.txt @@ -0,0 +1,202 @@ +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=NUSource +NODEUP_HASH=NUSHash + + + + +export http_proxy=http://example.com:80 +export https_proxy=${http_proxy} +export no_proxy= +export NO_PROXY=${no_proxy} +echo "export http_proxy=${http_proxy}" >> /etc/default/docker +echo "export https_proxy=${http_proxy}" >> /etc/default/docker +echo "export no_proxy=${no_proxy}" >> /etc/default/docker +echo "export NO_PROXY=${no_proxy}" >> /etc/default/docker +echo "export http_proxy=${http_proxy}" >> /etc/environment +echo "export https_proxy=${http_proxy}" >> /etc/environment +echo "export no_proxy=${no_proxy}" >> /etc/environment +echo "export NO_PROXY=${no_proxy}" >> /etc/environment +echo DefaultEnvironment=\"http_proxy=${http_proxy}\" \"https_proxy=${http_proxy}\"echo DefaultEnvironment=\"http_proxy=${http_proxy}\" \"https_proxy=${http_proxy}\" \"NO_PROXY=${no_proxy}\" \"no_proxy=${no_proxy}\" >> /etc/systemd/system.conf +source /etc/environment +systemctl daemon-reload +systemctl daemon-reexec +if [ -f /etc/lsb-release ] || [ -f /etc/debian_version ]; then + echo "Acquire::http::Proxy \"${http_proxy}\";" > /etc/apt/apt.conf.d/30proxy +elif [ -f /etc/redhat-release ]; then + echo "http_proxy=${http_proxy}" >> /etc/yum.conf +fi + + +function ensure-install-dir() { + INSTALL_DIR="/var/cache/kubernetes-install" + # On ContainerOS, we install to /var/lib/toolbox install (because of noexec) + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kubernetes-install" + fi + mkdir -p ${INSTALL_DIR} + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. Takes a hash and a set of URLs. +# +# $1 is the sha1 of the URL. Can be "" if the sha1 is unknown. +# $2+ are the URLs to download. +download-or-bust() { + local -r hash="$1" + shift 1 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + local file="${url##*/}" + rm -f "${file}" + if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; then + echo "== Failed to download ${url}. Retrying. ==" + elif [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha1sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + # TODO(zmerlynn): Now we REALLY have no excuse not to do the reboot + # optimization. + + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + local -r nodeup_filename="${nodeup_urls[0]##*/}" + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha1 (not found in env)" + download-or-bust "" "${nodeup_urls[@]/%/.sha1}" + local -r nodeup_hash=$(cat "${nodeup_filename}.sha1") + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + # In case of failure checking integrity of release, retry. + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > cluster_spec.yaml << __EOF_CLUSTER_SPEC +cloudConfig: + nodeTags: something +docker: + logLevel: INFO +hooks: +- execContainer: + command: + - sh + - -c + - chroot /rootfs apt-get update && chroot /rootfs apt-get install -y ceph-common + image: busybox + roles: + - Node +kubeProxy: + cpuRequest: 30m + featureGates: + AdvancedAuditing: "true" +kubelet: + kubeconfigPath: /etc/kubernetes/config.txt + +__EOF_CLUSTER_SPEC + +cat > ig_spec.yaml << __EOF_IG_SPEC +hooks: +- before: + - update-engine.service + - kubelet.service + manifest: |- + Type=oneshot + ExecStart=/usr/bin/systemctl stop update-engine.service + name: disable-update-engine.service + roles: + - Node +- manifest: |- + Type=oneshot + ExecStart=/usr/bin/systemctl start apply-to-all.service + name: apply-to-all.service +kubelet: + kubeconfigPath: /etc/kubernetes/igconfig.txt +nodeLabels: + label2: value2 + labelname: labelvalue +taints: +- key1=value1:NoSchedule +- key2=value2:NoExecute + +__EOF_IG_SPEC + +cat > kube_env.yaml << __EOF_KUBE_ENV +{} + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/pkg/model/tests/data/bootstrapscript_5.txt b/pkg/model/tests/data/bootstrapscript_5.txt new file mode 100644 index 0000000000..4c2ec2da70 --- /dev/null +++ b/pkg/model/tests/data/bootstrapscript_5.txt @@ -0,0 +1,204 @@ +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=NUSource +NODEUP_HASH=NUSHash + + + + +export http_proxy=http://example.com:80 +export https_proxy=${http_proxy} +export no_proxy= +export NO_PROXY=${no_proxy} +echo "export http_proxy=${http_proxy}" >> /etc/default/docker +echo "export https_proxy=${http_proxy}" >> /etc/default/docker +echo "export no_proxy=${no_proxy}" >> /etc/default/docker +echo "export NO_PROXY=${no_proxy}" >> /etc/default/docker +echo "export http_proxy=${http_proxy}" >> /etc/environment +echo "export https_proxy=${http_proxy}" >> /etc/environment +echo "export no_proxy=${no_proxy}" >> /etc/environment +echo "export NO_PROXY=${no_proxy}" >> /etc/environment +echo DefaultEnvironment=\"http_proxy=${http_proxy}\" \"https_proxy=${http_proxy}\"echo DefaultEnvironment=\"http_proxy=${http_proxy}\" \"https_proxy=${http_proxy}\" \"NO_PROXY=${no_proxy}\" \"no_proxy=${no_proxy}\" >> /etc/systemd/system.conf +source /etc/environment +systemctl daemon-reload +systemctl daemon-reexec +if [ -f /etc/lsb-release ] || [ -f /etc/debian_version ]; then + echo "Acquire::http::Proxy \"${http_proxy}\";" > /etc/apt/apt.conf.d/30proxy +elif [ -f /etc/redhat-release ]; then + echo "http_proxy=${http_proxy}" >> /etc/yum.conf +fi + + +function ensure-install-dir() { + INSTALL_DIR="/var/cache/kubernetes-install" + # On ContainerOS, we install to /var/lib/toolbox install (because of noexec) + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kubernetes-install" + fi + mkdir -p ${INSTALL_DIR} + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. Takes a hash and a set of URLs. +# +# $1 is the sha1 of the URL. Can be "" if the sha1 is unknown. +# $2+ are the URLs to download. +download-or-bust() { + local -r hash="$1" + shift 1 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + local file="${url##*/}" + rm -f "${file}" + if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; then + echo "== Failed to download ${url}. Retrying. ==" + elif [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha1sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + # TODO(zmerlynn): Now we REALLY have no excuse not to do the reboot + # optimization. + + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + local -r nodeup_filename="${nodeup_urls[0]##*/}" + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha1 (not found in env)" + download-or-bust "" "${nodeup_urls[@]/%/.sha1}" + local -r nodeup_hash=$(cat "${nodeup_filename}.sha1") + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + # In case of failure checking integrity of release, retry. + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > cluster_spec.yaml << __EOF_CLUSTER_SPEC +cloudConfig: + nodeTags: something +docker: + logLevel: INFO +hooks: +- execContainer: + command: + - sh + - -c + - chroot /rootfs apt-get update && chroot /rootfs apt-get install -y ceph-common + image: busybox + roles: + - Master + - Node +kubeProxy: + cpuRequest: 30m + featureGates: + AdvancedAuditing: "true" +kubelet: + kubeconfigPath: /etc/kubernetes/config.txt + +__EOF_CLUSTER_SPEC + +cat > ig_spec.yaml << __EOF_IG_SPEC +hooks: +- before: + - update-engine.service + - kubelet.service + manifest: |- + Type=oneshot + ExecStart=/usr/bin/systemctl stop update-engine.service + name: disable-update-engine.service + roles: + - Master + - Node +- manifest: |- + Type=oneshot + ExecStart=/usr/bin/systemctl start apply-to-all.service + name: apply-to-all.service +kubelet: + kubeconfigPath: /etc/kubernetes/igconfig.txt +nodeLabels: + label2: value2 + labelname: labelvalue +taints: +- key1=value1:NoSchedule +- key2=value2:NoExecute + +__EOF_IG_SPEC + +cat > kube_env.yaml << __EOF_KUBE_ENV +{} + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done =="