mirror of https://github.com/kubernetes/kops.git
Merge pull request #3195 from KashifSaadat/hooks-in-bootstrap-script
Automatic merge from submit-queue Add hooks to bootstrapscript output Ping @gambol99 Related to: - https://github.com/kubernetes/kops/pull/3120 - https://github.com/kubernetes/kops/pull/3063 Cluster Hooks are now added in their entirety to the nodeup user data scripts, dependant on whether the instance group roles match. This could increase the size of the user-data files significantly (getting closer to the 16KB mark) depending on users' hooks content. I considered hashing and fingerprinting the cluster hooks, but following discussions in https://github.com/kubernetes/kops/pull/3120 thought we should keep the full output instead.
This commit is contained in:
Kubernetes Submit Queue
GitHub
commit
32fda48285
|
|
@ -105,6 +105,11 @@ func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup, cs *kops.Cluste
|
|||
spec["masterKubelet"] = cs.MasterKubelet
|
||||
}
|
||||
|
||||
hooks := b.getRelevantHooks(cs.Hooks, ig.Spec.Role)
|
||||
if len(hooks) > 0 {
|
||||
spec["hooks"] = hooks
|
||||
}
|
||||
|
||||
content, err := yaml.Marshal(spec)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error converting cluster spec to yaml for inclusion within bootstrap script: %v", err)
|
||||
|
|
@ -117,6 +122,10 @@ func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup, cs *kops.Cluste
|
|||
spec["kubelet"] = ig.Spec.Kubelet
|
||||
spec["nodeLabels"] = ig.Spec.NodeLabels
|
||||
spec["taints"] = ig.Spec.Taints
|
||||
hooks := b.getRelevantHooks(ig.Spec.Hooks, ig.Spec.Role)
|
||||
if len(hooks) > 0 {
|
||||
spec["hooks"] = hooks
|
||||
}
|
||||
|
||||
content, err := yaml.Marshal(spec)
|
||||
if err != nil {
|
||||
|
|
@ -133,6 +142,24 @@ func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup, cs *kops.Cluste
|
|||
return fi.WrapResource(templateResource), nil
|
||||
}
|
||||
|
||||
// getRelevantHooks returns a list of hooks to be applied to the instance group
|
||||
func (b *BootstrapScript) getRelevantHooks(hooks []kops.HookSpec, role kops.InstanceGroupRole) []kops.HookSpec {
|
||||
relevantHooks := []kops.HookSpec{}
|
||||
for _, hook := range hooks {
|
||||
if len(hook.Roles) == 0 {
|
||||
relevantHooks = append(relevantHooks, hook)
|
||||
continue
|
||||
}
|
||||
for _, hookRole := range hook.Roles {
|
||||
if role == hookRole {
|
||||
relevantHooks = append(relevantHooks, hook)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return relevantHooks
|
||||
}
|
||||
|
||||
func (b *BootstrapScript) createProxyEnv(ps *kops.EgressProxySpec) string {
|
||||
var buffer bytes.Buffer
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import (
|
|||
|
||||
"k8s.io/kops/pkg/apis/kops"
|
||||
"k8s.io/kops/pkg/apis/nodeup"
|
||||
"k8s.io/kops/pkg/diff"
|
||||
)
|
||||
|
||||
func Test_ProxyFunc(t *testing.T) {
|
||||
|
|
@ -57,20 +58,53 @@ func TestBootstrapUserData(t *testing.T) {
|
|||
cs := []struct {
|
||||
Role kops.InstanceGroupRole
|
||||
ExpectedFilePath string
|
||||
HookSpecRoles []kops.InstanceGroupRole
|
||||
}{
|
||||
{
|
||||
Role: "Master",
|
||||
ExpectedFilePath: "tests/data/bootstrapscript_0.txt",
|
||||
HookSpecRoles: []kops.InstanceGroupRole{""},
|
||||
},
|
||||
{
|
||||
Role: "Master",
|
||||
ExpectedFilePath: "tests/data/bootstrapscript_0.txt",
|
||||
HookSpecRoles: []kops.InstanceGroupRole{"Node"},
|
||||
},
|
||||
{
|
||||
Role: "Master",
|
||||
ExpectedFilePath: "tests/data/bootstrapscript_1.txt",
|
||||
HookSpecRoles: []kops.InstanceGroupRole{"Master"},
|
||||
},
|
||||
{
|
||||
Role: "Master",
|
||||
ExpectedFilePath: "tests/data/bootstrapscript_2.txt",
|
||||
HookSpecRoles: []kops.InstanceGroupRole{"Master", "Node"},
|
||||
},
|
||||
{
|
||||
Role: "Node",
|
||||
ExpectedFilePath: "tests/data/bootstrapscript_1.txt",
|
||||
ExpectedFilePath: "tests/data/bootstrapscript_3.txt",
|
||||
HookSpecRoles: []kops.InstanceGroupRole{""},
|
||||
},
|
||||
{
|
||||
Role: "Node",
|
||||
ExpectedFilePath: "tests/data/bootstrapscript_4.txt",
|
||||
HookSpecRoles: []kops.InstanceGroupRole{"Node"},
|
||||
},
|
||||
{
|
||||
Role: "Node",
|
||||
ExpectedFilePath: "tests/data/bootstrapscript_3.txt",
|
||||
HookSpecRoles: []kops.InstanceGroupRole{"Master"},
|
||||
},
|
||||
{
|
||||
Role: "Node",
|
||||
ExpectedFilePath: "tests/data/bootstrapscript_5.txt",
|
||||
HookSpecRoles: []kops.InstanceGroupRole{"Master", "Node"},
|
||||
},
|
||||
}
|
||||
|
||||
for i, x := range cs {
|
||||
spec := makeTestCluster().Spec
|
||||
group := makeTestInstanceGroup(x.Role)
|
||||
spec := makeTestCluster(x.HookSpecRoles).Spec
|
||||
group := makeTestInstanceGroup(x.Role, x.HookSpecRoles)
|
||||
|
||||
renderNodeUpConfig := func(ig *kops.InstanceGroup) (*nodeup.Config, error) {
|
||||
return &nodeup.Config{}, nil
|
||||
|
|
@ -100,12 +134,14 @@ func TestBootstrapUserData(t *testing.T) {
|
|||
}
|
||||
|
||||
if actual != string(expectedBytes) {
|
||||
t.Errorf("case %d, expected: %s. got: %s", i, string(expectedBytes), actual)
|
||||
diffString := diff.FormatDiff(string(expectedBytes), actual)
|
||||
t.Errorf("case %d failed, actual output differed from expected.", i)
|
||||
t.Logf("diff:\n%s\n", diffString)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func makeTestCluster() *kops.Cluster {
|
||||
func makeTestCluster(hookSpecRoles []kops.InstanceGroupRole) *kops.Cluster {
|
||||
return &kops.Cluster{
|
||||
Spec: kops.ClusterSpec{
|
||||
CloudProvider: "aws",
|
||||
|
|
@ -159,11 +195,24 @@ func makeTestCluster() *kops.Cluster {
|
|||
Port: 80,
|
||||
},
|
||||
},
|
||||
Hooks: []kops.HookSpec{
|
||||
{
|
||||
ExecContainer: &kops.ExecContainerAction{
|
||||
Command: []string{
|
||||
"sh",
|
||||
"-c",
|
||||
"chroot /rootfs apt-get update && chroot /rootfs apt-get install -y ceph-common",
|
||||
},
|
||||
Image: "busybox",
|
||||
},
|
||||
Roles: hookSpecRoles,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func makeTestInstanceGroup(role kops.InstanceGroupRole) *kops.InstanceGroup {
|
||||
func makeTestInstanceGroup(role kops.InstanceGroupRole, hookSpecRoles []kops.InstanceGroupRole) *kops.InstanceGroup {
|
||||
return &kops.InstanceGroup{
|
||||
Spec: kops.InstanceGroupSpec{
|
||||
Kubelet: &kops.KubeletConfigSpec{
|
||||
|
|
@ -178,6 +227,20 @@ func makeTestInstanceGroup(role kops.InstanceGroupRole) *kops.InstanceGroup {
|
|||
"key1=value1:NoSchedule",
|
||||
"key2=value2:NoExecute",
|
||||
},
|
||||
Hooks: []kops.HookSpec{
|
||||
{
|
||||
Name: "disable-update-engine.service",
|
||||
Before: []string{
|
||||
"update-engine.service",
|
||||
"kubelet.service",
|
||||
},
|
||||
Manifest: "Type=oneshot\nExecStart=/usr/bin/systemctl stop update-engine.service",
|
||||
Roles: hookSpecRoles,
|
||||
}, {
|
||||
Name: "apply-to-all.service",
|
||||
Manifest: "Type=oneshot\nExecStart=/usr/bin/systemctl start apply-to-all.service",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -167,6 +167,11 @@ masterKubelet:
|
|||
__EOF_CLUSTER_SPEC
|
||||
|
||||
cat > ig_spec.yaml << __EOF_IG_SPEC
|
||||
hooks:
|
||||
- manifest: |-
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/systemctl start apply-to-all.service
|
||||
name: apply-to-all.service
|
||||
kubelet:
|
||||
kubeconfigPath: /etc/kubernetes/igconfig.txt
|
||||
nodeLabels:
|
||||
|
|
|
|||
|
|
@ -149,16 +149,47 @@ cloudConfig:
|
|||
nodeTags: something
|
||||
docker:
|
||||
logLevel: INFO
|
||||
hooks:
|
||||
- execContainer:
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- chroot /rootfs apt-get update && chroot /rootfs apt-get install -y ceph-common
|
||||
image: busybox
|
||||
roles:
|
||||
- Master
|
||||
kubeAPIServer:
|
||||
image: CoreOS
|
||||
kubeControllerManager:
|
||||
cloudProvider: aws
|
||||
kubeProxy:
|
||||
cpuRequest: 30m
|
||||
featureGates:
|
||||
AdvancedAuditing: "true"
|
||||
kubeScheduler:
|
||||
image: SomeImage
|
||||
kubelet:
|
||||
kubeconfigPath: /etc/kubernetes/config.txt
|
||||
masterKubelet:
|
||||
kubeconfigPath: /etc/kubernetes/config.cfg
|
||||
|
||||
__EOF_CLUSTER_SPEC
|
||||
|
||||
cat > ig_spec.yaml << __EOF_IG_SPEC
|
||||
hooks:
|
||||
- before:
|
||||
- update-engine.service
|
||||
- kubelet.service
|
||||
manifest: |-
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/systemctl stop update-engine.service
|
||||
name: disable-update-engine.service
|
||||
roles:
|
||||
- Master
|
||||
- manifest: |-
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/systemctl start apply-to-all.service
|
||||
name: apply-to-all.service
|
||||
kubelet:
|
||||
kubeconfigPath: /etc/kubernetes/igconfig.txt
|
||||
nodeLabels:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,212 @@
|
|||
#!/bin/bash
|
||||
# Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
NODEUP_URL=NUSource
|
||||
NODEUP_HASH=NUSHash
|
||||
|
||||
|
||||
|
||||
|
||||
export http_proxy=http://example.com:80
|
||||
export https_proxy=${http_proxy}
|
||||
export no_proxy=
|
||||
export NO_PROXY=${no_proxy}
|
||||
echo "export http_proxy=${http_proxy}" >> /etc/default/docker
|
||||
echo "export https_proxy=${http_proxy}" >> /etc/default/docker
|
||||
echo "export no_proxy=${no_proxy}" >> /etc/default/docker
|
||||
echo "export NO_PROXY=${no_proxy}" >> /etc/default/docker
|
||||
echo "export http_proxy=${http_proxy}" >> /etc/environment
|
||||
echo "export https_proxy=${http_proxy}" >> /etc/environment
|
||||
echo "export no_proxy=${no_proxy}" >> /etc/environment
|
||||
echo "export NO_PROXY=${no_proxy}" >> /etc/environment
|
||||
echo DefaultEnvironment=\"http_proxy=${http_proxy}\" \"https_proxy=${http_proxy}\"echo DefaultEnvironment=\"http_proxy=${http_proxy}\" \"https_proxy=${http_proxy}\" \"NO_PROXY=${no_proxy}\" \"no_proxy=${no_proxy}\" >> /etc/systemd/system.conf
|
||||
source /etc/environment
|
||||
systemctl daemon-reload
|
||||
systemctl daemon-reexec
|
||||
if [ -f /etc/lsb-release ] || [ -f /etc/debian_version ]; then
|
||||
echo "Acquire::http::Proxy \"${http_proxy}\";" > /etc/apt/apt.conf.d/30proxy
|
||||
elif [ -f /etc/redhat-release ]; then
|
||||
echo "http_proxy=${http_proxy}" >> /etc/yum.conf
|
||||
fi
|
||||
|
||||
|
||||
function ensure-install-dir() {
|
||||
INSTALL_DIR="/var/cache/kubernetes-install"
|
||||
# On ContainerOS, we install to /var/lib/toolbox install (because of noexec)
|
||||
if [[ -d /var/lib/toolbox ]]; then
|
||||
INSTALL_DIR="/var/lib/toolbox/kubernetes-install"
|
||||
fi
|
||||
mkdir -p ${INSTALL_DIR}
|
||||
cd ${INSTALL_DIR}
|
||||
}
|
||||
|
||||
# Retry a download until we get it. Takes a hash and a set of URLs.
|
||||
#
|
||||
# $1 is the sha1 of the URL. Can be "" if the sha1 is unknown.
|
||||
# $2+ are the URLs to download.
|
||||
download-or-bust() {
|
||||
local -r hash="$1"
|
||||
shift 1
|
||||
|
||||
urls=( $* )
|
||||
while true; do
|
||||
for url in "${urls[@]}"; do
|
||||
local file="${url##*/}"
|
||||
rm -f "${file}"
|
||||
if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; then
|
||||
echo "== Failed to download ${url}. Retrying. =="
|
||||
elif [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
|
||||
echo "== Hash validation of ${url} failed. Retrying. =="
|
||||
else
|
||||
if [[ -n "${hash}" ]]; then
|
||||
echo "== Downloaded ${url} (SHA1 = ${hash}) =="
|
||||
else
|
||||
echo "== Downloaded ${url} =="
|
||||
fi
|
||||
return
|
||||
fi
|
||||
done
|
||||
|
||||
echo "All downloads failed; sleeping before retrying"
|
||||
sleep 60
|
||||
done
|
||||
}
|
||||
|
||||
validate-hash() {
|
||||
local -r file="$1"
|
||||
local -r expected="$2"
|
||||
local actual
|
||||
|
||||
actual=$(sha1sum ${file} | awk '{ print $1 }') || true
|
||||
if [[ "${actual}" != "${expected}" ]]; then
|
||||
echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} =="
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
function split-commas() {
|
||||
echo $1 | tr "," "\n"
|
||||
}
|
||||
|
||||
function try-download-release() {
|
||||
# TODO(zmerlynn): Now we REALLY have no excuse not to do the reboot
|
||||
# optimization.
|
||||
|
||||
local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
|
||||
local -r nodeup_filename="${nodeup_urls[0]##*/}"
|
||||
if [[ -n "${NODEUP_HASH:-}" ]]; then
|
||||
local -r nodeup_hash="${NODEUP_HASH}"
|
||||
else
|
||||
# TODO: Remove?
|
||||
echo "Downloading sha1 (not found in env)"
|
||||
download-or-bust "" "${nodeup_urls[@]/%/.sha1}"
|
||||
local -r nodeup_hash=$(cat "${nodeup_filename}.sha1")
|
||||
fi
|
||||
|
||||
echo "Downloading nodeup (${nodeup_urls[@]})"
|
||||
download-or-bust "${nodeup_hash}" "${nodeup_urls[@]}"
|
||||
|
||||
chmod +x nodeup
|
||||
}
|
||||
|
||||
function download-release() {
|
||||
# In case of failure checking integrity of release, retry.
|
||||
until try-download-release; do
|
||||
sleep 15
|
||||
echo "Couldn't download release. Retrying..."
|
||||
done
|
||||
|
||||
echo "Running nodeup"
|
||||
# We can't run in the foreground because of https://github.com/docker/docker/issues/23793
|
||||
( cd ${INSTALL_DIR}; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/kube_env.yaml --v=8 )
|
||||
}
|
||||
|
||||
####################################################################################
|
||||
|
||||
/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
|
||||
|
||||
echo "== nodeup node config starting =="
|
||||
ensure-install-dir
|
||||
|
||||
cat > cluster_spec.yaml << __EOF_CLUSTER_SPEC
|
||||
cloudConfig:
|
||||
nodeTags: something
|
||||
docker:
|
||||
logLevel: INFO
|
||||
hooks:
|
||||
- execContainer:
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- chroot /rootfs apt-get update && chroot /rootfs apt-get install -y ceph-common
|
||||
image: busybox
|
||||
roles:
|
||||
- Master
|
||||
- Node
|
||||
kubeAPIServer:
|
||||
image: CoreOS
|
||||
kubeControllerManager:
|
||||
cloudProvider: aws
|
||||
kubeProxy:
|
||||
cpuRequest: 30m
|
||||
featureGates:
|
||||
AdvancedAuditing: "true"
|
||||
kubeScheduler:
|
||||
image: SomeImage
|
||||
kubelet:
|
||||
kubeconfigPath: /etc/kubernetes/config.txt
|
||||
masterKubelet:
|
||||
kubeconfigPath: /etc/kubernetes/config.cfg
|
||||
|
||||
__EOF_CLUSTER_SPEC
|
||||
|
||||
cat > ig_spec.yaml << __EOF_IG_SPEC
|
||||
hooks:
|
||||
- before:
|
||||
- update-engine.service
|
||||
- kubelet.service
|
||||
manifest: |-
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/systemctl stop update-engine.service
|
||||
name: disable-update-engine.service
|
||||
roles:
|
||||
- Master
|
||||
- Node
|
||||
- manifest: |-
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/systemctl start apply-to-all.service
|
||||
name: apply-to-all.service
|
||||
kubelet:
|
||||
kubeconfigPath: /etc/kubernetes/igconfig.txt
|
||||
nodeLabels:
|
||||
label2: value2
|
||||
labelname: labelvalue
|
||||
taints:
|
||||
- key1=value1:NoSchedule
|
||||
- key2=value2:NoExecute
|
||||
|
||||
__EOF_IG_SPEC
|
||||
|
||||
cat > kube_env.yaml << __EOF_KUBE_ENV
|
||||
{}
|
||||
|
||||
__EOF_KUBE_ENV
|
||||
|
||||
download-release
|
||||
echo "== nodeup node config done =="
|
||||
|
|
@ -0,0 +1,184 @@
|
|||
#!/bin/bash
|
||||
# Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
NODEUP_URL=NUSource
|
||||
NODEUP_HASH=NUSHash
|
||||
|
||||
|
||||
|
||||
|
||||
export http_proxy=http://example.com:80
|
||||
export https_proxy=${http_proxy}
|
||||
export no_proxy=
|
||||
export NO_PROXY=${no_proxy}
|
||||
echo "export http_proxy=${http_proxy}" >> /etc/default/docker
|
||||
echo "export https_proxy=${http_proxy}" >> /etc/default/docker
|
||||
echo "export no_proxy=${no_proxy}" >> /etc/default/docker
|
||||
echo "export NO_PROXY=${no_proxy}" >> /etc/default/docker
|
||||
echo "export http_proxy=${http_proxy}" >> /etc/environment
|
||||
echo "export https_proxy=${http_proxy}" >> /etc/environment
|
||||
echo "export no_proxy=${no_proxy}" >> /etc/environment
|
||||
echo "export NO_PROXY=${no_proxy}" >> /etc/environment
|
||||
echo DefaultEnvironment=\"http_proxy=${http_proxy}\" \"https_proxy=${http_proxy}\"echo DefaultEnvironment=\"http_proxy=${http_proxy}\" \"https_proxy=${http_proxy}\" \"NO_PROXY=${no_proxy}\" \"no_proxy=${no_proxy}\" >> /etc/systemd/system.conf
|
||||
source /etc/environment
|
||||
systemctl daemon-reload
|
||||
systemctl daemon-reexec
|
||||
if [ -f /etc/lsb-release ] || [ -f /etc/debian_version ]; then
|
||||
echo "Acquire::http::Proxy \"${http_proxy}\";" > /etc/apt/apt.conf.d/30proxy
|
||||
elif [ -f /etc/redhat-release ]; then
|
||||
echo "http_proxy=${http_proxy}" >> /etc/yum.conf
|
||||
fi
|
||||
|
||||
|
||||
function ensure-install-dir() {
|
||||
INSTALL_DIR="/var/cache/kubernetes-install"
|
||||
# On ContainerOS, we install to /var/lib/toolbox install (because of noexec)
|
||||
if [[ -d /var/lib/toolbox ]]; then
|
||||
INSTALL_DIR="/var/lib/toolbox/kubernetes-install"
|
||||
fi
|
||||
mkdir -p ${INSTALL_DIR}
|
||||
cd ${INSTALL_DIR}
|
||||
}
|
||||
|
||||
# Retry a download until we get it. Takes a hash and a set of URLs.
|
||||
#
|
||||
# $1 is the sha1 of the URL. Can be "" if the sha1 is unknown.
|
||||
# $2+ are the URLs to download.
|
||||
download-or-bust() {
|
||||
local -r hash="$1"
|
||||
shift 1
|
||||
|
||||
urls=( $* )
|
||||
while true; do
|
||||
for url in "${urls[@]}"; do
|
||||
local file="${url##*/}"
|
||||
rm -f "${file}"
|
||||
if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; then
|
||||
echo "== Failed to download ${url}. Retrying. =="
|
||||
elif [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
|
||||
echo "== Hash validation of ${url} failed. Retrying. =="
|
||||
else
|
||||
if [[ -n "${hash}" ]]; then
|
||||
echo "== Downloaded ${url} (SHA1 = ${hash}) =="
|
||||
else
|
||||
echo "== Downloaded ${url} =="
|
||||
fi
|
||||
return
|
||||
fi
|
||||
done
|
||||
|
||||
echo "All downloads failed; sleeping before retrying"
|
||||
sleep 60
|
||||
done
|
||||
}
|
||||
|
||||
validate-hash() {
|
||||
local -r file="$1"
|
||||
local -r expected="$2"
|
||||
local actual
|
||||
|
||||
actual=$(sha1sum ${file} | awk '{ print $1 }') || true
|
||||
if [[ "${actual}" != "${expected}" ]]; then
|
||||
echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} =="
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
function split-commas() {
|
||||
echo $1 | tr "," "\n"
|
||||
}
|
||||
|
||||
function try-download-release() {
|
||||
# TODO(zmerlynn): Now we REALLY have no excuse not to do the reboot
|
||||
# optimization.
|
||||
|
||||
local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
|
||||
local -r nodeup_filename="${nodeup_urls[0]##*/}"
|
||||
if [[ -n "${NODEUP_HASH:-}" ]]; then
|
||||
local -r nodeup_hash="${NODEUP_HASH}"
|
||||
else
|
||||
# TODO: Remove?
|
||||
echo "Downloading sha1 (not found in env)"
|
||||
download-or-bust "" "${nodeup_urls[@]/%/.sha1}"
|
||||
local -r nodeup_hash=$(cat "${nodeup_filename}.sha1")
|
||||
fi
|
||||
|
||||
echo "Downloading nodeup (${nodeup_urls[@]})"
|
||||
download-or-bust "${nodeup_hash}" "${nodeup_urls[@]}"
|
||||
|
||||
chmod +x nodeup
|
||||
}
|
||||
|
||||
function download-release() {
|
||||
# In case of failure checking integrity of release, retry.
|
||||
until try-download-release; do
|
||||
sleep 15
|
||||
echo "Couldn't download release. Retrying..."
|
||||
done
|
||||
|
||||
echo "Running nodeup"
|
||||
# We can't run in the foreground because of https://github.com/docker/docker/issues/23793
|
||||
( cd ${INSTALL_DIR}; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/kube_env.yaml --v=8 )
|
||||
}
|
||||
|
||||
####################################################################################
|
||||
|
||||
/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
|
||||
|
||||
echo "== nodeup node config starting =="
|
||||
ensure-install-dir
|
||||
|
||||
cat > cluster_spec.yaml << __EOF_CLUSTER_SPEC
|
||||
cloudConfig:
|
||||
nodeTags: something
|
||||
docker:
|
||||
logLevel: INFO
|
||||
kubeProxy:
|
||||
cpuRequest: 30m
|
||||
featureGates:
|
||||
AdvancedAuditing: "true"
|
||||
kubelet:
|
||||
kubeconfigPath: /etc/kubernetes/config.txt
|
||||
|
||||
__EOF_CLUSTER_SPEC
|
||||
|
||||
cat > ig_spec.yaml << __EOF_IG_SPEC
|
||||
hooks:
|
||||
- manifest: |-
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/systemctl start apply-to-all.service
|
||||
name: apply-to-all.service
|
||||
kubelet:
|
||||
kubeconfigPath: /etc/kubernetes/igconfig.txt
|
||||
nodeLabels:
|
||||
label2: value2
|
||||
labelname: labelvalue
|
||||
taints:
|
||||
- key1=value1:NoSchedule
|
||||
- key2=value2:NoExecute
|
||||
|
||||
__EOF_IG_SPEC
|
||||
|
||||
cat > kube_env.yaml << __EOF_KUBE_ENV
|
||||
{}
|
||||
|
||||
__EOF_KUBE_ENV
|
||||
|
||||
download-release
|
||||
echo "== nodeup node config done =="
|
||||
|
|
@ -0,0 +1,202 @@
|
|||
#!/bin/bash
|
||||
# Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
NODEUP_URL=NUSource
|
||||
NODEUP_HASH=NUSHash
|
||||
|
||||
|
||||
|
||||
|
||||
export http_proxy=http://example.com:80
|
||||
export https_proxy=${http_proxy}
|
||||
export no_proxy=
|
||||
export NO_PROXY=${no_proxy}
|
||||
echo "export http_proxy=${http_proxy}" >> /etc/default/docker
|
||||
echo "export https_proxy=${http_proxy}" >> /etc/default/docker
|
||||
echo "export no_proxy=${no_proxy}" >> /etc/default/docker
|
||||
echo "export NO_PROXY=${no_proxy}" >> /etc/default/docker
|
||||
echo "export http_proxy=${http_proxy}" >> /etc/environment
|
||||
echo "export https_proxy=${http_proxy}" >> /etc/environment
|
||||
echo "export no_proxy=${no_proxy}" >> /etc/environment
|
||||
echo "export NO_PROXY=${no_proxy}" >> /etc/environment
|
||||
echo DefaultEnvironment=\"http_proxy=${http_proxy}\" \"https_proxy=${http_proxy}\"echo DefaultEnvironment=\"http_proxy=${http_proxy}\" \"https_proxy=${http_proxy}\" \"NO_PROXY=${no_proxy}\" \"no_proxy=${no_proxy}\" >> /etc/systemd/system.conf
|
||||
source /etc/environment
|
||||
systemctl daemon-reload
|
||||
systemctl daemon-reexec
|
||||
if [ -f /etc/lsb-release ] || [ -f /etc/debian_version ]; then
|
||||
echo "Acquire::http::Proxy \"${http_proxy}\";" > /etc/apt/apt.conf.d/30proxy
|
||||
elif [ -f /etc/redhat-release ]; then
|
||||
echo "http_proxy=${http_proxy}" >> /etc/yum.conf
|
||||
fi
|
||||
|
||||
|
||||
function ensure-install-dir() {
|
||||
INSTALL_DIR="/var/cache/kubernetes-install"
|
||||
# On ContainerOS, we install to /var/lib/toolbox install (because of noexec)
|
||||
if [[ -d /var/lib/toolbox ]]; then
|
||||
INSTALL_DIR="/var/lib/toolbox/kubernetes-install"
|
||||
fi
|
||||
mkdir -p ${INSTALL_DIR}
|
||||
cd ${INSTALL_DIR}
|
||||
}
|
||||
|
||||
# Retry a download until we get it. Takes a hash and a set of URLs.
|
||||
#
|
||||
# $1 is the sha1 of the URL. Can be "" if the sha1 is unknown.
|
||||
# $2+ are the URLs to download.
|
||||
download-or-bust() {
|
||||
local -r hash="$1"
|
||||
shift 1
|
||||
|
||||
urls=( $* )
|
||||
while true; do
|
||||
for url in "${urls[@]}"; do
|
||||
local file="${url##*/}"
|
||||
rm -f "${file}"
|
||||
if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; then
|
||||
echo "== Failed to download ${url}. Retrying. =="
|
||||
elif [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
|
||||
echo "== Hash validation of ${url} failed. Retrying. =="
|
||||
else
|
||||
if [[ -n "${hash}" ]]; then
|
||||
echo "== Downloaded ${url} (SHA1 = ${hash}) =="
|
||||
else
|
||||
echo "== Downloaded ${url} =="
|
||||
fi
|
||||
return
|
||||
fi
|
||||
done
|
||||
|
||||
echo "All downloads failed; sleeping before retrying"
|
||||
sleep 60
|
||||
done
|
||||
}
|
||||
|
||||
validate-hash() {
|
||||
local -r file="$1"
|
||||
local -r expected="$2"
|
||||
local actual
|
||||
|
||||
actual=$(sha1sum ${file} | awk '{ print $1 }') || true
|
||||
if [[ "${actual}" != "${expected}" ]]; then
|
||||
echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} =="
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
function split-commas() {
|
||||
echo $1 | tr "," "\n"
|
||||
}
|
||||
|
||||
function try-download-release() {
|
||||
# TODO(zmerlynn): Now we REALLY have no excuse not to do the reboot
|
||||
# optimization.
|
||||
|
||||
local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
|
||||
local -r nodeup_filename="${nodeup_urls[0]##*/}"
|
||||
if [[ -n "${NODEUP_HASH:-}" ]]; then
|
||||
local -r nodeup_hash="${NODEUP_HASH}"
|
||||
else
|
||||
# TODO: Remove?
|
||||
echo "Downloading sha1 (not found in env)"
|
||||
download-or-bust "" "${nodeup_urls[@]/%/.sha1}"
|
||||
local -r nodeup_hash=$(cat "${nodeup_filename}.sha1")
|
||||
fi
|
||||
|
||||
echo "Downloading nodeup (${nodeup_urls[@]})"
|
||||
download-or-bust "${nodeup_hash}" "${nodeup_urls[@]}"
|
||||
|
||||
chmod +x nodeup
|
||||
}
|
||||
|
||||
function download-release() {
|
||||
# In case of failure checking integrity of release, retry.
|
||||
until try-download-release; do
|
||||
sleep 15
|
||||
echo "Couldn't download release. Retrying..."
|
||||
done
|
||||
|
||||
echo "Running nodeup"
|
||||
# We can't run in the foreground because of https://github.com/docker/docker/issues/23793
|
||||
( cd ${INSTALL_DIR}; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/kube_env.yaml --v=8 )
|
||||
}
|
||||
|
||||
####################################################################################
|
||||
|
||||
/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
|
||||
|
||||
echo "== nodeup node config starting =="
|
||||
ensure-install-dir
|
||||
|
||||
cat > cluster_spec.yaml << __EOF_CLUSTER_SPEC
|
||||
cloudConfig:
|
||||
nodeTags: something
|
||||
docker:
|
||||
logLevel: INFO
|
||||
hooks:
|
||||
- execContainer:
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- chroot /rootfs apt-get update && chroot /rootfs apt-get install -y ceph-common
|
||||
image: busybox
|
||||
roles:
|
||||
- Node
|
||||
kubeProxy:
|
||||
cpuRequest: 30m
|
||||
featureGates:
|
||||
AdvancedAuditing: "true"
|
||||
kubelet:
|
||||
kubeconfigPath: /etc/kubernetes/config.txt
|
||||
|
||||
__EOF_CLUSTER_SPEC
|
||||
|
||||
cat > ig_spec.yaml << __EOF_IG_SPEC
|
||||
hooks:
|
||||
- before:
|
||||
- update-engine.service
|
||||
- kubelet.service
|
||||
manifest: |-
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/systemctl stop update-engine.service
|
||||
name: disable-update-engine.service
|
||||
roles:
|
||||
- Node
|
||||
- manifest: |-
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/systemctl start apply-to-all.service
|
||||
name: apply-to-all.service
|
||||
kubelet:
|
||||
kubeconfigPath: /etc/kubernetes/igconfig.txt
|
||||
nodeLabels:
|
||||
label2: value2
|
||||
labelname: labelvalue
|
||||
taints:
|
||||
- key1=value1:NoSchedule
|
||||
- key2=value2:NoExecute
|
||||
|
||||
__EOF_IG_SPEC
|
||||
|
||||
cat > kube_env.yaml << __EOF_KUBE_ENV
|
||||
{}
|
||||
|
||||
__EOF_KUBE_ENV
|
||||
|
||||
download-release
|
||||
echo "== nodeup node config done =="
|
||||
|
|
@ -0,0 +1,204 @@
|
|||
#!/bin/bash
|
||||
# Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
NODEUP_URL=NUSource
|
||||
NODEUP_HASH=NUSHash
|
||||
|
||||
|
||||
|
||||
|
||||
export http_proxy=http://example.com:80
|
||||
export https_proxy=${http_proxy}
|
||||
export no_proxy=
|
||||
export NO_PROXY=${no_proxy}
|
||||
echo "export http_proxy=${http_proxy}" >> /etc/default/docker
|
||||
echo "export https_proxy=${http_proxy}" >> /etc/default/docker
|
||||
echo "export no_proxy=${no_proxy}" >> /etc/default/docker
|
||||
echo "export NO_PROXY=${no_proxy}" >> /etc/default/docker
|
||||
echo "export http_proxy=${http_proxy}" >> /etc/environment
|
||||
echo "export https_proxy=${http_proxy}" >> /etc/environment
|
||||
echo "export no_proxy=${no_proxy}" >> /etc/environment
|
||||
echo "export NO_PROXY=${no_proxy}" >> /etc/environment
|
||||
echo DefaultEnvironment=\"http_proxy=${http_proxy}\" \"https_proxy=${http_proxy}\"echo DefaultEnvironment=\"http_proxy=${http_proxy}\" \"https_proxy=${http_proxy}\" \"NO_PROXY=${no_proxy}\" \"no_proxy=${no_proxy}\" >> /etc/systemd/system.conf
|
||||
source /etc/environment
|
||||
systemctl daemon-reload
|
||||
systemctl daemon-reexec
|
||||
if [ -f /etc/lsb-release ] || [ -f /etc/debian_version ]; then
|
||||
echo "Acquire::http::Proxy \"${http_proxy}\";" > /etc/apt/apt.conf.d/30proxy
|
||||
elif [ -f /etc/redhat-release ]; then
|
||||
echo "http_proxy=${http_proxy}" >> /etc/yum.conf
|
||||
fi
|
||||
|
||||
|
||||
function ensure-install-dir() {
|
||||
INSTALL_DIR="/var/cache/kubernetes-install"
|
||||
# On ContainerOS, we install to /var/lib/toolbox install (because of noexec)
|
||||
if [[ -d /var/lib/toolbox ]]; then
|
||||
INSTALL_DIR="/var/lib/toolbox/kubernetes-install"
|
||||
fi
|
||||
mkdir -p ${INSTALL_DIR}
|
||||
cd ${INSTALL_DIR}
|
||||
}
|
||||
|
||||
# Retry a download until we get it. Takes a hash and a set of URLs.
|
||||
#
|
||||
# $1 is the sha1 of the URL. Can be "" if the sha1 is unknown.
|
||||
# $2+ are the URLs to download.
|
||||
download-or-bust() {
|
||||
local -r hash="$1"
|
||||
shift 1
|
||||
|
||||
urls=( $* )
|
||||
while true; do
|
||||
for url in "${urls[@]}"; do
|
||||
local file="${url##*/}"
|
||||
rm -f "${file}"
|
||||
if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; then
|
||||
echo "== Failed to download ${url}. Retrying. =="
|
||||
elif [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
|
||||
echo "== Hash validation of ${url} failed. Retrying. =="
|
||||
else
|
||||
if [[ -n "${hash}" ]]; then
|
||||
echo "== Downloaded ${url} (SHA1 = ${hash}) =="
|
||||
else
|
||||
echo "== Downloaded ${url} =="
|
||||
fi
|
||||
return
|
||||
fi
|
||||
done
|
||||
|
||||
echo "All downloads failed; sleeping before retrying"
|
||||
sleep 60
|
||||
done
|
||||
}
|
||||
|
||||
validate-hash() {
|
||||
local -r file="$1"
|
||||
local -r expected="$2"
|
||||
local actual
|
||||
|
||||
actual=$(sha1sum ${file} | awk '{ print $1 }') || true
|
||||
if [[ "${actual}" != "${expected}" ]]; then
|
||||
echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} =="
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
function split-commas() {
|
||||
echo $1 | tr "," "\n"
|
||||
}
|
||||
|
||||
function try-download-release() {
|
||||
# TODO(zmerlynn): Now we REALLY have no excuse not to do the reboot
|
||||
# optimization.
|
||||
|
||||
local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
|
||||
local -r nodeup_filename="${nodeup_urls[0]##*/}"
|
||||
if [[ -n "${NODEUP_HASH:-}" ]]; then
|
||||
local -r nodeup_hash="${NODEUP_HASH}"
|
||||
else
|
||||
# TODO: Remove?
|
||||
echo "Downloading sha1 (not found in env)"
|
||||
download-or-bust "" "${nodeup_urls[@]/%/.sha1}"
|
||||
local -r nodeup_hash=$(cat "${nodeup_filename}.sha1")
|
||||
fi
|
||||
|
||||
echo "Downloading nodeup (${nodeup_urls[@]})"
|
||||
download-or-bust "${nodeup_hash}" "${nodeup_urls[@]}"
|
||||
|
||||
chmod +x nodeup
|
||||
}
|
||||
|
||||
function download-release() {
|
||||
# In case of failure checking integrity of release, retry.
|
||||
until try-download-release; do
|
||||
sleep 15
|
||||
echo "Couldn't download release. Retrying..."
|
||||
done
|
||||
|
||||
echo "Running nodeup"
|
||||
# We can't run in the foreground because of https://github.com/docker/docker/issues/23793
|
||||
( cd ${INSTALL_DIR}; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/kube_env.yaml --v=8 )
|
||||
}
|
||||
|
||||
####################################################################################
|
||||
|
||||
/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
|
||||
|
||||
echo "== nodeup node config starting =="
|
||||
ensure-install-dir
|
||||
|
||||
cat > cluster_spec.yaml << __EOF_CLUSTER_SPEC
|
||||
cloudConfig:
|
||||
nodeTags: something
|
||||
docker:
|
||||
logLevel: INFO
|
||||
hooks:
|
||||
- execContainer:
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- chroot /rootfs apt-get update && chroot /rootfs apt-get install -y ceph-common
|
||||
image: busybox
|
||||
roles:
|
||||
- Master
|
||||
- Node
|
||||
kubeProxy:
|
||||
cpuRequest: 30m
|
||||
featureGates:
|
||||
AdvancedAuditing: "true"
|
||||
kubelet:
|
||||
kubeconfigPath: /etc/kubernetes/config.txt
|
||||
|
||||
__EOF_CLUSTER_SPEC
|
||||
|
||||
cat > ig_spec.yaml << __EOF_IG_SPEC
|
||||
hooks:
|
||||
- before:
|
||||
- update-engine.service
|
||||
- kubelet.service
|
||||
manifest: |-
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/systemctl stop update-engine.service
|
||||
name: disable-update-engine.service
|
||||
roles:
|
||||
- Master
|
||||
- Node
|
||||
- manifest: |-
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/systemctl start apply-to-all.service
|
||||
name: apply-to-all.service
|
||||
kubelet:
|
||||
kubeconfigPath: /etc/kubernetes/igconfig.txt
|
||||
nodeLabels:
|
||||
label2: value2
|
||||
labelname: labelvalue
|
||||
taints:
|
||||
- key1=value1:NoSchedule
|
||||
- key2=value2:NoExecute
|
||||
|
||||
__EOF_IG_SPEC
|
||||
|
||||
cat > kube_env.yaml << __EOF_KUBE_ENV
|
||||
{}
|
||||
|
||||
__EOF_KUBE_ENV
|
||||
|
||||
download-release
|
||||
echo "== nodeup node config done =="
|
||||
Loading…
Reference in New Issue