diff --git a/pkg/model/resources/BUILD.bazel b/pkg/model/resources/BUILD.bazel index a5ad76408e..b9ecb801c2 100644 --- a/pkg/model/resources/BUILD.bazel +++ b/pkg/model/resources/BUILD.bazel @@ -1,4 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", @@ -7,3 +7,9 @@ go_library( visibility = ["//visibility:public"], deps = ["//pkg/apis/kops:go_default_library"], ) + +go_test( + name = "go_default_test", + srcs = ["nodeup_test.go"], + embed = [":go_default_library"], +) diff --git a/pkg/model/resources/nodeup.go b/pkg/model/resources/nodeup.go index 93baa9a830..2b88f3e73b 100644 --- a/pkg/model/resources/nodeup.go +++ b/pkg/model/resources/nodeup.go @@ -60,11 +60,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/pkg/model/resources/nodeup_test.go b/pkg/model/resources/nodeup_test.go new file mode 100644 index 0000000000..2151743be6 --- /dev/null +++ b/pkg/model/resources/nodeup_test.go @@ -0,0 +1,30 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + "strings" + "testing" +) + +func Test_NodeUpTabs(t *testing.T) { + for i, line := range strings.Split(NodeUpTemplate, "\n") { + if strings.Contains(line, "\t") { + t.Errorf("NodeUpTemplate contains unexpected character %q on line %d: %q", "\t", i, line) + } + } +} diff --git a/pkg/model/tests/data/bootstrapscript_0.txt b/pkg/model/tests/data/bootstrapscript_0.txt index d19a0d0556..78fdcfca1a 100644 --- a/pkg/model/tests/data/bootstrapscript_0.txt +++ b/pkg/model/tests/data/bootstrapscript_0.txt @@ -50,11 +50,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/pkg/model/tests/data/bootstrapscript_1.txt b/pkg/model/tests/data/bootstrapscript_1.txt index ba030c58b2..96d66e5426 100644 --- a/pkg/model/tests/data/bootstrapscript_1.txt +++ b/pkg/model/tests/data/bootstrapscript_1.txt @@ -50,11 +50,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/pkg/model/tests/data/bootstrapscript_2.txt b/pkg/model/tests/data/bootstrapscript_2.txt index ba030c58b2..96d66e5426 100644 --- a/pkg/model/tests/data/bootstrapscript_2.txt +++ b/pkg/model/tests/data/bootstrapscript_2.txt @@ -50,11 +50,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/pkg/model/tests/data/bootstrapscript_3.txt b/pkg/model/tests/data/bootstrapscript_3.txt index 559e4738b9..a320a4267d 100644 --- a/pkg/model/tests/data/bootstrapscript_3.txt +++ b/pkg/model/tests/data/bootstrapscript_3.txt @@ -50,11 +50,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/pkg/model/tests/data/bootstrapscript_4.txt b/pkg/model/tests/data/bootstrapscript_4.txt index f070478a61..a16217e4ec 100644 --- a/pkg/model/tests/data/bootstrapscript_4.txt +++ b/pkg/model/tests/data/bootstrapscript_4.txt @@ -50,11 +50,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/pkg/model/tests/data/bootstrapscript_5.txt b/pkg/model/tests/data/bootstrapscript_5.txt index f070478a61..a16217e4ec 100644 --- a/pkg/model/tests/data/bootstrapscript_5.txt +++ b/pkg/model/tests/data/bootstrapscript_5.txt @@ -50,11 +50,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/apiservernodes/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/apiservernodes/cloudformation.json.extracted.yaml index c9bbcbe066..e0346d9415 100644 --- a/tests/integration/update_cluster/apiservernodes/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/apiservernodes/cloudformation.json.extracted.yaml @@ -1,245 +1,757 @@ -Resources.AWSEC2LaunchTemplateapiserverapiserversminimalexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n - \ hostnameOverride: '@aws'\n image: k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: - 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: systemd\n cgroupRoot: /\n - \ cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: cluster.local\n - \ enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n\n__EOF_CLUSTER_SPEC\n\ncat > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat - > conf/kube_env.yaml << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\nClusterName: - minimal.example.com\nConfigBase: memfs://clusters.example.com/minimal.example.com\nInstanceGroupName: - apiserver\nInstanceGroupRole: APIServer\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kubernetes.io/role: api-server\n - \ node-role.kubernetes.io/api-server: \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n - \ podManifestPath: /etc/kubernetes/manifests\nchannels:\n- memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nencryptionConfig: null\netcdClusters:\n events:\n version: - 3.4.13\n main:\n version: 3.4.13\nkubeAPIServer:\n allowPrivileged: true\n - \ anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n apiServerCount: - 1\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n cloudProvider: aws\n - \ enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - ServiceAccount\n - \ - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.21.0\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.minimal.example.com\n - \ serviceAccountJWKSURI: https://api.internal.minimal.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n storageBackend: etcd3\nkubeControllerManager:\n - \ allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: 1m0s\n cloudProvider: - aws\n clusterCIDR: 100.96.0.0/11\n clusterName: minimal.example.com\n configureCloudRoutes: - false\n image: k8s.gcr.io/kube-controller-manager:v1.21.0\n leaderElection:\n - \ leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: true\nkubeProxy:\n - \ clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: '@aws'\n image: - k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: 2\nkubeScheduler:\n image: k8s.gcr.io/kube-scheduler:v1.21.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\nkubelet:\n anonymousAuth: - false\n cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: - 100.64.0.10\n clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\nmasterKubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n registerSchedulable: false\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - minimal.example.com\nConfigBase: memfs://clusters.example.com/minimal.example.com\nInstanceGroupName: - master-us-test-1a\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/api-server: \"\"\n - \ node-role.kubernetes.io/control-plane: \"\"\n node-role.kubernetes.io/master: - \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: \"\"\n nonMasqueradeCIDR: - 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\n registerSchedulable: - false\nchannels:\n- memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml\n- memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatenodesminimalexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n - \ hostnameOverride: '@aws'\n image: k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: - 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: systemd\n cgroupRoot: /\n - \ cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: cluster.local\n - \ enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n\n__EOF_CLUSTER_SPEC\n\ncat > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat - > conf/kube_env.yaml << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\nClusterName: - minimal.example.com\nConfigBase: memfs://clusters.example.com/minimal.example.com\nInstanceGroupName: - nodes\nInstanceGroupRole: Node\nKubeletConfig:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kubernetes.io/role: node\n node-role.kubernetes.io/node: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\nchannels:\n- - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" +Resources.AWSEC2LaunchTemplateapiserverapiserversminimalexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + ClusterName: minimal.example.com + ConfigBase: memfs://clusters.example.com/minimal.example.com + InstanceGroupName: apiserver + InstanceGroupRole: APIServer + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: api-server + node-role.kubernetes.io/api-server: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.minimal.example.com + serviceAccountJWKSURI: https://api.internal.minimal.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: minimal.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: minimal.example.com + ConfigBase: memfs://clusters.example.com/minimal.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/api-server: "" + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodesminimalexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + ClusterName: minimal.example.com + ConfigBase: memfs://clusters.example.com/minimal.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/aws-lb-controller/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data b/tests/integration/update_cluster/aws-lb-controller/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data index 260794f2fe..351e6f65e7 100644 --- a/tests/integration/update_cluster/aws-lb-controller/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data +++ b/tests/integration/update_cluster/aws-lb-controller/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/aws-lb-controller/data/aws_launch_template_nodes.minimal.example.com_user_data b/tests/integration/update_cluster/aws-lb-controller/data/aws_launch_template_nodes.minimal.example.com_user_data index a2a6065018..a587cb76f0 100644 --- a/tests/integration/update_cluster/aws-lb-controller/data/aws_launch_template_nodes.minimal.example.com_user_data +++ b/tests/integration/update_cluster/aws-lb-controller/data/aws_launch_template_nodes.minimal.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_master-us-test-1a.masters.bastionuserdata.example.com_user_data b/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_master-us-test-1a.masters.bastionuserdata.example.com_user_data index 5db53a2aa3..8bd8522a10 100644 --- a/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_master-us-test-1a.masters.bastionuserdata.example.com_user_data +++ b/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_master-us-test-1a.masters.bastionuserdata.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_nodes.bastionuserdata.example.com_user_data b/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_nodes.bastionuserdata.example.com_user_data index 1b6fe483a3..b92db5bf3d 100644 --- a/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_nodes.bastionuserdata.example.com_user_data +++ b/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_nodes.bastionuserdata.example.com_user_data @@ -43,11 +43,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/complex/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/complex/cloudformation.json.extracted.yaml index 56a53440e7..4fc55c92dc 100644 --- a/tests/integration/update_cluster/complex/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/complex/cloudformation.json.extracted.yaml @@ -1,184 +1,580 @@ -Resources.AWSEC2LaunchTemplatemasterustest1amasterscomplexexamplecom.Properties.LaunchTemplateData.UserData: "Content-Type: - multipart/mixed; boundary=\"MIMEBOUNDARY\"\nMIME-Version: 1.0\n\n--MIMEBOUNDARY\nContent-Disposition: - attachment; filename=\"nodeup.sh\"\nContent-Transfer-Encoding: 7bit\nContent-Type: - text/x-shellscript\nMime-Version: 1.0\n\n#!/bin/bash\nset -o errexit\nset -o nounset\nset - -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nencryptionConfig: null\netcdClusters:\n events:\n version: - 3.4.13\n main:\n version: 3.4.13\nkubeAPIServer:\n allowPrivileged: true\n - \ anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n apiServerCount: - 1\n auditWebhookBatchThrottleQps: 3140m\n authorizationMode: AlwaysAllow\n bindAddress: - 0.0.0.0\n cloudProvider: aws\n cpuLimit: 500m\n cpuRequest: 200m\n enableAdmissionPlugins:\n - \ - NamespaceLifecycle\n - LimitRanger\n - ServiceAccount\n - PersistentVolumeLabel\n - \ - DefaultStorageClass\n - DefaultTolerationSeconds\n - MutatingAdmissionWebhook\n - \ - ValidatingAdmissionWebhook\n - NodeRestriction\n - ResourceQuota\n etcdServers:\n - \ - https://127.0.0.1:4001\n etcdServersOverrides:\n - /events#https://127.0.0.1:4002\n - \ image: k8s.gcr.io/kube-apiserver:v1.21.0\n kubeletPreferredAddressTypes:\n - - InternalIP\n - Hostname\n - ExternalIP\n logLevel: 2\n memoryLimit: 1000Mi\n - \ memoryRequest: 800Mi\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.complex.example.com\n - \ serviceAccountJWKSURI: https://api.internal.complex.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n serviceNodePortRange: 28000-32767\n storageBackend: - etcd3\nkubeControllerManager:\n allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: - 1m0s\n cloudProvider: aws\n clusterCIDR: 100.96.0.0/11\n clusterName: complex.example.com\n - \ configureCloudRoutes: false\n image: k8s.gcr.io/kube-controller-manager:v1.21.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: - true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: - '@aws'\n image: k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: 2\nkubeScheduler:\n - \ image: k8s.gcr.io/kube-scheduler:v1.21.0\n leaderElection:\n leaderElect: - true\n logLevel: 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: systemd\n - \ cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\nmasterKubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n registerSchedulable: false\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - complex.example.com\nConfigBase: memfs://clusters.example.com/complex.example.com\nInstanceGroupName: - master-us-test-1a\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\n - \ registerSchedulable: false\nchannels:\n- memfs://clusters.example.com/complex.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/complex.example.com/manifests/etcd/main.yaml\n- memfs://clusters.example.com/complex.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n\n--MIMEBOUNDARY\nContent-Disposition: attachment; - filename=\"myscript.sh\"\nContent-Transfer-Encoding: 7bit\nContent-Type: text/x-shellscript\nMime-Version: - 1.0\n\n#!/bin/sh\necho \"nodes: The time is now $(date -R)!\" | tee /root/output.txt\n\n--MIMEBOUNDARY--\n" -Resources.AWSEC2LaunchTemplatenodescomplexexamplecom.Properties.LaunchTemplateData.UserData: "Content-Type: - multipart/mixed; boundary=\"MIMEBOUNDARY\"\nMIME-Version: 1.0\n\n--MIMEBOUNDARY\nContent-Disposition: - attachment; filename=\"nodeup.sh\"\nContent-Transfer-Encoding: 7bit\nContent-Type: - text/x-shellscript\nMime-Version: 1.0\n\n#!/bin/bash\nset -o errexit\nset -o nounset\nset - -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n - \ hostnameOverride: '@aws'\n image: k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: - 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: systemd\n cgroupRoot: /\n - \ cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: cluster.local\n - \ enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n\n__EOF_CLUSTER_SPEC\n\ncat > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat - > conf/kube_env.yaml << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\nClusterName: - complex.example.com\nConfigBase: memfs://clusters.example.com/complex.example.com\nInstanceGroupName: - nodes\nInstanceGroupRole: Node\nKubeletConfig:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kubernetes.io/role: node\n node-role.kubernetes.io/node: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\nchannels:\n- - memfs://clusters.example.com/complex.example.com/addons/bootstrap-channel.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n\n--MIMEBOUNDARY\nContent-Disposition: attachment; - filename=\"myscript.sh\"\nContent-Transfer-Encoding: 7bit\nContent-Type: text/x-shellscript\nMime-Version: - 1.0\n\n#!/bin/sh\necho \"nodes: The time is now $(date -R)!\" | tee /root/output.txt\n\n--MIMEBOUNDARY--\n" +Resources.AWSEC2LaunchTemplatemasterustest1amasterscomplexexamplecom.Properties.LaunchTemplateData.UserData: | + Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" + MIME-Version: 1.0 + + --MIMEBOUNDARY + Content-Disposition: attachment; filename="nodeup.sh" + Content-Transfer-Encoding: 7bit + Content-Type: text/x-shellscript + Mime-Version: 1.0 + + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + auditWebhookBatchThrottleQps: 3140m + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + cpuLimit: 500m + cpuRequest: 200m + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + memoryLimit: 1000Mi + memoryRequest: 800Mi + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.complex.example.com + serviceAccountJWKSURI: https://api.internal.complex.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + serviceNodePortRange: 28000-32767 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: complex.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: complex.example.com + ConfigBase: memfs://clusters.example.com/complex.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/complex.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/complex.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/complex.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" + + --MIMEBOUNDARY + Content-Disposition: attachment; filename="myscript.sh" + Content-Transfer-Encoding: 7bit + Content-Type: text/x-shellscript + Mime-Version: 1.0 + + #!/bin/sh + echo "nodes: The time is now $(date -R)!" | tee /root/output.txt + + --MIMEBOUNDARY-- +Resources.AWSEC2LaunchTemplatenodescomplexexamplecom.Properties.LaunchTemplateData.UserData: | + Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" + MIME-Version: 1.0 + + --MIMEBOUNDARY + Content-Disposition: attachment; filename="nodeup.sh" + Content-Transfer-Encoding: 7bit + Content-Type: text/x-shellscript + Mime-Version: 1.0 + + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + ClusterName: complex.example.com + ConfigBase: memfs://clusters.example.com/complex.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/complex.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" + + --MIMEBOUNDARY + Content-Disposition: attachment; filename="myscript.sh" + Content-Transfer-Encoding: 7bit + Content-Type: text/x-shellscript + Mime-Version: 1.0 + + #!/bin/sh + echo "nodes: The time is now $(date -R)!" | tee /root/output.txt + + --MIMEBOUNDARY-- diff --git a/tests/integration/update_cluster/complex/data/aws_launch_template_master-us-test-1a.masters.complex.example.com_user_data b/tests/integration/update_cluster/complex/data/aws_launch_template_master-us-test-1a.masters.complex.example.com_user_data index 9751586939..cdaf364a44 100644 --- a/tests/integration/update_cluster/complex/data/aws_launch_template_master-us-test-1a.masters.complex.example.com_user_data +++ b/tests/integration/update_cluster/complex/data/aws_launch_template_master-us-test-1a.masters.complex.example.com_user_data @@ -43,11 +43,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/complex/data/aws_launch_template_nodes.complex.example.com_user_data b/tests/integration/update_cluster/complex/data/aws_launch_template_nodes.complex.example.com_user_data index b7bbcb2184..053073d8de 100644 --- a/tests/integration/update_cluster/complex/data/aws_launch_template_nodes.complex.example.com_user_data +++ b/tests/integration/update_cluster/complex/data/aws_launch_template_nodes.complex.example.com_user_data @@ -43,11 +43,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/compress/data/aws_launch_template_master-us-test-1a.masters.compress.example.com_user_data b/tests/integration/update_cluster/compress/data/aws_launch_template_master-us-test-1a.masters.compress.example.com_user_data index 460654a650..2d35a8ddb2 100644 --- a/tests/integration/update_cluster/compress/data/aws_launch_template_master-us-test-1a.masters.compress.example.com_user_data +++ b/tests/integration/update_cluster/compress/data/aws_launch_template_master-us-test-1a.masters.compress.example.com_user_data @@ -43,11 +43,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/compress/data/aws_launch_template_nodes.compress.example.com_user_data b/tests/integration/update_cluster/compress/data/aws_launch_template_nodes.compress.example.com_user_data index b067f97956..dde717e89f 100644 --- a/tests/integration/update_cluster/compress/data/aws_launch_template_nodes.compress.example.com_user_data +++ b/tests/integration/update_cluster/compress/data/aws_launch_template_nodes.compress.example.com_user_data @@ -43,11 +43,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/containerd-custom/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/containerd-custom/cloudformation.json.extracted.yaml index dc4f0335a4..439e3f596e 100644 --- a/tests/integration/update_cluster/containerd-custom/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/containerd-custom/cloudformation.json.extracted.yaml @@ -1,187 +1,570 @@ -Resources.AWSEC2LaunchTemplatemasterustest1amasterscontainerdexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n\n [plugins.\"io.containerd.grpc.v1.cri\".registry]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors]\n\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n - \ endpoint = [\"http://HostIP2:Port2\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"docker.io\"]\n - \ endpoint = [\"https://registry-1.docker.io\"]\n logLevel: info\n - \ packages:\n hashAmd64: \"0000000000000000000000000000000000000000000000000000000000000000\"\n - \ urlAmd64: https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz\n - \ registryMirrors:\n '*':\n - http://HostIP2:Port2\n docker.io:\n - - https://registry-1.docker.io\n version: 1.4.6\ndocker:\n skipInstall: true\nencryptionConfig: - null\netcdClusters:\n events:\n version: 3.4.13\n main:\n version: 3.4.13\nkubeAPIServer:\n - \ allowPrivileged: true\n anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n - \ apiServerCount: 1\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n - \ cloudProvider: aws\n enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - \ - ServiceAccount\n - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.21.0\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.containerd.example.com\n - \ serviceAccountJWKSURI: https://api.internal.containerd.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n storageBackend: etcd3\nkubeControllerManager:\n - \ allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: 1m0s\n cloudProvider: - aws\n clusterCIDR: 100.96.0.0/11\n clusterName: containerd.example.com\n configureCloudRoutes: - false\n image: k8s.gcr.io/kube-controller-manager:v1.21.0\n leaderElection:\n - \ leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: true\nkubeProxy:\n - \ clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: '@aws'\n image: - k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: 2\nkubeScheduler:\n image: k8s.gcr.io/kube-scheduler:v1.21.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\nkubelet:\n anonymousAuth: - false\n cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: - 100.64.0.10\n clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\nmasterKubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n registerSchedulable: false\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 0000000000000000000000000000000000000000000000000000000000000000@https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - containerd.example.com\nConfigBase: memfs://clusters.example.com/containerd.example.com\nInstanceGroupName: - master-us-test-1a\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\n - \ registerSchedulable: false\nchannels:\n- memfs://clusters.example.com/containerd.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/containerd.example.com/manifests/etcd/main.yaml\n- - memfs://clusters.example.com/containerd.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatenodescontainerdexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n\n [plugins.\"io.containerd.grpc.v1.cri\".registry]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors]\n\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n - \ endpoint = [\"http://HostIP2:Port2\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"docker.io\"]\n - \ endpoint = [\"https://registry-1.docker.io\"]\n logLevel: info\n - \ packages:\n hashAmd64: \"0000000000000000000000000000000000000000000000000000000000000000\"\n - \ urlAmd64: https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz\n - \ registryMirrors:\n '*':\n - http://HostIP2:Port2\n docker.io:\n - - https://registry-1.docker.io\n version: 1.4.6\ndocker:\n skipInstall: true\nkubeProxy:\n - \ clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: '@aws'\n image: - k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: 2\nkubelet:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n\n__EOF_CLUSTER_SPEC\n\ncat > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat - > conf/kube_env.yaml << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 0000000000000000000000000000000000000000000000000000000000000000@https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\nClusterName: - containerd.example.com\nConfigBase: memfs://clusters.example.com/containerd.example.com\nInstanceGroupName: - nodes\nInstanceGroupRole: Node\nKubeletConfig:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kubernetes.io/role: node\n node-role.kubernetes.io/node: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\nchannels:\n- - memfs://clusters.example.com/containerd.example.com/addons/bootstrap-channel.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" +Resources.AWSEC2LaunchTemplatemasterustest1amasterscontainerdexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + + [plugins."io.containerd.grpc.v1.cri".registry] + + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."*"] + endpoint = ["http://HostIP2:Port2"] + + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + logLevel: info + packages: + hashAmd64: "0000000000000000000000000000000000000000000000000000000000000000" + urlAmd64: https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz + registryMirrors: + '*': + - http://HostIP2:Port2 + docker.io: + - https://registry-1.docker.io + version: 1.4.6 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.containerd.example.com + serviceAccountJWKSURI: https://api.internal.containerd.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: containerd.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 0000000000000000000000000000000000000000000000000000000000000000@https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: containerd.example.com + ConfigBase: memfs://clusters.example.com/containerd.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/containerd.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/containerd.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/containerd.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodescontainerdexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + + [plugins."io.containerd.grpc.v1.cri".registry] + + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."*"] + endpoint = ["http://HostIP2:Port2"] + + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + logLevel: info + packages: + hashAmd64: "0000000000000000000000000000000000000000000000000000000000000000" + urlAmd64: https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz + registryMirrors: + '*': + - http://HostIP2:Port2 + docker.io: + - https://registry-1.docker.io + version: 1.4.6 + docker: + skipInstall: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 0000000000000000000000000000000000000000000000000000000000000000@https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + ClusterName: containerd.example.com + ConfigBase: memfs://clusters.example.com/containerd.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/containerd.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/containerd/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/containerd/cloudformation.json.extracted.yaml index 1a57148a90..ca301b0f22 100644 --- a/tests/integration/update_cluster/containerd/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/containerd/cloudformation.json.extracted.yaml @@ -1,174 +1,534 @@ -Resources.AWSEC2LaunchTemplatemasterustest1amasterscontainerdexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nencryptionConfig: null\netcdClusters:\n events:\n version: - 3.4.13\n main:\n version: 3.4.13\nkubeAPIServer:\n allowPrivileged: true\n - \ anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n apiServerCount: - 1\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n cloudProvider: aws\n - \ enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - ServiceAccount\n - \ - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.21.0\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.containerd.example.com\n - \ serviceAccountJWKSURI: https://api.internal.containerd.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n storageBackend: etcd3\nkubeControllerManager:\n - \ allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: 1m0s\n cloudProvider: - aws\n clusterCIDR: 100.96.0.0/11\n clusterName: containerd.example.com\n configureCloudRoutes: - false\n image: k8s.gcr.io/kube-controller-manager:v1.21.0\n leaderElection:\n - \ leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: true\nkubeProxy:\n - \ clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: '@aws'\n image: - k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: 2\nkubeScheduler:\n image: k8s.gcr.io/kube-scheduler:v1.21.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\nkubelet:\n anonymousAuth: - false\n cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: - 100.64.0.10\n clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\nmasterKubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n registerSchedulable: false\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - containerd.example.com\nConfigBase: memfs://clusters.example.com/containerd.example.com\nInstanceGroupName: - master-us-test-1a\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\n - \ registerSchedulable: false\nchannels:\n- memfs://clusters.example.com/containerd.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/containerd.example.com/manifests/etcd/main.yaml\n- - memfs://clusters.example.com/containerd.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatenodescontainerdexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n - \ hostnameOverride: '@aws'\n image: k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: - 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: systemd\n cgroupRoot: /\n - \ cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: cluster.local\n - \ enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n\n__EOF_CLUSTER_SPEC\n\ncat > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat - > conf/kube_env.yaml << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\nClusterName: - containerd.example.com\nConfigBase: memfs://clusters.example.com/containerd.example.com\nInstanceGroupName: - nodes\nInstanceGroupRole: Node\nKubeletConfig:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kubernetes.io/role: node\n node-role.kubernetes.io/node: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\nchannels:\n- - memfs://clusters.example.com/containerd.example.com/addons/bootstrap-channel.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" +Resources.AWSEC2LaunchTemplatemasterustest1amasterscontainerdexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.containerd.example.com + serviceAccountJWKSURI: https://api.internal.containerd.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: containerd.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: containerd.example.com + ConfigBase: memfs://clusters.example.com/containerd.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/containerd.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/containerd.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/containerd.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodescontainerdexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + ClusterName: containerd.example.com + ConfigBase: memfs://clusters.example.com/containerd.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/containerd.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/docker-custom/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/docker-custom/cloudformation.json.extracted.yaml index 66126cfac8..45aefe6846 100644 --- a/tests/integration/update_cluster/docker-custom/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/docker-custom/cloudformation.json.extracted.yaml @@ -1,180 +1,539 @@ -Resources.AWSEC2LaunchTemplatemasterustest1amastersdockerexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - docker\ncontainerd:\n configOverride: |\n disabled_plugins = [\"cri\"]\n logLevel: - info\ndocker:\n execOpt:\n - native.cgroupdriver=systemd\n ipMasq: false\n ipTables: - false\n logDriver: json-file\n logLevel: info\n logOpt:\n - max-size=10m\n - - max-file=5\n packages:\n hashAmd64: 000000000000000000000000000000000000000000000000000000000000000a\n - \ hashArm64: 000000000000000000000000000000000000000000000000000000000000000b\n - \ urlAmd64: https://download.docker.com/linux/static/stable/x86_64/docker-20.10.1.tgz\n - \ urlArm64: https://download.docker.com/linux/static/stable/aarch64/docker-20.10.1.tgz\n - \ storage: overlay2,overlay,aufs\n version: 20.10.5\nencryptionConfig: null\netcdClusters:\n - \ events:\n version: 3.4.13\n main:\n version: 3.4.13\nkubeAPIServer:\n allowPrivileged: - true\n anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n apiServerCount: - 1\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n cloudProvider: aws\n - \ enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - ServiceAccount\n - \ - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.21.0\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.docker.example.com\n - \ serviceAccountJWKSURI: https://api.internal.docker.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n storageBackend: etcd3\nkubeControllerManager:\n - \ allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: 1m0s\n cloudProvider: - aws\n clusterCIDR: 100.96.0.0/11\n clusterName: docker.example.com\n configureCloudRoutes: - false\n image: k8s.gcr.io/kube-controller-manager:v1.21.0\n leaderElection:\n - \ leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: true\nkubeProxy:\n - \ clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: '@aws'\n image: - k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: 2\nkubeScheduler:\n image: k8s.gcr.io/kube-scheduler:v1.21.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\nkubelet:\n anonymousAuth: - false\n cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: - 100.64.0.10\n clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podInfraContainerImage: - k8s.gcr.io/pause:3.2\n podManifestPath: /etc/kubernetes/manifests\nmasterKubelet:\n - \ anonymousAuth: false\n cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: - aws\n clusterDNS: 100.64.0.10\n clusterDomain: cluster.local\n enableDebuggingHandlers: - true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podInfraContainerImage: - k8s.gcr.io/pause:3.2\n podManifestPath: /etc/kubernetes/manifests\n registerSchedulable: - false\n\n__EOF_CLUSTER_SPEC\n\ncat > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat - > conf/kube_env.yaml << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 000000000000000000000000000000000000000000000000000000000000000a@https://download.docker.com/linux/static/stable/x86_64/docker-20.10.1.tgz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 000000000000000000000000000000000000000000000000000000000000000b@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.1.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - docker.example.com\nConfigBase: memfs://clusters.example.com/docker.example.com\nInstanceGroupName: - master-us-test-1a\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podInfraContainerImage: k8s.gcr.io/pause:3.2\n - \ podManifestPath: /etc/kubernetes/manifests\n registerSchedulable: false\nchannels:\n- - memfs://clusters.example.com/docker.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/docker.example.com/manifests/etcd/main.yaml\n- memfs://clusters.example.com/docker.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatenodesdockerexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - docker\ncontainerd:\n configOverride: |\n disabled_plugins = [\"cri\"]\n logLevel: - info\ndocker:\n execOpt:\n - native.cgroupdriver=systemd\n ipMasq: false\n ipTables: - false\n logDriver: json-file\n logLevel: info\n logOpt:\n - max-size=10m\n - - max-file=5\n packages:\n hashAmd64: 000000000000000000000000000000000000000000000000000000000000000a\n - \ hashArm64: 000000000000000000000000000000000000000000000000000000000000000b\n - \ urlAmd64: https://download.docker.com/linux/static/stable/x86_64/docker-20.10.1.tgz\n - \ urlArm64: https://download.docker.com/linux/static/stable/aarch64/docker-20.10.1.tgz\n - \ storage: overlay2,overlay,aufs\n version: 20.10.5\nkubeProxy:\n clusterCIDR: - 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: '@aws'\n image: k8s.gcr.io/kube-proxy:v1.21.0\n - \ logLevel: 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: systemd\n cgroupRoot: - /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: cluster.local\n - \ enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podInfraContainerImage: - k8s.gcr.io/pause:3.2\n podManifestPath: /etc/kubernetes/manifests\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 000000000000000000000000000000000000000000000000000000000000000a@https://download.docker.com/linux/static/stable/x86_64/docker-20.10.1.tgz\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 000000000000000000000000000000000000000000000000000000000000000b@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.1.tgz\nClusterName: - docker.example.com\nConfigBase: memfs://clusters.example.com/docker.example.com\nInstanceGroupName: - nodes\nInstanceGroupRole: Node\nKubeletConfig:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kubernetes.io/role: node\n node-role.kubernetes.io/node: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podInfraContainerImage: k8s.gcr.io/pause:3.2\n - \ podManifestPath: /etc/kubernetes/manifests\nchannels:\n- memfs://clusters.example.com/docker.example.com/addons/bootstrap-channel.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" +Resources.AWSEC2LaunchTemplatemasterustest1amastersdockerexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: docker + containerd: + configOverride: | + disabled_plugins = ["cri"] + logLevel: info + docker: + execOpt: + - native.cgroupdriver=systemd + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + packages: + hashAmd64: 000000000000000000000000000000000000000000000000000000000000000a + hashArm64: 000000000000000000000000000000000000000000000000000000000000000b + urlAmd64: https://download.docker.com/linux/static/stable/x86_64/docker-20.10.1.tgz + urlArm64: https://download.docker.com/linux/static/stable/aarch64/docker-20.10.1.tgz + storage: overlay2,overlay,aufs + version: 20.10.5 + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.docker.example.com + serviceAccountJWKSURI: https://api.internal.docker.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: docker.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 000000000000000000000000000000000000000000000000000000000000000a@https://download.docker.com/linux/static/stable/x86_64/docker-20.10.1.tgz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 000000000000000000000000000000000000000000000000000000000000000b@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.1.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: docker.example.com + ConfigBase: memfs://clusters.example.com/docker.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/docker.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/docker.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/docker.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodesdockerexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: docker + containerd: + configOverride: | + disabled_plugins = ["cri"] + logLevel: info + docker: + execOpt: + - native.cgroupdriver=systemd + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + packages: + hashAmd64: 000000000000000000000000000000000000000000000000000000000000000a + hashArm64: 000000000000000000000000000000000000000000000000000000000000000b + urlAmd64: https://download.docker.com/linux/static/stable/x86_64/docker-20.10.1.tgz + urlArm64: https://download.docker.com/linux/static/stable/aarch64/docker-20.10.1.tgz + storage: overlay2,overlay,aufs + version: 20.10.5 + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 000000000000000000000000000000000000000000000000000000000000000a@https://download.docker.com/linux/static/stable/x86_64/docker-20.10.1.tgz + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 000000000000000000000000000000000000000000000000000000000000000b@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.1.tgz + ClusterName: docker.example.com + ConfigBase: memfs://clusters.example.com/docker.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/docker.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1a.masters.existing-iam.example.com_user_data b/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1a.masters.existing-iam.example.com_user_data index debd9ecec9..ee0fe65dfa 100644 --- a/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1a.masters.existing-iam.example.com_user_data +++ b/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1a.masters.existing-iam.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1b.masters.existing-iam.example.com_user_data b/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1b.masters.existing-iam.example.com_user_data index a3a7784787..575779e5ee 100644 --- a/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1b.masters.existing-iam.example.com_user_data +++ b/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1b.masters.existing-iam.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1c.masters.existing-iam.example.com_user_data b/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1c.masters.existing-iam.example.com_user_data index 00768bd895..ad2793643f 100644 --- a/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1c.masters.existing-iam.example.com_user_data +++ b/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1c.masters.existing-iam.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/existing_iam/data/aws_launch_template_nodes.existing-iam.example.com_user_data b/tests/integration/update_cluster/existing_iam/data/aws_launch_template_nodes.existing-iam.example.com_user_data index bfc74a4e25..d6d9a05577 100644 --- a/tests/integration/update_cluster/existing_iam/data/aws_launch_template_nodes.existing-iam.example.com_user_data +++ b/tests/integration/update_cluster/existing_iam/data/aws_launch_template_nodes.existing-iam.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/existing_iam_cloudformation/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/existing_iam_cloudformation/cloudformation.json.extracted.yaml index 9c05fdcd2e..7fc8f2ed3d 100644 --- a/tests/integration/update_cluster/existing_iam_cloudformation/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/existing_iam_cloudformation/cloudformation.json.extracted.yaml @@ -1,173 +1,534 @@ -Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nencryptionConfig: null\netcdClusters:\n events:\n version: - 3.4.13\n main:\n version: 3.4.13\nkubeAPIServer:\n allowPrivileged: true\n - \ anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n apiServerCount: - 1\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n cloudProvider: aws\n - \ enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - ServiceAccount\n - \ - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.21.0\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.minimal.example.com\n - \ serviceAccountJWKSURI: https://api.internal.minimal.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n storageBackend: etcd3\nkubeControllerManager:\n - \ allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: 1m0s\n cloudProvider: - aws\n clusterCIDR: 100.96.0.0/11\n clusterName: minimal.example.com\n configureCloudRoutes: - false\n image: k8s.gcr.io/kube-controller-manager:v1.21.0\n leaderElection:\n - \ leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: true\nkubeProxy:\n - \ clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: '@aws'\n image: - k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: 2\nkubeScheduler:\n image: k8s.gcr.io/kube-scheduler:v1.21.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\nkubelet:\n anonymousAuth: - false\n cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: - 100.64.0.10\n clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\nmasterKubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n registerSchedulable: false\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - minimal.example.com\nConfigBase: memfs://clusters.example.com/minimal.example.com\nInstanceGroupName: - master-us-test-1a\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\n - \ registerSchedulable: false\nchannels:\n- memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml\n- memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatenodesminimalexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n - \ hostnameOverride: '@aws'\n image: k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: - 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: systemd\n cgroupRoot: /\n - \ cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: cluster.local\n - \ enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n\n__EOF_CLUSTER_SPEC\n\ncat > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat - > conf/kube_env.yaml << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\nClusterName: - minimal.example.com\nConfigBase: memfs://clusters.example.com/minimal.example.com\nInstanceGroupName: - nodes\nInstanceGroupRole: Node\nKubeletConfig:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kubernetes.io/role: node\n node-role.kubernetes.io/node: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\nchannels:\n- - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" +Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.minimal.example.com + serviceAccountJWKSURI: https://api.internal.minimal.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: minimal.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: minimal.example.com + ConfigBase: memfs://clusters.example.com/minimal.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodesminimalexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + ClusterName: minimal.example.com + ConfigBase: memfs://clusters.example.com/minimal.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1a.masters.existingsg.example.com_user_data b/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1a.masters.existingsg.example.com_user_data index 39c49a0769..b7c1dbe910 100644 --- a/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1a.masters.existingsg.example.com_user_data +++ b/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1a.masters.existingsg.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1b.masters.existingsg.example.com_user_data b/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1b.masters.existingsg.example.com_user_data index 0fcedf5fcc..85ff29af06 100644 --- a/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1b.masters.existingsg.example.com_user_data +++ b/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1b.masters.existingsg.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1c.masters.existingsg.example.com_user_data b/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1c.masters.existingsg.example.com_user_data index 3ebd0fc4af..37896b0557 100644 --- a/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1c.masters.existingsg.example.com_user_data +++ b/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1c.masters.existingsg.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/existing_sg/data/aws_launch_template_nodes.existingsg.example.com_user_data b/tests/integration/update_cluster/existing_sg/data/aws_launch_template_nodes.existingsg.example.com_user_data index 40282b03a2..5b12c81701 100644 --- a/tests/integration/update_cluster/existing_sg/data/aws_launch_template_nodes.existingsg.example.com_user_data +++ b/tests/integration/update_cluster/existing_sg/data/aws_launch_template_nodes.existingsg.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/externallb/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/externallb/cloudformation.json.extracted.yaml index dea9a87001..090e370d26 100644 --- a/tests/integration/update_cluster/externallb/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/externallb/cloudformation.json.extracted.yaml @@ -1,174 +1,534 @@ -Resources.AWSEC2LaunchTemplatemasterustest1amastersexternallbexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nencryptionConfig: null\netcdClusters:\n events:\n version: - 3.4.13\n main:\n version: 3.4.13\nkubeAPIServer:\n allowPrivileged: true\n - \ anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n apiServerCount: - 1\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n cloudProvider: aws\n - \ enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - ServiceAccount\n - \ - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.21.0\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.externallb.example.com\n - \ serviceAccountJWKSURI: https://api.internal.externallb.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n storageBackend: etcd3\nkubeControllerManager:\n - \ allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: 1m0s\n cloudProvider: - aws\n clusterCIDR: 100.96.0.0/11\n clusterName: externallb.example.com\n configureCloudRoutes: - false\n image: k8s.gcr.io/kube-controller-manager:v1.21.0\n leaderElection:\n - \ leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: true\nkubeProxy:\n - \ clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: '@aws'\n image: - k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: 2\nkubeScheduler:\n image: k8s.gcr.io/kube-scheduler:v1.21.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\nkubelet:\n anonymousAuth: - false\n cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: - 100.64.0.10\n clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\nmasterKubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n registerSchedulable: false\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - externallb.example.com\nConfigBase: memfs://clusters.example.com/externallb.example.com\nInstanceGroupName: - master-us-test-1a\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\n - \ registerSchedulable: false\nchannels:\n- memfs://clusters.example.com/externallb.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/externallb.example.com/manifests/etcd/main.yaml\n- - memfs://clusters.example.com/externallb.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatenodesexternallbexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n - \ hostnameOverride: '@aws'\n image: k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: - 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: systemd\n cgroupRoot: /\n - \ cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: cluster.local\n - \ enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n\n__EOF_CLUSTER_SPEC\n\ncat > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat - > conf/kube_env.yaml << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\nClusterName: - externallb.example.com\nConfigBase: memfs://clusters.example.com/externallb.example.com\nInstanceGroupName: - nodes\nInstanceGroupRole: Node\nKubeletConfig:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kubernetes.io/role: node\n node-role.kubernetes.io/node: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\nchannels:\n- - memfs://clusters.example.com/externallb.example.com/addons/bootstrap-channel.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" +Resources.AWSEC2LaunchTemplatemasterustest1amastersexternallbexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.externallb.example.com + serviceAccountJWKSURI: https://api.internal.externallb.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: externallb.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: externallb.example.com + ConfigBase: memfs://clusters.example.com/externallb.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/externallb.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/externallb.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/externallb.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodesexternallbexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + ClusterName: externallb.example.com + ConfigBase: memfs://clusters.example.com/externallb.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/externallb.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/externallb/data/aws_launch_template_master-us-test-1a.masters.externallb.example.com_user_data b/tests/integration/update_cluster/externallb/data/aws_launch_template_master-us-test-1a.masters.externallb.example.com_user_data index b7dd10c267..1d7bc4ec1d 100644 --- a/tests/integration/update_cluster/externallb/data/aws_launch_template_master-us-test-1a.masters.externallb.example.com_user_data +++ b/tests/integration/update_cluster/externallb/data/aws_launch_template_master-us-test-1a.masters.externallb.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/externallb/data/aws_launch_template_nodes.externallb.example.com_user_data b/tests/integration/update_cluster/externallb/data/aws_launch_template_nodes.externallb.example.com_user_data index 7988d77c26..ad8bfc17d0 100644 --- a/tests/integration/update_cluster/externallb/data/aws_launch_template_nodes.externallb.example.com_user_data +++ b/tests/integration/update_cluster/externallb/data/aws_launch_template_nodes.externallb.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/externalpolicies/data/aws_launch_template_master-us-test-1a.masters.externalpolicies.example.com_user_data b/tests/integration/update_cluster/externalpolicies/data/aws_launch_template_master-us-test-1a.masters.externalpolicies.example.com_user_data index 035e30a9f9..eeec799557 100644 --- a/tests/integration/update_cluster/externalpolicies/data/aws_launch_template_master-us-test-1a.masters.externalpolicies.example.com_user_data +++ b/tests/integration/update_cluster/externalpolicies/data/aws_launch_template_master-us-test-1a.masters.externalpolicies.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/externalpolicies/data/aws_launch_template_nodes.externalpolicies.example.com_user_data b/tests/integration/update_cluster/externalpolicies/data/aws_launch_template_nodes.externalpolicies.example.com_user_data index 2c6a980c95..094e10445b 100644 --- a/tests/integration/update_cluster/externalpolicies/data/aws_launch_template_nodes.externalpolicies.example.com_user_data +++ b/tests/integration/update_cluster/externalpolicies/data/aws_launch_template_nodes.externalpolicies.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1a.masters.ha.example.com_user_data b/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1a.masters.ha.example.com_user_data index 6dbab64029..84df476d9f 100644 --- a/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1a.masters.ha.example.com_user_data +++ b/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1a.masters.ha.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1b.masters.ha.example.com_user_data b/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1b.masters.ha.example.com_user_data index 740a1f2005..8d3ef743c7 100644 --- a/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1b.masters.ha.example.com_user_data +++ b/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1b.masters.ha.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1c.masters.ha.example.com_user_data b/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1c.masters.ha.example.com_user_data index 823b6ce808..c790fc5e06 100644 --- a/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1c.masters.ha.example.com_user_data +++ b/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1c.masters.ha.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/ha/data/aws_launch_template_nodes.ha.example.com_user_data b/tests/integration/update_cluster/ha/data/aws_launch_template_nodes.ha.example.com_user_data index 9cbf236cf1..99a8fa66fd 100644 --- a/tests/integration/update_cluster/ha/data/aws_launch_template_nodes.ha.example.com_user_data +++ b/tests/integration/update_cluster/ha/data/aws_launch_template_nodes.ha.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/ha_gce/data/google_compute_instance_template_master-us-test1-a-ha-gce-example-com_metadata_startup-script b/tests/integration/update_cluster/ha_gce/data/google_compute_instance_template_master-us-test1-a-ha-gce-example-com_metadata_startup-script index aac8a0cb19..ba36cefe91 100644 --- a/tests/integration/update_cluster/ha_gce/data/google_compute_instance_template_master-us-test1-a-ha-gce-example-com_metadata_startup-script +++ b/tests/integration/update_cluster/ha_gce/data/google_compute_instance_template_master-us-test1-a-ha-gce-example-com_metadata_startup-script @@ -33,11 +33,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/ha_gce/data/google_compute_instance_template_master-us-test1-b-ha-gce-example-com_metadata_startup-script b/tests/integration/update_cluster/ha_gce/data/google_compute_instance_template_master-us-test1-b-ha-gce-example-com_metadata_startup-script index 65f1a6f966..a92c89ce5e 100644 --- a/tests/integration/update_cluster/ha_gce/data/google_compute_instance_template_master-us-test1-b-ha-gce-example-com_metadata_startup-script +++ b/tests/integration/update_cluster/ha_gce/data/google_compute_instance_template_master-us-test1-b-ha-gce-example-com_metadata_startup-script @@ -33,11 +33,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/ha_gce/data/google_compute_instance_template_master-us-test1-c-ha-gce-example-com_metadata_startup-script b/tests/integration/update_cluster/ha_gce/data/google_compute_instance_template_master-us-test1-c-ha-gce-example-com_metadata_startup-script index e8dccfaa77..b7d8a8154a 100644 --- a/tests/integration/update_cluster/ha_gce/data/google_compute_instance_template_master-us-test1-c-ha-gce-example-com_metadata_startup-script +++ b/tests/integration/update_cluster/ha_gce/data/google_compute_instance_template_master-us-test1-c-ha-gce-example-com_metadata_startup-script @@ -33,11 +33,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/ha_gce/data/google_compute_instance_template_nodes-ha-gce-example-com_metadata_startup-script b/tests/integration/update_cluster/ha_gce/data/google_compute_instance_template_nodes-ha-gce-example-com_metadata_startup-script index 4c96139115..cbf25f9525 100644 --- a/tests/integration/update_cluster/ha_gce/data/google_compute_instance_template_nodes-ha-gce-example-com_metadata_startup-script +++ b/tests/integration/update_cluster/ha_gce/data/google_compute_instance_template_nodes-ha-gce-example-com_metadata_startup-script @@ -33,11 +33,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/irsa/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data b/tests/integration/update_cluster/irsa/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data index 6326f31347..d9eceff1ea 100644 --- a/tests/integration/update_cluster/irsa/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data +++ b/tests/integration/update_cluster/irsa/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/irsa/data/aws_launch_template_nodes.minimal.example.com_user_data b/tests/integration/update_cluster/irsa/data/aws_launch_template_nodes.minimal.example.com_user_data index e60606a428..99684fdd48 100644 --- a/tests/integration/update_cluster/irsa/data/aws_launch_template_nodes.minimal.example.com_user_data +++ b/tests/integration/update_cluster/irsa/data/aws_launch_template_nodes.minimal.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/minimal-etcd/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/minimal-etcd/cloudformation.json.extracted.yaml index f38a2a937d..6077560283 100644 --- a/tests/integration/update_cluster/minimal-etcd/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/minimal-etcd/cloudformation.json.extracted.yaml @@ -1,179 +1,550 @@ -Resources.AWSEC2LaunchTemplatemasterustest1amastersminimaletcdexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nencryptionConfig: null\netcdClusters:\n events:\n etcdMembers:\n - \ - name: us-test-1a\n volumeSize: 20\n manager:\n env:\n - - name: ETCD_MANAGER_HOURLY_BACKUPS_RETENTION\n value: 1d\n - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION\n - \ value: 30d\n image: gcr.io/k8s-staging-etcdadm/etcd:v20210430-v0.1.3-739-g7da12acc\n - \ version: 3.4.13\n main:\n etcdMembers:\n - name: us-test-1a\n volumeSize: - 20\n manager:\n image: gcr.io/k8s-staging-etcdadm/etcd:v20210430-v0.1.3-739-g7da12acc\n - \ logLevel: 10\n version: 3.4.13\nkubeAPIServer:\n allowPrivileged: true\n - \ anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n apiServerCount: - 1\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n cloudProvider: aws\n - \ enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - ServiceAccount\n - \ - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.21.0\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.minimal-etcd.example.com\n - \ serviceAccountJWKSURI: https://api.internal.minimal-etcd.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n storageBackend: etcd3\nkubeControllerManager:\n - \ allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: 1m0s\n cloudProvider: - aws\n clusterCIDR: 100.96.0.0/11\n clusterName: minimal-etcd.example.com\n configureCloudRoutes: - false\n image: k8s.gcr.io/kube-controller-manager:v1.21.0\n leaderElection:\n - \ leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: true\nkubeProxy:\n - \ clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: '@aws'\n image: - k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: 2\nkubeScheduler:\n image: k8s.gcr.io/kube-scheduler:v1.21.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\nkubelet:\n anonymousAuth: - false\n cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: - 100.64.0.10\n clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\nmasterKubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n registerSchedulable: false\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - minimal-etcd.example.com\nConfigBase: memfs://clusters.example.com/minimal-etcd.example.com\nInstanceGroupName: - master-us-test-1a\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\n - \ registerSchedulable: false\nchannels:\n- memfs://clusters.example.com/minimal-etcd.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/minimal-etcd.example.com/manifests/etcd/main.yaml\n- - memfs://clusters.example.com/minimal-etcd.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatenodesminimaletcdexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n - \ hostnameOverride: '@aws'\n image: k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: - 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: systemd\n cgroupRoot: /\n - \ cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: cluster.local\n - \ enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n\n__EOF_CLUSTER_SPEC\n\ncat > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat - > conf/kube_env.yaml << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\nClusterName: - minimal-etcd.example.com\nConfigBase: memfs://clusters.example.com/minimal-etcd.example.com\nInstanceGroupName: - nodes\nInstanceGroupRole: Node\nKubeletConfig:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kubernetes.io/role: node\n node-role.kubernetes.io/node: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\nchannels:\n- - memfs://clusters.example.com/minimal-etcd.example.com/addons/bootstrap-channel.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" +Resources.AWSEC2LaunchTemplatemasterustest1amastersminimaletcdexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + etcdMembers: + - name: us-test-1a + volumeSize: 20 + manager: + env: + - name: ETCD_MANAGER_HOURLY_BACKUPS_RETENTION + value: 1d + - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION + value: 30d + image: gcr.io/k8s-staging-etcdadm/etcd:v20210430-v0.1.3-739-g7da12acc + version: 3.4.13 + main: + etcdMembers: + - name: us-test-1a + volumeSize: 20 + manager: + image: gcr.io/k8s-staging-etcdadm/etcd:v20210430-v0.1.3-739-g7da12acc + logLevel: 10 + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.minimal-etcd.example.com + serviceAccountJWKSURI: https://api.internal.minimal-etcd.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: minimal-etcd.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: minimal-etcd.example.com + ConfigBase: memfs://clusters.example.com/minimal-etcd.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/minimal-etcd.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/minimal-etcd.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/minimal-etcd.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodesminimaletcdexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + ClusterName: minimal-etcd.example.com + ConfigBase: memfs://clusters.example.com/minimal-etcd.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/minimal-etcd.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/minimal-gp3/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/minimal-gp3/cloudformation.json.extracted.yaml index 769257e2dd..f957153a6a 100644 --- a/tests/integration/update_cluster/minimal-gp3/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/minimal-gp3/cloudformation.json.extracted.yaml @@ -1,174 +1,540 @@ -Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nencryptionConfig: null\netcdClusters:\n events:\n etcdMembers:\n - \ - name: us-test-1a\n volumeSize: 20\n version: 3.4.13\n main:\n etcdMembers:\n - \ - name: us-test-1a\n volumeSize: 50\n version: 3.4.13\nkubeAPIServer:\n - \ allowPrivileged: true\n anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n - \ apiServerCount: 1\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n - \ cloudProvider: aws\n enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - \ - ServiceAccount\n - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.21.0\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.minimal.example.com\n - \ serviceAccountJWKSURI: https://api.internal.minimal.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n storageBackend: etcd3\nkubeControllerManager:\n - \ allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: 1m0s\n cloudProvider: - aws\n clusterCIDR: 100.96.0.0/11\n clusterName: minimal.example.com\n configureCloudRoutes: - false\n image: k8s.gcr.io/kube-controller-manager:v1.21.0\n leaderElection:\n - \ leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: true\nkubeProxy:\n - \ clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: '@aws'\n image: - k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: 2\nkubeScheduler:\n image: k8s.gcr.io/kube-scheduler:v1.21.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\nkubelet:\n anonymousAuth: - false\n cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: - 100.64.0.10\n clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\nmasterKubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n registerSchedulable: false\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - minimal.example.com\nConfigBase: memfs://clusters.example.com/minimal.example.com\nInstanceGroupName: - master-us-test-1a\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\n - \ registerSchedulable: false\nchannels:\n- memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml\n- memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatenodesminimalexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n - \ hostnameOverride: '@aws'\n image: k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: - 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: systemd\n cgroupRoot: /\n - \ cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: cluster.local\n - \ enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n\n__EOF_CLUSTER_SPEC\n\ncat > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat - > conf/kube_env.yaml << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\nClusterName: - minimal.example.com\nConfigBase: memfs://clusters.example.com/minimal.example.com\nInstanceGroupName: - nodes\nInstanceGroupRole: Node\nKubeletConfig:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kubernetes.io/role: node\n node-role.kubernetes.io/node: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\nchannels:\n- - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" +Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + etcdMembers: + - name: us-test-1a + volumeSize: 20 + version: 3.4.13 + main: + etcdMembers: + - name: us-test-1a + volumeSize: 50 + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.minimal.example.com + serviceAccountJWKSURI: https://api.internal.minimal.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: minimal.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: minimal.example.com + ConfigBase: memfs://clusters.example.com/minimal.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodesminimalexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + ClusterName: minimal.example.com + ConfigBase: memfs://clusters.example.com/minimal.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/minimal-gp3/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data b/tests/integration/update_cluster/minimal-gp3/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data index ce5fe553fd..bda5538021 100644 --- a/tests/integration/update_cluster/minimal-gp3/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data +++ b/tests/integration/update_cluster/minimal-gp3/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/minimal-gp3/data/aws_launch_template_nodes.minimal.example.com_user_data b/tests/integration/update_cluster/minimal-gp3/data/aws_launch_template_nodes.minimal.example.com_user_data index a2a6065018..a587cb76f0 100644 --- a/tests/integration/update_cluster/minimal-gp3/data/aws_launch_template_nodes.minimal.example.com_user_data +++ b/tests/integration/update_cluster/minimal-gp3/data/aws_launch_template_nodes.minimal.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/minimal-ipv6/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/minimal-ipv6/cloudformation.json.extracted.yaml index f14a338ea5..50d301e53f 100644 --- a/tests/integration/update_cluster/minimal-ipv6/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/minimal-ipv6/cloudformation.json.extracted.yaml @@ -1,174 +1,534 @@ -Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalipv6examplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nencryptionConfig: null\netcdClusters:\n events:\n version: - 3.4.13\n main:\n version: 3.4.13\nkubeAPIServer:\n allowPrivileged: true\n - \ anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n apiServerCount: - 1\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n cloudProvider: aws\n - \ enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - ServiceAccount\n - \ - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.21.0\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.minimal-ipv6.example.com\n - \ serviceAccountJWKSURI: https://api.internal.minimal-ipv6.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n storageBackend: etcd3\nkubeControllerManager:\n - \ allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: 1m0s\n cloudProvider: - aws\n clusterCIDR: 100.96.0.0/11\n clusterName: minimal-ipv6.example.com\n configureCloudRoutes: - false\n image: k8s.gcr.io/kube-controller-manager:v1.21.0\n leaderElection:\n - \ leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: true\nkubeProxy:\n - \ clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: '@aws'\n image: - k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: 2\nkubeScheduler:\n image: k8s.gcr.io/kube-scheduler:v1.21.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\nkubelet:\n anonymousAuth: - false\n cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: - 100.64.0.10\n clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\nmasterKubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n registerSchedulable: false\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - minimal-ipv6.example.com\nConfigBase: memfs://clusters.example.com/minimal-ipv6.example.com\nInstanceGroupName: - master-us-test-1a\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\n - \ registerSchedulable: false\nchannels:\n- memfs://clusters.example.com/minimal-ipv6.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/minimal-ipv6.example.com/manifests/etcd/main.yaml\n- - memfs://clusters.example.com/minimal-ipv6.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatenodesminimalipv6examplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n - \ hostnameOverride: '@aws'\n image: k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: - 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: systemd\n cgroupRoot: /\n - \ cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: cluster.local\n - \ enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n\n__EOF_CLUSTER_SPEC\n\ncat > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat - > conf/kube_env.yaml << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\nClusterName: - minimal-ipv6.example.com\nConfigBase: memfs://clusters.example.com/minimal-ipv6.example.com\nInstanceGroupName: - nodes\nInstanceGroupRole: Node\nKubeletConfig:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kubernetes.io/role: node\n node-role.kubernetes.io/node: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\nchannels:\n- - memfs://clusters.example.com/minimal-ipv6.example.com/addons/bootstrap-channel.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" +Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalipv6examplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.minimal-ipv6.example.com + serviceAccountJWKSURI: https://api.internal.minimal-ipv6.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: minimal-ipv6.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: minimal-ipv6.example.com + ConfigBase: memfs://clusters.example.com/minimal-ipv6.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/minimal-ipv6.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/minimal-ipv6.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/minimal-ipv6.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodesminimalipv6examplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + ClusterName: minimal-ipv6.example.com + ConfigBase: memfs://clusters.example.com/minimal-ipv6.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/minimal-ipv6.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/minimal-ipv6/data/aws_launch_template_master-us-test-1a.masters.minimal-ipv6.example.com_user_data b/tests/integration/update_cluster/minimal-ipv6/data/aws_launch_template_master-us-test-1a.masters.minimal-ipv6.example.com_user_data index 1adc017a83..f5cc88e3cd 100644 --- a/tests/integration/update_cluster/minimal-ipv6/data/aws_launch_template_master-us-test-1a.masters.minimal-ipv6.example.com_user_data +++ b/tests/integration/update_cluster/minimal-ipv6/data/aws_launch_template_master-us-test-1a.masters.minimal-ipv6.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/minimal-ipv6/data/aws_launch_template_nodes.minimal-ipv6.example.com_user_data b/tests/integration/update_cluster/minimal-ipv6/data/aws_launch_template_nodes.minimal-ipv6.example.com_user_data index d6da4b52fa..5872945e11 100644 --- a/tests/integration/update_cluster/minimal-ipv6/data/aws_launch_template_nodes.minimal-ipv6.example.com_user_data +++ b/tests/integration/update_cluster/minimal-ipv6/data/aws_launch_template_nodes.minimal-ipv6.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/minimal-json/data/aws_launch_template_master-us-test-1a.masters.minimal-json.example.com_user_data b/tests/integration/update_cluster/minimal-json/data/aws_launch_template_master-us-test-1a.masters.minimal-json.example.com_user_data index 74e8ea8aad..0a2dc54878 100644 --- a/tests/integration/update_cluster/minimal-json/data/aws_launch_template_master-us-test-1a.masters.minimal-json.example.com_user_data +++ b/tests/integration/update_cluster/minimal-json/data/aws_launch_template_master-us-test-1a.masters.minimal-json.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/minimal-json/data/aws_launch_template_nodes.minimal-json.example.com_user_data b/tests/integration/update_cluster/minimal-json/data/aws_launch_template_nodes.minimal-json.example.com_user_data index 0cb6ae9be7..3f6651e8a8 100644 --- a/tests/integration/update_cluster/minimal-json/data/aws_launch_template_nodes.minimal-json.example.com_user_data +++ b/tests/integration/update_cluster/minimal-json/data/aws_launch_template_nodes.minimal-json.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/minimal/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/minimal/cloudformation.json.extracted.yaml index 9c05fdcd2e..7fc8f2ed3d 100644 --- a/tests/integration/update_cluster/minimal/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/minimal/cloudformation.json.extracted.yaml @@ -1,173 +1,534 @@ -Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nencryptionConfig: null\netcdClusters:\n events:\n version: - 3.4.13\n main:\n version: 3.4.13\nkubeAPIServer:\n allowPrivileged: true\n - \ anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n apiServerCount: - 1\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n cloudProvider: aws\n - \ enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - ServiceAccount\n - \ - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.21.0\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.minimal.example.com\n - \ serviceAccountJWKSURI: https://api.internal.minimal.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n storageBackend: etcd3\nkubeControllerManager:\n - \ allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: 1m0s\n cloudProvider: - aws\n clusterCIDR: 100.96.0.0/11\n clusterName: minimal.example.com\n configureCloudRoutes: - false\n image: k8s.gcr.io/kube-controller-manager:v1.21.0\n leaderElection:\n - \ leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: true\nkubeProxy:\n - \ clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: '@aws'\n image: - k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: 2\nkubeScheduler:\n image: k8s.gcr.io/kube-scheduler:v1.21.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\nkubelet:\n anonymousAuth: - false\n cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: - 100.64.0.10\n clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\nmasterKubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n registerSchedulable: false\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - minimal.example.com\nConfigBase: memfs://clusters.example.com/minimal.example.com\nInstanceGroupName: - master-us-test-1a\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\n - \ registerSchedulable: false\nchannels:\n- memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml\n- memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatenodesminimalexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n - \ hostnameOverride: '@aws'\n image: k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: - 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: systemd\n cgroupRoot: /\n - \ cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: cluster.local\n - \ enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n\n__EOF_CLUSTER_SPEC\n\ncat > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat - > conf/kube_env.yaml << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\nClusterName: - minimal.example.com\nConfigBase: memfs://clusters.example.com/minimal.example.com\nInstanceGroupName: - nodes\nInstanceGroupRole: Node\nKubeletConfig:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kubernetes.io/role: node\n node-role.kubernetes.io/node: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\nchannels:\n- - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" +Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.minimal.example.com + serviceAccountJWKSURI: https://api.internal.minimal.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: minimal.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: minimal.example.com + ConfigBase: memfs://clusters.example.com/minimal.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodesminimalexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + ClusterName: minimal.example.com + ConfigBase: memfs://clusters.example.com/minimal.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/minimal/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data b/tests/integration/update_cluster/minimal/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data index 90f3131765..e904ca22ab 100644 --- a/tests/integration/update_cluster/minimal/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data +++ b/tests/integration/update_cluster/minimal/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/minimal/data/aws_launch_template_nodes.minimal.example.com_user_data b/tests/integration/update_cluster/minimal/data/aws_launch_template_nodes.minimal.example.com_user_data index a2a6065018..a587cb76f0 100644 --- a/tests/integration/update_cluster/minimal/data/aws_launch_template_nodes.minimal.example.com_user_data +++ b/tests/integration/update_cluster/minimal/data/aws_launch_template_nodes.minimal.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/minimal_gce/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_startup-script b/tests/integration/update_cluster/minimal_gce/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_startup-script index 77da8ed569..6f80f202aa 100644 --- a/tests/integration/update_cluster/minimal_gce/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_startup-script +++ b/tests/integration/update_cluster/minimal_gce/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_startup-script @@ -33,11 +33,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/minimal_gce/data/google_compute_instance_template_nodes-minimal-gce-example-com_metadata_startup-script b/tests/integration/update_cluster/minimal_gce/data/google_compute_instance_template_nodes-minimal-gce-example-com_metadata_startup-script index 7709a7a1e6..bb69074d31 100644 --- a/tests/integration/update_cluster/minimal_gce/data/google_compute_instance_template_nodes-minimal-gce-example-com_metadata_startup-script +++ b/tests/integration/update_cluster/minimal_gce/data/google_compute_instance_template_nodes-minimal-gce-example-com_metadata_startup-script @@ -33,11 +33,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/minimal_gce_private/data/google_compute_instance_template_master-us-test1-a-minimal-gce-private-example-com_metadata_startup-script b/tests/integration/update_cluster/minimal_gce_private/data/google_compute_instance_template_master-us-test1-a-minimal-gce-private-example-com_metadata_startup-script index 52296795b5..a0360c0e15 100644 --- a/tests/integration/update_cluster/minimal_gce_private/data/google_compute_instance_template_master-us-test1-a-minimal-gce-private-example-com_metadata_startup-script +++ b/tests/integration/update_cluster/minimal_gce_private/data/google_compute_instance_template_master-us-test1-a-minimal-gce-private-example-com_metadata_startup-script @@ -33,11 +33,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/minimal_gce_private/data/google_compute_instance_template_nodes-minimal-gce-private-example-com_metadata_startup-script b/tests/integration/update_cluster/minimal_gce_private/data/google_compute_instance_template_nodes-minimal-gce-private-example-com_metadata_startup-script index 326d89c66b..d37fe1f445 100644 --- a/tests/integration/update_cluster/minimal_gce_private/data/google_compute_instance_template_nodes-minimal-gce-private-example-com_metadata_startup-script +++ b/tests/integration/update_cluster/minimal_gce_private/data/google_compute_instance_template_nodes-minimal-gce-private-example-com_metadata_startup-script @@ -33,11 +33,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/minimal_gossip/data/aws_launch_template_master-us-test-1a.masters.minimal.k8s.local_user_data b/tests/integration/update_cluster/minimal_gossip/data/aws_launch_template_master-us-test-1a.masters.minimal.k8s.local_user_data index 07785d0c03..6456337d9f 100644 --- a/tests/integration/update_cluster/minimal_gossip/data/aws_launch_template_master-us-test-1a.masters.minimal.k8s.local_user_data +++ b/tests/integration/update_cluster/minimal_gossip/data/aws_launch_template_master-us-test-1a.masters.minimal.k8s.local_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/minimal_gossip/data/aws_launch_template_nodes.minimal.k8s.local_user_data b/tests/integration/update_cluster/minimal_gossip/data/aws_launch_template_nodes.minimal.k8s.local_user_data index 82e056227a..00691868bd 100644 --- a/tests/integration/update_cluster/minimal_gossip/data/aws_launch_template_nodes.minimal.k8s.local_user_data +++ b/tests/integration/update_cluster/minimal_gossip/data/aws_launch_template_nodes.minimal.k8s.local_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/mixed_instances/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/mixed_instances/cloudformation.json.extracted.yaml index 253bd7c517..68f45a1cd8 100644 --- a/tests/integration/update_cluster/mixed_instances/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/mixed_instances/cloudformation.json.extracted.yaml @@ -1,384 +1,1164 @@ -Resources.AWSEC2LaunchTemplatemasterustest1amastersmixedinstancesexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nencryptionConfig: null\netcdClusters:\n events:\n version: - 3.4.13\n main:\n version: 3.4.13\nkubeAPIServer:\n allowPrivileged: true\n - \ anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n apiServerCount: - 3\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n cloudProvider: aws\n - \ enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - ServiceAccount\n - \ - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.21.0\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.mixedinstances.example.com\n - \ serviceAccountJWKSURI: https://api.internal.mixedinstances.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n storageBackend: etcd3\nkubeControllerManager:\n - \ allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: 1m0s\n cloudProvider: - aws\n clusterCIDR: 100.96.0.0/11\n clusterName: mixedinstances.example.com\n configureCloudRoutes: - false\n image: k8s.gcr.io/kube-controller-manager:v1.21.0\n leaderElection:\n - \ leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: true\nkubeProxy:\n - \ clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: '@aws'\n image: - k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: 2\nkubeScheduler:\n image: k8s.gcr.io/kube-scheduler:v1.21.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\nkubelet:\n anonymousAuth: - false\n cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: - 100.64.0.10\n clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\nmasterKubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n registerSchedulable: false\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - mixedinstances.example.com\nConfigBase: memfs://clusters.example.com/mixedinstances.example.com\nInstanceGroupName: - master-us-test-1a\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\n - \ registerSchedulable: false\nchannels:\n- memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/main.yaml\n- - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatemasterustest1bmastersmixedinstancesexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nencryptionConfig: null\netcdClusters:\n events:\n version: - 3.4.13\n main:\n version: 3.4.13\nkubeAPIServer:\n allowPrivileged: true\n - \ anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n apiServerCount: - 3\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n cloudProvider: aws\n - \ enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - ServiceAccount\n - \ - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.21.0\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.mixedinstances.example.com\n - \ serviceAccountJWKSURI: https://api.internal.mixedinstances.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n storageBackend: etcd3\nkubeControllerManager:\n - \ allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: 1m0s\n cloudProvider: - aws\n clusterCIDR: 100.96.0.0/11\n clusterName: mixedinstances.example.com\n configureCloudRoutes: - false\n image: k8s.gcr.io/kube-controller-manager:v1.21.0\n leaderElection:\n - \ leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: true\nkubeProxy:\n - \ clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: '@aws'\n image: - k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: 2\nkubeScheduler:\n image: k8s.gcr.io/kube-scheduler:v1.21.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\nkubelet:\n anonymousAuth: - false\n cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: - 100.64.0.10\n clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\nmasterKubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n registerSchedulable: false\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - mixedinstances.example.com\nConfigBase: memfs://clusters.example.com/mixedinstances.example.com\nInstanceGroupName: - master-us-test-1b\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\n - \ registerSchedulable: false\nchannels:\n- memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/main.yaml\n- - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatemasterustest1cmastersmixedinstancesexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nencryptionConfig: null\netcdClusters:\n events:\n version: - 3.4.13\n main:\n version: 3.4.13\nkubeAPIServer:\n allowPrivileged: true\n - \ anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n apiServerCount: - 3\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n cloudProvider: aws\n - \ enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - ServiceAccount\n - \ - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.21.0\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.mixedinstances.example.com\n - \ serviceAccountJWKSURI: https://api.internal.mixedinstances.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n storageBackend: etcd3\nkubeControllerManager:\n - \ allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: 1m0s\n cloudProvider: - aws\n clusterCIDR: 100.96.0.0/11\n clusterName: mixedinstances.example.com\n configureCloudRoutes: - false\n image: k8s.gcr.io/kube-controller-manager:v1.21.0\n leaderElection:\n - \ leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: true\nkubeProxy:\n - \ clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: '@aws'\n image: - k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: 2\nkubeScheduler:\n image: k8s.gcr.io/kube-scheduler:v1.21.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\nkubelet:\n anonymousAuth: - false\n cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: - 100.64.0.10\n clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\nmasterKubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n registerSchedulable: false\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - mixedinstances.example.com\nConfigBase: memfs://clusters.example.com/mixedinstances.example.com\nInstanceGroupName: - master-us-test-1c\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\n - \ registerSchedulable: false\nchannels:\n- memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/main.yaml\n- - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatenodesmixedinstancesexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n - \ hostnameOverride: '@aws'\n image: k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: - 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: systemd\n cgroupRoot: /\n - \ cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: cluster.local\n - \ enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n\n__EOF_CLUSTER_SPEC\n\ncat > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat - > conf/kube_env.yaml << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\nClusterName: - mixedinstances.example.com\nConfigBase: memfs://clusters.example.com/mixedinstances.example.com\nInstanceGroupName: - nodes\nInstanceGroupRole: Node\nKubeletConfig:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kubernetes.io/role: node\n node-role.kubernetes.io/node: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\nchannels:\n- - memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" +Resources.AWSEC2LaunchTemplatemasterustest1amastersmixedinstancesexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.mixedinstances.example.com + serviceAccountJWKSURI: https://api.internal.mixedinstances.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: mixedinstances.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: mixedinstances.example.com + ConfigBase: memfs://clusters.example.com/mixedinstances.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatemasterustest1bmastersmixedinstancesexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.mixedinstances.example.com + serviceAccountJWKSURI: https://api.internal.mixedinstances.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: mixedinstances.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: mixedinstances.example.com + ConfigBase: memfs://clusters.example.com/mixedinstances.example.com + InstanceGroupName: master-us-test-1b + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatemasterustest1cmastersmixedinstancesexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.mixedinstances.example.com + serviceAccountJWKSURI: https://api.internal.mixedinstances.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: mixedinstances.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: mixedinstances.example.com + ConfigBase: memfs://clusters.example.com/mixedinstances.example.com + InstanceGroupName: master-us-test-1c + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodesmixedinstancesexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + ClusterName: mixedinstances.example.com + ConfigBase: memfs://clusters.example.com/mixedinstances.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1a.masters.mixedinstances.example.com_user_data b/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1a.masters.mixedinstances.example.com_user_data index e05894bfaf..28e9ea7db9 100644 --- a/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1a.masters.mixedinstances.example.com_user_data +++ b/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1a.masters.mixedinstances.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1b.masters.mixedinstances.example.com_user_data b/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1b.masters.mixedinstances.example.com_user_data index 480d0b8d8d..4706226081 100644 --- a/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1b.masters.mixedinstances.example.com_user_data +++ b/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1b.masters.mixedinstances.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1c.masters.mixedinstances.example.com_user_data b/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1c.masters.mixedinstances.example.com_user_data index 4485bde392..b83e3c377a 100644 --- a/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1c.masters.mixedinstances.example.com_user_data +++ b/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1c.masters.mixedinstances.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_nodes.mixedinstances.example.com_user_data b/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_nodes.mixedinstances.example.com_user_data index ead5a024d2..a6b4293e01 100644 --- a/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_nodes.mixedinstances.example.com_user_data +++ b/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_nodes.mixedinstances.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/mixed_instances_spot/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/mixed_instances_spot/cloudformation.json.extracted.yaml index 253bd7c517..68f45a1cd8 100644 --- a/tests/integration/update_cluster/mixed_instances_spot/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/mixed_instances_spot/cloudformation.json.extracted.yaml @@ -1,384 +1,1164 @@ -Resources.AWSEC2LaunchTemplatemasterustest1amastersmixedinstancesexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nencryptionConfig: null\netcdClusters:\n events:\n version: - 3.4.13\n main:\n version: 3.4.13\nkubeAPIServer:\n allowPrivileged: true\n - \ anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n apiServerCount: - 3\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n cloudProvider: aws\n - \ enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - ServiceAccount\n - \ - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.21.0\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.mixedinstances.example.com\n - \ serviceAccountJWKSURI: https://api.internal.mixedinstances.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n storageBackend: etcd3\nkubeControllerManager:\n - \ allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: 1m0s\n cloudProvider: - aws\n clusterCIDR: 100.96.0.0/11\n clusterName: mixedinstances.example.com\n configureCloudRoutes: - false\n image: k8s.gcr.io/kube-controller-manager:v1.21.0\n leaderElection:\n - \ leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: true\nkubeProxy:\n - \ clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: '@aws'\n image: - k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: 2\nkubeScheduler:\n image: k8s.gcr.io/kube-scheduler:v1.21.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\nkubelet:\n anonymousAuth: - false\n cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: - 100.64.0.10\n clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\nmasterKubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n registerSchedulable: false\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - mixedinstances.example.com\nConfigBase: memfs://clusters.example.com/mixedinstances.example.com\nInstanceGroupName: - master-us-test-1a\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\n - \ registerSchedulable: false\nchannels:\n- memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/main.yaml\n- - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatemasterustest1bmastersmixedinstancesexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nencryptionConfig: null\netcdClusters:\n events:\n version: - 3.4.13\n main:\n version: 3.4.13\nkubeAPIServer:\n allowPrivileged: true\n - \ anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n apiServerCount: - 3\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n cloudProvider: aws\n - \ enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - ServiceAccount\n - \ - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.21.0\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.mixedinstances.example.com\n - \ serviceAccountJWKSURI: https://api.internal.mixedinstances.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n storageBackend: etcd3\nkubeControllerManager:\n - \ allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: 1m0s\n cloudProvider: - aws\n clusterCIDR: 100.96.0.0/11\n clusterName: mixedinstances.example.com\n configureCloudRoutes: - false\n image: k8s.gcr.io/kube-controller-manager:v1.21.0\n leaderElection:\n - \ leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: true\nkubeProxy:\n - \ clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: '@aws'\n image: - k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: 2\nkubeScheduler:\n image: k8s.gcr.io/kube-scheduler:v1.21.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\nkubelet:\n anonymousAuth: - false\n cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: - 100.64.0.10\n clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\nmasterKubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n registerSchedulable: false\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - mixedinstances.example.com\nConfigBase: memfs://clusters.example.com/mixedinstances.example.com\nInstanceGroupName: - master-us-test-1b\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\n - \ registerSchedulable: false\nchannels:\n- memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/main.yaml\n- - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatemasterustest1cmastersmixedinstancesexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nencryptionConfig: null\netcdClusters:\n events:\n version: - 3.4.13\n main:\n version: 3.4.13\nkubeAPIServer:\n allowPrivileged: true\n - \ anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n apiServerCount: - 3\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n cloudProvider: aws\n - \ enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - ServiceAccount\n - \ - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.21.0\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.mixedinstances.example.com\n - \ serviceAccountJWKSURI: https://api.internal.mixedinstances.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n storageBackend: etcd3\nkubeControllerManager:\n - \ allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: 1m0s\n cloudProvider: - aws\n clusterCIDR: 100.96.0.0/11\n clusterName: mixedinstances.example.com\n configureCloudRoutes: - false\n image: k8s.gcr.io/kube-controller-manager:v1.21.0\n leaderElection:\n - \ leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: true\nkubeProxy:\n - \ clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: '@aws'\n image: - k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: 2\nkubeScheduler:\n image: k8s.gcr.io/kube-scheduler:v1.21.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\nkubelet:\n anonymousAuth: - false\n cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: - 100.64.0.10\n clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\nmasterKubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n registerSchedulable: false\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - mixedinstances.example.com\nConfigBase: memfs://clusters.example.com/mixedinstances.example.com\nInstanceGroupName: - master-us-test-1c\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\n - \ registerSchedulable: false\nchannels:\n- memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/main.yaml\n- - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatenodesmixedinstancesexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n - \ hostnameOverride: '@aws'\n image: k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: - 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: systemd\n cgroupRoot: /\n - \ cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: cluster.local\n - \ enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n\n__EOF_CLUSTER_SPEC\n\ncat > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat - > conf/kube_env.yaml << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\nClusterName: - mixedinstances.example.com\nConfigBase: memfs://clusters.example.com/mixedinstances.example.com\nInstanceGroupName: - nodes\nInstanceGroupRole: Node\nKubeletConfig:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kubernetes.io/role: node\n node-role.kubernetes.io/node: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\nchannels:\n- - memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" +Resources.AWSEC2LaunchTemplatemasterustest1amastersmixedinstancesexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.mixedinstances.example.com + serviceAccountJWKSURI: https://api.internal.mixedinstances.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: mixedinstances.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: mixedinstances.example.com + ConfigBase: memfs://clusters.example.com/mixedinstances.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatemasterustest1bmastersmixedinstancesexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.mixedinstances.example.com + serviceAccountJWKSURI: https://api.internal.mixedinstances.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: mixedinstances.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: mixedinstances.example.com + ConfigBase: memfs://clusters.example.com/mixedinstances.example.com + InstanceGroupName: master-us-test-1b + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatemasterustest1cmastersmixedinstancesexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.mixedinstances.example.com + serviceAccountJWKSURI: https://api.internal.mixedinstances.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: mixedinstances.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: mixedinstances.example.com + ConfigBase: memfs://clusters.example.com/mixedinstances.example.com + InstanceGroupName: master-us-test-1c + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodesmixedinstancesexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + ClusterName: mixedinstances.example.com + ConfigBase: memfs://clusters.example.com/mixedinstances.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1a.masters.mixedinstances.example.com_user_data b/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1a.masters.mixedinstances.example.com_user_data index e05894bfaf..28e9ea7db9 100644 --- a/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1a.masters.mixedinstances.example.com_user_data +++ b/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1a.masters.mixedinstances.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1b.masters.mixedinstances.example.com_user_data b/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1b.masters.mixedinstances.example.com_user_data index 480d0b8d8d..4706226081 100644 --- a/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1b.masters.mixedinstances.example.com_user_data +++ b/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1b.masters.mixedinstances.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1c.masters.mixedinstances.example.com_user_data b/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1c.masters.mixedinstances.example.com_user_data index 4485bde392..b83e3c377a 100644 --- a/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1c.masters.mixedinstances.example.com_user_data +++ b/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1c.masters.mixedinstances.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_nodes.mixedinstances.example.com_user_data b/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_nodes.mixedinstances.example.com_user_data index ead5a024d2..a6b4293e01 100644 --- a/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_nodes.mixedinstances.example.com_user_data +++ b/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_nodes.mixedinstances.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/nth_sqs_resources/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/nth_sqs_resources/cloudformation.json.extracted.yaml index 168f6dd5d4..4c8f062da9 100644 --- a/tests/integration/update_cluster/nth_sqs_resources/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/nth_sqs_resources/cloudformation.json.extracted.yaml @@ -1,174 +1,534 @@ -Resources.AWSEC2LaunchTemplatemasterustest1amastersnthsqsresourcesexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nencryptionConfig: null\netcdClusters:\n events:\n version: - 3.4.13\n main:\n version: 3.4.13\nkubeAPIServer:\n allowPrivileged: true\n - \ anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n apiServerCount: - 1\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n cloudProvider: aws\n - \ enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - ServiceAccount\n - \ - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.20.0\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.nthsqsresources.example.com\n - \ serviceAccountJWKSURI: https://api.internal.nthsqsresources.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n storageBackend: etcd3\nkubeControllerManager:\n - \ allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: 1m0s\n cloudProvider: - aws\n clusterCIDR: 100.96.0.0/11\n clusterName: nthsqsresources.example.com\n - \ configureCloudRoutes: false\n image: k8s.gcr.io/kube-controller-manager:v1.20.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: - true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: - '@aws'\n image: k8s.gcr.io/kube-proxy:v1.20.0\n logLevel: 2\nkubeScheduler:\n - \ image: k8s.gcr.io/kube-scheduler:v1.20.0\n leaderElection:\n leaderElect: - true\n logLevel: 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: systemd\n - \ cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\nmasterKubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n registerSchedulable: false\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - ff2422571c4c1e9696e367f5f25466b96fb6e501f28aed29f414b1524a52dea0@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubelet\n - \ - a5895007f331f08d2e082eb12458764949559f30bcc5beae26c38f3e2724262c@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 47ab6c4273fc3bb0cb8ec9517271d915890c5a6b0e54b2991e7a8fbbe77b06e4@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubelet\n - \ - 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - nthsqsresources.example.com\nConfigBase: memfs://clusters.example.com/nthsqsresources.example.com\nInstanceGroupName: - master-us-test-1a\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\n - \ registerSchedulable: false\nchannels:\n- memfs://clusters.example.com/nthsqsresources.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/nthsqsresources.example.com/manifests/etcd/main.yaml\n- - memfs://clusters.example.com/nthsqsresources.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatenodesnthsqsresourcesexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n - \ hostnameOverride: '@aws'\n image: k8s.gcr.io/kube-proxy:v1.20.0\n logLevel: - 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: systemd\n cgroupRoot: /\n - \ cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: cluster.local\n - \ enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n\n__EOF_CLUSTER_SPEC\n\ncat > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat - > conf/kube_env.yaml << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - ff2422571c4c1e9696e367f5f25466b96fb6e501f28aed29f414b1524a52dea0@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubelet\n - \ - a5895007f331f08d2e082eb12458764949559f30bcc5beae26c38f3e2724262c@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ arm64:\n - 47ab6c4273fc3bb0cb8ec9517271d915890c5a6b0e54b2991e7a8fbbe77b06e4@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubelet\n - \ - 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\nClusterName: - nthsqsresources.example.com\nConfigBase: memfs://clusters.example.com/nthsqsresources.example.com\nInstanceGroupName: - nodes\nInstanceGroupRole: Node\nKubeletConfig:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kubernetes.io/role: node\n node-role.kubernetes.io/node: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\nchannels:\n- - memfs://clusters.example.com/nthsqsresources.example.com/addons/bootstrap-channel.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" +Resources.AWSEC2LaunchTemplatemasterustest1amastersnthsqsresourcesexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.20.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.nthsqsresources.example.com + serviceAccountJWKSURI: https://api.internal.nthsqsresources.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: nthsqsresources.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.20.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.20.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.20.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - ff2422571c4c1e9696e367f5f25466b96fb6e501f28aed29f414b1524a52dea0@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubelet + - a5895007f331f08d2e082eb12458764949559f30bcc5beae26c38f3e2724262c@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 47ab6c4273fc3bb0cb8ec9517271d915890c5a6b0e54b2991e7a8fbbe77b06e4@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubelet + - 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: nthsqsresources.example.com + ConfigBase: memfs://clusters.example.com/nthsqsresources.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/nthsqsresources.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/nthsqsresources.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/nthsqsresources.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodesnthsqsresourcesexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.20.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - ff2422571c4c1e9696e367f5f25466b96fb6e501f28aed29f414b1524a52dea0@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubelet + - a5895007f331f08d2e082eb12458764949559f30bcc5beae26c38f3e2724262c@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + arm64: + - 47ab6c4273fc3bb0cb8ec9517271d915890c5a6b0e54b2991e7a8fbbe77b06e4@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubelet + - 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623@https://storage.googleapis.com/kubernetes-release/release/v1.20.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + ClusterName: nthsqsresources.example.com + ConfigBase: memfs://clusters.example.com/nthsqsresources.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/nthsqsresources.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_master-us-test-1a.masters.nthsqsresources.example.com_user_data b/tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_master-us-test-1a.masters.nthsqsresources.example.com_user_data index 8ee778ce30..c8fbf9be5f 100644 --- a/tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_master-us-test-1a.masters.nthsqsresources.example.com_user_data +++ b/tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_master-us-test-1a.masters.nthsqsresources.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_nodes.nthsqsresources.example.com_user_data b/tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_nodes.nthsqsresources.example.com_user_data index 435fc3d16e..19ae94e583 100644 --- a/tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_nodes.nthsqsresources.example.com_user_data +++ b/tests/integration/update_cluster/nth_sqs_resources/data/aws_launch_template_nodes.nthsqsresources.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/private-shared-ip/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/private-shared-ip/cloudformation.json.extracted.yaml index 2a0ecd29a6..1b18f47cf2 100644 --- a/tests/integration/update_cluster/private-shared-ip/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/private-shared-ip/cloudformation.json.extracted.yaml @@ -1,175 +1,535 @@ Resources.AWSEC2LaunchTemplatebastionprivatesharedipexamplecom.Properties.LaunchTemplateData.UserData: "" -Resources.AWSEC2LaunchTemplatemasterustest1amastersprivatesharedipexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nencryptionConfig: null\netcdClusters:\n events:\n version: - 3.4.13\n main:\n version: 3.4.13\nkubeAPIServer:\n allowPrivileged: true\n - \ anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n apiServerCount: - 1\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n cloudProvider: aws\n - \ enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - ServiceAccount\n - \ - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.21.0\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.private-shared-ip.example.com\n - \ serviceAccountJWKSURI: https://api.internal.private-shared-ip.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n storageBackend: etcd3\nkubeControllerManager:\n - \ allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: 1m0s\n cloudProvider: - aws\n clusterCIDR: 100.96.0.0/11\n clusterName: private-shared-ip.example.com\n - \ configureCloudRoutes: false\n image: k8s.gcr.io/kube-controller-manager:v1.21.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: - true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: - '@aws'\n image: k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: 2\nkubeScheduler:\n - \ image: k8s.gcr.io/kube-scheduler:v1.21.0\n leaderElection:\n leaderElect: - true\n logLevel: 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: systemd\n - \ cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\nmasterKubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n registerSchedulable: false\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - private-shared-ip.example.com\nConfigBase: memfs://clusters.example.com/private-shared-ip.example.com\nInstanceGroupName: - master-us-test-1a\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\n - \ registerSchedulable: false\nchannels:\n- memfs://clusters.example.com/private-shared-ip.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/private-shared-ip.example.com/manifests/etcd/main.yaml\n- - memfs://clusters.example.com/private-shared-ip.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatenodesprivatesharedipexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n - \ hostnameOverride: '@aws'\n image: k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: - 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: systemd\n cgroupRoot: /\n - \ cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: cluster.local\n - \ enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n\n__EOF_CLUSTER_SPEC\n\ncat > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat - > conf/kube_env.yaml << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\nClusterName: - private-shared-ip.example.com\nConfigBase: memfs://clusters.example.com/private-shared-ip.example.com\nInstanceGroupName: - nodes\nInstanceGroupRole: Node\nKubeletConfig:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kubernetes.io/role: node\n node-role.kubernetes.io/node: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\nchannels:\n- - memfs://clusters.example.com/private-shared-ip.example.com/addons/bootstrap-channel.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" +Resources.AWSEC2LaunchTemplatemasterustest1amastersprivatesharedipexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.private-shared-ip.example.com + serviceAccountJWKSURI: https://api.internal.private-shared-ip.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: private-shared-ip.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: private-shared-ip.example.com + ConfigBase: memfs://clusters.example.com/private-shared-ip.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/private-shared-ip.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/private-shared-ip.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/private-shared-ip.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodesprivatesharedipexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + ClusterName: private-shared-ip.example.com + ConfigBase: memfs://clusters.example.com/private-shared-ip.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/private-shared-ip.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/private-shared-ip/data/aws_launch_template_master-us-test-1a.masters.private-shared-ip.example.com_user_data b/tests/integration/update_cluster/private-shared-ip/data/aws_launch_template_master-us-test-1a.masters.private-shared-ip.example.com_user_data index 3b5b38be33..369d3de66f 100644 --- a/tests/integration/update_cluster/private-shared-ip/data/aws_launch_template_master-us-test-1a.masters.private-shared-ip.example.com_user_data +++ b/tests/integration/update_cluster/private-shared-ip/data/aws_launch_template_master-us-test-1a.masters.private-shared-ip.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/private-shared-ip/data/aws_launch_template_nodes.private-shared-ip.example.com_user_data b/tests/integration/update_cluster/private-shared-ip/data/aws_launch_template_nodes.private-shared-ip.example.com_user_data index e53f4968fb..c8dbb9ab1b 100644 --- a/tests/integration/update_cluster/private-shared-ip/data/aws_launch_template_nodes.private-shared-ip.example.com_user_data +++ b/tests/integration/update_cluster/private-shared-ip/data/aws_launch_template_nodes.private-shared-ip.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/private-shared-subnet/data/aws_launch_template_master-us-test-1a.masters.private-shared-subnet.example.com_user_data b/tests/integration/update_cluster/private-shared-subnet/data/aws_launch_template_master-us-test-1a.masters.private-shared-subnet.example.com_user_data index 682a821da0..b24db11597 100644 --- a/tests/integration/update_cluster/private-shared-subnet/data/aws_launch_template_master-us-test-1a.masters.private-shared-subnet.example.com_user_data +++ b/tests/integration/update_cluster/private-shared-subnet/data/aws_launch_template_master-us-test-1a.masters.private-shared-subnet.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/private-shared-subnet/data/aws_launch_template_nodes.private-shared-subnet.example.com_user_data b/tests/integration/update_cluster/private-shared-subnet/data/aws_launch_template_nodes.private-shared-subnet.example.com_user_data index c72e346331..ca765c1636 100644 --- a/tests/integration/update_cluster/private-shared-subnet/data/aws_launch_template_nodes.private-shared-subnet.example.com_user_data +++ b/tests/integration/update_cluster/private-shared-subnet/data/aws_launch_template_nodes.private-shared-subnet.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/privatecalico/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/privatecalico/cloudformation.json.extracted.yaml index f85b63b3fc..50a5db0694 100644 --- a/tests/integration/update_cluster/privatecalico/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/privatecalico/cloudformation.json.extracted.yaml @@ -1,175 +1,535 @@ Resources.AWSEC2LaunchTemplatebastionprivatecalicoexamplecom.Properties.LaunchTemplateData.UserData: "" -Resources.AWSEC2LaunchTemplatemasterustest1amastersprivatecalicoexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nencryptionConfig: null\netcdClusters:\n events:\n version: - 3.4.13\n main:\n version: 3.4.13\nkubeAPIServer:\n allowPrivileged: true\n - \ anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n apiServerCount: - 1\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n cloudProvider: aws\n - \ enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - ServiceAccount\n - \ - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.21.0\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.privatecalico.example.com\n - \ serviceAccountJWKSURI: https://api.internal.privatecalico.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n storageBackend: etcd3\nkubeControllerManager:\n - \ allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: 1m0s\n cloudProvider: - aws\n clusterCIDR: 100.96.0.0/11\n clusterName: privatecalico.example.com\n configureCloudRoutes: - false\n image: k8s.gcr.io/kube-controller-manager:v1.21.0\n leaderElection:\n - \ leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: true\nkubeProxy:\n - \ clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: '@aws'\n image: - k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: 2\nkubeScheduler:\n image: k8s.gcr.io/kube-scheduler:v1.21.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\nkubelet:\n anonymousAuth: - false\n cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: - 100.64.0.10\n clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\nmasterKubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n registerSchedulable: false\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - privatecalico.example.com\nConfigBase: memfs://clusters.example.com/privatecalico.example.com\nInstanceGroupName: - master-us-test-1a\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\n - \ registerSchedulable: false\nchannels:\n- memfs://clusters.example.com/privatecalico.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/privatecalico.example.com/manifests/etcd/main.yaml\n- - memfs://clusters.example.com/privatecalico.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatenodesprivatecalicoexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n - \ hostnameOverride: '@aws'\n image: k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: - 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: systemd\n cgroupRoot: /\n - \ cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: cluster.local\n - \ enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n\n__EOF_CLUSTER_SPEC\n\ncat > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat - > conf/kube_env.yaml << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\nClusterName: - privatecalico.example.com\nConfigBase: memfs://clusters.example.com/privatecalico.example.com\nInstanceGroupName: - nodes\nInstanceGroupRole: Node\nKubeletConfig:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kubernetes.io/role: node\n node-role.kubernetes.io/node: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\nchannels:\n- - memfs://clusters.example.com/privatecalico.example.com/addons/bootstrap-channel.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" +Resources.AWSEC2LaunchTemplatemasterustest1amastersprivatecalicoexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.privatecalico.example.com + serviceAccountJWKSURI: https://api.internal.privatecalico.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: privatecalico.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: privatecalico.example.com + ConfigBase: memfs://clusters.example.com/privatecalico.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/privatecalico.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/privatecalico.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/privatecalico.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodesprivatecalicoexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + ClusterName: privatecalico.example.com + ConfigBase: memfs://clusters.example.com/privatecalico.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/privatecalico.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privatecalico/data/aws_launch_template_master-us-test-1a.masters.privatecalico.example.com_user_data b/tests/integration/update_cluster/privatecalico/data/aws_launch_template_master-us-test-1a.masters.privatecalico.example.com_user_data index c2d95a6225..b8fe41d957 100644 --- a/tests/integration/update_cluster/privatecalico/data/aws_launch_template_master-us-test-1a.masters.privatecalico.example.com_user_data +++ b/tests/integration/update_cluster/privatecalico/data/aws_launch_template_master-us-test-1a.masters.privatecalico.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/privatecalico/data/aws_launch_template_nodes.privatecalico.example.com_user_data b/tests/integration/update_cluster/privatecalico/data/aws_launch_template_nodes.privatecalico.example.com_user_data index 8feab314e8..2f4b710674 100644 --- a/tests/integration/update_cluster/privatecalico/data/aws_launch_template_nodes.privatecalico.example.com_user_data +++ b/tests/integration/update_cluster/privatecalico/data/aws_launch_template_nodes.privatecalico.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/privatecanal/data/aws_launch_template_master-us-test-1a.masters.privatecanal.example.com_user_data b/tests/integration/update_cluster/privatecanal/data/aws_launch_template_master-us-test-1a.masters.privatecanal.example.com_user_data index 43b17eef2a..f9f6d13f12 100644 --- a/tests/integration/update_cluster/privatecanal/data/aws_launch_template_master-us-test-1a.masters.privatecanal.example.com_user_data +++ b/tests/integration/update_cluster/privatecanal/data/aws_launch_template_master-us-test-1a.masters.privatecanal.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/privatecanal/data/aws_launch_template_nodes.privatecanal.example.com_user_data b/tests/integration/update_cluster/privatecanal/data/aws_launch_template_nodes.privatecanal.example.com_user_data index d29cb4ba3a..564c029a00 100644 --- a/tests/integration/update_cluster/privatecanal/data/aws_launch_template_nodes.privatecanal.example.com_user_data +++ b/tests/integration/update_cluster/privatecanal/data/aws_launch_template_nodes.privatecanal.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/privatecilium/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/privatecilium/cloudformation.json.extracted.yaml index 3172aee926..78fb6c7ae8 100644 --- a/tests/integration/update_cluster/privatecilium/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/privatecilium/cloudformation.json.extracted.yaml @@ -1,175 +1,535 @@ Resources.AWSEC2LaunchTemplatebastionprivateciliumexamplecom.Properties.LaunchTemplateData.UserData: "" -Resources.AWSEC2LaunchTemplatemasterustest1amastersprivateciliumexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nencryptionConfig: null\netcdClusters:\n events:\n version: - 3.4.13\n main:\n version: 3.4.13\nkubeAPIServer:\n allowPrivileged: true\n - \ anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n apiServerCount: - 1\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n cloudProvider: aws\n - \ enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - ServiceAccount\n - \ - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.21.0\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.privatecilium.example.com\n - \ serviceAccountJWKSURI: https://api.internal.privatecilium.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n storageBackend: etcd3\nkubeControllerManager:\n - \ allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: 1m0s\n cloudProvider: - aws\n clusterCIDR: 100.96.0.0/11\n clusterName: privatecilium.example.com\n configureCloudRoutes: - false\n image: k8s.gcr.io/kube-controller-manager:v1.21.0\n leaderElection:\n - \ leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: true\nkubeProxy:\n - \ clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: '@aws'\n image: - k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: 2\nkubeScheduler:\n image: k8s.gcr.io/kube-scheduler:v1.21.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\nkubelet:\n anonymousAuth: - false\n cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: - 100.64.0.10\n clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\nmasterKubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n registerSchedulable: false\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - privatecilium.example.com\nConfigBase: memfs://clusters.example.com/privatecilium.example.com\nInstanceGroupName: - master-us-test-1a\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\n - \ registerSchedulable: false\nchannels:\n- memfs://clusters.example.com/privatecilium.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/privatecilium.example.com/manifests/etcd/main.yaml\n- - memfs://clusters.example.com/privatecilium.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatenodesprivateciliumexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n - \ hostnameOverride: '@aws'\n image: k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: - 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: systemd\n cgroupRoot: /\n - \ cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: cluster.local\n - \ enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n\n__EOF_CLUSTER_SPEC\n\ncat > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat - > conf/kube_env.yaml << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\nClusterName: - privatecilium.example.com\nConfigBase: memfs://clusters.example.com/privatecilium.example.com\nInstanceGroupName: - nodes\nInstanceGroupRole: Node\nKubeletConfig:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kubernetes.io/role: node\n node-role.kubernetes.io/node: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\nchannels:\n- - memfs://clusters.example.com/privatecilium.example.com/addons/bootstrap-channel.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" +Resources.AWSEC2LaunchTemplatemasterustest1amastersprivateciliumexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.privatecilium.example.com + serviceAccountJWKSURI: https://api.internal.privatecilium.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: privatecilium.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: privatecilium.example.com + ConfigBase: memfs://clusters.example.com/privatecilium.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/privatecilium.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/privatecilium.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/privatecilium.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodesprivateciliumexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + ClusterName: privatecilium.example.com + ConfigBase: memfs://clusters.example.com/privatecilium.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/privatecilium.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privatecilium/data/aws_launch_template_master-us-test-1a.masters.privatecilium.example.com_user_data b/tests/integration/update_cluster/privatecilium/data/aws_launch_template_master-us-test-1a.masters.privatecilium.example.com_user_data index 91ad581df2..f90c2522f5 100644 --- a/tests/integration/update_cluster/privatecilium/data/aws_launch_template_master-us-test-1a.masters.privatecilium.example.com_user_data +++ b/tests/integration/update_cluster/privatecilium/data/aws_launch_template_master-us-test-1a.masters.privatecilium.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/privatecilium/data/aws_launch_template_nodes.privatecilium.example.com_user_data b/tests/integration/update_cluster/privatecilium/data/aws_launch_template_nodes.privatecilium.example.com_user_data index 39ae7233f2..485dc026c9 100644 --- a/tests/integration/update_cluster/privatecilium/data/aws_launch_template_nodes.privatecilium.example.com_user_data +++ b/tests/integration/update_cluster/privatecilium/data/aws_launch_template_nodes.privatecilium.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/privatecilium2/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/privatecilium2/cloudformation.json.extracted.yaml index 3bd68e4397..fc0e7aaf56 100644 --- a/tests/integration/update_cluster/privatecilium2/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/privatecilium2/cloudformation.json.extracted.yaml @@ -1,170 +1,517 @@ Resources.AWSEC2LaunchTemplatebastionprivateciliumexamplecom.Properties.LaunchTemplateData.UserData: "" -Resources.AWSEC2LaunchTemplatemasterustest1amastersprivateciliumexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - docker\ncontainerd:\n configOverride: |\n disabled_plugins = [\"cri\"]\n logLevel: - info\ndocker:\n ipMasq: false\n ipTables: false\n logDriver: json-file\n logLevel: - info\n logOpt:\n - max-size=10m\n - max-file=5\n storage: overlay2,overlay,aufs\n - \ version: 19.03.15\nencryptionConfig: null\netcdClusters:\n events:\n version: - 3.4.3\n main:\n version: 3.4.3\nkubeAPIServer:\n allowPrivileged: true\n anonymousAuth: - false\n apiServerCount: 1\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n - \ cloudProvider: aws\n enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - \ - ServiceAccount\n - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.17.15\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceClusterIPRange: 100.64.0.0/13\n storageBackend: - etcd3\nkubeControllerManager:\n allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: - 1m0s\n cloudProvider: aws\n clusterCIDR: 100.96.0.0/11\n clusterName: privatecilium.example.com\n - \ configureCloudRoutes: false\n image: k8s.gcr.io/kube-controller-manager:v1.17.15\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: - true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n hostnameOverride: - '@aws'\n image: k8s.gcr.io/kube-proxy:v1.17.15\n logLevel: 2\nkubeScheduler:\n - \ image: k8s.gcr.io/kube-scheduler:v1.17.15\n leaderElection:\n leaderElect: - true\n logLevel: 2\nkubelet:\n anonymousAuth: false\n cgroupRoot: /\n cloudProvider: - aws\n clusterDNS: 100.64.0.10\n clusterDomain: cluster.local\n enableDebuggingHandlers: - true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podInfraContainerImage: - k8s.gcr.io/pause:3.2\n podManifestPath: /etc/kubernetes/manifests\nmasterKubelet:\n - \ anonymousAuth: false\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podInfraContainerImage: - k8s.gcr.io/pause:3.2\n podManifestPath: /etc/kubernetes/manifests\n registerSchedulable: - false\n\n__EOF_CLUSTER_SPEC\n\ncat > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat - > conf/kube_env.yaml << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 090a2a9829f1c5913672b679bfa24a97d434cbf4bc1edd05f4a3a37f97dfeb75@https://storage.googleapis.com/kubernetes-release/release/v1.17.15/bin/linux/amd64/kubelet\n - \ - a94f33ab8c5c68a2d9f177ad1e6654bbbd7ea52e80ed7ed9938fe6d6ca1f7d26@https://storage.googleapis.com/kubernetes-release/release/v1.17.15/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 5504d190eef37355231325c176686d51ade6e0cabe2da526d561a38d8611506f@https://download.docker.com/linux/static/stable/x86_64/docker-19.03.15.tgz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 3515d2ec6698371f1dcaeb3ff79829a0e04a277d738a33d844249b33678306c6@https://storage.googleapis.com/kubernetes-release/release/v1.17.15/bin/linux/arm64/kubelet\n - \ - a75af21eae2913aacd521cc8a052f7b9f1cb8b195f7bffbab478833abe024b0e@https://storage.googleapis.com/kubernetes-release/release/v1.17.15/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 264f3396630507606a8646fda6a28a98d3ced8927df84be8ee9a74ab73cc1566@https://download.docker.com/linux/static/stable/aarch64/docker-19.03.15.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - privatecilium.example.com\nConfigBase: memfs://clusters.example.com/privatecilium.example.com\nInstanceGroupName: - master-us-test-1a\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podInfraContainerImage: k8s.gcr.io/pause:3.2\n - \ podManifestPath: /etc/kubernetes/manifests\n registerSchedulable: false\nchannels:\n- - memfs://clusters.example.com/privatecilium.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/privatecilium.example.com/manifests/etcd/main.yaml\n- - memfs://clusters.example.com/privatecilium.example.com/manifests/etcd/events.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatenodesprivateciliumexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - docker\ncontainerd:\n configOverride: |\n disabled_plugins = [\"cri\"]\n logLevel: - info\ndocker:\n ipMasq: false\n ipTables: false\n logDriver: json-file\n logLevel: - info\n logOpt:\n - max-size=10m\n - max-file=5\n storage: overlay2,overlay,aufs\n - \ version: 19.03.15\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n - \ hostnameOverride: '@aws'\n image: k8s.gcr.io/kube-proxy:v1.17.15\n logLevel: - 2\nkubelet:\n anonymousAuth: false\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: - 100.64.0.10\n clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podInfraContainerImage: - k8s.gcr.io/pause:3.2\n podManifestPath: /etc/kubernetes/manifests\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 090a2a9829f1c5913672b679bfa24a97d434cbf4bc1edd05f4a3a37f97dfeb75@https://storage.googleapis.com/kubernetes-release/release/v1.17.15/bin/linux/amd64/kubelet\n - \ - a94f33ab8c5c68a2d9f177ad1e6654bbbd7ea52e80ed7ed9938fe6d6ca1f7d26@https://storage.googleapis.com/kubernetes-release/release/v1.17.15/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 5504d190eef37355231325c176686d51ade6e0cabe2da526d561a38d8611506f@https://download.docker.com/linux/static/stable/x86_64/docker-19.03.15.tgz\n - \ arm64:\n - 3515d2ec6698371f1dcaeb3ff79829a0e04a277d738a33d844249b33678306c6@https://storage.googleapis.com/kubernetes-release/release/v1.17.15/bin/linux/arm64/kubelet\n - \ - a75af21eae2913aacd521cc8a052f7b9f1cb8b195f7bffbab478833abe024b0e@https://storage.googleapis.com/kubernetes-release/release/v1.17.15/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 264f3396630507606a8646fda6a28a98d3ced8927df84be8ee9a74ab73cc1566@https://download.docker.com/linux/static/stable/aarch64/docker-19.03.15.tgz\nClusterName: - privatecilium.example.com\nConfigBase: memfs://clusters.example.com/privatecilium.example.com\nInstanceGroupName: - nodes\nInstanceGroupRole: Node\nKubeletConfig:\n anonymousAuth: false\n cgroupRoot: - /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: cluster.local\n - \ enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kubernetes.io/role: node\n node-role.kubernetes.io/node: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podInfraContainerImage: k8s.gcr.io/pause:3.2\n - \ podManifestPath: /etc/kubernetes/manifests\nchannels:\n- memfs://clusters.example.com/privatecilium.example.com/addons/bootstrap-channel.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" +Resources.AWSEC2LaunchTemplatemasterustest1amastersprivateciliumexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: docker + containerd: + configOverride: | + disabled_plugins = ["cri"] + logLevel: info + docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 19.03.15 + encryptionConfig: null + etcdClusters: + events: + version: 3.4.3 + main: + version: 3.4.3 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.17.15 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: privatecilium.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.17.15 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.17.15 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.17.15 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 090a2a9829f1c5913672b679bfa24a97d434cbf4bc1edd05f4a3a37f97dfeb75@https://storage.googleapis.com/kubernetes-release/release/v1.17.15/bin/linux/amd64/kubelet + - a94f33ab8c5c68a2d9f177ad1e6654bbbd7ea52e80ed7ed9938fe6d6ca1f7d26@https://storage.googleapis.com/kubernetes-release/release/v1.17.15/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 5504d190eef37355231325c176686d51ade6e0cabe2da526d561a38d8611506f@https://download.docker.com/linux/static/stable/x86_64/docker-19.03.15.tgz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 3515d2ec6698371f1dcaeb3ff79829a0e04a277d738a33d844249b33678306c6@https://storage.googleapis.com/kubernetes-release/release/v1.17.15/bin/linux/arm64/kubelet + - a75af21eae2913aacd521cc8a052f7b9f1cb8b195f7bffbab478833abe024b0e@https://storage.googleapis.com/kubernetes-release/release/v1.17.15/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 264f3396630507606a8646fda6a28a98d3ced8927df84be8ee9a74ab73cc1566@https://download.docker.com/linux/static/stable/aarch64/docker-19.03.15.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: privatecilium.example.com + ConfigBase: memfs://clusters.example.com/privatecilium.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/privatecilium.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/privatecilium.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/privatecilium.example.com/manifests/etcd/events.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodesprivateciliumexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: docker + containerd: + configOverride: | + disabled_plugins = ["cri"] + logLevel: info + docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 19.03.15 + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.17.15 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 090a2a9829f1c5913672b679bfa24a97d434cbf4bc1edd05f4a3a37f97dfeb75@https://storage.googleapis.com/kubernetes-release/release/v1.17.15/bin/linux/amd64/kubelet + - a94f33ab8c5c68a2d9f177ad1e6654bbbd7ea52e80ed7ed9938fe6d6ca1f7d26@https://storage.googleapis.com/kubernetes-release/release/v1.17.15/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 5504d190eef37355231325c176686d51ade6e0cabe2da526d561a38d8611506f@https://download.docker.com/linux/static/stable/x86_64/docker-19.03.15.tgz + arm64: + - 3515d2ec6698371f1dcaeb3ff79829a0e04a277d738a33d844249b33678306c6@https://storage.googleapis.com/kubernetes-release/release/v1.17.15/bin/linux/arm64/kubelet + - a75af21eae2913aacd521cc8a052f7b9f1cb8b195f7bffbab478833abe024b0e@https://storage.googleapis.com/kubernetes-release/release/v1.17.15/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 264f3396630507606a8646fda6a28a98d3ced8927df84be8ee9a74ab73cc1566@https://download.docker.com/linux/static/stable/aarch64/docker-19.03.15.tgz + ClusterName: privatecilium.example.com + ConfigBase: memfs://clusters.example.com/privatecilium.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/privatecilium.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privatecilium2/data/aws_launch_template_master-us-test-1a.masters.privatecilium.example.com_user_data b/tests/integration/update_cluster/privatecilium2/data/aws_launch_template_master-us-test-1a.masters.privatecilium.example.com_user_data index 842bc70391..2493899819 100644 --- a/tests/integration/update_cluster/privatecilium2/data/aws_launch_template_master-us-test-1a.masters.privatecilium.example.com_user_data +++ b/tests/integration/update_cluster/privatecilium2/data/aws_launch_template_master-us-test-1a.masters.privatecilium.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/privatecilium2/data/aws_launch_template_nodes.privatecilium.example.com_user_data b/tests/integration/update_cluster/privatecilium2/data/aws_launch_template_nodes.privatecilium.example.com_user_data index 47cc051944..c6b80a298e 100644 --- a/tests/integration/update_cluster/privatecilium2/data/aws_launch_template_nodes.privatecilium.example.com_user_data +++ b/tests/integration/update_cluster/privatecilium2/data/aws_launch_template_nodes.privatecilium.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/privateciliumadvanced/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/privateciliumadvanced/cloudformation.json.extracted.yaml index a4514103b6..0d367c997e 100644 --- a/tests/integration/update_cluster/privateciliumadvanced/cloudformation.json.extracted.yaml +++ b/tests/integration/update_cluster/privateciliumadvanced/cloudformation.json.extracted.yaml @@ -1,176 +1,540 @@ Resources.AWSEC2LaunchTemplatebastionprivateciliumadvancedexamplecom.Properties.LaunchTemplateData.UserData: "" -Resources.AWSEC2LaunchTemplatemasterustest1amastersprivateciliumadvancedexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nencryptionConfig: null\netcdClusters:\n cilium:\n version: - 3.4.13\n events:\n version: 3.4.13\n main:\n version: 3.4.13\nkubeAPIServer:\n - \ allowPrivileged: true\n anonymousAuth: false\n apiAudiences:\n - kubernetes.svc.default\n - \ apiServerCount: 1\n authorizationMode: AlwaysAllow\n bindAddress: 0.0.0.0\n - \ cloudProvider: aws\n enableAdmissionPlugins:\n - NamespaceLifecycle\n - LimitRanger\n - \ - ServiceAccount\n - PersistentVolumeLabel\n - DefaultStorageClass\n - DefaultTolerationSeconds\n - \ - MutatingAdmissionWebhook\n - ValidatingAdmissionWebhook\n - NodeRestriction\n - \ - ResourceQuota\n etcdServers:\n - https://127.0.0.1:4001\n etcdServersOverrides:\n - \ - /events#https://127.0.0.1:4002\n image: k8s.gcr.io/kube-apiserver:v1.21.0\n - \ kubeletPreferredAddressTypes:\n - InternalIP\n - Hostname\n - ExternalIP\n - \ logLevel: 2\n requestheaderAllowedNames:\n - aggregator\n requestheaderExtraHeaderPrefixes:\n - \ - X-Remote-Extra-\n requestheaderGroupHeaders:\n - X-Remote-Group\n requestheaderUsernameHeaders:\n - \ - X-Remote-User\n securePort: 443\n serviceAccountIssuer: https://api.internal.privateciliumadvanced.example.com\n - \ serviceAccountJWKSURI: https://api.internal.privateciliumadvanced.example.com/openid/v1/jwks\n - \ serviceClusterIPRange: 100.64.0.0/13\n storageBackend: etcd3\nkubeControllerManager:\n - \ allocateNodeCIDRs: true\n attachDetachReconcileSyncPeriod: 1m0s\n cloudProvider: - aws\n clusterCIDR: 100.96.0.0/11\n clusterName: privateciliumadvanced.example.com\n - \ configureCloudRoutes: false\n image: k8s.gcr.io/kube-controller-manager:v1.21.0\n - \ leaderElection:\n leaderElect: true\n logLevel: 2\n useServiceAccountCredentials: - true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n enabled: false\n - \ hostnameOverride: '@aws'\n image: k8s.gcr.io/kube-proxy:v1.21.0\n logLevel: - 2\nkubeScheduler:\n image: k8s.gcr.io/kube-scheduler:v1.21.0\n leaderElection:\n - \ leaderElect: true\n logLevel: 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\nmasterKubelet:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n registerSchedulable: false\n\n__EOF_CLUSTER_SPEC\n\ncat - > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat > conf/kube_env.yaml - << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64\n - \ - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\n - \ - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64\n - \ - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64\nClusterName: - privateciliumadvanced.example.com\nConfigBase: memfs://clusters.example.com/privateciliumadvanced.example.com\nInstanceGroupName: - master-us-test-1a\nInstanceGroupRole: Master\nKubeletConfig:\n anonymousAuth: false\n - \ cgroupDriver: systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n - \ clusterDomain: cluster.local\n enableDebuggingHandlers: true\n evictionHard: - memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kops.k8s.io/kops-controller-pki: - \"\"\n kubernetes.io/role: master\n node-role.kubernetes.io/control-plane: - \"\"\n node-role.kubernetes.io/master: \"\"\n node.kubernetes.io/exclude-from-external-load-balancers: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\n - \ registerSchedulable: false\nchannels:\n- memfs://clusters.example.com/privateciliumadvanced.example.com/addons/bootstrap-channel.yaml\netcdManifests:\n- - memfs://clusters.example.com/privateciliumadvanced.example.com/manifests/etcd/main.yaml\n- - memfs://clusters.example.com/privateciliumadvanced.example.com/manifests/etcd/events.yaml\n- - memfs://clusters.example.com/privateciliumadvanced.example.com/manifests/etcd/cilium.yaml\nstaticManifests:\n- - key: kube-apiserver-healthcheck\n path: manifests/static/kube-apiserver-healthcheck.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" -Resources.AWSEC2LaunchTemplatenodesprivateciliumadvancedexamplecom.Properties.LaunchTemplateData.UserData: "#!/bin/bash\nset - -o errexit\nset -o nounset\nset -o pipefail\n\nNODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64\nNODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924\nNODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64\nNODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865\n\nexport - AWS_REGION=us-test-1\n\n\n\n\nsysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' - || true\n\n\nfunction ensure-install-dir() {\n INSTALL_DIR=\"/opt/kops\"\n # On - ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec\n if [[ -d - /var/lib/toolbox ]]; then\n INSTALL_DIR=\"/var/lib/toolbox/kops\"\n fi\n mkdir - -p ${INSTALL_DIR}/bin\n mkdir -p ${INSTALL_DIR}/conf\n cd ${INSTALL_DIR}\n}\n\n# - Retry a download until we get it. args: name, sha, urls\ndownload-or-bust() {\n - \ local -r file=\"$1\"\n local -r hash=\"$2\"\n local -r urls=( $(split-commas - \"$3\") )\n\n if [[ -f \"${file}\" ]]; then\n\tif ! validate-hash \"${file}\" \"${hash}\"; - then\n\t rm -f \"${file}\"\n\telse\n\t return\n\tfi\n fi\n\n while true; do\n - \ for url in \"${urls[@]}\"; do\n commands=(\n \"curl -f --ipv4 --compressed - -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay 10\"\n \"wget - --inet4-only --compression=auto -O \"${file}\" --connect-timeout=20 --tries=6 --wait=10\"\n - \ \"curl -f --ipv4 -Lo \"${file}\" --connect-timeout 20 --retry 6 --retry-delay - 10\"\n \"wget --inet4-only -O \"${file}\" --connect-timeout=20 --tries=6 - --wait=10\"\n )\n for cmd in \"${commands[@]}\"; do\n echo \"Attempting - download with: ${cmd} {url}\"\n if ! (${cmd} \"${url}\"); then\n echo - \"== Download failed with ${cmd} ==\"\n continue\n fi\n if - ! validate-hash \"${file}\" \"${hash}\"; then\n echo \"== Hash validation - of ${url} failed. Retrying. ==\"\n rm -f \"${file}\"\n else\n echo - \"== Downloaded ${url} (SHA256 = ${hash}) ==\"\n return\n fi\n done\n - \ done\n\n echo \"All downloads failed; sleeping before retrying\"\n sleep - 60\n done\n}\n\nvalidate-hash() {\n local -r file=\"$1\"\n local -r expected=\"$2\"\n - \ local actual\n\n actual=$(sha256sum ${file} | awk '{ print $1 }') || true\n if - [[ \"${actual}\" != \"${expected}\" ]]; then\n echo \"== ${file} corrupted, hash - ${actual} doesn't match expected ${expected} ==\"\n return 1\n fi\n}\n\nfunction - split-commas() {\n echo $1 | tr \",\" \"\\n\"\n}\n\nfunction download-release() - {\n case \"$(uname -m)\" in\n x86_64*|i?86_64*|amd64*)\n NODEUP_URL=\"${NODEUP_URL_AMD64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_AMD64}\"\n ;;\n aarch64*|arm64*)\n NODEUP_URL=\"${NODEUP_URL_ARM64}\"\n - \ NODEUP_HASH=\"${NODEUP_HASH_ARM64}\"\n ;;\n *)\n echo \"Unsupported host - arch: $(uname -m)\" >&2\n exit 1\n ;;\n esac\n\n cd ${INSTALL_DIR}/bin\n - \ download-or-bust nodeup \"${NODEUP_HASH}\" \"${NODEUP_URL}\"\n\n chmod +x nodeup\n\n - \ echo \"Running nodeup\"\n # We can't run in the foreground because of https://github.com/docker/docker/issues/23793\n - \ ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml - --v=8 )\n}\n\n####################################################################################\n\n/bin/systemd-machine-id-setup - || echo \"failed to set up ensure machine-id configured\"\n\necho \"== nodeup node - config starting ==\"\nensure-install-dir\n\ncat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'\ncloudConfig:\n - \ awsEBSCSIDriver:\n enabled: false\n manageStorageClasses: true\ncontainerRuntime: - containerd\ncontainerd:\n configOverride: |\n version = 2\n\n [plugins]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\"]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n\n - \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n - \ runtime_type = \"io.containerd.runc.v2\"\n\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\n - \ SystemdCgroup = true\n logLevel: info\n version: 1.4.6\ndocker:\n - \ skipInstall: true\nkubeProxy:\n clusterCIDR: 100.96.0.0/11\n cpuRequest: 100m\n - \ enabled: false\n hostnameOverride: '@aws'\n image: k8s.gcr.io/kube-proxy:v1.21.0\n - \ logLevel: 2\nkubelet:\n anonymousAuth: false\n cgroupDriver: systemd\n cgroupRoot: - /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: cluster.local\n - \ enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: - /etc/kubernetes/manifests\n\n__EOF_CLUSTER_SPEC\n\ncat > conf/ig_spec.yaml << '__EOF_IG_SPEC'\n{}\n\n__EOF_IG_SPEC\n\ncat - > conf/kube_env.yaml << '__EOF_KUBE_ENV'\nAssets:\n amd64:\n - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet\n - \ - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl\n - \ - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz\n - \ - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz\n - \ arm64:\n - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet\n - \ - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl\n - \ - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz\n - \ - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz\nClusterName: - privateciliumadvanced.example.com\nConfigBase: memfs://clusters.example.com/privateciliumadvanced.example.com\nInstanceGroupName: - nodes\nInstanceGroupRole: Node\nKubeletConfig:\n anonymousAuth: false\n cgroupDriver: - systemd\n cgroupRoot: /\n cloudProvider: aws\n clusterDNS: 100.64.0.10\n clusterDomain: - cluster.local\n enableDebuggingHandlers: true\n evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%\n - \ hostnameOverride: '@aws'\n kubeconfigPath: /var/lib/kubelet/kubeconfig\n logLevel: - 2\n networkPluginName: cni\n nodeLabels:\n kubernetes.io/role: node\n node-role.kubernetes.io/node: - \"\"\n nonMasqueradeCIDR: 100.64.0.0/10\n podManifestPath: /etc/kubernetes/manifests\nchannels:\n- - memfs://clusters.example.com/privateciliumadvanced.example.com/addons/bootstrap-channel.yaml\n\n__EOF_KUBE_ENV\n\ndownload-release\necho - \"== nodeup node config done ==\"\n" +Resources.AWSEC2LaunchTemplatemasterustest1amastersprivateciliumadvancedexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + cilium: + version: 3.4.13 + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.privateciliumadvanced.example.com + serviceAccountJWKSURI: https://api.internal.privateciliumadvanced.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: privateciliumadvanced.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + enabled: false + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 + ClusterName: privateciliumadvanced.example.com + ConfigBase: memfs://clusters.example.com/privateciliumadvanced.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/privateciliumadvanced.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/privateciliumadvanced.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/privateciliumadvanced.example.com/manifests/etcd/events.yaml + - memfs://clusters.example.com/privateciliumadvanced.example.com/manifests/etcd/cilium.yaml + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodesprivateciliumadvancedexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.ipv4.tcp_rmem='4096 12582912 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + configOverride: | + version = 2 + + [plugins] + + [plugins."io.containerd.grpc.v1.cri"] + + [plugins."io.containerd.grpc.v1.cri".containerd] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + logLevel: info + version: 1.4.6 + docker: + skipInstall: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + enabled: false + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 6ae4763598c9583f8b50605f19d6c7e9ef93c216706465e73dfc84ee6b63a238@https://github.com/containerd/containerd/releases/download/v1.4.6/cri-containerd-cni-1.4.6-linux-amd64.tar.gz + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 998b3b6669335f1a1d8c475fb7c211ed1e41c2ff37275939e2523666ccb7d910@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.6.tgz + ClusterName: privateciliumadvanced.example.com + ConfigBase: memfs://clusters.example.com/privateciliumadvanced.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/privateciliumadvanced.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_master-us-test-1a.masters.privateciliumadvanced.example.com_user_data b/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_master-us-test-1a.masters.privateciliumadvanced.example.com_user_data index 5d456ea6ec..2318a300ba 100644 --- a/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_master-us-test-1a.masters.privateciliumadvanced.example.com_user_data +++ b/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_master-us-test-1a.masters.privateciliumadvanced.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_nodes.privateciliumadvanced.example.com_user_data b/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_nodes.privateciliumadvanced.example.com_user_data index 7c53547fb9..370ba9b888 100644 --- a/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_nodes.privateciliumadvanced.example.com_user_data +++ b/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_nodes.privateciliumadvanced.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/privatedns1/data/aws_launch_template_master-us-test-1a.masters.privatedns1.example.com_user_data b/tests/integration/update_cluster/privatedns1/data/aws_launch_template_master-us-test-1a.masters.privatedns1.example.com_user_data index 36c36991d1..c9d248b5ab 100644 --- a/tests/integration/update_cluster/privatedns1/data/aws_launch_template_master-us-test-1a.masters.privatedns1.example.com_user_data +++ b/tests/integration/update_cluster/privatedns1/data/aws_launch_template_master-us-test-1a.masters.privatedns1.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/privatedns1/data/aws_launch_template_nodes.privatedns1.example.com_user_data b/tests/integration/update_cluster/privatedns1/data/aws_launch_template_nodes.privatedns1.example.com_user_data index 7b06bf8192..a5b10dda32 100644 --- a/tests/integration/update_cluster/privatedns1/data/aws_launch_template_nodes.privatedns1.example.com_user_data +++ b/tests/integration/update_cluster/privatedns1/data/aws_launch_template_nodes.privatedns1.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/privatedns2/data/aws_launch_template_master-us-test-1a.masters.privatedns2.example.com_user_data b/tests/integration/update_cluster/privatedns2/data/aws_launch_template_master-us-test-1a.masters.privatedns2.example.com_user_data index 548ab5bd1a..853c4b082f 100644 --- a/tests/integration/update_cluster/privatedns2/data/aws_launch_template_master-us-test-1a.masters.privatedns2.example.com_user_data +++ b/tests/integration/update_cluster/privatedns2/data/aws_launch_template_master-us-test-1a.masters.privatedns2.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/privatedns2/data/aws_launch_template_nodes.privatedns2.example.com_user_data b/tests/integration/update_cluster/privatedns2/data/aws_launch_template_nodes.privatedns2.example.com_user_data index 5fada23560..7175be3d48 100644 --- a/tests/integration/update_cluster/privatedns2/data/aws_launch_template_nodes.privatedns2.example.com_user_data +++ b/tests/integration/update_cluster/privatedns2/data/aws_launch_template_nodes.privatedns2.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/privateflannel/data/aws_launch_template_master-us-test-1a.masters.privateflannel.example.com_user_data b/tests/integration/update_cluster/privateflannel/data/aws_launch_template_master-us-test-1a.masters.privateflannel.example.com_user_data index 1871a05562..afcb31f248 100644 --- a/tests/integration/update_cluster/privateflannel/data/aws_launch_template_master-us-test-1a.masters.privateflannel.example.com_user_data +++ b/tests/integration/update_cluster/privateflannel/data/aws_launch_template_master-us-test-1a.masters.privateflannel.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/privateflannel/data/aws_launch_template_nodes.privateflannel.example.com_user_data b/tests/integration/update_cluster/privateflannel/data/aws_launch_template_nodes.privateflannel.example.com_user_data index 5161811452..1d3627cd47 100644 --- a/tests/integration/update_cluster/privateflannel/data/aws_launch_template_nodes.privateflannel.example.com_user_data +++ b/tests/integration/update_cluster/privateflannel/data/aws_launch_template_nodes.privateflannel.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/privatekopeio/data/aws_launch_template_master-us-test-1a.masters.privatekopeio.example.com_user_data b/tests/integration/update_cluster/privatekopeio/data/aws_launch_template_master-us-test-1a.masters.privatekopeio.example.com_user_data index 7eeab5cd5e..41065e3093 100644 --- a/tests/integration/update_cluster/privatekopeio/data/aws_launch_template_master-us-test-1a.masters.privatekopeio.example.com_user_data +++ b/tests/integration/update_cluster/privatekopeio/data/aws_launch_template_master-us-test-1a.masters.privatekopeio.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/privatekopeio/data/aws_launch_template_nodes.privatekopeio.example.com_user_data b/tests/integration/update_cluster/privatekopeio/data/aws_launch_template_nodes.privatekopeio.example.com_user_data index 60ce4e5624..8bee0152e2 100644 --- a/tests/integration/update_cluster/privatekopeio/data/aws_launch_template_nodes.privatekopeio.example.com_user_data +++ b/tests/integration/update_cluster/privatekopeio/data/aws_launch_template_nodes.privatekopeio.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/privateweave/data/aws_launch_template_master-us-test-1a.masters.privateweave.example.com_user_data b/tests/integration/update_cluster/privateweave/data/aws_launch_template_master-us-test-1a.masters.privateweave.example.com_user_data index 57ee829341..4c7dbaf330 100644 --- a/tests/integration/update_cluster/privateweave/data/aws_launch_template_master-us-test-1a.masters.privateweave.example.com_user_data +++ b/tests/integration/update_cluster/privateweave/data/aws_launch_template_master-us-test-1a.masters.privateweave.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/privateweave/data/aws_launch_template_nodes.privateweave.example.com_user_data b/tests/integration/update_cluster/privateweave/data/aws_launch_template_nodes.privateweave.example.com_user_data index de75c2f697..37de39d08b 100644 --- a/tests/integration/update_cluster/privateweave/data/aws_launch_template_nodes.privateweave.example.com_user_data +++ b/tests/integration/update_cluster/privateweave/data/aws_launch_template_nodes.privateweave.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/public-jwks-apiserver/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data b/tests/integration/update_cluster/public-jwks-apiserver/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data index 1d682976bf..6688cc0306 100644 --- a/tests/integration/update_cluster/public-jwks-apiserver/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data +++ b/tests/integration/update_cluster/public-jwks-apiserver/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/public-jwks-apiserver/data/aws_launch_template_nodes.minimal.example.com_user_data b/tests/integration/update_cluster/public-jwks-apiserver/data/aws_launch_template_nodes.minimal.example.com_user_data index 7995b7fbca..6595be60e2 100644 --- a/tests/integration/update_cluster/public-jwks-apiserver/data/aws_launch_template_nodes.minimal.example.com_user_data +++ b/tests/integration/update_cluster/public-jwks-apiserver/data/aws_launch_template_nodes.minimal.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/shared_subnet/data/aws_launch_template_master-us-test-1a.masters.sharedsubnet.example.com_user_data b/tests/integration/update_cluster/shared_subnet/data/aws_launch_template_master-us-test-1a.masters.sharedsubnet.example.com_user_data index 79c2d7617e..036fb4ac65 100644 --- a/tests/integration/update_cluster/shared_subnet/data/aws_launch_template_master-us-test-1a.masters.sharedsubnet.example.com_user_data +++ b/tests/integration/update_cluster/shared_subnet/data/aws_launch_template_master-us-test-1a.masters.sharedsubnet.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/shared_subnet/data/aws_launch_template_nodes.sharedsubnet.example.com_user_data b/tests/integration/update_cluster/shared_subnet/data/aws_launch_template_nodes.sharedsubnet.example.com_user_data index 6a71a48a94..b4f11d86ae 100644 --- a/tests/integration/update_cluster/shared_subnet/data/aws_launch_template_nodes.sharedsubnet.example.com_user_data +++ b/tests/integration/update_cluster/shared_subnet/data/aws_launch_template_nodes.sharedsubnet.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/shared_vpc/data/aws_launch_template_master-us-test-1a.masters.sharedvpc.example.com_user_data b/tests/integration/update_cluster/shared_vpc/data/aws_launch_template_master-us-test-1a.masters.sharedvpc.example.com_user_data index 87760c40af..4ff44cbf6c 100644 --- a/tests/integration/update_cluster/shared_vpc/data/aws_launch_template_master-us-test-1a.masters.sharedvpc.example.com_user_data +++ b/tests/integration/update_cluster/shared_vpc/data/aws_launch_template_master-us-test-1a.masters.sharedvpc.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/shared_vpc/data/aws_launch_template_nodes.sharedvpc.example.com_user_data b/tests/integration/update_cluster/shared_vpc/data/aws_launch_template_nodes.sharedvpc.example.com_user_data index e1e56f4d43..f498149e6a 100644 --- a/tests/integration/update_cluster/shared_vpc/data/aws_launch_template_nodes.sharedvpc.example.com_user_data +++ b/tests/integration/update_cluster/shared_vpc/data/aws_launch_template_nodes.sharedvpc.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/unmanaged/data/aws_launch_template_master-us-test-1a.masters.unmanaged.example.com_user_data b/tests/integration/update_cluster/unmanaged/data/aws_launch_template_master-us-test-1a.masters.unmanaged.example.com_user_data index 36b9a60277..c135a5059a 100644 --- a/tests/integration/update_cluster/unmanaged/data/aws_launch_template_master-us-test-1a.masters.unmanaged.example.com_user_data +++ b/tests/integration/update_cluster/unmanaged/data/aws_launch_template_master-us-test-1a.masters.unmanaged.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/unmanaged/data/aws_launch_template_nodes.unmanaged.example.com_user_data b/tests/integration/update_cluster/unmanaged/data/aws_launch_template_nodes.unmanaged.example.com_user_data index cd40e5850f..c50ce428fa 100644 --- a/tests/integration/update_cluster/unmanaged/data/aws_launch_template_nodes.unmanaged.example.com_user_data +++ b/tests/integration/update_cluster/unmanaged/data/aws_launch_template_nodes.unmanaged.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/vfs-said/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data b/tests/integration/update_cluster/vfs-said/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data index 260794f2fe..351e6f65e7 100644 --- a/tests/integration/update_cluster/vfs-said/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data +++ b/tests/integration/update_cluster/vfs-said/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do diff --git a/tests/integration/update_cluster/vfs-said/data/aws_launch_template_nodes.minimal.example.com_user_data b/tests/integration/update_cluster/vfs-said/data/aws_launch_template_nodes.minimal.example.com_user_data index a2a6065018..a587cb76f0 100644 --- a/tests/integration/update_cluster/vfs-said/data/aws_launch_template_nodes.minimal.example.com_user_data +++ b/tests/integration/update_cluster/vfs-said/data/aws_launch_template_nodes.minimal.example.com_user_data @@ -34,11 +34,11 @@ download-or-bust() { local -r urls=( $(split-commas "$3") ) if [[ -f "${file}" ]]; then - if ! validate-hash "${file}" "${hash}"; then - rm -f "${file}" - else - return - fi + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi fi while true; do