kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #14179 from hakman/hetzner_terraform 

Add Terraform target support for Hetzner
This commit is contained in:
Kubernetes Prow Robot 2022-08-25 00:36:19 -07:00 committed by GitHub
parent e23b852aa7 272ded0187
commit 9b52b30399
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
39 changed files with 3793 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
cmd/kops/create_cluster_integration_test.go
View File

@ -55,6 +55,19 @@ func TestCreateClusterMinimal(t *testing.T) {
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal-1.26", "v1alpha2")
}
// TestCreateClusterHetzner runs kops create cluster minimal.k8s.local --zones fsn1
func TestCreateClusterHetzner(t *testing.T) {
t.Setenv("HCLOUD_TOKEN", "REDACTED")
featureflag.ParseFlags("+Hetzner")
unsetFeatureFlags := func() {
featureflag.ParseFlags("-Hetzner")
}
defer unsetFeatureFlags()
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/ha_hetzner", "v1alpha2")
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/minimal_hetzner", "v1alpha2")
}
// TestCreateClusterOverride tests the override flag
func TestCreateClusterOverride(t *testing.T) {
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/overrides", "v1alpha2")

47
cmd/kops/integration_test.go
View File

@ -267,6 +267,19 @@ func TestMinimal_v1_26(t *testing.T) {
newIntegrationTest("minimal.example.com", "minimal").runTestCloudformation(t)
}
// TestHetzner runs the test on a minimum configuration
func TestHetzner(t *testing.T) {
t.Setenv("HCLOUD_TOKEN", "REDACTED")
featureflag.ParseFlags("+Hetzner")
unsetFeatureFlags := func() {
featureflag.ParseFlags("-Hetzner")
}
defer unsetFeatureFlags()
newIntegrationTest("minimal.k8s.local", "minimal_hetzner").
runTestTerraformHetzner(t)
}
func TestNvidia(t *testing.T) {
newIntegrationTest("minimal.example.com", "nvidia").
withAddons(
@ -1430,6 +1443,40 @@ func (i *integrationTest) runTestTerraformGCE(t *testing.T) {
i.runTest(t, h, expectedFilenames, "", "", nil)
}
func (i *integrationTest) runTestTerraformHetzner(t *testing.T) {
h := testutils.NewIntegrationTestHarness(t)
defer h.Close()
h.MockKopsVersion("1.21.0-alpha.1")
expectedFilenames := i.expectTerraformFilenames
expectedFilenames = append(expectedFilenames,
"aws_s3_object_cluster-completed.spec_content",
"aws_s3_object_etcd-cluster-spec-events_content",
"aws_s3_object_etcd-cluster-spec-main_content",
"aws_s3_object_kops-version.txt_content",
"aws_s3_object_manifests-etcdmanager-events-master-fsn1_content",
"aws_s3_object_manifests-etcdmanager-main-master-fsn1_content",
"aws_s3_object_manifests-static-kube-apiserver-healthcheck_content",
"aws_s3_object_nodeupconfig-master-fsn1_content",
"aws_s3_object_nodeupconfig-nodes-fsn1_content",
"aws_s3_object_"+i.clusterName+"-addons-bootstrap_content",
"aws_s3_object_"+i.clusterName+"-addons-coredns.addons.k8s.io-k8s-1.12_content",
"aws_s3_object_"+i.clusterName+"-addons-dns-controller.addons.k8s.io-k8s-1.12_content",
"aws_s3_object_"+i.clusterName+"-addons-hcloud-cloud-controller.addons.k8s.io-k8s-1.22_content",
"aws_s3_object_"+i.clusterName+"-addons-hcloud-csi-driver.addons.k8s.io-k8s-1.22_content",
"aws_s3_object_"+i.clusterName+"-addons-kops-controller.addons.k8s.io-k8s-1.16_content",
"aws_s3_object_"+i.clusterName+"-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content",
"aws_s3_object_"+i.clusterName+"-addons-limit-range.addons.k8s.io_content",
"aws_s3_object_"+i.clusterName+"-addons-rbac.addons.k8s.io-k8s-1.8_content",
"hcloud_server_master-fsn1_user_data",
"hcloud_server_nodes-fsn1_user_data",
)
i.runTest(t, h, expectedFilenames, "", "", nil)
}
func (i *integrationTest) runTestCloudformation(t *testing.T) {
ctx := context.Background()

133
tests/integration/create_cluster/ha_hetzner/expected-v1alpha2.yaml Normal file
View File

@ -0,0 +1,133 @@
apiVersion: kops.k8s.io/v1alpha2
kind: Cluster
metadata:
creationTimestamp: "2017-01-01T00:00:00Z"
name: minimal.k8s.local
spec:
api:
loadBalancer:
type: Public
authorization:
rbac: {}
channel: stable
cloudProvider: hetzner
configBase: memfs://tests/minimal.k8s.local
etcdClusters:
- cpuRequest: 200m
etcdMembers:
- instanceGroup: master-fsn1-1
name: etcd-1
- instanceGroup: master-fsn1-2
name: etcd-2
- instanceGroup: master-fsn1-3
name: etcd-3
memoryRequest: 100Mi
name: main
- cpuRequest: 100m
etcdMembers:
- instanceGroup: master-fsn1-1
name: etcd-1
- instanceGroup: master-fsn1-2
name: etcd-2
- instanceGroup: master-fsn1-3
name: etcd-3
memoryRequest: 100Mi
name: events
iam:
allowContainerRegistry: true
legacy: false
kubelet:
anonymousAuth: false
kubernetesApiAccess:
- 0.0.0.0/0
- ::/0
kubernetesVersion: v1.25.0
masterPublicName: api.minimal.k8s.local
networkCIDR: 10.0.0.0/16
networking:
cni: {}
nonMasqueradeCIDR: 100.64.0.0/10
sshAccess:
- 0.0.0.0/0
- ::/0
subnets:
- name: fsn1
type: Public
zone: fsn1
topology:
dns:
type: Public
masters: public
nodes: public
---
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: "2017-01-01T00:00:00Z"
labels:
kops.k8s.io/cluster: minimal.k8s.local
name: master-fsn1-1
spec:
image: ubuntu-20.04
machineType: cx21
maxSize: 1
minSize: 1
role: Master
subnets:
- fsn1
---
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: "2017-01-01T00:00:00Z"
labels:
kops.k8s.io/cluster: minimal.k8s.local
name: master-fsn1-2
spec:
image: ubuntu-20.04
machineType: cx21
maxSize: 1
minSize: 1
role: Master
subnets:
- fsn1
---
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: "2017-01-01T00:00:00Z"
labels:
kops.k8s.io/cluster: minimal.k8s.local
name: master-fsn1-3
spec:
image: ubuntu-20.04
machineType: cx21
maxSize: 1
minSize: 1
role: Master
subnets:
- fsn1
---
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: "2017-01-01T00:00:00Z"
labels:
kops.k8s.io/cluster: minimal.k8s.local
name: nodes-fsn1
spec:
image: ubuntu-20.04
machineType: cx21
maxSize: 1
minSize: 1
role: Node
subnets:
- fsn1

9
tests/integration/create_cluster/ha_hetzner/options.yaml Normal file
View File

@ -0,0 +1,9 @@
CloudProvider: hetzner
ClusterName: minimal.k8s.local
Image: ubuntu-20.04
KubernetesVersion: v1.25.0
MasterCount: 3
NetworkCIDR: 10.0.0.0/16
Networking: cni
Zones:
- fsn1

89
tests/integration/create_cluster/minimal_hetzner/expected-v1alpha2.yaml Normal file
View File

@ -0,0 +1,89 @@
apiVersion: kops.k8s.io/v1alpha2
kind: Cluster
metadata:
creationTimestamp: "2017-01-01T00:00:00Z"
name: minimal.k8s.local
spec:
api:
loadBalancer:
type: Public
authorization:
rbac: {}
channel: stable
cloudProvider: hetzner
configBase: memfs://tests/minimal.k8s.local
etcdClusters:
- cpuRequest: 200m
etcdMembers:
- instanceGroup: master-fsn1
name: etcd-1
memoryRequest: 100Mi
name: main
- cpuRequest: 100m
etcdMembers:
- instanceGroup: master-fsn1
name: etcd-1
memoryRequest: 100Mi
name: events
iam:
allowContainerRegistry: true
legacy: false
kubelet:
anonymousAuth: false
kubernetesApiAccess:
- 0.0.0.0/0
- ::/0
kubernetesVersion: v1.25.0
masterPublicName: api.minimal.k8s.local
networkCIDR: 10.0.0.0/16
networking:
cni: {}
nonMasqueradeCIDR: 100.64.0.0/10
sshAccess:
- 0.0.0.0/0
- ::/0
subnets:
- name: fsn1
type: Public
zone: fsn1
topology:
dns:
type: Public
masters: public
nodes: public
---
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: "2017-01-01T00:00:00Z"
labels:
kops.k8s.io/cluster: minimal.k8s.local
name: master-fsn1
spec:
image: ubuntu-20.04
machineType: cx21
maxSize: 1
minSize: 1
role: Master
subnets:
- fsn1
---
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: "2017-01-01T00:00:00Z"
labels:
kops.k8s.io/cluster: minimal.k8s.local
name: nodes-fsn1
spec:
image: ubuntu-20.04
machineType: cx21
maxSize: 1
minSize: 1
role: Node
subnets:
- fsn1

8
tests/integration/create_cluster/minimal_hetzner/options.yaml Normal file
View File

@ -0,0 +1,8 @@
CloudProvider: hetzner
ClusterName: minimal.k8s.local
Image: ubuntu-20.04
KubernetesVersion: v1.25.0
NetworkCIDR: 10.0.0.0/16
Networking: cni
Zones:
- fsn1

187
tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_cluster-completed.spec_content Normal file
View File

@ -0,0 +1,187 @@
apiVersion: kops.k8s.io/v1alpha2
kind: Cluster
metadata:
creationTimestamp: "2017-01-01T00:00:00Z"
name: minimal.k8s.local
spec:
api:
loadBalancer:
type: Public
authorization:
rbac: {}
channel: stable
cloudConfig:
manageStorageClasses: true
cloudProvider: hetzner
clusterDNSDomain: cluster.local
configBase: memfs://tests/minimal.k8s.local
configStore: memfs://tests/minimal.k8s.local
containerRuntime: containerd
containerd:
logLevel: info
runc:
version: 1.1.3
version: 1.6.8
docker:
skipInstall: true
etcdClusters:
- backups:
backupStore: memfs://tests/minimal.k8s.local/backups/etcd/main
cpuRequest: 200m
etcdMembers:
- instanceGroup: master-fsn1
name: etcd-1
memoryRequest: 100Mi
name: main
version: 3.5.4
- backups:
backupStore: memfs://tests/minimal.k8s.local/backups/etcd/events
cpuRequest: 100m
etcdMembers:
- instanceGroup: master-fsn1
name: etcd-1
memoryRequest: 100Mi
name: events
version: 3.5.4
externalDns:
provider: dns-controller
iam:
allowContainerRegistry: true
legacy: false
keyStore: memfs://tests/minimal.k8s.local/pki
kubeAPIServer:
allowPrivileged: true
anonymousAuth: false
apiAudiences:
- kubernetes.svc.default
apiServerCount: 1
authorizationMode: Node,RBAC
bindAddress: 0.0.0.0
cloudProvider: external
enableAdmissionPlugins:
- NamespaceLifecycle
- LimitRanger
- ServiceAccount
- DefaultStorageClass
- DefaultTolerationSeconds
- MutatingAdmissionWebhook
- ValidatingAdmissionWebhook
- NodeRestriction
- ResourceQuota
etcdServers:
- https://127.0.0.1:4001
etcdServersOverrides:
- /events#https://127.0.0.1:4002
image: registry.k8s.io/kube-apiserver:v1.25.0
kubeletPreferredAddressTypes:
- InternalIP
- Hostname
- ExternalIP
logLevel: 2
requestheaderAllowedNames:
- aggregator
requestheaderExtraHeaderPrefixes:
- X-Remote-Extra-
requestheaderGroupHeaders:
- X-Remote-Group
requestheaderUsernameHeaders:
- X-Remote-User
securePort: 443
serviceAccountIssuer: https://api.internal.minimal.k8s.local
serviceAccountJWKSURI: https://api.internal.minimal.k8s.local/openid/v1/jwks
serviceClusterIPRange: 100.64.0.0/13
storageBackend: etcd3
kubeControllerManager:
allocateNodeCIDRs: true
attachDetachReconcileSyncPeriod: 1m0s
cloudProvider: external
clusterCIDR: 100.96.0.0/11
clusterName: minimal.k8s.local
configureCloudRoutes: false
image: registry.k8s.io/kube-controller-manager:v1.25.0
leaderElection:
leaderElect: true
logLevel: 2
useServiceAccountCredentials: true
kubeDNS:
cacheMaxConcurrent: 150
cacheMaxSize: 1000
cpuRequest: 100m
domain: cluster.local
memoryLimit: 170Mi
memoryRequest: 70Mi
nodeLocalDNS:
cpuRequest: 25m
enabled: false
image: registry.k8s.io/dns/k8s-dns-node-cache:1.21.3
memoryRequest: 5Mi
provider: CoreDNS
serverIP: 100.64.0.10
kubeProxy:
clusterCIDR: 100.96.0.0/11
cpuRequest: 100m
image: registry.k8s.io/kube-proxy:v1.25.0
logLevel: 2
kubeScheduler:
image: registry.k8s.io/kube-scheduler:v1.25.0
leaderElection:
leaderElect: true
logLevel: 2
kubelet:
anonymousAuth: false
cgroupDriver: systemd
cgroupRoot: /
cloudProvider: external
clusterDNS: 100.64.0.10
clusterDomain: cluster.local
enableDebuggingHandlers: true
evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
kubeconfigPath: /var/lib/kubelet/kubeconfig
logLevel: 2
podInfraContainerImage: registry.k8s.io/pause:3.6
podManifestPath: /etc/kubernetes/manifests
protectKernelDefaults: true
shutdownGracePeriod: 30s
shutdownGracePeriodCriticalPods: 10s
kubernetesApiAccess:
- 0.0.0.0/0
- ::/0
kubernetesVersion: 1.25.0
masterInternalName: api.internal.minimal.k8s.local
masterKubelet:
anonymousAuth: false
cgroupDriver: systemd
cgroupRoot: /
cloudProvider: external
clusterDNS: 100.64.0.10
clusterDomain: cluster.local
enableDebuggingHandlers: true
evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
kubeconfigPath: /var/lib/kubelet/kubeconfig
logLevel: 2
podInfraContainerImage: registry.k8s.io/pause:3.6
podManifestPath: /etc/kubernetes/manifests
protectKernelDefaults: true
registerSchedulable: false
shutdownGracePeriod: 30s
shutdownGracePeriodCriticalPods: 10s
masterPublicName: api.minimal.k8s.local
networkCIDR: 10.0.0.0/16
networking:
cni: {}
nonMasqueradeCIDR: 100.64.0.0/10
podCIDR: 100.96.0.0/11
secretStore: memfs://tests/minimal.k8s.local/secrets
serviceClusterIPRange: 100.64.0.0/13
sshAccess:
- 0.0.0.0/0
- ::/0
subnets:
- name: fsn1
type: Public
zone: fsn1
topology:
dns:
type: Public
masters: public
nodes: public

4
tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_etcd-cluster-spec-events_content Normal file
View File

@ -0,0 +1,4 @@
{
"memberCount": 1,
"etcdVersion": "3.5.4"
}

4
tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_etcd-cluster-spec-main_content Normal file
View File

@ -0,0 +1,4 @@
{
"memberCount": 1,
"etcdVersion": "3.5.4"
}

1
tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_kops-version.txt_content Normal file
View File

@ -0,0 +1 @@
1.21.0-alpha.1

63
tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_manifests-etcdmanager-events-master-fsn1_content Normal file
View File

@ -0,0 +1,63 @@
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
labels:
k8s-app: etcd-manager-events
name: etcd-manager-events
namespace: kube-system
spec:
containers:
- command:
- /bin/sh
- -c
- mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /etcd-manager
--backup-store=memfs://tests/minimal.k8s.local/backups/etcd/events --client-urls=https://__name__:4002
--cluster-name=etcd-events --containerized=true --dns-suffix=internal.minimal.k8s.local
--grpc-port=3997 --peer-urls=https://__name__:2381 --quarantine-client-urls=https://__name__:3995
--v=6 --volume-provider=hetzner --volume-tag=kops.k8s.io/cluster=minimal.k8s.local
--volume-tag=kops.k8s.io/volume-role=events > /tmp/pipe 2>&1
env:
- name: HCLOUD_TOKEN
value: REDACTED
image: registry.k8s.io/etcdadm/etcd-manager:v3.0.20220727
name: etcd-manager
resources:
requests:
cpu: 100m
memory: 100Mi
securityContext:
privileged: true
volumeMounts:
- mountPath: /rootfs
name: rootfs
- mountPath: /run
name: run
- mountPath: /etc/kubernetes/pki/etcd-manager
name: pki
- mountPath: /var/log/etcd.log
name: varlogetcd
hostNetwork: true
hostPID: true
priorityClassName: system-cluster-critical
tolerations:
- key: CriticalAddonsOnly
operator: Exists
volumes:
- hostPath:
path: /
type: Directory
name: rootfs
- hostPath:
path: /run
type: DirectoryOrCreate
name: run
- hostPath:
path: /etc/kubernetes/pki/etcd-manager-events
type: DirectoryOrCreate
name: pki
- hostPath:
path: /var/log/etcd-events.log
type: FileOrCreate
name: varlogetcd
status: {}

63
tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_manifests-etcdmanager-main-master-fsn1_content Normal file
View File

@ -0,0 +1,63 @@
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
labels:
k8s-app: etcd-manager-main
name: etcd-manager-main
namespace: kube-system
spec:
containers:
- command:
- /bin/sh
- -c
- mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /etcd-manager
--backup-store=memfs://tests/minimal.k8s.local/backups/etcd/main --client-urls=https://__name__:4001
--cluster-name=etcd --containerized=true --dns-suffix=internal.minimal.k8s.local
--grpc-port=3996 --peer-urls=https://__name__:2380 --quarantine-client-urls=https://__name__:3994
--v=6 --volume-provider=hetzner --volume-tag=kops.k8s.io/cluster=minimal.k8s.local
--volume-tag=kops.k8s.io/volume-role=main > /tmp/pipe 2>&1
env:
- name: HCLOUD_TOKEN
value: REDACTED
image: registry.k8s.io/etcdadm/etcd-manager:v3.0.20220727
name: etcd-manager
resources:
requests:
cpu: 200m
memory: 100Mi
securityContext:
privileged: true
volumeMounts:
- mountPath: /rootfs
name: rootfs
- mountPath: /run
name: run
- mountPath: /etc/kubernetes/pki/etcd-manager
name: pki
- mountPath: /var/log/etcd.log
name: varlogetcd
hostNetwork: true
hostPID: true
priorityClassName: system-cluster-critical
tolerations:
- key: CriticalAddonsOnly
operator: Exists
volumes:
- hostPath:
path: /
type: Directory
name: rootfs
- hostPath:
path: /run
type: DirectoryOrCreate
name: run
- hostPath:
path: /etc/kubernetes/pki/etcd-manager-main
type: DirectoryOrCreate
name: pki
- hostPath:
path: /var/log/etcd.log
type: FileOrCreate
name: varlogetcd
status: {}

33
tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content Normal file
View File

@ -0,0 +1,33 @@
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
spec:
containers:
- args:
- --ca-cert=/secrets/ca.crt
- --client-cert=/secrets/client.crt
- --client-key=/secrets/client.key
image: registry.k8s.io/kops/kube-apiserver-healthcheck:1.25.0-alpha.2
livenessProbe:
httpGet:
host: 127.0.0.1
path: /.kube-apiserver-healthcheck/healthz
port: 3990
initialDelaySeconds: 5
timeoutSeconds: 5
name: healthcheck
resources: {}
securityContext:
runAsNonRoot: true
runAsUser: 10012
volumeMounts:
- mountPath: /secrets
name: healthcheck-secrets
readOnly: true
volumes:
- hostPath:
path: /etc/kubernetes/kube-apiserver-healthcheck/secrets
type: Directory
name: healthcheck-secrets
status: {}

62
tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_minimal.k8s.local-addons-bootstrap_content Normal file
View File

@ -0,0 +1,62 @@
kind: Addons
metadata:
creationTimestamp: null
name: bootstrap
spec:
addons:
- id: k8s-1.16
manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
manifestHash: 327e09a6528c0e3c091d36830517196d7c01bf419c6fb965dd4731ecdb8233f0
name: kops-controller.addons.k8s.io
needsRollingUpdate: control-plane
selector:
k8s-addon: kops-controller.addons.k8s.io
version: 9.99.0
- id: k8s-1.12
manifest: coredns.addons.k8s.io/k8s-1.12.yaml
manifestHash: a38cdb6f83a2b6cb1a47136f5d2c137edfa366a4ccdc61e424949cd4a4ff518b
name: coredns.addons.k8s.io
selector:
k8s-addon: coredns.addons.k8s.io
version: 9.99.0
- id: k8s-1.8
manifest: rbac.addons.k8s.io/k8s-1.8.yaml
manifestHash: f81bd7c57bc1902ca342635d7ad7d01b82dfeaff01a1192b076e66907d87871e
name: rbac.addons.k8s.io
selector:
k8s-addon: rbac.addons.k8s.io
version: 9.99.0
- id: k8s-1.9
manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml
manifestHash: 01c120e887bd98d82ef57983ad58a0b22bc85efb48108092a24c4b82e4c9ea81
name: kubelet-api.rbac.addons.k8s.io
selector:
k8s-addon: kubelet-api.rbac.addons.k8s.io
version: 9.99.0
- manifest: limit-range.addons.k8s.io/v1.5.0.yaml
manifestHash: 2d55c3bc5e354e84a3730a65b42f39aba630a59dc8d32b30859fcce3d3178bc2
name: limit-range.addons.k8s.io
selector:
k8s-addon: limit-range.addons.k8s.io
version: 9.99.0
- id: k8s-1.12
manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml
manifestHash: b94765b2c424cb3345dbf21c0e953854a78c3efdb59c41ada8e35440208408b6
name: dns-controller.addons.k8s.io
selector:
k8s-addon: dns-controller.addons.k8s.io
version: 9.99.0
- id: k8s-1.22
manifest: hcloud-cloud-controller.addons.k8s.io/k8s-1.22.yaml
manifestHash: 9f8a6974c1b1b7c32082fbb69866cd122132d3167fa829817588537ddea71c99
name: hcloud-cloud-controller.addons.k8s.io
selector:
k8s-addon: hcloud-cloud-controller.addons.k8s.io
version: 9.99.0
- id: k8s-1.22
manifest: hcloud-csi-driver.addons.k8s.io/k8s-1.22.yaml
manifestHash: ab12002aa9a1c17f7568acc659dd38f73f14c2547ce75dafef0d95a15cb0b189
name: hcloud-csi-driver.addons.k8s.io
selector:
k8s-addon: hcloud-csi-driver.addons.k8s.io
version: 9.99.0

394
tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_minimal.k8s.local-addons-coredns.addons.k8s.io-k8s-1.12_content Normal file
View File

@ -0,0 +1,394 @@
apiVersion: v1
kind: ServiceAccount
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: coredns.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: coredns.addons.k8s.io
kubernetes.io/cluster-service: "true"
name: coredns
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: coredns.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: coredns.addons.k8s.io
kubernetes.io/bootstrapping: rbac-defaults
name: system:coredns
rules:
- apiGroups:
- ""
resources:
- endpoints
- services
- pods
- namespaces
verbs:
- list
- watch
- apiGroups:
- discovery.k8s.io
resources:
- endpointslices
verbs:
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
creationTimestamp: null
labels:
addon.kops.k8s.io/name: coredns.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: coredns.addons.k8s.io
kubernetes.io/bootstrapping: rbac-defaults
name: system:coredns
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:coredns
subjects:
- kind: ServiceAccount
name: coredns
namespace: kube-system
---
apiVersion: v1
data:
Corefile: |-
.:53 {
errors
health {
lameduck 5s
}
ready
kubernetes cluster.local. in-addr.arpa ip6.arpa {
pods insecure
fallthrough in-addr.arpa ip6.arpa
ttl 30
}
hosts /rootfs/etc/hosts k8s.local {
ttl 30
fallthrough
}
prometheus :9153
forward . /etc/resolv.conf {
max_concurrent 1000
}
cache 30
loop
reload
loadbalance
}
kind: ConfigMap
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: coredns.addons.k8s.io
addonmanager.kubernetes.io/mode: EnsureExists
app.kubernetes.io/managed-by: kops
k8s-addon: coredns.addons.k8s.io
name: coredns
namespace: kube-system
---
apiVersion: apps/v1
kind: Deployment
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: coredns.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: coredns.addons.k8s.io
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
kubernetes.io/name: CoreDNS
name: coredns
namespace: kube-system
spec:
selector:
matchLabels:
k8s-app: kube-dns
strategy:
rollingUpdate:
maxSurge: 10%
maxUnavailable: 1
type: RollingUpdate
template:
metadata:
creationTimestamp: null
labels:
k8s-app: kube-dns
kops.k8s.io/managed-by: kops
spec:
containers:
- args:
- -conf
- /etc/coredns/Corefile
image: registry.k8s.io/coredns/coredns:v1.9.3
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 5
httpGet:
path: /health
port: 8080
scheme: HTTP
initialDelaySeconds: 60
successThreshold: 1
timeoutSeconds: 5
name: coredns
ports:
- containerPort: 53
name: dns
protocol: UDP
- containerPort: 53
name: dns-tcp
protocol: TCP
- containerPort: 9153
name: metrics
protocol: TCP
readinessProbe:
httpGet:
path: /ready
port: 8181
scheme: HTTP
resources:
limits:
memory: 170Mi
requests:
cpu: 100m
memory: 70Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- NET_BIND_SERVICE
drop:
- all
readOnlyRootFilesystem: true
volumeMounts:
- mountPath: /etc/coredns
name: config-volume
readOnly: true
- mountPath: /rootfs/etc/hosts
name: etc-hosts
readOnly: true
dnsPolicy: Default
nodeSelector:
kubernetes.io/os: linux
priorityClassName: system-cluster-critical
serviceAccountName: coredns
tolerations:
- key: CriticalAddonsOnly
operator: Exists
topologySpreadConstraints:
- labelSelector:
matchLabels:
k8s-app: kube-dns
maxSkew: 1
topologyKey: topology.kubernetes.io/zone
whenUnsatisfiable: ScheduleAnyway
- labelSelector:
matchLabels:
k8s-app: kube-dns
maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: ScheduleAnyway
volumes:
- configMap:
name: coredns
name: config-volume
- hostPath:
path: /etc/hosts
type: File
name: etc-hosts
---
apiVersion: v1
kind: Service
metadata:
annotations:
prometheus.io/port: "9153"
prometheus.io/scrape: "true"
creationTimestamp: null
labels:
addon.kops.k8s.io/name: coredns.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: coredns.addons.k8s.io
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
kubernetes.io/name: CoreDNS
name: kube-dns
namespace: kube-system
resourceVersion: "0"
spec:
clusterIP: 100.64.0.10
ports:
- name: dns
port: 53
protocol: UDP
- name: dns-tcp
port: 53
protocol: TCP
- name: metrics
port: 9153
protocol: TCP
selector:
k8s-app: kube-dns
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: coredns.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: coredns.addons.k8s.io
name: kube-dns
namespace: kube-system
spec:
maxUnavailable: 50%
selector:
matchLabels:
k8s-app: kube-dns
---
apiVersion: v1
kind: ServiceAccount
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: coredns.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: coredns.addons.k8s.io
name: coredns-autoscaler
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: coredns.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: coredns.addons.k8s.io
name: coredns-autoscaler
rules:
- apiGroups:
- ""
resources:
- nodes
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- replicationcontrollers/scale
verbs:
- get
- update
- apiGroups:
- extensions
- apps
resources:
- deployments/scale
- replicasets/scale
verbs:
- get
- update
- apiGroups:
- ""
resources:
- configmaps
verbs:
- get
- create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: coredns.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: coredns.addons.k8s.io
name: coredns-autoscaler
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: coredns-autoscaler
subjects:
- kind: ServiceAccount
name: coredns-autoscaler
namespace: kube-system
---
apiVersion: apps/v1
kind: Deployment
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: coredns.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: coredns.addons.k8s.io
k8s-app: coredns-autoscaler
kubernetes.io/cluster-service: "true"
name: coredns-autoscaler
namespace: kube-system
spec:
selector:
matchLabels:
k8s-app: coredns-autoscaler
template:
metadata:
creationTimestamp: null
labels:
k8s-app: coredns-autoscaler
kops.k8s.io/managed-by: kops
spec:
containers:
- command:
- /cluster-proportional-autoscaler
- --namespace=kube-system
- --configmap=coredns-autoscaler
- --target=Deployment/coredns
- --default-params={"linear":{"coresPerReplica":256,"nodesPerReplica":16,"preventSinglePointFailure":true}}
- --logtostderr=true
- --v=2
image: registry.k8s.io/cpa/cluster-proportional-autoscaler:1.8.5
name: autoscaler
resources:
requests:
cpu: 20m
memory: 10Mi
nodeSelector:
kubernetes.io/os: linux
priorityClassName: system-cluster-critical
serviceAccountName: coredns-autoscaler
tolerations:
- key: CriticalAddonsOnly
operator: Exists

141
tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_minimal.k8s.local-addons-dns-controller.addons.k8s.io-k8s-1.12_content Normal file
View File

@ -0,0 +1,141 @@
apiVersion: apps/v1
kind: Deployment
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: dns-controller.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: dns-controller.addons.k8s.io
k8s-app: dns-controller
version: v1.25.0-alpha.2
name: dns-controller
namespace: kube-system
spec:
replicas: 1
selector:
matchLabels:
k8s-app: dns-controller
strategy:
type: Recreate
template:
metadata:
creationTimestamp: null
labels:
k8s-addon: dns-controller.addons.k8s.io
k8s-app: dns-controller
kops.k8s.io/managed-by: kops
version: v1.25.0-alpha.2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/control-plane
operator: Exists
- matchExpressions:
- key: node-role.kubernetes.io/master
operator: Exists
containers:
- args:
- --watch-ingress=false
- --dns=gossip
- --gossip-seed=127.0.0.1:3999
- --gossip-protocol-secondary=memberlist
- --gossip-listen-secondary=0.0.0.0:3993
- --gossip-seed-secondary=127.0.0.1:4000
- --internal-ipv4
- --zone=*/*
- -v=2
command: null
env:
- name: KUBERNETES_SERVICE_HOST
value: 127.0.0.1
image: registry.k8s.io/kops/dns-controller:1.25.0-alpha.2
name: dns-controller
resources:
requests:
cpu: 50m
memory: 50Mi
securityContext:
runAsNonRoot: true
dnsPolicy: Default
hostNetwork: true
nodeSelector: null
priorityClassName: system-cluster-critical
serviceAccount: dns-controller
tolerations:
- key: node.cloudprovider.kubernetes.io/uninitialized
operator: Exists
- key: node.kubernetes.io/not-ready
operator: Exists
- key: node-role.kubernetes.io/control-plane
operator: Exists
- key: node-role.kubernetes.io/master
operator: Exists
---
apiVersion: v1
kind: ServiceAccount
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: dns-controller.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: dns-controller.addons.k8s.io
name: dns-controller
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: dns-controller.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: dns-controller.addons.k8s.io
name: kops:dns-controller
rules:
- apiGroups:
- ""
resources:
- endpoints
- services
- pods
- ingress
- nodes
verbs:
- get
- list
- watch
- apiGroups:
- networking.k8s.io
resources:
- ingresses
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: dns-controller.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: dns-controller.addons.k8s.io
name: kops:dns-controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: kops:dns-controller
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: system:serviceaccount:kube-system:dns-controller

117
tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_minimal.k8s.local-addons-hcloud-cloud-controller.addons.k8s.io-k8s-1.22_content Normal file
View File

@ -0,0 +1,117 @@
apiVersion: v1
kind: Secret
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: hcloud-cloud-controller.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: hcloud-cloud-controller.addons.k8s.io
name: hcloud
namespace: kube-system
stringData:
network: minimal.k8s.local
token: REDACTED
---
apiVersion: v1
kind: ServiceAccount
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: hcloud-cloud-controller.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: hcloud-cloud-controller.addons.k8s.io
name: cloud-controller-manager
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: hcloud-cloud-controller.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: hcloud-cloud-controller.addons.k8s.io
name: system:cloud-controller-manager
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: cloud-controller-manager
namespace: kube-system
---
apiVersion: apps/v1
kind: Deployment
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: hcloud-cloud-controller.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: hcloud-cloud-controller.addons.k8s.io
name: hcloud-cloud-controller-manager
namespace: kube-system
spec:
replicas: 1
revisionHistoryLimit: 2
selector:
matchLabels:
app: hcloud-cloud-controller-manager
template:
metadata:
creationTimestamp: null
labels:
app: hcloud-cloud-controller-manager
kops.k8s.io/managed-by: kops
spec:
containers:
- command:
- /bin/hcloud-cloud-controller-manager
- --cloud-provider=hcloud
- --leader-elect=false
- --allow-untagged-cloud
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: HCLOUD_TOKEN
valueFrom:
secretKeyRef:
key: token
name: hcloud
- name: HCLOUD_NETWORK
valueFrom:
secretKeyRef:
key: network
name: hcloud
image: hetznercloud/hcloud-cloud-controller-manager:v1.12.1
name: hcloud-cloud-controller-manager
resources:
requests:
cpu: 100m
memory: 50Mi
dnsPolicy: Default
hostNetwork: true
priorityClassName: system-node-critical
serviceAccountName: cloud-controller-manager
tolerations:
- effect: NoSchedule
key: node.cloudprovider.kubernetes.io/uninitialized
value: "true"
- key: CriticalAddonsOnly
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
operator: Exists
- effect: NoSchedule
key: node.kubernetes.io/not-ready

506
tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_minimal.k8s.local-addons-hcloud-csi-driver.addons.k8s.io-k8s-1.22_content Normal file
View File

@ -0,0 +1,506 @@
apiVersion: v1
kind: Secret
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: hcloud-csi-driver.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: hcloud-csi-driver.addons.k8s.io
name: hcloud-csi
namespace: kube-system
stringData:
token: REDACTED
---
apiVersion: storage.k8s.io/v1
kind: CSIDriver
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: hcloud-csi-driver.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: hcloud-csi-driver.addons.k8s.io
name: csi.hetzner.cloud
spec:
attachRequired: true
podInfoOnMount: true
volumeLifecycleModes:
- Persistent
---
allowVolumeExpansion: true
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
annotations:
storageclass.kubernetes.io/is-default-class: "true"
creationTimestamp: null
labels:
addon.kops.k8s.io/name: hcloud-csi-driver.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: hcloud-csi-driver.addons.k8s.io
name: hcloud-volumes
namespace: kube-system
provisioner: csi.hetzner.cloud
volumeBindingMode: WaitForFirstConsumer
---
apiVersion: v1
kind: ServiceAccount
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: hcloud-csi-driver.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: hcloud-csi-driver.addons.k8s.io
name: hcloud-csi
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: hcloud-csi-driver.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: hcloud-csi-driver.addons.k8s.io
name: hcloud-csi
rules:
- apiGroups:
- ""
resources:
- persistentvolumes
verbs:
- get
- list
- watch
- update
- patch
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- list
- watch
- apiGroups:
- csi.storage.k8s.io
resources:
- csinodeinfos
verbs:
- get
- list
- watch
- apiGroups:
- storage.k8s.io
resources:
- csinodes
verbs:
- get
- list
- watch
- apiGroups:
- storage.k8s.io
resources:
- volumeattachments
verbs:
- get
- list
- watch
- update
- patch
- apiGroups:
- storage.k8s.io
resources:
- volumeattachments/status
verbs:
- patch
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- list
- apiGroups:
- ""
resources:
- persistentvolumes
verbs:
- get
- list
- watch
- create
- delete
- patch
- apiGroups:
- ""
resources:
- persistentvolumeclaims
- persistentvolumeclaims/status
verbs:
- get
- list
- watch
- update
- patch
- apiGroups:
- storage.k8s.io
resources:
- storageclasses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- list
- watch
- create
- update
- patch
- apiGroups:
- snapshot.storage.k8s.io
resources:
- volumesnapshots
verbs:
- get
- list
- apiGroups:
- snapshot.storage.k8s.io
resources:
- volumesnapshotcontents
verbs:
- get
- list
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- get
- list
- watch
- create
- update
- patch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: hcloud-csi-driver.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: hcloud-csi-driver.addons.k8s.io
name: hcloud-csi
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: hcloud-csi
subjects:
- kind: ServiceAccount
name: hcloud-csi
namespace: kube-system
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: hcloud-csi-driver.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: hcloud-csi-driver.addons.k8s.io
name: hcloud-csi-controller
namespace: kube-system
spec:
replicas: 1
selector:
matchLabels:
app: hcloud-csi-controller
serviceName: hcloud-csi-controller
template:
metadata:
labels:
app: hcloud-csi-controller
spec:
containers:
- image: k8s.gcr.io/sig-storage/csi-attacher:v3.2.1
name: csi-attacher
securityContext:
allowPrivilegeEscalation: true
capabilities:
add:
- SYS_ADMIN
privileged: true
volumeMounts:
- mountPath: /run/csi
name: socket-dir
- image: k8s.gcr.io/sig-storage/csi-resizer:v1.2.0
name: csi-resizer
securityContext:
allowPrivilegeEscalation: true
capabilities:
add:
- SYS_ADMIN
privileged: true
volumeMounts:
- mountPath: /run/csi
name: socket-dir
- args:
- --feature-gates=Topology=true
- --default-fstype=ext4
image: k8s.gcr.io/sig-storage/csi-provisioner:v2.2.2
name: csi-provisioner
securityContext:
allowPrivilegeEscalation: true
capabilities:
add:
- SYS_ADMIN
privileged: true
volumeMounts:
- mountPath: /run/csi
name: socket-dir
- env:
- name: CSI_ENDPOINT
value: unix:///run/csi/socket
- name: METRICS_ENDPOINT
value: 0.0.0.0:9189
- name: ENABLE_METRICS
value: "true"
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
- name: HCLOUD_TOKEN
valueFrom:
secretKeyRef:
key: token
name: hcloud-csi
image: hetznercloud/hcloud-csi-driver:1.6.0
imagePullPolicy: Always
livenessProbe:
failureThreshold: 5
httpGet:
path: /healthz
port: healthz
initialDelaySeconds: 10
periodSeconds: 2
timeoutSeconds: 3
name: hcloud-csi-driver
ports:
- containerPort: 9189
name: metrics
- containerPort: 9808
name: healthz
protocol: TCP
securityContext:
allowPrivilegeEscalation: true
capabilities:
add:
- SYS_ADMIN
privileged: true
volumeMounts:
- mountPath: /run/csi
name: socket-dir
- image: k8s.gcr.io/sig-storage/livenessprobe:v2.3.0
imagePullPolicy: Always
name: liveness-probe
volumeMounts:
- mountPath: /run/csi
name: socket-dir
serviceAccount: hcloud-csi
volumes:
- emptyDir: {}
name: socket-dir
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: hcloud-csi-driver.addons.k8s.io
app: hcloud-csi
app.kubernetes.io/managed-by: kops
k8s-addon: hcloud-csi-driver.addons.k8s.io
name: hcloud-csi-node
namespace: kube-system
spec:
selector:
matchLabels:
app: hcloud-csi
template:
metadata:
creationTimestamp: null
labels:
app: hcloud-csi
kops.k8s.io/managed-by: kops
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: instance.hetzner.cloud/is-root-server
operator: NotIn
values:
- "true"
containers:
- args:
- --kubelet-registration-path=/var/lib/kubelet/plugins/csi.hetzner.cloud/socket
env:
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0
name: csi-node-driver-registrar
securityContext:
privileged: true
volumeMounts:
- mountPath: /run/csi
name: plugin-dir
- mountPath: /registration
name: registration-dir
- env:
- name: CSI_ENDPOINT
value: unix:///run/csi/socket
- name: METRICS_ENDPOINT
value: 0.0.0.0:9189
- name: ENABLE_METRICS
value: "true"
- name: HCLOUD_TOKEN
valueFrom:
secretKeyRef:
key: token
name: hcloud-csi
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
image: hetznercloud/hcloud-csi-driver:1.6.0
imagePullPolicy: Always
livenessProbe:
failureThreshold: 5
httpGet:
path: /healthz
port: healthz
initialDelaySeconds: 10
periodSeconds: 2
timeoutSeconds: 3
name: hcloud-csi-driver
ports:
- containerPort: 9189
name: metrics
- containerPort: 9808
name: healthz
protocol: TCP
securityContext:
privileged: true
volumeMounts:
- mountPath: /var/lib/kubelet
mountPropagation: Bidirectional
name: kubelet-dir
- mountPath: /run/csi
name: plugin-dir
- mountPath: /dev
name: device-dir
- image: k8s.gcr.io/sig-storage/livenessprobe:v2.3.0
imagePullPolicy: Always
name: liveness-probe
volumeMounts:
- mountPath: /run/csi
name: plugin-dir
serviceAccount: hcloud-csi
tolerations:
- effect: NoExecute
operator: Exists
- effect: NoSchedule
operator: Exists
- key: CriticalAddonsOnly
operator: Exists
volumes:
- hostPath:
path: /var/lib/kubelet
type: Directory
name: kubelet-dir
- hostPath:
path: /var/lib/kubelet/plugins/csi.hetzner.cloud/
type: DirectoryOrCreate
name: plugin-dir
- hostPath:
path: /var/lib/kubelet/plugins_registry/
type: Directory
name: registration-dir
- hostPath:
path: /dev
type: Directory
name: device-dir
---
apiVersion: v1
kind: Service
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: hcloud-csi-driver.addons.k8s.io
app: hcloud-csi
app.kubernetes.io/managed-by: kops
k8s-addon: hcloud-csi-driver.addons.k8s.io
name: hcloud-csi-controller-metrics
namespace: kube-system
spec:
ports:
- name: metrics
port: 9189
targetPort: metrics
selector:
app: hcloud-csi-controller
---
apiVersion: v1
kind: Service
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: hcloud-csi-driver.addons.k8s.io
app: hcloud-csi
app.kubernetes.io/managed-by: kops
k8s-addon: hcloud-csi-driver.addons.k8s.io
name: hcloud-csi-node-metrics
namespace: kube-system
spec:
ports:
- name: metrics
port: 9189
targetPort: metrics
selector:
app: hcloud-csi

291
tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_minimal.k8s.local-addons-kops-controller.addons.k8s.io-k8s-1.16_content Normal file
View File

@ -0,0 +1,291 @@
apiVersion: v1
data:
config.yaml: |
{"cloud":"hetzner","configBase":"memfs://tests/minimal.k8s.local","discovery":{"enabled":true}}
kind: ConfigMap
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: kops-controller.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: kops-controller.addons.k8s.io
name: kops-controller
namespace: kube-system
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: kops-controller.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: kops-controller.addons.k8s.io
k8s-app: kops-controller
version: v1.25.0-alpha.2
name: kops-controller
namespace: kube-system
spec:
selector:
matchLabels:
k8s-app: kops-controller
template:
metadata:
creationTimestamp: null
labels:
k8s-addon: kops-controller.addons.k8s.io
k8s-app: kops-controller
kops.k8s.io/managed-by: kops
version: v1.25.0-alpha.2
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/control-plane
operator: Exists
- key: kops.k8s.io/kops-controller-pki
operator: Exists
- matchExpressions:
- key: node-role.kubernetes.io/master
operator: Exists
- key: kops.k8s.io/kops-controller-pki
operator: Exists
containers:
- args:
- --v=2
- --conf=/etc/kubernetes/kops-controller/config/config.yaml
command: null
env:
- name: KUBERNETES_SERVICE_HOST
value: 127.0.0.1
- name: HCLOUD_TOKEN
value: REDACTED
image: registry.k8s.io/kops/kops-controller:1.25.0-alpha.2
name: kops-controller
resources:
requests:
cpu: 50m
memory: 50Mi
securityContext:
runAsNonRoot: true
runAsUser: 10011
volumeMounts:
- mountPath: /etc/kubernetes/kops-controller/config/
name: kops-controller-config
- mountPath: /etc/kubernetes/kops-controller/pki/
name: kops-controller-pki
dnsPolicy: Default
hostNetwork: true
nodeSelector: null
priorityClassName: system-cluster-critical
serviceAccount: kops-controller
tolerations:
- key: node.cloudprovider.kubernetes.io/uninitialized
operator: Exists
- key: node.kubernetes.io/not-ready
operator: Exists
- key: node-role.kubernetes.io/master
operator: Exists
- key: node-role.kubernetes.io/control-plane
operator: Exists
volumes:
- configMap:
name: kops-controller
name: kops-controller-config
- hostPath:
path: /etc/kubernetes/kops-controller/
type: Directory
name: kops-controller-pki
updateStrategy:
type: OnDelete
---
apiVersion: v1
kind: ServiceAccount
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: kops-controller.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: kops-controller.addons.k8s.io
name: kops-controller
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: kops-controller.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: kops-controller.addons.k8s.io
name: kops-controller
rules:
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- list
- watch
- patch
- apiGroups:
- ""
resources:
- endpoints
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: kops-controller.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: kops-controller.addons.k8s.io
name: kops-controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: kops-controller
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: system:serviceaccount:kube-system:kops-controller
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: kops-controller.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: kops-controller.addons.k8s.io
name: kops-controller
namespace: kube-system
rules:
- apiGroups:
- ""
resources:
- events
verbs:
- get
- list
- watch
- create
- apiGroups:
- ""
- coordination.k8s.io
resourceNames:
- kops-controller-leader
resources:
- configmaps
- leases
verbs:
- get
- list
- watch
- patch
- update
- delete
- apiGroups:
- ""
- coordination.k8s.io
resources:
- configmaps
- leases
verbs:
- create
- apiGroups:
- ""
resourceNames:
- coredns
resources:
- configmaps
verbs:
- get
- watch
- patch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: kops-controller.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: kops-controller.addons.k8s.io
name: kops-controller
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: kops-controller
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: system:serviceaccount:kube-system:kops-controller
---
apiVersion: v1
kind: Service
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: kops-controller.addons.k8s.io
app.kubernetes.io/managed-by: kops
discovery.kops.k8s.io/internal-name: api.internal.minimal.k8s.local
k8s-addon: kops-controller.addons.k8s.io
name: api-internal
namespace: kube-system
spec:
clusterIP: None
ports:
- name: https
port: 443
protocol: TCP
targetPort: 443
selector:
k8s-app: kops-controller
type: ClusterIP
---
apiVersion: v1
kind: Service
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: kops-controller.addons.k8s.io
app.kubernetes.io/managed-by: kops
discovery.kops.k8s.io/internal-name: kops-controller.internal.minimal.k8s.local
k8s-addon: kops-controller.addons.k8s.io
name: kops-controller-internal
namespace: kube-system
spec:
clusterIP: None
ports:
- name: https
port: 3988
protocol: TCP
targetPort: 3988
selector:
k8s-app: kops-controller
type: ClusterIP

17
tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_minimal.k8s.local-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content Normal file
View File

@ -0,0 +1,17 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: kubelet-api.rbac.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: kubelet-api.rbac.addons.k8s.io
name: kops:system:kubelet-api-admin
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:kubelet-api-admin
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: kubelet-api

15
tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_minimal.k8s.local-addons-limit-range.addons.k8s.io_content Normal file
View File

@ -0,0 +1,15 @@
apiVersion: v1
kind: LimitRange
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: limit-range.addons.k8s.io
app.kubernetes.io/managed-by: kops
k8s-addon: limit-range.addons.k8s.io
name: limits
namespace: default
spec:
limits:
- defaultRequest:
cpu: 100m
type: Container

19
tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_minimal.k8s.local-addons-rbac.addons.k8s.io-k8s-1.8_content Normal file
View File

@ -0,0 +1,19 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
creationTimestamp: null
labels:
addon.kops.k8s.io/name: rbac.addons.k8s.io
addonmanager.kubernetes.io/mode: Reconcile
app.kubernetes.io/managed-by: kops
k8s-addon: rbac.addons.k8s.io
kubernetes.io/cluster-service: "true"
name: kubelet-cluster-admin
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:node
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: kubelet

276
tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_nodeupconfig-master-fsn1_content Normal file
View File

@ -0,0 +1,276 @@
APIServerConfig:
KubeAPIServer:
allowPrivileged: true
anonymousAuth: false
apiAudiences:
- kubernetes.svc.default
apiServerCount: 1
authorizationMode: Node,RBAC
bindAddress: 0.0.0.0
cloudProvider: external
enableAdmissionPlugins:
- NamespaceLifecycle
- LimitRanger
- ServiceAccount
- DefaultStorageClass
- DefaultTolerationSeconds
- MutatingAdmissionWebhook
- ValidatingAdmissionWebhook
- NodeRestriction
- ResourceQuota
etcdServers:
- https://127.0.0.1:4001
etcdServersOverrides:
- /events#https://127.0.0.1:4002
image: registry.k8s.io/kube-apiserver:v1.25.0
kubeletPreferredAddressTypes:
- InternalIP
- Hostname
- ExternalIP
logLevel: 2
requestheaderAllowedNames:
- aggregator
requestheaderExtraHeaderPrefixes:
- X-Remote-Extra-
requestheaderGroupHeaders:
- X-Remote-Group
requestheaderUsernameHeaders:
- X-Remote-User
securePort: 443
serviceAccountIssuer: https://api.internal.minimal.k8s.local
serviceAccountJWKSURI: https://api.internal.minimal.k8s.local/openid/v1/jwks
serviceClusterIPRange: 100.64.0.0/13
storageBackend: etcd3
ServiceAccountPublicKeys: |
-----BEGIN RSA PUBLIC KEY-----
MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm
XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ==
-----END RSA PUBLIC KEY-----
-----BEGIN RSA PUBLIC KEY-----
MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF
Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ==
-----END RSA PUBLIC KEY-----
Assets:
amd64:
- 7f9183fce12606818612ce80b6c09757452c4fb50aefea5fc5843951c5020e24@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/amd64/kubelet
- e23cc7092218c95c22d8ee36fb9499194a36ac5b5349ca476886b7edc0203885@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/amd64/kubectl
- 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
- 3a1322c18ee5ff4b9bd5af6b7b30c923a3eab8af1df05554f530ef8e2b24ac5e@https://github.com/containerd/containerd/releases/download/v1.6.8/containerd-1.6.8-linux-amd64.tar.gz
- 6e8b24be90fffce6b025d254846da9d2ca6d65125f9139b6354bab0272253d01@https://github.com/opencontainers/runc/releases/download/v1.1.3/runc.amd64
- f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64
- 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64
arm64:
- 69572a7b3d179d4a479aa2e0f90e2f091d8d84ef33a35422fc89975dc137a590@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/arm64/kubelet
- 24db547bbae294c5c44f2b4a777e45f0e2f3d6295eace0d0c4be2b2dfa45330d@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/arm64/kubectl
- ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
- b114e36ecce78cef9d611416c01b784a420928c82766d6df7dc02b10d9da94cd@https://github.com/containerd/containerd/releases/download/v1.6.8/containerd-1.6.8-linux-arm64.tar.gz
- 00c9ad161a77a01d9dcbd25b1d76fa9822e57d8e4abf26ba8907c98f6bcfcd0f@https://github.com/opencontainers/runc/releases/download/v1.1.3/runc.arm64
- 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64
- 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64
CAs:
apiserver-aggregator-ca: |
-----BEGIN CERTIFICATE-----
MIIBgjCCASygAwIBAgIMFo3gINaZLHjisEcbMA0GCSqGSIb3DQEBCwUAMCIxIDAe
BgNVBAMTF2FwaXNlcnZlci1hZ2dyZWdhdG9yLWNhMB4XDTIxMDYzMDA0NTExMloX
DTMxMDYzMDA0NTExMlowIjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3It
Y2EwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAyyE71AOU3go5XFegLQ6fidI0LhhM
x7CzpTzh2xWKcHUfbNI7itgJvC/+GlyG5W+DF5V7ba0IJiQLsFve0oLdewIDAQAB
o0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU
ALfqF5ZmfqvqORuJIFilZYKF3d0wDQYJKoZIhvcNAQELBQADQQAHAomFKsF4jvYX
WM/UzQXDj9nSAFTf8dBPCXyZZNotsOH7+P6W4mMiuVs8bAuGiXGUdbsQ2lpiT/Rk
CzMeMdr4
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIBgjCCASygAwIBAgIMFo3gM0nxQpiX/agfMA0GCSqGSIb3DQEBCwUAMCIxIDAe
BgNVBAMTF2FwaXNlcnZlci1hZ2dyZWdhdG9yLWNhMB4XDTIxMDYzMDA0NTIzMVoX
DTMxMDYzMDA0NTIzMVowIjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3It
Y2EwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAyyE71AOU3go5XFegLQ6fidI0LhhM
x7CzpTzh2xWKcHUfbNI7itgJvC/+GlyG5W+DF5V7ba0IJiQLsFve0oLdewIDAQAB
o0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU
ALfqF5ZmfqvqORuJIFilZYKF3d0wDQYJKoZIhvcNAQELBQADQQCXsoezoxXu2CEN
QdlXZOfmBT6cqxIX/RMHXhpHwRiqPsTO8IO2bVA8CSzxNwMuSv/ZtrMHoh8+PcVW
HLtkTXH8
-----END CERTIFICATE-----
etcd-clients-ca: |
-----BEGIN CERTIFICATE-----
MIIBcjCCARygAwIBAgIMFo1ogHnr26DL9YkqMA0GCSqGSIb3DQEBCwUAMBoxGDAW
BgNVBAMTD2V0Y2QtY2xpZW50cy1jYTAeFw0yMTA2MjgxNjE5MDFaFw0zMTA2Mjgx
NjE5MDFaMBoxGDAWBgNVBAMTD2V0Y2QtY2xpZW50cy1jYTBcMA0GCSqGSIb3DQEB
AQUAA0sAMEgCQQDYlt4Xx03Cp8QooPrloaVWznx9aQDSpl1UsrDyoBPNEElOLWep
uPaQBHiDLL8LwzGi7G9r+ib13tKrwprnlPv7AgMBAAGjQjBAMA4GA1UdDwEB/wQE
AwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQjlt4Ue54AbJPWlDpRM51s
x+PeBDANBgkqhkiG9w0BAQsFAANBAAZAdf8ROEVkr3Rf7I+s+CQOil2toadlKWOY
qCeJ2XaEROfp9aUTEIU1MGM3g57MPyAPPU7mURskuOQz6B1UFaY=
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIBcjCCARygAwIBAgIMFo1olfBnC/CsT+dqMA0GCSqGSIb3DQEBCwUAMBoxGDAW
BgNVBAMTD2V0Y2QtY2xpZW50cy1jYTAeFw0yMTA2MjgxNjIwMzNaFw0zMTA2Mjgx
NjIwMzNaMBoxGDAWBgNVBAMTD2V0Y2QtY2xpZW50cy1jYTBcMA0GCSqGSIb3DQEB
AQUAA0sAMEgCQQDYlt4Xx03Cp8QooPrloaVWznx9aQDSpl1UsrDyoBPNEElOLWep
uPaQBHiDLL8LwzGi7G9r+ib13tKrwprnlPv7AgMBAAGjQjBAMA4GA1UdDwEB/wQE
AwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQjlt4Ue54AbJPWlDpRM51s
x+PeBDANBgkqhkiG9w0BAQsFAANBAF1xUz77PlUVUnd9duF8F7plou0TONC9R6/E
YQ8C6vM1b+9NSDGjCW8YmwEU2fBgskb/BBX2lwVZ32/RUEju4Co=
-----END CERTIFICATE-----
etcd-manager-ca-events: |
-----BEGIN CERTIFICATE-----
MIIBgDCCASqgAwIBAgIMFo+bKjm04vB4rNtaMA0GCSqGSIb3DQEBCwUAMCExHzAd
BgNVBAMTFmV0Y2QtbWFuYWdlci1jYS1ldmVudHMwHhcNMjEwNzA1MjAwOTU2WhcN
MzEwNzA1MjAwOTU2WjAhMR8wHQYDVQQDExZldGNkLW1hbmFnZXItY2EtZXZlbnRz
MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKiC8tndMlEFZ7qzeKxeKqFVjaYpsh/H
g7RxWo15+1kgH3suO0lxp9+RxSVv97hnsfbySTPZVhy2cIQj7eZtZt8CAwEAAaNC
MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFBg6
CEZkQNnRkARBwFce03AEWa+sMA0GCSqGSIb3DQEBCwUAA0EAJMnBThok/uUe8q8O
sS5q19KUuE8YCTUzMDj36EBKf6NX4NoakCa1h6kfQVtlMtEIMWQZCjbm8xGK5ffs
GS/VUw==
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIBgDCCASqgAwIBAgIMFo+bQ+EgIiBmGghjMA0GCSqGSIb3DQEBCwUAMCExHzAd
BgNVBAMTFmV0Y2QtbWFuYWdlci1jYS1ldmVudHMwHhcNMjEwNzA1MjAxMTQ2WhcN
MzEwNzA1MjAxMTQ2WjAhMR8wHQYDVQQDExZldGNkLW1hbmFnZXItY2EtZXZlbnRz
MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKFhHVVxxDGv8d1jBvtdSxz7KIVoBOjL
DMxsmTsINiQkTQaFlb+XPlnY1ar4+RhE519AFUkqfhypk4Zxqf1YFXUCAwEAAaNC
MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNuW
LLH5c8kDubDbr6BHgedW0iJ9MA0GCSqGSIb3DQEBCwUAA0EAiKUoBoaGu7XzboFE
hjfKlX0TujqWuW3qMxDEJwj4dVzlSLrAoB/G01MJ+xxYKh456n48aG6N827UPXhV
cPfVNg==
-----END CERTIFICATE-----
etcd-manager-ca-main: |
-----BEGIN CERTIFICATE-----
MIIBfDCCASagAwIBAgIMFo+bKjm1c3jfv6hIMA0GCSqGSIb3DQEBCwUAMB8xHTAb
BgNVBAMTFGV0Y2QtbWFuYWdlci1jYS1tYWluMB4XDTIxMDcwNTIwMDk1NloXDTMx
MDcwNTIwMDk1NlowHzEdMBsGA1UEAxMUZXRjZC1tYW5hZ2VyLWNhLW1haW4wXDAN
BgkqhkiG9w0BAQEFAANLADBIAkEAxbkDbGYmCSShpRG3r+lzTOFujyuruRfjOhYm
ZRX4w1Utd5y63dUc98sjc9GGUYMHd+0k1ql/a48tGhnK6N6jJwIDAQABo0IwQDAO
BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUWZLkbBFx
GAgPU4i62c52unSo7RswDQYJKoZIhvcNAQELBQADQQAj6Pgd0va/8FtkyMlnohLu
Gf4v8RJO6zk3Y6jJ4+cwWziipFM1ielMzSOZfFcCZgH3m5Io40is4hPSqyq2TOA6
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIBfDCCASagAwIBAgIMFo+bQ+Eg8Si30gr4MA0GCSqGSIb3DQEBCwUAMB8xHTAb
BgNVBAMTFGV0Y2QtbWFuYWdlci1jYS1tYWluMB4XDTIxMDcwNTIwMTE0NloXDTMx
MDcwNTIwMTE0NlowHzEdMBsGA1UEAxMUZXRjZC1tYW5hZ2VyLWNhLW1haW4wXDAN
BgkqhkiG9w0BAQEFAANLADBIAkEAw33jzcd/iosN04b0WXbDt7B0c3sJ3aafcGLP
vG3xRB9N5bYr9+qZAq3mzAFkxscn4j1ce5b1/GKTDEAClmZgdQIDAQABo0IwQDAO
BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUE/h+3gDP
DvKwHRyiYlXM8voZ1wowDQYJKoZIhvcNAQELBQADQQBXuimeEoAOu5HN4hG7NqL9
t40K3ZRhRZv3JQWnRVJCBDjg1rD0GQJR/n+DoWvbeijI5C9pNjr2pWSIYR1eYCvd
-----END CERTIFICATE-----
etcd-peers-ca-events: |
-----BEGIN CERTIFICATE-----
MIIBfDCCASagAwIBAgIMFo+bKjmxTPh3/lYJMA0GCSqGSIb3DQEBCwUAMB8xHTAb
BgNVBAMTFGV0Y2QtcGVlcnMtY2EtZXZlbnRzMB4XDTIxMDcwNTIwMDk1NloXDTMx
MDcwNTIwMDk1NlowHzEdMBsGA1UEAxMUZXRjZC1wZWVycy1jYS1ldmVudHMwXDAN
BgkqhkiG9w0BAQEFAANLADBIAkEAv5g4HF2xmrYyouJfY9jXx1M3gPLD/pupvxPY
xyjJw5pNCy5M5XGS3iTqRD5RDE0fWudVHFZKLIe8WPc06NApXwIDAQABo0IwQDAO
BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUf6xiDI+O
Yph1ziCGr2hZaQYt+fUwDQYJKoZIhvcNAQELBQADQQBBxj5hqEQstonTb8lnqeGB
DEYtUeAk4eR/HzvUMjF52LVGuvN3XVt+JTrFeKNvb6/RDUbBNRj3azalcUkpPh6V
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIBfDCCASagAwIBAgIMFo+bQ+Eq69jgzpKwMA0GCSqGSIb3DQEBCwUAMB8xHTAb
BgNVBAMTFGV0Y2QtcGVlcnMtY2EtZXZlbnRzMB4XDTIxMDcwNTIwMTE0NloXDTMx
MDcwNTIwMTE0NlowHzEdMBsGA1UEAxMUZXRjZC1wZWVycy1jYS1ldmVudHMwXDAN
BgkqhkiG9w0BAQEFAANLADBIAkEAo5Nj2CjX1qp3mEPw1H5nHAFWLoGNSLSlRFJW
03NxaNPMFzL5PrCoyOXrX8/MWczuZYw0Crf8EPOOQWi2+W0XLwIDAQABo0IwQDAO
BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUxauhhKQh
cvdZND78rHe0RQVTTiswDQYJKoZIhvcNAQELBQADQQB+cq4jIS9q0zXslaRa+ViI
J+dviA3sMygbmSJO0s4DxYmoazKJblux5q0ASSvS9iL1l9ShuZ1dWyp2tpZawHyb
-----END CERTIFICATE-----
etcd-peers-ca-main: |
-----BEGIN CERTIFICATE-----
MIIBeDCCASKgAwIBAgIMFo+bKjmuLDDLcDHsMA0GCSqGSIb3DQEBCwUAMB0xGzAZ
BgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjAeFw0yMTA3MDUyMDA5NTZaFw0zMTA3
MDUyMDA5NTZaMB0xGzAZBgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjBcMA0GCSqG
SIb3DQEBAQUAA0sAMEgCQQCyRaXWpwgN6INQqws9p/BvPElJv2Rno9dVTFhlQqDA
aUJXe7MBmiO4NJcW76EozeBh5ztR3/4NE1FM2x8TisS3AgMBAAGjQjBAMA4GA1Ud
DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQtE1d49uSvpURf
OQ25Vlu6liY20DANBgkqhkiG9w0BAQsFAANBAAgLVaetJZcfOA3OIMMvQbz2Ydrt
uWF9BKkIad8jrcIrm3IkOtR8bKGmDIIaRKuG/ZUOL6NMe2fky3AAfKwleL4=
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIBeDCCASKgAwIBAgIMFo+bQ+EuVthBfuZvMA0GCSqGSIb3DQEBCwUAMB0xGzAZ
BgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjAeFw0yMTA3MDUyMDExNDZaFw0zMTA3
MDUyMDExNDZaMB0xGzAZBgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjBcMA0GCSqG
SIb3DQEBAQUAA0sAMEgCQQCxNbycDZNx5V1ZOiXxZSvaFpHRwKeHDfcuMUitdoPt
naVMlMTGDWAMuCVmFHFAWohIYynemEegmZkZ15S7AErfAgMBAAGjQjBAMA4GA1Ud
DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTAjQ8T4HclPIsC
qipEfUIcLP6jqTANBgkqhkiG9w0BAQsFAANBAJdZ17TN3HlWrH7HQgfR12UBwz8K
G9DurDznVaBVUYaHY8Sg5AvAXeb+yIF2JMmRR+bK+/G1QYY2D3/P31Ic2Oo=
-----END CERTIFICATE-----
kubernetes-ca: |
-----BEGIN CERTIFICATE-----
MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU
BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw
ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1
jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA
MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8
tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w==
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU
BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw
OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7
WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn
MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA
9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw==
-----END CERTIFICATE-----
ClusterName: minimal.k8s.local
FileAssets:
- content: |
apiVersion: kubescheduler.config.k8s.io/v1beta2
clientConnection:
kubeconfig: /var/lib/kube-scheduler/kubeconfig
kind: KubeSchedulerConfiguration
path: /var/lib/kube-scheduler/config.yaml
Hooks:
- null
- null
KeypairIDs:
apiserver-aggregator-ca: "6980187172486667078076483355"
etcd-clients-ca: "6979622252718071085282986282"
etcd-manager-ca-events: "6982279354000777253151890266"
etcd-manager-ca-main: "6982279354000936168671127624"
etcd-peers-ca-events: "6982279353999767935825892873"
etcd-peers-ca-main: "6982279353998887468930183660"
kubernetes-ca: "6982820025135291416230495506"
service-account: "2"
KubeletConfig:
anonymousAuth: false
cgroupDriver: systemd
cgroupRoot: /
cloudProvider: external
clusterDNS: 100.64.0.10
clusterDomain: cluster.local
enableDebuggingHandlers: true
evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
kubeconfigPath: /var/lib/kubelet/kubeconfig
logLevel: 2
nodeLabels:
kops.k8s.io/kops-controller-pki: ""
node-role.kubernetes.io/control-plane: ""
node.kubernetes.io/exclude-from-external-load-balancers: ""
podInfraContainerImage: registry.k8s.io/pause:3.6
podManifestPath: /etc/kubernetes/manifests
protectKernelDefaults: true
registerSchedulable: false
shutdownGracePeriod: 30s
shutdownGracePeriodCriticalPods: 10s
UpdatePolicy: automatic
channels:
- memfs://tests/minimal.k8s.local/addons/bootstrap-channel.yaml
containerdConfig:
logLevel: info
runc:
version: 1.1.3
version: 1.6.8
etcdManifests:
- memfs://tests/minimal.k8s.local/manifests/etcd/main-master-fsn1.yaml
- memfs://tests/minimal.k8s.local/manifests/etcd/events-master-fsn1.yaml
staticManifests:
- key: kube-apiserver-healthcheck
path: manifests/static/kube-apiserver-healthcheck.yaml

73
tests/integration/update_cluster/minimal_hetzner/data/aws_s3_object_nodeupconfig-nodes-fsn1_content Normal file
View File

@ -0,0 +1,73 @@
Assets:
amd64:
- 7f9183fce12606818612ce80b6c09757452c4fb50aefea5fc5843951c5020e24@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/amd64/kubelet
- e23cc7092218c95c22d8ee36fb9499194a36ac5b5349ca476886b7edc0203885@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/amd64/kubectl
- 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
- 3a1322c18ee5ff4b9bd5af6b7b30c923a3eab8af1df05554f530ef8e2b24ac5e@https://github.com/containerd/containerd/releases/download/v1.6.8/containerd-1.6.8-linux-amd64.tar.gz
- 6e8b24be90fffce6b025d254846da9d2ca6d65125f9139b6354bab0272253d01@https://github.com/opencontainers/runc/releases/download/v1.1.3/runc.amd64
- f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64
- 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64
arm64:
- 69572a7b3d179d4a479aa2e0f90e2f091d8d84ef33a35422fc89975dc137a590@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/arm64/kubelet
- 24db547bbae294c5c44f2b4a777e45f0e2f3d6295eace0d0c4be2b2dfa45330d@https://storage.googleapis.com/kubernetes-release/release/v1.25.0/bin/linux/arm64/kubectl
- ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
- b114e36ecce78cef9d611416c01b784a420928c82766d6df7dc02b10d9da94cd@https://github.com/containerd/containerd/releases/download/v1.6.8/containerd-1.6.8-linux-arm64.tar.gz
- 00c9ad161a77a01d9dcbd25b1d76fa9822e57d8e4abf26ba8907c98f6bcfcd0f@https://github.com/opencontainers/runc/releases/download/v1.1.3/runc.arm64
- 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64
- 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64
CAs:
kubernetes-ca: |
-----BEGIN CERTIFICATE-----
MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU
BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw
ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1
jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA
MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8
tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w==
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU
BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw
OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD
SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7
WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG
MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn
MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA
9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw==
-----END CERTIFICATE-----
ClusterName: minimal.k8s.local
Hooks:
- null
- null
KeypairIDs:
kube-proxy: "6986354184403674830529235586"
kubelet: "6986354184404014133128804066"
kubernetes-ca: "6982820025135291416230495506"
KubeletConfig:
anonymousAuth: false
cgroupDriver: systemd
cgroupRoot: /
cloudProvider: external
clusterDNS: 100.64.0.10
clusterDomain: cluster.local
enableDebuggingHandlers: true
evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
kubeconfigPath: /var/lib/kubelet/kubeconfig
logLevel: 2
nodeLabels:
node-role.kubernetes.io/node: ""
podInfraContainerImage: registry.k8s.io/pause:3.6
podManifestPath: /etc/kubernetes/manifests
protectKernelDefaults: true
shutdownGracePeriod: 30s
shutdownGracePeriodCriticalPods: 10s
UpdatePolicy: automatic
channels:
- memfs://tests/minimal.k8s.local/addons/bootstrap-channel.yaml
containerdConfig:
logLevel: info
runc:
version: 1.1.3
version: 1.6.8

256
tests/integration/update_cluster/minimal_hetzner/data/hcloud_server_master-fsn1_user_data Normal file
View File

@ -0,0 +1,256 @@
#!/bin/bash
set -o errexit
set -o nounset
set -o pipefail
NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64
NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924
NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64
NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865
export HCLOUD_TOKEN=REDACTED
sysctl -w net.core.rmem_max=16777216 || true
sysctl -w net.core.wmem_max=16777216 || true
sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true
sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true
function ensure-install-dir() {
INSTALL_DIR="/opt/kops"
# On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
if [[ -d /var/lib/toolbox ]]; then
INSTALL_DIR="/var/lib/toolbox/kops"
fi
mkdir -p ${INSTALL_DIR}/bin
mkdir -p ${INSTALL_DIR}/conf
cd ${INSTALL_DIR}
}
# Retry a download until we get it. args: name, sha, urls
download-or-bust() {
local -r file="$1"
local -r hash="$2"
local -r urls=( $(split-commas "$3") )
if [[ -f "${file}" ]]; then
if ! validate-hash "${file}" "${hash}"; then
rm -f "${file}"
else
return 0
fi
fi
while true; do
for url in "${urls[@]}"; do
commands=(
"curl -f --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
"wget --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
"curl -f -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
"wget -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
)
for cmd in "${commands[@]}"; do
echo "Attempting download with: ${cmd} {url}"
if ! (${cmd} "${url}"); then
echo "== Download failed with ${cmd} =="
continue
fi
if ! validate-hash "${file}" "${hash}"; then
echo "== Hash validation of ${url} failed. Retrying. =="
rm -f "${file}"
else
echo "== Downloaded ${url} (SHA256 = ${hash}) =="
return 0
fi
done
done
echo "All downloads failed; sleeping before retrying"
sleep 60
done
}
validate-hash() {
local -r file="$1"
local -r expected="$2"
local actual
actual=$(sha256sum ${file} | awk '{ print $1 }') || true
if [[ "${actual}" != "${expected}" ]]; then
echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} =="
return 1
fi
}
function split-commas() {
echo $1 | tr "," "\n"
}
function download-release() {
case "$(uname -m)" in
x86_64*|i?86_64*|amd64*)
NODEUP_URL="${NODEUP_URL_AMD64}"
NODEUP_HASH="${NODEUP_HASH_AMD64}"
;;
aarch64*|arm64*)
NODEUP_URL="${NODEUP_URL_ARM64}"
NODEUP_HASH="${NODEUP_HASH_ARM64}"
;;
*)
echo "Unsupported host arch: $(uname -m)" >&2
exit 1
;;
esac
cd ${INSTALL_DIR}/bin
download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}"
chmod +x nodeup
echo "Running nodeup"
# We can't run in the foreground because of https://github.com/docker/docker/issues/23793
( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 )
}
####################################################################################
/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
echo "== nodeup node config starting =="
ensure-install-dir
cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
cloudConfig:
manageStorageClasses: true
containerRuntime: containerd
containerd:
logLevel: info
runc:
version: 1.1.3
version: 1.6.8
docker:
skipInstall: true
encryptionConfig: null
etcdClusters:
events:
cpuRequest: 100m
memoryRequest: 100Mi
version: 3.5.4
main:
cpuRequest: 200m
memoryRequest: 100Mi
version: 3.5.4
kubeAPIServer:
allowPrivileged: true
anonymousAuth: false
apiAudiences:
- kubernetes.svc.default
apiServerCount: 1
authorizationMode: Node,RBAC
bindAddress: 0.0.0.0
cloudProvider: external
enableAdmissionPlugins:
- NamespaceLifecycle
- LimitRanger
- ServiceAccount
- DefaultStorageClass
- DefaultTolerationSeconds
- MutatingAdmissionWebhook
- ValidatingAdmissionWebhook
- NodeRestriction
- ResourceQuota
etcdServers:
- https://127.0.0.1:4001
etcdServersOverrides:
- /events#https://127.0.0.1:4002
image: registry.k8s.io/kube-apiserver:v1.25.0
kubeletPreferredAddressTypes:
- InternalIP
- Hostname
- ExternalIP
logLevel: 2
requestheaderAllowedNames:
- aggregator
requestheaderExtraHeaderPrefixes:
- X-Remote-Extra-
requestheaderGroupHeaders:
- X-Remote-Group
requestheaderUsernameHeaders:
- X-Remote-User
securePort: 443
serviceAccountIssuer: https://api.internal.minimal.k8s.local
serviceAccountJWKSURI: https://api.internal.minimal.k8s.local/openid/v1/jwks
serviceClusterIPRange: 100.64.0.0/13
storageBackend: etcd3
kubeControllerManager:
allocateNodeCIDRs: true
attachDetachReconcileSyncPeriod: 1m0s
cloudProvider: external
clusterCIDR: 100.96.0.0/11
clusterName: minimal.k8s.local
configureCloudRoutes: false
image: registry.k8s.io/kube-controller-manager:v1.25.0
leaderElection:
leaderElect: true
logLevel: 2
useServiceAccountCredentials: true
kubeProxy:
clusterCIDR: 100.96.0.0/11
cpuRequest: 100m
image: registry.k8s.io/kube-proxy:v1.25.0
logLevel: 2
kubeScheduler:
image: registry.k8s.io/kube-scheduler:v1.25.0
leaderElection:
leaderElect: true
logLevel: 2
kubelet:
anonymousAuth: false
cgroupDriver: systemd
cgroupRoot: /
cloudProvider: external
clusterDNS: 100.64.0.10
clusterDomain: cluster.local
enableDebuggingHandlers: true
evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
kubeconfigPath: /var/lib/kubelet/kubeconfig
logLevel: 2
podInfraContainerImage: registry.k8s.io/pause:3.6
podManifestPath: /etc/kubernetes/manifests
protectKernelDefaults: true
shutdownGracePeriod: 30s
shutdownGracePeriodCriticalPods: 10s
masterKubelet:
anonymousAuth: false
cgroupDriver: systemd
cgroupRoot: /
cloudProvider: external
clusterDNS: 100.64.0.10
clusterDomain: cluster.local
enableDebuggingHandlers: true
evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
kubeconfigPath: /var/lib/kubelet/kubeconfig
logLevel: 2
podInfraContainerImage: registry.k8s.io/pause:3.6
podManifestPath: /etc/kubernetes/manifests
protectKernelDefaults: true
registerSchedulable: false
shutdownGracePeriod: 30s
shutdownGracePeriodCriticalPods: 10s
__EOF_CLUSTER_SPEC
cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
CloudProvider: hetzner
ConfigBase: memfs://tests/minimal.k8s.local
InstanceGroupName: master-fsn1
InstanceGroupRole: Master
NodeupConfigHash: AQ0Tb2H1o5VR7ckXuEeo1dtYXguWwlMrWgDBefTICdQ=
__EOF_KUBE_ENV
download-release
echo "== nodeup node config done =="

170
tests/integration/update_cluster/minimal_hetzner/data/hcloud_server_nodes-fsn1_user_data Normal file
View File

@ -0,0 +1,170 @@
#!/bin/bash
set -o errexit
set -o nounset
set -o pipefail
NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64
NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924
NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64
NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865
export HCLOUD_TOKEN=REDACTED
sysctl -w net.core.rmem_max=16777216 || true
sysctl -w net.core.wmem_max=16777216 || true
sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true
sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true
function ensure-install-dir() {
INSTALL_DIR="/opt/kops"
# On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
if [[ -d /var/lib/toolbox ]]; then
INSTALL_DIR="/var/lib/toolbox/kops"
fi
mkdir -p ${INSTALL_DIR}/bin
mkdir -p ${INSTALL_DIR}/conf
cd ${INSTALL_DIR}
}
# Retry a download until we get it. args: name, sha, urls
download-or-bust() {
local -r file="$1"
local -r hash="$2"
local -r urls=( $(split-commas "$3") )
if [[ -f "${file}" ]]; then
if ! validate-hash "${file}" "${hash}"; then
rm -f "${file}"
else
return 0
fi
fi
while true; do
for url in "${urls[@]}"; do
commands=(
"curl -f --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
"wget --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
"curl -f -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
"wget -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
)
for cmd in "${commands[@]}"; do
echo "Attempting download with: ${cmd} {url}"
if ! (${cmd} "${url}"); then
echo "== Download failed with ${cmd} =="
continue
fi
if ! validate-hash "${file}" "${hash}"; then
echo "== Hash validation of ${url} failed. Retrying. =="
rm -f "${file}"
else
echo "== Downloaded ${url} (SHA256 = ${hash}) =="
return 0
fi
done
done
echo "All downloads failed; sleeping before retrying"
sleep 60
done
}
validate-hash() {
local -r file="$1"
local -r expected="$2"
local actual
actual=$(sha256sum ${file} | awk '{ print $1 }') || true
if [[ "${actual}" != "${expected}" ]]; then
echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} =="
return 1
fi
}
function split-commas() {
echo $1 | tr "," "\n"
}
function download-release() {
case "$(uname -m)" in
x86_64*|i?86_64*|amd64*)
NODEUP_URL="${NODEUP_URL_AMD64}"
NODEUP_HASH="${NODEUP_HASH_AMD64}"
;;
aarch64*|arm64*)
NODEUP_URL="${NODEUP_URL_ARM64}"
NODEUP_HASH="${NODEUP_HASH_ARM64}"
;;
*)
echo "Unsupported host arch: $(uname -m)" >&2
exit 1
;;
esac
cd ${INSTALL_DIR}/bin
download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}"
chmod +x nodeup
echo "Running nodeup"
# We can't run in the foreground because of https://github.com/docker/docker/issues/23793
( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 )
}
####################################################################################
/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
echo "== nodeup node config starting =="
ensure-install-dir
cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
cloudConfig:
manageStorageClasses: true
containerRuntime: containerd
containerd:
logLevel: info
runc:
version: 1.1.3
version: 1.6.8
docker:
skipInstall: true
kubeProxy:
clusterCIDR: 100.96.0.0/11
cpuRequest: 100m
image: registry.k8s.io/kube-proxy:v1.25.0
logLevel: 2
kubelet:
anonymousAuth: false
cgroupDriver: systemd
cgroupRoot: /
cloudProvider: external
clusterDNS: 100.64.0.10
clusterDomain: cluster.local
enableDebuggingHandlers: true
evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
kubeconfigPath: /var/lib/kubelet/kubeconfig
logLevel: 2
podInfraContainerImage: registry.k8s.io/pause:3.6
podManifestPath: /etc/kubernetes/manifests
protectKernelDefaults: true
shutdownGracePeriod: 30s
shutdownGracePeriodCriticalPods: 10s
__EOF_CLUSTER_SPEC
cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
CloudProvider: hetzner
ConfigBase: memfs://tests/minimal.k8s.local
InstanceGroupName: nodes-fsn1
InstanceGroupRole: Node
NodeupConfigHash: GUWrj4JRGMwY7t63XODDLhZuHqkebReNXmgyEzCZE0c=
__EOF_KUBE_ENV
download-release
echo "== nodeup node config done =="

1
tests/integration/update_cluster/minimal_hetzner/id_rsa.pub Executable file
View File

@ -0,0 +1 @@
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==

89
tests/integration/update_cluster/minimal_hetzner/in-v1alpha2.yaml Normal file
View File

@ -0,0 +1,89 @@
apiVersion: kops.k8s.io/v1alpha2
kind: Cluster
metadata:
creationTimestamp: "2017-01-01T00:00:00Z"
name: minimal.k8s.local
spec:
api:
loadBalancer:
type: Public
authorization:
rbac: {}
channel: stable
cloudProvider: hetzner
configBase: memfs://tests/minimal.k8s.local
etcdClusters:
- cpuRequest: 200m
etcdMembers:
- instanceGroup: master-fsn1
name: etcd-1
memoryRequest: 100Mi
name: main
- cpuRequest: 100m
etcdMembers:
- instanceGroup: master-fsn1
name: etcd-1
memoryRequest: 100Mi
name: events
iam:
allowContainerRegistry: true
legacy: false
kubelet:
anonymousAuth: false
kubernetesApiAccess:
- 0.0.0.0/0
- ::/0
kubernetesVersion: v1.25.0
masterPublicName: api.minimal.k8s.local
networkCIDR: 10.0.0.0/16
networking:
cni: {}
nonMasqueradeCIDR: 100.64.0.0/10
sshAccess:
- 0.0.0.0/0
- ::/0
subnets:
- name: fsn1
type: Public
zone: fsn1
topology:
dns:
type: Public
masters: public
nodes: public
---
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: "2017-01-01T00:00:00Z"
labels:
kops.k8s.io/cluster: minimal.k8s.local
name: master-fsn1
spec:
image: ubuntu-20.04
machineType: cx21
maxSize: 1
minSize: 1
role: Master
subnets:
- fsn1
---
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: "2017-01-01T00:00:00Z"
labels:
kops.k8s.io/cluster: minimal.k8s.local
name: nodes-fsn1
spec:
image: ubuntu-20.04
machineType: cx21
maxSize: 1
minSize: 1
role: Node
subnets:
- fsn1

325
tests/integration/update_cluster/minimal_hetzner/kubernetes.tf Normal file
View File

@ -0,0 +1,325 @@
locals {
cluster_name = "minimal.k8s.local"
region = "eu-central"
}
output "cluster_name" {
value = "minimal.k8s.local"
}
output "region" {
value = "eu-central"
}
provider "hcloud" {
}
provider "aws" {
alias = "files"
region = "us-test-1"
}
resource "aws_s3_object" "cluster-completed-spec" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_cluster-completed.spec_content")
key = "tests/minimal.k8s.local/cluster-completed.spec"
provider = aws.files
server_side_encryption = "AES256"
}
resource "aws_s3_object" "etcd-cluster-spec-events" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_etcd-cluster-spec-events_content")
key = "tests/minimal.k8s.local/backups/etcd/events/control/etcd-cluster-spec"
provider = aws.files
server_side_encryption = "AES256"
}
resource "aws_s3_object" "etcd-cluster-spec-main" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_etcd-cluster-spec-main_content")
key = "tests/minimal.k8s.local/backups/etcd/main/control/etcd-cluster-spec"
provider = aws.files
server_side_encryption = "AES256"
}
resource "aws_s3_object" "kops-version-txt" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_kops-version.txt_content")
key = "tests/minimal.k8s.local/kops-version.txt"
provider = aws.files
server_side_encryption = "AES256"
}
resource "aws_s3_object" "manifests-etcdmanager-events-master-fsn1" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events-master-fsn1_content")
key = "tests/minimal.k8s.local/manifests/etcd/events-master-fsn1.yaml"
provider = aws.files
server_side_encryption = "AES256"
}
resource "aws_s3_object" "manifests-etcdmanager-main-master-fsn1" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main-master-fsn1_content")
key = "tests/minimal.k8s.local/manifests/etcd/main-master-fsn1.yaml"
provider = aws.files
server_side_encryption = "AES256"
}
resource "aws_s3_object" "manifests-static-kube-apiserver-healthcheck" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content")
key = "tests/minimal.k8s.local/manifests/static/kube-apiserver-healthcheck.yaml"
provider = aws.files
server_side_encryption = "AES256"
}
resource "aws_s3_object" "minimal-k8s-local-addons-bootstrap" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_minimal.k8s.local-addons-bootstrap_content")
key = "tests/minimal.k8s.local/addons/bootstrap-channel.yaml"
provider = aws.files
server_side_encryption = "AES256"
}
resource "aws_s3_object" "minimal-k8s-local-addons-coredns-addons-k8s-io-k8s-1-12" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_minimal.k8s.local-addons-coredns.addons.k8s.io-k8s-1.12_content")
key = "tests/minimal.k8s.local/addons/coredns.addons.k8s.io/k8s-1.12.yaml"
provider = aws.files
server_side_encryption = "AES256"
}
resource "aws_s3_object" "minimal-k8s-local-addons-dns-controller-addons-k8s-io-k8s-1-12" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_minimal.k8s.local-addons-dns-controller.addons.k8s.io-k8s-1.12_content")
key = "tests/minimal.k8s.local/addons/dns-controller.addons.k8s.io/k8s-1.12.yaml"
provider = aws.files
server_side_encryption = "AES256"
}
resource "aws_s3_object" "minimal-k8s-local-addons-hcloud-cloud-controller-addons-k8s-io-k8s-1-22" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_minimal.k8s.local-addons-hcloud-cloud-controller.addons.k8s.io-k8s-1.22_content")
key = "tests/minimal.k8s.local/addons/hcloud-cloud-controller.addons.k8s.io/k8s-1.22.yaml"
provider = aws.files
server_side_encryption = "AES256"
}
resource "aws_s3_object" "minimal-k8s-local-addons-hcloud-csi-driver-addons-k8s-io-k8s-1-22" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_minimal.k8s.local-addons-hcloud-csi-driver.addons.k8s.io-k8s-1.22_content")
key = "tests/minimal.k8s.local/addons/hcloud-csi-driver.addons.k8s.io/k8s-1.22.yaml"
provider = aws.files
server_side_encryption = "AES256"
}
resource "aws_s3_object" "minimal-k8s-local-addons-kops-controller-addons-k8s-io-k8s-1-16" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_minimal.k8s.local-addons-kops-controller.addons.k8s.io-k8s-1.16_content")
key = "tests/minimal.k8s.local/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml"
provider = aws.files
server_side_encryption = "AES256"
}
resource "aws_s3_object" "minimal-k8s-local-addons-kubelet-api-rbac-addons-k8s-io-k8s-1-9" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_minimal.k8s.local-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content")
key = "tests/minimal.k8s.local/addons/kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml"
provider = aws.files
server_side_encryption = "AES256"
}
resource "aws_s3_object" "minimal-k8s-local-addons-limit-range-addons-k8s-io" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_minimal.k8s.local-addons-limit-range.addons.k8s.io_content")
key = "tests/minimal.k8s.local/addons/limit-range.addons.k8s.io/v1.5.0.yaml"
provider = aws.files
server_side_encryption = "AES256"
}
resource "aws_s3_object" "minimal-k8s-local-addons-rbac-addons-k8s-io-k8s-1-8" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_minimal.k8s.local-addons-rbac.addons.k8s.io-k8s-1.8_content")
key = "tests/minimal.k8s.local/addons/rbac.addons.k8s.io/k8s-1.8.yaml"
provider = aws.files
server_side_encryption = "AES256"
}
resource "aws_s3_object" "nodeupconfig-master-fsn1" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_nodeupconfig-master-fsn1_content")
key = "tests/minimal.k8s.local/igconfig/master/master-fsn1/nodeupconfig.yaml"
provider = aws.files
server_side_encryption = "AES256"
}
resource "aws_s3_object" "nodeupconfig-nodes-fsn1" {
bucket = "testingBucket"
content = file("${path.module}/data/aws_s3_object_nodeupconfig-nodes-fsn1_content")
key = "tests/minimal.k8s.local/igconfig/node/nodes-fsn1/nodeupconfig.yaml"
provider = aws.files
server_side_encryption = "AES256"
}
resource "hcloud_firewall" "control-plane-minimal-k8s-local" {
apply_to {
label_selector = "kops.k8s.io/cluster=minimal.k8s.local,kops.k8s.io/instance-role=Master"
}
labels = {
"kops.k8s.io/cluster" = "minimal.k8s.local"
"kops.k8s.io/firewall-role" = "control-plane"
}
name = "control-plane.minimal.k8s.local"
rule {
direction = "in"
port = "22"
protocol = "tcp"
source_ips = ["0.0.0.0/0", "::/0"]
}
}
resource "hcloud_firewall" "nodes-minimal-k8s-local" {
apply_to {
label_selector = "kops.k8s.io/cluster=minimal.k8s.local,kops.k8s.io/instance-role=Node"
}
labels = {
"kops.k8s.io/cluster" = "minimal.k8s.local"
"kops.k8s.io/firewall-role" = "nodes"
}
name = "nodes.minimal.k8s.local"
rule {
direction = "in"
port = "22"
protocol = "tcp"
source_ips = ["0.0.0.0/0", "::/0"]
}
}
resource "hcloud_load_balancer" "api-minimal-k8s-local" {
labels = {
"kops.k8s.io/cluster" = "minimal.k8s.local"
}
load_balancer_type = "lb11"
location = "fsn1"
name = "api.minimal.k8s.local"
}
resource "hcloud_load_balancer_network" "api-minimal-k8s-local" {
load_balancer_id = hcloud_load_balancer.api-minimal-k8s-local.id
network_id = hcloud_network.minimal-k8s-local.id
}
resource "hcloud_load_balancer_service" "api-minimal-k8s-local" {
destination_port = 443
listen_port = 443
load_balancer_id = hcloud_load_balancer.api-minimal-k8s-local.id
protocol = "tcp"
}
resource "hcloud_load_balancer_target" "api-minimal-k8s-local" {
label_selector = "kops.k8s.io/cluster=minimal.k8s.local,kops.k8s.io/instance-role=Master"
load_balancer_id = hcloud_load_balancer.api-minimal-k8s-local.id
type = "label_selector"
use_private_ip = true
}
resource "hcloud_network" "minimal-k8s-local" {
ip_range = "10.0.0.0/16"
labels = {
"kops.k8s.io/cluster" = "minimal.k8s.local"
}
name = "minimal.k8s.local"
}
resource "hcloud_network_subnet" "minimal-k8s-local-10-0-0-0--16" {
ip_range = "10.0.0.0/16"
network_id = hcloud_network.minimal-k8s-local.id
network_zone = "eu-central"
type = "cloud"
}
resource "hcloud_server" "master-fsn1" {
count = 1
image = "ubuntu-20.04"
labels = {
"kops.k8s.io/cluster" = "minimal.k8s.local"
"kops.k8s.io/instance-group" = "master-fsn1"
"kops.k8s.io/instance-role" = "Master"
}
location = "fsn1"
name = "master-fsn1-${count.index}"
network {
network_id = hcloud_network.minimal-k8s-local.id
}
public_net {
ipv4_enabled = true
ipv6_enabled = false
}
server_type = "cx21"
ssh_keys = [hcloud_ssh_key.minimal-k8s-local-c4_a6_ed_9a_a8_89_b9_e2_c3_9c_d6_63_eb_9c_71_57.id]
user_data = filebase64("${path.module}/data/hcloud_server_master-fsn1_user_data")
}
resource "hcloud_server" "nodes-fsn1" {
count = 1
image = "ubuntu-20.04"
labels = {
"kops.k8s.io/cluster" = "minimal.k8s.local"
"kops.k8s.io/instance-group" = "nodes-fsn1"
"kops.k8s.io/instance-role" = "Node"
}
location = "fsn1"
name = "nodes-fsn1-${count.index}"
network {
network_id = hcloud_network.minimal-k8s-local.id
}
public_net {
ipv4_enabled = true
ipv6_enabled = false
}
server_type = "cx21"
ssh_keys = [hcloud_ssh_key.minimal-k8s-local-c4_a6_ed_9a_a8_89_b9_e2_c3_9c_d6_63_eb_9c_71_57.id]
user_data = filebase64("${path.module}/data/hcloud_server_nodes-fsn1_user_data")
}
resource "hcloud_ssh_key" "minimal-k8s-local-c4_a6_ed_9a_a8_89_b9_e2_c3_9c_d6_63_eb_9c_71_57" {
labels = {
"kops.k8s.io/cluster" = "minimal.k8s.local"
}
name = "minimal.k8s.local-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57"
public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ=="
}
resource "hcloud_volume" "etcd-1-etcd-events-minimal-k8s-local" {
labels = {
"kops.k8s.io/cluster" = "minimal.k8s.local"
"kops.k8s.io/instance-group" = "master-fsn1"
"kops.k8s.io/volume-role" = "events"
}
location = "fsn1"
name = "etcd-1.etcd-events.minimal.k8s.local"
size = 20
}
resource "hcloud_volume" "etcd-1-etcd-main-minimal-k8s-local" {
labels = {
"kops.k8s.io/cluster" = "minimal.k8s.local"
"kops.k8s.io/instance-group" = "master-fsn1"
"kops.k8s.io/volume-role" = "main"
}
location = "fsn1"
name = "etcd-1.etcd-main.minimal.k8s.local"
size = 20
}
terraform {
required_version = ">= 0.15.0"
required_providers {
hcloud = {
"source" = "hetznercloud/hcloud"
"version" = ">= 1.35.1"
}
}
}

6
upup/pkg/fi/cloudup/apply_cluster.go
View File

@ -88,7 +88,11 @@ const (
)
// TerraformCloudProviders is the list of cloud providers with terraform target support
var TerraformCloudProviders = []kops.CloudProviderID{kops.CloudProviderAWS, kops.CloudProviderGCE}
var TerraformCloudProviders = []kops.CloudProviderID{
kops.CloudProviderAWS,
kops.CloudProviderGCE,
kops.CloudProviderHetzner,
}
type ApplyClusterCmd struct {
Cloud fi.Cloud

52
upup/pkg/fi/cloudup/hetznertasks/firewall.go
View File

@ -24,6 +24,7 @@ import (
"github.com/hetznercloud/hcloud-go/hcloud"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
)
// +kops:fitask
@ -205,3 +206,54 @@ var _ fi.HasDependencies = &FirewallRule{}
func (e *FirewallRule) GetDependencies(tasks map[string]fi.Task) []fi.Task {
return nil
}
type terraformFirewall struct {
Name *string `cty:"name"`
ApplyTo []*terraformFirewallApplyTo `cty:"apply_to"`
Rules []*terraformFirewallRule `cty:"rule"`
Labels map[string]string `cty:"labels"`
}
type terraformFirewallApplyTo struct {
LabelSelector *string `cty:"label_selector"`
}
type terraformFirewallRule struct {
Direction *string `cty:"direction"`
SourceIPs []*string `cty:"source_ips"`
Protocol *string `cty:"protocol"`
Port *string `cty:"port"`
}
func (_ *Firewall) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Firewall) error {
{
tf := &terraformFirewall{
Name: e.Name,
ApplyTo: []*terraformFirewallApplyTo{
{
LabelSelector: fi.String(e.Selector),
},
},
Labels: e.Labels,
}
for _, rule := range e.Rules {
tfr := &terraformFirewallRule{
Direction: fi.String(string(rule.Direction)),
Protocol: fi.String(string(rule.Protocol)),
Port: rule.Port,
}
for _, ip := range rule.SourceIPs {
tfr.SourceIPs = append(tfr.SourceIPs, fi.String(ip.String()))
}
tf.Rules = append(tf.Rules, tfr)
}
err := t.RenderResource("hcloud_firewall", *e.Name, tf)
if err != nil {
return err
}
}
return nil
}

114
upup/pkg/fi/cloudup/hetznertasks/loadbalancer.go
View File

@ -20,10 +20,13 @@ import (
"context"
"fmt"
"strconv"
"strings"
"github.com/hetznercloud/hcloud-go/hcloud"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
"k8s.io/kops/upup/pkg/fi/cloudup/terraformWriter"
)
// +kops:fitask
@ -54,6 +57,11 @@ func (e *LoadBalancer) IsForAPIServer() bool {
}
func (v *LoadBalancer) FindAddresses(c *fi.Context) ([]string, error) {
// TODO(hakman): Use mock to handle this more gracefully
if strings.HasPrefix(c.ClusterConfigBase.Path(), "memfs://tests/") {
return nil, nil
}
ctx := context.TODO()
cloud := c.Cloud.(hetzner.HetznerCloud)
client := cloud.LoadBalancerClient()
@ -203,13 +211,6 @@ func (_ *LoadBalancer) RenderHetzner(t *hetzner.HetznerAPITarget, a, e, changes
Name: e.Location,
},
Labels: e.Labels,
Services: []hcloud.LoadBalancerCreateOptsService{
{
Protocol: hcloud.LoadBalancerServiceProtocolTCP,
ListenPort: fi.Int(443),
DestinationPort: fi.Int(443),
},
},
Targets: []hcloud.LoadBalancerCreateOptsTarget{
{
Type: hcloud.LoadBalancerTargetTypeLabelSelector,
@ -223,6 +224,15 @@ func (_ *LoadBalancer) RenderHetzner(t *hetzner.HetznerAPITarget, a, e, changes
ID: fi.IntValue(e.Network.ID),
},
}
for _, service := range e.Services {
opts.Services = append(opts.Services, hcloud.LoadBalancerCreateOptsService{
Protocol: hcloud.LoadBalancerServiceProtocol(service.Protocol),
ListenPort: service.ListenerPort,
DestinationPort: service.DestinationPort,
})
}
result, _, err := client.Create(ctx, opts)
if err != nil {
return err
@ -301,3 +311,93 @@ var _ fi.HasDependencies = &LoadBalancerService{}
func (e *LoadBalancerService) GetDependencies(tasks map[string]fi.Task) []fi.Task {
return nil
}
type terraformLoadBalancer struct {
Name *string `cty:"name"`
Type *string `cty:"load_balancer_type"`
Location *string `cty:"location"`
Target *terraformLoadBalancerTarget `cty:"target"`
Network *terraformWriter.Literal `cty:"network"`
Labels map[string]string `cty:"labels"`
}
type terraformLoadBalancerNetwork struct {
LoadBalancerID *terraformWriter.Literal `cty:"load_balancer_id"`
NetworkID *terraformWriter.Literal `cty:"network_id"`
}
type terraformLoadBalancerService struct {
LoadBalancerID *terraformWriter.Literal `cty:"load_balancer_id"`
Protocol *string `cty:"protocol"`
ListenPort *int `cty:"listen_port"`
DestinationPort *int `cty:"destination_port"`
}
type terraformLoadBalancerTarget struct {
LoadBalancerID *terraformWriter.Literal `cty:"load_balancer_id"`
Type *string `cty:"type"`
LabelSelector *string `cty:"label_selector"`
UsePrivateIP *bool `cty:"use_private_ip"`
}
func (_ *LoadBalancer) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *LoadBalancer) error {
{
tf := &terraformLoadBalancer{
Name: e.Name,
Type: &e.Type,
Location: &e.Location,
Labels: e.Labels,
}
err := t.RenderResource("hcloud_load_balancer", *e.Name, tf)
if err != nil {
return err
}
}
{
tf := &terraformLoadBalancerNetwork{
LoadBalancerID: e.TerraformLink(),
NetworkID: e.Network.TerraformLink(),
}
err := t.RenderResource("hcloud_load_balancer_network", *e.Name, tf)
if err != nil {
return err
}
}
for _, service := range e.Services {
tf := &terraformLoadBalancerService{
LoadBalancerID: e.TerraformLink(),
Protocol: fi.String(service.Protocol),
ListenPort: service.ListenerPort,
DestinationPort: service.DestinationPort,
}
err := t.RenderResource("hcloud_load_balancer_service", *e.Name, tf)
if err != nil {
return err
}
}
{
tf := &terraformLoadBalancerTarget{
LoadBalancerID: e.TerraformLink(),
Type: fi.String(string(hcloud.LoadBalancerTargetTypeLabelSelector)),
LabelSelector: fi.String(e.Target),
UsePrivateIP: fi.Bool(true),
}
err := t.RenderResource("hcloud_load_balancer_target", *e.Name, tf)
if err != nil {
return err
}
}
return nil
}
func (e *LoadBalancer) TerraformLink() *terraformWriter.Literal {
return terraformWriter.LiteralProperty("hcloud_load_balancer", *e.Name, "id")
}

55
upup/pkg/fi/cloudup/hetznertasks/network.go
View File

@ -25,6 +25,8 @@ import (
"github.com/hetznercloud/hcloud-go/hcloud"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
"k8s.io/kops/upup/pkg/fi/cloudup/terraformWriter"
)
// +kops:fitask
@ -188,3 +190,56 @@ func (_ *Network) RenderHetzner(t *hetzner.HetznerAPITarget, a, e, changes *Netw
return nil
}
type terraformNetwork struct {
Name *string `cty:"name"`
IPRange *string `cty:"ip_range"`
Labels map[string]string `cty:"labels"`
}
type terraformNetworkSubnet struct {
NetworkID *terraformWriter.Literal `cty:"network_id"`
Type *string `cty:"type"`
NetworkZone *string `cty:"network_zone"`
IPRange *string `cty:"ip_range"`
}
func (_ *Network) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Network) error {
{
tf := &terraformNetwork{
Name: e.Name,
IPRange: fi.String(e.IPRange),
Labels: e.Labels,
}
err := t.RenderResource("hcloud_network", *e.Name, tf)
if err != nil {
return err
}
}
for _, subnet := range e.Subnets {
_, subnetIpRange, err := net.ParseCIDR(subnet)
if err != nil {
return err
}
tf := &terraformNetworkSubnet{
NetworkID: e.TerraformLink(),
Type: fi.String(string(hcloud.NetworkSubnetTypeCloud)),
IPRange: fi.String(subnetIpRange.String()),
NetworkZone: fi.String(e.Region),
}
err = t.RenderResource("hcloud_network_subnet", *e.Name+"-"+subnet, tf)
if err != nil {
return err
}
}
return nil
}
func (e *Network) TerraformLink() *terraformWriter.Literal {
return terraformWriter.LiteralProperty("hcloud_network", *e.Name, "id")
}

75
upup/pkg/fi/cloudup/hetznertasks/servergroup.go
View File

@ -27,6 +27,8 @@ import (
"github.com/hetznercloud/hcloud-go/hcloud"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
"k8s.io/kops/upup/pkg/fi/cloudup/terraformWriter"
)
// +kops:fitask
@ -57,7 +59,7 @@ func (v *ServerGroup) Find(c *fi.Context) (*ServerGroup, error) {
labelSelector := []string{
fmt.Sprintf("%s=%s", hetzner.TagKubernetesClusterName, c.Cluster.Name),
fmt.Sprintf("%s=%s", hetzner.TagKubernetesInstanceGroup, v.Labels[hetzner.TagKubernetesInstanceGroup]),
fmt.Sprintf("%s=%s", hetzner.TagKubernetesInstanceGroup, fi.StringValue(v.Name)),
}
listOptions := hcloud.ListOpts{
PerPage: 50,
@ -181,10 +183,10 @@ func (_ *ServerGroup) RenderHetzner(t *hetzner.HetznerAPITarget, a, e, changes *
}
if len(e.SSHKeys) == 0 {
return fmt.Errorf("failed to find ssh keys for server %q", fi.StringValue(e.Name))
return fmt.Errorf("failed to find ssh keys for server group %q", fi.StringValue(e.Name))
}
if e.Network == nil {
return fmt.Errorf("failed to find network for server %q", fi.StringValue(e.Name))
return fmt.Errorf("failed to find network for server group %q", fi.StringValue(e.Name))
}
userData, err := fi.ResourceAsString(e.UserData)
@ -199,7 +201,7 @@ func (_ *ServerGroup) RenderHetzner(t *hetzner.HetznerAPITarget, a, e, changes *
for i := 1; i <= expectedCount-actualCount; i++ {
// Append a random/unique ID to the node name
name := fmt.Sprintf("%s-%x", e.Labels[hetzner.TagKubernetesInstanceGroup], rand.Int63())
name := fmt.Sprintf("%s-%x", fi.StringValue(e.Name), rand.Int63())
opts := hcloud.ServerCreateOpts{
Name: name,
@ -257,3 +259,68 @@ func safeBytesHash(data []byte) string {
return fmt.Sprintf("sha256.%s", safe256)
}
type terraformServer struct {
Count *int `cty:"count"`
Name *terraformWriter.Literal `cty:"name"`
Location *string `cty:"location"`
ServerType *string `cty:"server_type"`
Image *string `cty:"image"`
SSHKeys []*terraformWriter.Literal `cty:"ssh_keys"`
Network []*terraformServerNetwork `cty:"network"`
PublicNet *terraformServerPublicNet `cty:"public_net"`
UserData *terraformWriter.Literal `cty:"user_data"`
Labels map[string]string `cty:"labels"`
}
type terraformServerNetwork struct {
ID *terraformWriter.Literal `cty:"network_id"`
IP *string `cty:"ip"`
AliasIPs *string `cty:"alias_ips"`
MACAddress *string `cty:"mac_address"`
}
type terraformServerPublicNet struct {
EnableIPv4 *bool `cty:"ipv4_enabled"`
EnableIPv6 *bool `cty:"ipv6_enabled"`
}
func (_ *ServerGroup) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *ServerGroup) error {
name := terraformWriter.LiteralWithIndex(fi.StringValue(e.Name))
tf := &terraformServer{
Count: fi.Int(e.Count),
Name: name,
Location: fi.String(e.Location),
ServerType: fi.String(e.Size),
Image: fi.String(e.Image),
Network: []*terraformServerNetwork{
{
ID: e.Network.TerraformLink(),
},
},
PublicNet: &terraformServerPublicNet{
EnableIPv4: fi.Bool(e.EnableIPv4),
EnableIPv6: fi.Bool(e.EnableIPv6),
},
Labels: e.Labels,
}
for _, sshkey := range e.SSHKeys {
tf.SSHKeys = append(tf.SSHKeys, sshkey.TerraformLink())
}
if e.UserData != nil {
data, err := fi.ResourceAsBytes(e.UserData)
if err != nil {
return err
}
if data != nil {
tf.UserData, err = t.AddFileBytes("hcloud_server", fi.StringValue(e.Name), "user_data", data, true)
if err != nil {
return err
}
}
}
return t.RenderResource("hcloud_server", fi.StringValue(e.Name), tf)
}

22
upup/pkg/fi/cloudup/hetznertasks/sshkey.go
View File

@ -26,6 +26,8 @@ import (
"k8s.io/kops/pkg/pki"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
"k8s.io/kops/upup/pkg/fi/cloudup/terraformWriter"
)
// +kops:fitask
@ -127,3 +129,23 @@ func (_ *SSHKey) RenderHetzner(t *hetzner.HetznerAPITarget, a, e, changes *SSHKe
return nil
}
type terraformSSHKey struct {
Name *string `cty:"name"`
PublicKey *string `cty:"public_key"`
Labels map[string]string `cty:"labels"`
}
func (_ *SSHKey) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *SSHKey) error {
tf := &terraformSSHKey{
Name: e.Name,
PublicKey: fi.String(e.PublicKey),
Labels: e.Labels,
}
return t.RenderResource("hcloud_ssh_key", *e.Name, tf)
}
func (e *SSHKey) TerraformLink() *terraformWriter.Literal {
return terraformWriter.LiteralProperty("hcloud_ssh_key", *e.Name, "id")
}

23
upup/pkg/fi/cloudup/hetznertasks/volume.go
View File

@ -23,6 +23,7 @@ import (
"github.com/hetznercloud/hcloud-go/hcloud"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
"k8s.io/kops/upup/pkg/fi/cloudup/terraform"
)
// +kops:fitask
@ -118,14 +119,14 @@ func (_ *Volume) RenderHetzner(t *hetzner.HetznerAPITarget, a, e, changes *Volum
}
} else {
sshkey, _, err := client.Get(context.TODO(), strconv.Itoa(fi.IntValue(a.ID)))
volume, _, err := client.Get(context.TODO(), strconv.Itoa(fi.IntValue(a.ID)))
if err != nil {
return err
}
// Update the labels
if changes.Name != nil || len(changes.Labels) != 0 {
_, _, err := client.Update(context.TODO(), sshkey, hcloud.VolumeUpdateOpts{
_, _, err := client.Update(context.TODO(), volume, hcloud.VolumeUpdateOpts{
Name: fi.StringValue(e.Name),
Labels: e.Labels,
})
@ -137,3 +138,21 @@ func (_ *Volume) RenderHetzner(t *hetzner.HetznerAPITarget, a, e, changes *Volum
return nil
}
type terraformVolume struct {
Name *string `cty:"name"`
Size *int `cty:"size"`
Location *string `cty:"location"`
Labels map[string]string `cty:"labels"`
}
func (_ *Volume) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Volume) error {
tf := &terraformVolume{
Name: e.Name,
Size: fi.Int(e.Size),
Location: fi.String(e.Location),
Labels: e.Labels,
}
return t.RenderResource("hcloud_volume", *e.Name, tf)
}

36
upup/pkg/fi/cloudup/terraform/hcl2.go
View File

@ -112,6 +112,42 @@ func writeLiteral(body *hclwrite.Body, key string, literal *terraformWriter.Lite
},
}
body.SetAttributeRaw(key, tokens)
} else if literal.Index {
tokens := hclwrite.Tokens{
{
Type: hclsyntax.TokenOQuote,
Bytes: []byte(`"`),
},
{
Type: hclsyntax.TokenQuotedLit,
Bytes: []byte(literal.Value),
},
{
Type: hclsyntax.TokenQuotedLit,
Bytes: []byte(`-`),
},
{
Type: hclsyntax.TokenTemplateInterp,
Bytes: []byte(`${`),
},
{
Type: hclsyntax.TokenQuotedLit,
Bytes: []byte(`count.index`),
},
{
Type: hclsyntax.TokenTemplateSeqEnd,
Bytes: []byte(`}`),
},
{
Type: hclsyntax.TokenCQuote,
Bytes: []byte(`"`),
},
{
Type: hclsyntax.TokenEOF,
Bytes: []byte{},
},
}
body.SetAttributeRaw(key, tokens)
} else if len(literal.Tokens) == 0 {
body.SetAttributeValue(key, cty.StringVal(literal.Value))
} else {

12
upup/pkg/fi/cloudup/terraform/target_hcl2.go
View File

@ -42,12 +42,17 @@ func (t *TerraformTarget) finishHCL2() error {
if t.Cloud.ProviderID() == kops.CloudProviderGCE {
providerName = "google"
}
if t.Cloud.ProviderID() == kops.CloudProviderHetzner {
providerName = "hcloud"
}
providerBlock := rootBody.AppendNewBlock("provider", []string{providerName})
providerBody := providerBlock.Body()
if t.Cloud.ProviderID() == kops.CloudProviderGCE {
providerBody.SetAttributeValue("project", cty.StringVal(t.Project))
}
providerBody.SetAttributeValue("region", cty.StringVal(t.Cloud.Region()))
if t.Cloud.ProviderID() != kops.CloudProviderHetzner {
providerBody.SetAttributeValue("region", cty.StringVal(t.Cloud.Region()))
}
for k, v := range tfGetProviderExtraConfig(t.clusterSpecTarget) {
providerBody.SetAttributeValue(k, cty.StringVal(v))
}
@ -110,6 +115,11 @@ func (t *TerraformTarget) finishHCL2() error {
"source": cty.StringVal("hashicorp/google"),
"version": cty.StringVal(">= 2.19.0"),
})
} else if t.Cloud.ProviderID() == kops.CloudProviderHetzner {
writeMap(requiredProvidersBody, "hcloud", map[string]cty.Value{
"source": cty.StringVal("hetznercloud/hcloud"),
"version": cty.StringVal(">= 1.35.1"),
})
} else if t.Cloud.ProviderID() == kops.CloudProviderAWS {
configurationAliases := []*terraformWriter.Literal{terraformWriter.LiteralTokens("aws", "files")}
aliasesType, err := gocty.ImpliedType(configurationAliases)

8
upup/pkg/fi/cloudup/terraformWriter/literal.go
View File

@ -27,8 +27,10 @@ import (
// Literal represents a literal in terraform syntax
type Literal struct {
// Value is used to support Terraform's "${}" interpolation
// Value is used to support Terraform's "${}" interpolation.
Value string `cty:"value"`
// Index to support the index of the count meta-argument.
Index bool `cty:"index"`
// Tokens are portions of a literal reference joined by periods.
// example: {"aws_vpc", "foo", "id"}
@ -80,6 +82,10 @@ func LiteralFromStringValue(s string) *Literal {
return &Literal{Value: s}
}
func LiteralWithIndex(s string) *Literal {
return &Literal{Value: s, Index: true}
}
type literalWithJSON struct {
literal *Literal
key string