From 67e6b3486fca096020103608b5db546b5be04072 Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Mon, 6 Dec 2021 10:13:14 +0100 Subject: [PATCH 01/13] Add integration test for karpenter --- cmd/kops/integration_test.go | 6 + .../karpenter/cloudformation.json | 1312 +++++++++++++++++ .../cloudformation.json.extracted.yaml | 418 ++++++ ...am_role_masters.minimal.example.com_policy | 10 + ..._iam_role_nodes.minimal.example.com_policy | 10 + ..._policy_masters.minimal.example.com_policy | 240 +++ ...le_policy_nodes.minimal.example.com_policy | 41 + ...4a6ed9aa889b9e2c39cd663eb9c7157_public_key | 1 + ...t-1a.masters.minimal.example.com_user_data | 248 ++++ ...mplate_nodes.minimal.example.com_user_data | 168 +++ ...cket_object_cluster-completed.spec_content | 177 +++ ...et_object_etcd-cluster-spec-events_content | 4 + ...cket_object_etcd-cluster-spec-main_content | 4 + ..._s3_bucket_object_kops-version.txt_content | 1 + ...bject_manifests-etcdmanager-events_content | 61 + ..._object_manifests-etcdmanager-main_content | 61 + ...-static-kube-apiserver-healthcheck_content | 32 + ...nimal.example.com-addons-bootstrap_content | 54 + ...mple.com-addons-core.addons.k8s.io_content | 56 + ...ons-coredns.addons.k8s.io-k8s-1.12_content | 380 +++++ ...-controller.addons.k8s.io-k8s-1.12_content | 127 ++ ...-controller.addons.k8s.io-k8s-1.16_content | 208 +++ ...let-api.rbac.addons.k8s.io-k8s-1.9_content | 17 + ...m-addons-limit-range.addons.k8s.io_content | 15 + ...-storage-aws.addons.k8s.io-v1.15.0_content | 98 ++ ...ect_nodeupconfig-master-us-test-1a_content | 265 ++++ ...3_bucket_object_nodeupconfig-nodes_content | 62 + .../update_cluster/karpenter/id_rsa.pub | 1 + .../update_cluster/karpenter/in-v1alpha2.yaml | 78 + .../update_cluster/karpenter/kubernetes.tf | 824 +++++++++++ 30 files changed, 4979 insertions(+) create mode 100644 tests/integration/update_cluster/karpenter/cloudformation.json create mode 100644 tests/integration/update_cluster/karpenter/cloudformation.json.extracted.yaml create mode 100644 tests/integration/update_cluster/karpenter/data/aws_iam_role_masters.minimal.example.com_policy create mode 100644 tests/integration/update_cluster/karpenter/data/aws_iam_role_nodes.minimal.example.com_policy create mode 100644 tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_masters.minimal.example.com_policy create mode 100644 tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_nodes.minimal.example.com_policy create mode 100644 tests/integration/update_cluster/karpenter/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key create mode 100644 tests/integration/update_cluster/karpenter/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data create mode 100644 tests/integration/update_cluster/karpenter/data/aws_launch_template_nodes.minimal.example.com_user_data create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_cluster-completed.spec_content create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_etcd-cluster-spec-events_content create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_etcd-cluster-spec-main_content create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_kops-version.txt_content create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_manifests-etcdmanager-events_content create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_manifests-etcdmanager-main_content create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_manifests-static-kube-apiserver-healthcheck_content create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-core.addons.k8s.io_content create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-limit-range.addons.k8s.io_content create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-nodes_content create mode 100755 tests/integration/update_cluster/karpenter/id_rsa.pub create mode 100644 tests/integration/update_cluster/karpenter/in-v1alpha2.yaml create mode 100644 tests/integration/update_cluster/karpenter/kubernetes.tf diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index a20cb87da9..fea4e06c22 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -594,6 +594,12 @@ func TestExternalDNSIRSA(t *testing.T) { runTestTerraformAWS(t) } +func TestKarpenter(t *testing.T) { + newIntegrationTest("minimal.example.com", "karpenter"). + withAddons(dnsControllerAddon). + runTestTerraformAWS(t) +} + // TestSharedSubnet runs the test on a configuration with a shared subnet (and VPC) func TestSharedSubnet(t *testing.T) { newIntegrationTest("sharedsubnet.example.com", "shared_subnet"). diff --git a/tests/integration/update_cluster/karpenter/cloudformation.json b/tests/integration/update_cluster/karpenter/cloudformation.json new file mode 100644 index 0000000000..db82e8a455 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/cloudformation.json @@ -0,0 +1,1312 @@ +{ + "Resources": { + "AWSAutoScalingAutoScalingGroupmasterustest1amastersminimalexamplecom": { + "Type": "AWS::AutoScaling::AutoScalingGroup", + "Properties": { + "AutoScalingGroupName": "master-us-test-1a.masters.minimal.example.com", + "LaunchTemplate": { + "LaunchTemplateId": { + "Ref": "AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom" + }, + "Version": { + "Fn::GetAtt": [ + "AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom", + "LatestVersionNumber" + ] + } + }, + "MaxSize": "1", + "MinSize": "1", + "VPCZoneIdentifier": [ + { + "Ref": "AWSEC2Subnetustest1aminimalexamplecom" + } + ], + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com", + "PropagateAtLaunch": true + }, + { + "Key": "Name", + "Value": "master-us-test-1a.masters.minimal.example.com", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki", + "Value": "", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "master", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane", + "Value": "", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master", + "Value": "", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers", + "Value": "", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/role/master", + "Value": "1", + "PropagateAtLaunch": true + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "master-us-test-1a", + "PropagateAtLaunch": true + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned", + "PropagateAtLaunch": true + } + ], + "MetricsCollection": [ + { + "Granularity": "1Minute", + "Metrics": [ + "GroupDesiredCapacity", + "GroupInServiceInstances", + "GroupMaxSize", + "GroupMinSize", + "GroupPendingInstances", + "GroupStandbyInstances", + "GroupTerminatingInstances", + "GroupTotalInstances" + ] + } + ] + } + }, + "AWSAutoScalingAutoScalingGroupnodesminimalexamplecom": { + "Type": "AWS::AutoScaling::AutoScalingGroup", + "Properties": { + "AutoScalingGroupName": "nodes.minimal.example.com", + "LaunchTemplate": { + "LaunchTemplateId": { + "Ref": "AWSEC2LaunchTemplatenodesminimalexamplecom" + }, + "Version": { + "Fn::GetAtt": [ + "AWSEC2LaunchTemplatenodesminimalexamplecom", + "LatestVersionNumber" + ] + } + }, + "MaxSize": "2", + "MinSize": "2", + "VPCZoneIdentifier": [ + { + "Ref": "AWSEC2Subnetustest1aminimalexamplecom" + } + ], + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com", + "PropagateAtLaunch": true + }, + { + "Key": "Name", + "Value": "nodes.minimal.example.com", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "node", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node", + "Value": "", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/role/node", + "Value": "1", + "PropagateAtLaunch": true + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "nodes", + "PropagateAtLaunch": true + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned", + "PropagateAtLaunch": true + } + ], + "MetricsCollection": [ + { + "Granularity": "1Minute", + "Metrics": [ + "GroupDesiredCapacity", + "GroupInServiceInstances", + "GroupMaxSize", + "GroupMinSize", + "GroupPendingInstances", + "GroupStandbyInstances", + "GroupTerminatingInstances", + "GroupTotalInstances" + ] + } + ] + } + }, + "AWSEC2DHCPOptionsminimalexamplecom": { + "Type": "AWS::EC2::DHCPOptions", + "Properties": { + "DomainName": "us-test-1.compute.internal", + "DomainNameServers": [ + "AmazonProvidedDNS" + ], + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "minimal.example.com" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2InternetGatewayminimalexamplecom": { + "Type": "AWS::EC2::InternetGateway", + "Properties": { + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "minimal.example.com" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom": { + "Type": "AWS::EC2::LaunchTemplate", + "Properties": { + "LaunchTemplateName": "master-us-test-1a.masters.minimal.example.com", + "LaunchTemplateData": { + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "VolumeType": "gp3", + "VolumeSize": 64, + "Iops": 3000, + "Throughput": 125, + "DeleteOnTermination": true, + "Encrypted": true + } + }, + { + "DeviceName": "/dev/sdc", + "VirtualName": "ephemeral0" + } + ], + "IamInstanceProfile": { + "Name": { + "Ref": "AWSIAMInstanceProfilemastersminimalexamplecom" + } + }, + "ImageId": "ami-12345678", + "InstanceType": "m3.medium", + "KeyName": "kubernetes.minimal.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", + "MetadataOptions": { + "HttpPutResponseHopLimit": 1, + "HttpTokens": "optional" + }, + "Monitoring": { + "Enabled": false + }, + "NetworkInterfaces": [ + { + "AssociatePublicIpAddress": true, + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Ipv6AddressCount": 0, + "Groups": [ + { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + } + ] + } + ], + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "master-us-test-1a.masters.minimal.example.com" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki", + "Value": "" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "master" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane", + "Value": "" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master", + "Value": "" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers", + "Value": "" + }, + { + "Key": "k8s.io/role/master", + "Value": "1" + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "master-us-test-1a" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + }, + { + "ResourceType": "volume", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "master-us-test-1a.masters.minimal.example.com" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki", + "Value": "" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "master" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane", + "Value": "" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master", + "Value": "" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers", + "Value": "" + }, + { + "Key": "k8s.io/role/master", + "Value": "1" + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "master-us-test-1a" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + ], + "UserData": "extracted" + } + } + }, + "AWSEC2LaunchTemplatenodesminimalexamplecom": { + "Type": "AWS::EC2::LaunchTemplate", + "Properties": { + "LaunchTemplateName": "nodes.minimal.example.com", + "LaunchTemplateData": { + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "VolumeType": "gp3", + "VolumeSize": 128, + "Iops": 3000, + "Throughput": 125, + "DeleteOnTermination": true, + "Encrypted": true + } + } + ], + "IamInstanceProfile": { + "Name": { + "Ref": "AWSIAMInstanceProfilenodesminimalexamplecom" + } + }, + "ImageId": "ami-12345678", + "InstanceType": "t2.medium", + "KeyName": "kubernetes.minimal.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", + "MetadataOptions": { + "HttpPutResponseHopLimit": 1, + "HttpTokens": "optional" + }, + "Monitoring": { + "Enabled": false + }, + "NetworkInterfaces": [ + { + "AssociatePublicIpAddress": true, + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Ipv6AddressCount": 0, + "Groups": [ + { + "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" + } + ] + } + ], + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "nodes.minimal.example.com" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "node" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node", + "Value": "" + }, + { + "Key": "k8s.io/role/node", + "Value": "1" + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "nodes" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + }, + { + "ResourceType": "volume", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "nodes.minimal.example.com" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "node" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node", + "Value": "" + }, + { + "Key": "k8s.io/role/node", + "Value": "1" + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "nodes" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + ], + "UserData": "extracted" + } + } + }, + "AWSEC2Route0": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "AWSEC2RouteTableminimalexamplecom" + }, + "DestinationIpv6CidrBlock": "::/0", + "GatewayId": { + "Ref": "AWSEC2InternetGatewayminimalexamplecom" + } + } + }, + "AWSEC2Route00000": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "AWSEC2RouteTableminimalexamplecom" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "AWSEC2InternetGatewayminimalexamplecom" + } + } + }, + "AWSEC2RouteTableminimalexamplecom": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "AWSEC2VPCminimalexamplecom" + }, + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "minimal.example.com" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + }, + { + "Key": "kubernetes.io/kops/role", + "Value": "public" + } + ] + } + }, + "AWSEC2SecurityGroupEgressfrommastersminimalexamplecomegressall0to00": { + "Type": "AWS::EC2::SecurityGroupEgress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1", + "CidrIpv6": "::/0" + } + }, + "AWSEC2SecurityGroupEgressfrommastersminimalexamplecomegressall0to000000": { + "Type": "AWS::EC2::SecurityGroupEgress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1", + "CidrIp": "0.0.0.0/0" + } + }, + "AWSEC2SecurityGroupEgressfromnodesminimalexamplecomegressall0to00": { + "Type": "AWS::EC2::SecurityGroupEgress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1", + "CidrIpv6": "::/0" + } + }, + "AWSEC2SecurityGroupEgressfromnodesminimalexamplecomegressall0to000000": { + "Type": "AWS::EC2::SecurityGroupEgress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1", + "CidrIp": "0.0.0.0/0" + } + }, + "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22mastersminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + }, + "FromPort": 22, + "ToPort": 22, + "IpProtocol": "tcp", + "CidrIp": "0.0.0.0/0" + } + }, + "AWSEC2SecurityGroupIngressfrom00000ingresstcp22to22nodesminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" + }, + "FromPort": 22, + "ToPort": 22, + "IpProtocol": "tcp", + "CidrIp": "0.0.0.0/0" + } + }, + "AWSEC2SecurityGroupIngressfrom00000ingresstcp443to443mastersminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + }, + "FromPort": 443, + "ToPort": 443, + "IpProtocol": "tcp", + "CidrIp": "0.0.0.0/0" + } + }, + "AWSEC2SecurityGroupIngressfrommastersminimalexamplecomingressall0to0mastersminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1" + } + }, + "AWSEC2SecurityGroupIngressfrommastersminimalexamplecomingressall0to0nodesminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1" + } + }, + "AWSEC2SecurityGroupIngressfromnodesminimalexamplecomingressall0to0nodesminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1" + } + }, + "AWSEC2SecurityGroupIngressfromnodesminimalexamplecomingresstcp1to2379mastersminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" + }, + "FromPort": 1, + "ToPort": 2379, + "IpProtocol": "tcp" + } + }, + "AWSEC2SecurityGroupIngressfromnodesminimalexamplecomingresstcp2382to4000mastersminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" + }, + "FromPort": 2382, + "ToPort": 4000, + "IpProtocol": "tcp" + } + }, + "AWSEC2SecurityGroupIngressfromnodesminimalexamplecomingresstcp4003to65535mastersminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" + }, + "FromPort": 4003, + "ToPort": 65535, + "IpProtocol": "tcp" + } + }, + "AWSEC2SecurityGroupIngressfromnodesminimalexamplecomingressudp1to65535mastersminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersminimalexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupnodesminimalexamplecom" + }, + "FromPort": 1, + "ToPort": 65535, + "IpProtocol": "udp" + } + }, + "AWSEC2SecurityGroupmastersminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupName": "masters.minimal.example.com", + "VpcId": { + "Ref": "AWSEC2VPCminimalexamplecom" + }, + "GroupDescription": "Security group for masters", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "masters.minimal.example.com" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2SecurityGroupnodesminimalexamplecom": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupName": "nodes.minimal.example.com", + "VpcId": { + "Ref": "AWSEC2VPCminimalexamplecom" + }, + "GroupDescription": "Security group for nodes", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "nodes.minimal.example.com" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2SubnetRouteTableAssociationustest1aminimalexamplecom": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "SubnetId": { + "Ref": "AWSEC2Subnetustest1aminimalexamplecom" + }, + "RouteTableId": { + "Ref": "AWSEC2RouteTableminimalexamplecom" + } + } + }, + "AWSEC2Subnetustest1aminimalexamplecom": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "VpcId": { + "Ref": "AWSEC2VPCminimalexamplecom" + }, + "CidrBlock": "172.20.32.0/19", + "AvailabilityZone": "us-test-1a", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "us-test-1a.minimal.example.com" + }, + { + "Key": "SubnetType", + "Value": "Public" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + }, + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "kubernetes.io/role/internal-elb", + "Value": "1" + } + ] + } + }, + "AWSEC2VPCCidrBlockAmazonIPv6": { + "Type": "AWS::EC2::VPCCidrBlock", + "Properties": { + "VpcId": { + "Ref": "AWSEC2VPCminimalexamplecom" + }, + "AmazonProvidedIpv6CidrBlock": true + } + }, + "AWSEC2VPCDHCPOptionsAssociationminimalexamplecom": { + "Type": "AWS::EC2::VPCDHCPOptionsAssociation", + "Properties": { + "VpcId": { + "Ref": "AWSEC2VPCminimalexamplecom" + }, + "DhcpOptionsId": { + "Ref": "AWSEC2DHCPOptionsminimalexamplecom" + } + } + }, + "AWSEC2VPCGatewayAttachmentminimalexamplecom": { + "Type": "AWS::EC2::VPCGatewayAttachment", + "Properties": { + "VpcId": { + "Ref": "AWSEC2VPCminimalexamplecom" + }, + "InternetGatewayId": { + "Ref": "AWSEC2InternetGatewayminimalexamplecom" + } + } + }, + "AWSEC2VPCminimalexamplecom": { + "Type": "AWS::EC2::VPC", + "Properties": { + "CidrBlock": "172.20.0.0/16", + "EnableDnsHostnames": true, + "EnableDnsSupport": true, + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "minimal.example.com" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2Volumeustest1aetcdeventsminimalexamplecom": { + "Type": "AWS::EC2::Volume", + "Properties": { + "AvailabilityZone": "us-test-1a", + "Size": 20, + "VolumeType": "gp3", + "Iops": 3000, + "Throughput": 125, + "Encrypted": false, + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "us-test-1a.etcd-events.minimal.example.com" + }, + { + "Key": "k8s.io/etcd/events", + "Value": "us-test-1a/us-test-1a" + }, + { + "Key": "k8s.io/role/master", + "Value": "1" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2Volumeustest1aetcdmainminimalexamplecom": { + "Type": "AWS::EC2::Volume", + "Properties": { + "AvailabilityZone": "us-test-1a", + "Size": 20, + "VolumeType": "gp3", + "Iops": 3000, + "Throughput": 125, + "Encrypted": false, + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "us-test-1a.etcd-main.minimal.example.com" + }, + { + "Key": "k8s.io/etcd/main", + "Value": "us-test-1a/us-test-1a" + }, + { + "Key": "k8s.io/role/master", + "Value": "1" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + }, + "AWSIAMInstanceProfilemastersminimalexamplecom": { + "Type": "AWS::IAM::InstanceProfile", + "Properties": { + "InstanceProfileName": "masters.minimal.example.com", + "Roles": [ + { + "Ref": "AWSIAMRolemastersminimalexamplecom" + } + ] + } + }, + "AWSIAMInstanceProfilenodesminimalexamplecom": { + "Type": "AWS::IAM::InstanceProfile", + "Properties": { + "InstanceProfileName": "nodes.minimal.example.com", + "Roles": [ + { + "Ref": "AWSIAMRolenodesminimalexamplecom" + } + ] + } + }, + "AWSIAMPolicymastersminimalexamplecom": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyName": "masters.minimal.example.com", + "Roles": [ + { + "Ref": "AWSIAMRolemastersminimalexamplecom" + } + ], + "PolicyDocument": { + "Statement": [ + { + "Action": "ec2:AttachVolume", + "Condition": { + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "minimal.example.com", + "aws:ResourceTag/k8s.io/role/master": "1" + } + }, + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "s3:Get*" + ], + "Effect": "Allow", + "Resource": "arn:aws-test:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/*" + }, + { + "Action": [ + "s3:GetObject", + "s3:DeleteObject", + "s3:DeleteObjectVersion", + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": "arn:aws-test:s3:::placeholder-write-bucket/clusters.example.com/minimal.example.com/backups/etcd/main/*" + }, + { + "Action": [ + "s3:GetObject", + "s3:DeleteObject", + "s3:DeleteObjectVersion", + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": "arn:aws-test:s3:::placeholder-write-bucket/clusters.example.com/minimal.example.com/backups/etcd/events/*" + }, + { + "Action": [ + "s3:GetBucketLocation", + "s3:GetEncryptionConfiguration", + "s3:ListBucket", + "s3:ListBucketVersions" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:s3:::placeholder-read-bucket" + ] + }, + { + "Action": [ + "s3:GetBucketLocation", + "s3:GetEncryptionConfiguration", + "s3:ListBucket", + "s3:ListBucketVersions" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:s3:::placeholder-write-bucket" + ] + }, + { + "Action": [ + "route53:ChangeResourceRecordSets", + "route53:ListResourceRecordSets", + "route53:GetHostedZone" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:route53:::hostedzone/Z1AFAKE1ZON3YO" + ] + }, + { + "Action": [ + "route53:GetChange" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:route53:::change/*" + ] + }, + { + "Action": [ + "route53:ListHostedZones", + "route53:ListTagsForResource" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "ec2:CreateAction": [ + "CreateVolume", + "CreateSnapshot" + ] + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:volume/*", + "arn:aws-test:ec2:*:*:snapshot/*" + ] + }, + { + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "ec2:CreateAction": [ + "CreateVolume", + "CreateSnapshot" + ] + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:volume/*", + "arn:aws-test:ec2:*:*:snapshot/*" + ] + }, + { + "Action": "ec2:DeleteTags", + "Condition": { + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:volume/*", + "arn:aws-test:ec2:*:*:snapshot/*" + ] + }, + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:DescribeAccountAttributes", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeTags", + "ec2:DescribeVolumes", + "ec2:DescribeVolumesModifications", + "ec2:DescribeVpcs", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:RegisterTargets", + "iam:GetServerCertificate", + "iam:ListServerCertificates", + "kms:DescribeKey", + "kms:GenerateRandom" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:DeleteRoute", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume", + "ec2:RevokeSecurityGroupIngress", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:DeregisterTargets", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ec2:CreateSecurityGroup", + "ec2:CreateVolume", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateTargetGroup" + ], + "Condition": { + "StringEquals": { + "aws:RequestTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + } + } + }, + "AWSIAMPolicynodesminimalexamplecom": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyName": "nodes.minimal.example.com", + "Roles": [ + { + "Ref": "AWSIAMRolenodesminimalexamplecom" + } + ], + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:Get*" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/addons/*", + "arn:aws-test:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/cluster-completed.spec", + "arn:aws-test:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/igconfig/node/*", + "arn:aws-test:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/secrets/dockerconfig" + ] + }, + { + "Action": [ + "s3:GetBucketLocation", + "s3:GetEncryptionConfiguration", + "s3:ListBucket", + "s3:ListBucketVersions" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:s3:::placeholder-read-bucket" + ] + }, + { + "Action": [ + "autoscaling:DescribeAutoScalingInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstances", + "iam:GetServerCertificate", + "iam:ListServerCertificates", + "kms:GenerateRandom" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + } + } + }, + "AWSIAMRolemastersminimalexamplecom": { + "Type": "AWS::IAM::Role", + "Properties": { + "RoleName": "masters.minimal.example.com", + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "masters.minimal.example.com" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + }, + "AWSIAMRolenodesminimalexamplecom": { + "Type": "AWS::IAM::Role", + "Properties": { + "RoleName": "nodes.minimal.example.com", + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "minimal.example.com" + }, + { + "Key": "Name", + "Value": "nodes.minimal.example.com" + }, + { + "Key": "kubernetes.io/cluster/minimal.example.com", + "Value": "owned" + } + ] + } + } + } +} diff --git a/tests/integration/update_cluster/karpenter/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/karpenter/cloudformation.json.extracted.yaml new file mode 100644 index 0000000000..48f3097682 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/cloudformation.json.extracted.yaml @@ -0,0 +1,418 @@ +Resources.AWSEC2LaunchTemplatemasterustest1amastersminimalexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.core.rmem_max=16777216 || true + sysctl -w net.core.wmem_max=16777216 || true + sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true + sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + logLevel: info + version: 1.4.12 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.minimal.example.com + serviceAccountJWKSURI: https://api.internal.minimal.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: minimal.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + CloudProvider: aws + ConfigBase: memfs://clusters.example.com/minimal.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + NodeupConfigHash: LFwTDQ1M/AxVLdvKc8ZPsktDgr836JEsdQRwn2TU+iM= + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodesminimalexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 + NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 + NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + + export AWS_REGION=us-test-1 + + + + + sysctl -w net.core.rmem_max=16777216 || true + sysctl -w net.core.wmem_max=16777216 || true + sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true + sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, urls + download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + containerRuntime: containerd + containerd: + logLevel: info + version: 1.4.12 + docker: + skipInstall: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + CloudProvider: aws + ConfigBase: memfs://clusters.example.com/minimal.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + NodeupConfigHash: ehZK5PooPMXQw0YD3dy5oARwClEXIj8ymh6DR1XYbQ0= + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/karpenter/data/aws_iam_role_masters.minimal.example.com_policy b/tests/integration/update_cluster/karpenter/data/aws_iam_role_masters.minimal.example.com_policy new file mode 100644 index 0000000000..66d5de1d5a --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_iam_role_masters.minimal.example.com_policy @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { "Service": "ec2.amazonaws.com"}, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/tests/integration/update_cluster/karpenter/data/aws_iam_role_nodes.minimal.example.com_policy b/tests/integration/update_cluster/karpenter/data/aws_iam_role_nodes.minimal.example.com_policy new file mode 100644 index 0000000000..66d5de1d5a --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_iam_role_nodes.minimal.example.com_policy @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { "Service": "ec2.amazonaws.com"}, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_masters.minimal.example.com_policy b/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_masters.minimal.example.com_policy new file mode 100644 index 0000000000..8fa5a8517a --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_masters.minimal.example.com_policy @@ -0,0 +1,240 @@ +{ + "Statement": [ + { + "Action": "ec2:AttachVolume", + "Condition": { + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "minimal.example.com", + "aws:ResourceTag/k8s.io/role/master": "1" + } + }, + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "s3:Get*" + ], + "Effect": "Allow", + "Resource": "arn:aws-test:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/*" + }, + { + "Action": [ + "s3:GetObject", + "s3:DeleteObject", + "s3:DeleteObjectVersion", + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": "arn:aws-test:s3:::placeholder-write-bucket/clusters.example.com/minimal.example.com/backups/etcd/main/*" + }, + { + "Action": [ + "s3:GetObject", + "s3:DeleteObject", + "s3:DeleteObjectVersion", + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": "arn:aws-test:s3:::placeholder-write-bucket/clusters.example.com/minimal.example.com/backups/etcd/events/*" + }, + { + "Action": [ + "s3:GetBucketLocation", + "s3:GetEncryptionConfiguration", + "s3:ListBucket", + "s3:ListBucketVersions" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:s3:::placeholder-read-bucket" + ] + }, + { + "Action": [ + "s3:GetBucketLocation", + "s3:GetEncryptionConfiguration", + "s3:ListBucket", + "s3:ListBucketVersions" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:s3:::placeholder-write-bucket" + ] + }, + { + "Action": [ + "route53:ChangeResourceRecordSets", + "route53:ListResourceRecordSets", + "route53:GetHostedZone" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:route53:::hostedzone/Z1AFAKE1ZON3YO" + ] + }, + { + "Action": [ + "route53:GetChange" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:route53:::change/*" + ] + }, + { + "Action": [ + "route53:ListHostedZones", + "route53:ListTagsForResource" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "ec2:CreateAction": [ + "CreateVolume", + "CreateSnapshot" + ] + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:volume/*", + "arn:aws-test:ec2:*:*:snapshot/*" + ] + }, + { + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "ec2:CreateAction": [ + "CreateVolume", + "CreateSnapshot" + ] + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:volume/*", + "arn:aws-test:ec2:*:*:snapshot/*" + ] + }, + { + "Action": "ec2:DeleteTags", + "Condition": { + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:volume/*", + "arn:aws-test:ec2:*:*:snapshot/*" + ] + }, + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:DescribeAccountAttributes", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeTags", + "ec2:DescribeVolumes", + "ec2:DescribeVolumesModifications", + "ec2:DescribeVpcs", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:RegisterTargets", + "iam:GetServerCertificate", + "iam:ListServerCertificates", + "kms:DescribeKey", + "kms:GenerateRandom" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:DeleteRoute", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume", + "ec2:RevokeSecurityGroupIngress", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:DeregisterTargets", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ec2:CreateSecurityGroup", + "ec2:CreateVolume", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateTargetGroup" + ], + "Condition": { + "StringEquals": { + "aws:RequestTag/KubernetesCluster": "minimal.example.com" + } + }, + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_nodes.minimal.example.com_policy b/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_nodes.minimal.example.com_policy new file mode 100644 index 0000000000..0f89b6f1d2 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_nodes.minimal.example.com_policy @@ -0,0 +1,41 @@ +{ + "Statement": [ + { + "Action": [ + "s3:Get*" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/addons/*", + "arn:aws-test:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/cluster-completed.spec", + "arn:aws-test:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/igconfig/node/*", + "arn:aws-test:s3:::placeholder-read-bucket/clusters.example.com/minimal.example.com/secrets/dockerconfig" + ] + }, + { + "Action": [ + "s3:GetBucketLocation", + "s3:GetEncryptionConfiguration", + "s3:ListBucket", + "s3:ListBucketVersions" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:s3:::placeholder-read-bucket" + ] + }, + { + "Action": [ + "autoscaling:DescribeAutoScalingInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstances", + "iam:GetServerCertificate", + "iam:ListServerCertificates", + "kms:GenerateRandom" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/karpenter/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key b/tests/integration/update_cluster/karpenter/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key new file mode 100644 index 0000000000..81cb012783 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/karpenter/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data b/tests/integration/update_cluster/karpenter/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data new file mode 100644 index 0000000000..899013be00 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data @@ -0,0 +1,248 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 +NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 +NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 +NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + +export AWS_REGION=us-test-1 + + + + +sysctl -w net.core.rmem_max=16777216 || true +sysctl -w net.core.wmem_max=16777216 || true +sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true +sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, urls +download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true +containerRuntime: containerd +containerd: + logLevel: info + version: 1.4.12 +docker: + skipInstall: true +encryptionConfig: null +etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.minimal.example.com + serviceAccountJWKSURI: https://api.internal.minimal.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: minimal.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +CloudProvider: aws +ConfigBase: memfs://clusters.example.com/minimal.example.com +InstanceGroupName: master-us-test-1a +InstanceGroupRole: Master +NodeupConfigHash: LFwTDQ1M/AxVLdvKc8ZPsktDgr836JEsdQRwn2TU+iM= + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/karpenter/data/aws_launch_template_nodes.minimal.example.com_user_data b/tests/integration/update_cluster/karpenter/data/aws_launch_template_nodes.minimal.example.com_user_data new file mode 100644 index 0000000000..853009f820 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_launch_template_nodes.minimal.example.com_user_data @@ -0,0 +1,168 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 +NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 +NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 +NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + +export AWS_REGION=us-test-1 + + + + +sysctl -w net.core.rmem_max=16777216 || true +sysctl -w net.core.wmem_max=16777216 || true +sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true +sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, urls +download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true +containerRuntime: containerd +containerd: + logLevel: info + version: 1.4.12 +docker: + skipInstall: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +CloudProvider: aws +ConfigBase: memfs://clusters.example.com/minimal.example.com +InstanceGroupName: nodes +InstanceGroupRole: Node +NodeupConfigHash: ehZK5PooPMXQw0YD3dy5oARwClEXIj8ymh6DR1XYbQ0= + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_cluster-completed.spec_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_cluster-completed.spec_content new file mode 100644 index 0000000000..a86eb9b99f --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_cluster-completed.spec_content @@ -0,0 +1,177 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2016-12-10T22:42:27Z" + name: minimal.example.com +spec: + api: + dns: {} + authorization: + alwaysAllow: {} + channel: stable + cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true + cloudProvider: aws + clusterDNSDomain: cluster.local + configBase: memfs://clusters.example.com/minimal.example.com + configStore: memfs://clusters.example.com/minimal.example.com + containerRuntime: containerd + containerd: + logLevel: info + version: 1.4.12 + dnsZone: Z1AFAKE1ZON3YO + docker: + skipInstall: true + etcdClusters: + - backups: + backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd/main + etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: main + version: 3.4.13 + - backups: + backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd/events + etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: events + version: 3.4.13 + externalDns: + provider: dns-controller + iam: + legacy: false + keyStore: memfs://clusters.example.com/minimal.example.com/pki + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.minimal.example.com + serviceAccountJWKSURI: https://api.internal.minimal.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: minimal.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeDNS: + cacheMaxConcurrent: 150 + cacheMaxSize: 1000 + cpuRequest: 100m + domain: cluster.local + memoryLimit: 170Mi + memoryRequest: 70Mi + nodeLocalDNS: + cpuRequest: 25m + enabled: false + image: k8s.gcr.io/dns/k8s-dns-node-cache:1.21.3 + memoryRequest: 5Mi + provider: CoreDNS + serverIP: 100.64.0.10 + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.21.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + kubernetesApiAccess: + - 0.0.0.0/0 + kubernetesVersion: 1.21.0 + masterInternalName: api.internal.minimal.example.com + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + masterPublicName: api.minimal.example.com + networkCIDR: 172.20.0.0/16 + networking: + cni: {} + nonMasqueradeCIDR: 100.64.0.0/10 + podCIDR: 100.96.0.0/11 + secretStore: memfs://clusters.example.com/minimal.example.com/secrets + serviceClusterIPRange: 100.64.0.0/13 + sshAccess: + - 0.0.0.0/0 + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Public + zone: us-test-1a + topology: + dns: + type: Public + masters: public + nodes: public diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_etcd-cluster-spec-events_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_etcd-cluster-spec-events_content new file mode 100644 index 0000000000..bb8ddb0e2e --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_etcd-cluster-spec-events_content @@ -0,0 +1,4 @@ +{ + "memberCount": 1, + "etcdVersion": "3.4.13" +} diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_etcd-cluster-spec-main_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_etcd-cluster-spec-main_content new file mode 100644 index 0000000000..bb8ddb0e2e --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_etcd-cluster-spec-main_content @@ -0,0 +1,4 @@ +{ + "memberCount": 1, + "etcdVersion": "3.4.13" +} diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_kops-version.txt_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_kops-version.txt_content new file mode 100644 index 0000000000..b7340298dc --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_kops-version.txt_content @@ -0,0 +1 @@ +1.21.0-alpha.1 diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_manifests-etcdmanager-events_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_manifests-etcdmanager-events_content new file mode 100644 index 0000000000..3554815f1b --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_manifests-etcdmanager-events_content @@ -0,0 +1,61 @@ +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + labels: + k8s-app: etcd-manager-events + name: etcd-manager-events + namespace: kube-system +spec: + containers: + - command: + - /bin/sh + - -c + - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /etcd-manager + --backup-store=memfs://clusters.example.com/minimal.example.com/backups/etcd/events + --client-urls=https://__name__:4002 --cluster-name=etcd-events --containerized=true + --dns-suffix=.internal.minimal.example.com --grpc-port=3997 --peer-urls=https://__name__:2381 + --quarantine-client-urls=https://__name__:3995 --v=6 --volume-name-tag=k8s.io/etcd/events + --volume-provider=aws --volume-tag=k8s.io/etcd/events --volume-tag=k8s.io/role/master=1 + --volume-tag=kubernetes.io/cluster/minimal.example.com=owned > /tmp/pipe 2>&1 + image: k8s.gcr.io/etcdadm/etcd-manager:v3.0.20211124 + name: etcd-manager + resources: + requests: + cpu: 200m + memory: 100Mi + securityContext: + privileged: true + volumeMounts: + - mountPath: /rootfs + name: rootfs + - mountPath: /run + name: run + - mountPath: /etc/kubernetes/pki/etcd-manager + name: pki + - mountPath: /var/log/etcd.log + name: varlogetcd + hostNetwork: true + hostPID: true + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: / + type: Directory + name: rootfs + - hostPath: + path: /run + type: DirectoryOrCreate + name: run + - hostPath: + path: /etc/kubernetes/pki/etcd-manager-events + type: DirectoryOrCreate + name: pki + - hostPath: + path: /var/log/etcd-events.log + type: FileOrCreate + name: varlogetcd +status: {} diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_manifests-etcdmanager-main_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_manifests-etcdmanager-main_content new file mode 100644 index 0000000000..87bb8e843f --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_manifests-etcdmanager-main_content @@ -0,0 +1,61 @@ +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + labels: + k8s-app: etcd-manager-main + name: etcd-manager-main + namespace: kube-system +spec: + containers: + - command: + - /bin/sh + - -c + - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /etcd-manager + --backup-store=memfs://clusters.example.com/minimal.example.com/backups/etcd/main + --client-urls=https://__name__:4001 --cluster-name=etcd --containerized=true + --dns-suffix=.internal.minimal.example.com --grpc-port=3996 --peer-urls=https://__name__:2380 + --quarantine-client-urls=https://__name__:3994 --v=6 --volume-name-tag=k8s.io/etcd/main + --volume-provider=aws --volume-tag=k8s.io/etcd/main --volume-tag=k8s.io/role/master=1 + --volume-tag=kubernetes.io/cluster/minimal.example.com=owned > /tmp/pipe 2>&1 + image: k8s.gcr.io/etcdadm/etcd-manager:v3.0.20211124 + name: etcd-manager + resources: + requests: + cpu: 200m + memory: 100Mi + securityContext: + privileged: true + volumeMounts: + - mountPath: /rootfs + name: rootfs + - mountPath: /run + name: run + - mountPath: /etc/kubernetes/pki/etcd-manager + name: pki + - mountPath: /var/log/etcd.log + name: varlogetcd + hostNetwork: true + hostPID: true + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: / + type: Directory + name: rootfs + - hostPath: + path: /run + type: DirectoryOrCreate + name: run + - hostPath: + path: /etc/kubernetes/pki/etcd-manager-main + type: DirectoryOrCreate + name: pki + - hostPath: + path: /var/log/etcd.log + type: FileOrCreate + name: varlogetcd +status: {} diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_manifests-static-kube-apiserver-healthcheck_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_manifests-static-kube-apiserver-healthcheck_content new file mode 100644 index 0000000000..b3f50e9cde --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_manifests-static-kube-apiserver-healthcheck_content @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null +spec: + containers: + - args: + - --ca-cert=/secrets/ca.crt + - --client-cert=/secrets/client.crt + - --client-key=/secrets/client.key + command: + - /kube-apiserver-healthcheck + image: k8s.gcr.io/kops/kube-apiserver-healthcheck:1.24.0-alpha.1 + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /.kube-apiserver-healthcheck/healthz + port: 3990 + initialDelaySeconds: 5 + timeoutSeconds: 5 + name: healthcheck + resources: {} + volumeMounts: + - mountPath: /secrets + name: healthcheck-secrets + readOnly: true + volumes: + - hostPath: + path: /etc/kubernetes/kube-apiserver-healthcheck/secrets + type: Directory + name: healthcheck-secrets +status: {} diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content new file mode 100644 index 0000000000..d89afb3021 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content @@ -0,0 +1,54 @@ +kind: Addons +metadata: + creationTimestamp: null + name: bootstrap +spec: + addons: + - id: k8s-1.16 + manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml + manifestHash: 7eb2e21eff3c1f501f7aaf1007d533fcfa5ae8bd862dcea053ae62559a478e81 + name: kops-controller.addons.k8s.io + needsRollingUpdate: control-plane + selector: + k8s-addon: kops-controller.addons.k8s.io + version: 9.99.0 + - manifest: core.addons.k8s.io/v1.4.0.yaml + manifestHash: 18233793a8442224d052e44891e737c67ccfb4e051e95216392319653f4cb0e5 + name: core.addons.k8s.io + selector: + k8s-addon: core.addons.k8s.io + version: 9.99.0 + - id: k8s-1.12 + manifest: coredns.addons.k8s.io/k8s-1.12.yaml + manifestHash: 8c3daed1d84f622f3db7a8833fe1b317268d696e28a56e5ae79ff1167385463f + name: coredns.addons.k8s.io + selector: + k8s-addon: coredns.addons.k8s.io + version: 9.99.0 + - id: k8s-1.9 + manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml + manifestHash: 01c120e887bd98d82ef57983ad58a0b22bc85efb48108092a24c4b82e4c9ea81 + name: kubelet-api.rbac.addons.k8s.io + selector: + k8s-addon: kubelet-api.rbac.addons.k8s.io + version: 9.99.0 + - manifest: limit-range.addons.k8s.io/v1.5.0.yaml + manifestHash: 2d55c3bc5e354e84a3730a65b42f39aba630a59dc8d32b30859fcce3d3178bc2 + name: limit-range.addons.k8s.io + selector: + k8s-addon: limit-range.addons.k8s.io + version: 9.99.0 + - id: k8s-1.12 + manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml + manifestHash: 158ef7d0836127e699c62529da8cb601f4aca8135c8126416dcd2c70086963af + name: dns-controller.addons.k8s.io + selector: + k8s-addon: dns-controller.addons.k8s.io + version: 9.99.0 + - id: v1.15.0 + manifest: storage-aws.addons.k8s.io/v1.15.0.yaml + manifestHash: 065ae832ddac8d0931e9992d6a76f43a33a36975a38003b34f4c5d86a7d42780 + name: storage-aws.addons.k8s.io + selector: + k8s-addon: storage-aws.addons.k8s.io + version: 9.99.0 diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-core.addons.k8s.io_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-core.addons.k8s.io_content new file mode 100644 index 0000000000..a334917278 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-core.addons.k8s.io_content @@ -0,0 +1,56 @@ +apiVersion: v1 +kind: Namespace +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: core.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: core.addons.k8s.io + name: kube-system + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: core.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: core.addons.k8s.io + name: kube-dns + namespace: kube-system + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: core.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: core.addons.k8s.io + name: kube-proxy + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: core.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: core.addons.k8s.io + name: kubeadm:node-proxier +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:node-proxier +subjects: +- apiGroup: "" + kind: ServiceAccount + name: kube-proxy + namespace: kube-system diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content new file mode 100644 index 0000000000..a7daeb37c9 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content @@ -0,0 +1,380 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + kubernetes.io/cluster-service: "true" + name: coredns + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns +subjects: +- kind: ServiceAccount + name: coredns + namespace: kube-system + +--- + +apiVersion: v1 +data: + Corefile: |- + .:53 { + errors + health { + lameduck 5s + } + ready + kubernetes cluster.local. in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + } + prometheus :9153 + forward . /etc/resolv.conf { + max_concurrent 1000 + } + cache 30 + loop + reload + loadbalance + } +kind: ConfigMap +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + addonmanager.kubernetes.io/mode: EnsureExists + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: coredns + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: CoreDNS + name: coredns + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-dns + strategy: + rollingUpdate: + maxSurge: 10% + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + k8s-app: kube-dns + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: k8s-app + operator: In + values: + - kube-dns + topologyKey: kubernetes.io/hostname + weight: 100 + containers: + - args: + - -conf + - /etc/coredns/Corefile + image: k8s.gcr.io/coredns/coredns:v1.8.5 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + successThreshold: 1 + timeoutSeconds: 5 + name: coredns + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - all + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /etc/coredns + name: config-volume + readOnly: true + dnsPolicy: Default + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: coredns + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - configMap: + name: coredns + name: config-volume + +--- + +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: CoreDNS + name: kube-dns + namespace: kube-system + resourceVersion: "0" +spec: + clusterIP: 100.64.0.10 + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP + selector: + k8s-app: kube-dns + +--- + +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: kube-dns + namespace: kube-system +spec: + maxUnavailable: 50% + selector: + matchLabels: + k8s-app: kube-dns + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: coredns-autoscaler + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: coredns-autoscaler +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - replicationcontrollers/scale + verbs: + - get + - update +- apiGroups: + - extensions + - apps + resources: + - deployments/scale + - replicasets/scale + verbs: + - get + - update +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: coredns-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: coredns-autoscaler +subjects: +- kind: ServiceAccount + name: coredns-autoscaler + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + k8s-app: coredns-autoscaler + kubernetes.io/cluster-service: "true" + name: coredns-autoscaler + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: coredns-autoscaler + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + k8s-app: coredns-autoscaler + spec: + containers: + - command: + - /cluster-proportional-autoscaler + - --namespace=kube-system + - --configmap=coredns-autoscaler + - --target=Deployment/coredns + - --default-params={"linear":{"coresPerReplica":256,"nodesPerReplica":16,"preventSinglePointFailure":true}} + - --logtostderr=true + - --v=2 + image: k8s.gcr.io/cpa/cluster-proportional-autoscaler:1.8.4 + name: autoscaler + resources: + requests: + cpu: 20m + memory: 10Mi + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: coredns-autoscaler + tolerations: + - key: CriticalAddonsOnly + operator: Exists diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content new file mode 100644 index 0000000000..0bb4449b50 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content @@ -0,0 +1,127 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: dns-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: dns-controller.addons.k8s.io + k8s-app: dns-controller + version: v1.24.0-alpha.1 + name: dns-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: dns-controller + strategy: + type: Recreate + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + k8s-addon: dns-controller.addons.k8s.io + k8s-app: dns-controller + version: v1.24.0-alpha.1 + spec: + containers: + - command: + - /dns-controller + - --watch-ingress=false + - --dns=aws-route53 + - --zone=*/Z1AFAKE1ZON3YO + - --internal-ipv4 + - --zone=*/* + - -v=2 + env: + - name: KUBERNETES_SERVICE_HOST + value: 127.0.0.1 + image: k8s.gcr.io/kops/dns-controller:1.24.0-alpha.1 + name: dns-controller + resources: + requests: + cpu: 50m + memory: 50Mi + securityContext: + runAsNonRoot: true + dnsPolicy: Default + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + priorityClassName: system-cluster-critical + serviceAccount: dns-controller + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + - key: node.kubernetes.io/not-ready + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: dns-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: dns-controller.addons.k8s.io + name: dns-controller + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: dns-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: dns-controller.addons.k8s.io + name: kops:dns-controller +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - ingress + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - networking + resources: + - ingresses + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: dns-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: dns-controller.addons.k8s.io + name: kops:dns-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kops:dns-controller +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:kube-system:dns-controller diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content new file mode 100644 index 0000000000..ed3da7ae3c --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content @@ -0,0 +1,208 @@ +apiVersion: v1 +data: + config.yaml: | + {"cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["nodes.minimal.example.com"],"Region":"us-test-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"]}} +kind: ConfigMap +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + k8s-app: kops-controller + version: v1.24.0-alpha.1 + name: kops-controller + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kops-controller + template: + metadata: + annotations: + dns.alpha.kubernetes.io/internal: kops-controller.internal.minimal.example.com + labels: + k8s-addon: kops-controller.addons.k8s.io + k8s-app: kops-controller + version: v1.24.0-alpha.1 + spec: + containers: + - command: + - /kops-controller + - --v=2 + - --conf=/etc/kubernetes/kops-controller/config/config.yaml + env: + - name: KUBERNETES_SERVICE_HOST + value: 127.0.0.1 + image: k8s.gcr.io/kops/kops-controller:1.24.0-alpha.1 + name: kops-controller + resources: + requests: + cpu: 50m + memory: 50Mi + securityContext: + runAsNonRoot: true + volumeMounts: + - mountPath: /etc/kubernetes/kops-controller/config/ + name: kops-controller-config + - mountPath: /etc/kubernetes/kops-controller/pki/ + name: kops-controller-pki + dnsPolicy: Default + hostNetwork: true + nodeSelector: + kops.k8s.io/kops-controller-pki: "" + node-role.kubernetes.io/master: "" + priorityClassName: system-cluster-critical + serviceAccount: kops-controller + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + - key: node.kubernetes.io/not-ready + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + volumes: + - configMap: + name: kops-controller + name: kops-controller-config + - hostPath: + path: /etc/kubernetes/kops-controller/ + type: Directory + name: kops-controller-pki + updateStrategy: + type: OnDelete + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - patch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kops-controller +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:kube-system:kops-controller + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - create +- apiGroups: + - "" + - coordination.k8s.io + resourceNames: + - kops-controller-leader + resources: + - configmaps + - leases + verbs: + - get + - list + - watch + - patch + - update + - delete +- apiGroups: + - "" + - coordination.k8s.io + resources: + - configmaps + - leases + verbs: + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kops-controller +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:kube-system:kops-controller diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content new file mode 100644 index 0000000000..36761e1c56 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kubelet-api.rbac.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kubelet-api.rbac.addons.k8s.io + name: kops:system:kubelet-api-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kubelet-api-admin +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: kubelet-api diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-limit-range.addons.k8s.io_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-limit-range.addons.k8s.io_content new file mode 100644 index 0000000000..4dcdce48b9 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-limit-range.addons.k8s.io_content @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: LimitRange +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: limit-range.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: limit-range.addons.k8s.io + name: limits + namespace: default +spec: + limits: + - defaultRequest: + cpu: 100m + type: Container diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content new file mode 100644 index 0000000000..21efd54326 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content @@ -0,0 +1,98 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: default +parameters: + type: gp2 +provisioner: kubernetes.io/aws-ebs + +--- + +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: + storageclass.kubernetes.io/is-default-class: "false" + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: gp2 +parameters: + type: gp2 +provisioner: kubernetes.io/aws-ebs + +--- + +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: + storageclass.kubernetes.io/is-default-class: "true" + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: kops-ssd-1-17 +parameters: + encrypted: "true" + type: gp2 +provisioner: kubernetes.io/aws-ebs +volumeBindingMode: WaitForFirstConsumer + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: system:aws-cloud-provider +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - patch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: system:aws-cloud-provider +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:aws-cloud-provider +subjects: +- kind: ServiceAccount + name: aws-cloud-provider + namespace: kube-system diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content new file mode 100644 index 0000000000..7d06ee5690 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content @@ -0,0 +1,265 @@ +APIServerConfig: + KubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.21.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.minimal.example.com + serviceAccountJWKSURI: https://api.internal.minimal.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + ServiceAccountPublicKeys: | + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm + XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ== + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF + Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ== + -----END RSA PUBLIC KEY----- +Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - f6120552408175ca332fd3b5d31c5edd115d8426d6731664e4ea3951c5eee3b4@https://github.com/containerd/containerd/releases/download/v1.4.12/cri-containerd-cni-1.4.12-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 87a4219c54552797ffd38790b72832372a90eceb7c8e451c36a682093d57dae6@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.11.tgz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 +CAs: + apiserver-aggregator-ca: | + -----BEGIN CERTIFICATE----- + MIIBgjCCASygAwIBAgIMFo3gINaZLHjisEcbMA0GCSqGSIb3DQEBCwUAMCIxIDAe + BgNVBAMTF2FwaXNlcnZlci1hZ2dyZWdhdG9yLWNhMB4XDTIxMDYzMDA0NTExMloX + DTMxMDYzMDA0NTExMlowIjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3It + Y2EwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAyyE71AOU3go5XFegLQ6fidI0LhhM + x7CzpTzh2xWKcHUfbNI7itgJvC/+GlyG5W+DF5V7ba0IJiQLsFve0oLdewIDAQAB + o0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU + ALfqF5ZmfqvqORuJIFilZYKF3d0wDQYJKoZIhvcNAQELBQADQQAHAomFKsF4jvYX + WM/UzQXDj9nSAFTf8dBPCXyZZNotsOH7+P6W4mMiuVs8bAuGiXGUdbsQ2lpiT/Rk + CzMeMdr4 + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBgjCCASygAwIBAgIMFo3gM0nxQpiX/agfMA0GCSqGSIb3DQEBCwUAMCIxIDAe + BgNVBAMTF2FwaXNlcnZlci1hZ2dyZWdhdG9yLWNhMB4XDTIxMDYzMDA0NTIzMVoX + DTMxMDYzMDA0NTIzMVowIjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3It + Y2EwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAyyE71AOU3go5XFegLQ6fidI0LhhM + x7CzpTzh2xWKcHUfbNI7itgJvC/+GlyG5W+DF5V7ba0IJiQLsFve0oLdewIDAQAB + o0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU + ALfqF5ZmfqvqORuJIFilZYKF3d0wDQYJKoZIhvcNAQELBQADQQCXsoezoxXu2CEN + QdlXZOfmBT6cqxIX/RMHXhpHwRiqPsTO8IO2bVA8CSzxNwMuSv/ZtrMHoh8+PcVW + HLtkTXH8 + -----END CERTIFICATE----- + etcd-clients-ca: | + -----BEGIN CERTIFICATE----- + MIIBcjCCARygAwIBAgIMFo1ogHnr26DL9YkqMA0GCSqGSIb3DQEBCwUAMBoxGDAW + BgNVBAMTD2V0Y2QtY2xpZW50cy1jYTAeFw0yMTA2MjgxNjE5MDFaFw0zMTA2Mjgx + NjE5MDFaMBoxGDAWBgNVBAMTD2V0Y2QtY2xpZW50cy1jYTBcMA0GCSqGSIb3DQEB + AQUAA0sAMEgCQQDYlt4Xx03Cp8QooPrloaVWznx9aQDSpl1UsrDyoBPNEElOLWep + uPaQBHiDLL8LwzGi7G9r+ib13tKrwprnlPv7AgMBAAGjQjBAMA4GA1UdDwEB/wQE + AwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQjlt4Ue54AbJPWlDpRM51s + x+PeBDANBgkqhkiG9w0BAQsFAANBAAZAdf8ROEVkr3Rf7I+s+CQOil2toadlKWOY + qCeJ2XaEROfp9aUTEIU1MGM3g57MPyAPPU7mURskuOQz6B1UFaY= + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBcjCCARygAwIBAgIMFo1olfBnC/CsT+dqMA0GCSqGSIb3DQEBCwUAMBoxGDAW + BgNVBAMTD2V0Y2QtY2xpZW50cy1jYTAeFw0yMTA2MjgxNjIwMzNaFw0zMTA2Mjgx + NjIwMzNaMBoxGDAWBgNVBAMTD2V0Y2QtY2xpZW50cy1jYTBcMA0GCSqGSIb3DQEB + AQUAA0sAMEgCQQDYlt4Xx03Cp8QooPrloaVWznx9aQDSpl1UsrDyoBPNEElOLWep + uPaQBHiDLL8LwzGi7G9r+ib13tKrwprnlPv7AgMBAAGjQjBAMA4GA1UdDwEB/wQE + AwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQjlt4Ue54AbJPWlDpRM51s + x+PeBDANBgkqhkiG9w0BAQsFAANBAF1xUz77PlUVUnd9duF8F7plou0TONC9R6/E + YQ8C6vM1b+9NSDGjCW8YmwEU2fBgskb/BBX2lwVZ32/RUEju4Co= + -----END CERTIFICATE----- + etcd-manager-ca-events: | + -----BEGIN CERTIFICATE----- + MIIBgDCCASqgAwIBAgIMFo+bKjm04vB4rNtaMA0GCSqGSIb3DQEBCwUAMCExHzAd + BgNVBAMTFmV0Y2QtbWFuYWdlci1jYS1ldmVudHMwHhcNMjEwNzA1MjAwOTU2WhcN + MzEwNzA1MjAwOTU2WjAhMR8wHQYDVQQDExZldGNkLW1hbmFnZXItY2EtZXZlbnRz + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKiC8tndMlEFZ7qzeKxeKqFVjaYpsh/H + g7RxWo15+1kgH3suO0lxp9+RxSVv97hnsfbySTPZVhy2cIQj7eZtZt8CAwEAAaNC + MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFBg6 + CEZkQNnRkARBwFce03AEWa+sMA0GCSqGSIb3DQEBCwUAA0EAJMnBThok/uUe8q8O + sS5q19KUuE8YCTUzMDj36EBKf6NX4NoakCa1h6kfQVtlMtEIMWQZCjbm8xGK5ffs + GS/VUw== + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBgDCCASqgAwIBAgIMFo+bQ+EgIiBmGghjMA0GCSqGSIb3DQEBCwUAMCExHzAd + BgNVBAMTFmV0Y2QtbWFuYWdlci1jYS1ldmVudHMwHhcNMjEwNzA1MjAxMTQ2WhcN + MzEwNzA1MjAxMTQ2WjAhMR8wHQYDVQQDExZldGNkLW1hbmFnZXItY2EtZXZlbnRz + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKFhHVVxxDGv8d1jBvtdSxz7KIVoBOjL + DMxsmTsINiQkTQaFlb+XPlnY1ar4+RhE519AFUkqfhypk4Zxqf1YFXUCAwEAAaNC + MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNuW + LLH5c8kDubDbr6BHgedW0iJ9MA0GCSqGSIb3DQEBCwUAA0EAiKUoBoaGu7XzboFE + hjfKlX0TujqWuW3qMxDEJwj4dVzlSLrAoB/G01MJ+xxYKh456n48aG6N827UPXhV + cPfVNg== + -----END CERTIFICATE----- + etcd-manager-ca-main: | + -----BEGIN CERTIFICATE----- + MIIBfDCCASagAwIBAgIMFo+bKjm1c3jfv6hIMA0GCSqGSIb3DQEBCwUAMB8xHTAb + BgNVBAMTFGV0Y2QtbWFuYWdlci1jYS1tYWluMB4XDTIxMDcwNTIwMDk1NloXDTMx + MDcwNTIwMDk1NlowHzEdMBsGA1UEAxMUZXRjZC1tYW5hZ2VyLWNhLW1haW4wXDAN + BgkqhkiG9w0BAQEFAANLADBIAkEAxbkDbGYmCSShpRG3r+lzTOFujyuruRfjOhYm + ZRX4w1Utd5y63dUc98sjc9GGUYMHd+0k1ql/a48tGhnK6N6jJwIDAQABo0IwQDAO + BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUWZLkbBFx + GAgPU4i62c52unSo7RswDQYJKoZIhvcNAQELBQADQQAj6Pgd0va/8FtkyMlnohLu + Gf4v8RJO6zk3Y6jJ4+cwWziipFM1ielMzSOZfFcCZgH3m5Io40is4hPSqyq2TOA6 + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBfDCCASagAwIBAgIMFo+bQ+Eg8Si30gr4MA0GCSqGSIb3DQEBCwUAMB8xHTAb + BgNVBAMTFGV0Y2QtbWFuYWdlci1jYS1tYWluMB4XDTIxMDcwNTIwMTE0NloXDTMx + MDcwNTIwMTE0NlowHzEdMBsGA1UEAxMUZXRjZC1tYW5hZ2VyLWNhLW1haW4wXDAN + BgkqhkiG9w0BAQEFAANLADBIAkEAw33jzcd/iosN04b0WXbDt7B0c3sJ3aafcGLP + vG3xRB9N5bYr9+qZAq3mzAFkxscn4j1ce5b1/GKTDEAClmZgdQIDAQABo0IwQDAO + BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUE/h+3gDP + DvKwHRyiYlXM8voZ1wowDQYJKoZIhvcNAQELBQADQQBXuimeEoAOu5HN4hG7NqL9 + t40K3ZRhRZv3JQWnRVJCBDjg1rD0GQJR/n+DoWvbeijI5C9pNjr2pWSIYR1eYCvd + -----END CERTIFICATE----- + etcd-peers-ca-events: | + -----BEGIN CERTIFICATE----- + MIIBfDCCASagAwIBAgIMFo+bKjmxTPh3/lYJMA0GCSqGSIb3DQEBCwUAMB8xHTAb + BgNVBAMTFGV0Y2QtcGVlcnMtY2EtZXZlbnRzMB4XDTIxMDcwNTIwMDk1NloXDTMx + MDcwNTIwMDk1NlowHzEdMBsGA1UEAxMUZXRjZC1wZWVycy1jYS1ldmVudHMwXDAN + BgkqhkiG9w0BAQEFAANLADBIAkEAv5g4HF2xmrYyouJfY9jXx1M3gPLD/pupvxPY + xyjJw5pNCy5M5XGS3iTqRD5RDE0fWudVHFZKLIe8WPc06NApXwIDAQABo0IwQDAO + BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUf6xiDI+O + Yph1ziCGr2hZaQYt+fUwDQYJKoZIhvcNAQELBQADQQBBxj5hqEQstonTb8lnqeGB + DEYtUeAk4eR/HzvUMjF52LVGuvN3XVt+JTrFeKNvb6/RDUbBNRj3azalcUkpPh6V + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBfDCCASagAwIBAgIMFo+bQ+Eq69jgzpKwMA0GCSqGSIb3DQEBCwUAMB8xHTAb + BgNVBAMTFGV0Y2QtcGVlcnMtY2EtZXZlbnRzMB4XDTIxMDcwNTIwMTE0NloXDTMx + MDcwNTIwMTE0NlowHzEdMBsGA1UEAxMUZXRjZC1wZWVycy1jYS1ldmVudHMwXDAN + BgkqhkiG9w0BAQEFAANLADBIAkEAo5Nj2CjX1qp3mEPw1H5nHAFWLoGNSLSlRFJW + 03NxaNPMFzL5PrCoyOXrX8/MWczuZYw0Crf8EPOOQWi2+W0XLwIDAQABo0IwQDAO + BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUxauhhKQh + cvdZND78rHe0RQVTTiswDQYJKoZIhvcNAQELBQADQQB+cq4jIS9q0zXslaRa+ViI + J+dviA3sMygbmSJO0s4DxYmoazKJblux5q0ASSvS9iL1l9ShuZ1dWyp2tpZawHyb + -----END CERTIFICATE----- + etcd-peers-ca-main: | + -----BEGIN CERTIFICATE----- + MIIBeDCCASKgAwIBAgIMFo+bKjmuLDDLcDHsMA0GCSqGSIb3DQEBCwUAMB0xGzAZ + BgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjAeFw0yMTA3MDUyMDA5NTZaFw0zMTA3 + MDUyMDA5NTZaMB0xGzAZBgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjBcMA0GCSqG + SIb3DQEBAQUAA0sAMEgCQQCyRaXWpwgN6INQqws9p/BvPElJv2Rno9dVTFhlQqDA + aUJXe7MBmiO4NJcW76EozeBh5ztR3/4NE1FM2x8TisS3AgMBAAGjQjBAMA4GA1Ud + DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQtE1d49uSvpURf + OQ25Vlu6liY20DANBgkqhkiG9w0BAQsFAANBAAgLVaetJZcfOA3OIMMvQbz2Ydrt + uWF9BKkIad8jrcIrm3IkOtR8bKGmDIIaRKuG/ZUOL6NMe2fky3AAfKwleL4= + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBeDCCASKgAwIBAgIMFo+bQ+EuVthBfuZvMA0GCSqGSIb3DQEBCwUAMB0xGzAZ + BgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjAeFw0yMTA3MDUyMDExNDZaFw0zMTA3 + MDUyMDExNDZaMB0xGzAZBgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjBcMA0GCSqG + SIb3DQEBAQUAA0sAMEgCQQCxNbycDZNx5V1ZOiXxZSvaFpHRwKeHDfcuMUitdoPt + naVMlMTGDWAMuCVmFHFAWohIYynemEegmZkZ15S7AErfAgMBAAGjQjBAMA4GA1Ud + DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTAjQ8T4HclPIsC + qipEfUIcLP6jqTANBgkqhkiG9w0BAQsFAANBAJdZ17TN3HlWrH7HQgfR12UBwz8K + G9DurDznVaBVUYaHY8Sg5AvAXeb+yIF2JMmRR+bK+/G1QYY2D3/P31Ic2Oo= + -----END CERTIFICATE----- + kubernetes-ca: | + -----BEGIN CERTIFICATE----- + MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU + BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw + ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD + SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1 + jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG + MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA + MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8 + tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w== + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU + BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw + OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD + SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7 + WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG + MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn + MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA + 9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw== + -----END CERTIFICATE----- +ClusterName: minimal.example.com +Hooks: +- null +- null +KeypairIDs: + apiserver-aggregator-ca: "6980187172486667078076483355" + etcd-clients-ca: "6979622252718071085282986282" + etcd-manager-ca-events: "6982279354000777253151890266" + etcd-manager-ca-main: "6982279354000936168671127624" + etcd-peers-ca-events: "6982279353999767935825892873" + etcd-peers-ca-main: "6982279353998887468930183660" + kubernetes-ca: "6982820025135291416230495506" + service-account: "2" +KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + kubernetes.io/role: master + node-role.kubernetes.io/control-plane: "" + node-role.kubernetes.io/master: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false +UpdatePolicy: automatic +channels: +- memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml +containerdConfig: + logLevel: info + version: 1.4.12 +etcdManifests: +- memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml +staticManifests: +- key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-nodes_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-nodes_content new file mode 100644 index 0000000000..014862648a --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-nodes_content @@ -0,0 +1,62 @@ +Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - f6120552408175ca332fd3b5d31c5edd115d8426d6731664e4ea3951c5eee3b4@https://github.com/containerd/containerd/releases/download/v1.4.12/cri-containerd-cni-1.4.12-linux-amd64.tar.gz + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 87a4219c54552797ffd38790b72832372a90eceb7c8e451c36a682093d57dae6@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.11.tgz +CAs: + kubernetes-ca: | + -----BEGIN CERTIFICATE----- + MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU + BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw + ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD + SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1 + jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG + MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA + MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8 + tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w== + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU + BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw + OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD + SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7 + WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG + MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn + MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA + 9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw== + -----END CERTIFICATE----- +ClusterName: minimal.example.com +Hooks: +- null +- null +KeypairIDs: + kubernetes-ca: "6982820025135291416230495506" +KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests +UpdatePolicy: automatic +channels: +- memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml +containerdConfig: + logLevel: info + version: 1.4.12 diff --git a/tests/integration/update_cluster/karpenter/id_rsa.pub b/tests/integration/update_cluster/karpenter/id_rsa.pub new file mode 100755 index 0000000000..81cb012783 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml b/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml new file mode 100644 index 0000000000..613e156008 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml @@ -0,0 +1,78 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2016-12-10T22:42:27Z" + name: minimal.example.com +spec: + kubernetesApiAccess: + - 0.0.0.0/0 + channel: stable + cloudProvider: aws + configBase: memfs://clusters.example.com/minimal.example.com + etcdClusters: + - etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: main + - etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: events + iam: {} + kubelet: + anonymousAuth: false + kubernetesVersion: v1.21.0 + masterInternalName: api.internal.minimal.example.com + masterPublicName: api.minimal.example.com + networkCIDR: 172.20.0.0/16 + networking: + cni: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + topology: + masters: public + nodes: public + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Public + zone: us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-10T22:42:28Z" + name: nodes + labels: + kops.k8s.io/cluster: minimal.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 + machineType: t2.medium + maxSize: 2 + minSize: 2 + role: Node + subnets: + - us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-10T22:42:28Z" + name: master-us-test-1a + labels: + kops.k8s.io/cluster: minimal.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 + machineType: m3.medium + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test-1a diff --git a/tests/integration/update_cluster/karpenter/kubernetes.tf b/tests/integration/update_cluster/karpenter/kubernetes.tf new file mode 100644 index 0000000000..c75a09b559 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/kubernetes.tf @@ -0,0 +1,824 @@ +locals { + cluster_name = "minimal.example.com" + master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id] + master_security_group_ids = [aws_security_group.masters-minimal-example-com.id] + masters_role_arn = aws_iam_role.masters-minimal-example-com.arn + masters_role_name = aws_iam_role.masters-minimal-example-com.name + node_autoscaling_group_ids = [aws_autoscaling_group.nodes-minimal-example-com.id] + node_security_group_ids = [aws_security_group.nodes-minimal-example-com.id] + node_subnet_ids = [aws_subnet.us-test-1a-minimal-example-com.id] + nodes_role_arn = aws_iam_role.nodes-minimal-example-com.arn + nodes_role_name = aws_iam_role.nodes-minimal-example-com.name + region = "us-test-1" + route_table_public_id = aws_route_table.minimal-example-com.id + subnet_us-test-1a_id = aws_subnet.us-test-1a-minimal-example-com.id + vpc_cidr_block = aws_vpc.minimal-example-com.cidr_block + vpc_id = aws_vpc.minimal-example-com.id +} + +output "cluster_name" { + value = "minimal.example.com" +} + +output "master_autoscaling_group_ids" { + value = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id] +} + +output "master_security_group_ids" { + value = [aws_security_group.masters-minimal-example-com.id] +} + +output "masters_role_arn" { + value = aws_iam_role.masters-minimal-example-com.arn +} + +output "masters_role_name" { + value = aws_iam_role.masters-minimal-example-com.name +} + +output "node_autoscaling_group_ids" { + value = [aws_autoscaling_group.nodes-minimal-example-com.id] +} + +output "node_security_group_ids" { + value = [aws_security_group.nodes-minimal-example-com.id] +} + +output "node_subnet_ids" { + value = [aws_subnet.us-test-1a-minimal-example-com.id] +} + +output "nodes_role_arn" { + value = aws_iam_role.nodes-minimal-example-com.arn +} + +output "nodes_role_name" { + value = aws_iam_role.nodes-minimal-example-com.name +} + +output "region" { + value = "us-test-1" +} + +output "route_table_public_id" { + value = aws_route_table.minimal-example-com.id +} + +output "subnet_us-test-1a_id" { + value = aws_subnet.us-test-1a-minimal-example-com.id +} + +output "vpc_cidr_block" { + value = aws_vpc.minimal-example-com.cidr_block +} + +output "vpc_id" { + value = aws_vpc.minimal-example-com.id +} + +provider "aws" { + region = "us-test-1" +} + +provider "aws" { + alias = "files" + region = "us-test-1" +} + +resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" { + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + launch_template { + id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id + version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version + } + max_size = 1 + metrics_granularity = "1Minute" + min_size = 1 + name = "master-us-test-1a.masters.minimal.example.com" + protect_from_scale_in = false + tag { + key = "KubernetesCluster" + propagate_at_launch = true + value = "minimal.example.com" + } + tag { + key = "Name" + propagate_at_launch = true + value = "master-us-test-1a.masters.minimal.example.com" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" + propagate_at_launch = true + value = "master" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/role/master" + propagate_at_launch = true + value = "1" + } + tag { + key = "kops.k8s.io/instancegroup" + propagate_at_launch = true + value = "master-us-test-1a" + } + tag { + key = "kubernetes.io/cluster/minimal.example.com" + propagate_at_launch = true + value = "owned" + } + vpc_zone_identifier = [aws_subnet.us-test-1a-minimal-example-com.id] +} + +resource "aws_autoscaling_group" "nodes-minimal-example-com" { + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + launch_template { + id = aws_launch_template.nodes-minimal-example-com.id + version = aws_launch_template.nodes-minimal-example-com.latest_version + } + max_size = 2 + metrics_granularity = "1Minute" + min_size = 2 + name = "nodes.minimal.example.com" + protect_from_scale_in = false + tag { + key = "KubernetesCluster" + propagate_at_launch = true + value = "minimal.example.com" + } + tag { + key = "Name" + propagate_at_launch = true + value = "nodes.minimal.example.com" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" + propagate_at_launch = true + value = "node" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/role/node" + propagate_at_launch = true + value = "1" + } + tag { + key = "kops.k8s.io/instancegroup" + propagate_at_launch = true + value = "nodes" + } + tag { + key = "kubernetes.io/cluster/minimal.example.com" + propagate_at_launch = true + value = "owned" + } + vpc_zone_identifier = [aws_subnet.us-test-1a-minimal-example-com.id] +} + +resource "aws_ebs_volume" "us-test-1a-etcd-events-minimal-example-com" { + availability_zone = "us-test-1a" + encrypted = false + iops = 3000 + size = 20 + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.etcd-events.minimal.example.com" + "k8s.io/etcd/events" = "us-test-1a/us-test-1a" + "k8s.io/role/master" = "1" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + throughput = 125 + type = "gp3" +} + +resource "aws_ebs_volume" "us-test-1a-etcd-main-minimal-example-com" { + availability_zone = "us-test-1a" + encrypted = false + iops = 3000 + size = 20 + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.etcd-main.minimal.example.com" + "k8s.io/etcd/main" = "us-test-1a/us-test-1a" + "k8s.io/role/master" = "1" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + throughput = 125 + type = "gp3" +} + +resource "aws_iam_instance_profile" "masters-minimal-example-com" { + name = "masters.minimal.example.com" + role = aws_iam_role.masters-minimal-example-com.name + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "masters.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_iam_instance_profile" "nodes-minimal-example-com" { + name = "nodes.minimal.example.com" + role = aws_iam_role.nodes-minimal-example-com.name + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_iam_role" "masters-minimal-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_masters.minimal.example.com_policy") + name = "masters.minimal.example.com" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "masters.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_iam_role" "nodes-minimal-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_nodes.minimal.example.com_policy") + name = "nodes.minimal.example.com" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_iam_role_policy" "masters-minimal-example-com" { + name = "masters.minimal.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_masters.minimal.example.com_policy") + role = aws_iam_role.masters-minimal-example-com.name +} + +resource "aws_iam_role_policy" "nodes-minimal-example-com" { + name = "nodes.minimal.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_nodes.minimal.example.com_policy") + role = aws_iam_role.nodes-minimal-example-com.name +} + +resource "aws_internet_gateway" "minimal-example-com" { + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_key_pair" "kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" { + key_name = "kubernetes.minimal.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57" + public_key = file("${path.module}/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key") + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" { + block_device_mappings { + device_name = "/dev/xvda" + ebs { + delete_on_termination = true + encrypted = true + iops = 3000 + throughput = 125 + volume_size = 64 + volume_type = "gp3" + } + } + block_device_mappings { + device_name = "/dev/sdc" + virtual_name = "ephemeral0" + } + iam_instance_profile { + name = aws_iam_instance_profile.masters-minimal-example-com.id + } + image_id = "ami-12345678" + instance_type = "m3.medium" + key_name = aws_key_pair.kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id + lifecycle { + create_before_destroy = true + } + metadata_options { + http_endpoint = "enabled" + http_protocol_ipv6 = "disabled" + http_put_response_hop_limit = 1 + http_tokens = "optional" + } + monitoring { + enabled = false + } + name = "master-us-test-1a.masters.minimal.example.com" + network_interfaces { + associate_public_ip_address = true + delete_on_termination = true + ipv6_address_count = 0 + security_groups = [aws_security_group.masters-minimal-example-com.id] + } + tag_specifications { + resource_type = "instance" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "master-us-test-1a.masters.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "master" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" = "" + "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" + "k8s.io/role/master" = "1" + "kops.k8s.io/instancegroup" = "master-us-test-1a" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + } + tag_specifications { + resource_type = "volume" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "master-us-test-1a.masters.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "master" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" = "" + "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" + "k8s.io/role/master" = "1" + "kops.k8s.io/instancegroup" = "master-us-test-1a" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + } + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "master-us-test-1a.masters.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "master" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master" = "" + "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" + "k8s.io/role/master" = "1" + "kops.k8s.io/instancegroup" = "master-us-test-1a" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + user_data = filebase64("${path.module}/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data") +} + +resource "aws_launch_template" "nodes-minimal-example-com" { + block_device_mappings { + device_name = "/dev/xvda" + ebs { + delete_on_termination = true + encrypted = true + iops = 3000 + throughput = 125 + volume_size = 128 + volume_type = "gp3" + } + } + iam_instance_profile { + name = aws_iam_instance_profile.nodes-minimal-example-com.id + } + image_id = "ami-12345678" + instance_type = "t2.medium" + key_name = aws_key_pair.kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id + lifecycle { + create_before_destroy = true + } + metadata_options { + http_endpoint = "enabled" + http_protocol_ipv6 = "disabled" + http_put_response_hop_limit = 1 + http_tokens = "optional" + } + monitoring { + enabled = false + } + name = "nodes.minimal.example.com" + network_interfaces { + associate_public_ip_address = true + delete_on_termination = true + ipv6_address_count = 0 + security_groups = [aws_security_group.nodes-minimal-example-com.id] + } + tag_specifications { + resource_type = "instance" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "nodes" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + } + tag_specifications { + resource_type = "volume" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "nodes" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + } + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "nodes" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + user_data = filebase64("${path.module}/data/aws_launch_template_nodes.minimal.example.com_user_data") +} + +resource "aws_route" "route-0-0-0-0--0" { + destination_cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.minimal-example-com.id + route_table_id = aws_route_table.minimal-example-com.id +} + +resource "aws_route" "route-__--0" { + destination_ipv6_cidr_block = "::/0" + gateway_id = aws_internet_gateway.minimal-example-com.id + route_table_id = aws_route_table.minimal-example-com.id +} + +resource "aws_route_table" "minimal-example-com" { + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/kops/role" = "public" + } + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_route_table_association" "us-test-1a-minimal-example-com" { + route_table_id = aws_route_table.minimal-example-com.id + subnet_id = aws_subnet.us-test-1a-minimal-example-com.id +} + +resource "aws_s3_bucket_object" "cluster-completed-spec" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_cluster-completed.spec_content") + key = "clusters.example.com/minimal.example.com/cluster-completed.spec" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_bucket_object" "etcd-cluster-spec-events" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_etcd-cluster-spec-events_content") + key = "clusters.example.com/minimal.example.com/backups/etcd/events/control/etcd-cluster-spec" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_bucket_object" "etcd-cluster-spec-main" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_etcd-cluster-spec-main_content") + key = "clusters.example.com/minimal.example.com/backups/etcd/main/control/etcd-cluster-spec" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_bucket_object" "kops-version-txt" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_kops-version.txt_content") + key = "clusters.example.com/minimal.example.com/kops-version.txt" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_bucket_object" "manifests-etcdmanager-events" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_manifests-etcdmanager-events_content") + key = "clusters.example.com/minimal.example.com/manifests/etcd/events.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_bucket_object" "manifests-etcdmanager-main" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_manifests-etcdmanager-main_content") + key = "clusters.example.com/minimal.example.com/manifests/etcd/main.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_bucket_object" "manifests-static-kube-apiserver-healthcheck" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_manifests-static-kube-apiserver-healthcheck_content") + key = "clusters.example.com/minimal.example.com/manifests/static/kube-apiserver-healthcheck.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_bucket_object" "minimal-example-com-addons-bootstrap" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content") + key = "clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_bucket_object" "minimal-example-com-addons-core-addons-k8s-io" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_minimal.example.com-addons-core.addons.k8s.io_content") + key = "clusters.example.com/minimal.example.com/addons/core.addons.k8s.io/v1.4.0.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_bucket_object" "minimal-example-com-addons-coredns-addons-k8s-io-k8s-1-12" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content") + key = "clusters.example.com/minimal.example.com/addons/coredns.addons.k8s.io/k8s-1.12.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_bucket_object" "minimal-example-com-addons-dns-controller-addons-k8s-io-k8s-1-12" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_minimal.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content") + key = "clusters.example.com/minimal.example.com/addons/dns-controller.addons.k8s.io/k8s-1.12.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_bucket_object" "minimal-example-com-addons-kops-controller-addons-k8s-io-k8s-1-16" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_minimal.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content") + key = "clusters.example.com/minimal.example.com/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_bucket_object" "minimal-example-com-addons-kubelet-api-rbac-addons-k8s-io-k8s-1-9" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_minimal.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content") + key = "clusters.example.com/minimal.example.com/addons/kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_bucket_object" "minimal-example-com-addons-limit-range-addons-k8s-io" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_minimal.example.com-addons-limit-range.addons.k8s.io_content") + key = "clusters.example.com/minimal.example.com/addons/limit-range.addons.k8s.io/v1.5.0.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_bucket_object" "minimal-example-com-addons-storage-aws-addons-k8s-io-v1-15-0" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_minimal.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content") + key = "clusters.example.com/minimal.example.com/addons/storage-aws.addons.k8s.io/v1.15.0.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_bucket_object" "nodeupconfig-master-us-test-1a" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content") + key = "clusters.example.com/minimal.example.com/igconfig/master/master-us-test-1a/nodeupconfig.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_bucket_object" "nodeupconfig-nodes" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_nodeupconfig-nodes_content") + key = "clusters.example.com/minimal.example.com/igconfig/node/nodes/nodeupconfig.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_security_group" "masters-minimal-example-com" { + description = "Security group for masters" + name = "masters.minimal.example.com" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "masters.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_security_group" "nodes-minimal-example-com" { + description = "Security group for nodes" + name = "nodes.minimal.example.com" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-masters-minimal-example-com" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-nodes-minimal-example-com" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-masters-minimal-example-com" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 443 + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 443 + type = "ingress" +} + +resource "aws_security_group_rule" "from-masters-minimal-example-com-egress-all-0to0-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-masters-minimal-example-com-egress-all-0to0-__--0" { + from_port = 0 + ipv6_cidr_blocks = ["::/0"] + protocol = "-1" + security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-masters-minimal-example-com-ingress-all-0to0-masters-minimal-example-com" { + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.masters-minimal-example-com.id + source_security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 0 + type = "ingress" +} + +resource "aws_security_group_rule" "from-masters-minimal-example-com-ingress-all-0to0-nodes-minimal-example-com" { + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes-minimal-example-com.id + source_security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 0 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-minimal-example-com-egress-all-0to0-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-nodes-minimal-example-com-egress-all-0to0-__--0" { + from_port = 0 + ipv6_cidr_blocks = ["::/0"] + protocol = "-1" + security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-all-0to0-nodes-minimal-example-com" { + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes-minimal-example-com.id + source_security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 0 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-tcp-1to2379-masters-minimal-example-com" { + from_port = 1 + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + source_security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 2379 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-tcp-2382to4000-masters-minimal-example-com" { + from_port = 2382 + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + source_security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 4000 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-tcp-4003to65535-masters-minimal-example-com" { + from_port = 4003 + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + source_security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 65535 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-minimal-example-com-ingress-udp-1to65535-masters-minimal-example-com" { + from_port = 1 + protocol = "udp" + security_group_id = aws_security_group.masters-minimal-example-com.id + source_security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 65535 + type = "ingress" +} + +resource "aws_subnet" "us-test-1a-minimal-example-com" { + availability_zone = "us-test-1a" + cidr_block = "172.20.32.0/19" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + "kubernetes.io/role/internal-elb" = "1" + } + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_vpc" "minimal-example-com" { + assign_generated_ipv6_cidr_block = true + cidr_block = "172.20.0.0/16" + enable_dns_hostnames = true + enable_dns_support = true + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_vpc_dhcp_options" "minimal-example-com" { + domain_name = "us-test-1.compute.internal" + domain_name_servers = ["AmazonProvidedDNS"] + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_vpc_dhcp_options_association" "minimal-example-com" { + dhcp_options_id = aws_vpc_dhcp_options.minimal-example-com.id + vpc_id = aws_vpc.minimal-example-com.id +} + +terraform { + required_version = ">= 0.15.0" + required_providers { + aws = { + "configuration_aliases" = [aws.files] + "source" = "hashicorp/aws" + "version" = ">= 3.59.0" + } + } +} From 794cb72112d78bd99081a000bda32d177d5ad44d Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Sat, 4 Dec 2021 12:23:51 +0100 Subject: [PATCH 02/13] Karpenter addon Constrain the instance types to what is supported by the AMI Add taints and label to karpenter provisioner Add instance types to karpenter provisioner --- cmd/kops/integration_test.go | 12 +- k8s/crds/kops.k8s.io_clusters.yaml | 6 + k8s/crds/kops.k8s.io_instancegroups.yaml | 4 + pkg/apis/kops/cluster.go | 7 + pkg/apis/kops/instancegroup.go | 9 + pkg/apis/kops/v1alpha2/cluster.go | 6 + pkg/apis/kops/v1alpha2/instancegroup.go | 4 + .../kops/v1alpha2/zz_generated.conversion.go | 50 ++ .../kops/v1alpha2/zz_generated.deepcopy.go | 21 + pkg/apis/kops/v1alpha3/cluster.go | 7 + pkg/apis/kops/v1alpha3/instancegroup.go | 4 + .../kops/v1alpha3/zz_generated.conversion.go | 50 ++ .../kops/v1alpha3/zz_generated.deepcopy.go | 21 + pkg/apis/kops/zz_generated.deepcopy.go | 21 + pkg/model/awsmodel/network.go | 12 +- .../components/addonmanifests/BUILD.bazel | 1 + .../addonmanifests/karpenter/BUILD.bazel | 12 + .../addonmanifests/karpenter/iam.go | 66 ++ pkg/model/components/addonmanifests/remap.go | 4 + pkg/model/iam/iam_builder.go | 87 +- ....kube-system.sa.minimal.example.com_policy | 17 + ....kube-system.sa.minimal.example.com_policy | 17 + ....kube-system.sa.minimal.example.com_policy | 35 + ....kube-system.sa.minimal.example.com_policy | 24 + ..._policy_masters.minimal.example.com_policy | 36 - ...penter-nodes.minimal.example.com_user_data | 168 ++++ ...t-1a.masters.minimal.example.com_user_data | 6 +- ...cket_object_cluster-completed.spec_content | 10 +- ...ws_s3_bucket_object_discovery.json_content | 18 + .../aws_s3_bucket_object_keys.json_content | 20 + ...nimal.example.com-addons-bootstrap_content | 10 +- ...-controller.addons.k8s.io-k8s-1.12_content | 19 + ...e.com-addons-karpenter.sh-k8s-1.19_content | 848 ++++++++++++++++++ ...bject_nodeupconfig-karpenter-nodes_content | 62 ++ ...ect_nodeupconfig-master-us-test-1a_content | 4 +- .../update_cluster/karpenter/in-v1alpha2.yaml | 25 +- .../update_cluster/karpenter/kubernetes.tf | 259 +++++- upup/models/BUILD.bazel | 1 + .../karpenter.sh/k8s-1.19.yaml.template | 677 ++++++++++++++ .../bootstrapchannelbuilder/BUILD.bazel | 1 + .../bootstrapchannelbuilder.go | 20 + upup/pkg/fi/cloudup/template_functions.go | 52 +- 42 files changed, 2625 insertions(+), 108 deletions(-) create mode 100644 pkg/model/components/addonmanifests/karpenter/BUILD.bazel create mode 100644 pkg/model/components/addonmanifests/karpenter/iam.go create mode 100644 tests/integration/update_cluster/karpenter/data/aws_iam_role_dns-controller.kube-system.sa.minimal.example.com_policy create mode 100644 tests/integration/update_cluster/karpenter/data/aws_iam_role_karpenter.kube-system.sa.minimal.example.com_policy create mode 100644 tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_dns-controller.kube-system.sa.minimal.example.com_policy create mode 100644 tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_karpenter.kube-system.sa.minimal.example.com_policy create mode 100644 tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes.minimal.example.com_user_data create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_discovery.json_content create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_keys.json_content create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content create mode 100644 tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-karpenter-nodes_content create mode 100644 upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index fea4e06c22..afdce7aacc 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -595,9 +595,17 @@ func TestExternalDNSIRSA(t *testing.T) { } func TestKarpenter(t *testing.T) { - newIntegrationTest("minimal.example.com", "karpenter"). + test := newIntegrationTest("minimal.example.com", "karpenter"). + withOIDCDiscovery(). withAddons(dnsControllerAddon). - runTestTerraformAWS(t) + withServiceAccountRole("dns-controller.kube-system", true). + withAddons("karpenter.sh-k8s-1.19"). + withServiceAccountRole("karpenter.kube-system", true) + test.expectTerraformFilenames = append(test.expectTerraformFilenames, + "aws_launch_template_karpenter-nodes.minimal.example.com_user_data", + "aws_s3_bucket_object_nodeupconfig-karpenter-nodes_content", + ) + test.runTestTerraformAWS(t) } // TestSharedSubnet runs the test on a configuration with a shared subnet (and VPC) diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index eff5ec6c5b..de9fd3a551 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -1319,6 +1319,12 @@ spec: kube-proxy on the master * enable debugging handlers on the master, so kubectl logs works' type: boolean + karpenter: + description: Karpenter defines the Karpenter configuration. + properties: + enabled: + type: boolean + type: object keyStore: description: KeyStore is the VFS path to where SSL keys and certificates are stored diff --git a/k8s/crds/kops.k8s.io_instancegroups.yaml b/k8s/crds/kops.k8s.io_instancegroups.yaml index c9f320c5fc..3fad5c5bdf 100644 --- a/k8s/crds/kops.k8s.io_instancegroups.yaml +++ b/k8s/crds/kops.k8s.io_instancegroups.yaml @@ -233,6 +233,10 @@ spec: description: InstanceInterruptionBehavior defines if a spot instance should be terminated, hibernated, or stopped after interruption type: string + instanceManager: + description: InstanceManager determines what is managing the node + lifecycle + type: string instanceMetadata: description: InstanceMetadata defines the EC2 instance metadata service options (AWS Only) diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index ebef13b10b..9fd97c4d55 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -23,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/kops/pkg/apis/kops/util" "k8s.io/kops/upup/pkg/fi/utils" ) @@ -213,6 +214,12 @@ type ClusterSpec struct { ServiceAccountIssuerDiscovery *ServiceAccountIssuerDiscoveryConfig `json:"serviceAccountIssuerDiscovery,omitempty"` // SnapshotController defines the CSI Snapshot Controller configuration. SnapshotController *SnapshotControllerConfig `json:"snapshotController,omitempty"` + // Karpenter defines the Karpenter configuration. + Karpenter *KarpenterConfig `json:"karpenter,omitempty"` +} + +type KarpenterConfig struct { + Enabled bool `json:"enabled,omitempty"` } // ServiceAccountIssuerDiscoveryConfig configures an OIDC Issuer. diff --git a/pkg/apis/kops/instancegroup.go b/pkg/apis/kops/instancegroup.go index 5076c47921..55265f14b2 100644 --- a/pkg/apis/kops/instancegroup.go +++ b/pkg/apis/kops/instancegroup.go @@ -82,8 +82,17 @@ const ( // SupportedFilesystems is a list of supported filesystems to format as var SupportedFilesystems = []string{BtfsFilesystem, Ext4Filesystem, XFSFilesystem} +type InstanceManager string + +const ( + InstanceManagerCloudGroup InstanceManager = "CloudGroup" + InstanceManagerKarpenter InstanceManager = "Karpenter" +) + // InstanceGroupSpec is the specification for an InstanceGroup type InstanceGroupSpec struct { + // InstanceManager determines what is managing the node lifecycle + InstanceManager InstanceManager `json:"instanceManager,omitempty"` // Type determines the role of instances in this instance group: masters or nodes Role InstanceGroupRole `json:"role,omitempty"` // Image is the instance (ami etc) we should use diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index 6970b6e254..16584183c9 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -211,6 +211,12 @@ type ClusterSpec struct { ServiceAccountIssuerDiscovery *ServiceAccountIssuerDiscoveryConfig `json:"serviceAccountIssuerDiscovery,omitempty"` // SnapshotController defines the CSI Snapshot Controller configuration. SnapshotController *SnapshotControllerConfig `json:"snapshotController,omitempty"` + // Karpenter defines the Karpenter configuration. + Karpenter *KarpenterConfig `json:"karpenter,omitempty"` +} + +type KarpenterConfig struct { + Enabled bool `json:"enabled,omitempty"` } // ServiceAccountIssuerDiscoveryConfig configures an OIDC Issuer. diff --git a/pkg/apis/kops/v1alpha2/instancegroup.go b/pkg/apis/kops/v1alpha2/instancegroup.go index e15e163f22..7800d9845b 100644 --- a/pkg/apis/kops/v1alpha2/instancegroup.go +++ b/pkg/apis/kops/v1alpha2/instancegroup.go @@ -49,8 +49,12 @@ type InstanceGroupList struct { // InstanceGroupRole string describes the roles of the nodes in this InstanceGroup (master or nodes) type InstanceGroupRole string +type InstanceManager string + // InstanceGroupSpec is the specification for an InstanceGroup type InstanceGroupSpec struct { + // InstanceManager determines what is managing the node lifecycle + InstanceManager InstanceManager `json:"instanceManager,omitempty"` // Type determines the role of instances in this instance group: masters or nodes Role InstanceGroupRole `json:"role,omitempty"` // Image is the instance (ami etc) we should use diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 38622f303f..7440864d05 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -584,6 +584,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*KarpenterConfig)(nil), (*kops.KarpenterConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_KarpenterConfig_To_kops_KarpenterConfig(a.(*KarpenterConfig), b.(*kops.KarpenterConfig), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.KarpenterConfig)(nil), (*KarpenterConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_KarpenterConfig_To_v1alpha2_KarpenterConfig(a.(*kops.KarpenterConfig), b.(*KarpenterConfig), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*Keyset)(nil), (*kops.Keyset)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_Keyset_To_kops_Keyset(a.(*Keyset), b.(*kops.Keyset), scope) }); err != nil { @@ -2724,6 +2734,15 @@ func autoConvert_v1alpha2_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out * } else { out.SnapshotController = nil } + if in.Karpenter != nil { + in, out := &in.Karpenter, &out.Karpenter + *out = new(kops.KarpenterConfig) + if err := Convert_v1alpha2_KarpenterConfig_To_kops_KarpenterConfig(*in, *out, s); err != nil { + return err + } + } else { + out.Karpenter = nil + } return nil } @@ -3131,6 +3150,15 @@ func autoConvert_kops_ClusterSpec_To_v1alpha2_ClusterSpec(in *kops.ClusterSpec, } else { out.SnapshotController = nil } + if in.Karpenter != nil { + in, out := &in.Karpenter, &out.Karpenter + *out = new(KarpenterConfig) + if err := Convert_kops_KarpenterConfig_To_v1alpha2_KarpenterConfig(*in, *out, s); err != nil { + return err + } + } else { + out.Karpenter = nil + } return nil } @@ -4169,6 +4197,7 @@ func Convert_kops_InstanceGroupList_To_v1alpha2_InstanceGroupList(in *kops.Insta } func autoConvert_v1alpha2_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *InstanceGroupSpec, out *kops.InstanceGroupSpec, s conversion.Scope) error { + out.InstanceManager = kops.InstanceManager(in.InstanceManager) out.Role = kops.InstanceGroupRole(in.Role) out.Image = in.Image out.MinSize = in.MinSize @@ -4331,6 +4360,7 @@ func Convert_v1alpha2_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *InstanceGr } func autoConvert_kops_InstanceGroupSpec_To_v1alpha2_InstanceGroupSpec(in *kops.InstanceGroupSpec, out *InstanceGroupSpec, s conversion.Scope) error { + out.InstanceManager = InstanceManager(in.InstanceManager) out.Role = InstanceGroupRole(in.Role) out.Image = in.Image out.MinSize = in.MinSize @@ -4513,6 +4543,26 @@ func Convert_kops_InstanceMetadataOptions_To_v1alpha2_InstanceMetadataOptions(in return autoConvert_kops_InstanceMetadataOptions_To_v1alpha2_InstanceMetadataOptions(in, out, s) } +func autoConvert_v1alpha2_KarpenterConfig_To_kops_KarpenterConfig(in *KarpenterConfig, out *kops.KarpenterConfig, s conversion.Scope) error { + out.Enabled = in.Enabled + return nil +} + +// Convert_v1alpha2_KarpenterConfig_To_kops_KarpenterConfig is an autogenerated conversion function. +func Convert_v1alpha2_KarpenterConfig_To_kops_KarpenterConfig(in *KarpenterConfig, out *kops.KarpenterConfig, s conversion.Scope) error { + return autoConvert_v1alpha2_KarpenterConfig_To_kops_KarpenterConfig(in, out, s) +} + +func autoConvert_kops_KarpenterConfig_To_v1alpha2_KarpenterConfig(in *kops.KarpenterConfig, out *KarpenterConfig, s conversion.Scope) error { + out.Enabled = in.Enabled + return nil +} + +// Convert_kops_KarpenterConfig_To_v1alpha2_KarpenterConfig is an autogenerated conversion function. +func Convert_kops_KarpenterConfig_To_v1alpha2_KarpenterConfig(in *kops.KarpenterConfig, out *KarpenterConfig, s conversion.Scope) error { + return autoConvert_kops_KarpenterConfig_To_v1alpha2_KarpenterConfig(in, out, s) +} + func autoConvert_v1alpha2_Keyset_To_kops_Keyset(in *Keyset, out *kops.Keyset, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta if err := Convert_v1alpha2_KeysetSpec_To_kops_KeysetSpec(&in.Spec, &out.Spec, s); err != nil { diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index 7a8b9ebbf1..188d384cd2 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -1257,6 +1257,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = new(SnapshotControllerConfig) (*in).DeepCopyInto(*out) } + if in.Karpenter != nil { + in, out := &in.Karpenter, &out.Karpenter + *out = new(KarpenterConfig) + **out = **in + } return } @@ -2490,6 +2495,22 @@ func (in *InstanceMetadataOptions) DeepCopy() *InstanceMetadataOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KarpenterConfig) DeepCopyInto(out *KarpenterConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KarpenterConfig. +func (in *KarpenterConfig) DeepCopy() *KarpenterConfig { + if in == nil { + return nil + } + out := new(KarpenterConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Keyset) DeepCopyInto(out *Keyset) { *out = *in diff --git a/pkg/apis/kops/v1alpha3/cluster.go b/pkg/apis/kops/v1alpha3/cluster.go index a8654a7b60..57635c4850 100644 --- a/pkg/apis/kops/v1alpha3/cluster.go +++ b/pkg/apis/kops/v1alpha3/cluster.go @@ -21,6 +21,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/kops/pkg/apis/kops" ) @@ -211,6 +212,12 @@ type ClusterSpec struct { ServiceAccountIssuerDiscovery *ServiceAccountIssuerDiscoveryConfig `json:"serviceAccountIssuerDiscovery,omitempty"` // SnapshotController defines the CSI Snapshot Controller configuration. SnapshotController *SnapshotControllerConfig `json:"snapshotController,omitempty"` + // Karpenter defines the Karpenter configuration. + Karpenter *KarpenterConfig `json:"karpenter,omitempty"` +} + +type KarpenterConfig struct { + Enabled bool `json:"enabled,omitempty"` } // ServiceAccountIssuerDiscoveryConfig configures an OIDC Issuer. diff --git a/pkg/apis/kops/v1alpha3/instancegroup.go b/pkg/apis/kops/v1alpha3/instancegroup.go index 154dfebdf7..e9a6cceaee 100644 --- a/pkg/apis/kops/v1alpha3/instancegroup.go +++ b/pkg/apis/kops/v1alpha3/instancegroup.go @@ -49,8 +49,12 @@ type InstanceGroupList struct { // InstanceGroupRole string describes the roles of the nodes in this InstanceGroup (master or nodes) type InstanceGroupRole string +type InstanceManager string + // InstanceGroupSpec is the specification for an InstanceGroup type InstanceGroupSpec struct { + // InstanceManager determines what is managing the node lifecycle + InstanceManager InstanceManager `json:"instanceManager,omitempty"` // Type determines the role of instances in this instance group: masters or nodes Role InstanceGroupRole `json:"role,omitempty"` // Image is the instance (ami etc) we should use diff --git a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go index 6022edfadb..751bc7cf6f 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go @@ -614,6 +614,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*KarpenterConfig)(nil), (*kops.KarpenterConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha3_KarpenterConfig_To_kops_KarpenterConfig(a.(*KarpenterConfig), b.(*kops.KarpenterConfig), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.KarpenterConfig)(nil), (*KarpenterConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_KarpenterConfig_To_v1alpha3_KarpenterConfig(a.(*kops.KarpenterConfig), b.(*KarpenterConfig), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*Keyset)(nil), (*kops.Keyset)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_Keyset_To_kops_Keyset(a.(*Keyset), b.(*kops.Keyset), scope) }); err != nil { @@ -2607,6 +2617,15 @@ func autoConvert_v1alpha3_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out * } else { out.SnapshotController = nil } + if in.Karpenter != nil { + in, out := &in.Karpenter, &out.Karpenter + *out = new(kops.KarpenterConfig) + if err := Convert_v1alpha3_KarpenterConfig_To_kops_KarpenterConfig(*in, *out, s); err != nil { + return err + } + } else { + out.Karpenter = nil + } return nil } @@ -3011,6 +3030,15 @@ func autoConvert_kops_ClusterSpec_To_v1alpha3_ClusterSpec(in *kops.ClusterSpec, } else { out.SnapshotController = nil } + if in.Karpenter != nil { + in, out := &in.Karpenter, &out.Karpenter + *out = new(KarpenterConfig) + if err := Convert_kops_KarpenterConfig_To_v1alpha3_KarpenterConfig(*in, *out, s); err != nil { + return err + } + } else { + out.Karpenter = nil + } return nil } @@ -4060,6 +4088,7 @@ func Convert_kops_InstanceGroupList_To_v1alpha3_InstanceGroupList(in *kops.Insta } func autoConvert_v1alpha3_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *InstanceGroupSpec, out *kops.InstanceGroupSpec, s conversion.Scope) error { + out.InstanceManager = kops.InstanceManager(in.InstanceManager) out.Role = kops.InstanceGroupRole(in.Role) out.Image = in.Image out.MinSize = in.MinSize @@ -4221,6 +4250,7 @@ func Convert_v1alpha3_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *InstanceGr } func autoConvert_kops_InstanceGroupSpec_To_v1alpha3_InstanceGroupSpec(in *kops.InstanceGroupSpec, out *InstanceGroupSpec, s conversion.Scope) error { + out.InstanceManager = InstanceManager(in.InstanceManager) out.Role = InstanceGroupRole(in.Role) out.Image = in.Image out.MinSize = in.MinSize @@ -4403,6 +4433,26 @@ func Convert_kops_InstanceMetadataOptions_To_v1alpha3_InstanceMetadataOptions(in return autoConvert_kops_InstanceMetadataOptions_To_v1alpha3_InstanceMetadataOptions(in, out, s) } +func autoConvert_v1alpha3_KarpenterConfig_To_kops_KarpenterConfig(in *KarpenterConfig, out *kops.KarpenterConfig, s conversion.Scope) error { + out.Enabled = in.Enabled + return nil +} + +// Convert_v1alpha3_KarpenterConfig_To_kops_KarpenterConfig is an autogenerated conversion function. +func Convert_v1alpha3_KarpenterConfig_To_kops_KarpenterConfig(in *KarpenterConfig, out *kops.KarpenterConfig, s conversion.Scope) error { + return autoConvert_v1alpha3_KarpenterConfig_To_kops_KarpenterConfig(in, out, s) +} + +func autoConvert_kops_KarpenterConfig_To_v1alpha3_KarpenterConfig(in *kops.KarpenterConfig, out *KarpenterConfig, s conversion.Scope) error { + out.Enabled = in.Enabled + return nil +} + +// Convert_kops_KarpenterConfig_To_v1alpha3_KarpenterConfig is an autogenerated conversion function. +func Convert_kops_KarpenterConfig_To_v1alpha3_KarpenterConfig(in *kops.KarpenterConfig, out *KarpenterConfig, s conversion.Scope) error { + return autoConvert_kops_KarpenterConfig_To_v1alpha3_KarpenterConfig(in, out, s) +} + func autoConvert_v1alpha3_Keyset_To_kops_Keyset(in *Keyset, out *kops.Keyset, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta if err := Convert_v1alpha3_KeysetSpec_To_kops_KeysetSpec(&in.Spec, &out.Spec, s); err != nil { diff --git a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go index ab6da989fe..8ef416d7cb 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go @@ -1168,6 +1168,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = new(SnapshotControllerConfig) (*in).DeepCopyInto(*out) } + if in.Karpenter != nil { + in, out := &in.Karpenter, &out.Karpenter + *out = new(KarpenterConfig) + **out = **in + } return } @@ -2396,6 +2401,22 @@ func (in *InstanceMetadataOptions) DeepCopy() *InstanceMetadataOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KarpenterConfig) DeepCopyInto(out *KarpenterConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KarpenterConfig. +func (in *KarpenterConfig) DeepCopy() *KarpenterConfig { + if in == nil { + return nil + } + out := new(KarpenterConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Keyset) DeepCopyInto(out *Keyset) { *out = *in diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 8316f22d00..74dc6579f0 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -1260,6 +1260,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = new(SnapshotControllerConfig) (*in).DeepCopyInto(*out) } + if in.Karpenter != nil { + in, out := &in.Karpenter, &out.Karpenter + *out = new(KarpenterConfig) + **out = **in + } return } @@ -2554,6 +2559,22 @@ func (in *InstanceMetadataOptions) DeepCopy() *InstanceMetadataOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KarpenterConfig) DeepCopyInto(out *KarpenterConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KarpenterConfig. +func (in *KarpenterConfig) DeepCopy() *KarpenterConfig { + if in == nil { + return nil + } + out := new(KarpenterConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Keyset) DeepCopyInto(out *Keyset) { *out = *in diff --git a/pkg/model/awsmodel/network.go b/pkg/model/awsmodel/network.go index a4a5fc7115..04eb11bedc 100644 --- a/pkg/model/awsmodel/network.go +++ b/pkg/model/awsmodel/network.go @@ -21,11 +21,12 @@ import ( "strings" "k8s.io/klog/v2" + "k8s.io/legacy-cloud-providers/aws" + "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awstasks" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" - "k8s.io/legacy-cloud-providers/aws" ) // NetworkModelBuilder configures network objects @@ -243,6 +244,15 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { default: klog.V(2).Infof("unable to properly tag subnet %q because it has unknown type %q. Load balancers may be created in incorrect subnets", subnetSpec.Name, subnetSpec.Type) } + for _, ig := range b.InstanceGroups { + if ig.Spec.InstanceManager == kops.InstanceManagerKarpenter { + for _, igSubnet := range ig.Spec.Subnets { + if igSubnet == subnetSpec.Name { + tags["kops.k8s.io/karpenter/"+ig.Name] = "1" + } + } + } + } } else { klog.V(2).Infof("skipping subnet tags. Ensure these are maintained externally.") } diff --git a/pkg/model/components/addonmanifests/BUILD.bazel b/pkg/model/components/addonmanifests/BUILD.bazel index 0cb15d50b9..7dc0a00dfe 100644 --- a/pkg/model/components/addonmanifests/BUILD.bazel +++ b/pkg/model/components/addonmanifests/BUILD.bazel @@ -16,6 +16,7 @@ go_library( "//pkg/model/components/addonmanifests/clusterautoscaler:go_default_library", "//pkg/model/components/addonmanifests/dnscontroller:go_default_library", "//pkg/model/components/addonmanifests/externaldns:go_default_library", + "//pkg/model/components/addonmanifests/karpenter:go_default_library", "//pkg/model/components/addonmanifests/nodeterminationhandler:go_default_library", "//pkg/model/iam:go_default_library", "//upup/pkg/fi:go_default_library", diff --git a/pkg/model/components/addonmanifests/karpenter/BUILD.bazel b/pkg/model/components/addonmanifests/karpenter/BUILD.bazel new file mode 100644 index 0000000000..8a19deb0e5 --- /dev/null +++ b/pkg/model/components/addonmanifests/karpenter/BUILD.bazel @@ -0,0 +1,12 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["iam.go"], + importpath = "k8s.io/kops/pkg/model/components/addonmanifests/karpenter", + visibility = ["//visibility:public"], + deps = [ + "//pkg/model/iam:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + ], +) diff --git a/pkg/model/components/addonmanifests/karpenter/iam.go b/pkg/model/components/addonmanifests/karpenter/iam.go new file mode 100644 index 0000000000..60b72b58cb --- /dev/null +++ b/pkg/model/components/addonmanifests/karpenter/iam.go @@ -0,0 +1,66 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package karpenter + +import ( + "k8s.io/apimachinery/pkg/types" + + "k8s.io/kops/pkg/model/iam" +) + +// ServiceAccount represents the service-account used by the dns-controller. +// It implements iam.Subject to get AWS IAM permissions. +type ServiceAccount struct{} + +var _ iam.Subject = &ServiceAccount{} + +// BuildAWSPolicy generates a custom policy for a ServiceAccount IAM role. +func (r *ServiceAccount) BuildAWSPolicy(b *iam.PolicyBuilder) (*iam.Policy, error) { + clusterName := b.Cluster.ObjectMeta.Name + p := iam.NewPolicy(clusterName) + + addKarpenterPermissions(p) + + return p, nil +} + +// ServiceAccount returns the kubernetes service account used. +func (r *ServiceAccount) ServiceAccount() (types.NamespacedName, bool) { + return types.NamespacedName{ + Namespace: "kube-system", + Name: "karpenter", + }, true +} + +func addKarpenterPermissions(p *iam.Policy) { + p.UnconditionalAction.Insert( + // "ec2:CreateLaunchTemplate", + "ec2:CreateFleet", + "ec2:RunInstances", + "ec2:CreateTags", + "iam:PassRole", + "ec2:TerminateInstances", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeInstances", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeAvailabilityZones", + "ssm:GetParameter", + ) +} diff --git a/pkg/model/components/addonmanifests/remap.go b/pkg/model/components/addonmanifests/remap.go index 39bb66e2a2..da6690bad4 100644 --- a/pkg/model/components/addonmanifests/remap.go +++ b/pkg/model/components/addonmanifests/remap.go @@ -23,6 +23,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" + addonsapi "k8s.io/kops/channels/pkg/api" "k8s.io/kops/pkg/assets" "k8s.io/kops/pkg/kubemanifest" @@ -33,6 +34,7 @@ import ( "k8s.io/kops/pkg/model/components/addonmanifests/clusterautoscaler" "k8s.io/kops/pkg/model/components/addonmanifests/dnscontroller" "k8s.io/kops/pkg/model/components/addonmanifests/externaldns" + "k8s.io/kops/pkg/model/components/addonmanifests/karpenter" "k8s.io/kops/pkg/model/components/addonmanifests/nodeterminationhandler" "k8s.io/kops/pkg/model/iam" "k8s.io/kops/upup/pkg/fi" @@ -128,6 +130,8 @@ func getWellknownServiceAccount(name string) iam.Subject { return &awscloudcontrollermanager.ServiceAccount{} case "external-dns": return &externaldns.ServiceAccount{} + case "karpenter": + return &karpenter.ServiceAccount{} default: return nil } diff --git a/pkg/model/iam/iam_builder.go b/pkg/model/iam/iam_builder.go index e7aaaafaf9..f94affe119 100644 --- a/pkg/model/iam/iam_builder.go +++ b/pkg/model/iam/iam_builder.go @@ -35,6 +35,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" + "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/kops/model" "k8s.io/kops/pkg/util/stringorslice" @@ -49,26 +50,26 @@ const PolicyDefaultVersion = "2012-10-17" // Policy Struct is a collection of fields that form a valid AWS policy document type Policy struct { clusterName string - unconditionalAction sets.String - clusterTaggedAction sets.String - clusterTaggedCreateAction sets.String + UnconditionalAction sets.String `json:"-"` + ClusterTaggedAction sets.String `json:"-"` + ClusterTaggedCreateAction sets.String `json:"-"` Statement []*Statement Version string } // AsJSON converts the policy document to JSON format (parsable by AWS) func (p *Policy) AsJSON() (string, error) { - if len(p.unconditionalAction) > 0 { + if len(p.UnconditionalAction) > 0 { p.Statement = append(p.Statement, &Statement{ Effect: StatementEffectAllow, - Action: stringorslice.Of(p.unconditionalAction.List()...), + Action: stringorslice.Of(p.UnconditionalAction.List()...), Resource: stringorslice.String("*"), }) } - if len(p.clusterTaggedAction) > 0 { + if len(p.ClusterTaggedAction) > 0 { p.Statement = append(p.Statement, &Statement{ Effect: StatementEffectAllow, - Action: stringorslice.Of(p.clusterTaggedAction.List()...), + Action: stringorslice.Of(p.ClusterTaggedAction.List()...), Resource: stringorslice.String("*"), Condition: Condition{ "StringEquals": map[string]string{ @@ -77,10 +78,10 @@ func (p *Policy) AsJSON() (string, error) { }, }) } - if len(p.clusterTaggedCreateAction) > 0 { + if len(p.ClusterTaggedCreateAction) > 0 { p.Statement = append(p.Statement, &Statement{ Effect: StatementEffectAllow, - Action: stringorslice.Of(p.clusterTaggedCreateAction.List()...), + Action: stringorslice.Of(p.ClusterTaggedCreateAction.List()...), Resource: stringorslice.String("*"), Condition: Condition{ "StringEquals": map[string]string{ @@ -276,9 +277,9 @@ func NewPolicy(clusterName string) *Policy { p := &Policy{ Version: PolicyDefaultVersion, clusterName: clusterName, - unconditionalAction: sets.NewString(), - clusterTaggedAction: sets.NewString(), - clusterTaggedCreateAction: sets.NewString(), + UnconditionalAction: sets.NewString(), + ClusterTaggedAction: sets.NewString(), + ClusterTaggedCreateAction: sets.NewString(), } return p } @@ -420,7 +421,7 @@ func (r *NodeRoleBastion) BuildAWSPolicy(b *PolicyBuilder) (*Policy, error) { // Bastion hosts currently don't require any specific permissions. // A trivial permission is granted, because empty policies are not allowed. - p.unconditionalAction.Insert("ec2:DescribeRegions") + p.UnconditionalAction.Insert("ec2:DescribeRegions") return p, nil } @@ -732,7 +733,7 @@ func addECRPermissions(p *Policy) { // We shouldn't be running lots of pods on the master, but it is perfectly reasonable to run // a private logging pod or similar. // At this point we allow all regions with ECR, since ECR is region specific. - p.unconditionalAction.Insert( + p.UnconditionalAction.Insert( "ecr:GetAuthorizationToken", "ecr:BatchCheckLayerAvailability", "ecr:GetDownloadUrlForLayer", @@ -744,7 +745,7 @@ func addECRPermissions(p *Policy) { } func addCalicoSrcDstCheckPermissions(p *Policy) { - p.unconditionalAction.Insert( + p.UnconditionalAction.Insert( "ec2:DescribeInstances", "ec2:ModifyNetworkInterfaceAttribute", ) @@ -754,26 +755,26 @@ func (b *PolicyBuilder) addNodeupPermissions(p *Policy, enableHookSupport bool) addCertIAMPolicies(p) addKMSGenerateRandomPolicies(p) addASLifecyclePolicies(p, enableHookSupport) - p.unconditionalAction.Insert( + p.UnconditionalAction.Insert( "ec2:DescribeInstances", // aws.go "ec2:DescribeInstanceTypes", ) if b.Cluster.Spec.IsKopsControllerIPAM() { - p.unconditionalAction.Insert( + p.UnconditionalAction.Insert( "ec2:AssignIpv6Addresses", ) } } func addKopsControllerIPAMPermissions(p *Policy) { - p.unconditionalAction.Insert( + p.UnconditionalAction.Insert( "ec2:DescribeNetworkInterfaces", ) } func addEtcdManagerPermissions(p *Policy) { - p.unconditionalAction.Insert( + p.UnconditionalAction.Insert( "ec2:DescribeVolumes", // aws.go ) @@ -795,7 +796,7 @@ func addEtcdManagerPermissions(p *Policy) { } func AddLegacyCCMPermissions(p *Policy) { - p.unconditionalAction.Insert( + p.UnconditionalAction.Insert( "ec2:CreateSecurityGroup", "ec2:CreateTags", "elasticloadbalancing:CreateTargetGroup", @@ -806,7 +807,7 @@ func AddLegacyCCMPermissions(p *Policy) { } func AddCCMPermissions(p *Policy, partition string, cloudRoutes bool) { - p.unconditionalAction.Insert( + p.UnconditionalAction.Insert( "autoscaling:DescribeAutoScalingGroups", "autoscaling:DescribeTags", "ec2:DescribeInstances", @@ -825,7 +826,7 @@ func AddCCMPermissions(p *Policy, partition string, cloudRoutes bool) { "kms:DescribeKey", ) - p.clusterTaggedAction.Insert( + p.ClusterTaggedAction.Insert( "ec2:ModifyInstanceAttribute", "ec2:ModifyVolume", "ec2:AttachVolume", @@ -858,7 +859,7 @@ func AddCCMPermissions(p *Policy, partition string, cloudRoutes bool) { "elasticloadbalancing:SetLoadBalancerPoliciesOfListener", ) - p.clusterTaggedCreateAction.Insert( + p.ClusterTaggedCreateAction.Insert( "elasticloadbalancing:CreateLoadBalancer", "ec2:CreateSecurityGroup", "ec2:CreateVolume", @@ -889,7 +890,7 @@ func AddCCMPermissions(p *Policy, partition string, cloudRoutes bool) { }, ) if cloudRoutes { - p.clusterTaggedAction.Insert( + p.ClusterTaggedAction.Insert( "ec2:CreateRoute", "ec2:DeleteRoute", ) @@ -898,7 +899,7 @@ func AddCCMPermissions(p *Policy, partition string, cloudRoutes bool) { // AddAWSLoadbalancerControllerPermissions adds the permissions needed for the aws load balancer controller to the givnen policy func AddAWSLoadbalancerControllerPermissions(p *Policy) { - p.unconditionalAction.Insert( + p.UnconditionalAction.Insert( "ec2:DescribeAvailabilityZones", "ec2:DescribeNetworkInterfaces", "elasticloadbalancing:DescribeTags", @@ -910,7 +911,7 @@ func AddAWSLoadbalancerControllerPermissions(p *Policy) { "acm:ListCertificates", "acm:DescribeCertificate", ) - p.clusterTaggedAction.Insert( + p.ClusterTaggedAction.Insert( "ec2:AuthorizeSecurityGroupIngress", // aws.go "ec2:DeleteSecurityGroup", // aws.go "ec2:RevokeSecurityGroupIngress", // aws.go @@ -925,11 +926,11 @@ func AddAWSLoadbalancerControllerPermissions(p *Policy) { } func AddClusterAutoscalerPermissions(p *Policy) { - p.clusterTaggedAction.Insert( + p.ClusterTaggedAction.Insert( "autoscaling:SetDesiredCapacity", "autoscaling:TerminateInstanceInAutoScalingGroup", ) - p.unconditionalAction.Insert( + p.UnconditionalAction.Insert( "autoscaling:DescribeAutoScalingGroups", "autoscaling:DescribeAutoScalingInstances", "autoscaling:DescribeLaunchConfigurations", @@ -943,21 +944,21 @@ func AddAWSEBSCSIDriverPermissions(p *Policy, partition string, appendSnapshotPe addSnapshotPersmissions(p) } - p.unconditionalAction.Insert( + p.UnconditionalAction.Insert( "ec2:DescribeAccountAttributes", // aws.go "ec2:DescribeInstances", // aws.go "ec2:DescribeVolumes", // aws.go "ec2:DescribeVolumesModifications", // aws.go "ec2:DescribeTags", // aws.go ) - p.clusterTaggedAction.Insert( + p.ClusterTaggedAction.Insert( "ec2:ModifyVolume", // aws.go "ec2:ModifyInstanceAttribute", // aws.go "ec2:AttachVolume", // aws.go "ec2:DeleteVolume", // aws.go "ec2:DetachVolume", // aws.go ) - p.clusterTaggedCreateAction.Insert( + p.ClusterTaggedCreateAction.Insert( "ec2:CreateVolume", // aws.go ) @@ -1004,12 +1005,12 @@ func AddAWSEBSCSIDriverPermissions(p *Policy, partition string, appendSnapshotPe } func addSnapshotPersmissions(p *Policy) { - p.unconditionalAction.Insert( + p.UnconditionalAction.Insert( "ec2:CreateSnapshot", "ec2:DescribeAvailabilityZones", "ec2:DescribeSnapshots", ) - p.clusterTaggedAction.Insert( + p.ClusterTaggedAction.Insert( "ec2:DeleteSnapshot", ) } @@ -1051,7 +1052,7 @@ func AddDNSControllerPermissions(b *PolicyBuilder, p *Policy) { func addKMSIAMPolicies(p *Policy, resource stringorslice.StringOrSlice) { // TODO could use "kms:ViaService" Condition Key here? - p.unconditionalAction.Insert( + p.UnconditionalAction.Insert( "kms:CreateGrant", "kms:Decrypt", "kms:DescribeKey", @@ -1063,35 +1064,35 @@ func addKMSIAMPolicies(p *Policy, resource stringorslice.StringOrSlice) { func addKMSGenerateRandomPolicies(p *Policy) { // For nodeup to seed the instance's random number generator. - p.unconditionalAction.Insert( + p.UnconditionalAction.Insert( "kms:GenerateRandom", ) } func addASLifecyclePolicies(p *Policy, enableHookSupport bool) { if enableHookSupport { - p.clusterTaggedAction.Insert( + p.ClusterTaggedAction.Insert( "autoscaling:CompleteLifecycleAction", // aws_manager.go ) - p.unconditionalAction.Insert( + p.UnconditionalAction.Insert( "autoscaling:DescribeLifecycleHooks", ) } - p.unconditionalAction.Insert( + p.UnconditionalAction.Insert( "autoscaling:DescribeAutoScalingInstances", ) } func addCertIAMPolicies(p *Policy) { // TODO: Make optional only if using IAM SSL Certs on ELBs - p.unconditionalAction.Insert( + p.UnconditionalAction.Insert( "iam:ListServerCertificates", "iam:GetServerCertificate", ) } func addCiliumEniPermissions(p *Policy) { - p.unconditionalAction.Insert( + p.UnconditionalAction.Insert( "ec2:DescribeSubnets", "ec2:AttachNetworkInterface", "ec2:AssignPrivateIpAddresses", @@ -1108,7 +1109,7 @@ func addCiliumEniPermissions(p *Policy) { } func addAmazonVPCCNIPermissions(p *Policy, partition string) { - p.unconditionalAction.Insert( + p.UnconditionalAction.Insert( "ec2:AssignPrivateIpAddresses", "ec2:AttachNetworkInterface", "ec2:CreateNetworkInterface", @@ -1135,7 +1136,7 @@ func addAmazonVPCCNIPermissions(p *Policy, partition string) { } func AddNodeTerminationHandlerSQSPermissions(p *Policy) { - p.unconditionalAction.Insert( + p.UnconditionalAction.Insert( "autoscaling:DescribeAutoScalingInstances", "autoscaling:DescribeTags", "ec2:DescribeInstances", @@ -1143,7 +1144,7 @@ func AddNodeTerminationHandlerSQSPermissions(p *Policy) { "sqs:DeleteMessage", "sqs:ReceiveMessage", ) - p.clusterTaggedAction.Insert( + p.ClusterTaggedAction.Insert( "autoscaling:CompleteLifecycleAction", ) } diff --git a/tests/integration/update_cluster/karpenter/data/aws_iam_role_dns-controller.kube-system.sa.minimal.example.com_policy b/tests/integration/update_cluster/karpenter/data/aws_iam_role_dns-controller.kube-system.sa.minimal.example.com_policy new file mode 100644 index 0000000000..a1f5c8e2f8 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_iam_role_dns-controller.kube-system.sa.minimal.example.com_policy @@ -0,0 +1,17 @@ +{ + "Statement": [ + { + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "discovery.example.com/minimal.example.com:sub": "system:serviceaccount:kube-system:dns-controller" + } + }, + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws-test:iam::123456789012:oidc-provider/discovery.example.com/minimal.example.com" + } + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/karpenter/data/aws_iam_role_karpenter.kube-system.sa.minimal.example.com_policy b/tests/integration/update_cluster/karpenter/data/aws_iam_role_karpenter.kube-system.sa.minimal.example.com_policy new file mode 100644 index 0000000000..93a839ea52 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_iam_role_karpenter.kube-system.sa.minimal.example.com_policy @@ -0,0 +1,17 @@ +{ + "Statement": [ + { + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "discovery.example.com/minimal.example.com:sub": "system:serviceaccount:kube-system:karpenter" + } + }, + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws-test:iam::123456789012:oidc-provider/discovery.example.com/minimal.example.com" + } + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_dns-controller.kube-system.sa.minimal.example.com_policy b/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_dns-controller.kube-system.sa.minimal.example.com_policy new file mode 100644 index 0000000000..48c0a0297b --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_dns-controller.kube-system.sa.minimal.example.com_policy @@ -0,0 +1,35 @@ +{ + "Statement": [ + { + "Action": [ + "route53:ChangeResourceRecordSets", + "route53:ListResourceRecordSets", + "route53:GetHostedZone" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:route53:::hostedzone/Z1AFAKE1ZON3YO" + ] + }, + { + "Action": [ + "route53:GetChange" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:route53:::change/*" + ] + }, + { + "Action": [ + "route53:ListHostedZones", + "route53:ListTagsForResource" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_karpenter.kube-system.sa.minimal.example.com_policy b/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_karpenter.kube-system.sa.minimal.example.com_policy new file mode 100644 index 0000000000..eb011e04da --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_karpenter.kube-system.sa.minimal.example.com_policy @@ -0,0 +1,24 @@ +{ + "Statement": [ + { + "Action": [ + "ec2:CreateFleet", + "ec2:CreateTags", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:RunInstances", + "ec2:TerminateInstances", + "iam:PassRole", + "ssm:GetParameter" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_masters.minimal.example.com_policy b/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_masters.minimal.example.com_policy index 8fa5a8517a..e7a32649b0 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_masters.minimal.example.com_policy +++ b/tests/integration/update_cluster/karpenter/data/aws_iam_role_policy_masters.minimal.example.com_policy @@ -110,54 +110,20 @@ "arn:aws-test:ec2:*:*:snapshot/*" ] }, - { - "Action": "ec2:CreateTags", - "Condition": { - "StringEquals": { - "ec2:CreateAction": [ - "CreateVolume", - "CreateSnapshot" - ] - } - }, - "Effect": "Allow", - "Resource": [ - "arn:aws-test:ec2:*:*:volume/*", - "arn:aws-test:ec2:*:*:snapshot/*" - ] - }, - { - "Action": "ec2:DeleteTags", - "Condition": { - "StringEquals": { - "aws:ResourceTag/KubernetesCluster": "minimal.example.com" - } - }, - "Effect": "Allow", - "Resource": [ - "arn:aws-test:ec2:*:*:volume/*", - "arn:aws-test:ec2:*:*:snapshot/*" - ] - }, { "Action": [ "autoscaling:DescribeAutoScalingGroups", "autoscaling:DescribeAutoScalingInstances", - "autoscaling:DescribeLaunchConfigurations", "autoscaling:DescribeTags", "ec2:CreateSecurityGroup", "ec2:CreateTags", - "ec2:DescribeAccountAttributes", "ec2:DescribeInstanceTypes", "ec2:DescribeInstances", - "ec2:DescribeLaunchTemplateVersions", "ec2:DescribeRegions", "ec2:DescribeRouteTables", "ec2:DescribeSecurityGroups", "ec2:DescribeSubnets", - "ec2:DescribeTags", "ec2:DescribeVolumes", - "ec2:DescribeVolumesModifications", "ec2:DescribeVpcs", "elasticloadbalancing:AddTags", "elasticloadbalancing:CreateListener", @@ -179,8 +145,6 @@ }, { "Action": [ - "autoscaling:SetDesiredCapacity", - "autoscaling:TerminateInstanceInAutoScalingGroup", "ec2:AttachVolume", "ec2:AuthorizeSecurityGroupIngress", "ec2:DeleteRoute", diff --git a/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes.minimal.example.com_user_data b/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes.minimal.example.com_user_data new file mode 100644 index 0000000000..d3ff3ce9b2 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes.minimal.example.com_user_data @@ -0,0 +1,168 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 +NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 +NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 +NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + +export AWS_REGION=us-test-1 + + + + +sysctl -w net.core.rmem_max=16777216 || true +sysctl -w net.core.wmem_max=16777216 || true +sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true +sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, urls +download-or-bust() { + local -r file="$1" + local -r hash="$2" + local -r urls=( $(split-commas "$3") ) + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + echo "== Downloaded ${url} (SHA256 = ${hash}) ==" + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: + awsEBSCSIDriver: + enabled: false + manageStorageClasses: true +containerRuntime: containerd +containerd: + logLevel: info + version: 1.4.12 +docker: + skipInstall: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + image: k8s.gcr.io/kube-proxy:v1.21.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +CloudProvider: aws +ConfigBase: memfs://clusters.example.com/minimal.example.com +InstanceGroupName: karpenter-nodes +InstanceGroupRole: Node +NodeupConfigHash: ehZK5PooPMXQw0YD3dy5oARwClEXIj8ymh6DR1XYbQ0= + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/karpenter/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data b/tests/integration/update_cluster/karpenter/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data index 899013be00..b0b0eebd70 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data +++ b/tests/integration/update_cluster/karpenter/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data @@ -177,8 +177,8 @@ kubeAPIServer: requestheaderUsernameHeaders: - X-Remote-User securePort: 443 - serviceAccountIssuer: https://api.internal.minimal.example.com - serviceAccountJWKSURI: https://api.internal.minimal.example.com/openid/v1/jwks + serviceAccountIssuer: https://discovery.example.com/minimal.example.com + serviceAccountJWKSURI: https://discovery.example.com/minimal.example.com/openid/v1/jwks serviceClusterIPRange: 100.64.0.0/13 storageBackend: etcd3 kubeControllerManager: @@ -240,7 +240,7 @@ CloudProvider: aws ConfigBase: memfs://clusters.example.com/minimal.example.com InstanceGroupName: master-us-test-1a InstanceGroupRole: Master -NodeupConfigHash: LFwTDQ1M/AxVLdvKc8ZPsktDgr836JEsdQRwn2TU+iM= +NodeupConfigHash: Z4IU80+07IRL4JdtasAA0cGjl3hQHN6rUgqsA2L/R+E= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_cluster-completed.spec_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_cluster-completed.spec_content index a86eb9b99f..20f1287f38 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_cluster-completed.spec_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_cluster-completed.spec_content @@ -43,6 +43,9 @@ spec: provider: dns-controller iam: legacy: false + useServiceAccountExternalPermissions: true + karpenter: + enabled: true keyStore: memfs://clusters.example.com/minimal.example.com/pki kubeAPIServer: allowPrivileged: true @@ -82,8 +85,8 @@ spec: requestheaderUsernameHeaders: - X-Remote-User securePort: 443 - serviceAccountIssuer: https://api.internal.minimal.example.com - serviceAccountJWKSURI: https://api.internal.minimal.example.com/openid/v1/jwks + serviceAccountIssuer: https://discovery.example.com/minimal.example.com + serviceAccountJWKSURI: https://discovery.example.com/minimal.example.com/openid/v1/jwks serviceClusterIPRange: 100.64.0.0/13 storageBackend: etcd3 kubeControllerManager: @@ -162,6 +165,9 @@ spec: nonMasqueradeCIDR: 100.64.0.0/10 podCIDR: 100.96.0.0/11 secretStore: memfs://clusters.example.com/minimal.example.com/secrets + serviceAccountIssuerDiscovery: + discoveryStore: memfs://discovery.example.com/minimal.example.com + enableAWSOIDCProvider: true serviceClusterIPRange: 100.64.0.0/13 sshAccess: - 0.0.0.0/0 diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_discovery.json_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_discovery.json_content new file mode 100644 index 0000000000..aba05dfd1a --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_discovery.json_content @@ -0,0 +1,18 @@ +{ +"issuer": "https://discovery.example.com/minimal.example.com", +"jwks_uri": "https://discovery.example.com/minimal.example.com/openid/v1/jwks", +"authorization_endpoint": "urn:kubernetes:programmatic_authorization", +"response_types_supported": [ +"id_token" +], +"subject_types_supported": [ +"public" +], +"id_token_signing_alg_values_supported": [ +"RS256" +], +"claims_supported": [ +"sub", +"iss" +] +} diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_keys.json_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_keys.json_content new file mode 100644 index 0000000000..ddcbc6ed75 --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_keys.json_content @@ -0,0 +1,20 @@ +{ +"keys": [ +{ +"use": "sig", +"kty": "RSA", +"kid": "3mNcULfgtWECYyZWY5ow1rOHjiRwEZHx28HQcRec3Ew", +"alg": "RS256", +"n": "2JbeF8dNwqfEKKD65aGlVs58fWkA0qZdVLKw8qATzRBJTi1nqbj2kAR4gyy_C8Mxouxva_om9d7Sq8Ka55T7-w", +"e": "AQAB" +}, +{ +"use": "sig", +"kty": "RSA", +"kid": "G-cZ10iKJqrXhR15ivI7Lg2q_cuL0zN9ouL0vF67FLc", +"alg": "RS256", +"n": "o4Tridlsf4Yz3UAiup_scSTiG_OqxkUW3Fz7zGKvVcLeYj9GEIKuzoB1VFk1nboDq4cCuGLfdzaQdCQKPIsDuw", +"e": "AQAB" +} +] +} diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content index d89afb3021..d309547774 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content @@ -40,7 +40,7 @@ spec: version: 9.99.0 - id: k8s-1.12 manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml - manifestHash: 158ef7d0836127e699c62529da8cb601f4aca8135c8126416dcd2c70086963af + manifestHash: 595f7ea5bff4f4668b9fdac3cb76ffce2fef68e3ae08e1781d657474a46b8df9 name: dns-controller.addons.k8s.io selector: k8s-addon: dns-controller.addons.k8s.io @@ -52,3 +52,11 @@ spec: selector: k8s-addon: storage-aws.addons.k8s.io version: 9.99.0 + - id: k8s-1.19 + manifest: karpenter.sh/k8s-1.19.yaml + manifestHash: c0a94c541234427f6ff9f07d4b83cb6ba042a2430373058c168ad249827d610b + name: karpenter.sh + needsPKI: true + selector: + k8s-addon: karpenter.sh + version: 9.99.0 diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content index 0bb4449b50..a864212863 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content @@ -38,6 +38,10 @@ spec: env: - name: KUBERNETES_SERVICE_HOST value: 127.0.0.1 + - name: AWS_ROLE_ARN + value: arn:aws-test:iam::123456789012:role/dns-controller.kube-system.sa.minimal.example.com + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: /var/run/secrets/amazonaws.com/token image: k8s.gcr.io/kops/dns-controller:1.24.0-alpha.1 name: dns-controller resources: @@ -46,11 +50,17 @@ spec: memory: 50Mi securityContext: runAsNonRoot: true + volumeMounts: + - mountPath: /var/run/secrets/amazonaws.com/ + name: token-amazonaws-com + readOnly: true dnsPolicy: Default hostNetwork: true nodeSelector: node-role.kubernetes.io/master: "" priorityClassName: system-cluster-critical + securityContext: + fsGroup: 10001 serviceAccount: dns-controller tolerations: - key: node.cloudprovider.kubernetes.io/uninitialized @@ -59,6 +69,15 @@ spec: operator: Exists - key: node-role.kubernetes.io/master operator: Exists + volumes: + - name: token-amazonaws-com + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: amazonaws.com + expirationSeconds: 86400 + path: token --- diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content new file mode 100644 index 0000000000..23bd852a3f --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content @@ -0,0 +1,848 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: provisioners.karpenter.sh +spec: + group: karpenter.sh + names: + kind: Provisioner + listKind: ProvisionerList + plural: provisioners + singular: provisioner + scope: Cluster + versions: + - name: v1alpha5 + schema: + openAPIV3Schema: + description: Provisioner is the Schema for the Provisioners API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ProvisionerSpec is the top level provisioner specification. + Provisioners launch nodes in response to pods that are unschedulable. + A single provisioner is capable of managing a diverse set of nodes. + Node properties are determined from a combination of provisioner and + pod scheduling constraints. + properties: + labels: + additionalProperties: + type: string + description: Labels are layered with Requirements and applied to every + node. + type: object + limits: + description: Limits define a set of bounds for provisioning capacity. + properties: + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources contains all the allocatable resources + that Karpenter supports for limiting. + type: object + type: object + provider: + description: Provider contains fields specific to your cloudprovider. + type: object + x-kubernetes-preserve-unknown-fields: true + requirements: + description: Requirements are layered with Labels and applied to every + node. + items: + description: A node selector requirement is a selector that contains + values, a key, and an operator that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and + Lt. + type: string + values: + description: An array of string values. If the operator is In + or NotIn, the values array must be non-empty. If the operator + is Exists or DoesNotExist, the values array must be empty. + If the operator is Gt or Lt, the values array must have a + single element, which will be interpreted as an integer. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + taints: + description: Taints will be applied to every node launched by the + Provisioner. If specified, the provisioner will not provision nodes + for pods that do not have matching tolerations. Additional taints + will be created that match pod tolerations on a per-node basis. + items: + description: The node this Taint is attached to has the "effect" + on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods that + do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule + and NoExecute. + type: string + key: + description: Required. The taint key to be applied to a node. + type: string + timeAdded: + description: TimeAdded represents the time at which the taint + was added. It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + type: array + ttlSecondsAfterEmpty: + description: "TTLSecondsAfterEmpty is the number of seconds the controller + will wait before attempting to delete a node, measured from when + the node is detected to be empty. A Node is considered to be empty + when it does not have pods scheduled to it, excluding daemonsets. + \n Termination due to underutilization is disabled if this field + is not set." + format: int64 + type: integer + ttlSecondsUntilExpired: + description: "TTLSecondsUntilExpired is the number of seconds the + controller will wait before terminating a node, measured from when + the node is created. This is useful to implement features like eventually + consistent node upgrade, memory leak protection, and disruption + testing. \n Termination due to expiration is disabled if this field + is not set." + format: int64 + type: integer + type: object + status: + description: ProvisionerStatus defines the observed state of Provisioner + properties: + conditions: + description: Conditions is the set of conditions required for this + provisioner to scale its target, and indicates whether or not those + conditions are met. + items: + description: 'Condition defines a readiness condition for a Knative + resource. See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties' + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the condition + transitioned from one status to another. We use VolatileTime + in place of metav1.Time to exclude this from creating equality.Semantic + differences (all other things held constant). + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + severity: + description: Severity with which to treat failures of this type + of condition. When this is not specified, it defaults to Error. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - status + - type + type: object + type: array + lastScaleTime: + description: LastScaleTime is the last time the Provisioner scaled + the number of nodes + format: date-time + type: string + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is the list of resources that have been provisioned. + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: karpenter + namespace: kube-system + +--- + +apiVersion: v1 +data: {} +kind: Secret +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: karpenter-webhook-cert + namespace: kube-system + +--- + +apiVersion: v1 +data: + zap-logger-config: | + { + "level": "debug", + "development": true, + "disableStacktrace": true, + "disableCaller": true, + "sampling": { + "initial": 100, + "thereafter": 100 + }, + "outputPaths": ["stdout"], + "errorOutputPaths": ["stderr"], + "encoding": "console", + "encoderConfig": { + "timeKey": "time", + "levelKey": "level", + "nameKey": "logger", + "callerKey": "caller", + "messageKey": "message", + "stacktraceKey": "stacktrace", + "levelEncoder": "capital", + "timeEncoder": "iso8601" + } + } +kind: ConfigMap +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + app.kubernetes.io/part-of: karpenter + k8s-addon: karpenter.sh + name: config-logging + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: karpenter-controller +rules: +- apiGroups: + - karpenter.sh + resources: + - provisioners + - provisioners/status + verbs: + - create + - delete + - patch + - get + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - patch + - update + - watch +- apiGroups: + - "" + resources: + - nodes + - pods + verbs: + - get + - list + - watch + - patch + - delete +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - create +- apiGroups: + - "" + resources: + - pods/binding + - pods/eviction + verbs: + - create +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: karpenter-webhook +rules: +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - get + - watch + - list + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: karpenter-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: karpenter-controller +subjects: +- kind: ServiceAccount + name: karpenter + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: karpenter-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: karpenter-webhook +subjects: +- kind: ServiceAccount + name: karpenter + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: karpenter-controller + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - configmaps/status + verbs: + - get + - update + - patch +- apiGroups: + - "" + resources: + - events + verbs: + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: karpenter-webhook + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - configmaps + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - create + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: karpenter-controller + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: karpenter-controller +subjects: +- kind: ServiceAccount + name: karpenter + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: karpenter-webhook + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: karpenter-webhook +subjects: +- kind: ServiceAccount + name: karpenter + namespace: kube-system + +--- + +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: karpenter-metrics + namespace: kube-system +spec: + ports: + - port: 8080 + targetPort: metrics + selector: + karpenter: controller + +--- + +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: karpenter-webhook + namespace: kube-system +spec: + ports: + - port: 443 + targetPort: webhook + selector: + karpenter: webhook + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: karpenter-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + karpenter: controller + strategy: + type: Recreate + template: + metadata: + labels: + karpenter: controller + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + - key: karpenter.sh/provisioner-name + operator: DoesNotExist + containers: + - env: + - name: AWS_REGION + value: us-test-1 + - name: CLUSTER_NAME + value: minimal.example.com + - name: CLUSTER_ENDPOINT + value: https://api.internal.minimal.example.com + - name: SYSTEM_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: AWS_ROLE_ARN + value: arn:aws-test:iam::123456789012:role/karpenter.kube-system.sa.minimal.example.com + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: /var/run/secrets/amazonaws.com/token + image: public.ecr.aws/karpenter/controller:v0.5.2 + livenessProbe: + httpGet: + path: /healthz + port: 8081 + name: manager + ports: + - containerPort: 8080 + name: metrics + - containerPort: 8081 + name: health-probe + readinessProbe: + httpGet: + path: /readyz + port: 8081 + resources: + limits: + cpu: "1" + memory: 1Gi + requests: + cpu: "1" + memory: 1Gi + volumeMounts: + - mountPath: /var/run/secrets/amazonaws.com/ + name: token-amazonaws-com + readOnly: true + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 10001 + serviceAccountName: karpenter + volumes: + - name: token-amazonaws-com + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: amazonaws.com + expirationSeconds: 86400 + path: token + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: karpenter-webhook + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + karpenter: webhook + strategy: + type: Recreate + template: + metadata: + labels: + karpenter: webhook + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + - key: karpenter.sh/provisioner-name + operator: DoesNotExist + containers: + - args: + - -port=8443 + env: + - name: AWS_REGION + value: us-test-1 + - name: CLUSTER_NAME + value: minimal.example.com + - name: CLUSTER_ENDPOINT + value: https://api.internal.minimal.example.com + - name: SYSTEM_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: AWS_ROLE_ARN + value: arn:aws-test:iam::123456789012:role/karpenter.kube-system.sa.minimal.example.com + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: /var/run/secrets/amazonaws.com/token + image: public.ecr.aws/karpenter/webhook:v0.5.2 + livenessProbe: + httpGet: + port: 8443 + scheme: HTTPS + name: webhook + ports: + - containerPort: 8443 + name: webhook + readinessProbe: + httpGet: + port: 8443 + scheme: HTTPS + resources: + limits: + cpu: 100m + memory: 50Mi + requests: + cpu: 100m + memory: 50Mi + volumeMounts: + - mountPath: /var/run/secrets/amazonaws.com/ + name: token-amazonaws-com + readOnly: true + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 10001 + serviceAccountName: karpenter + volumes: + - name: token-amazonaws-com + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: amazonaws.com + expirationSeconds: 86400 + path: token + +--- + +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: defaulting.webhook.provisioners.karpenter.sh +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: karpenter-webhook + namespace: kube-system + failurePolicy: Fail + name: defaulting.webhook.provisioners.karpenter.sh + rules: + - apiGroups: + - karpenter.sh + apiVersions: + - v1alpha5 + operations: + - CREATE + - UPDATE + resources: + - provisioners provisioners/status + sideEffects: None + +--- + +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: validation.webhook.provisioners.karpenter.sh +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: karpenter-webhook + namespace: kube-system + failurePolicy: Fail + name: validation.webhook.provisioners.karpenter.sh + rules: + - apiGroups: + - karpenter.sh + apiVersions: + - v1alpha5 + operations: + - CREATE + - UPDATE + resources: + - provisioners provisioners/status + sideEffects: None + +--- + +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: validation.webhook.config.karpenter.sh +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: karpenter-webhook + namespace: kube-system + failurePolicy: Fail + name: validation.webhook.config.karpenter.sh + objectSelector: + matchLabels: + app.kubernetes.io/part-of: karpenter + sideEffects: None diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-karpenter-nodes_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-karpenter-nodes_content new file mode 100644 index 0000000000..014862648a --- /dev/null +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-karpenter-nodes_content @@ -0,0 +1,62 @@ +Assets: + amd64: + - 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubelet + - 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - f6120552408175ca332fd3b5d31c5edd115d8426d6731664e4ea3951c5eee3b4@https://github.com/containerd/containerd/releases/download/v1.4.12/cri-containerd-cni-1.4.12-linux-amd64.tar.gz + arm64: + - 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubelet + - a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d@https://storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 87a4219c54552797ffd38790b72832372a90eceb7c8e451c36a682093d57dae6@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.11.tgz +CAs: + kubernetes-ca: | + -----BEGIN CERTIFICATE----- + MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU + BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw + ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD + SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1 + jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG + MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA + MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8 + tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w== + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU + BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw + OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD + SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7 + WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG + MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn + MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA + 9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw== + -----END CERTIFICATE----- +ClusterName: minimal.example.com +Hooks: +- null +- null +KeypairIDs: + kubernetes-ca: "6982820025135291416230495506" +KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests +UpdatePolicy: automatic +channels: +- memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml +containerdConfig: + logLevel: info + version: 1.4.12 diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content index 7d06ee5690..a66607232b 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content @@ -37,8 +37,8 @@ APIServerConfig: requestheaderUsernameHeaders: - X-Remote-User securePort: 443 - serviceAccountIssuer: https://api.internal.minimal.example.com - serviceAccountJWKSURI: https://api.internal.minimal.example.com/openid/v1/jwks + serviceAccountIssuer: https://discovery.example.com/minimal.example.com + serviceAccountJWKSURI: https://discovery.example.com/minimal.example.com/openid/v1/jwks serviceClusterIPRange: 100.64.0.0/13 storageBackend: etcd3 ServiceAccountPublicKeys: | diff --git a/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml b/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml index 613e156008..0375017eef 100644 --- a/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml @@ -18,7 +18,10 @@ spec: - instanceGroup: master-us-test-1a name: us-test-1a name: events - iam: {} + iam: + useServiceAccountExternalPermissions: true + karpenter: + enabled: true kubelet: anonymousAuth: false kubernetesVersion: v1.21.0 @@ -28,6 +31,9 @@ spec: networking: cni: {} nonMasqueradeCIDR: 100.64.0.0/10 + serviceAccountIssuerDiscovery: + enableAWSOIDCProvider: true + discoveryStore: memfs://discovery.example.com/minimal.example.com sshAccess: - 0.0.0.0/0 topology: @@ -76,3 +82,20 @@ spec: role: Master subnets: - us-test-1a +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-10T22:42:28Z" + name: karpenter-nodes + labels: + kops.k8s.io/cluster: minimal.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 + machineType: t2.medium + role: Node + subnets: + - us-test-1a + diff --git a/tests/integration/update_cluster/karpenter/kubernetes.tf b/tests/integration/update_cluster/karpenter/kubernetes.tf index c75a09b559..b64406c9db 100644 --- a/tests/integration/update_cluster/karpenter/kubernetes.tf +++ b/tests/integration/update_cluster/karpenter/kubernetes.tf @@ -1,25 +1,55 @@ locals { - cluster_name = "minimal.example.com" - master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id] - master_security_group_ids = [aws_security_group.masters-minimal-example-com.id] - masters_role_arn = aws_iam_role.masters-minimal-example-com.arn - masters_role_name = aws_iam_role.masters-minimal-example-com.name - node_autoscaling_group_ids = [aws_autoscaling_group.nodes-minimal-example-com.id] - node_security_group_ids = [aws_security_group.nodes-minimal-example-com.id] - node_subnet_ids = [aws_subnet.us-test-1a-minimal-example-com.id] - nodes_role_arn = aws_iam_role.nodes-minimal-example-com.arn - nodes_role_name = aws_iam_role.nodes-minimal-example-com.name - region = "us-test-1" - route_table_public_id = aws_route_table.minimal-example-com.id - subnet_us-test-1a_id = aws_subnet.us-test-1a-minimal-example-com.id - vpc_cidr_block = aws_vpc.minimal-example-com.cidr_block - vpc_id = aws_vpc.minimal-example-com.id + cluster_name = "minimal.example.com" + iam_openid_connect_provider_arn = aws_iam_openid_connect_provider.minimal-example-com.arn + iam_openid_connect_provider_issuer = "discovery.example.com/minimal.example.com" + kube-system-dns-controller_role_arn = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.arn + kube-system-dns-controller_role_name = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.name + kube-system-karpenter_role_arn = aws_iam_role.karpenter-kube-system-sa-minimal-example-com.arn + kube-system-karpenter_role_name = aws_iam_role.karpenter-kube-system-sa-minimal-example-com.name + master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id] + master_security_group_ids = [aws_security_group.masters-minimal-example-com.id] + masters_role_arn = aws_iam_role.masters-minimal-example-com.arn + masters_role_name = aws_iam_role.masters-minimal-example-com.name + node_autoscaling_group_ids = [aws_autoscaling_group.karpenter-nodes-minimal-example-com.id, aws_autoscaling_group.nodes-minimal-example-com.id] + node_security_group_ids = [aws_security_group.nodes-minimal-example-com.id] + node_subnet_ids = [aws_subnet.us-test-1a-minimal-example-com.id] + nodes_role_arn = aws_iam_role.nodes-minimal-example-com.arn + nodes_role_name = aws_iam_role.nodes-minimal-example-com.name + region = "us-test-1" + route_table_public_id = aws_route_table.minimal-example-com.id + subnet_us-test-1a_id = aws_subnet.us-test-1a-minimal-example-com.id + vpc_cidr_block = aws_vpc.minimal-example-com.cidr_block + vpc_id = aws_vpc.minimal-example-com.id } output "cluster_name" { value = "minimal.example.com" } +output "iam_openid_connect_provider_arn" { + value = aws_iam_openid_connect_provider.minimal-example-com.arn +} + +output "iam_openid_connect_provider_issuer" { + value = "discovery.example.com/minimal.example.com" +} + +output "kube-system-dns-controller_role_arn" { + value = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.arn +} + +output "kube-system-dns-controller_role_name" { + value = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.name +} + +output "kube-system-karpenter_role_arn" { + value = aws_iam_role.karpenter-kube-system-sa-minimal-example-com.arn +} + +output "kube-system-karpenter_role_name" { + value = aws_iam_role.karpenter-kube-system-sa-minimal-example-com.name +} + output "master_autoscaling_group_ids" { value = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id] } @@ -37,7 +67,7 @@ output "masters_role_name" { } output "node_autoscaling_group_ids" { - value = [aws_autoscaling_group.nodes-minimal-example-com.id] + value = [aws_autoscaling_group.karpenter-nodes-minimal-example-com.id, aws_autoscaling_group.nodes-minimal-example-com.id] } output "node_security_group_ids" { @@ -85,6 +115,55 @@ provider "aws" { region = "us-test-1" } +resource "aws_autoscaling_group" "karpenter-nodes-minimal-example-com" { + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + launch_template { + id = aws_launch_template.karpenter-nodes-minimal-example-com.id + version = aws_launch_template.karpenter-nodes-minimal-example-com.latest_version + } + max_size = 2 + metrics_granularity = "1Minute" + min_size = 2 + name = "karpenter-nodes.minimal.example.com" + protect_from_scale_in = false + tag { + key = "KubernetesCluster" + propagate_at_launch = true + value = "minimal.example.com" + } + tag { + key = "Name" + propagate_at_launch = true + value = "karpenter-nodes.minimal.example.com" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" + propagate_at_launch = true + value = "node" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/role/node" + propagate_at_launch = true + value = "1" + } + tag { + key = "kops.k8s.io/instancegroup" + propagate_at_launch = true + value = "karpenter-nodes" + } + tag { + key = "kubernetes.io/cluster/minimal.example.com" + propagate_at_launch = true + value = "owned" + } + vpc_zone_identifier = [aws_subnet.us-test-1a-minimal-example-com.id] +} + resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" { enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] launch_template { @@ -250,6 +329,37 @@ resource "aws_iam_instance_profile" "nodes-minimal-example-com" { } } +resource "aws_iam_openid_connect_provider" "minimal-example-com" { + client_id_list = ["amazonaws.com"] + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + thumbprint_list = ["9e99a48a9960b14926bb7f3b02e22da2b0ab7280", "a9d53002e97e00e043244f3d170d6f4c414104fd"] + url = "https://discovery.example.com/minimal.example.com" +} + +resource "aws_iam_role" "dns-controller-kube-system-sa-minimal-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_dns-controller.kube-system.sa.minimal.example.com_policy") + name = "dns-controller.kube-system.sa.minimal.example.com" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "dns-controller.kube-system.sa.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_iam_role" "karpenter-kube-system-sa-minimal-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_karpenter.kube-system.sa.minimal.example.com_policy") + name = "karpenter.kube-system.sa.minimal.example.com" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "karpenter.kube-system.sa.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + resource "aws_iam_role" "masters-minimal-example-com" { assume_role_policy = file("${path.module}/data/aws_iam_role_masters.minimal.example.com_policy") name = "masters.minimal.example.com" @@ -270,6 +380,18 @@ resource "aws_iam_role" "nodes-minimal-example-com" { } } +resource "aws_iam_role_policy" "dns-controller-kube-system-sa-minimal-example-com" { + name = "dns-controller.kube-system.sa.minimal.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_dns-controller.kube-system.sa.minimal.example.com_policy") + role = aws_iam_role.dns-controller-kube-system-sa-minimal-example-com.name +} + +resource "aws_iam_role_policy" "karpenter-kube-system-sa-minimal-example-com" { + name = "karpenter.kube-system.sa.minimal.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_karpenter.kube-system.sa.minimal.example.com_policy") + role = aws_iam_role.karpenter-kube-system-sa-minimal-example-com.name +} + resource "aws_iam_role_policy" "masters-minimal-example-com" { name = "masters.minimal.example.com" policy = file("${path.module}/data/aws_iam_role_policy_masters.minimal.example.com_policy") @@ -301,6 +423,79 @@ resource "aws_key_pair" "kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663 } } +resource "aws_launch_template" "karpenter-nodes-minimal-example-com" { + block_device_mappings { + device_name = "/dev/xvda" + ebs { + delete_on_termination = true + encrypted = true + iops = 3000 + throughput = 125 + volume_size = 128 + volume_type = "gp3" + } + } + iam_instance_profile { + name = aws_iam_instance_profile.nodes-minimal-example-com.id + } + image_id = "ami-12345678" + instance_type = "t2.medium" + key_name = aws_key_pair.kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id + lifecycle { + create_before_destroy = true + } + metadata_options { + http_endpoint = "enabled" + http_protocol_ipv6 = "disabled" + http_put_response_hop_limit = 1 + http_tokens = "optional" + } + monitoring { + enabled = false + } + name = "karpenter-nodes.minimal.example.com" + network_interfaces { + associate_public_ip_address = true + delete_on_termination = true + ipv6_address_count = 0 + security_groups = [aws_security_group.nodes-minimal-example-com.id] + } + tag_specifications { + resource_type = "instance" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "karpenter-nodes.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "karpenter-nodes" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + } + tag_specifications { + resource_type = "volume" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "karpenter-nodes.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "karpenter-nodes" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + } + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "karpenter-nodes.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "karpenter-nodes" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + user_data = filebase64("${path.module}/data/aws_launch_template_karpenter-nodes.minimal.example.com_user_data") +} + resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" { block_device_mappings { device_name = "/dev/xvda" @@ -495,6 +690,14 @@ resource "aws_s3_bucket_object" "cluster-completed-spec" { server_side_encryption = "AES256" } +resource "aws_s3_bucket_object" "discovery-json" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_discovery.json_content") + key = "discovery.example.com/minimal.example.com/.well-known/openid-configuration" + provider = aws.files + server_side_encryption = "AES256" +} + resource "aws_s3_bucket_object" "etcd-cluster-spec-events" { bucket = "testingBucket" content = file("${path.module}/data/aws_s3_bucket_object_etcd-cluster-spec-events_content") @@ -511,6 +714,14 @@ resource "aws_s3_bucket_object" "etcd-cluster-spec-main" { server_side_encryption = "AES256" } +resource "aws_s3_bucket_object" "keys-json" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_keys.json_content") + key = "discovery.example.com/minimal.example.com/openid/v1/jwks" + provider = aws.files + server_side_encryption = "AES256" +} + resource "aws_s3_bucket_object" "kops-version-txt" { bucket = "testingBucket" content = file("${path.module}/data/aws_s3_bucket_object_kops-version.txt_content") @@ -575,6 +786,14 @@ resource "aws_s3_bucket_object" "minimal-example-com-addons-dns-controller-addon server_side_encryption = "AES256" } +resource "aws_s3_bucket_object" "minimal-example-com-addons-karpenter-sh-k8s-1-19" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content") + key = "clusters.example.com/minimal.example.com/addons/karpenter.sh/k8s-1.19.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + resource "aws_s3_bucket_object" "minimal-example-com-addons-kops-controller-addons-k8s-io-k8s-1-16" { bucket = "testingBucket" content = file("${path.module}/data/aws_s3_bucket_object_minimal.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content") @@ -607,6 +826,14 @@ resource "aws_s3_bucket_object" "minimal-example-com-addons-storage-aws-addons-k server_side_encryption = "AES256" } +resource "aws_s3_bucket_object" "nodeupconfig-karpenter-nodes" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_bucket_object_nodeupconfig-karpenter-nodes_content") + key = "clusters.example.com/minimal.example.com/igconfig/node/karpenter-nodes/nodeupconfig.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + resource "aws_s3_bucket_object" "nodeupconfig-master-us-test-1a" { bucket = "testingBucket" content = file("${path.module}/data/aws_s3_bucket_object_nodeupconfig-master-us-test-1a_content") diff --git a/upup/models/BUILD.bazel b/upup/models/BUILD.bazel index d3c39612dd..82b751e5b3 100644 --- a/upup/models/BUILD.bazel +++ b/upup/models/BUILD.bazel @@ -55,6 +55,7 @@ go_library( "cloudup/resources/addons/external-dns.addons.k8s.io/k8s-1.19.yaml.template", "cloudup/resources/addons/leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml", "cloudup/resources/addons/leader-migration.rbac.addons.k8s.io/k8s-1.25.yaml", + "cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template", ], importpath = "k8s.io/kops/upup/models", visibility = ["//visibility:public"], diff --git a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template new file mode 100644 index 0000000000..151a235840 --- /dev/null +++ b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template @@ -0,0 +1,677 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + name: provisioners.karpenter.sh +spec: + group: karpenter.sh + names: + kind: Provisioner + listKind: ProvisionerList + plural: provisioners + singular: provisioner + scope: Cluster + versions: + - name: v1alpha5 + schema: + openAPIV3Schema: + description: Provisioner is the Schema for the Provisioners API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ProvisionerSpec is the top level provisioner specification. + Provisioners launch nodes in response to pods that are unschedulable. + A single provisioner is capable of managing a diverse set of nodes. + Node properties are determined from a combination of provisioner and + pod scheduling constraints. + properties: + labels: + additionalProperties: + type: string + description: Labels are layered with Requirements and applied to every + node. + type: object + limits: + description: Limits define a set of bounds for provisioning capacity. + properties: + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources contains all the allocatable resources + that Karpenter supports for limiting. + type: object + type: object + provider: + description: Provider contains fields specific to your cloudprovider. + type: object + x-kubernetes-preserve-unknown-fields: true + requirements: + description: Requirements are layered with Labels and applied to every + node. + items: + description: A node selector requirement is a selector that contains + values, a key, and an operator that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and + Lt. + type: string + values: + description: An array of string values. If the operator is In + or NotIn, the values array must be non-empty. If the operator + is Exists or DoesNotExist, the values array must be empty. + If the operator is Gt or Lt, the values array must have a + single element, which will be interpreted as an integer. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + taints: + description: Taints will be applied to every node launched by the + Provisioner. If specified, the provisioner will not provision nodes + for pods that do not have matching tolerations. Additional taints + will be created that match pod tolerations on a per-node basis. + items: + description: The node this Taint is attached to has the "effect" + on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods that + do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule + and NoExecute. + type: string + key: + description: Required. The taint key to be applied to a node. + type: string + timeAdded: + description: TimeAdded represents the time at which the taint + was added. It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + type: array + ttlSecondsAfterEmpty: + description: "TTLSecondsAfterEmpty is the number of seconds the controller + will wait before attempting to delete a node, measured from when + the node is detected to be empty. A Node is considered to be empty + when it does not have pods scheduled to it, excluding daemonsets. + \n Termination due to underutilization is disabled if this field + is not set." + format: int64 + type: integer + ttlSecondsUntilExpired: + description: "TTLSecondsUntilExpired is the number of seconds the + controller will wait before terminating a node, measured from when + the node is created. This is useful to implement features like eventually + consistent node upgrade, memory leak protection, and disruption + testing. \n Termination due to expiration is disabled if this field + is not set." + format: int64 + type: integer + type: object + status: + description: ProvisionerStatus defines the observed state of Provisioner + properties: + conditions: + description: Conditions is the set of conditions required for this + provisioner to scale its target, and indicates whether or not those + conditions are met. + items: + description: 'Condition defines a readiness condition for a Knative + resource. See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties' + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the condition + transitioned from one status to another. We use VolatileTime + in place of metav1.Time to exclude this from creating equality.Semantic + differences (all other things held constant). + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + severity: + description: Severity with which to treat failures of this type + of condition. When this is not specified, it defaults to Error. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - status + - type + type: object + type: array + lastScaleTime: + description: LastScaleTime is the last time the Provisioner scaled + the number of nodes + format: date-time + type: string + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is the list of resources that have been provisioned. + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +# Source: karpenter/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: karpenter + namespace: kube-system +--- +# Source: karpenter/templates/webhook/deployment.yaml +apiVersion: v1 +kind: Secret +metadata: + name: karpenter-webhook-cert + namespace: kube-system +data: {} # Injected by karpenter-webhook +--- +# Source: karpenter/templates/100-config-logging.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-logging + namespace: kube-system + labels: + app.kubernetes.io/part-of: karpenter +data: + # https://github.com/uber-go/zap/blob/aa3e73ec0896f8b066ddf668597a02f89628ee50/config.go + zap-logger-config: | + { + "level": "debug", + "development": true, + "disableStacktrace": true, + "disableCaller": true, + "sampling": { + "initial": 100, + "thereafter": 100 + }, + "outputPaths": ["stdout"], + "errorOutputPaths": ["stderr"], + "encoding": "console", + "encoderConfig": { + "timeKey": "time", + "levelKey": "level", + "nameKey": "logger", + "callerKey": "caller", + "messageKey": "message", + "stacktraceKey": "stacktrace", + "levelEncoder": "capital", + "timeEncoder": "iso8601" + } + } + # Log level overrides + # loglevel.controller: info # debug + # loglevel.webhook: info # debug +--- +# Source: karpenter/templates/controller/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: karpenter-controller +rules: +- apiGroups: ["karpenter.sh"] + resources: ["provisioners", "provisioners/status"] + verbs: ["create", "delete", "patch", "get", "list", "watch"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create", "get", "patch", "update", "watch"] +- apiGroups: [""] + resources: ["nodes", "pods"] + verbs: ["get", "list", "watch", "patch", "delete"] +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "update"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["create"] +- apiGroups: [""] + resources: ["pods/binding", "pods/eviction"] + verbs: ["create"] +- apiGroups: ["apps"] + resources: ["daemonsets"] + verbs: ["list", "watch"] +--- +# Source: karpenter/templates/webhook/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: karpenter-webhook +rules: +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["validatingwebhookconfigurations", "mutatingwebhookconfigurations"] + verbs: ["get", "watch", "list", "update"] +--- +# Source: karpenter/templates/controller/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: karpenter-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: karpenter-controller +subjects: +- kind: ServiceAccount + name: karpenter + namespace: kube-system +--- +# Source: karpenter/templates/webhook/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: karpenter-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: karpenter-webhook +subjects: +- kind: ServiceAccount + name: karpenter + namespace: kube-system +--- +# Source: karpenter/templates/controller/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: karpenter-controller + namespace: kube-system +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: [""] + resources: ["configmaps/status"] + verbs: ["get", "update", "patch"] +- apiGroups: [""] + resources: ["events"] + verbs: ["create"] +--- +# Source: karpenter/templates/webhook/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: karpenter-webhook + namespace: kube-system +rules: +- apiGroups: [""] + resources: ["configmaps", "namespaces"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch", "update"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "create", "update"] +--- +# Source: karpenter/templates/controller/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: karpenter-controller + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: karpenter-controller +subjects: +- kind: ServiceAccount + name: karpenter + namespace: kube-system +--- +# Source: karpenter/templates/webhook/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: karpenter-webhook + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: karpenter-webhook +subjects: +- kind: ServiceAccount + name: karpenter + namespace: kube-system +--- +# Source: karpenter/templates/controller/deployment.yaml +apiVersion: v1 +kind: Service +metadata: + name: karpenter-metrics + namespace: kube-system +spec: + ports: + - port: 8080 + targetPort: metrics + selector: + karpenter: controller +--- +# Source: karpenter/templates/webhook/deployment.yaml +apiVersion: v1 +kind: Service +metadata: + name: karpenter-webhook + namespace: kube-system +spec: + ports: + - port: 443 + targetPort: webhook + selector: + karpenter: webhook +--- +# Source: karpenter/templates/controller/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: karpenter-controller + namespace: kube-system +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + karpenter: controller + template: + metadata: + labels: + karpenter: controller + spec: + priorityClassName: system-cluster-critical + serviceAccountName: karpenter + containers: + - name: manager + image: public.ecr.aws/karpenter/controller:v0.5.2 + resources: + limits: + cpu: 1 + memory: 1Gi + requests: + cpu: 1 + memory: 1Gi + ports: + - name: metrics + containerPort: 8080 + - name: health-probe + containerPort: 8081 + livenessProbe: + httpGet: + path: /healthz + port: 8081 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + env: + - name: AWS_REGION + value: {{ Region }} + - name: CLUSTER_NAME + value: {{ ClusterName }} + - name: CLUSTER_ENDPOINT + value: https://{{ .MasterInternalName }} + - name: SYSTEM_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + - key: karpenter.sh/provisioner-name + operator: DoesNotExist +--- +# Source: karpenter/templates/webhook/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: karpenter-webhook + namespace: kube-system +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + karpenter: webhook + template: + metadata: + labels: + karpenter: webhook + spec: + priorityClassName: system-cluster-critical + serviceAccountName: karpenter + containers: + - name: webhook + image: public.ecr.aws/karpenter/webhook:v0.5.2 + args: + - -port=8443 + resources: + limits: + cpu: 100m + memory: 50Mi + requests: + cpu: 100m + memory: 50Mi + ports: + - name: webhook + containerPort: 8443 + livenessProbe: + httpGet: + scheme: HTTPS + port: 8443 + readinessProbe: + httpGet: + scheme: HTTPS + port: 8443 + env: + - name: AWS_REGION + value: {{ Region }} + - name: CLUSTER_NAME + value: {{ ClusterName }} + - name: CLUSTER_ENDPOINT + value: https://{{ .MasterInternalName }} + - name: SYSTEM_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + - key: karpenter.sh/provisioner-name + operator: DoesNotExist +--- +# Source: karpenter/templates/webhook/webhooks.yaml +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: defaulting.webhook.provisioners.karpenter.sh +webhooks: +- admissionReviewVersions: ["v1"] + clientConfig: + service: + name: karpenter-webhook + namespace: 'kube-system' + failurePolicy: Fail + sideEffects: None + name: defaulting.webhook.provisioners.karpenter.sh + rules: + - apiGroups: + - karpenter.sh + apiVersions: + - v1alpha5 + resources: + - provisioners + provisioners/status + operations: + - CREATE + - UPDATE +--- +# Source: karpenter/templates/webhook/webhooks.yaml +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validation.webhook.provisioners.karpenter.sh +webhooks: +- admissionReviewVersions: ["v1"] + clientConfig: + service: + name: karpenter-webhook + namespace: 'kube-system' + failurePolicy: Fail + sideEffects: None + name: validation.webhook.provisioners.karpenter.sh + rules: + - apiGroups: + - karpenter.sh + apiVersions: + - v1alpha5 + resources: + - provisioners + provisioners/status + operations: + - CREATE + - UPDATE +--- +# Source: karpenter/templates/webhook/webhooks.yaml +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validation.webhook.config.karpenter.sh +webhooks: +- admissionReviewVersions: ["v1"] + clientConfig: + service: + name: karpenter-webhook + namespace: 'kube-system' + failurePolicy: Fail + sideEffects: None + name: validation.webhook.config.karpenter.sh + objectSelector: + matchLabels: + app.kubernetes.io/part-of: karpenter + +{{ range $name, $spec := GetNodeInstanceGroups }} +{{ if eq $spec.InstanceManager "Karpenter" }} +--- +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: {{ $name }} +spec: + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["spot"] + - key: kubernetes.io/arch + operator: In + values: ["{{ SupportedArchitecture $spec.Image }}"] + - key: "node.kubernetes.io/instance-type" + operator: In + values: + - {{ $spec.MachineType }} + {{ with $spec.MixedInstancesPolicy }} + {{ range $key := .Instances }} + - {{ $key }} + {{ end }} + {{ end }} +{{ with $spec.Taints }} + taints: + {{ range $taintString := $spec.Taints }} + {{ $taint := ParseTaint $taintString }} + - key: {{ $taint.key }} + effect: {{ $taint.effect }} + {{ if $taint.value }} + value: {{ $taint.value }} + {{ end }} +{{ end }} +{{ end }} +{{ with $spec.NodeLabels }} + labels: + {{ range $key, $value := . }} + {{ $key }}: {{ $value }} + {{ end }} +{{ end }} + limits: + resources: + cpu: 1000 + provider: + instanceProfile: nodes.{{ ClusterName }} + launchTemplate: {{ $name }} + securityGroupSelector: + Name: nodes.{{ ClusterName }} + subnetSelector: + kops.k8s.io/karpenter/{{ $name }}: "*" + ttlSecondsAfterEmpty: 30 +{{ end }} +{{ end }} \ No newline at end of file diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/BUILD.bazel b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/BUILD.bazel index cc45e57c72..5cdd5548eb 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/BUILD.bazel +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/BUILD.bazel @@ -24,6 +24,7 @@ go_library( "//pkg/model/components/addonmanifests/clusterautoscaler:go_default_library", "//pkg/model/components/addonmanifests/dnscontroller:go_default_library", "//pkg/model/components/addonmanifests/externaldns:go_default_library", + "//pkg/model/components/addonmanifests/karpenter:go_default_library", "//pkg/model/components/addonmanifests/nodeterminationhandler:go_default_library", "//pkg/model/iam:go_default_library", "//pkg/templates:go_default_library", diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go index dd4c48dc0d..a550405227 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go @@ -21,6 +21,7 @@ import ( "strings" "k8s.io/klog/v2" + channelsapi "k8s.io/kops/channels/pkg/api" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/assets" @@ -35,6 +36,7 @@ import ( "k8s.io/kops/pkg/model/components/addonmanifests/clusterautoscaler" "k8s.io/kops/pkg/model/components/addonmanifests/dnscontroller" "k8s.io/kops/pkg/model/components/addonmanifests/externaldns" + "k8s.io/kops/pkg/model/components/addonmanifests/karpenter" "k8s.io/kops/pkg/model/components/addonmanifests/nodeterminationhandler" "k8s.io/kops/pkg/model/iam" "k8s.io/kops/pkg/templates" @@ -1058,6 +1060,24 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*Addon }) } } + if b.Cluster.Spec.Karpenter != nil && fi.BoolValue(&b.Cluster.Spec.Karpenter.Enabled) { + key := "karpenter.sh" + + { + id := "k8s-1.19" + location := key + "/" + id + ".yaml" + addons.Add(&channelsapi.AddonSpec{ + Name: fi.String(key), + Manifest: fi.String(location), + Selector: map[string]string{"k8s-addon": key}, + NeedsPKI: true, + Id: id, + }) + if b.UseServiceAccountExternalPermissions() { + serviceAccountRoles = append(serviceAccountRoles, &karpenter.ServiceAccount{}) + } + } + } if b.Cluster.Spec.KubeScheduler.UsePolicyConfigMap != nil { key := "scheduler.addons.k8s.io" diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 3ec3c1db4a..add1d3d7f8 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -43,6 +43,8 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" + "sigs.k8s.io/yaml" + kopscontrollerconfig "k8s.io/kops/cmd/kops-controller/pkg/config" "k8s.io/kops/pkg/apis/kops" apiModel "k8s.io/kops/pkg/apis/kops/model" @@ -59,7 +61,6 @@ import ( "k8s.io/kops/upup/pkg/fi/cloudup/gce" gcetpm "k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm" "k8s.io/kops/util/pkg/env" - "sigs.k8s.io/yaml" ) // TemplateFunctions provides a collection of methods used throughout the templates @@ -280,6 +281,10 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS dest["EnableSQSTerminationDraining"] = func() bool { return *cluster.Spec.NodeTerminationHandler.EnableSQSTerminationDraining } } + dest["SupportedArchitecture"] = tf.SupportedArchitecture + + dest["ParseTaint"] = parseTaint + return nil } @@ -735,3 +740,48 @@ func (tf *TemplateFunctions) GetNodeInstanceGroups() map[string]kops.InstanceGro } return nodegroups } + +func (tf *TemplateFunctions) SupportedArchitecture(amiID string) string { + image, _ := tf.cloud.(awsup.AWSCloud).ResolveImage(amiID) + switch *image.Architecture { + case "x86_64": + return "amd64" + } + return "arm64" +} + +// parseTaint takes a string and returns a map of its value +// it mimics the function from https://github.com/kubernetes/kubernetes/blob/master/pkg/util/taints/taints.go +// but returns a map instead of a v1.Taint +func parseTaint(st string) (map[string]string, error) { + taint := make(map[string]string) + + var key string + var value string + var effect string + + parts := strings.Split(st, ":") + switch len(parts) { + case 1: + key = parts[0] + case 2: + effect = parts[1] + + partsKV := strings.Split(parts[0], "=") + if len(partsKV) > 2 { + return taint, fmt.Errorf("invalid taint spec: %v", st) + } + key = partsKV[0] + if len(partsKV) == 2 { + value = partsKV[1] + } + default: + return taint, fmt.Errorf("invalid taint spec: %v", st) + } + + taint["key"] = key + taint["value"] = value + taint["effect"] = effect + + return taint, nil +} From 043dd91417684e51ff9c2577c7159b04e4cd86d2 Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Tue, 7 Dec 2021 22:44:04 +0100 Subject: [PATCH 03/13] Add karpenter feature flag --- cmd/kops/integration_test.go | 6 +++ pkg/apis/kops/validation/validation.go | 7 ++++ pkg/featureflag/featureflag.go | 2 + ...nimal.example.com-addons-bootstrap_content | 2 +- ...e.com-addons-karpenter.sh-k8s-1.19_content | 37 +++++++++++++++++++ .../update_cluster/karpenter/in-v1alpha2.yaml | 1 + .../update_cluster/karpenter/kubernetes.tf | 1 + 7 files changed, 55 insertions(+), 1 deletion(-) diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index afdce7aacc..10a7e4c890 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -595,6 +595,12 @@ func TestExternalDNSIRSA(t *testing.T) { } func TestKarpenter(t *testing.T) { + featureflag.ParseFlags("+Karpenter") + unsetFeatureFlags := func() { + featureflag.ParseFlags("-Karpenter") + } + defer unsetFeatureFlags() + test := newIntegrationTest("minimal.example.com", "karpenter"). withOIDCDiscovery(). withAddons(dnsControllerAddon). diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index 2369cad05f..a9cac6adc4 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -37,6 +37,7 @@ import ( "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/dns" + "k8s.io/kops/pkg/featureflag" "k8s.io/kops/pkg/model/components" "k8s.io/kops/pkg/model/iam" "k8s.io/kops/upup/pkg/fi" @@ -265,6 +266,12 @@ func validateClusterSpec(spec *kops.ClusterSpec, c *kops.Cluster, fieldPath *fie } } + if spec.Karpenter != nil && spec.Karpenter.Enabled { + if !featureflag.Karpenter.Enabled() { + allErrs = append(allErrs, field.Forbidden(fieldPath.Child("karpenter", "enabled"), "karpenter requires the Karpenter feature flag")) + } + } + return allErrs } diff --git a/pkg/featureflag/featureflag.go b/pkg/featureflag/featureflag.go index 2d408d3d19..c749adba2b 100644 --- a/pkg/featureflag/featureflag.go +++ b/pkg/featureflag/featureflag.go @@ -90,6 +90,8 @@ var ( TerraformManagedFiles = new("TerraformManagedFiles", Bool(true)) // AlphaAllowGCE is a feature flag that gates GCE support while it is alpha. AlphaAllowGCE = new("AlphaAllowGCE", Bool(false)) + // Karpenter enables karpenter-managed Instance Groups + Karpenter = new("Karpenter", Bool(false)) ) // FeatureFlag defines a feature flag diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content index d309547774..18d504a801 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content @@ -54,7 +54,7 @@ spec: version: 9.99.0 - id: k8s-1.19 manifest: karpenter.sh/k8s-1.19.yaml - manifestHash: c0a94c541234427f6ff9f07d4b83cb6ba042a2430373058c168ad249827d610b + manifestHash: d31ab5979f05095a9ec57bbbf1a2ac14a52a9e321d7732832dcd14f44be39551 name: karpenter.sh needsPKI: true selector: diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content index 23bd852a3f..ece199ae8c 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content @@ -846,3 +846,40 @@ webhooks: matchLabels: app.kubernetes.io/part-of: karpenter sideEffects: None + +--- + +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: karpenter.sh + app.kubernetes.io/managed-by: kops + k8s-addon: karpenter.sh + name: karpenter-nodes +spec: + limits: + resources: + cpu: 1000 + provider: + instanceProfile: nodes.minimal.example.com + launchTemplate: karpenter-nodes + securityGroupSelector: + Name: nodes.minimal.example.com + subnetSelector: + kops.k8s.io/karpenter/karpenter-nodes: '*' + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: + - spot + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: node.kubernetes.io/instance-type + operator: In + values: + - t2.medium + ttlSecondsAfterEmpty: 30 diff --git a/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml b/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml index 0375017eef..23d092af34 100644 --- a/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml @@ -92,6 +92,7 @@ metadata: labels: kops.k8s.io/cluster: minimal.example.com spec: + instanceManager: Karpenter associatePublicIp: true image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 machineType: t2.medium diff --git a/tests/integration/update_cluster/karpenter/kubernetes.tf b/tests/integration/update_cluster/karpenter/kubernetes.tf index b64406c9db..1e923988c7 100644 --- a/tests/integration/update_cluster/karpenter/kubernetes.tf +++ b/tests/integration/update_cluster/karpenter/kubernetes.tf @@ -1005,6 +1005,7 @@ resource "aws_subnet" "us-test-1a-minimal-example-com" { "KubernetesCluster" = "minimal.example.com" "Name" = "us-test-1a.minimal.example.com" "SubnetType" = "Public" + "kops.k8s.io/karpenter/karpenter-nodes" = "1" "kubernetes.io/cluster/minimal.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" From d4ec7cc7e06411a120c09b40ed4d100baea1429b Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Mon, 6 Dec 2021 20:27:25 +0100 Subject: [PATCH 04/13] Make kops get instances work with karpenter --- pkg/model/awsmodel/autoscalinggroup.go | 13 +- pkg/model/awsmodel/nodeterminationhandler.go | 9 +- .../complex/expected-v1alpha2.yaml | 2 + .../gce_byo_sa/expected-v1alpha2.yaml | 2 + .../create_cluster/ha/expected-v1alpha2.yaml | 6 + .../ha_encrypt/expected-v1alpha2.yaml | 6 + .../ha_gce/expected-v1alpha2.yaml | 6 + .../ha_shared_zone/expected-v1alpha2.yaml | 4 + .../ha_shared_zones/expected-v1alpha2.yaml | 7 + .../ingwspecified/expected-v1alpha2.yaml | 3 + .../ipv6/expected-v1alpha2.yaml | 2 + .../minimal-1.18/expected-v1alpha2.yaml | 2 + .../minimal-1.19/expected-v1alpha2.yaml | 2 + .../minimal-1.20/expected-v1alpha2.yaml | 2 + .../minimal-1.21/expected-v1alpha2.yaml | 2 + .../minimal-1.22/expected-v1alpha2.yaml | 2 + .../minimal-1.23/expected-v1alpha2.yaml | 2 + .../ngwspecified/expected-v1alpha2.yaml | 3 + .../overrides/expected-v1alpha2.yaml | 2 + .../private/expected-v1alpha2.yaml | 3 + .../private_gce/expected-v1alpha2.yaml | 3 + .../expected-v1alpha2.yaml | 2 + .../shared_subnets/expected-v1alpha2.yaml | 2 + .../expected-v1alpha2.yaml | 2 + .../shared_vpc/expected-v1alpha2.yaml | 2 + ...nimal.example.com-addons-bootstrap_content | 2 +- ...e.com-addons-karpenter.sh-k8s-1.19_content | 2 +- .../update_cluster/karpenter/kubernetes.tf | 53 +------- .../karpenter.sh/k8s-1.19.yaml.template | 2 +- upup/pkg/fi/cloudup/awsup/aws_cloud.go | 123 +++++++++++++++++- .../fi/cloudup/populate_instancegroup_spec.go | 5 + 31 files changed, 214 insertions(+), 64 deletions(-) diff --git a/pkg/model/awsmodel/autoscalinggroup.go b/pkg/model/awsmodel/autoscalinggroup.go index 394e2d2e9b..3de993608e 100644 --- a/pkg/model/awsmodel/autoscalinggroup.go +++ b/pkg/model/awsmodel/autoscalinggroup.go @@ -24,6 +24,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "k8s.io/klog/v2" + "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/featureflag" "k8s.io/kops/pkg/model" @@ -79,12 +80,14 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { c.AddTask(task) // @step: now lets build the autoscaling group task - tsk, err := b.buildAutoScalingGroupTask(c, name, ig) - if err != nil { - return err + if ig.Spec.InstanceManager != "Karpenter" { + tsk, err := b.buildAutoScalingGroupTask(c, name, ig) + if err != nil { + return err + } + tsk.LaunchTemplate = task + c.AddTask(tsk) } - tsk.LaunchTemplate = task - c.AddTask(tsk) warmPool := b.Cluster.Spec.WarmPool.ResolveDefaults(ig) { diff --git a/pkg/model/awsmodel/nodeterminationhandler.go b/pkg/model/awsmodel/nodeterminationhandler.go index 772413e2fa..3287e2dc9e 100644 --- a/pkg/model/awsmodel/nodeterminationhandler.go +++ b/pkg/model/awsmodel/nodeterminationhandler.go @@ -25,6 +25,7 @@ import ( "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "github.com/aws/aws-sdk-go/aws" + "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi" ) @@ -80,9 +81,11 @@ type NodeTerminationHandlerBuilder struct { func (b *NodeTerminationHandlerBuilder) Build(c *fi.ModelBuilderContext) error { for _, ig := range b.InstanceGroups { - err := b.configureASG(c, ig) - if err != nil { - return err + if ig.Spec.InstanceManager == kops.InstanceManagerCloudGroup { + err := b.configureASG(c, ig) + if err != nil { + return err + } } } diff --git a/tests/integration/create_cluster/complex/expected-v1alpha2.yaml b/tests/integration/create_cluster/complex/expected-v1alpha2.yaml index 1b416599ca..ecf6f14ac7 100644 --- a/tests/integration/create_cluster/complex/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/complex/expected-v1alpha2.yaml @@ -64,6 +64,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -84,6 +85,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml b/tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml index d433f0a5be..bf0f00317f 100644 --- a/tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml @@ -65,6 +65,7 @@ metadata: name: master-us-test1-a spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: n1-standard-1 maxSize: 1 minSize: 1 @@ -88,6 +89,7 @@ metadata: name: nodes-us-test1-a spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: n1-standard-2 maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/ha/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha/expected-v1alpha2.yaml index a2131e0256..4142d2e51d 100644 --- a/tests/integration/create_cluster/ha/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha/expected-v1alpha2.yaml @@ -85,6 +85,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -105,6 +106,7 @@ metadata: name: master-us-test-1b spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -125,6 +127,7 @@ metadata: name: master-us-test-1c spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -145,6 +148,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 @@ -165,6 +169,7 @@ metadata: name: nodes-us-test-1b spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 @@ -185,6 +190,7 @@ metadata: name: nodes-us-test-1c spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/ha_encrypt/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha_encrypt/expected-v1alpha2.yaml index a2131e0256..4142d2e51d 100644 --- a/tests/integration/create_cluster/ha_encrypt/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha_encrypt/expected-v1alpha2.yaml @@ -85,6 +85,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -105,6 +106,7 @@ metadata: name: master-us-test-1b spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -125,6 +127,7 @@ metadata: name: master-us-test-1c spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -145,6 +148,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 @@ -165,6 +169,7 @@ metadata: name: nodes-us-test-1b spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 @@ -185,6 +190,7 @@ metadata: name: nodes-us-test-1c spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml index e1ab6da893..22cc833ed4 100644 --- a/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml @@ -73,6 +73,7 @@ metadata: name: master-us-test1-a spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: n1-standard-1 maxSize: 1 minSize: 1 @@ -96,6 +97,7 @@ metadata: name: master-us-test1-b spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: n1-standard-1 maxSize: 1 minSize: 1 @@ -119,6 +121,7 @@ metadata: name: master-us-test1-c spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: n1-standard-1 maxSize: 1 minSize: 1 @@ -142,6 +145,7 @@ metadata: name: nodes-us-test1-a spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: n1-standard-2 maxSize: 1 minSize: 1 @@ -165,6 +169,7 @@ metadata: name: nodes-us-test1-b spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: n1-standard-2 maxSize: 1 minSize: 1 @@ -188,6 +193,7 @@ metadata: name: nodes-us-test1-c spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: n1-standard-2 maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/ha_shared_zone/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha_shared_zone/expected-v1alpha2.yaml index 1e10778abf..d1f94b6bc0 100644 --- a/tests/integration/create_cluster/ha_shared_zone/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha_shared_zone/expected-v1alpha2.yaml @@ -77,6 +77,7 @@ metadata: name: master-us-test-1a-1 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -97,6 +98,7 @@ metadata: name: master-us-test-1a-2 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -117,6 +119,7 @@ metadata: name: master-us-test-1a-3 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -137,6 +140,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/ha_shared_zones/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha_shared_zones/expected-v1alpha2.yaml index 411454465c..16da87cbc0 100644 --- a/tests/integration/create_cluster/ha_shared_zones/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha_shared_zones/expected-v1alpha2.yaml @@ -93,6 +93,7 @@ metadata: name: master-us-test-1a-1 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -113,6 +114,7 @@ metadata: name: master-us-test-1a-2 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -133,6 +135,7 @@ metadata: name: master-us-test-1a-3 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -153,6 +156,7 @@ metadata: name: master-us-test-1b-1 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -173,6 +177,7 @@ metadata: name: master-us-test-1b-2 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -193,6 +198,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 @@ -213,6 +219,7 @@ metadata: name: nodes-us-test-1b spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/ingwspecified/expected-v1alpha2.yaml b/tests/integration/create_cluster/ingwspecified/expected-v1alpha2.yaml index 729838a4f8..8e7178a6a7 100644 --- a/tests/integration/create_cluster/ingwspecified/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ingwspecified/expected-v1alpha2.yaml @@ -74,6 +74,7 @@ metadata: name: bastions spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.micro maxSize: 1 minSize: 1 @@ -94,6 +95,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -114,6 +116,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/ipv6/expected-v1alpha2.yaml b/tests/integration/create_cluster/ipv6/expected-v1alpha2.yaml index b62c91b66e..5304026462 100644 --- a/tests/integration/create_cluster/ipv6/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ipv6/expected-v1alpha2.yaml @@ -67,6 +67,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup instanceMetadata: httpPutResponseHopLimit: 3 httpTokens: required @@ -90,6 +91,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup instanceMetadata: httpPutResponseHopLimit: 1 httpTokens: required diff --git a/tests/integration/create_cluster/minimal-1.18/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.18/expected-v1alpha2.yaml index f8a2a4e58c..ec4a6fb0ee 100644 --- a/tests/integration/create_cluster/minimal-1.18/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.18/expected-v1alpha2.yaml @@ -65,6 +65,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -85,6 +86,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/minimal-1.19/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.19/expected-v1alpha2.yaml index 8aa10625ee..4cec7f0f24 100644 --- a/tests/integration/create_cluster/minimal-1.19/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.19/expected-v1alpha2.yaml @@ -65,6 +65,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -85,6 +86,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/minimal-1.20/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.20/expected-v1alpha2.yaml index bbb082cb66..3960ae47c9 100644 --- a/tests/integration/create_cluster/minimal-1.20/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.20/expected-v1alpha2.yaml @@ -65,6 +65,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -85,6 +86,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/minimal-1.21/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.21/expected-v1alpha2.yaml index bbb082cb66..3960ae47c9 100644 --- a/tests/integration/create_cluster/minimal-1.21/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.21/expected-v1alpha2.yaml @@ -65,6 +65,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -85,6 +86,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/minimal-1.22/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.22/expected-v1alpha2.yaml index 60ece11212..7dde515496 100644 --- a/tests/integration/create_cluster/minimal-1.22/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.22/expected-v1alpha2.yaml @@ -65,6 +65,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup instanceMetadata: httpPutResponseHopLimit: 3 httpTokens: required @@ -88,6 +89,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup instanceMetadata: httpPutResponseHopLimit: 1 httpTokens: required diff --git a/tests/integration/create_cluster/minimal-1.23/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.23/expected-v1alpha2.yaml index 67d0345559..2462129ec2 100644 --- a/tests/integration/create_cluster/minimal-1.23/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.23/expected-v1alpha2.yaml @@ -65,6 +65,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup instanceMetadata: httpPutResponseHopLimit: 3 httpTokens: required @@ -88,6 +89,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup instanceMetadata: httpPutResponseHopLimit: 1 httpTokens: required diff --git a/tests/integration/create_cluster/ngwspecified/expected-v1alpha2.yaml b/tests/integration/create_cluster/ngwspecified/expected-v1alpha2.yaml index b432f78124..0a798de54d 100644 --- a/tests/integration/create_cluster/ngwspecified/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ngwspecified/expected-v1alpha2.yaml @@ -74,6 +74,7 @@ metadata: name: bastions spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.micro maxSize: 1 minSize: 1 @@ -94,6 +95,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -114,6 +116,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/overrides/expected-v1alpha2.yaml b/tests/integration/create_cluster/overrides/expected-v1alpha2.yaml index a0a1174099..7f435ad173 100644 --- a/tests/integration/create_cluster/overrides/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/overrides/expected-v1alpha2.yaml @@ -68,6 +68,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -88,6 +89,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/private/expected-v1alpha2.yaml b/tests/integration/create_cluster/private/expected-v1alpha2.yaml index 9d5e89339c..693f50ecdc 100644 --- a/tests/integration/create_cluster/private/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/private/expected-v1alpha2.yaml @@ -77,6 +77,7 @@ metadata: name: bastions spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.micro maxSize: 1 minSize: 1 @@ -100,6 +101,7 @@ spec: - sg-exampleid3 - sg-exampleid4 image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -123,6 +125,7 @@ spec: - sg-exampleid - sg-exampleid2 image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/private_gce/expected-v1alpha2.yaml b/tests/integration/create_cluster/private_gce/expected-v1alpha2.yaml index 4c8f859085..59aa216768 100644 --- a/tests/integration/create_cluster/private_gce/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/private_gce/expected-v1alpha2.yaml @@ -72,6 +72,7 @@ metadata: name: bastions spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: f1-micro maxSize: 1 minSize: 1 @@ -98,6 +99,7 @@ spec: - sg-exampleid3 - sg-exampleid4 image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: n1-standard-1 maxSize: 1 minSize: 1 @@ -124,6 +126,7 @@ spec: - sg-exampleid - sg-exampleid2 image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 + instanceManager: CloudGroup machineType: n1-standard-2 maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/private_shared_subnets/expected-v1alpha2.yaml b/tests/integration/create_cluster/private_shared_subnets/expected-v1alpha2.yaml index 10ca073d3c..6e70791362 100644 --- a/tests/integration/create_cluster/private_shared_subnets/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/private_shared_subnets/expected-v1alpha2.yaml @@ -74,6 +74,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -94,6 +95,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/shared_subnets/expected-v1alpha2.yaml b/tests/integration/create_cluster/shared_subnets/expected-v1alpha2.yaml index a2847cf5bc..36357e5a5f 100644 --- a/tests/integration/create_cluster/shared_subnets/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/shared_subnets/expected-v1alpha2.yaml @@ -67,6 +67,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -87,6 +88,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/shared_subnets_vpc_lookup/expected-v1alpha2.yaml b/tests/integration/create_cluster/shared_subnets_vpc_lookup/expected-v1alpha2.yaml index a2847cf5bc..36357e5a5f 100644 --- a/tests/integration/create_cluster/shared_subnets_vpc_lookup/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/shared_subnets_vpc_lookup/expected-v1alpha2.yaml @@ -67,6 +67,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -87,6 +88,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/create_cluster/shared_vpc/expected-v1alpha2.yaml b/tests/integration/create_cluster/shared_vpc/expected-v1alpha2.yaml index 86dafd4d05..b00d62cefa 100644 --- a/tests/integration/create_cluster/shared_vpc/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/shared_vpc/expected-v1alpha2.yaml @@ -66,6 +66,7 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: m3.medium maxSize: 1 minSize: 1 @@ -86,6 +87,7 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 + instanceManager: CloudGroup machineType: t2.medium maxSize: 1 minSize: 1 diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content index 18d504a801..a3798692b0 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content @@ -54,7 +54,7 @@ spec: version: 9.99.0 - id: k8s-1.19 manifest: karpenter.sh/k8s-1.19.yaml - manifestHash: d31ab5979f05095a9ec57bbbf1a2ac14a52a9e321d7732832dcd14f44be39551 + manifestHash: f5bf5ee51335406c3ad93bfa6cea5778dfddba76f353da2b89bfdc30b925bd13 name: karpenter.sh needsPKI: true selector: diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content index ece199ae8c..e45fa59d72 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content @@ -864,7 +864,7 @@ spec: cpu: 1000 provider: instanceProfile: nodes.minimal.example.com - launchTemplate: karpenter-nodes + launchTemplate: karpenter-nodes.minimal.example.com securityGroupSelector: Name: nodes.minimal.example.com subnetSelector: diff --git a/tests/integration/update_cluster/karpenter/kubernetes.tf b/tests/integration/update_cluster/karpenter/kubernetes.tf index 1e923988c7..6e38358a48 100644 --- a/tests/integration/update_cluster/karpenter/kubernetes.tf +++ b/tests/integration/update_cluster/karpenter/kubernetes.tf @@ -10,7 +10,7 @@ locals { master_security_group_ids = [aws_security_group.masters-minimal-example-com.id] masters_role_arn = aws_iam_role.masters-minimal-example-com.arn masters_role_name = aws_iam_role.masters-minimal-example-com.name - node_autoscaling_group_ids = [aws_autoscaling_group.karpenter-nodes-minimal-example-com.id, aws_autoscaling_group.nodes-minimal-example-com.id] + node_autoscaling_group_ids = [aws_autoscaling_group.nodes-minimal-example-com.id] node_security_group_ids = [aws_security_group.nodes-minimal-example-com.id] node_subnet_ids = [aws_subnet.us-test-1a-minimal-example-com.id] nodes_role_arn = aws_iam_role.nodes-minimal-example-com.arn @@ -67,7 +67,7 @@ output "masters_role_name" { } output "node_autoscaling_group_ids" { - value = [aws_autoscaling_group.karpenter-nodes-minimal-example-com.id, aws_autoscaling_group.nodes-minimal-example-com.id] + value = [aws_autoscaling_group.nodes-minimal-example-com.id] } output "node_security_group_ids" { @@ -115,55 +115,6 @@ provider "aws" { region = "us-test-1" } -resource "aws_autoscaling_group" "karpenter-nodes-minimal-example-com" { - enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] - launch_template { - id = aws_launch_template.karpenter-nodes-minimal-example-com.id - version = aws_launch_template.karpenter-nodes-minimal-example-com.latest_version - } - max_size = 2 - metrics_granularity = "1Minute" - min_size = 2 - name = "karpenter-nodes.minimal.example.com" - protect_from_scale_in = false - tag { - key = "KubernetesCluster" - propagate_at_launch = true - value = "minimal.example.com" - } - tag { - key = "Name" - propagate_at_launch = true - value = "karpenter-nodes.minimal.example.com" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" - propagate_at_launch = true - value = "node" - } - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" - propagate_at_launch = true - value = "" - } - tag { - key = "k8s.io/role/node" - propagate_at_launch = true - value = "1" - } - tag { - key = "kops.k8s.io/instancegroup" - propagate_at_launch = true - value = "karpenter-nodes" - } - tag { - key = "kubernetes.io/cluster/minimal.example.com" - propagate_at_launch = true - value = "owned" - } - vpc_zone_identifier = [aws_subnet.us-test-1a-minimal-example-com.id] -} - resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" { enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] launch_template { diff --git a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template index 151a235840..26f2605958 100644 --- a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template +++ b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template @@ -667,7 +667,7 @@ spec: cpu: 1000 provider: instanceProfile: nodes.{{ ClusterName }} - launchTemplate: {{ $name }} + launchTemplate: {{ $name }}.{{ ClusterName }} securityGroupSelector: Name: nodes.{{ ClusterName }} subnetSelector: diff --git a/upup/pkg/fi/cloudup/awsup/aws_cloud.go b/upup/pkg/fi/cloudup/awsup/aws_cloud.go index df96ea5d69..072d5d571e 100644 --- a/upup/pkg/fi/cloudup/awsup/aws_cloud.go +++ b/upup/pkg/fi/cloudup/awsup/aws_cloud.go @@ -51,6 +51,8 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" + k8s_aws "k8s.io/legacy-cloud-providers/aws" + "k8s.io/kops/dnsprovider/pkg/dnsprovider" dnsproviderroute53 "k8s.io/kops/dnsprovider/pkg/dnsprovider/providers/aws/route53" "k8s.io/kops/pkg/apis/kops" @@ -60,7 +62,6 @@ import ( identity_aws "k8s.io/kops/pkg/nodeidentity/aws" "k8s.io/kops/pkg/resources/spotinst" "k8s.io/kops/upup/pkg/fi" - k8s_aws "k8s.io/legacy-cloud-providers/aws" ) // By default, aws-sdk-go only retries 3 times, which doesn't give @@ -622,7 +623,125 @@ func (c *awsCloudImplementation) GetCloudGroups(cluster *kops.Cluster, instanceg return sgroups, nil } - return getCloudGroups(c, cluster, instancegroups, warnUnmatched, nodes) + cloudGroups, err := getCloudGroups(c, cluster, instancegroups, warnUnmatched, nodes) + if err != nil { + return nil, err + } + karpenterGroups, err := getKarpenterGroups(c, cluster, instancegroups, nodes) + if err != nil { + return nil, err + } + + for name, group := range karpenterGroups { + cloudGroups[name] = group + } + return cloudGroups, nil +} + +func getKarpenterGroups(c AWSCloud, cluster *kops.Cluster, instancegroups []*kops.InstanceGroup, nodes []v1.Node) (map[string]*cloudinstances.CloudInstanceGroup, error) { + cloudGroups := make(map[string]*cloudinstances.CloudInstanceGroup) + for _, ig := range instancegroups { + if ig.Spec.InstanceManager == kops.InstanceManagerKarpenter { + group, err := buildKarpenterGroup(c, cluster, ig, nodes) + if err != nil { + return nil, err + } + cloudGroups[ig.ObjectMeta.Name] = group + } + } + return cloudGroups, nil +} + +func buildKarpenterGroup(c AWSCloud, cluster *kops.Cluster, ig *kops.InstanceGroup, nodes []v1.Node) (*cloudinstances.CloudInstanceGroup, error) { + nodeMap := cloudinstances.GetNodeMap(nodes, cluster) + instances := make(map[string]*ec2.Instance) + updatedInstances := make(map[string]*ec2.Instance) + clusterName := c.Tags()[TagClusterName] + var version string + + { + result, err := c.EC2().DescribeLaunchTemplates(&ec2.DescribeLaunchTemplatesInput{ + Filters: []*ec2.Filter{ + NewEC2Filter("tag:"+identity_aws.CloudTagInstanceGroupName, ig.ObjectMeta.Name), + NewEC2Filter("tag:"+TagClusterName, clusterName), + }, + }) + if err != nil { + return nil, err + } + lt := result.LaunchTemplates[0] + versionNumber := *lt.LatestVersionNumber + version = strconv.Itoa(int(versionNumber)) + + } + + karpenterGroup := &cloudinstances.CloudInstanceGroup{ + InstanceGroup: ig, + HumanName: ig.ObjectMeta.Name, + } + { + req := &ec2.DescribeInstancesInput{ + Filters: []*ec2.Filter{ + NewEC2Filter("tag:"+identity_aws.CloudTagInstanceGroupName, ig.ObjectMeta.Name), + NewEC2Filter("tag:"+TagClusterName, clusterName), + NewEC2Filter("instance-state-name", "pending", "running", "stopping", "stopped"), + }, + } + + result, err := c.EC2().DescribeInstances(req) + if err != nil { + return nil, err + } + + for _, r := range result.Reservations { + for _, i := range r.Instances { + id := aws.StringValue(i.InstanceId) + instances[id] = i + } + } + } + + klog.Infof("found %d karpenter instances", len(instances)) + + { + req := &ec2.DescribeInstancesInput{ + Filters: []*ec2.Filter{ + NewEC2Filter("tag:"+identity_aws.CloudTagInstanceGroupName, ig.ObjectMeta.Name), + NewEC2Filter("tag:"+TagClusterName, clusterName), + NewEC2Filter("instance-state-name", "pending", "running", "stopping", "stopped"), + NewEC2Filter("tag:aws:ec2launchtemplate:version", version), + }, + } + + result, err := c.EC2().DescribeInstances(req) + if err != nil { + return nil, err + } + + for _, r := range result.Reservations { + for _, i := range r.Instances { + id := aws.StringValue(i.InstanceId) + updatedInstances[id] = i + } + } + } + klog.Infof("found %d updated instances", len(updatedInstances)) + + { + for _, instance := range instances { + id := *instance.InstanceId + _, ready := updatedInstances[id] + var status string + if ready { + status = cloudinstances.CloudInstanceStatusUpToDate + } else { + status = cloudinstances.CloudInstanceStatusNeedsUpdate + } + cloudInstance, _ := karpenterGroup.NewCloudInstance(id, status, nodeMap[id]) + addCloudInstanceData(cloudInstance, instance) + } + } + return karpenterGroup, nil } func getCloudGroups(c AWSCloud, cluster *kops.Cluster, instancegroups []*kops.InstanceGroup, warnUnmatched bool, nodes []v1.Node) (map[string]*cloudinstances.CloudInstanceGroup, error) { diff --git a/upup/pkg/fi/cloudup/populate_instancegroup_spec.go b/upup/pkg/fi/cloudup/populate_instancegroup_spec.go index 1f9d241d48..d817ca7744 100644 --- a/upup/pkg/fi/cloudup/populate_instancegroup_spec.go +++ b/upup/pkg/fi/cloudup/populate_instancegroup_spec.go @@ -22,6 +22,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/blang/semver/v4" "k8s.io/klog/v2" + "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/kops/util" "k8s.io/kops/pkg/apis/kops/validation" @@ -173,6 +174,10 @@ func PopulateInstanceGroupSpec(cluster *kops.Cluster, input *kops.InstanceGroup, } } } + + if ig.Spec.InstanceManager == "" { + ig.Spec.InstanceManager = kops.InstanceManagerCloudGroup + } return ig, nil } From 1ccb7840ac60ac8053898a54d6f6dca33ad8b88d Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Tue, 7 Dec 2021 13:06:08 +0100 Subject: [PATCH 05/13] make rolling update work --- pkg/instancegroups/settings.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/instancegroups/settings.go b/pkg/instancegroups/settings.go index e6247f3544..0dd208a8e5 100644 --- a/pkg/instancegroups/settings.go +++ b/pkg/instancegroups/settings.go @@ -18,6 +18,7 @@ package instancegroups import ( "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/featureflag" "k8s.io/kops/upup/pkg/fi" @@ -47,7 +48,7 @@ func resolveSettings(cluster *kops.Cluster, group *kops.InstanceGroup, numInstan if rollingUpdate.MaxSurge == nil { val := intstr.FromInt(0) - if kops.CloudProviderID(cluster.Spec.CloudProvider) == kops.CloudProviderAWS && !featureflag.Spotinst.Enabled() { + if kops.CloudProviderID(cluster.Spec.CloudProvider) == kops.CloudProviderAWS && !featureflag.Spotinst.Enabled() && group.Spec.InstanceManager != kops.InstanceManagerKarpenter { val = intstr.FromInt(1) } rollingUpdate.MaxSurge = &val From 75d2b97754db464aff0a9cedf95f9771e3d7ea99 Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Tue, 7 Dec 2021 20:59:12 +0100 Subject: [PATCH 06/13] Set provider name on karpenter-managed node --- pkg/nodeidentity/aws/identify.go | 14 +++++- pkg/nodelabels/builder.go | 4 ++ ...penter-nodes.minimal.example.com_user_data | 2 +- ...bject_nodeupconfig-karpenter-nodes_content | 1 + .../update_cluster/karpenter/kubernetes.tf | 45 ++++++++++--------- 5 files changed, 42 insertions(+), 24 deletions(-) diff --git a/pkg/nodeidentity/aws/identify.go b/pkg/nodeidentity/aws/identify.go index 83beaa0620..13bcd6cc2e 100644 --- a/pkg/nodeidentity/aws/identify.go +++ b/pkg/nodeidentity/aws/identify.go @@ -31,6 +31,7 @@ import ( corev1 "k8s.io/api/core/v1" expirationcache "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" + "k8s.io/kops/pkg/nodeidentity" ) @@ -41,7 +42,8 @@ const ( // ClusterAutoscalerNodeTemplateLabel is the prefix used on node labels when copying to cloud tags. ClusterAutoscalerNodeTemplateLabel = "k8s.io/cluster-autoscaler/node-template/label/" // The expiration time of nodeidentity.Info cache. - cacheTTL = 60 * time.Minute + cacheTTL = 60 * time.Minute + KarpenterNodeLabel = "karpenter.sh/" ) // nodeIdentifier identifies a node from EC2 @@ -145,10 +147,18 @@ func (i *nodeIdentifier) IdentifyNode(ctx context.Context, node *corev1.Node) (* Labels: labels, } + isKarpenterManaged := false for _, tag := range instance.Tags { - if strings.HasPrefix(aws.StringValue(tag.Key), ClusterAutoscalerNodeTemplateLabel) { + key := aws.StringValue(tag.Key) + if strings.HasPrefix(key, ClusterAutoscalerNodeTemplateLabel) { info.Labels[strings.TrimPrefix(aws.StringValue(tag.Key), ClusterAutoscalerNodeTemplateLabel)] = aws.StringValue(tag.Value) } + if strings.HasPrefix(key, KarpenterNodeLabel) { + isKarpenterManaged = true + } + } + if isKarpenterManaged { + info.Labels["karpenter.sh/provisioner-name"] = info.Labels[CloudTagInstanceGroupName] } // If caching is enabled add the nodeidentity.Info to cache. diff --git a/pkg/nodelabels/builder.go b/pkg/nodelabels/builder.go index a6ea671959..f01d56e4b5 100644 --- a/pkg/nodelabels/builder.go +++ b/pkg/nodelabels/builder.go @@ -92,6 +92,10 @@ func BuildNodeLabels(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) m nodeLabels[k] = v } + if instanceGroup.Spec.InstanceManager == kops.InstanceManagerKarpenter { + nodeLabels["karpenter.sh/provisioner-name"] = instanceGroup.ObjectMeta.Name + } + return nodeLabels } diff --git a/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes.minimal.example.com_user_data b/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes.minimal.example.com_user_data index d3ff3ce9b2..ae9d9bc182 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes.minimal.example.com_user_data +++ b/tests/integration/update_cluster/karpenter/data/aws_launch_template_karpenter-nodes.minimal.example.com_user_data @@ -160,7 +160,7 @@ CloudProvider: aws ConfigBase: memfs://clusters.example.com/minimal.example.com InstanceGroupName: karpenter-nodes InstanceGroupRole: Node -NodeupConfigHash: ehZK5PooPMXQw0YD3dy5oARwClEXIj8ymh6DR1XYbQ0= +NodeupConfigHash: 82WPA9hO1RdHX0zXhyshQojhW4qQug4izPAWZkgZkN4= __EOF_KUBE_ENV diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-karpenter-nodes_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-karpenter-nodes_content index 014862648a..7dcfc40cd3 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-karpenter-nodes_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_nodeupconfig-karpenter-nodes_content @@ -50,6 +50,7 @@ KubeletConfig: logLevel: 2 networkPluginName: cni nodeLabels: + karpenter.sh/provisioner-name: karpenter-nodes kubernetes.io/role: node node-role.kubernetes.io/node: "" nonMasqueradeCIDR: 100.64.0.0/10 diff --git a/tests/integration/update_cluster/karpenter/kubernetes.tf b/tests/integration/update_cluster/karpenter/kubernetes.tf index 6e38358a48..77a268e08c 100644 --- a/tests/integration/update_cluster/karpenter/kubernetes.tf +++ b/tests/integration/update_cluster/karpenter/kubernetes.tf @@ -414,35 +414,38 @@ resource "aws_launch_template" "karpenter-nodes-minimal-example-com" { tag_specifications { resource_type = "instance" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "karpenter-nodes.minimal.example.com" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "karpenter-nodes" - "kubernetes.io/cluster/minimal.example.com" = "owned" + "KubernetesCluster" = "minimal.example.com" + "Name" = "karpenter-nodes.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "karpenter-nodes" + "kubernetes.io/cluster/minimal.example.com" = "owned" } } tag_specifications { resource_type = "volume" tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "karpenter-nodes.minimal.example.com" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "karpenter-nodes" - "kubernetes.io/cluster/minimal.example.com" = "owned" + "KubernetesCluster" = "minimal.example.com" + "Name" = "karpenter-nodes.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "karpenter-nodes" + "kubernetes.io/cluster/minimal.example.com" = "owned" } } tags = { - "KubernetesCluster" = "minimal.example.com" - "Name" = "karpenter-nodes.minimal.example.com" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" - "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" - "k8s.io/role/node" = "1" - "kops.k8s.io/instancegroup" = "karpenter-nodes" - "kubernetes.io/cluster/minimal.example.com" = "owned" + "KubernetesCluster" = "minimal.example.com" + "Name" = "karpenter-nodes.minimal.example.com" + "k8s.io/cluster-autoscaler/node-template/label/karpenter.sh/provisioner-name" = "karpenter-nodes" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role" = "node" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "karpenter-nodes" + "kubernetes.io/cluster/minimal.example.com" = "owned" } user_data = filebase64("${path.module}/data/aws_launch_template_karpenter-nodes.minimal.example.com_user_data") } From 0cfea4925025efcb0c08d82dfcaa85c5f484e901 Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Sun, 12 Dec 2021 08:25:43 +0100 Subject: [PATCH 07/13] Do not expose the policy actions sets out of package --- .../addonmanifests/karpenter/iam.go | 2 +- pkg/model/iam/iam_builder.go | 90 ++++++++++--------- 2 files changed, 48 insertions(+), 44 deletions(-) diff --git a/pkg/model/components/addonmanifests/karpenter/iam.go b/pkg/model/components/addonmanifests/karpenter/iam.go index 60b72b58cb..c3b4b9e4cd 100644 --- a/pkg/model/components/addonmanifests/karpenter/iam.go +++ b/pkg/model/components/addonmanifests/karpenter/iam.go @@ -47,7 +47,7 @@ func (r *ServiceAccount) ServiceAccount() (types.NamespacedName, bool) { } func addKarpenterPermissions(p *iam.Policy) { - p.UnconditionalAction.Insert( + p.AddUnconditionalActions( // "ec2:CreateLaunchTemplate", "ec2:CreateFleet", "ec2:RunInstances", diff --git a/pkg/model/iam/iam_builder.go b/pkg/model/iam/iam_builder.go index f94affe119..f9862f8a63 100644 --- a/pkg/model/iam/iam_builder.go +++ b/pkg/model/iam/iam_builder.go @@ -50,26 +50,30 @@ const PolicyDefaultVersion = "2012-10-17" // Policy Struct is a collection of fields that form a valid AWS policy document type Policy struct { clusterName string - UnconditionalAction sets.String `json:"-"` - ClusterTaggedAction sets.String `json:"-"` - ClusterTaggedCreateAction sets.String `json:"-"` + unconditionalAction sets.String + clusterTaggedAction sets.String + clusterTaggedCreateAction sets.String Statement []*Statement Version string } +func (p *Policy) AddUnconditionalActions(actions ...string) { + p.unconditionalAction.Insert(actions...) +} + // AsJSON converts the policy document to JSON format (parsable by AWS) func (p *Policy) AsJSON() (string, error) { - if len(p.UnconditionalAction) > 0 { + if len(p.unconditionalAction) > 0 { p.Statement = append(p.Statement, &Statement{ Effect: StatementEffectAllow, - Action: stringorslice.Of(p.UnconditionalAction.List()...), + Action: stringorslice.Of(p.unconditionalAction.List()...), Resource: stringorslice.String("*"), }) } - if len(p.ClusterTaggedAction) > 0 { + if len(p.clusterTaggedAction) > 0 { p.Statement = append(p.Statement, &Statement{ Effect: StatementEffectAllow, - Action: stringorslice.Of(p.ClusterTaggedAction.List()...), + Action: stringorslice.Of(p.clusterTaggedAction.List()...), Resource: stringorslice.String("*"), Condition: Condition{ "StringEquals": map[string]string{ @@ -78,10 +82,10 @@ func (p *Policy) AsJSON() (string, error) { }, }) } - if len(p.ClusterTaggedCreateAction) > 0 { + if len(p.clusterTaggedCreateAction) > 0 { p.Statement = append(p.Statement, &Statement{ Effect: StatementEffectAllow, - Action: stringorslice.Of(p.ClusterTaggedCreateAction.List()...), + Action: stringorslice.Of(p.clusterTaggedCreateAction.List()...), Resource: stringorslice.String("*"), Condition: Condition{ "StringEquals": map[string]string{ @@ -277,9 +281,9 @@ func NewPolicy(clusterName string) *Policy { p := &Policy{ Version: PolicyDefaultVersion, clusterName: clusterName, - UnconditionalAction: sets.NewString(), - ClusterTaggedAction: sets.NewString(), - ClusterTaggedCreateAction: sets.NewString(), + unconditionalAction: sets.NewString(), + clusterTaggedAction: sets.NewString(), + clusterTaggedCreateAction: sets.NewString(), } return p } @@ -421,7 +425,7 @@ func (r *NodeRoleBastion) BuildAWSPolicy(b *PolicyBuilder) (*Policy, error) { // Bastion hosts currently don't require any specific permissions. // A trivial permission is granted, because empty policies are not allowed. - p.UnconditionalAction.Insert("ec2:DescribeRegions") + p.unconditionalAction.Insert("ec2:DescribeRegions") return p, nil } @@ -733,7 +737,7 @@ func addECRPermissions(p *Policy) { // We shouldn't be running lots of pods on the master, but it is perfectly reasonable to run // a private logging pod or similar. // At this point we allow all regions with ECR, since ECR is region specific. - p.UnconditionalAction.Insert( + p.unconditionalAction.Insert( "ecr:GetAuthorizationToken", "ecr:BatchCheckLayerAvailability", "ecr:GetDownloadUrlForLayer", @@ -745,7 +749,7 @@ func addECRPermissions(p *Policy) { } func addCalicoSrcDstCheckPermissions(p *Policy) { - p.UnconditionalAction.Insert( + p.unconditionalAction.Insert( "ec2:DescribeInstances", "ec2:ModifyNetworkInterfaceAttribute", ) @@ -755,26 +759,26 @@ func (b *PolicyBuilder) addNodeupPermissions(p *Policy, enableHookSupport bool) addCertIAMPolicies(p) addKMSGenerateRandomPolicies(p) addASLifecyclePolicies(p, enableHookSupport) - p.UnconditionalAction.Insert( + p.unconditionalAction.Insert( "ec2:DescribeInstances", // aws.go "ec2:DescribeInstanceTypes", ) if b.Cluster.Spec.IsKopsControllerIPAM() { - p.UnconditionalAction.Insert( + p.unconditionalAction.Insert( "ec2:AssignIpv6Addresses", ) } } func addKopsControllerIPAMPermissions(p *Policy) { - p.UnconditionalAction.Insert( + p.unconditionalAction.Insert( "ec2:DescribeNetworkInterfaces", ) } func addEtcdManagerPermissions(p *Policy) { - p.UnconditionalAction.Insert( + p.unconditionalAction.Insert( "ec2:DescribeVolumes", // aws.go ) @@ -796,7 +800,7 @@ func addEtcdManagerPermissions(p *Policy) { } func AddLegacyCCMPermissions(p *Policy) { - p.UnconditionalAction.Insert( + p.unconditionalAction.Insert( "ec2:CreateSecurityGroup", "ec2:CreateTags", "elasticloadbalancing:CreateTargetGroup", @@ -807,7 +811,7 @@ func AddLegacyCCMPermissions(p *Policy) { } func AddCCMPermissions(p *Policy, partition string, cloudRoutes bool) { - p.UnconditionalAction.Insert( + p.unconditionalAction.Insert( "autoscaling:DescribeAutoScalingGroups", "autoscaling:DescribeTags", "ec2:DescribeInstances", @@ -826,7 +830,7 @@ func AddCCMPermissions(p *Policy, partition string, cloudRoutes bool) { "kms:DescribeKey", ) - p.ClusterTaggedAction.Insert( + p.clusterTaggedAction.Insert( "ec2:ModifyInstanceAttribute", "ec2:ModifyVolume", "ec2:AttachVolume", @@ -859,7 +863,7 @@ func AddCCMPermissions(p *Policy, partition string, cloudRoutes bool) { "elasticloadbalancing:SetLoadBalancerPoliciesOfListener", ) - p.ClusterTaggedCreateAction.Insert( + p.clusterTaggedCreateAction.Insert( "elasticloadbalancing:CreateLoadBalancer", "ec2:CreateSecurityGroup", "ec2:CreateVolume", @@ -890,7 +894,7 @@ func AddCCMPermissions(p *Policy, partition string, cloudRoutes bool) { }, ) if cloudRoutes { - p.ClusterTaggedAction.Insert( + p.clusterTaggedAction.Insert( "ec2:CreateRoute", "ec2:DeleteRoute", ) @@ -899,7 +903,7 @@ func AddCCMPermissions(p *Policy, partition string, cloudRoutes bool) { // AddAWSLoadbalancerControllerPermissions adds the permissions needed for the aws load balancer controller to the givnen policy func AddAWSLoadbalancerControllerPermissions(p *Policy) { - p.UnconditionalAction.Insert( + p.unconditionalAction.Insert( "ec2:DescribeAvailabilityZones", "ec2:DescribeNetworkInterfaces", "elasticloadbalancing:DescribeTags", @@ -911,7 +915,7 @@ func AddAWSLoadbalancerControllerPermissions(p *Policy) { "acm:ListCertificates", "acm:DescribeCertificate", ) - p.ClusterTaggedAction.Insert( + p.clusterTaggedAction.Insert( "ec2:AuthorizeSecurityGroupIngress", // aws.go "ec2:DeleteSecurityGroup", // aws.go "ec2:RevokeSecurityGroupIngress", // aws.go @@ -926,11 +930,11 @@ func AddAWSLoadbalancerControllerPermissions(p *Policy) { } func AddClusterAutoscalerPermissions(p *Policy) { - p.ClusterTaggedAction.Insert( + p.clusterTaggedAction.Insert( "autoscaling:SetDesiredCapacity", "autoscaling:TerminateInstanceInAutoScalingGroup", ) - p.UnconditionalAction.Insert( + p.unconditionalAction.Insert( "autoscaling:DescribeAutoScalingGroups", "autoscaling:DescribeAutoScalingInstances", "autoscaling:DescribeLaunchConfigurations", @@ -944,21 +948,21 @@ func AddAWSEBSCSIDriverPermissions(p *Policy, partition string, appendSnapshotPe addSnapshotPersmissions(p) } - p.UnconditionalAction.Insert( + p.unconditionalAction.Insert( "ec2:DescribeAccountAttributes", // aws.go "ec2:DescribeInstances", // aws.go "ec2:DescribeVolumes", // aws.go "ec2:DescribeVolumesModifications", // aws.go "ec2:DescribeTags", // aws.go ) - p.ClusterTaggedAction.Insert( + p.clusterTaggedAction.Insert( "ec2:ModifyVolume", // aws.go "ec2:ModifyInstanceAttribute", // aws.go "ec2:AttachVolume", // aws.go "ec2:DeleteVolume", // aws.go "ec2:DetachVolume", // aws.go ) - p.ClusterTaggedCreateAction.Insert( + p.clusterTaggedCreateAction.Insert( "ec2:CreateVolume", // aws.go ) @@ -1005,12 +1009,12 @@ func AddAWSEBSCSIDriverPermissions(p *Policy, partition string, appendSnapshotPe } func addSnapshotPersmissions(p *Policy) { - p.UnconditionalAction.Insert( + p.unconditionalAction.Insert( "ec2:CreateSnapshot", "ec2:DescribeAvailabilityZones", "ec2:DescribeSnapshots", ) - p.ClusterTaggedAction.Insert( + p.clusterTaggedAction.Insert( "ec2:DeleteSnapshot", ) } @@ -1052,7 +1056,7 @@ func AddDNSControllerPermissions(b *PolicyBuilder, p *Policy) { func addKMSIAMPolicies(p *Policy, resource stringorslice.StringOrSlice) { // TODO could use "kms:ViaService" Condition Key here? - p.UnconditionalAction.Insert( + p.unconditionalAction.Insert( "kms:CreateGrant", "kms:Decrypt", "kms:DescribeKey", @@ -1064,35 +1068,35 @@ func addKMSIAMPolicies(p *Policy, resource stringorslice.StringOrSlice) { func addKMSGenerateRandomPolicies(p *Policy) { // For nodeup to seed the instance's random number generator. - p.UnconditionalAction.Insert( + p.unconditionalAction.Insert( "kms:GenerateRandom", ) } func addASLifecyclePolicies(p *Policy, enableHookSupport bool) { if enableHookSupport { - p.ClusterTaggedAction.Insert( + p.clusterTaggedAction.Insert( "autoscaling:CompleteLifecycleAction", // aws_manager.go ) - p.UnconditionalAction.Insert( + p.unconditionalAction.Insert( "autoscaling:DescribeLifecycleHooks", ) } - p.UnconditionalAction.Insert( + p.unconditionalAction.Insert( "autoscaling:DescribeAutoScalingInstances", ) } func addCertIAMPolicies(p *Policy) { // TODO: Make optional only if using IAM SSL Certs on ELBs - p.UnconditionalAction.Insert( + p.unconditionalAction.Insert( "iam:ListServerCertificates", "iam:GetServerCertificate", ) } func addCiliumEniPermissions(p *Policy) { - p.UnconditionalAction.Insert( + p.unconditionalAction.Insert( "ec2:DescribeSubnets", "ec2:AttachNetworkInterface", "ec2:AssignPrivateIpAddresses", @@ -1109,7 +1113,7 @@ func addCiliumEniPermissions(p *Policy) { } func addAmazonVPCCNIPermissions(p *Policy, partition string) { - p.UnconditionalAction.Insert( + p.unconditionalAction.Insert( "ec2:AssignPrivateIpAddresses", "ec2:AttachNetworkInterface", "ec2:CreateNetworkInterface", @@ -1136,7 +1140,7 @@ func addAmazonVPCCNIPermissions(p *Policy, partition string) { } func AddNodeTerminationHandlerSQSPermissions(p *Policy) { - p.UnconditionalAction.Insert( + p.unconditionalAction.Insert( "autoscaling:DescribeAutoScalingInstances", "autoscaling:DescribeTags", "ec2:DescribeInstances", @@ -1144,7 +1148,7 @@ func AddNodeTerminationHandlerSQSPermissions(p *Policy) { "sqs:DeleteMessage", "sqs:ReceiveMessage", ) - p.ClusterTaggedAction.Insert( + p.clusterTaggedAction.Insert( "autoscaling:CompleteLifecycleAction", ) } From b785965c50268668f843a2532038239203619d70 Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Sun, 12 Dec 2021 08:40:40 +0100 Subject: [PATCH 08/13] Rename InstanceManager to Manager --- k8s/crds/kops.k8s.io_instancegroups.yaml | 7 +++---- pkg/apis/kops/instancegroup.go | 4 ++-- pkg/apis/kops/v1alpha2/instancegroup.go | 4 ++-- pkg/apis/kops/v1alpha2/zz_generated.conversion.go | 4 ++-- pkg/apis/kops/v1alpha3/instancegroup.go | 4 ++-- pkg/apis/kops/v1alpha3/zz_generated.conversion.go | 4 ++-- pkg/instancegroups/settings.go | 2 +- pkg/model/awsmodel/autoscalinggroup.go | 2 +- pkg/model/awsmodel/network.go | 2 +- pkg/model/awsmodel/nodeterminationhandler.go | 2 +- pkg/nodelabels/builder.go | 2 +- .../create_cluster/complex/expected-v1alpha2.yaml | 4 ++-- .../gce_byo_sa/expected-v1alpha2.yaml | 4 ++-- .../create_cluster/ha/expected-v1alpha2.yaml | 12 ++++++------ .../ha_encrypt/expected-v1alpha2.yaml | 12 ++++++------ .../create_cluster/ha_gce/expected-v1alpha2.yaml | 12 ++++++------ .../ha_shared_zone/expected-v1alpha2.yaml | 8 ++++---- .../ha_shared_zones/expected-v1alpha2.yaml | 14 +++++++------- .../ingwspecified/expected-v1alpha2.yaml | 6 +++--- .../create_cluster/ipv6/expected-v1alpha2.yaml | 4 ++-- .../minimal-1.18/expected-v1alpha2.yaml | 4 ++-- .../minimal-1.19/expected-v1alpha2.yaml | 4 ++-- .../minimal-1.20/expected-v1alpha2.yaml | 4 ++-- .../minimal-1.21/expected-v1alpha2.yaml | 4 ++-- .../minimal-1.22/expected-v1alpha2.yaml | 4 ++-- .../minimal-1.23/expected-v1alpha2.yaml | 4 ++-- .../ngwspecified/expected-v1alpha2.yaml | 6 +++--- .../overrides/expected-v1alpha2.yaml | 4 ++-- .../create_cluster/private/expected-v1alpha2.yaml | 6 +++--- .../private_gce/expected-v1alpha2.yaml | 6 +++--- .../private_shared_subnets/expected-v1alpha2.yaml | 4 ++-- .../shared_subnets/expected-v1alpha2.yaml | 4 ++-- .../expected-v1alpha2.yaml | 4 ++-- .../shared_vpc/expected-v1alpha2.yaml | 4 ++-- .../update_cluster/karpenter/in-v1alpha2.yaml | 2 +- .../addons/karpenter.sh/k8s-1.19.yaml.template | 2 +- upup/pkg/fi/cloudup/awsup/aws_cloud.go | 2 +- upup/pkg/fi/cloudup/populate_instancegroup_spec.go | 4 ++-- 38 files changed, 92 insertions(+), 93 deletions(-) diff --git a/k8s/crds/kops.k8s.io_instancegroups.yaml b/k8s/crds/kops.k8s.io_instancegroups.yaml index 3fad5c5bdf..46166b3541 100644 --- a/k8s/crds/kops.k8s.io_instancegroups.yaml +++ b/k8s/crds/kops.k8s.io_instancegroups.yaml @@ -233,10 +233,6 @@ spec: description: InstanceInterruptionBehavior defines if a spot instance should be terminated, hibernated, or stopped after interruption type: string - instanceManager: - description: InstanceManager determines what is managing the node - lifecycle - type: string instanceMetadata: description: InstanceMetadata defines the EC2 instance metadata service options (AWS Only) @@ -665,6 +661,9 @@ spec: machineType: description: MachineType is the instance class type: string + manager: + description: Manager determines what is managing the node lifecycle + type: string maxPrice: description: MaxPrice indicates this is a spot-pricing group, with the specified value as our max-price bid diff --git a/pkg/apis/kops/instancegroup.go b/pkg/apis/kops/instancegroup.go index 55265f14b2..5332ea3f30 100644 --- a/pkg/apis/kops/instancegroup.go +++ b/pkg/apis/kops/instancegroup.go @@ -91,8 +91,8 @@ const ( // InstanceGroupSpec is the specification for an InstanceGroup type InstanceGroupSpec struct { - // InstanceManager determines what is managing the node lifecycle - InstanceManager InstanceManager `json:"instanceManager,omitempty"` + // Manager determines what is managing the node lifecycle + Manager InstanceManager `json:"manager,omitempty"` // Type determines the role of instances in this instance group: masters or nodes Role InstanceGroupRole `json:"role,omitempty"` // Image is the instance (ami etc) we should use diff --git a/pkg/apis/kops/v1alpha2/instancegroup.go b/pkg/apis/kops/v1alpha2/instancegroup.go index 7800d9845b..8fd13ad656 100644 --- a/pkg/apis/kops/v1alpha2/instancegroup.go +++ b/pkg/apis/kops/v1alpha2/instancegroup.go @@ -53,8 +53,8 @@ type InstanceManager string // InstanceGroupSpec is the specification for an InstanceGroup type InstanceGroupSpec struct { - // InstanceManager determines what is managing the node lifecycle - InstanceManager InstanceManager `json:"instanceManager,omitempty"` + // Manager determines what is managing the node lifecycle + Manager InstanceManager `json:"manager,omitempty"` // Type determines the role of instances in this instance group: masters or nodes Role InstanceGroupRole `json:"role,omitempty"` // Image is the instance (ami etc) we should use diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 7440864d05..af971a1f6f 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -4197,7 +4197,7 @@ func Convert_kops_InstanceGroupList_To_v1alpha2_InstanceGroupList(in *kops.Insta } func autoConvert_v1alpha2_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *InstanceGroupSpec, out *kops.InstanceGroupSpec, s conversion.Scope) error { - out.InstanceManager = kops.InstanceManager(in.InstanceManager) + out.Manager = kops.InstanceManager(in.Manager) out.Role = kops.InstanceGroupRole(in.Role) out.Image = in.Image out.MinSize = in.MinSize @@ -4360,7 +4360,7 @@ func Convert_v1alpha2_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *InstanceGr } func autoConvert_kops_InstanceGroupSpec_To_v1alpha2_InstanceGroupSpec(in *kops.InstanceGroupSpec, out *InstanceGroupSpec, s conversion.Scope) error { - out.InstanceManager = InstanceManager(in.InstanceManager) + out.Manager = InstanceManager(in.Manager) out.Role = InstanceGroupRole(in.Role) out.Image = in.Image out.MinSize = in.MinSize diff --git a/pkg/apis/kops/v1alpha3/instancegroup.go b/pkg/apis/kops/v1alpha3/instancegroup.go index e9a6cceaee..cf47cb73e4 100644 --- a/pkg/apis/kops/v1alpha3/instancegroup.go +++ b/pkg/apis/kops/v1alpha3/instancegroup.go @@ -53,8 +53,8 @@ type InstanceManager string // InstanceGroupSpec is the specification for an InstanceGroup type InstanceGroupSpec struct { - // InstanceManager determines what is managing the node lifecycle - InstanceManager InstanceManager `json:"instanceManager,omitempty"` + // Manager determines what is managing the node lifecycle + Manager InstanceManager `json:"manager,omitempty"` // Type determines the role of instances in this instance group: masters or nodes Role InstanceGroupRole `json:"role,omitempty"` // Image is the instance (ami etc) we should use diff --git a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go index 751bc7cf6f..46c3a52d30 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go @@ -4088,7 +4088,7 @@ func Convert_kops_InstanceGroupList_To_v1alpha3_InstanceGroupList(in *kops.Insta } func autoConvert_v1alpha3_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *InstanceGroupSpec, out *kops.InstanceGroupSpec, s conversion.Scope) error { - out.InstanceManager = kops.InstanceManager(in.InstanceManager) + out.Manager = kops.InstanceManager(in.Manager) out.Role = kops.InstanceGroupRole(in.Role) out.Image = in.Image out.MinSize = in.MinSize @@ -4250,7 +4250,7 @@ func Convert_v1alpha3_InstanceGroupSpec_To_kops_InstanceGroupSpec(in *InstanceGr } func autoConvert_kops_InstanceGroupSpec_To_v1alpha3_InstanceGroupSpec(in *kops.InstanceGroupSpec, out *InstanceGroupSpec, s conversion.Scope) error { - out.InstanceManager = InstanceManager(in.InstanceManager) + out.Manager = InstanceManager(in.Manager) out.Role = InstanceGroupRole(in.Role) out.Image = in.Image out.MinSize = in.MinSize diff --git a/pkg/instancegroups/settings.go b/pkg/instancegroups/settings.go index 0dd208a8e5..6d41b8863d 100644 --- a/pkg/instancegroups/settings.go +++ b/pkg/instancegroups/settings.go @@ -48,7 +48,7 @@ func resolveSettings(cluster *kops.Cluster, group *kops.InstanceGroup, numInstan if rollingUpdate.MaxSurge == nil { val := intstr.FromInt(0) - if kops.CloudProviderID(cluster.Spec.CloudProvider) == kops.CloudProviderAWS && !featureflag.Spotinst.Enabled() && group.Spec.InstanceManager != kops.InstanceManagerKarpenter { + if kops.CloudProviderID(cluster.Spec.CloudProvider) == kops.CloudProviderAWS && !featureflag.Spotinst.Enabled() && group.Spec.Manager != kops.InstanceManagerKarpenter { val = intstr.FromInt(1) } rollingUpdate.MaxSurge = &val diff --git a/pkg/model/awsmodel/autoscalinggroup.go b/pkg/model/awsmodel/autoscalinggroup.go index 3de993608e..6cd960e1d5 100644 --- a/pkg/model/awsmodel/autoscalinggroup.go +++ b/pkg/model/awsmodel/autoscalinggroup.go @@ -80,7 +80,7 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { c.AddTask(task) // @step: now lets build the autoscaling group task - if ig.Spec.InstanceManager != "Karpenter" { + if ig.Spec.Manager != "Karpenter" { tsk, err := b.buildAutoScalingGroupTask(c, name, ig) if err != nil { return err diff --git a/pkg/model/awsmodel/network.go b/pkg/model/awsmodel/network.go index 04eb11bedc..841d96b4fa 100644 --- a/pkg/model/awsmodel/network.go +++ b/pkg/model/awsmodel/network.go @@ -245,7 +245,7 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { klog.V(2).Infof("unable to properly tag subnet %q because it has unknown type %q. Load balancers may be created in incorrect subnets", subnetSpec.Name, subnetSpec.Type) } for _, ig := range b.InstanceGroups { - if ig.Spec.InstanceManager == kops.InstanceManagerKarpenter { + if ig.Spec.Manager == kops.InstanceManagerKarpenter { for _, igSubnet := range ig.Spec.Subnets { if igSubnet == subnetSpec.Name { tags["kops.k8s.io/karpenter/"+ig.Name] = "1" diff --git a/pkg/model/awsmodel/nodeterminationhandler.go b/pkg/model/awsmodel/nodeterminationhandler.go index 3287e2dc9e..b1c369a96b 100644 --- a/pkg/model/awsmodel/nodeterminationhandler.go +++ b/pkg/model/awsmodel/nodeterminationhandler.go @@ -81,7 +81,7 @@ type NodeTerminationHandlerBuilder struct { func (b *NodeTerminationHandlerBuilder) Build(c *fi.ModelBuilderContext) error { for _, ig := range b.InstanceGroups { - if ig.Spec.InstanceManager == kops.InstanceManagerCloudGroup { + if ig.Spec.Manager == kops.InstanceManagerCloudGroup { err := b.configureASG(c, ig) if err != nil { return err diff --git a/pkg/nodelabels/builder.go b/pkg/nodelabels/builder.go index f01d56e4b5..fd8c7b42e0 100644 --- a/pkg/nodelabels/builder.go +++ b/pkg/nodelabels/builder.go @@ -92,7 +92,7 @@ func BuildNodeLabels(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) m nodeLabels[k] = v } - if instanceGroup.Spec.InstanceManager == kops.InstanceManagerKarpenter { + if instanceGroup.Spec.Manager == kops.InstanceManagerKarpenter { nodeLabels["karpenter.sh/provisioner-name"] = instanceGroup.ObjectMeta.Name } diff --git a/tests/integration/create_cluster/complex/expected-v1alpha2.yaml b/tests/integration/create_cluster/complex/expected-v1alpha2.yaml index ecf6f14ac7..54d1cb0de8 100644 --- a/tests/integration/create_cluster/complex/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/complex/expected-v1alpha2.yaml @@ -64,8 +64,8 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -85,8 +85,8 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml b/tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml index bf0f00317f..2cd5d06244 100644 --- a/tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml @@ -65,8 +65,8 @@ metadata: name: master-us-test1-a spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 - instanceManager: CloudGroup machineType: n1-standard-1 + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -89,8 +89,8 @@ metadata: name: nodes-us-test1-a spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 - instanceManager: CloudGroup machineType: n1-standard-2 + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/ha/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha/expected-v1alpha2.yaml index 4142d2e51d..5e45d844a6 100644 --- a/tests/integration/create_cluster/ha/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha/expected-v1alpha2.yaml @@ -85,8 +85,8 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -106,8 +106,8 @@ metadata: name: master-us-test-1b spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -127,8 +127,8 @@ metadata: name: master-us-test-1c spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -148,8 +148,8 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -169,8 +169,8 @@ metadata: name: nodes-us-test-1b spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -190,8 +190,8 @@ metadata: name: nodes-us-test-1c spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/ha_encrypt/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha_encrypt/expected-v1alpha2.yaml index 4142d2e51d..5e45d844a6 100644 --- a/tests/integration/create_cluster/ha_encrypt/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha_encrypt/expected-v1alpha2.yaml @@ -85,8 +85,8 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -106,8 +106,8 @@ metadata: name: master-us-test-1b spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -127,8 +127,8 @@ metadata: name: master-us-test-1c spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -148,8 +148,8 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -169,8 +169,8 @@ metadata: name: nodes-us-test-1b spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -190,8 +190,8 @@ metadata: name: nodes-us-test-1c spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml index 22cc833ed4..a39b21149f 100644 --- a/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml @@ -73,8 +73,8 @@ metadata: name: master-us-test1-a spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 - instanceManager: CloudGroup machineType: n1-standard-1 + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -97,8 +97,8 @@ metadata: name: master-us-test1-b spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 - instanceManager: CloudGroup machineType: n1-standard-1 + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -121,8 +121,8 @@ metadata: name: master-us-test1-c spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 - instanceManager: CloudGroup machineType: n1-standard-1 + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -145,8 +145,8 @@ metadata: name: nodes-us-test1-a spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 - instanceManager: CloudGroup machineType: n1-standard-2 + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -169,8 +169,8 @@ metadata: name: nodes-us-test1-b spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 - instanceManager: CloudGroup machineType: n1-standard-2 + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -193,8 +193,8 @@ metadata: name: nodes-us-test1-c spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 - instanceManager: CloudGroup machineType: n1-standard-2 + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/ha_shared_zone/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha_shared_zone/expected-v1alpha2.yaml index d1f94b6bc0..4752fdafd2 100644 --- a/tests/integration/create_cluster/ha_shared_zone/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha_shared_zone/expected-v1alpha2.yaml @@ -77,8 +77,8 @@ metadata: name: master-us-test-1a-1 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -98,8 +98,8 @@ metadata: name: master-us-test-1a-2 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -119,8 +119,8 @@ metadata: name: master-us-test-1a-3 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -140,8 +140,8 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/ha_shared_zones/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha_shared_zones/expected-v1alpha2.yaml index 16da87cbc0..c23d4911de 100644 --- a/tests/integration/create_cluster/ha_shared_zones/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha_shared_zones/expected-v1alpha2.yaml @@ -93,8 +93,8 @@ metadata: name: master-us-test-1a-1 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -114,8 +114,8 @@ metadata: name: master-us-test-1a-2 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -135,8 +135,8 @@ metadata: name: master-us-test-1a-3 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -156,8 +156,8 @@ metadata: name: master-us-test-1b-1 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -177,8 +177,8 @@ metadata: name: master-us-test-1b-2 spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -198,8 +198,8 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -219,8 +219,8 @@ metadata: name: nodes-us-test-1b spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/ingwspecified/expected-v1alpha2.yaml b/tests/integration/create_cluster/ingwspecified/expected-v1alpha2.yaml index 8e7178a6a7..d79c1b0d96 100644 --- a/tests/integration/create_cluster/ingwspecified/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ingwspecified/expected-v1alpha2.yaml @@ -74,8 +74,8 @@ metadata: name: bastions spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.micro + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -95,8 +95,8 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -116,8 +116,8 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/ipv6/expected-v1alpha2.yaml b/tests/integration/create_cluster/ipv6/expected-v1alpha2.yaml index 5304026462..8a561a977f 100644 --- a/tests/integration/create_cluster/ipv6/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ipv6/expected-v1alpha2.yaml @@ -67,11 +67,11 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup instanceMetadata: httpPutResponseHopLimit: 3 httpTokens: required machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -91,11 +91,11 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup instanceMetadata: httpPutResponseHopLimit: 1 httpTokens: required machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/minimal-1.18/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.18/expected-v1alpha2.yaml index ec4a6fb0ee..97c21720cd 100644 --- a/tests/integration/create_cluster/minimal-1.18/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.18/expected-v1alpha2.yaml @@ -65,8 +65,8 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -86,8 +86,8 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/minimal-1.19/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.19/expected-v1alpha2.yaml index 4cec7f0f24..5962345e1b 100644 --- a/tests/integration/create_cluster/minimal-1.19/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.19/expected-v1alpha2.yaml @@ -65,8 +65,8 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -86,8 +86,8 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/minimal-1.20/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.20/expected-v1alpha2.yaml index 3960ae47c9..00fde6e97a 100644 --- a/tests/integration/create_cluster/minimal-1.20/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.20/expected-v1alpha2.yaml @@ -65,8 +65,8 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -86,8 +86,8 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/minimal-1.21/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.21/expected-v1alpha2.yaml index 3960ae47c9..00fde6e97a 100644 --- a/tests/integration/create_cluster/minimal-1.21/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.21/expected-v1alpha2.yaml @@ -65,8 +65,8 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -86,8 +86,8 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/minimal-1.22/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.22/expected-v1alpha2.yaml index 7dde515496..69487b5590 100644 --- a/tests/integration/create_cluster/minimal-1.22/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.22/expected-v1alpha2.yaml @@ -65,11 +65,11 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup instanceMetadata: httpPutResponseHopLimit: 3 httpTokens: required machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -89,11 +89,11 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup instanceMetadata: httpPutResponseHopLimit: 1 httpTokens: required machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/minimal-1.23/expected-v1alpha2.yaml b/tests/integration/create_cluster/minimal-1.23/expected-v1alpha2.yaml index 2462129ec2..cf5af49db9 100644 --- a/tests/integration/create_cluster/minimal-1.23/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/minimal-1.23/expected-v1alpha2.yaml @@ -65,11 +65,11 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup instanceMetadata: httpPutResponseHopLimit: 3 httpTokens: required machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -89,11 +89,11 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup instanceMetadata: httpPutResponseHopLimit: 1 httpTokens: required machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/ngwspecified/expected-v1alpha2.yaml b/tests/integration/create_cluster/ngwspecified/expected-v1alpha2.yaml index 0a798de54d..2a447c0594 100644 --- a/tests/integration/create_cluster/ngwspecified/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ngwspecified/expected-v1alpha2.yaml @@ -74,8 +74,8 @@ metadata: name: bastions spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.micro + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -95,8 +95,8 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -116,8 +116,8 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/overrides/expected-v1alpha2.yaml b/tests/integration/create_cluster/overrides/expected-v1alpha2.yaml index 7f435ad173..acfb38aa21 100644 --- a/tests/integration/create_cluster/overrides/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/overrides/expected-v1alpha2.yaml @@ -68,8 +68,8 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -89,8 +89,8 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/private/expected-v1alpha2.yaml b/tests/integration/create_cluster/private/expected-v1alpha2.yaml index 693f50ecdc..e8bf0ba296 100644 --- a/tests/integration/create_cluster/private/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/private/expected-v1alpha2.yaml @@ -77,8 +77,8 @@ metadata: name: bastions spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.micro + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -101,8 +101,8 @@ spec: - sg-exampleid3 - sg-exampleid4 image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -125,8 +125,8 @@ spec: - sg-exampleid - sg-exampleid2 image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/private_gce/expected-v1alpha2.yaml b/tests/integration/create_cluster/private_gce/expected-v1alpha2.yaml index 59aa216768..7c602e9f59 100644 --- a/tests/integration/create_cluster/private_gce/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/private_gce/expected-v1alpha2.yaml @@ -72,8 +72,8 @@ metadata: name: bastions spec: image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 - instanceManager: CloudGroup machineType: f1-micro + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -99,8 +99,8 @@ spec: - sg-exampleid3 - sg-exampleid4 image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 - instanceManager: CloudGroup machineType: n1-standard-1 + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -126,8 +126,8 @@ spec: - sg-exampleid - sg-exampleid2 image: ubuntu-os-cloud/ubuntu-2004-focal-v20211118 - instanceManager: CloudGroup machineType: n1-standard-2 + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/private_shared_subnets/expected-v1alpha2.yaml b/tests/integration/create_cluster/private_shared_subnets/expected-v1alpha2.yaml index 6e70791362..c53ea11789 100644 --- a/tests/integration/create_cluster/private_shared_subnets/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/private_shared_subnets/expected-v1alpha2.yaml @@ -74,8 +74,8 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -95,8 +95,8 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/shared_subnets/expected-v1alpha2.yaml b/tests/integration/create_cluster/shared_subnets/expected-v1alpha2.yaml index 36357e5a5f..ca717cbc04 100644 --- a/tests/integration/create_cluster/shared_subnets/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/shared_subnets/expected-v1alpha2.yaml @@ -67,8 +67,8 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -88,8 +88,8 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/shared_subnets_vpc_lookup/expected-v1alpha2.yaml b/tests/integration/create_cluster/shared_subnets_vpc_lookup/expected-v1alpha2.yaml index 36357e5a5f..ca717cbc04 100644 --- a/tests/integration/create_cluster/shared_subnets_vpc_lookup/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/shared_subnets_vpc_lookup/expected-v1alpha2.yaml @@ -67,8 +67,8 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -88,8 +88,8 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/create_cluster/shared_vpc/expected-v1alpha2.yaml b/tests/integration/create_cluster/shared_vpc/expected-v1alpha2.yaml index b00d62cefa..fed192e949 100644 --- a/tests/integration/create_cluster/shared_vpc/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/shared_vpc/expected-v1alpha2.yaml @@ -66,8 +66,8 @@ metadata: name: master-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: m3.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: @@ -87,8 +87,8 @@ metadata: name: nodes-us-test-1a spec: image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20211118 - instanceManager: CloudGroup machineType: t2.medium + manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: diff --git a/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml b/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml index 23d092af34..a21681df55 100644 --- a/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/karpenter/in-v1alpha2.yaml @@ -92,7 +92,7 @@ metadata: labels: kops.k8s.io/cluster: minimal.example.com spec: - instanceManager: Karpenter + manager: Karpenter associatePublicIp: true image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 machineType: t2.medium diff --git a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template index 26f2605958..d4e29b66ca 100644 --- a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template +++ b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template @@ -622,7 +622,7 @@ webhooks: app.kubernetes.io/part-of: karpenter {{ range $name, $spec := GetNodeInstanceGroups }} -{{ if eq $spec.InstanceManager "Karpenter" }} +{{ if eq $spec.Manager "Karpenter" }} --- apiVersion: karpenter.sh/v1alpha5 kind: Provisioner diff --git a/upup/pkg/fi/cloudup/awsup/aws_cloud.go b/upup/pkg/fi/cloudup/awsup/aws_cloud.go index 072d5d571e..baadb62e06 100644 --- a/upup/pkg/fi/cloudup/awsup/aws_cloud.go +++ b/upup/pkg/fi/cloudup/awsup/aws_cloud.go @@ -641,7 +641,7 @@ func (c *awsCloudImplementation) GetCloudGroups(cluster *kops.Cluster, instanceg func getKarpenterGroups(c AWSCloud, cluster *kops.Cluster, instancegroups []*kops.InstanceGroup, nodes []v1.Node) (map[string]*cloudinstances.CloudInstanceGroup, error) { cloudGroups := make(map[string]*cloudinstances.CloudInstanceGroup) for _, ig := range instancegroups { - if ig.Spec.InstanceManager == kops.InstanceManagerKarpenter { + if ig.Spec.Manager == kops.InstanceManagerKarpenter { group, err := buildKarpenterGroup(c, cluster, ig, nodes) if err != nil { return nil, err diff --git a/upup/pkg/fi/cloudup/populate_instancegroup_spec.go b/upup/pkg/fi/cloudup/populate_instancegroup_spec.go index d817ca7744..f0bcfb291a 100644 --- a/upup/pkg/fi/cloudup/populate_instancegroup_spec.go +++ b/upup/pkg/fi/cloudup/populate_instancegroup_spec.go @@ -175,8 +175,8 @@ func PopulateInstanceGroupSpec(cluster *kops.Cluster, input *kops.InstanceGroup, } } - if ig.Spec.InstanceManager == "" { - ig.Spec.InstanceManager = kops.InstanceManagerCloudGroup + if ig.Spec.Manager == "" { + ig.Spec.Manager = kops.InstanceManagerCloudGroup } return ig, nil } From d987d4ac1a811961b2845c7ef8eadb5c418c50b4 Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Mon, 13 Dec 2021 09:12:10 +0100 Subject: [PATCH 09/13] Rename SupportedArchitecture to architectureOfAMI --- .../resources/addons/karpenter.sh/k8s-1.19.yaml.template | 2 +- upup/pkg/fi/cloudup/template_functions.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template index d4e29b66ca..6dfb007f5a 100644 --- a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template +++ b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template @@ -635,7 +635,7 @@ spec: values: ["spot"] - key: kubernetes.io/arch operator: In - values: ["{{ SupportedArchitecture $spec.Image }}"] + values: ["{{ ArchitectureOfAMI $spec.Image }}"] - key: "node.kubernetes.io/instance-type" operator: In values: diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index add1d3d7f8..b26d0e3c8d 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -281,7 +281,7 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS dest["EnableSQSTerminationDraining"] = func() bool { return *cluster.Spec.NodeTerminationHandler.EnableSQSTerminationDraining } } - dest["SupportedArchitecture"] = tf.SupportedArchitecture + dest["ArchitectureOfAMI"] = tf.architectureOfAMI dest["ParseTaint"] = parseTaint @@ -741,7 +741,7 @@ func (tf *TemplateFunctions) GetNodeInstanceGroups() map[string]kops.InstanceGro return nodegroups } -func (tf *TemplateFunctions) SupportedArchitecture(amiID string) string { +func (tf *TemplateFunctions) architectureOfAMI(amiID string) string { image, _ := tf.cloud.(awsup.AWSCloud).ResolveImage(amiID) switch *image.Architecture { case "x86_64": From 0a6e329477516044ec59b53a63c1978b21dbb5a2 Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Mon, 13 Dec 2021 20:57:49 +0100 Subject: [PATCH 10/13] Use internal-elb as karpenter subnet selector --- pkg/model/awsmodel/network.go | 9 --------- ...t_object_minimal.example.com-addons-bootstrap_content | 2 +- ...imal.example.com-addons-karpenter.sh-k8s-1.19_content | 2 +- tests/integration/update_cluster/karpenter/kubernetes.tf | 1 - .../resources/addons/karpenter.sh/k8s-1.19.yaml.template | 2 +- 5 files changed, 3 insertions(+), 13 deletions(-) diff --git a/pkg/model/awsmodel/network.go b/pkg/model/awsmodel/network.go index 841d96b4fa..d75157e1c1 100644 --- a/pkg/model/awsmodel/network.go +++ b/pkg/model/awsmodel/network.go @@ -244,15 +244,6 @@ func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error { default: klog.V(2).Infof("unable to properly tag subnet %q because it has unknown type %q. Load balancers may be created in incorrect subnets", subnetSpec.Name, subnetSpec.Type) } - for _, ig := range b.InstanceGroups { - if ig.Spec.Manager == kops.InstanceManagerKarpenter { - for _, igSubnet := range ig.Spec.Subnets { - if igSubnet == subnetSpec.Name { - tags["kops.k8s.io/karpenter/"+ig.Name] = "1" - } - } - } - } } else { klog.V(2).Infof("skipping subnet tags. Ensure these are maintained externally.") } diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content index a3798692b0..416c53ce09 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content @@ -54,7 +54,7 @@ spec: version: 9.99.0 - id: k8s-1.19 manifest: karpenter.sh/k8s-1.19.yaml - manifestHash: f5bf5ee51335406c3ad93bfa6cea5778dfddba76f353da2b89bfdc30b925bd13 + manifestHash: 2caf3600648b924f1174a3a02b030c500bccda7a4eb7bfb9af0e1a99b48da72a name: karpenter.sh needsPKI: true selector: diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content index e45fa59d72..1213d717c4 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content @@ -868,7 +868,7 @@ spec: securityGroupSelector: Name: nodes.minimal.example.com subnetSelector: - kops.k8s.io/karpenter/karpenter-nodes: '*' + kubernetes.io/role/internal-elb: 1 requirements: - key: karpenter.sh/capacity-type operator: In diff --git a/tests/integration/update_cluster/karpenter/kubernetes.tf b/tests/integration/update_cluster/karpenter/kubernetes.tf index 77a268e08c..a9f1e5985b 100644 --- a/tests/integration/update_cluster/karpenter/kubernetes.tf +++ b/tests/integration/update_cluster/karpenter/kubernetes.tf @@ -959,7 +959,6 @@ resource "aws_subnet" "us-test-1a-minimal-example-com" { "KubernetesCluster" = "minimal.example.com" "Name" = "us-test-1a.minimal.example.com" "SubnetType" = "Public" - "kops.k8s.io/karpenter/karpenter-nodes" = "1" "kubernetes.io/cluster/minimal.example.com" = "owned" "kubernetes.io/role/elb" = "1" "kubernetes.io/role/internal-elb" = "1" diff --git a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template index 6dfb007f5a..5fdd3c153e 100644 --- a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template +++ b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template @@ -671,7 +671,7 @@ spec: securityGroupSelector: Name: nodes.{{ ClusterName }} subnetSelector: - kops.k8s.io/karpenter/{{ $name }}: "*" + kubernetes.io/role/internal-elb: 1 ttlSecondsAfterEmpty: 30 {{ end }} {{ end }} \ No newline at end of file From c89e318701f73fcd840381be5418b53bee08a4f4 Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Tue, 14 Dec 2021 10:34:48 +0100 Subject: [PATCH 11/13] Karpenter does not need PKI --- .../cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go | 1 - 1 file changed, 1 deletion(-) diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go index a550405227..1a9a1b9317 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go @@ -1070,7 +1070,6 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*Addon Name: fi.String(key), Manifest: fi.String(location), Selector: map[string]string{"k8s-addon": key}, - NeedsPKI: true, Id: id, }) if b.UseServiceAccountExternalPermissions() { From eef4652c34eb6697d310c847a45f4c3c55a365a6 Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Tue, 14 Dec 2021 10:39:49 +0100 Subject: [PATCH 12/13] Quote 1 in label selector --- ..._bucket_object_minimal.example.com-addons-bootstrap_content | 3 +-- ...ct_minimal.example.com-addons-karpenter.sh-k8s-1.19_content | 2 +- .../resources/addons/karpenter.sh/k8s-1.19.yaml.template | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content index 416c53ce09..dd61a918f3 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content @@ -54,9 +54,8 @@ spec: version: 9.99.0 - id: k8s-1.19 manifest: karpenter.sh/k8s-1.19.yaml - manifestHash: 2caf3600648b924f1174a3a02b030c500bccda7a4eb7bfb9af0e1a99b48da72a + manifestHash: 5dd23539e34c16de00719cc23c7369b38b159fa21945063c81b5e83f0af1eb6b name: karpenter.sh - needsPKI: true selector: k8s-addon: karpenter.sh version: 9.99.0 diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content index 1213d717c4..ee79756ade 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content @@ -868,7 +868,7 @@ spec: securityGroupSelector: Name: nodes.minimal.example.com subnetSelector: - kubernetes.io/role/internal-elb: 1 + kubernetes.io/role/internal-elb: "1" requirements: - key: karpenter.sh/capacity-type operator: In diff --git a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template index 5fdd3c153e..71baa09f9d 100644 --- a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template +++ b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template @@ -671,7 +671,7 @@ spec: securityGroupSelector: Name: nodes.{{ ClusterName }} subnetSelector: - kubernetes.io/role/internal-elb: 1 + kubernetes.io/role/internal-elb: "1" ttlSecondsAfterEmpty: 30 {{ end }} {{ end }} \ No newline at end of file From 9529073f3a3897e092ed885f6ff9cbfdf1db3627 Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Tue, 14 Dec 2021 10:54:00 +0100 Subject: [PATCH 13/13] Add KubernetesCluster as selector as well to avoid trying to schedule outside the cluster --- ...3_bucket_object_minimal.example.com-addons-bootstrap_content | 2 +- ...ect_minimal.example.com-addons-karpenter.sh-k8s-1.19_content | 1 + .../resources/addons/karpenter.sh/k8s-1.19.yaml.template | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content index dd61a918f3..49e3231033 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content @@ -54,7 +54,7 @@ spec: version: 9.99.0 - id: k8s-1.19 manifest: karpenter.sh/k8s-1.19.yaml - manifestHash: 5dd23539e34c16de00719cc23c7369b38b159fa21945063c81b5e83f0af1eb6b + manifestHash: c93d4c515254e2e8d9cc61a900a15f6ada60afed44fd7ba9ae0dd16cda9ca027 name: karpenter.sh selector: k8s-addon: karpenter.sh diff --git a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content index ee79756ade..89ffff1272 100644 --- a/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content +++ b/tests/integration/update_cluster/karpenter/data/aws_s3_bucket_object_minimal.example.com-addons-karpenter.sh-k8s-1.19_content @@ -868,6 +868,7 @@ spec: securityGroupSelector: Name: nodes.minimal.example.com subnetSelector: + KubernetesCluster: minimal.example.com kubernetes.io/role/internal-elb: "1" requirements: - key: karpenter.sh/capacity-type diff --git a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template index 71baa09f9d..ac5110d6c7 100644 --- a/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template +++ b/upup/models/cloudup/resources/addons/karpenter.sh/k8s-1.19.yaml.template @@ -672,6 +672,7 @@ spec: Name: nodes.{{ ClusterName }} subnetSelector: kubernetes.io/role/internal-elb: "1" + KubernetesCluster: {{ ClusterName }} ttlSecondsAfterEmpty: 30 {{ end }} {{ end }} \ No newline at end of file