diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index 62ff9519bb..9986d856fb 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -57,6 +57,11 @@ func TestMinimal(t *testing.T) { runTestAWS(t, "minimal.example.com", "minimal", "v1alpha2", false, 1, true, false, nil, true, false) } +// TestMinimalGCE runs tests on a minimal GCE configuration +func TestMinimalGCE(t *testing.T) { + runTestGCE(t, "minimal-gce.example.com", "minimal_gce", nil, "v1alpha2", false, 1, true) +} + // TestRestrictAccess runs the test on a simple SG configuration, similar to kops create cluster minimal.example.com --ssh-access=$(IPS) --admin-access=$(IPS) --master-count=3 func TestRestrictAccess(t *testing.T) { runTestAWS(t, "restrictaccess.example.com", "restrict_access", "v1alpha2", false, 1, true, false, nil, true, false) @@ -71,7 +76,7 @@ func TestHA(t *testing.T) { // TestHighAvailabilityGCE runs the test on a simple HA GCE configuration, similar to kops create cluster ha-gce.example.com // --zones us-test1-a,us-test1-b,us-test1-c --master-count=3 func TestHighAvailabilityGCE(t *testing.T) { - runTestGCE(t, "ha-gce.example.com", "ha_gce", "v1alpha2", false, 3, true) + runTestGCE(t, "ha-gce.example.com", "ha_gce", nil, "v1alpha2", false, 3, true) } // TestComplex runs the test on a more complex configuration, intended to hit more of the edge cases @@ -517,7 +522,7 @@ func runTestPhase(t *testing.T, clusterName string, srcDir string, version strin runTest(t, h, clusterName, srcDir, version, private, zones, expectedFilenames, tfFileName, "", &phase, nil, sshKey) } -func runTestGCE(t *testing.T, clusterName string, srcDir string, version string, private bool, zones int, sshKey bool) { +func runTestGCE(t *testing.T, clusterName string, srcDir string, lifecycleOverrides []string, version string, private bool, zones int, sshKey bool) { featureflag.ParseFlags("+AlphaAllowGCE") h := testutils.NewIntegrationTestHarness(t) @@ -543,7 +548,7 @@ func runTestGCE(t *testing.T, clusterName string, srcDir string, version string, expectedFilenames = append(expectedFilenames, prefix+"kops-k8s-io-instance-group-name") } - runTest(t, h, clusterName, srcDir, version, private, zones, expectedFilenames, "", "", nil, nil, sshKey) + runTest(t, h, clusterName, srcDir, version, private, zones, expectedFilenames, "", "", nil, lifecycleOverrides, sshKey) } func runTestCloudformation(t *testing.T, clusterName string, srcDir string, version string, private bool, lifecycleOverrides []string, sshKey bool) { diff --git a/tests/integration/minimal_gce/expected-v1alpha2.yaml b/tests/integration/minimal_gce/expected-v1alpha2.yaml new file mode 100644 index 0000000000..66c00c6a94 --- /dev/null +++ b/tests/integration/minimal_gce/expected-v1alpha2.yaml @@ -0,0 +1,97 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + name: minimal-gce.example.com +spec: + api: + dns: {} + authorization: + rbac: {} + channel: stable + cloudProvider: gce + configBase: memfs://tests/ha-gce.example.com + containerRuntime: docker + etcdClusters: + - cpuRequest: 200m + etcdMembers: + - instanceGroup: master-us-test1-a + name: a + memoryRequest: 100Mi + name: main + - cpuRequest: 100m + etcdMembers: + - instanceGroup: master-us-test1-a + name: a + memoryRequest: 100Mi + name: events + iam: + allowContainerRegistry: true + legacy: false + kubelet: + anonymousAuth: false + kubernetesApiAccess: + - 0.0.0.0/0 + kubernetesVersion: v1.16.0 + masterPublicName: api.minimal-gce.example.com + networking: + kubenet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + project: testproject + sshAccess: + - 0.0.0.0/0 + subnets: + - name: us-test1 + region: us-test1 + type: Public + topology: + dns: + type: Public + masters: public + nodes: public + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: minimal-gce.example.com + name: master-us-test1-a +spec: + image: cos-cloud/cos-stable-65-10323-99-0 + machineType: n1-standard-1 + maxSize: 1 + minSize: 1 + nodeLabels: + cloud.google.com/metadata-proxy-ready: "true" + kops.k8s.io/instancegroup: master-us-test1-a + role: Master + subnets: + - us-test1 + zones: + - us-test1-a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: minimal-gce.example.com + name: nodes +spec: + image: cos-cloud/cos-stable-65-10323-99-0 + machineType: n1-standard-2 + maxSize: 2 + minSize: 2 + nodeLabels: + cloud.google.com/metadata-proxy-ready: "true" + kops.k8s.io/instancegroup: nodes + role: Node + subnets: + - us-test1 + zones: + - us-test1-a diff --git a/tests/integration/minimal_gce/options.yaml b/tests/integration/minimal_gce/options.yaml new file mode 100644 index 0000000000..c3cf9d1a6b --- /dev/null +++ b/tests/integration/minimal_gce/options.yaml @@ -0,0 +1,5 @@ +ClusterName: minimal-gce.example.com +Zones: +- us-test1-a +Cloud: gce +KubernetesVersion: v1.16.0 diff --git a/tests/integration/update_cluster/minimal_gce/id_rsa.pub b/tests/integration/update_cluster/minimal_gce/id_rsa.pub new file mode 100755 index 0000000000..81cb012783 --- /dev/null +++ b/tests/integration/update_cluster/minimal_gce/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/minimal_gce/in-v1alpha2.yaml b/tests/integration/update_cluster/minimal_gce/in-v1alpha2.yaml new file mode 100644 index 0000000000..9cebb11884 --- /dev/null +++ b/tests/integration/update_cluster/minimal_gce/in-v1alpha2.yaml @@ -0,0 +1,83 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + name: minimal-gce.example.com +spec: + api: + dns: {} + authorization: + alwaysAllow: {} + channel: stable + cloudProvider: gce + configBase: memfs://tests/minimal-gce.example.com + etcdClusters: + - etcdMembers: + - instanceGroup: master-us-test1-a + name: "1" + name: main + - etcdMembers: + - instanceGroup: master-us-test1-a + name: "1" + name: events + iam: + legacy: false + kubernetesApiAccess: + - 0.0.0.0/0 + kubernetesVersion: v1.14.0 + masterPublicName: api.minimal-gce.example.com + networking: + kubenet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + project: testproject + sshAccess: + - 0.0.0.0/0 + subnets: + - name: us-test1 + region: us-test1 + type: Public + topology: + dns: + type: Public + masters: public + nodes: public + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: minimal-gce.example.com + name: master-us-test1-a +spec: + image: cos-cloud/cos-stable-57-9202-64-0 + machineType: n1-standard-1 + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test1 + zones: + - us-test1-a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: minimal-gce.example.com + name: nodes +spec: + image: cos-cloud/cos-stable-57-9202-64-0 + machineType: n1-standard-2 + maxSize: 2 + minSize: 2 + role: Node + subnets: + - us-test1 + zones: + - us-test1-a diff --git a/tests/integration/update_cluster/minimal_gce/kubernetes.tf b/tests/integration/update_cluster/minimal_gce/kubernetes.tf new file mode 100644 index 0000000000..b1da433d04 --- /dev/null +++ b/tests/integration/update_cluster/minimal_gce/kubernetes.tf @@ -0,0 +1,385 @@ +locals = { + cluster_name = "minimal-gce.example.com" + project = "us-test1" + region = "us-test1" +} + +output "cluster_name" { + value = "minimal-gce.example.com" +} + +output "project" { + value = "us-test1" +} + +output "region" { + value = "us-test1" +} + +provider "google" { + project = "us-test1" + region = "us-test1" + version = ">= 3.0.0" +} + +resource "google_compute_disk" "d1-etcd-events-minimal-gce-example-com" { + name = "d1-etcd-events-minimal-gce-example-com" + type = "pd-ssd" + size = 20 + zone = "us-test1-a" + + labels = { + k8s-io-cluster-name = "minimal-gce-example-com" + k8s-io-etcd-events = "1-2f1" + k8s-io-role-master = "master" + } +} + +resource "google_compute_disk" "d1-etcd-main-minimal-gce-example-com" { + name = "d1-etcd-main-minimal-gce-example-com" + type = "pd-ssd" + size = 20 + zone = "us-test1-a" + + labels = { + k8s-io-cluster-name = "minimal-gce-example-com" + k8s-io-etcd-main = "1-2f1" + k8s-io-role-master = "master" + } +} + +resource "google_compute_firewall" "cidr-to-master-minimal-gce-example-com" { + name = "cidr-to-master-minimal-gce-example-com" + network = "${google_compute_network.default.name}" + + allow = { + protocol = "tcp" + ports = ["443"] + } + + allow = { + protocol = "tcp" + ports = ["4194"] + } + + source_ranges = ["100.64.0.0/10"] + target_tags = ["minimal-gce-example-com-k8s-io-role-master"] +} + +resource "google_compute_firewall" "cidr-to-node-minimal-gce-example-com" { + name = "cidr-to-node-minimal-gce-example-com" + network = "${google_compute_network.default.name}" + + allow = { + protocol = "tcp" + } + + allow = { + protocol = "udp" + } + + allow = { + protocol = "icmp" + } + + allow = { + protocol = "esp" + } + + allow = { + protocol = "ah" + } + + allow = { + protocol = "sctp" + } + + source_ranges = ["100.64.0.0/10"] + target_tags = ["minimal-gce-example-com-k8s-io-role-node"] +} + +resource "google_compute_firewall" "kubernetes-master-https-minimal-gce-example-com" { + name = "kubernetes-master-https-minimal-gce-example-com" + network = "${google_compute_network.default.name}" + + allow = { + protocol = "tcp" + ports = ["443"] + } + + source_ranges = ["0.0.0.0/0"] + target_tags = ["minimal-gce-example-com-k8s-io-role-master"] +} + +resource "google_compute_firewall" "master-to-master-minimal-gce-example-com" { + name = "master-to-master-minimal-gce-example-com" + network = "${google_compute_network.default.name}" + + allow = { + protocol = "tcp" + } + + allow = { + protocol = "udp" + } + + allow = { + protocol = "icmp" + } + + allow = { + protocol = "esp" + } + + allow = { + protocol = "ah" + } + + allow = { + protocol = "sctp" + } + + source_tags = ["minimal-gce-example-com-k8s-io-role-master"] + target_tags = ["minimal-gce-example-com-k8s-io-role-master"] +} + +resource "google_compute_firewall" "master-to-node-minimal-gce-example-com" { + name = "master-to-node-minimal-gce-example-com" + network = "${google_compute_network.default.name}" + + allow = { + protocol = "tcp" + } + + allow = { + protocol = "udp" + } + + allow = { + protocol = "icmp" + } + + allow = { + protocol = "esp" + } + + allow = { + protocol = "ah" + } + + allow = { + protocol = "sctp" + } + + source_tags = ["minimal-gce-example-com-k8s-io-role-master"] + target_tags = ["minimal-gce-example-com-k8s-io-role-node"] +} + +resource "google_compute_firewall" "node-to-master-minimal-gce-example-com" { + name = "node-to-master-minimal-gce-example-com" + network = "${google_compute_network.default.name}" + + allow = { + protocol = "tcp" + ports = ["443"] + } + + allow = { + protocol = "tcp" + ports = ["4194"] + } + + source_tags = ["minimal-gce-example-com-k8s-io-role-node"] + target_tags = ["minimal-gce-example-com-k8s-io-role-master"] +} + +resource "google_compute_firewall" "node-to-node-minimal-gce-example-com" { + name = "node-to-node-minimal-gce-example-com" + network = "${google_compute_network.default.name}" + + allow = { + protocol = "tcp" + } + + allow = { + protocol = "udp" + } + + allow = { + protocol = "icmp" + } + + allow = { + protocol = "esp" + } + + allow = { + protocol = "ah" + } + + allow = { + protocol = "sctp" + } + + source_tags = ["minimal-gce-example-com-k8s-io-role-node"] + target_tags = ["minimal-gce-example-com-k8s-io-role-node"] +} + +resource "google_compute_firewall" "nodeport-external-to-node-minimal-gce-example-com" { + name = "nodeport-external-to-node-minimal-gce-example-com" + network = "${google_compute_network.default.name}" + + allow = { + protocol = "tcp" + ports = ["30000-32767"] + } + + allow = { + protocol = "udp" + ports = ["30000-32767"] + } + + source_tags = ["minimal-gce-example-com-k8s-io-role-node"] + target_tags = ["minimal-gce-example-com-k8s-io-role-node"] +} + +resource "google_compute_firewall" "ssh-external-to-master-minimal-gce-example-com" { + name = "ssh-external-to-master-minimal-gce-example-com" + network = "${google_compute_network.default.name}" + + allow = { + protocol = "tcp" + ports = ["22"] + } + + source_ranges = ["0.0.0.0/0"] + target_tags = ["minimal-gce-example-com-k8s-io-role-master"] +} + +resource "google_compute_firewall" "ssh-external-to-node-minimal-gce-example-com" { + name = "ssh-external-to-node-minimal-gce-example-com" + network = "${google_compute_network.default.name}" + + allow = { + protocol = "tcp" + ports = ["22"] + } + + source_ranges = ["0.0.0.0/0"] + target_tags = ["minimal-gce-example-com-k8s-io-role-node"] +} + +resource "google_compute_instance_group_manager" "a-master-us-test1-a-minimal-gce-example-com" { + name = "a-master-us-test1-a-minimal-gce-example-com" + zone = "us-test1-a" + base_instance_name = "master-us-test1-a" + + version = { + instance_template = "${google_compute_instance_template.master-us-test1-a-minimal-gce-example-com.self_link}" + } + + target_size = 1 +} + +resource "google_compute_instance_group_manager" "a-nodes-minimal-gce-example-com" { + name = "a-nodes-minimal-gce-example-com" + zone = "us-test1-a" + base_instance_name = "nodes" + + version = { + instance_template = "${google_compute_instance_template.nodes-minimal-gce-example-com.self_link}" + } + + target_size = 2 +} + +resource "google_compute_instance_template" "master-us-test1-a-minimal-gce-example-com" { + can_ip_forward = true + machine_type = "n1-standard-1" + + service_account = { + scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_write", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"] + } + + scheduling = { + automatic_restart = true + on_host_maintenance = "MIGRATE" + preemptible = false + } + + disk = { + auto_delete = true + device_name = "persistent-disks-0" + type = "PERSISTENT" + boot = true + source_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-57-9202-64-0" + mode = "READ_WRITE" + disk_type = "pd-standard" + disk_size_gb = 64 + } + + network_interface = { + network = "${google_compute_network.default.name}" + access_config = {} + } + + metadata = { + cluster-name = "${file("${path.module}/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_cluster-name")}" + kops-k8s-io-instance-group-name = "${file("${path.module}/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_kops-k8s-io-instance-group-name")}" + ssh-keys = "${file("${path.module}/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_ssh-keys")}" + startup-script = "${file("${path.module}/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_startup-script")}" + } + + tags = ["minimal-gce-example-com-k8s-io-role-master"] + name_prefix = "master-us-test1-a-minimal-do16cp-" +} + +resource "google_compute_instance_template" "nodes-minimal-gce-example-com" { + can_ip_forward = true + machine_type = "n1-standard-2" + + service_account = { + scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_only"] + } + + scheduling = { + automatic_restart = true + on_host_maintenance = "MIGRATE" + preemptible = false + } + + disk = { + auto_delete = true + device_name = "persistent-disks-0" + type = "PERSISTENT" + boot = true + source_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-57-9202-64-0" + mode = "READ_WRITE" + disk_type = "pd-standard" + disk_size_gb = 128 + } + + network_interface = { + network = "${google_compute_network.default.name}" + access_config = {} + } + + metadata = { + cluster-name = "${file("${path.module}/data/google_compute_instance_template_nodes-minimal-gce-example-com_metadata_cluster-name")}" + kops-k8s-io-instance-group-name = "${file("${path.module}/data/google_compute_instance_template_nodes-minimal-gce-example-com_metadata_kops-k8s-io-instance-group-name")}" + ssh-keys = "${file("${path.module}/data/google_compute_instance_template_nodes-minimal-gce-example-com_metadata_ssh-keys")}" + startup-script = "${file("${path.module}/data/google_compute_instance_template_nodes-minimal-gce-example-com_metadata_startup-script")}" + } + + tags = ["minimal-gce-example-com-k8s-io-role-node"] + name_prefix = "nodes-minimal-gce-example-com-" +} + +resource "google_compute_network" "default" { + name = "default" + auto_create_subnetworks = true +} + +terraform = { + required_version = ">= 0.9.3" +}