Merge pull request #8760 from geojaz/gce_tests_lifecycle

Gce tests lifecycle

cmd/kops/integration_test.go

@@ -57,6 +57,11 @@ func TestMinimal(t *testing.T) {
 	runTestAWS(t, "minimal.example.com", "minimal", "v1alpha2", false, 1, true, false, nil, true, false)
 }
 
+// TestMinimalGCE runs tests on a minimal GCE configuration
+func TestMinimalGCE(t *testing.T) {
+	runTestGCE(t, "minimal-gce.example.com", "minimal_gce", nil, "v1alpha2", false, 1, true)
+}
+
 // TestRestrictAccess runs the test on a simple SG configuration, similar to kops create cluster minimal.example.com --ssh-access=$(IPS) --admin-access=$(IPS) --master-count=3
 func TestRestrictAccess(t *testing.T) {
 	runTestAWS(t, "restrictaccess.example.com", "restrict_access", "v1alpha2", false, 1, true, false, nil, true, false)
@@ -71,7 +76,7 @@ func TestHA(t *testing.T) {
 // TestHighAvailabilityGCE runs the test on a simple HA GCE configuration, similar to kops create cluster ha-gce.example.com
 // --zones us-test1-a,us-test1-b,us-test1-c --master-count=3
 func TestHighAvailabilityGCE(t *testing.T) {
-	runTestGCE(t, "ha-gce.example.com", "ha_gce", "v1alpha2", false, 3, true)
+	runTestGCE(t, "ha-gce.example.com", "ha_gce", nil, "v1alpha2", false, 3, true)
 }
 
 // TestComplex runs the test on a more complex configuration, intended to hit more of the edge cases
@@ -517,7 +522,7 @@ func runTestPhase(t *testing.T, clusterName string, srcDir string, version strin
 	runTest(t, h, clusterName, srcDir, version, private, zones, expectedFilenames, tfFileName, "", &phase, nil, sshKey)
 }
 
-func runTestGCE(t *testing.T, clusterName string, srcDir string, version string, private bool, zones int, sshKey bool) {
+func runTestGCE(t *testing.T, clusterName string, srcDir string, lifecycleOverrides []string, version string, private bool, zones int, sshKey bool) {
 	featureflag.ParseFlags("+AlphaAllowGCE")
 
 	h := testutils.NewIntegrationTestHarness(t)
@@ -543,7 +548,7 @@ func runTestGCE(t *testing.T, clusterName string, srcDir string, version string,
 		expectedFilenames = append(expectedFilenames, prefix+"kops-k8s-io-instance-group-name")
 	}
 
-	runTest(t, h, clusterName, srcDir, version, private, zones, expectedFilenames, "", "", nil, nil, sshKey)
+	runTest(t, h, clusterName, srcDir, version, private, zones, expectedFilenames, "", "", nil, lifecycleOverrides, sshKey)
 }
 
 func runTestCloudformation(t *testing.T, clusterName string, srcDir string, version string, private bool, lifecycleOverrides []string, sshKey bool) {

tests/integration/minimal_gce/expected-v1alpha2.yaml

@@ -0,0 +1,97 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  name: minimal-gce.example.com
+spec:
+  api:
+    dns: {}
+  authorization:
+    rbac: {}
+  channel: stable
+  cloudProvider: gce
+  configBase: memfs://tests/ha-gce.example.com
+  containerRuntime: docker
+  etcdClusters:
+  - cpuRequest: 200m
+    etcdMembers:
+    - instanceGroup: master-us-test1-a
+      name: a
+    memoryRequest: 100Mi
+    name: main
+  - cpuRequest: 100m
+    etcdMembers:
+    - instanceGroup: master-us-test1-a
+      name: a
+    memoryRequest: 100Mi
+    name: events
+  iam:
+    allowContainerRegistry: true
+    legacy: false
+  kubelet:
+    anonymousAuth: false
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  kubernetesVersion: v1.16.0
+  masterPublicName: api.minimal-gce.example.com
+  networking:
+    kubenet: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  project: testproject
+  sshAccess:
+  - 0.0.0.0/0
+  subnets:
+  - name: us-test1
+    region: us-test1
+    type: Public
+  topology:
+    dns:
+      type: Public
+    masters: public
+    nodes: public
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: minimal-gce.example.com
+  name: master-us-test1-a
+spec:
+  image: cos-cloud/cos-stable-65-10323-99-0
+  machineType: n1-standard-1
+  maxSize: 1
+  minSize: 1
+  nodeLabels:
+    cloud.google.com/metadata-proxy-ready: "true"
+    kops.k8s.io/instancegroup: master-us-test1-a
+  role: Master
+  subnets:
+  - us-test1
+  zones:
+  - us-test1-a
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: minimal-gce.example.com
+  name: nodes
+spec:
+  image: cos-cloud/cos-stable-65-10323-99-0
+  machineType: n1-standard-2
+  maxSize: 2
+  minSize: 2
+  nodeLabels:
+    cloud.google.com/metadata-proxy-ready: "true"
+    kops.k8s.io/instancegroup: nodes
+  role: Node
+  subnets:
+  - us-test1
+  zones:
+  - us-test1-a

tests/integration/minimal_gce/options.yaml

@@ -0,0 +1,5 @@
+ClusterName: minimal-gce.example.com
+Zones:
+- us-test1-a
+Cloud: gce
+KubernetesVersion: v1.16.0

tests/integration/update_cluster/minimal_gce/id_rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==

tests/integration/update_cluster/minimal_gce/in-v1alpha2.yaml

@@ -0,0 +1,83 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  name: minimal-gce.example.com
+spec:
+  api:
+    dns: {}
+  authorization:
+    alwaysAllow: {}
+  channel: stable
+  cloudProvider: gce
+  configBase: memfs://tests/minimal-gce.example.com
+  etcdClusters:
+  - etcdMembers:
+    - instanceGroup: master-us-test1-a
+      name: "1"
+    name: main
+  - etcdMembers:
+    - instanceGroup: master-us-test1-a
+      name: "1"
+    name: events
+  iam:
+    legacy: false
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  kubernetesVersion: v1.14.0
+  masterPublicName: api.minimal-gce.example.com
+  networking:
+    kubenet: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  project: testproject
+  sshAccess:
+  - 0.0.0.0/0
+  subnets:
+  - name: us-test1
+    region: us-test1
+    type: Public
+  topology:
+    dns:
+      type: Public
+    masters: public
+    nodes: public
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: minimal-gce.example.com
+  name: master-us-test1-a
+spec:
+  image: cos-cloud/cos-stable-57-9202-64-0
+  machineType: n1-standard-1
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+  - us-test1
+  zones:
+  - us-test1-a
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: minimal-gce.example.com
+  name: nodes
+spec:
+  image: cos-cloud/cos-stable-57-9202-64-0
+  machineType: n1-standard-2
+  maxSize: 2
+  minSize: 2
+  role: Node
+  subnets:
+  - us-test1
+  zones:
+  - us-test1-a

tests/integration/update_cluster/minimal_gce/kubernetes.tf

@@ -0,0 +1,385 @@
+locals = {
+  cluster_name = "minimal-gce.example.com"
+  project      = "us-test1"
+  region       = "us-test1"
+}
+
+output "cluster_name" {
+  value = "minimal-gce.example.com"
+}
+
+output "project" {
+  value = "us-test1"
+}
+
+output "region" {
+  value = "us-test1"
+}
+
+provider "google" {
+  project = "us-test1"
+  region  = "us-test1"
+  version = ">= 3.0.0"
+}
+
+resource "google_compute_disk" "d1-etcd-events-minimal-gce-example-com" {
+  name = "d1-etcd-events-minimal-gce-example-com"
+  type = "pd-ssd"
+  size = 20
+  zone = "us-test1-a"
+
+  labels = {
+    k8s-io-cluster-name = "minimal-gce-example-com"
+    k8s-io-etcd-events  = "1-2f1"
+    k8s-io-role-master  = "master"
+  }
+}
+
+resource "google_compute_disk" "d1-etcd-main-minimal-gce-example-com" {
+  name = "d1-etcd-main-minimal-gce-example-com"
+  type = "pd-ssd"
+  size = 20
+  zone = "us-test1-a"
+
+  labels = {
+    k8s-io-cluster-name = "minimal-gce-example-com"
+    k8s-io-etcd-main    = "1-2f1"
+    k8s-io-role-master  = "master"
+  }
+}
+
+resource "google_compute_firewall" "cidr-to-master-minimal-gce-example-com" {
+  name    = "cidr-to-master-minimal-gce-example-com"
+  network = "${google_compute_network.default.name}"
+
+  allow = {
+    protocol = "tcp"
+    ports    = ["443"]
+  }
+
+  allow = {
+    protocol = "tcp"
+    ports    = ["4194"]
+  }
+
+  source_ranges = ["100.64.0.0/10"]
+  target_tags   = ["minimal-gce-example-com-k8s-io-role-master"]
+}
+
+resource "google_compute_firewall" "cidr-to-node-minimal-gce-example-com" {
+  name    = "cidr-to-node-minimal-gce-example-com"
+  network = "${google_compute_network.default.name}"
+
+  allow = {
+    protocol = "tcp"
+  }
+
+  allow = {
+    protocol = "udp"
+  }
+
+  allow = {
+    protocol = "icmp"
+  }
+
+  allow = {
+    protocol = "esp"
+  }
+
+  allow = {
+    protocol = "ah"
+  }
+
+  allow = {
+    protocol = "sctp"
+  }
+
+  source_ranges = ["100.64.0.0/10"]
+  target_tags   = ["minimal-gce-example-com-k8s-io-role-node"]
+}
+
+resource "google_compute_firewall" "kubernetes-master-https-minimal-gce-example-com" {
+  name    = "kubernetes-master-https-minimal-gce-example-com"
+  network = "${google_compute_network.default.name}"
+
+  allow = {
+    protocol = "tcp"
+    ports    = ["443"]
+  }
+
+  source_ranges = ["0.0.0.0/0"]
+  target_tags   = ["minimal-gce-example-com-k8s-io-role-master"]
+}
+
+resource "google_compute_firewall" "master-to-master-minimal-gce-example-com" {
+  name    = "master-to-master-minimal-gce-example-com"
+  network = "${google_compute_network.default.name}"
+
+  allow = {
+    protocol = "tcp"
+  }
+
+  allow = {
+    protocol = "udp"
+  }
+
+  allow = {
+    protocol = "icmp"
+  }
+
+  allow = {
+    protocol = "esp"
+  }
+
+  allow = {
+    protocol = "ah"
+  }
+
+  allow = {
+    protocol = "sctp"
+  }
+
+  source_tags = ["minimal-gce-example-com-k8s-io-role-master"]
+  target_tags = ["minimal-gce-example-com-k8s-io-role-master"]
+}
+
+resource "google_compute_firewall" "master-to-node-minimal-gce-example-com" {
+  name    = "master-to-node-minimal-gce-example-com"
+  network = "${google_compute_network.default.name}"
+
+  allow = {
+    protocol = "tcp"
+  }
+
+  allow = {
+    protocol = "udp"
+  }
+
+  allow = {
+    protocol = "icmp"
+  }
+
+  allow = {
+    protocol = "esp"
+  }
+
+  allow = {
+    protocol = "ah"
+  }
+
+  allow = {
+    protocol = "sctp"
+  }
+
+  source_tags = ["minimal-gce-example-com-k8s-io-role-master"]
+  target_tags = ["minimal-gce-example-com-k8s-io-role-node"]
+}
+
+resource "google_compute_firewall" "node-to-master-minimal-gce-example-com" {
+  name    = "node-to-master-minimal-gce-example-com"
+  network = "${google_compute_network.default.name}"
+
+  allow = {
+    protocol = "tcp"
+    ports    = ["443"]
+  }
+
+  allow = {
+    protocol = "tcp"
+    ports    = ["4194"]
+  }
+
+  source_tags = ["minimal-gce-example-com-k8s-io-role-node"]
+  target_tags = ["minimal-gce-example-com-k8s-io-role-master"]
+}
+
+resource "google_compute_firewall" "node-to-node-minimal-gce-example-com" {
+  name    = "node-to-node-minimal-gce-example-com"
+  network = "${google_compute_network.default.name}"
+
+  allow = {
+    protocol = "tcp"
+  }
+
+  allow = {
+    protocol = "udp"
+  }
+
+  allow = {
+    protocol = "icmp"
+  }
+
+  allow = {
+    protocol = "esp"
+  }
+
+  allow = {
+    protocol = "ah"
+  }
+
+  allow = {
+    protocol = "sctp"
+  }
+
+  source_tags = ["minimal-gce-example-com-k8s-io-role-node"]
+  target_tags = ["minimal-gce-example-com-k8s-io-role-node"]
+}
+
+resource "google_compute_firewall" "nodeport-external-to-node-minimal-gce-example-com" {
+  name    = "nodeport-external-to-node-minimal-gce-example-com"
+  network = "${google_compute_network.default.name}"
+
+  allow = {
+    protocol = "tcp"
+    ports    = ["30000-32767"]
+  }
+
+  allow = {
+    protocol = "udp"
+    ports    = ["30000-32767"]
+  }
+
+  source_tags = ["minimal-gce-example-com-k8s-io-role-node"]
+  target_tags = ["minimal-gce-example-com-k8s-io-role-node"]
+}
+
+resource "google_compute_firewall" "ssh-external-to-master-minimal-gce-example-com" {
+  name    = "ssh-external-to-master-minimal-gce-example-com"
+  network = "${google_compute_network.default.name}"
+
+  allow = {
+    protocol = "tcp"
+    ports    = ["22"]
+  }
+
+  source_ranges = ["0.0.0.0/0"]
+  target_tags   = ["minimal-gce-example-com-k8s-io-role-master"]
+}
+
+resource "google_compute_firewall" "ssh-external-to-node-minimal-gce-example-com" {
+  name    = "ssh-external-to-node-minimal-gce-example-com"
+  network = "${google_compute_network.default.name}"
+
+  allow = {
+    protocol = "tcp"
+    ports    = ["22"]
+  }
+
+  source_ranges = ["0.0.0.0/0"]
+  target_tags   = ["minimal-gce-example-com-k8s-io-role-node"]
+}
+
+resource "google_compute_instance_group_manager" "a-master-us-test1-a-minimal-gce-example-com" {
+  name               = "a-master-us-test1-a-minimal-gce-example-com"
+  zone               = "us-test1-a"
+  base_instance_name = "master-us-test1-a"
+
+  version = {
+    instance_template = "${google_compute_instance_template.master-us-test1-a-minimal-gce-example-com.self_link}"
+  }
+
+  target_size = 1
+}
+
+resource "google_compute_instance_group_manager" "a-nodes-minimal-gce-example-com" {
+  name               = "a-nodes-minimal-gce-example-com"
+  zone               = "us-test1-a"
+  base_instance_name = "nodes"
+
+  version = {
+    instance_template = "${google_compute_instance_template.nodes-minimal-gce-example-com.self_link}"
+  }
+
+  target_size = 2
+}
+
+resource "google_compute_instance_template" "master-us-test1-a-minimal-gce-example-com" {
+  can_ip_forward = true
+  machine_type   = "n1-standard-1"
+
+  service_account = {
+    scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_write", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"]
+  }
+
+  scheduling = {
+    automatic_restart   = true
+    on_host_maintenance = "MIGRATE"
+    preemptible         = false
+  }
+
+  disk = {
+    auto_delete  = true
+    device_name  = "persistent-disks-0"
+    type         = "PERSISTENT"
+    boot         = true
+    source_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-57-9202-64-0"
+    mode         = "READ_WRITE"
+    disk_type    = "pd-standard"
+    disk_size_gb = 64
+  }
+
+  network_interface = {
+    network       = "${google_compute_network.default.name}"
+    access_config = {}
+  }
+
+  metadata = {
+    cluster-name                    = "${file("${path.module}/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_cluster-name")}"
+    kops-k8s-io-instance-group-name = "${file("${path.module}/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_kops-k8s-io-instance-group-name")}"
+    ssh-keys                        = "${file("${path.module}/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_ssh-keys")}"
+    startup-script                  = "${file("${path.module}/data/google_compute_instance_template_master-us-test1-a-minimal-gce-example-com_metadata_startup-script")}"
+  }
+
+  tags        = ["minimal-gce-example-com-k8s-io-role-master"]
+  name_prefix = "master-us-test1-a-minimal-do16cp-"
+}
+
+resource "google_compute_instance_template" "nodes-minimal-gce-example-com" {
+  can_ip_forward = true
+  machine_type   = "n1-standard-2"
+
+  service_account = {
+    scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_only"]
+  }
+
+  scheduling = {
+    automatic_restart   = true
+    on_host_maintenance = "MIGRATE"
+    preemptible         = false
+  }
+
+  disk = {
+    auto_delete  = true
+    device_name  = "persistent-disks-0"
+    type         = "PERSISTENT"
+    boot         = true
+    source_image = "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/images/cos-stable-57-9202-64-0"
+    mode         = "READ_WRITE"
+    disk_type    = "pd-standard"
+    disk_size_gb = 128
+  }
+
+  network_interface = {
+    network       = "${google_compute_network.default.name}"
+    access_config = {}
+  }
+
+  metadata = {
+    cluster-name                    = "${file("${path.module}/data/google_compute_instance_template_nodes-minimal-gce-example-com_metadata_cluster-name")}"
+    kops-k8s-io-instance-group-name = "${file("${path.module}/data/google_compute_instance_template_nodes-minimal-gce-example-com_metadata_kops-k8s-io-instance-group-name")}"
+    ssh-keys                        = "${file("${path.module}/data/google_compute_instance_template_nodes-minimal-gce-example-com_metadata_ssh-keys")}"
+    startup-script                  = "${file("${path.module}/data/google_compute_instance_template_nodes-minimal-gce-example-com_metadata_startup-script")}"
+  }
+
+  tags        = ["minimal-gce-example-com-k8s-io-role-node"]
+  name_prefix = "nodes-minimal-gce-example-com-"
+}
+
+resource "google_compute_network" "default" {
+  name                    = "default"
+  auto_create_subnetworks = true
+}
+
+terraform = {
+  required_version = ">= 0.9.3"
+}