kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enabling JSON output for Terraform instead of writing the HCL syntax tf file. JSON syntax is officially supported in 0.12 and a terraform version requirement will be set. For previous installations you need to delete the .tf file by hand. JSON generation will fail if kubernetes.tf is present. 

Added Integration Test using minimal test setup

Added documentation. For terraform 0.12 support the resource names need to be changed still
This commit is contained in:
Christian van der Leeden 2019-12-15 12:38:23 +01:00
parent 85a667ad05
commit 549f54de48
8 changed files with 671 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
cmd/kops/integration_test.go
View File

@ -52,20 +52,20 @@ const updateClusterTestBase = "../../tests/integration/update_cluster/"
// TestMinimal runs the test on a minimum configuration, similar to kops create cluster minimal.example.com --zones us-west-1a
func TestMinimal(t *testing.T) {
runTestAWS(t, "minimal.example.com", "minimal", "v1alpha0", false, 1, true, false, nil, true)
runTestAWS(t, "minimal.example.com", "minimal", "v1alpha1", false, 1, true, false, nil, true)
runTestAWS(t, "minimal.example.com", "minimal", "v1alpha2", false, 1, true, false, nil, true)
runTestAWS(t, "minimal.example.com", "minimal", "v1alpha0", false, 1, true, false, nil, true, false)
runTestAWS(t, "minimal.example.com", "minimal", "v1alpha1", false, 1, true, false, nil, true, false)
runTestAWS(t, "minimal.example.com", "minimal", "v1alpha2", false, 1, true, false, nil, true, false)
}
// TestRestrictAccess runs the test on a simple SG configuration, similar to kops create cluster minimal.example.com --ssh-access=$(IPS) --admin-access=$(IPS) --master-count=3
func TestRestrictAccess(t *testing.T) {
runTestAWS(t, "restrictaccess.example.com", "restrict_access", "v1alpha2", false, 1, true, false, nil, true)
runTestAWS(t, "restrictaccess.example.com", "restrict_access", "v1alpha2", false, 1, true, false, nil, true, false)
}
// TestHA runs the test on a simple HA configuration, similar to kops create cluster minimal.example.com --zones us-west-1a,us-west-1b,us-west-1c --master-count=3
func TestHA(t *testing.T) {
runTestAWS(t, "ha.example.com", "ha", "v1alpha1", false, 3, true, false, nil, true)
runTestAWS(t, "ha.example.com", "ha", "v1alpha2", false, 3, true, false, nil, true)
runTestAWS(t, "ha.example.com", "ha", "v1alpha1", false, 3, true, false, nil, true, false)
runTestAWS(t, "ha.example.com", "ha", "v1alpha2", false, 3, true, false, nil, true, false)
}
// TestHighAvailabilityGCE runs the test on a simple HA GCE configuration, similar to kops create cluster ha-gce.example.com
@ -76,14 +76,14 @@ func TestHighAvailabilityGCE(t *testing.T) {
// TestComplex runs the test on a more complex configuration, intended to hit more of the edge cases
func TestComplex(t *testing.T) {
runTestAWS(t, "complex.example.com", "complex", "v1alpha2", false, 1, true, false, nil, true)
runTestAWS(t, "complex.example.com", "complex", "legacy-v1alpha2", false, 1, true, false, nil, true)
runTestAWS(t, "complex.example.com", "complex", "v1alpha2", false, 1, true, false, nil, true, false)
runTestAWS(t, "complex.example.com", "complex", "legacy-v1alpha2", false, 1, true, false, nil, true, false)
runTestCloudformation(t, "complex.example.com", "complex", "v1alpha2", false, nil, true)
}
func TestNoSSHKey(t *testing.T) {
runTestAWS(t, "nosshkey.example.com", "nosshkey", "v1alpha2", false, 1, true, false, nil, false)
runTestAWS(t, "nosshkey.example.com", "nosshkey", "v1alpha2", false, 1, true, false, nil, false, false)
}
func TestNoSSHKeyCloudformation(t *testing.T) {
@ -92,7 +92,7 @@ func TestNoSSHKeyCloudformation(t *testing.T) {
// TestCrossZone tests that the cross zone setting on the API ELB is set properly
func TestCrossZone(t *testing.T) {
runTestAWS(t, "crosszone.example.com", "api_elb_cross_zone", "v1alpha2", false, 1, true, false, nil, true)
runTestAWS(t, "crosszone.example.com", "api_elb_cross_zone", "v1alpha2", false, 1, true, false, nil, true, false)
}
// TestMinimalCloudformation runs the test on a minimum configuration, similar to kops create cluster minimal.example.com --zones us-west-1a
@ -108,7 +108,7 @@ func TestExistingIAMCloudformation(t *testing.T) {
// TestExistingSG runs the test with existing Security Group, similar to kops create cluster minimal.example.com --zones us-west-1a
func TestExistingSG(t *testing.T) {
runTestAWS(t, "existingsg.example.com", "existing_sg", "v1alpha2", false, 3, true, false, nil, true)
runTestAWS(t, "existingsg.example.com", "existing_sg", "v1alpha2", false, 3, true, false, nil, true, false)
}
// TestAdditionalUserData runs the test on passing additional user-data to an instance at bootstrap.
@ -118,83 +118,93 @@ func TestAdditionalUserData(t *testing.T) {
// TestBastionAdditionalUserData runs the test on passing additional user-data to a bastion instance group
func TestBastionAdditionalUserData(t *testing.T) {
runTestAWS(t, "bastionuserdata.example.com", "bastionadditional_user-data", "v1alpha2", true, 1, true, false, nil, true)
runTestAWS(t, "bastionuserdata.example.com", "bastionadditional_user-data", "v1alpha2", true, 1, true, false, nil, true, false)
}
// TestMinimal_JSON runs the test on a minimal data set and outputs JSON
func TestMinimal_json(t *testing.T) {
featureflag.ParseFlags("+TerraformJSON")
unsetFeaureFlag := func() {
featureflag.ParseFlags("-TerraformJSON")
}
defer unsetFeaureFlag()
runTestAWS(t, "minimal-json.example.com", "minimal-json", "v1alpha0", false, 1, true, false, nil, true, true)
}
// TestMinimal_141 runs the test on a configuration from 1.4.1 release
func TestMinimal_141(t *testing.T) {
runTestAWS(t, "minimal-141.example.com", "minimal-141", "v1alpha0", false, 1, true, false, nil, true)
runTestAWS(t, "minimal-141.example.com", "minimal-141", "v1alpha0", false, 1, true, false, nil, true, false)
}
// TestPrivateWeave runs the test on a configuration with private topology, weave networking
func TestPrivateWeave(t *testing.T) {
runTestAWS(t, "privateweave.example.com", "privateweave", "v1alpha1", true, 1, true, false, nil, true)
runTestAWS(t, "privateweave.example.com", "privateweave", "v1alpha2", true, 1, true, false, nil, true)
runTestAWS(t, "privateweave.example.com", "privateweave", "v1alpha1", true, 1, true, false, nil, true, false)
runTestAWS(t, "privateweave.example.com", "privateweave", "v1alpha2", true, 1, true, false, nil, true, false)
}
// TestPrivateFlannel runs the test on a configuration with private topology, flannel networking
func TestPrivateFlannel(t *testing.T) {
runTestAWS(t, "privateflannel.example.com", "privateflannel", "v1alpha1", true, 1, true, false, nil, true)
runTestAWS(t, "privateflannel.example.com", "privateflannel", "v1alpha2", true, 1, true, false, nil, true)
runTestAWS(t, "privateflannel.example.com", "privateflannel", "v1alpha1", true, 1, true, false, nil, true, false)
runTestAWS(t, "privateflannel.example.com", "privateflannel", "v1alpha2", true, 1, true, false, nil, true, false)
}
// TestPrivateCalico runs the test on a configuration with private topology, calico networking
func TestPrivateCalico(t *testing.T) {
runTestAWS(t, "privatecalico.example.com", "privatecalico", "v1alpha1", true, 1, true, false, nil, true)
runTestAWS(t, "privatecalico.example.com", "privatecalico", "v1alpha2", true, 1, true, false, nil, true)
runTestAWS(t, "privatecalico.example.com", "privatecalico", "v1alpha1", true, 1, true, false, nil, true, false)
runTestAWS(t, "privatecalico.example.com", "privatecalico", "v1alpha2", true, 1, true, false, nil, true, false)
runTestCloudformation(t, "privatecalico.example.com", "privatecalico", "v1alpha2", true, nil, true)
}
// TestPrivateCanal runs the test on a configuration with private topology, canal networking
func TestPrivateCanal(t *testing.T) {
runTestAWS(t, "privatecanal.example.com", "privatecanal", "v1alpha1", true, 1, true, false, nil, true)
runTestAWS(t, "privatecanal.example.com", "privatecanal", "v1alpha2", true, 1, true, false, nil, true)
runTestAWS(t, "privatecanal.example.com", "privatecanal", "v1alpha1", true, 1, true, false, nil, true, false)
runTestAWS(t, "privatecanal.example.com", "privatecanal", "v1alpha2", true, 1, true, false, nil, true, false)
}
// TestPrivateKopeio runs the test on a configuration with private topology, kopeio networking
func TestPrivateKopeio(t *testing.T) {
runTestAWS(t, "privatekopeio.example.com", "privatekopeio", "v1alpha2", true, 1, true, false, nil, true)
runTestAWS(t, "privatekopeio.example.com", "privatekopeio", "v1alpha2", true, 1, true, false, nil, true, false)
}
// TestUnmanaged is a test where all the subnets opt-out of route management
func TestUnmanaged(t *testing.T) {
runTestAWS(t, "unmanaged.example.com", "unmanaged", "v1alpha2", true, 1, true, false, nil, true)
runTestAWS(t, "unmanaged.example.com", "unmanaged", "v1alpha2", true, 1, true, false, nil, true, false)
}
// TestPrivateSharedSubnet runs the test on a configuration with private topology & shared subnets
func TestPrivateSharedSubnet(t *testing.T) {
runTestAWS(t, "private-shared-subnet.example.com", "private-shared-subnet", "v1alpha2", true, 1, true, false, nil, true)
runTestAWS(t, "private-shared-subnet.example.com", "private-shared-subnet", "v1alpha2", true, 1, true, false, nil, true, false)
}
// TestPrivateDns1 runs the test on a configuration with private topology, private dns
func TestPrivateDns1(t *testing.T) {
runTestAWS(t, "privatedns1.example.com", "privatedns1", "v1alpha2", true, 1, true, false, nil, true)
runTestAWS(t, "privatedns1.example.com", "privatedns1", "v1alpha2", true, 1, true, false, nil, true, false)
}
// TestPrivateDns2 runs the test on a configuration with private topology, private dns, extant vpc
func TestPrivateDns2(t *testing.T) {
runTestAWS(t, "privatedns2.example.com", "privatedns2", "v1alpha2", true, 1, true, false, nil, true)
runTestAWS(t, "privatedns2.example.com", "privatedns2", "v1alpha2", true, 1, true, false, nil, true, false)
}
// TestSharedSubnet runs the test on a configuration with a shared subnet (and VPC)
func TestSharedSubnet(t *testing.T) {
runTestAWS(t, "sharedsubnet.example.com", "shared_subnet", "v1alpha2", false, 1, true, false, nil, true)
runTestAWS(t, "sharedsubnet.example.com", "shared_subnet", "v1alpha2", false, 1, true, false, nil, true, false)
}
// TestSharedVPC runs the test on a configuration with a shared VPC
func TestSharedVPC(t *testing.T) {
runTestAWS(t, "sharedvpc.example.com", "shared_vpc", "v1alpha2", false, 1, true, false, nil, true)
runTestAWS(t, "sharedvpc.example.com", "shared_vpc", "v1alpha2", false, 1, true, false, nil, true, false)
}
// TestExistingIAM runs the test on a configuration with existing IAM instance profiles
func TestExistingIAM(t *testing.T) {
lifecycleOverrides := []string{"IAMRole=ExistsAndWarnIfChanges", "IAMRolePolicy=ExistsAndWarnIfChanges", "IAMInstanceProfileRole=ExistsAndWarnIfChanges"}
runTestAWS(t, "existing-iam.example.com", "existing_iam", "v1alpha2", false, 3, false, false, lifecycleOverrides, true)
runTestAWS(t, "existing-iam.example.com", "existing_iam", "v1alpha2", false, 3, false, false, lifecycleOverrides, true, false)
}
// TestAdditionalCIDR runs the test on a configuration with a shared VPC
func TestAdditionalCIDR(t *testing.T) {
runTestAWS(t, "additionalcidr.example.com", "additional_cidr", "v1alpha3", false, 3, true, false, nil, true)
runTestAWS(t, "additionalcidr.example.com", "additional_cidr", "v1alpha3", false, 3, true, false, nil, true, false)
runTestCloudformation(t, "additionalcidr.example.com", "additional_cidr", "v1alpha2", false, nil, true)
}
@ -204,7 +214,7 @@ func TestPhaseNetwork(t *testing.T) {
}
func TestExternalLoadBalancer(t *testing.T) {
runTestAWS(t, "externallb.example.com", "externallb", "v1alpha2", false, 1, true, false, nil, true)
runTestAWS(t, "externallb.example.com", "externallb", "v1alpha2", false, 1, true, false, nil, true, false)
runTestCloudformation(t, "externallb.example.com", "externallb", "v1alpha2", false, nil, true)
}
@ -223,13 +233,13 @@ func TestPhaseCluster(t *testing.T) {
// TestMixedInstancesASG tests ASGs using a mixed instance policy
func TestMixedInstancesASG(t *testing.T) {
runTestAWS(t, "mixedinstances.example.com", "mixed_instances", "v1alpha2", false, 3, true, true, nil, true)
runTestAWS(t, "mixedinstances.example.com", "mixed_instances", "v1alpha2", false, 3, true, true, nil, true, false)
runTestCloudformation(t, "mixedinstances.example.com", "mixed_instances", "v1alpha2", false, nil, true)
}
// TestMixedInstancesSpotASG tests ASGs using a mixed instance policy and spot instances
func TestMixedInstancesSpotASG(t *testing.T) {
runTestAWS(t, "mixedinstances.example.com", "mixed_instances_spot", "v1alpha2", false, 3, true, true, nil, true)
runTestAWS(t, "mixedinstances.example.com", "mixed_instances_spot", "v1alpha2", false, 3, true, true, nil, true, false)
runTestCloudformation(t, "mixedinstances.example.com", "mixed_instances_spot", "v1alpha2", false, nil, true)
}
@ -238,7 +248,7 @@ func TestContainerdCloudformation(t *testing.T) {
runTestCloudformation(t, "containerd.example.com", "containerd-cloudformation", "v1alpha2", false, nil, true)
}
func runTest(t *testing.T, h *testutils.IntegrationTestHarness, clusterName string, srcDir string, version string, private bool, zones int, expectedDataFilenames []string, tfFileName string, phase *cloudup.Phase, lifecycleOverrides []string, sshKey bool) {
func runTest(t *testing.T, h *testutils.IntegrationTestHarness, clusterName string, srcDir string, version string, private bool, zones int, expectedDataFilenames []string, tfFileName string, expectedTfFileName string, phase *cloudup.Phase, lifecycleOverrides []string, sshKey bool) {
var stdout bytes.Buffer
srcDir = updateClusterTestBase + srcDir
@ -250,6 +260,10 @@ func runTest(t *testing.T, h *testutils.IntegrationTestHarness, clusterName stri
testDataTFPath = tfFileName
}
if expectedTfFileName != "" {
actualTFPath = expectedTfFileName
}
factoryOptions := &util.FactoryOptions{}
factoryOptions.RegistryPath = "memfs://tests"
@ -312,10 +326,10 @@ func runTest(t *testing.T, h *testutils.IntegrationTestHarness, clusterName stri
sort.Strings(fileNames)
actualFilenames := strings.Join(fileNames, ",")
expectedFilenames := "kubernetes.tf"
expectedFilenames := actualTFPath
if len(expectedDataFilenames) > 0 {
expectedFilenames = "data,kubernetes.tf"
expectedFilenames = "data," + actualTFPath
}
if actualFilenames != expectedFilenames {
@ -392,10 +406,15 @@ func runTest(t *testing.T, h *testutils.IntegrationTestHarness, clusterName stri
}
}
func runTestAWS(t *testing.T, clusterName string, srcDir string, version string, private bool, zones int, expectPolicies bool, launchTemplate bool, lifecycleOverrides []string, sshKey bool) {
func runTestAWS(t *testing.T, clusterName string, srcDir string, version string, private bool, zones int, expectPolicies bool, launchTemplate bool, lifecycleOverrides []string, sshKey bool, jsonOutput bool) {
tfFileName := ""
h := testutils.NewIntegrationTestHarness(t)
defer h.Close()
if jsonOutput {
tfFileName = "kubernetes.tf.json"
}
h.MockKopsVersion("1.15.0")
h.SetupMockAWS()
@ -431,7 +450,7 @@ func runTestAWS(t *testing.T, clusterName string, srcDir string, version string,
}...)
}
}
runTest(t, h, clusterName, srcDir, version, private, zones, expectedFilenames, "", nil, lifecycleOverrides, sshKey)
runTest(t, h, clusterName, srcDir, version, private, zones, expectedFilenames, tfFileName, tfFileName, nil, lifecycleOverrides, sshKey)
}
func runTestPhase(t *testing.T, clusterName string, srcDir string, version string, private bool, zones int, phase cloudup.Phase, sshKey bool) {
@ -475,7 +494,7 @@ func runTestPhase(t *testing.T, clusterName string, srcDir string, version strin
}
}
runTest(t, h, clusterName, srcDir, version, private, zones, expectedFilenames, tfFileName, &phase, nil, sshKey)
runTest(t, h, clusterName, srcDir, version, private, zones, expectedFilenames, tfFileName, "", &phase, nil, sshKey)
}
func runTestGCE(t *testing.T, clusterName string, srcDir string, version string, private bool, zones int, sshKey bool) {
@ -504,7 +523,7 @@ func runTestGCE(t *testing.T, clusterName string, srcDir string, version string,
expectedFilenames = append(expectedFilenames, prefix+"kops-k8s-io-instance-group-name")
}
runTest(t, h, clusterName, srcDir, version, private, zones, expectedFilenames, "", nil, nil, sshKey)
runTest(t, h, clusterName, srcDir, version, private, zones, expectedFilenames, "", "", nil, nil, sshKey)
}
func runTestCloudformation(t *testing.T, clusterName string, srcDir string, version string, private bool, lifecycleOverrides []string, sshKey bool) {

1
docs/advanced/experimental.md
View File

@ -21,3 +21,4 @@ The following experimental features are currently available:
* `+Spotinst` - Enables the use of the Spotinst cloud provider
* `+SpotinstOcean` - Enables the use of Spotinst Ocean instance groups
* `+SkipEtcdVersionCheck` - Bypasses the check that etcd-manager is using a supported etcd version
* `+TerraformJSON` - Produce kubernetes.ts.json file instead of writing HCL v1 syntax. Can be consumed by terraform 0.12

13
docs/terraform.md
View File

@ -159,3 +159,16 @@ $ terraform apply
```
You should still run `kops delete cluster ${CLUSTER_NAME}`, to remove the kops cluster specification and any dynamically created Kubernetes resources (ELBs or volumes), but under this workaround also to remove the primary ELB volumes from the `proto` phase.
#### Terraform JSON output
With terraform 0.12 JSON is now officially supported as configuration language. To enable JSON output instead of HCLv1 output you need to enable it through a feature flag.
```
export KOPS_FEATURE_FLAGS=TerraformJSON
kops update cluster .....
```
This is an alternative to of using terraforms own configuration syntax HCL. Be sure to delete the existing kubernetes.tf file. Terraform will otherwise use both and then complain.
Kops will require terraform 0.12 for JSON configuration. Inofficially (partially) it was also supported with terraform 0.11, so you can try and remove the `required_version` in `kubernetes.tf.json`.

2
pkg/featureflag/featureflag.go
View File

@ -82,6 +82,8 @@ var (
VSphereCloudProvider = New("VSphereCloudProvider", Bool(false))
// SkipEtcdVersionCheck will bypass the check that etcd-manager is using a supported etcd version
SkipEtcdVersionCheck = New("SkipEtcdVersionCheck", Bool(false))
// Enable terraform JSON output instead of hcl output. JSON output can be also parsed by terraform 0.12
TerraformJSON = New("TerraformJSON", Bool(false))
)
// FeatureFlag defines a feature flag

1
tests/integration/update_cluster/minimal-json/id_rsa.pub Executable file
View File

@ -0,0 +1 @@
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==

76
tests/integration/update_cluster/minimal-json/in-v1alpha0.yaml Normal file
View File

@ -0,0 +1,76 @@
apiVersion: kops.k8s.io/v1alpha1
kind: Cluster
metadata:
creationTimestamp: "2016-12-10T22:42:27Z"
name: minimal-json.example.com
spec:
adminAccess:
- 0.0.0.0/0
channel: stable
cloudProvider: aws
configBase: memfs://clusters.example.com/minimal-json.example.com
etcdClusters:
- etcdMembers:
- name: us-test-1a
zone: us-test-1a
name: main
- etcdMembers:
- name: us-test-1a
zone: us-test-1a
name: events
kubernetesVersion: v1.14.0
masterInternalName: api.internal.minimal-json.example.com
masterPublicName: api.minimal-json.example.com
networkCIDR: 172.20.0.0/16
networking:
kubenet: {}
nonMasqueradeCIDR: 100.64.0.0/10
topology:
bastion:
idleTimeout: 120
machineType: t2.medium
masters: public
nodes: public
zones:
- cidr: 172.20.32.0/19
name: us-test-1a
---
apiVersion: kops.k8s.io/v1alpha1
kind: InstanceGroup
metadata:
creationTimestamp: "2016-12-10T22:42:28Z"
name: nodes
labels:
kops.k8s.io/cluster: minimal-json.example.com
spec:
associatePublicIp: true
image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21
machineType: t2.medium
maxSize: 2
minSize: 2
role: Node
zones:
- us-test-1a
---
apiVersion: kops.k8s.io/v1alpha1
kind: InstanceGroup
metadata:
creationTimestamp: "2016-12-10T22:42:28Z"
name: master-us-test-1a
labels:
kops.k8s.io/cluster: minimal-json.example.com
spec:
associatePublicIp: true
image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21
machineType: m3.medium
maxSize: 1
minSize: 1
role: Master
zones:
- us-test-1a

507
tests/integration/update_cluster/minimal-json/kubernetes.tf.json Normal file
View File

@ -0,0 +1,507 @@
{
"locals": {
"cluster_name": "minimal-json.example.com",
"master_autoscaling_group_ids": [
"${aws_autoscaling_group.master-us-test-1a-masters-minimal-json-example-com.id}"
],
"master_security_group_ids": [
"${aws_security_group.masters-minimal-json-example-com.id}"
],
"masters_role_arn": "${aws_iam_role.masters-minimal-json-example-com.arn}",
"masters_role_name": "${aws_iam_role.masters-minimal-json-example-com.name}",
"node_autoscaling_group_ids": [
"${aws_autoscaling_group.nodes-minimal-json-example-com.id}"
],
"node_security_group_ids": [
"${aws_security_group.nodes-minimal-json-example-com.id}"
],
"node_subnet_ids": [
"${aws_subnet.us-test-1a-minimal-json-example-com.id}"
],
"nodes_role_arn": "${aws_iam_role.nodes-minimal-json-example-com.arn}",
"nodes_role_name": "${aws_iam_role.nodes-minimal-json-example-com.name}",
"region": "us-test-1",
"route_table_public_id": "${aws_route_table.minimal-json-example-com.id}",
"subnet_us-test-1a_id": "${aws_subnet.us-test-1a-minimal-json-example-com.id}",
"vpc_cidr_block": "${aws_vpc.minimal-json-example-com.cidr_block}",
"vpc_id": "${aws_vpc.minimal-json-example-com.id}"
},
"output": {
"cluster_name": {
"value": "minimal-json.example.com"
},
"master_autoscaling_group_ids": {
"value": [
"${aws_autoscaling_group.master-us-test-1a-masters-minimal-json-example-com.id}"
]
},
"master_security_group_ids": {
"value": [
"${aws_security_group.masters-minimal-json-example-com.id}"
]
},
"masters_role_arn": {
"value": "${aws_iam_role.masters-minimal-json-example-com.arn}"
},
"masters_role_name": {
"value": "${aws_iam_role.masters-minimal-json-example-com.name}"
},
"node_autoscaling_group_ids": {
"value": [
"${aws_autoscaling_group.nodes-minimal-json-example-com.id}"
]
},
"node_security_group_ids": {
"value": [
"${aws_security_group.nodes-minimal-json-example-com.id}"
]
},
"node_subnet_ids": {
"value": [
"${aws_subnet.us-test-1a-minimal-json-example-com.id}"
]
},
"nodes_role_arn": {
"value": "${aws_iam_role.nodes-minimal-json-example-com.arn}"
},
"nodes_role_name": {
"value": "${aws_iam_role.nodes-minimal-json-example-com.name}"
},
"region": {
"value": "us-test-1"
},
"route_table_public_id": {
"value": "${aws_route_table.minimal-json-example-com.id}"
},
"subnet_us-test-1a_id": {
"value": "${aws_subnet.us-test-1a-minimal-json-example-com.id}"
},
"vpc_cidr_block": {
"value": "${aws_vpc.minimal-json-example-com.cidr_block}"
},
"vpc_id": {
"value": "${aws_vpc.minimal-json-example-com.id}"
}
},
"provider": {
"aws": {
"region": "us-test-1"
}
},
"resource": {
"aws_autoscaling_group": {
"master-us-test-1a-masters-minimal-json-example-com": {
"name": "master-us-test-1a.masters.minimal-json.example.com",
"launch_configuration": "${aws_launch_configuration.master-us-test-1a-masters-minimal-json-example-com.id}",
"max_size": 1,
"min_size": 1,
"vpc_zone_identifier": [
"${aws_subnet.us-test-1a-minimal-json-example-com.id}"
],
"tag": [
{
"key": "KubernetesCluster",
"value": "minimal-json.example.com",
"propagate_at_launch": true
},
{
"key": "Name",
"value": "master-us-test-1a.masters.minimal-json.example.com",
"propagate_at_launch": true
},
{
"key": "k8s.io/role/master",
"value": "1",
"propagate_at_launch": true
},
{
"key": "kops.k8s.io/instancegroup",
"value": "master-us-test-1a",
"propagate_at_launch": true
}
],
"metrics_granularity": "1Minute",
"enabled_metrics": [
"GroupDesiredCapacity",
"GroupInServiceInstances",
"GroupMaxSize",
"GroupMinSize",
"GroupPendingInstances",
"GroupStandbyInstances",
"GroupTerminatingInstances",
"GroupTotalInstances"
]
},
"nodes-minimal-json-example-com": {
"name": "nodes.minimal-json.example.com",
"launch_configuration": "${aws_launch_configuration.nodes-minimal-json-example-com.id}",
"max_size": 2,
"min_size": 2,
"vpc_zone_identifier": [
"${aws_subnet.us-test-1a-minimal-json-example-com.id}"
],
"tag": [
{
"key": "KubernetesCluster",
"value": "minimal-json.example.com",
"propagate_at_launch": true
},
{
"key": "Name",
"value": "nodes.minimal-json.example.com",
"propagate_at_launch": true
},
{
"key": "k8s.io/role/node",
"value": "1",
"propagate_at_launch": true
},
{
"key": "kops.k8s.io/instancegroup",
"value": "nodes",
"propagate_at_launch": true
}
],
"metrics_granularity": "1Minute",
"enabled_metrics": [
"GroupDesiredCapacity",
"GroupInServiceInstances",
"GroupMaxSize",
"GroupMinSize",
"GroupPendingInstances",
"GroupStandbyInstances",
"GroupTerminatingInstances",
"GroupTotalInstances"
]
}
},
"aws_ebs_volume": {
"us-test-1a-etcd-events-minimal-json-example-com": {
"availability_zone": "us-test-1a",
"size": 20,
"type": "gp2",
"encrypted": false,
"tags": {
"KubernetesCluster": "minimal-json.example.com",
"Name": "us-test-1a.etcd-events.minimal-json.example.com",
"k8s.io/etcd/events": "us-test-1a/us-test-1a",
"k8s.io/role/master": "1",
"kubernetes.io/cluster/minimal-json.example.com": "owned"
}
},
"us-test-1a-etcd-main-minimal-json-example-com": {
"availability_zone": "us-test-1a",
"size": 20,
"type": "gp2",
"encrypted": false,
"tags": {
"KubernetesCluster": "minimal-json.example.com",
"Name": "us-test-1a.etcd-main.minimal-json.example.com",
"k8s.io/etcd/main": "us-test-1a/us-test-1a",
"k8s.io/role/master": "1",
"kubernetes.io/cluster/minimal-json.example.com": "owned"
}
}
},
"aws_iam_instance_profile": {
"masters-minimal-json-example-com": {
"name": "masters.minimal-json.example.com",
"role": "${aws_iam_role.masters-minimal-json-example-com.name}"
},
"nodes-minimal-json-example-com": {
"name": "nodes.minimal-json.example.com",
"role": "${aws_iam_role.nodes-minimal-json-example-com.name}"
}
},
"aws_iam_role": {
"masters-minimal-json-example-com": {
"name": "masters.minimal-json.example.com",
"assume_role_policy": "${file(\"${path.module}/data/aws_iam_role_masters.minimal-json.example.com_policy\")}"
},
"nodes-minimal-json-example-com": {
"name": "nodes.minimal-json.example.com",
"assume_role_policy": "${file(\"${path.module}/data/aws_iam_role_nodes.minimal-json.example.com_policy\")}"
}
},
"aws_iam_role_policy": {
"masters-minimal-json-example-com": {
"name": "masters.minimal-json.example.com",
"role": "${aws_iam_role.masters-minimal-json-example-com.name}",
"policy": "${file(\"${path.module}/data/aws_iam_role_policy_masters.minimal-json.example.com_policy\")}"
},
"nodes-minimal-json-example-com": {
"name": "nodes.minimal-json.example.com",
"role": "${aws_iam_role.nodes-minimal-json-example-com.name}",
"policy": "${file(\"${path.module}/data/aws_iam_role_policy_nodes.minimal-json.example.com_policy\")}"
}
},
"aws_internet_gateway": {
"minimal-json-example-com": {
"vpc_id": "${aws_vpc.minimal-json-example-com.id}",
"tags": {
"KubernetesCluster": "minimal-json.example.com",
"Name": "minimal-json.example.com",
"kubernetes.io/cluster/minimal-json.example.com": "owned"
}
}
},
"aws_key_pair": {
"kubernetes-minimal-json-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157": {
"key_name": "kubernetes.minimal-json.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57",
"public_key": "${file(\"${path.module}/data/aws_key_pair_kubernetes.minimal-json.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key\")}"
}
},
"aws_launch_configuration": {
"master-us-test-1a-masters-minimal-json-example-com": {
"name_prefix": "master-us-test-1a.masters.minimal-json.example.com-",
"image_id": "ami-12345678",
"instance_type": "m3.medium",
"key_name": "${aws_key_pair.kubernetes-minimal-json-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id}",
"iam_instance_profile": "${aws_iam_instance_profile.masters-minimal-json-example-com.id}",
"security_groups": [
"${aws_security_group.masters-minimal-json-example-com.id}"
],
"associate_public_ip_address": true,
"user_data": "${file(\"${path.module}/data/aws_launch_configuration_master-us-test-1a.masters.minimal-json.example.com_user_data\")}",
"root_block_device": {
"volume_type": "gp2",
"volume_size": 64,
"delete_on_termination": true
},
"ephemeral_block_device": [
{
"device_name": "/dev/sdc",
"virtual_name": "ephemeral0"
}
],
"lifecycle": {
"create_before_destroy": true
},
"enable_monitoring": false
},
"nodes-minimal-json-example-com": {
"name_prefix": "nodes.minimal-json.example.com-",
"image_id": "ami-12345678",
"instance_type": "t2.medium",
"key_name": "${aws_key_pair.kubernetes-minimal-json-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id}",
"iam_instance_profile": "${aws_iam_instance_profile.nodes-minimal-json-example-com.id}",
"security_groups": [
"${aws_security_group.nodes-minimal-json-example-com.id}"
],
"associate_public_ip_address": true,
"user_data": "${file(\"${path.module}/data/aws_launch_configuration_nodes.minimal-json.example.com_user_data\")}",
"root_block_device": {
"volume_type": "gp2",
"volume_size": 128,
"delete_on_termination": true
},
"lifecycle": {
"create_before_destroy": true
},
"enable_monitoring": false
}
},
"aws_route": {
"route-0-0-0-0--0": {
"route_table_id": "${aws_route_table.minimal-json-example-com.id}",
"destination_cidr_block": "0.0.0.0/0",
"gateway_id": "${aws_internet_gateway.minimal-json-example-com.id}"
}
},
"aws_route_table": {
"minimal-json-example-com": {
"vpc_id": "${aws_vpc.minimal-json-example-com.id}",
"tags": {
"KubernetesCluster": "minimal-json.example.com",
"Name": "minimal-json.example.com",
"kubernetes.io/cluster/minimal-json.example.com": "owned",
"kubernetes.io/kops/role": "public"
}
}
},
"aws_route_table_association": {
"us-test-1a-minimal-json-example-com": {
"subnet_id": "${aws_subnet.us-test-1a-minimal-json-example-com.id}",
"route_table_id": "${aws_route_table.minimal-json-example-com.id}"
}
},
"aws_security_group": {
"masters-minimal-json-example-com": {
"name": "masters.minimal-json.example.com",
"vpc_id": "${aws_vpc.minimal-json-example-com.id}",
"description": "Security group for masters",
"tags": {
"KubernetesCluster": "minimal-json.example.com",
"Name": "masters.minimal-json.example.com",
"kubernetes.io/cluster/minimal-json.example.com": "owned"
}
},
"nodes-minimal-json-example-com": {
"name": "nodes.minimal-json.example.com",
"vpc_id": "${aws_vpc.minimal-json-example-com.id}",
"description": "Security group for nodes",
"tags": {
"KubernetesCluster": "minimal-json.example.com",
"Name": "nodes.minimal-json.example.com",
"kubernetes.io/cluster/minimal-json.example.com": "owned"
}
}
},
"aws_security_group_rule": {
"all-master-to-master": {
"type": "ingress",
"security_group_id": "${aws_security_group.masters-minimal-json-example-com.id}",
"source_security_group_id": "${aws_security_group.masters-minimal-json-example-com.id}",
"from_port": 0,
"to_port": 0,
"protocol": "-1"
},
"all-master-to-node": {
"type": "ingress",
"security_group_id": "${aws_security_group.nodes-minimal-json-example-com.id}",
"source_security_group_id": "${aws_security_group.masters-minimal-json-example-com.id}",
"from_port": 0,
"to_port": 0,
"protocol": "-1"
},
"all-node-to-node": {
"type": "ingress",
"security_group_id": "${aws_security_group.nodes-minimal-json-example-com.id}",
"source_security_group_id": "${aws_security_group.nodes-minimal-json-example-com.id}",
"from_port": 0,
"to_port": 0,
"protocol": "-1"
},
"https-external-to-master-0-0-0-0--0": {
"type": "ingress",
"security_group_id": "${aws_security_group.masters-minimal-json-example-com.id}",
"from_port": 443,
"to_port": 443,
"protocol": "tcp",
"cidr_blocks": [
"0.0.0.0/0"
]
},
"master-egress": {
"type": "egress",
"security_group_id": "${aws_security_group.masters-minimal-json-example-com.id}",
"from_port": 0,
"to_port": 0,
"protocol": "-1",
"cidr_blocks": [
"0.0.0.0/0"
]
},
"node-egress": {
"type": "egress",
"security_group_id": "${aws_security_group.nodes-minimal-json-example-com.id}",
"from_port": 0,
"to_port": 0,
"protocol": "-1",
"cidr_blocks": [
"0.0.0.0/0"
]
},
"node-to-master-tcp-1-2379": {
"type": "ingress",
"security_group_id": "${aws_security_group.masters-minimal-json-example-com.id}",
"source_security_group_id": "${aws_security_group.nodes-minimal-json-example-com.id}",
"from_port": 1,
"to_port": 2379,
"protocol": "tcp"
},
"node-to-master-tcp-2382-4000": {
"type": "ingress",
"security_group_id": "${aws_security_group.masters-minimal-json-example-com.id}",
"source_security_group_id": "${aws_security_group.nodes-minimal-json-example-com.id}",
"from_port": 2382,
"to_port": 4000,
"protocol": "tcp"
},
"node-to-master-tcp-4003-65535": {
"type": "ingress",
"security_group_id": "${aws_security_group.masters-minimal-json-example-com.id}",
"source_security_group_id": "${aws_security_group.nodes-minimal-json-example-com.id}",
"from_port": 4003,
"to_port": 65535,
"protocol": "tcp"
},
"node-to-master-udp-1-65535": {
"type": "ingress",
"security_group_id": "${aws_security_group.masters-minimal-json-example-com.id}",
"source_security_group_id": "${aws_security_group.nodes-minimal-json-example-com.id}",
"from_port": 1,
"to_port": 65535,
"protocol": "udp"
},
"ssh-external-to-master-0-0-0-0--0": {
"type": "ingress",
"security_group_id": "${aws_security_group.masters-minimal-json-example-com.id}",
"from_port": 22,
"to_port": 22,
"protocol": "tcp",
"cidr_blocks": [
"0.0.0.0/0"
]
},
"ssh-external-to-node-0-0-0-0--0": {
"type": "ingress",
"security_group_id": "${aws_security_group.nodes-minimal-json-example-com.id}",
"from_port": 22,
"to_port": 22,
"protocol": "tcp",
"cidr_blocks": [
"0.0.0.0/0"
]
}
},
"aws_subnet": {
"us-test-1a-minimal-json-example-com": {
"vpc_id": "${aws_vpc.minimal-json-example-com.id}",
"cidr_block": "172.20.32.0/19",
"availability_zone": "us-test-1a",
"tags": {
"KubernetesCluster": "minimal-json.example.com",
"Name": "us-test-1a.minimal-json.example.com",
"SubnetType": "Public",
"kubernetes.io/cluster/minimal-json.example.com": "owned",
"kubernetes.io/role/elb": "1"
}
}
},
"aws_vpc": {
"minimal-json-example-com": {
"cidr_block": "172.20.0.0/16",
"enable_dns_hostnames": true,
"enable_dns_support": true,
"tags": {
"KubernetesCluster": "minimal-json.example.com",
"Name": "minimal-json.example.com",
"kubernetes.io/cluster/minimal-json.example.com": "owned"
}
}
},
"aws_vpc_dhcp_options": {
"minimal-json-example-com": {
"domain_name": "us-test-1.compute.internal",
"domain_name_servers": [
"AmazonProvidedDNS"
],
"tags": {
"KubernetesCluster": "minimal-json.example.com",
"Name": "minimal-json.example.com",
"kubernetes.io/cluster/minimal-json.example.com": "owned"
}
}
},
"aws_vpc_dhcp_options_association": {
"minimal-json-example-com": {
"vpc_id": "${aws_vpc.minimal-json-example-com.id}",
"dhcp_options_id": "${aws_vpc_dhcp_options.minimal-json-example-com.id}"
}
}
},
"terraform": {
"required_version": "\u003e= 0.12.0"
}
}

17
upup/pkg/fi/cloudup/terraform/target.go
View File

@ -28,6 +28,7 @@ import (
hcl_parser "github.com/hashicorp/hcl/json/parser"
"k8s.io/klog"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/featureflag"
"k8s.io/kops/upup/pkg/fi"
)
@ -258,7 +259,11 @@ func (t *TerraformTarget) Finish(taskMap map[string]fi.Task) error {
// See https://github.com/kubernetes/kops/pull/2424 for why we require 0.9.3
terraformConfiguration := make(map[string]interface{})
terraformConfiguration["required_version"] = ">= 0.9.3"
if featureflag.TerraformJSON.Enabled() {
terraformConfiguration["required_version"] = ">= 0.12.0"
} else {
terraformConfiguration["required_version"] = ">= 0.9.3"
}
data := make(map[string]interface{})
data["terraform"] = terraformConfiguration
@ -278,10 +283,12 @@ func (t *TerraformTarget) Finish(taskMap map[string]fi.Task) error {
return fmt.Errorf("error marshaling terraform data to json: %v", err)
}
useJson := false
if useJson {
t.files["kubernetes.tf"] = jsonBytes
if featureflag.TerraformJSON.Enabled() {
t.files["kubernetes.tf.json"] = jsonBytes
p := path.Join(t.outDir, "kubernetes.tf")
if _, err := os.Stat(p); err == nil {
return fmt.Errorf("Error generating kubernetes.tf.json: If you are upgrading from terraform 0.11 or earlier please read the release notes. Also, the kubernetes.tf file is already present. Please move the file away since it will be replaced by the kubernetes.tf.json file. ")
}
} else {
f, err := hcl_parser.Parse(jsonBytes)
if err != nil {