From 6d2cac777b33a9277d4fb8c55d02852cce9fcc02 Mon Sep 17 00:00:00 2001 From: Matt Trachier Date: Wed, 13 Mar 2024 16:31:24 -0500 Subject: [PATCH] fix: make sure default ip is always available, send owners from aws to image outputs, validate image names (#57) Signed-off-by: Matt Trachier --- .github/workflows/release.yaml | 3 ++ examples/imagetype/basic/main.tf | 20 +++++++ examples/imagetype/basic/outputs.tf | 56 +++++++++++++++++++ examples/imagetype/basic/variables.tf | 4 ++ examples/imagetype/basic/versions.tf | 9 ++++ examples/os/ubuntu22/main.tf | 5 +- flake.nix | 2 + modules/image/main.tf | 4 +- modules/image/outputs.tf | 8 ++- modules/image/types.tf | 32 +++++------ modules/server/main.tf | 2 +- tests/imagetype_test.go | 42 +++++++++++++++ validate-image.sh | 78 +++++++++++++++++++++++++++ 13 files changed, 243 insertions(+), 22 deletions(-) create mode 100644 examples/imagetype/basic/main.tf create mode 100644 examples/imagetype/basic/outputs.tf create mode 100644 examples/imagetype/basic/variables.tf create mode 100644 examples/imagetype/basic/versions.tf create mode 100644 tests/imagetype_test.go create mode 100755 validate-image.sh diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 39a8ba4..9875911 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -64,6 +64,9 @@ jobs: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} GITHUB_OWNER: rancher IDENTIFIER: ${{github.job}}-${{github.run_id}}-${{github.run_number}}-${{github.run_attempt}} + - run: ./validate-image.sh + name: 'ImageCheck' + if: steps.release-please.outputs.pr - uses: peter-evans/create-or-update-comment@v4 name: 'Report Success' if: steps.release-please.outputs.pr diff --git a/examples/imagetype/basic/main.tf b/examples/imagetype/basic/main.tf new file mode 100644 index 0000000..c4ed21b --- /dev/null +++ b/examples/imagetype/basic/main.tf @@ -0,0 +1,20 @@ +## This example shows how to use the image type search to find images in AWS +# If you need to find a different AMI or add to the types this should help +provider "aws" { + default_tags { + tags = { + Id = local.identifier + } + } +} + +locals { + identifier = var.identifier # this is a random unique string that can be used to identify resources in the cloud provider + types = ["sles-15", "sles-15-cis", "rhel-8-cis", "ubuntu-20", "ubuntu-22", "rocky-8", "rhel-9", "rhel-8"] +} + +module "this" { + for_each = toset(local.types) + source = "../../../modules/image" + type = each.key +} diff --git a/examples/imagetype/basic/outputs.tf b/examples/imagetype/basic/outputs.tf new file mode 100644 index 0000000..c18fae8 --- /dev/null +++ b/examples/imagetype/basic/outputs.tf @@ -0,0 +1,56 @@ +output "sles-15" { + value = { + name = module.this["sles-15"].name, + id = module.this["sles-15"].id, + owners = module.this["sles-15"].owners, + } +} +output "sles-15-cis" { + value = { + name = module.this["sles-15-cis"].name, + id = module.this["sles-15-cis"].id, + owners = module.this["sles-15-cis"].owners, + } +} +output "rhel-8-cis" { + value = { + name = module.this["rhel-8-cis"].name, + id = module.this["rhel-8-cis"].id, + owners = module.this["rhel-8-cis"].owners, + } +} +output "ubuntu-20" { + value = { + name = module.this["ubuntu-20"].name, + id = module.this["ubuntu-20"].id, + owners = module.this["ubuntu-20"].owners, + } +} +output "ubuntu-22" { + value = { + name = module.this["ubuntu-22"].name, + id = module.this["ubuntu-22"].id, + owners = module.this["ubuntu-22"].owners, + } +} +output "rocky-8" { + value = { + name = module.this["rocky-8"].name, + id = module.this["rocky-8"].id, + owners = module.this["rocky-8"].owners, + } +} +output "rhel-9" { + value = { + name = module.this["rhel-9"].name, + id = module.this["rhel-9"].id, + owners = module.this["rhel-9"].owners, + } +} +output "rhel-8" { + value = { + name = module.this["rhel-8"].name, + id = module.this["rhel-8"].id, + owners = module.this["rhel-8"].owners, + } +} diff --git a/examples/imagetype/basic/variables.tf b/examples/imagetype/basic/variables.tf new file mode 100644 index 0000000..a5c2894 --- /dev/null +++ b/examples/imagetype/basic/variables.tf @@ -0,0 +1,4 @@ +variable "identifier" { + type = string + default = "test" +} diff --git a/examples/imagetype/basic/versions.tf b/examples/imagetype/basic/versions.tf new file mode 100644 index 0000000..efe2bf2 --- /dev/null +++ b/examples/imagetype/basic/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.5.0, < 1.6" + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.11" + } + } +} \ No newline at end of file diff --git a/examples/os/ubuntu22/main.tf b/examples/os/ubuntu22/main.tf index 5b997f6..1d24ddc 100644 --- a/examples/os/ubuntu22/main.tf +++ b/examples/os/ubuntu22/main.tf @@ -26,7 +26,7 @@ module "access" { vpc_cidr = "10.0.255.0/24" # gives 256 usable addresses from .1 to .254, but AWS reserves .1 to .4 and .255, leaving .5 to .254 subnet_name = local.name subnet_cidr = "10.0.255.224/28" # gives 14 usable addresses from .225 to .238, but AWS reserves .225 to .227 and .238, leaving .227 to .237 - security_group_name = local.name + security_group_name = local.name # WARNING: security_group.name isn't the same as security_group->tags->Name security_group_type = "specific" skip_ssh = true } @@ -47,5 +47,6 @@ module "this" { ssh_key = local.public_ssh_key ssh_key_name = local.key_name subnet_name = local.name - security_group_name = local.name # WARNING: security_group.name isn't the same as security_group->tags->Name + security_group_name = local.name + add_public_ip = true } diff --git a/flake.nix b/flake.nix index 12055a3..7ae699f 100644 --- a/flake.nix +++ b/flake.nix @@ -94,6 +94,7 @@ devShells.default = pkgs.mkShell { buildInputs = with pkgs; [ actionlint + awscli bashInteractive curl git @@ -107,6 +108,7 @@ tflint tfswitch vim + which ]; shellHook = '' homebin=$HOME/bin; diff --git a/modules/image/main.tf b/modules/image/main.tf index 7b64176..b61675b 100644 --- a/modules/image/main.tf +++ b/modules/image/main.tf @@ -3,7 +3,7 @@ locals { search = (local.id == "" ? true : false) # search if no id is given select = (local.id == "" ? false : true) # select if id is given type = (local.search ? local.types[var.type] : null) - owner = (local.search ? local.type.owner : null) + owners = (local.search ? local.type.owners : []) architecture = (local.search ? local.type.architecture : null) name = (local.search ? local.type.name : null) @@ -15,7 +15,7 @@ locals { data "aws_ami" "search" { count = (local.search ? 1 : 0) most_recent = true - owners = [local.owner] + owners = local.owners filter { name = "virtualization-type" diff --git a/modules/image/outputs.tf b/modules/image/outputs.tf index 88c451b..95f79bb 100644 --- a/modules/image/outputs.tf +++ b/modules/image/outputs.tf @@ -8,6 +8,9 @@ output "ami" { output "name" { value = (local.search ? data.aws_ami.search[0].name : data.aws_ami.select[0].name) } +output "owners" { + value = (local.search ? data.aws_ami.search[0].owners : data.aws_ami.select[0].owners) +} output "initial_user" { value = local.initial_user } @@ -16,4 +19,7 @@ output "admin_group" { } output "workfolder" { value = local.workfolder -} \ No newline at end of file +} +output "full" { + value = data.aws_ami.search[0] +} diff --git a/modules/image/types.tf b/modules/image/types.tf index 9957750..87f8411 100644 --- a/modules/image/types.tf +++ b/modules/image/types.tf @@ -3,48 +3,48 @@ locals { sles-15 = { user = "ec2-user", group = "wheel", - name = "suse-sles-15-sp5-v*-hvm-*", - owner = "amazon", + name = "suse-sles-15-sp5-v2024*", + owners = ["013907871322", "679593333241"] architecture = "x86_64", workfolder = "~" }, sles-15-cis = { # WARNING! this AMI requires subscription to a service, it is not free user = "ec2-user", group = "wheel", - name = "CIS SUSE Linux Enterprise 15*", - owner = "aws-marketplace", + name = "CIS SUSE*15*", + owners = ["679593333241"], architecture = "x86_64", workfolder = "~" }, rhel-8-cis = { # WARNING! this AMI requires subscription to a service, it is not free https://aws.amazon.com/marketplace/server/procurement?productId=ca1fe94d-9237-41c7-8fc8-78b6b0658c9f user = "ec2-user", group = "wheel", - name = "CIS Red Hat Enterprise Linux 8 STIG Benchmark*", - owner = "aws-marketplace", + name = "CIS Red Hat*8*STIG*", + owners = ["679593333241"], architecture = "x86_64", workfolder = "/var/tmp" }, ubuntu-20 = { # WARNING! you must subscribe and accept the terms to use this image user = "ubuntu", group = "admin", - name = "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-*", - owner = "aws-marketplace", + name = "ubuntu/images/*ubuntu-focal-20.04-amd64-server-2024*-*-*-*-*-*", + owners = ["679593333241", "099720109477", "513442679011", "837727238323"], architecture = "x86_64", workfolder = "~" }, ubuntu-22 = { # WARNING! you must subscribe and accept the terms to use this image user = "ubuntu", group = "admin", - name = "ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-*", - owner = "aws-marketplace", + name = "ubuntu/images/*ubuntu-jammy-22.04-amd64-server-2024*-*-*-*-*-*", + owners = ["679593333241", "099720109477", "513442679011", "837727238323"], architecture = "x86_64", workfolder = "~" }, rocky-8 = { # WARNING! you must subscribe and accept the terms to use this image user = "ec2-user", group = "wheel", - name = "Rocky-8-EC2-Base-8*", - owner = "aws-marketplace", + name = "Rocky-8-*Base*.x86_64-*-*-*-*-*", + owners = ["679593333241"], architecture = "x86_64", workfolder = "~" }, @@ -56,8 +56,8 @@ locals { rhel-9 = { user = "ec2-user", group = "wheel", - name = "RHEL-9.2.*_HVM-*-x86_64-*-Hourly2-GP3", - owner = "amazon", + name = "RHEL-9.3*_HVM-2024*-x86_64-*-Hourly2-GP3", + owners = ["309956199498"], architecture = "x86_64", workfolder = "~" }, @@ -66,8 +66,8 @@ locals { rhel-8 = { user = "ec2-user", group = "wheel", - name = "RHEL-8.8.*_HVM-*-x86_64-*-Hourly2-GP3", - owner = "amazon", + name = "RHEL-8.9*_HVM-2024*-x86_64-*-Hourly2-GP3", + owners = ["309956199498"], architecture = "x86_64", workfolder = "~" }, diff --git a/modules/server/main.tf b/modules/server/main.tf index 03d1283..5075c7c 100644 --- a/modules/server/main.tf +++ b/modules/server/main.tf @@ -7,7 +7,7 @@ locals { user = var.user subnet = var.subnet # the name of the subnet to find eip = var.eip # should we deploy a public elastic ip with the server? - default_ip = cidrhost(data.aws_subnet.general_info[0].cidr_block, -2) + default_ip = (length(data.aws_subnet.general_info) > 0 ? cidrhost(data.aws_subnet.general_info[0].cidr_block, -2) : "") ip = (var.ip == "" ? local.default_ip : var.ip) # specify the private ip to assign to the server (must be within the subnet) ipv4 = (strcontains(local.ip, ":") ? "" : local.ip) ipv6 = (strcontains(local.ip, ":") ? local.ip : "") diff --git a/tests/imagetype_test.go b/tests/imagetype_test.go new file mode 100644 index 0000000..5baabdd --- /dev/null +++ b/tests/imagetype_test.go @@ -0,0 +1,42 @@ +package test + +import ( + "testing" + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/stretchr/testify/assert" +) + +func TestImageTypesBasic(t *testing.T) { + t.Parallel() + category := "imagetype" + directory := "basic" + region := "us-west-1" + owner := "terraform-ci@suse.com" + terraformOptions, keypair := setup(t, category, directory, region, owner) + + defer teardown(t, category, directory, keypair) + defer terraform.Destroy(t, terraformOptions) + // don't pass key or key_name to the module + delete(terraformOptions.Vars, "key") + delete(terraformOptions.Vars, "key_name") + terraform.InitAndApply(t, terraformOptions) + + rhel8 := terraform.OutputMap(t, terraformOptions, "rhel-8") + rhel8Cis := terraform.OutputMap(t, terraformOptions, "rhel-8-cis") + rhel9 := terraform.OutputMap(t, terraformOptions, "rhel-9") + rocky8 := terraform.OutputMap(t, terraformOptions, "rocky-8") + sles15 := terraform.OutputMap(t, terraformOptions, "sles-15") + sles15Cis := terraform.OutputMap(t, terraformOptions, "sles-15-cis") + ubuntu20 := terraform.OutputMap(t, terraformOptions, "ubuntu-20") + ubuntu22 := terraform.OutputMap(t, terraformOptions, "ubuntu-22") + + // Validate the names of the images + assert.Contains(t, rhel8["name"], "RHEL", "RHEL image name should contain 'RHEL'") + assert.Contains(t, rhel8Cis["name"], "CIS Red Hat", "RHEL image name should contain 'CIS Red Hat'") + assert.Contains(t, rhel9["name"], "RHEL", "RHEL image name should contain 'RHEL'") + assert.Contains(t, rocky8["name"], "Rocky", "Rocky image name should contain 'Rocky'") + assert.Contains(t, sles15["name"], "sles", "SLES image name should contain 'sles'") + assert.Contains(t, sles15Cis["name"], "CIS SUSE", "SLES image name should contain 'CIS SUSE'") + assert.Contains(t, ubuntu20["name"], "ubuntu", "Ubuntu image name should contain 'ubuntu'") + assert.Contains(t, ubuntu22["name"], "ubuntu", "Ubuntu image name should contain 'ubuntu'") +} diff --git a/validate-image.sh b/validate-image.sh new file mode 100755 index 0000000..f69a044 --- /dev/null +++ b/validate-image.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# shellcheck disable=SC2317,SC2034 +# shell check can't see that we are using these variables in the eval statement + +TYPES="sles15 sles15cis rhel8cis ubuntu20 ubuntu22 rocky8 rhel9 rhel8" +REGIONS="us-west-1 us-west-2 us-east-1 us-east-2" + +# IMPORTANT!! The filters in this file should match the filters in the module/image/types.tf +## please make sure to validate these image filters with QA + + +FILTER_sles15='[{"Name": "name", "Values": ["suse-sles-15-sp5-v2024*"]},{"Name": "architecture", "Values": ["x86_64"]}]' +OWNER_sles15='["013907871322","679593333241"]' + +FILTER_sles15cis='[{"Name": "name", "Values": ["CIS SUSE*15*"]},{"Name": "architecture", "Values": ["x86_64"]}]' +OWNER_sles15cis='["679593333241"]' + +FILTER_rhel8cis='[{"Name": "name", "Values": ["CIS Red Hat*8*STIG*"]},{"Name": "architecture", "Values": ["x86_64"]}]' +OWNER_rhel8cis='["679593333241"]' + +FILTER_ubuntu20='[{"Name": "name", "Values": ["ubuntu/images/*ubuntu-focal-20.04-amd64-server-2024*-*-*-*-*-*"]},{"Name": "architecture", "Values": ["x86_64"]}]' +OWNER_ubuntu20='["679593333241","099720109477","513442679011","837727238323"]' + +FILTER_ubuntu22='[{"Name": "name", "Values": ["ubuntu/images/*ubuntu-jammy-22.04-amd64-server-2024*-*-*-*-*-*"]},{"Name": "architecture", "Values": ["x86_64"]}]' +OWNER_ubuntu22='["679593333241","099720109477","513442679011","837727238323"]' + +FILTER_rocky8='[{"Name": "name", "Values": ["Rocky-8-*Base*.x86_64-*-*-*-*-*"]},{"Name": "architecture", "Values": ["x86_64"]}]' +OWNER_rocky8='["679593333241"]' + +FILTER_rhel9='[{"Name": "name", "Values": ["RHEL-9.3*_HVM-2024*-x86_64-*-Hourly2-GP3"]},{"Name": "architecture", "Values": ["x86_64"]}]' +OWNER_rhel9='["309956199498"]' + +FILTER_rhel8='[{"Name": "name", "Values": ["RHEL-8.9*_HVM-2024*-x86_64-*-Hourly2-GP3"]},{"Name": "architecture", "Values": ["x86_64"]}]' +OWNER_rhel8='["309956199498"]' + +E=0 + +main() { + for TYPE in $TYPES; do + eval "region_check \"$TYPE\" \"\$FILTER_$TYPE\" \"\$OWNER_$TYPE\" \"\$REGIONS\"" + done +} + +region_check() { + TYPE="$1" + FILTER="$2" + OWNER="$3" + REGIONS="$4" + for REGION in $REGIONS; do + image="$(find_image "$REGION" "$FILTER" "$OWNER")" + if [ "null" = "$image" ]; then echo "no images found in $REGION for $TYPE"; E=1; return; fi + if [ -z "$image" ]; then echo "no images found in $REGION for $TYPE"; E=1; return; fi + id="$(jq '. | keys | .[0]' <<<"$image")" + name="$(jq '.[].name' <<<"$image")" + created="$(jq '.[].created' <<<"$image")" + echo "image $id for $TYPE found in $REGION created at $created titled $name..." + done +} + +find_image() { + region="$1" + filter="$2" + owner="$3" + if [ -z "$region" ]; then echo "need region"; return; fi + if [ -z "$filter" ]; then echo "need a filter"; return; fi + if [ -z "$owner" ]; then echo "need an owner"; return; fi + IMAGES="$(aws ec2 describe-images \ + --region="$region" \ + --filter="$filter" \ + --owners="$owner")" + if [ -z "$IMAGES" ]; then echo "no images found..."; E=1; return; fi + if [ "null" = "$IMAGES" ]; then echo "no images found..."; E=1; return; fi + jq '.Images | sort_by(.CreationDate) | reverse | map({(.ImageId): { "name": .Name, "created": .CreationDate}}) | .[0]' <<<"$IMAGES" +} + +main +exit $E \ No newline at end of file