tests/int: Add separate resource cleanup step

Introduce a destroy-only mode in the test runner to run terraform
destroy for the respective cloud provider configurations. This can be
used to destroy cloud resources without going through the whole
provision-test process.

Add a new step in github actions workflow to run the test binary in
destoy-only mode at the very end irrespective of the result of the
previous steps. This ensures that the infrastructure is always
destroyed, even if the CI job is cancelled.

Signed-off-by: Sunny <darkowlzz@protonmail.com>

.github/workflows/integration-azure.yaml

@@ -70,3 +70,12 @@ jobs:
           ARM_SUBSCRIPTION_ID: ${{ secrets.IRC_E2E_AZ_ARM_SUBSCRIPTION_ID }}
           ARM_TENANT_ID: ${{ secrets.IRC_E2E_AZ_ARM_TENANT_ID }}
           TF_VAR_azure_location: ${{ vars.TF_VAR_azure_location }}
+      - name: Ensure resource cleanup
+        if: ${{ always() }}
+        run: . .env && make destroy-azure
+        env:
+          ARM_CLIENT_ID: ${{ secrets.IRC_E2E_AZ_ARM_CLIENT_ID }}
+          ARM_CLIENT_SECRET: ${{ secrets.IRC_E2E_AZ_ARM_CLIENT_SECRET }}
+          ARM_SUBSCRIPTION_ID: ${{ secrets.IRC_E2E_AZ_ARM_SUBSCRIPTION_ID }}
+          ARM_TENANT_ID: ${{ secrets.IRC_E2E_AZ_ARM_TENANT_ID }}
+          TF_VAR_azure_location: ${{ vars.TF_VAR_azure_location }}

.github/workflows/integration-gcp.yaml

@@ -84,3 +84,10 @@ jobs:
           TF_VAR_gcp_project_id: ${{ vars.TF_VAR_gcp_project_id }}
           TF_VAR_gcp_region: ${{ vars.TF_VAR_gcp_region }}
           TF_VAR_gcp_zone: ${{ vars.TF_VAR_gcp_zone }}
+      - name: Ensure resource cleanup
+        if: ${{ always() }}
+        run: . .env && make destroy-gcp
+        env:
+          TF_VAR_gcp_project_id: ${{ vars.TF_VAR_gcp_project_id }}
+          TF_VAR_gcp_region: ${{ vars.TF_VAR_gcp_region }}
+          TF_VAR_gcp_zone: ${{ vars.TF_VAR_gcp_zone }}

tests/integration/Makefile

@@ -32,3 +32,15 @@ test-azure:
 
 test-gcp:
 	$(MAKE) test PROVIDER_ARG="-provider gcp"
+
+destroy:
+	go test -timeout $(TEST_TIMEOUT) -v $(GO_TEST_PATH) $(GO_TEST_ARGS) $(PROVIDER_ARG) -destroy-only
+
+destroy-aws:
+	$(MAKE) destroy PROVIDER_ARG="-provider aws"
+
+destroy-azure:
+	$(MAKE) destroy PROVIDER_ARG="-provider azure"
+
+destroy-gcp:
+	$(MAKE) destroy PROVIDER_ARG="-provider gcp"

tests/integration/README.md

@@ -49,6 +49,10 @@ variables using
 use the terraform configuration below. Please make sure all the requirements of
 azure-gh-actions are followed before running it.
 
+**NOTE:** When running the following for a repo under an organization, set the
+environment variable `GITHUB_ORGANIZATION` if setting the `owner` in the
+`github` provider doesn't work.
+
 ```hcl
 provider "github" {
   owner = "fluxcd"
@@ -149,6 +153,10 @@ variables using
 use the terraform configuration below. Please make sure all the requirements of
 gcp-gh-actions are followed before running it.
 
+**NOTE:** When running the following for a repo under an organization, set the
+environment variable `GITHUB_ORGANIZATION` if setting the `owner` in the
+`github` provider doesn't work.
+
 ```hcl
 provider "google" {}
 
@@ -256,9 +264,13 @@ using the initial `build/flux.yaml` manifest.
 Once the environment is ready, the individual go tests are executed. After the
 tests end, the environment is destroyed automatically.
 
-**IMPORTANT**: In case the terraform infrastructure results in a bad state,
-maybe due to a crash during the apply, the whole infrastructure can be destroyed
-by running `terraform destroy` in `terraform/<provider>` directory.
+If not configured explicitly to retain the infrastructure, at the end of the
+test, the test infrastructure is deleted. In case of any failure due to which
+the resources don't get deleted, the `make destroy-*` commands can be run for
+the respective provider. This will run terraform destroy in the respective
+provider's terraform configuration directory. This can be used to quickly
+destroy the infrastructure without going through the provision-test-destroy
+steps.
 
 ## Debugging the tests
 

tests/integration/go.mod

@@ -6,7 +6,8 @@ replace github.com/fluxcd/image-reflector-controller/api => ../../api
 
 require (
 	github.com/fluxcd/image-reflector-controller/api v0.0.0
-	github.com/fluxcd/test-infra/tftestenv v0.0.0-20230530120643-bdcf7573fb2f
+	github.com/fluxcd/test-infra/tftestenv v0.0.0-20240108135005-b58e0c4e0cfa
+	github.com/hashicorp/terraform-exec v0.18.1
 	github.com/hashicorp/terraform-json v0.15.0
 	github.com/onsi/gomega v1.30.0
 	k8s.io/apimachinery v0.28.4
@@ -40,7 +41,6 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/hashicorp/hc-install v0.5.0 // indirect
-	github.com/hashicorp/terraform-exec v0.18.1 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect

tests/integration/go.sum

@@ -185,8 +185,8 @@ github.com/fluxcd/pkg/apis/acl v0.1.0 h1:EoAl377hDQYL3WqanWCdifauXqXbMyFuK82NnX6
 github.com/fluxcd/pkg/apis/acl v0.1.0/go.mod h1:zfEZzz169Oap034EsDhmCAGgnWlcWmIObZjYMusoXS8=
 github.com/fluxcd/pkg/apis/meta v1.2.0 h1:O766PzGAdMdQKybSflGL8oV0+GgCNIkdsxfalRyzeO8=
 github.com/fluxcd/pkg/apis/meta v1.2.0/go.mod h1:fU/Az9AoVyIxC0oI4ihG0NVMNnvrcCzdEym3wxjIQsc=
-github.com/fluxcd/test-infra/tftestenv v0.0.0-20230530120643-bdcf7573fb2f h1:aTwZxfVUtm15HyAtgK1kfNTVvyeRoP4oUjDmEh5FZcY=
-github.com/fluxcd/test-infra/tftestenv v0.0.0-20230530120643-bdcf7573fb2f/go.mod h1:liFlLEXgambGVdWSJ4JzbIHf1Vjpp1HwUyPazPIVZug=
+github.com/fluxcd/test-infra/tftestenv v0.0.0-20240108135005-b58e0c4e0cfa h1:JdI+rVwGF5gBYt+UBijOVzXtq7aAU80vgksMNXSCCfU=
+github.com/fluxcd/test-infra/tftestenv v0.0.0-20240108135005-b58e0c4e0cfa/go.mod h1:liFlLEXgambGVdWSJ4JzbIHf1Vjpp1HwUyPazPIVZug=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=

tests/integration/suite_test.go

@@ -26,6 +26,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/hashicorp/terraform-exec/tfexec"
 	tfjson "github.com/hashicorp/terraform-json"
 	"k8s.io/apimachinery/pkg/runtime"
 
@@ -69,6 +70,9 @@ var (
 	// verbose flag to enable output of terraform execution.
 	verbose = flag.Bool("verbose", false, "verbose output of the environment setup")
 
+	// destroyOnly flag to destroy any provisioned infrastructure.
+	destroyOnly = flag.Bool("destroy-only", false, "run in destroy-only mode and delete any existing infrastructure")
+
 	// testRepos is a map of registry common name and URL of the test
 	// repositories. This is used as the test cases to run the tests against.
 	// The registry common name need not be the actual registry address but an
@@ -126,15 +130,6 @@ func TestMain(m *testing.M) {
 	flag.Parse()
 	ctx := context.TODO()
 
-	ircImg := os.Getenv("TEST_IMG")
-	if ircImg == "" {
-		log.Fatal("TEST_IMG must be set to the test image-reflector-controller image, cannot be empty")
-	}
-
-	localImgs := map[string]string{
-		"image-reflector-controller": ircImg,
-	}
-
 	// Validate the provider.
 	if *targetProvider == "" {
 		log.Fatalf("-provider flag must be set to one of %v", supportedProviders)
@@ -154,6 +149,30 @@ func TestMain(m *testing.M) {
 		log.Fatalf("Failed to get provider config for %q", *targetProvider)
 	}
 
+	// Run destroy-only mode if enabled.
+	if *destroyOnly {
+		log.Println("Running in destroy-only mode...")
+		envOpts := []tftestenv.EnvironmentOption{
+			tftestenv.WithVerbose(*verbose),
+			// Ignore any state lock in destroy-only mode.
+			tftestenv.WithTfDestroyOptions(tfexec.Lock(false)),
+		}
+		if err := tftestenv.Destroy(ctx, providerCfg.terraformPath, envOpts...); err != nil {
+			panic(err)
+		}
+		os.Exit(0)
+	}
+
+	// Check the test app image.
+	ircImg := os.Getenv("TEST_IMG")
+	if ircImg == "" {
+		log.Fatal("TEST_IMG must be set to the test image-reflector-controller image, cannot be empty")
+	}
+
+	localImgs := map[string]string{
+		"image-reflector-controller": ircImg,
+	}
+
 	// Construct scheme to be added to the kubeclient.
 	scheme := runtime.NewScheme()
 	err := imagev1.AddToScheme(scheme)