feat: update workflows and make idempotent (#9)

Signed-off-by: matttrach <matttrach@gmail.com>
Co-authored-by: Vivian Ta <ta.vivian@gmail.com>

.aliases

@@ -1,8 +1,10 @@
 alias gs='git status'
-alias gm='git add -A;git commit -s -m '
+alias gd='git diff'
 alias tf='terraform'
 alias tfa='terraform apply --auto-approve'
 alias tfd='terraform destroy --auto-approve'
 alias tfp='terraform init || terraform providers && terraform validate && terraform plan'
 alias tfr='terraform destroy --auto-approve; terraform apply --auto-approve'
 alias k='kubectl'
+alias tt='cd tests; go test -v -parallel=10 -timeout=80m'
+alias nix='nix --extra-experimental-features nix-command --extra-experimental-features flakes'

.envrc

@@ -1,8 +1,30 @@
-if [ -z "${name}" ]; then
+if [ -z "${NIX_ENV_LOADED}" ]; then
 	echo "entering dev environment..."
-	nix develop . --extra-experimental-features nix-command --extra-experimental-features flakes
+  export NIX_ENV_LOADED=$(pwd)
+
+  nix develop \
+    --ignore-environment \
+    --extra-experimental-features nix-command \
+    --extra-experimental-features flakes \
+    --keep HOME \
+    --keep SSH_AUTH_SOCK \
+    --keep GITHUB_TOKEN \
+    --keep AWS_ROLE \
+    --keep AWS_REGION \
+    --keep AWS_DEFAULT_REGION \
+    --keep AWS_ACCESS_KEY_ID \
+    --keep AWS_SECRET_ACCESS_KEY \
+    --keep AWS_SESSION_TOKEN \
+    --keep UPDATECLI_GPGTOKEN \
+    --keep UPDATECLI_GITHUB_TOKEN \
+    --keep UPDATECLI_GITHUB_ACTOR \
+    --keep GPG_SIGNING_KEY \
+    --keep NIX_ENV_LOADED \
+    --keep TERM \
+    $(pwd)
 else
 	echo "setting up dev environment..."
+
 	source .aliases
 	source .functions
 	source .variables

.github/workflows/release.yaml

@@ -1,18 +1,85 @@
-name: 'Testing'
+name: release
 
 on:
   push:
     branches:
     - main
 
-permissions:
+env:
-  contents: write
+  GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
-  pull-requests: write
+
+permissions: write-all
 
 jobs:
   release:
     runs-on: ubuntu-latest
     steps:
-      - uses: google-github-actions/release-please-action@v3
+      - uses: google-github-actions/release-please-action@v4
+        id: release-please
         with:
           release-type: terraform-module
+      - uses: peter-evans/create-or-update-comment@v4
+        name: 'Remind to wait'
+        if: steps.release-please.outputs.pr
+        with:
+          issue-number: ${{ fromJson(steps.release-please.outputs.pr).number }}
+          body: |
+            Please make sure e2e tests pass before merging this PR!
+            ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+      - uses: actions/checkout@v4
+        if: steps.release-please.outputs.pr
+        with:
+          token: ${{secrets.GITHUB_TOKEN}}
+      - uses: DeterminateSystems/nix-installer-action@main
+        if: steps.release-please.outputs.pr
+      - uses: nicknovitski/nix-develop@v1.1.0
+        if: steps.release-please.outputs.pr
+        with:
+          arguments: |
+            --ignore-environment \
+            --extra-experimental-features nix-command \
+            --extra-experimental-features flakes \
+            --keep HOME \
+            --keep SSH_AUTH_SOCK \
+            --keep GITHUB_TOKEN \
+            --keep AWS_ROLE \
+            --keep AWS_REGION \
+            --keep AWS_DEFAULT_REGION \
+            --keep AWS_ACCESS_KEY_ID \
+            --keep AWS_SECRET_ACCESS_KEY \
+            --keep AWS_SESSION_TOKEN \
+            --keep UPDATECLI_GPGTOKEN \
+            --keep UPDATECLI_GITHUB_TOKEN \
+            --keep UPDATECLI_GITHUB_ACTOR \
+            --keep GPG_SIGNING_KEY \
+            --keep NIX_ENV_LOADED \
+            --keep TERM \
+            ${{ github.workspace }}
+      - uses: actions/cache/restore@v4
+        id: cache-terraform-restore
+        if: steps.release-please.outputs.pr
+        with:
+          path: ${{ github.workspace }}/.terraform
+          key: terraform-${{hashFiles('**/versions.tf','**/main.tf')}}
+      - run: terraform init -upgrade
+        if: steps.release-please.outputs.pr
+      - uses: actions/cache/save@v4
+        id: cache-terraform-save
+        if: steps.release-please.outputs.pr
+        with:
+          path: ${{ github.workspace }}/.terraform
+          key: ${{ steps.cache-terraform-restore.outputs.cache-primary-key }}
+      - run: go version && cd ${{github.workspace}}/tests && go test -v -timeout=40m -parallel=10 && cd ${{github.workspace}}
+        if: steps.release-please.outputs.pr
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+          GITHUB_OWNER: rancher
+          IDENTIFIER: ${{github.job}}-${{github.run_id}}-${{github.run_number}}-${{github.run_attempt}}
+      - uses: peter-evans/create-or-update-comment@v4
+        name: 'Report Success'
+        if: steps.release-please.outputs.pr
+        with:
+          issue-number: ${{ fromJson(steps.release-please.outputs.pr).number }}
+          body: |
+            End to End Tests Passed!
+            ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

.github/workflows/testing.yaml

@@ -1,58 +0,0 @@
-name: 'Testing'
-
-on:
-  pull_request:
-    branches:
-    - main 
-
-jobs:
-  terraform:
-    name: 'Terraform'
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-      
-    - name: Setup Terraform
-      uses: hashicorp/setup-terraform@v1
-      with:
-        terraform_version: 1.5.6
-        with_wrapper: false
-
-    - name: Init Basic
-      run: cd examples/basic && terraform init -upgrade
-
-    - name: Validate Basic
-      run: cd examples/basic && terraform validate
-
-  tflint:
-    name: 'TFLint'
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-
-    - uses: terraform-linters/setup-tflint@v3
-      name: Setup TFLint
-      with:
-        tflint_version: latest
-
-    - name: Show version
-      run: tflint --version
-
-    - name: Init TFLint
-      run: tflint --init
-
-    - name: Run TFLint
-      run: tflint -f compact
-
-  terratest:
-    name: 'Terratest'
-    runs-on: ubuntu-latest
-    environment: test
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-
-    - name: Run Terratest
-      run: cd ./tests && go test -v -timeout 10m -parallel 10

.github/workflows/validate.yaml

@@ -0,0 +1,136 @@
+name: validate
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  terraform:
+    name: 'Terraform'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: DeterminateSystems/nix-installer-action@main
+    - uses: nicknovitski/nix-develop@v1.1.0
+      with:
+        arguments: |
+          --ignore-environment \
+          --extra-experimental-features nix-command \
+          --extra-experimental-features flakes \
+          --keep HOME \
+          --keep SSH_AUTH_SOCK \
+          --keep GITHUB_TOKEN \
+          --keep AWS_ROLE \
+          --keep AWS_REGION \
+          --keep AWS_DEFAULT_REGION \
+          --keep AWS_ACCESS_KEY_ID \
+          --keep AWS_SECRET_ACCESS_KEY \
+          --keep AWS_SESSION_TOKEN \
+          --keep UPDATECLI_GPGTOKEN \
+          --keep UPDATECLI_GITHUB_TOKEN \
+          --keep UPDATECLI_GITHUB_ACTOR \
+          --keep GPG_SIGNING_KEY \
+          --keep NIX_ENV_LOADED \
+          --keep TERM \
+          ${{ github.workspace }}
+    - uses: actions/cache/restore@v4
+      id: cache-terraform-restore
+      with:
+        path: ${{ github.workspace }}/.terraform
+        key: terraform
+    - run: terraform init -upgrade
+    - uses: actions/cache/save@v4
+      id: cache-terraform-save
+      with:
+        path: ${{ github.workspace }}/.terraform
+        key: ${{ steps.cache-terraform-restore.outputs.cache-primary-key }}
+    - run: cd ${{ github.workspace }}/examples/basic && terraform version && terraform init -upgrade && terraform validate && cd ${{ github.workspace }}
+    - run: terraform fmt -check -recursive
+
+  actionlint:
+    name: 'Lint Workflows'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: DeterminateSystems/nix-installer-action@main
+    - uses: nicknovitski/nix-develop@v1.1.0
+      with:
+        arguments: |
+          --ignore-environment \
+          --extra-experimental-features nix-command \
+          --extra-experimental-features flakes \
+          --keep HOME \
+          --keep SSH_AUTH_SOCK \
+          --keep GITHUB_TOKEN \
+          --keep AWS_ROLE \
+          --keep AWS_REGION \
+          --keep AWS_DEFAULT_REGION \
+          --keep AWS_ACCESS_KEY_ID \
+          --keep AWS_SECRET_ACCESS_KEY \
+          --keep AWS_SESSION_TOKEN \
+          --keep UPDATECLI_GPGTOKEN \
+          --keep UPDATECLI_GITHUB_TOKEN \
+          --keep UPDATECLI_GITHUB_ACTOR \
+          --keep GPG_SIGNING_KEY \
+          --keep NIX_ENV_LOADED \
+          --keep TERM \
+          ${{ github.workspace }}
+    - run: actionlint
+
+  tflint:
+    name: 'TFLint'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: terraform-linters/setup-tflint@v4
+      with:
+        tflint_version: latest
+    - run: tflint --version
+    - run: tflint --init
+    - run: tflint -f compact
+
+  shellcheck:
+    name: Shellcheck
+    runs-on: ubuntu-latest
+    steps:
+      - uses: ludeeus/action-shellcheck@master
+
+  validate-commit-message:
+    name: Validate Commit Message
+    runs-on: ubuntu-latest
+    steps:
+      - uses: amannn/action-semantic-pull-request@v5
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  gitleaks:
+    name: 'Scan for Secrets'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: DeterminateSystems/nix-installer-action@main
+    - uses: nicknovitski/nix-develop@v1.1.0
+      with:
+        arguments: |
+          --ignore-environment \
+          --extra-experimental-features nix-command \
+          --extra-experimental-features flakes \
+          --keep HOME \
+          --keep SSH_AUTH_SOCK \
+          --keep GITHUB_TOKEN \
+          --keep AWS_ROLE \
+          --keep AWS_REGION \
+          --keep AWS_DEFAULT_REGION \
+          --keep AWS_ACCESS_KEY_ID \
+          --keep AWS_SECRET_ACCESS_KEY \
+          --keep AWS_SESSION_TOKEN \
+          --keep UPDATECLI_GPGTOKEN \
+          --keep UPDATECLI_GITHUB_TOKEN \
+          --keep UPDATECLI_GITHUB_ACTOR \
+          --keep GPG_SIGNING_KEY \
+          --keep NIX_ENV_LOADED \
+          --keep TERM \
+          ${{ github.workspace }}
+    - run: gitleaks detect --no-banner -v --no-git
+    - run: gitleaks detect --no-banner -v

README.md

@@ -1,8 +1,7 @@
 # Terraform RKE2 Download
 
-WARNING! this module is experimental
-
 This module downloads the proper files from the RKE2 release specified and names them appropriately for the install script.
+The download can be triggered again by changing the release variable.
 
 ## Usage
 

flake.lock

@@ -5,11 +5,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1692799911,
+        "lastModified": 1705309234,
-        "narHash": "sha256-3eihraek4qL744EvQXsK1Ha6C3CR7nnT8X2qWap4RNk=",
+        "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "f9e7cf818399d17d347f847525c5a5a8032e4e44",
+        "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26",
         "type": "github"
       },
       "original": {
@@ -20,11 +20,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1693844670,
+        "lastModified": 1706925685,
-        "narHash": "sha256-t69F2nBB8DNQUWHD809oJZJVE+23XBrth4QZuVd6IE0=",
+        "narHash": "sha256-hVInjWMmgH4yZgA4ZtbgJM1qEAel72SYhP5nOWX4UIM=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "3c15feef7770eb5500a4b8792623e2d6f598c9c1",
+        "rev": "79a13f1437e149dc7be2d1290c74d378dad60814",
         "type": "github"
       },
       "original": {

flake.nix

@@ -1,8 +1,8 @@
 {
   # validate a flake with 'nix flake check .'
   # alias the use of flakes with: "alias nix='nix --extra-experimental-features nix-command --extra-experimental-features flakes'"
-  #  you can also set a config file at ~/.config/nix/nix.conf or /etc/nix.conf, but I wanted to remove that dependency
 
+  # WARNING! this is linux/mac only!
   description = "A reliable testing environment";
 
   # https://status.nixos.org/ has the latest channels, it is recommended to use a commit hash
@@ -15,34 +15,105 @@
   # select a tag
   #inputs.nixpkgs.url = "github:NixOS/nixpkgs/21.11";
 
-  # select HEAD on a branch
+  # select packages from another flake
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
-
-  # follows allows idempotent loading of nixpkgs in dependent flakes
   #inputs.nixpkgs.follows = "nixpkgs/0228346f7b58f1a284fdb1b72df6298b06677495";
-
+  inputs = {
-  # install flake utils
+    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
-  inputs.flake-utils.url = "github:numtide/flake-utils";
+    flake-utils.url = "github:numtide/flake-utils";
-
+  };
-  outputs = { self, nixpkgs, flake-utils }:
+  outputs = { self, nixpkgs, flake-utils, ... }:
     flake-utils.lib.eachDefaultSystem (system:
       # 'legacy' is not bad, it looks for previously imported nixpkgs
       #  this allows idempotent loading of nixpkgs in dependent flakes
       # https://discourse.nixos.org/t/using-nixpkgs-legacypackages-system-vs-import/17462/8
-      let pkgs = nixpkgs.legacyPackages.${system};
+      let
+        pkgs = nixpkgs.legacyPackages.${system};
+
+        # get "updatecli" tar from release, unpack it, and add it to shell
+        updatecli-version = {
+          "selected" = "v0.66.0";
+        };
+        updatecli-prep = {
+          "x86_64-darwin"   = {
+            "url" = "https://github.com/updatecli/updatecli/releases/download/${updatecli-version.selected}/updatecli_Darwin_x86_64.tar.gz";
+            "sha" = "sha256-AkYYaCp/a4KkZ4zcYc3GepllyPE6bHb7x7K25JSyljY=";
+          };
+          "aarch64-darwin"  = {
+            "url" = "https://github.com/updatecli/updatecli/releases/download/${updatecli-version.selected}/updatecli_Darwin_arm64.tar.gz";
+            "sha" = "sha256-5ctfz4DcHRmjImYzL9hgOaC9pyeYKQxkpdIAr3J1a6g=";
+          };
+          "x86_64-linux"    = {
+            "url" = "https://github.com/updatecli/updatecli/releases/download/${updatecli-version.selected}/updatecli_Linux_x86_64.tar.gz";
+            "sha" = "sha256-tmboI0ew+LApo3uLVqebaa8VA/6rgonGJH2onQEbSyk=";
+          };
+        };
+        updatecli = pkgs.runCommand "updatecli-${updatecli-version.selected}" {} ''
+          cp ${pkgs.fetchzip { # when fetching archives use fetchzip instead of fetchurl to automatically unpack
+            url = updatecli-prep."${system}".url;
+            sha256 = updatecli-prep."${system}".sha;
+            stripRoot = false;
+          }}/updatecli $out
+          chmod +x $out
+        '';
+        updatecli-wrapper = pkgs.writeShellScriptBin "updatecli" ''
+          exec ${updatecli} "$@"
+        '';
+
+        # get "leftovers" bin from release and add it to shell
+        leftovers-version = {
+          # remember when updating the version to also update the shas
+          # to get the sha, download the file and run 'nix hash file <file>'
+          "selected" = "v0.70.0";
+        };
+        leftovers-prep = {
+          "x86_64-darwin"   = {
+            "url" = "https://github.com/genevieve/leftovers/releases/download/${leftovers-version.selected}/leftovers-${leftovers-version.selected}-darwin-amd64";
+            "sha" = "sha256-HV12kHqB14lGDm1rh9nD1n7Jvw0rCnxmjC9gusw7jfo=";
+          };
+          "aarch64-darwin"  = {
+            "url" = "https://github.com/genevieve/leftovers/releases/download/${leftovers-version.selected}/leftovers-${leftovers-version.selected}-darwin-arm64";
+            "sha" = "sha256-Tw7G538RYZrwIauN7kI68u6aKS4d/0Efh+dirL/kzoM=";
+          };
+          "x86_64-linux"    = {
+            "url" = "https://github.com/genevieve/leftovers/releases/download/${leftovers-version.selected}/leftovers-${leftovers-version.selected}-linux-amd64";
+            "sha" = "sha256-D2OPjLlV5xR3f+dVHu0ld6bQajD5Rv9GLCMCk9hXlu8=";
+          };
+        };
+        leftovers = pkgs.runCommand "leftovers-${leftovers-version.selected}" {} ''
+          cp ${pkgs.fetchurl {
+            url = leftovers-prep."${system}".url;
+            sha256 = leftovers-prep."${system}".sha;
+          }} $out
+          chmod +x $out
+        '';
+        leftovers-wrapper = pkgs.writeShellScriptBin "leftovers" ''
+          exec ${leftovers} "$@"
+        '';
       in
       {
-        devShell = pkgs.mkShell {
+        devShells.default = pkgs.mkShell {
           buildInputs = with pkgs; [
+            actionlint
             bashInteractive
+            curl
             git
-            terraform
+            gitleaks
+            go # need go for terratest
+            gnupg # need gpg for signing commits
+            jq
+            less
+            openssh # need openssh for running remote provisioners
+            shellcheck
             tflint
-            tfsec
+            tfswitch
-            terraform-docs
+            vim
           ];
           shellHook = ''
-            source .envrc
+            homebin=$HOME/bin;
+            install -d $homebin;
+            tfswitch -b $homebin/terraform 1.5.7 &>/dev/null;
+            export PATH="${updatecli-wrapper}/bin:${leftovers-wrapper}/bin:$homebin:$PATH";
+            export PS1="nix:# ";
           '';
         };
       }

main.tf

@@ -47,16 +47,17 @@ resource "local_file" "download_dir" {
 }
 
 # requires curl to be installed in the environment running terraform
-resource "null_resource" "download" {
+resource "terraform_data" "download" {
   depends_on = [
     data.github_release.selected,
     data.github_release.latest,
     local_file.download_dir,
   ]
   for_each         = local.files
+  triggers_replace = each.value
   provisioner "local-exec" {
     command = <<-EOT
-      curl -L -s -o ${abspath("${local.path}/${each.key}")} ${each.value}
+      curl --clobber -L -s -o ${"${local.path}/${each.key}"} ${each.value}
     EOT
   }
 }

tests/selected_test.go

@@ -9,7 +9,8 @@ import (
 func TestSelected(t *testing.T) {
 	t.Parallel()
 	directory := "selected"
-	release := getLatestRelease(t, "rancher", "rke2")
+	release := "v1.28.5+rke2r1"
+
 	terraformVars := map[string]interface{}{
 		"release": release,
 		"path":    "./rke2",
@@ -19,4 +20,11 @@ func TestSelected(t *testing.T) {
 	defer teardown(t, directory)
 	defer terraform.Destroy(t, terraformOptions)
 	terraform.InitAndApply(t, terraformOptions)
+	newRelease := getLatestRelease(t, "rancher", "rke2")
+	newTerraformVars := map[string]interface{}{
+		"release": newRelease,
+		"path":    "./rke2",
+	}
+	newTerraformOptions := setup(t, directory, newTerraformVars)
+	terraform.InitAndApply(t, newTerraformOptions)
 }