From b3e70a2c73b85f1815d1d295aed76faeeb208389 Mon Sep 17 00:00:00 2001 From: Matt Trachier Date: Mon, 5 Feb 2024 16:44:53 -0600 Subject: [PATCH] feat: update workflows and make idempotent (#9) Signed-off-by: matttrach Co-authored-by: Vivian Ta --- .aliases | 6 +- .envrc | 28 ++++++- .github/workflows/release.yaml | 79 +++++++++++++++++-- .github/workflows/testing.yaml | 58 -------------- .github/workflows/validate.yaml | 136 ++++++++++++++++++++++++++++++++ README.md | 3 +- flake.lock | 12 +-- flake.nix | 103 ++++++++++++++++++++---- main.tf | 7 +- tests/selected_test.go | 10 ++- 10 files changed, 345 insertions(+), 97 deletions(-) delete mode 100644 .github/workflows/testing.yaml create mode 100644 .github/workflows/validate.yaml diff --git a/.aliases b/.aliases index b24d53f..88f4b3d 100644 --- a/.aliases +++ b/.aliases @@ -1,8 +1,10 @@ alias gs='git status' -alias gm='git add -A;git commit -s -m ' +alias gd='git diff' alias tf='terraform' alias tfa='terraform apply --auto-approve' alias tfd='terraform destroy --auto-approve' alias tfp='terraform init || terraform providers && terraform validate && terraform plan' alias tfr='terraform destroy --auto-approve; terraform apply --auto-approve' -alias k='kubectl' \ No newline at end of file +alias k='kubectl' +alias tt='cd tests; go test -v -parallel=10 -timeout=80m' +alias nix='nix --extra-experimental-features nix-command --extra-experimental-features flakes' \ No newline at end of file diff --git a/.envrc b/.envrc index 0b03339..6fd35ce 100644 --- a/.envrc +++ b/.envrc @@ -1,10 +1,32 @@ -if [ -z "${name}" ]; then +if [ -z "${NIX_ENV_LOADED}" ]; then echo "entering dev environment..." - nix develop . --extra-experimental-features nix-command --extra-experimental-features flakes + export NIX_ENV_LOADED=$(pwd) + + nix develop \ + --ignore-environment \ + --extra-experimental-features nix-command \ + --extra-experimental-features flakes \ + --keep HOME \ + --keep SSH_AUTH_SOCK \ + --keep GITHUB_TOKEN \ + --keep AWS_ROLE \ + --keep AWS_REGION \ + --keep AWS_DEFAULT_REGION \ + --keep AWS_ACCESS_KEY_ID \ + --keep AWS_SECRET_ACCESS_KEY \ + --keep AWS_SESSION_TOKEN \ + --keep UPDATECLI_GPGTOKEN \ + --keep UPDATECLI_GITHUB_TOKEN \ + --keep UPDATECLI_GITHUB_ACTOR \ + --keep GPG_SIGNING_KEY \ + --keep NIX_ENV_LOADED \ + --keep TERM \ + $(pwd) else echo "setting up dev environment..." + source .aliases source .functions source .variables source .rcs -fi \ No newline at end of file +fi diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1e78990..57ddcac 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,18 +1,85 @@ -name: 'Testing' +name: release on: push: branches: - main -permissions: - contents: write - pull-requests: write +env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + +permissions: write-all jobs: release: runs-on: ubuntu-latest steps: - - uses: google-github-actions/release-please-action@v3 + - uses: google-github-actions/release-please-action@v4 + id: release-please with: - release-type: terraform-module \ No newline at end of file + release-type: terraform-module + - uses: peter-evans/create-or-update-comment@v4 + name: 'Remind to wait' + if: steps.release-please.outputs.pr + with: + issue-number: ${{ fromJson(steps.release-please.outputs.pr).number }} + body: | + Please make sure e2e tests pass before merging this PR! + ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + - uses: actions/checkout@v4 + if: steps.release-please.outputs.pr + with: + token: ${{secrets.GITHUB_TOKEN}} + - uses: DeterminateSystems/nix-installer-action@main + if: steps.release-please.outputs.pr + - uses: nicknovitski/nix-develop@v1.1.0 + if: steps.release-please.outputs.pr + with: + arguments: | + --ignore-environment \ + --extra-experimental-features nix-command \ + --extra-experimental-features flakes \ + --keep HOME \ + --keep SSH_AUTH_SOCK \ + --keep GITHUB_TOKEN \ + --keep AWS_ROLE \ + --keep AWS_REGION \ + --keep AWS_DEFAULT_REGION \ + --keep AWS_ACCESS_KEY_ID \ + --keep AWS_SECRET_ACCESS_KEY \ + --keep AWS_SESSION_TOKEN \ + --keep UPDATECLI_GPGTOKEN \ + --keep UPDATECLI_GITHUB_TOKEN \ + --keep UPDATECLI_GITHUB_ACTOR \ + --keep GPG_SIGNING_KEY \ + --keep NIX_ENV_LOADED \ + --keep TERM \ + ${{ github.workspace }} + - uses: actions/cache/restore@v4 + id: cache-terraform-restore + if: steps.release-please.outputs.pr + with: + path: ${{ github.workspace }}/.terraform + key: terraform-${{hashFiles('**/versions.tf','**/main.tf')}} + - run: terraform init -upgrade + if: steps.release-please.outputs.pr + - uses: actions/cache/save@v4 + id: cache-terraform-save + if: steps.release-please.outputs.pr + with: + path: ${{ github.workspace }}/.terraform + key: ${{ steps.cache-terraform-restore.outputs.cache-primary-key }} + - run: go version && cd ${{github.workspace}}/tests && go test -v -timeout=40m -parallel=10 && cd ${{github.workspace}} + if: steps.release-please.outputs.pr + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + GITHUB_OWNER: rancher + IDENTIFIER: ${{github.job}}-${{github.run_id}}-${{github.run_number}}-${{github.run_attempt}} + - uses: peter-evans/create-or-update-comment@v4 + name: 'Report Success' + if: steps.release-please.outputs.pr + with: + issue-number: ${{ fromJson(steps.release-please.outputs.pr).number }} + body: | + End to End Tests Passed! + ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml deleted file mode 100644 index 4bdac89..0000000 --- a/.github/workflows/testing.yaml +++ /dev/null @@ -1,58 +0,0 @@ -name: 'Testing' - -on: - pull_request: - branches: - - main - -jobs: - terraform: - name: 'Terraform' - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v1 - with: - terraform_version: 1.5.6 - with_wrapper: false - - - name: Init Basic - run: cd examples/basic && terraform init -upgrade - - - name: Validate Basic - run: cd examples/basic && terraform validate - - tflint: - name: 'TFLint' - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - uses: terraform-linters/setup-tflint@v3 - name: Setup TFLint - with: - tflint_version: latest - - - name: Show version - run: tflint --version - - - name: Init TFLint - run: tflint --init - - - name: Run TFLint - run: tflint -f compact - - terratest: - name: 'Terratest' - runs-on: ubuntu-latest - environment: test - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Run Terratest - run: cd ./tests && go test -v -timeout 10m -parallel 10 diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml new file mode 100644 index 0000000..7f620ba --- /dev/null +++ b/.github/workflows/validate.yaml @@ -0,0 +1,136 @@ +name: validate + +on: + pull_request: + branches: + - main + +jobs: + terraform: + name: 'Terraform' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/nix-installer-action@main + - uses: nicknovitski/nix-develop@v1.1.0 + with: + arguments: | + --ignore-environment \ + --extra-experimental-features nix-command \ + --extra-experimental-features flakes \ + --keep HOME \ + --keep SSH_AUTH_SOCK \ + --keep GITHUB_TOKEN \ + --keep AWS_ROLE \ + --keep AWS_REGION \ + --keep AWS_DEFAULT_REGION \ + --keep AWS_ACCESS_KEY_ID \ + --keep AWS_SECRET_ACCESS_KEY \ + --keep AWS_SESSION_TOKEN \ + --keep UPDATECLI_GPGTOKEN \ + --keep UPDATECLI_GITHUB_TOKEN \ + --keep UPDATECLI_GITHUB_ACTOR \ + --keep GPG_SIGNING_KEY \ + --keep NIX_ENV_LOADED \ + --keep TERM \ + ${{ github.workspace }} + - uses: actions/cache/restore@v4 + id: cache-terraform-restore + with: + path: ${{ github.workspace }}/.terraform + key: terraform + - run: terraform init -upgrade + - uses: actions/cache/save@v4 + id: cache-terraform-save + with: + path: ${{ github.workspace }}/.terraform + key: ${{ steps.cache-terraform-restore.outputs.cache-primary-key }} + - run: cd ${{ github.workspace }}/examples/basic && terraform version && terraform init -upgrade && terraform validate && cd ${{ github.workspace }} + - run: terraform fmt -check -recursive + + actionlint: + name: 'Lint Workflows' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/nix-installer-action@main + - uses: nicknovitski/nix-develop@v1.1.0 + with: + arguments: | + --ignore-environment \ + --extra-experimental-features nix-command \ + --extra-experimental-features flakes \ + --keep HOME \ + --keep SSH_AUTH_SOCK \ + --keep GITHUB_TOKEN \ + --keep AWS_ROLE \ + --keep AWS_REGION \ + --keep AWS_DEFAULT_REGION \ + --keep AWS_ACCESS_KEY_ID \ + --keep AWS_SECRET_ACCESS_KEY \ + --keep AWS_SESSION_TOKEN \ + --keep UPDATECLI_GPGTOKEN \ + --keep UPDATECLI_GITHUB_TOKEN \ + --keep UPDATECLI_GITHUB_ACTOR \ + --keep GPG_SIGNING_KEY \ + --keep NIX_ENV_LOADED \ + --keep TERM \ + ${{ github.workspace }} + - run: actionlint + + tflint: + name: 'TFLint' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: terraform-linters/setup-tflint@v4 + with: + tflint_version: latest + - run: tflint --version + - run: tflint --init + - run: tflint -f compact + + shellcheck: + name: Shellcheck + runs-on: ubuntu-latest + steps: + - uses: ludeeus/action-shellcheck@master + + validate-commit-message: + name: Validate Commit Message + runs-on: ubuntu-latest + steps: + - uses: amannn/action-semantic-pull-request@v5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + gitleaks: + name: 'Scan for Secrets' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/nix-installer-action@main + - uses: nicknovitski/nix-develop@v1.1.0 + with: + arguments: | + --ignore-environment \ + --extra-experimental-features nix-command \ + --extra-experimental-features flakes \ + --keep HOME \ + --keep SSH_AUTH_SOCK \ + --keep GITHUB_TOKEN \ + --keep AWS_ROLE \ + --keep AWS_REGION \ + --keep AWS_DEFAULT_REGION \ + --keep AWS_ACCESS_KEY_ID \ + --keep AWS_SECRET_ACCESS_KEY \ + --keep AWS_SESSION_TOKEN \ + --keep UPDATECLI_GPGTOKEN \ + --keep UPDATECLI_GITHUB_TOKEN \ + --keep UPDATECLI_GITHUB_ACTOR \ + --keep GPG_SIGNING_KEY \ + --keep NIX_ENV_LOADED \ + --keep TERM \ + ${{ github.workspace }} + - run: gitleaks detect --no-banner -v --no-git + - run: gitleaks detect --no-banner -v diff --git a/README.md b/README.md index 28a2de8..0252a5e 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@ # Terraform RKE2 Download -WARNING! this module is experimental - This module downloads the proper files from the RKE2 release specified and names them appropriately for the install script. +The download can be triggered again by changing the release variable. ## Usage diff --git a/flake.lock b/flake.lock index f9394e4..1f709e6 100644 --- a/flake.lock +++ b/flake.lock @@ -5,11 +5,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1692799911, - "narHash": "sha256-3eihraek4qL744EvQXsK1Ha6C3CR7nnT8X2qWap4RNk=", + "lastModified": 1705309234, + "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", "owner": "numtide", "repo": "flake-utils", - "rev": "f9e7cf818399d17d347f847525c5a5a8032e4e44", + "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", "type": "github" }, "original": { @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1693844670, - "narHash": "sha256-t69F2nBB8DNQUWHD809oJZJVE+23XBrth4QZuVd6IE0=", + "lastModified": 1706925685, + "narHash": "sha256-hVInjWMmgH4yZgA4ZtbgJM1qEAel72SYhP5nOWX4UIM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "3c15feef7770eb5500a4b8792623e2d6f598c9c1", + "rev": "79a13f1437e149dc7be2d1290c74d378dad60814", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 30d55b0..12055a3 100644 --- a/flake.nix +++ b/flake.nix @@ -1,8 +1,8 @@ { # validate a flake with 'nix flake check .' # alias the use of flakes with: "alias nix='nix --extra-experimental-features nix-command --extra-experimental-features flakes'" - # you can also set a config file at ~/.config/nix/nix.conf or /etc/nix.conf, but I wanted to remove that dependency + # WARNING! this is linux/mac only! description = "A reliable testing environment"; # https://status.nixos.org/ has the latest channels, it is recommended to use a commit hash @@ -15,34 +15,105 @@ # select a tag #inputs.nixpkgs.url = "github:NixOS/nixpkgs/21.11"; - # select HEAD on a branch - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; - - # follows allows idempotent loading of nixpkgs in dependent flakes + # select packages from another flake #inputs.nixpkgs.follows = "nixpkgs/0228346f7b58f1a284fdb1b72df6298b06677495"; - - # install flake utils - inputs.flake-utils.url = "github:numtide/flake-utils"; - - outputs = { self, nixpkgs, flake-utils }: + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + outputs = { self, nixpkgs, flake-utils, ... }: flake-utils.lib.eachDefaultSystem (system: # 'legacy' is not bad, it looks for previously imported nixpkgs # this allows idempotent loading of nixpkgs in dependent flakes # https://discourse.nixos.org/t/using-nixpkgs-legacypackages-system-vs-import/17462/8 - let pkgs = nixpkgs.legacyPackages.${system}; + let + pkgs = nixpkgs.legacyPackages.${system}; + + # get "updatecli" tar from release, unpack it, and add it to shell + updatecli-version = { + "selected" = "v0.66.0"; + }; + updatecli-prep = { + "x86_64-darwin" = { + "url" = "https://github.com/updatecli/updatecli/releases/download/${updatecli-version.selected}/updatecli_Darwin_x86_64.tar.gz"; + "sha" = "sha256-AkYYaCp/a4KkZ4zcYc3GepllyPE6bHb7x7K25JSyljY="; + }; + "aarch64-darwin" = { + "url" = "https://github.com/updatecli/updatecli/releases/download/${updatecli-version.selected}/updatecli_Darwin_arm64.tar.gz"; + "sha" = "sha256-5ctfz4DcHRmjImYzL9hgOaC9pyeYKQxkpdIAr3J1a6g="; + }; + "x86_64-linux" = { + "url" = "https://github.com/updatecli/updatecli/releases/download/${updatecli-version.selected}/updatecli_Linux_x86_64.tar.gz"; + "sha" = "sha256-tmboI0ew+LApo3uLVqebaa8VA/6rgonGJH2onQEbSyk="; + }; + }; + updatecli = pkgs.runCommand "updatecli-${updatecli-version.selected}" {} '' + cp ${pkgs.fetchzip { # when fetching archives use fetchzip instead of fetchurl to automatically unpack + url = updatecli-prep."${system}".url; + sha256 = updatecli-prep."${system}".sha; + stripRoot = false; + }}/updatecli $out + chmod +x $out + ''; + updatecli-wrapper = pkgs.writeShellScriptBin "updatecli" '' + exec ${updatecli} "$@" + ''; + + # get "leftovers" bin from release and add it to shell + leftovers-version = { + # remember when updating the version to also update the shas + # to get the sha, download the file and run 'nix hash file ' + "selected" = "v0.70.0"; + }; + leftovers-prep = { + "x86_64-darwin" = { + "url" = "https://github.com/genevieve/leftovers/releases/download/${leftovers-version.selected}/leftovers-${leftovers-version.selected}-darwin-amd64"; + "sha" = "sha256-HV12kHqB14lGDm1rh9nD1n7Jvw0rCnxmjC9gusw7jfo="; + }; + "aarch64-darwin" = { + "url" = "https://github.com/genevieve/leftovers/releases/download/${leftovers-version.selected}/leftovers-${leftovers-version.selected}-darwin-arm64"; + "sha" = "sha256-Tw7G538RYZrwIauN7kI68u6aKS4d/0Efh+dirL/kzoM="; + }; + "x86_64-linux" = { + "url" = "https://github.com/genevieve/leftovers/releases/download/${leftovers-version.selected}/leftovers-${leftovers-version.selected}-linux-amd64"; + "sha" = "sha256-D2OPjLlV5xR3f+dVHu0ld6bQajD5Rv9GLCMCk9hXlu8="; + }; + }; + leftovers = pkgs.runCommand "leftovers-${leftovers-version.selected}" {} '' + cp ${pkgs.fetchurl { + url = leftovers-prep."${system}".url; + sha256 = leftovers-prep."${system}".sha; + }} $out + chmod +x $out + ''; + leftovers-wrapper = pkgs.writeShellScriptBin "leftovers" '' + exec ${leftovers} "$@" + ''; in { - devShell = pkgs.mkShell { + devShells.default = pkgs.mkShell { buildInputs = with pkgs; [ + actionlint bashInteractive + curl git - terraform + gitleaks + go # need go for terratest + gnupg # need gpg for signing commits + jq + less + openssh # need openssh for running remote provisioners + shellcheck tflint - tfsec - terraform-docs + tfswitch + vim ]; shellHook = '' - source .envrc + homebin=$HOME/bin; + install -d $homebin; + tfswitch -b $homebin/terraform 1.5.7 &>/dev/null; + export PATH="${updatecli-wrapper}/bin:${leftovers-wrapper}/bin:$homebin:$PATH"; + export PS1="nix:# "; ''; }; } diff --git a/main.tf b/main.tf index 5ebaeb5..90a0829 100644 --- a/main.tf +++ b/main.tf @@ -47,16 +47,17 @@ resource "local_file" "download_dir" { } # requires curl to be installed in the environment running terraform -resource "null_resource" "download" { +resource "terraform_data" "download" { depends_on = [ data.github_release.selected, data.github_release.latest, local_file.download_dir, ] - for_each = local.files + for_each = local.files + triggers_replace = each.value provisioner "local-exec" { command = <<-EOT - curl -L -s -o ${abspath("${local.path}/${each.key}")} ${each.value} + curl --clobber -L -s -o ${"${local.path}/${each.key}"} ${each.value} EOT } } diff --git a/tests/selected_test.go b/tests/selected_test.go index 02b4243..afa7c71 100644 --- a/tests/selected_test.go +++ b/tests/selected_test.go @@ -9,7 +9,8 @@ import ( func TestSelected(t *testing.T) { t.Parallel() directory := "selected" - release := getLatestRelease(t, "rancher", "rke2") + release := "v1.28.5+rke2r1" + terraformVars := map[string]interface{}{ "release": release, "path": "./rke2", @@ -19,4 +20,11 @@ func TestSelected(t *testing.T) { defer teardown(t, directory) defer terraform.Destroy(t, terraformOptions) terraform.InitAndApply(t, terraformOptions) + newRelease := getLatestRelease(t, "rancher", "rke2") + newTerraformVars := map[string]interface{}{ + "release": newRelease, + "path": "./rke2", + } + newTerraformOptions := setup(t, directory, newTerraformVars) + terraform.InitAndApply(t, newTerraformOptions) }