diff --git a/.gitignore b/.gitignore index 774dc18..3c5b5f9 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,5 @@ install_rancher.sh.bk # Ignore CLI configuration files .terraformrc terraform.rc +tests/backuprestore/migration/restore-migration.yaml +resources/terraform/modules/ec2/encryption-provider-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2450a46..313af17 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,3 +43,10 @@ repos: entry: go mod tidy files: ^go\.mod$|^go\.sum$ stages: [manual] + + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.77.1 # Check for latest tag at https://github.com/antonbabenko/pre-commit-terraform/releases + hooks: + - id: terraform_fmt # Formats Terraform code using terraform fmt -recursive + - id: terraform_validate # Validates Terraform configuration + - id: terraform_docs # Updates README.md with module docs (if using modules) diff --git a/resources/rancher/clusters.go b/resources/rancher/clusters.go index 129d8be..57ca2d0 100644 --- a/resources/rancher/clusters.go +++ b/resources/rancher/clusters.go @@ -267,8 +267,8 @@ func CreateRKE2Cluster(rancherClient *rancher.Client, cloudCredentialName string } err = VerifyCluster(rancherClient, config.ClusterSpec.Metadata.Name) if err != nil { - err := fmt.Errorf("cluster %s is now Active", config.ClusterSpec.Metadata.Name) - return "nil", err + err := fmt.Errorf("cluster %s is not Active", config.ClusterSpec.Metadata.Name) + return config.ClusterSpec.Metadata.Name, err } return config.ClusterSpec.Metadata.Name, nil } diff --git a/resources/rancher/install.go b/resources/rancher/install.go new file mode 100644 index 0000000..036c4c1 --- /dev/null +++ b/resources/rancher/install.go @@ -0,0 +1,79 @@ +package rancher + +import ( + "fmt" + "strings" + "time" + + e2e "k8s.io/kubernetes/test/e2e/framework" + + "github.com/rancher/observability-e2e/tests/helper/helm" +) + +// AddRancherHelmRepo adds the Rancher Helm repository and updates it. +func AddRancherHelmRepo(kubeconfig, helmRepoURL, repoName string) error { + + e2e.Logf("Adding Helm repo: %s -> %s", repoName, helmRepoURL) + _, err := helm.Execute(kubeconfig, "repo", "add", repoName, helmRepoURL) + if err != nil { + return fmt.Errorf("failed to add helm repo: %w", err) + } + + e2e.Logf("Updating Helm repos...") + _, err = helm.Execute(kubeconfig, "repo", "update") + if err != nil { + return fmt.Errorf("failed to update helm repos: %w", err) + } + + e2e.Logf("Helm repo added and updated successfully") + return nil +} + +// InstallRancher installs Rancher based on the repo URL and version +func InstallRancher(kubeconfig, helmRepoURL, rancherVersion, hostname, password string) error { + repoName := fmt.Sprintf("rancher-%d", time.Now().Unix()) + if err := AddRancherHelmRepo(kubeconfig, helmRepoURL, repoName); err != nil { + return err + } + + namespace := "cattle-system" + chart := fmt.Sprintf("%s/rancher", repoName) + version := strings.TrimPrefix(rancherVersion, "v") + + commonArgs := []string{ + "install", "rancher", chart, + "--namespace", namespace, + "--version", version, + "--set", fmt.Sprintf("hostname=%s", hostname), + "--set", "replicas=2", + "--set", fmt.Sprintf("bootstrapPassword=%s", password), + "--set", "global.cattle.psp.enabled=false", + "--set", "insecure=true", + "--wait", + "--timeout=10m", + "--create-namespace", + "--devel", + } + + if strings.Contains(helmRepoURL, "releases.rancher.com") { + e2e.Logf("Installing Rancher using official release chart...") + } else { + e2e.Logf("Installing Rancher using SUSE private registry chart...") + extraArgs := []string{ + "--set", fmt.Sprintf("rancherImageTag=%s", rancherVersion), + "--set", "rancherImage=stgregistry.suse.com/rancher/rancher", + "--set", "rancherImagePullPolicy=Always", + "--set", "extraEnv[0].name=CATTLE_AGENT_IMAGE", + "--set", fmt.Sprintf("extraEnv[0].value=stgregistry.suse.com/rancher/rancher-agent:%s", rancherVersion), + } + commonArgs = append(commonArgs, extraArgs...) + } + + output, err := helm.Execute(kubeconfig, commonArgs...) + if err != nil { + return fmt.Errorf("helm install failed: %w\nOutput: %s", err, output) + } + + e2e.Logf("Rancher installed successfully: %s", output) + return nil +} diff --git a/resources/terraform/config/main.tf b/resources/terraform/config/main.tf index 2206f8e..043922c 100644 --- a/resources/terraform/config/main.tf +++ b/resources/terraform/config/main.tf @@ -23,10 +23,14 @@ module "ec2" { vpc_id = module.vpc.vpc_id security_group_id = module.vpc.security_group_id private_key_path = var.private_key_path - preserve_eip = false + preserve_eip = var.preserve_eip rke2_version = var.rke2_version cert_manager_version = var.cert_manager_version encryption_secret_key = var.encryption_secret_key input_cluster_config = var.input_cluster_config cattle_config = var.cattle_config + rancher_password = var.rancher_password + rancher_version = var.rancher_version + rancher_repo_url = var.rancher_repo_url + install_rancher = var.install_rancher } diff --git a/resources/terraform/config/outputs.tf b/resources/terraform/config/outputs.tf new file mode 100644 index 0000000..5dd8563 --- /dev/null +++ b/resources/terraform/config/outputs.tf @@ -0,0 +1,15 @@ +output "ec2_public_ip" { + value = module.ec2.public_ip +} + +output "vpc_id" { + value = module.vpc.vpc_id +} + +output "subnet_id" { + value = module.vpc.subnet_id +} + +output "s3_bucket_name" { + value = module.s3.s3_bucket_name +} diff --git a/resources/terraform/config/terraform.safe.auto.tfvars b/resources/terraform/config/terraform.safe.auto.tfvars index 3a72c53..b916e45 100644 --- a/resources/terraform/config/terraform.safe.auto.tfvars +++ b/resources/terraform/config/terraform.safe.auto.tfvars @@ -1,10 +1,12 @@ -prefix = "auto-backup-restore-test" -aws_region_instance = "us-east-2" -aws_region_s3 = "us-east-2" -vpc_cidr = "10.0.0.0/16" -subnet_cidr = "10.0.0.0/24" -aws_zone = "us-east-2a" -ami_id = "ami-00eb69d236edcfaf8" -instance_type = "t2.2xlarge" -private_key_path = "~/.ssh/id_rsa" -root_volume_size = 60 +prefix = "auto-backup-restore-test" +aws_region_instance = "us-east-2" +aws_region_s3 = "us-east-2" +vpc_cidr = "10.0.0.0/16" +subnet_cidr = "10.0.0.0/24" +aws_zone = "us-east-2a" +ami_id = "ami-00eb69d236edcfaf8" +instance_type = "t2.2xlarge" +private_key_path = "~/.ssh/id_rsa" +root_volume_size = 60 +cattle_config = "./../../../cattle-config.yaml" +input_cluster_config = "./../../../tests/helper/yamls/inputClusterConfig.yaml" diff --git a/resources/terraform/config/variables.tf b/resources/terraform/config/variables.tf index 76bb4ee..4804f4d 100644 --- a/resources/terraform/config/variables.tf +++ b/resources/terraform/config/variables.tf @@ -44,6 +44,7 @@ variable "key_name" { description = "Key pair name for EC2" type = string default = "test" + sensitive = true } variable "private_key_path" { @@ -66,7 +67,7 @@ variable "prefix" { variable "rke2_version" { description = "RKE2 version to install" type = string - default = "v1.32.2+rke2r1" + default = "v1.32.5+rke2r1" } variable "cert_manager_version" { @@ -96,5 +97,24 @@ variable "input_cluster_config" { variable "preserve_eip" { description = "create the static eip and attach that to instance for migration scenario" type = bool - default = false + default = true +} +variable "rancher_version" { + description = "version of rancher under test" +} + +variable "rancher_password" { + description = "Bootstrap password for Rancher" + type = string + sensitive = true +} +variable "rancher_repo_url" { + description = "Helm repository URL to install Rancher" + type = string +} + +variable "install_rancher" { + type = bool + default = true + description = "Whether to install Rancher after installing RKE2" } diff --git a/resources/terraform/modules/ec2/encryption-provider-config.tpl b/resources/terraform/modules/ec2/encryption-provider-config.tpl index 1a69aeb..6b5a5e1 100644 --- a/resources/terraform/modules/ec2/encryption-provider-config.tpl +++ b/resources/terraform/modules/ec2/encryption-provider-config.tpl @@ -3,9 +3,11 @@ kind: EncryptionConfiguration resources: - resources: - secrets + - configmaps providers: - aescbc: keys: - name: key1 secret: "${encryption_secret_key}" - - identity: {} + - identity: {} # this fallback allows reading unencrypted secrets; + # for example, during initial migration diff --git a/resources/terraform/modules/ec2/install_rancher.sh b/resources/terraform/modules/ec2/install_rancher.sh new file mode 100644 index 0000000..6ad6522 --- /dev/null +++ b/resources/terraform/modules/ec2/install_rancher.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +set -euxo pipefail + +# Positional args + +RANCHER_VERSION="${1}" +RANCHER_PASSWORD="${2}" +HELM_REPO_URL="${3}" +INSTALL_RANCHER="${4:-true}" # Default to true if not provided + +if [[ "$INSTALL_RANCHER" != "true" ]]; then + echo "⏭️ Skipping Rancher installation because INSTALL_RANCHER=$INSTALL_RANCHER" + exit 0 +fi + +# Proceed with Rancher installation +echo "📦 Installing Rancher version: $RANCHER_VERSION" +echo "🔐 Using password: [REDACTED]" +echo "📦 Helm repo: $HELM_REPO_URL" + +# Add Helm repo for Rancher +helm repo add rancher "$HELM_REPO_URL" +helm repo update + +kubectl create namespace cattle-system || true + +# Get public IP and set hostname +PUBLIC_IP=$(curl -s ifconfig.me) +RANCHER_HOSTNAME="rancher.${PUBLIC_IP}.sslip.io" + +# Install Rancher +if echo "$HELM_REPO_URL" | grep -q "releases.rancher.com"; then + echo "📦 Installing Rancher using official release chart..." + helm install rancher rancher/rancher --namespace cattle-system \ + --version "$(echo "$RANCHER_VERSION" | tr -d 'v')" \ + --set hostname=$RANCHER_HOSTNAME \ + --set replicas=2 \ + --set bootstrapPassword=$RANCHER_PASSWORD \ + --set global.cattle.psp.enabled=false \ + --set insecure=true \ + --wait \ + --timeout=10m \ + --create-namespace \ + --devel +else + echo "📦 Installing Rancher using SUSE private registry chart..." + helm install rancher rancher/rancher --namespace cattle-system \ + --version "$(echo "$RANCHER_VERSION" | tr -d 'v')" \ + --set hostname=$RANCHER_HOSTNAME \ + --set replicas=2 \ + --set bootstrapPassword="$RANCHER_PASSWORD" \ + --set global.cattle.psp.enabled=false \ + --set insecure=true \ + --set rancherImageTag="$RANCHER_VERSION" \ + --set rancherImage='stgregistry.suse.com/rancher/rancher' \ + --set rancherImagePullPolicy=Always \ + --set extraEnv[0].name=CATTLE_AGENT_IMAGE \ + --set extraEnv[0].value="stgregistry.suse.com/rancher/rancher-agent:$RANCHER_VERSION" \ + --wait \ + --timeout=10m \ + --create-namespace \ + --devel +fi + +# Wait for Rancher to start +sleep 180 + +# Post-install setup +RANCHER_URL="https://${RANCHER_HOSTNAME}" +echo "::add-mask::$RANCHER_PASSWORD" + +LOGIN_RESPONSE=$(curl --silent -X POST -H 'Content-Type: application/json' \ + -d "{\"username\":\"admin\",\"password\":\"${RANCHER_PASSWORD}\"}" \ + "${RANCHER_URL}/v3-public/localProviders/local?action=login" \ + --insecure) + +TOKEN=$(echo "$LOGIN_RESPONSE" | jq -r .token) +echo "::add-mask::$TOKEN" + +if [[ -z "$TOKEN" || "$TOKEN" == "null" ]]; then + echo "❌ Failed to login with admin password" >&2 + exit 1 +fi + +# Accept telemetry +curl --silent -X PUT -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' \ + -d '{"name":"telemetry-opt","value":"out"}' \ + "${RANCHER_URL}/v3/settings/telemetry-opt" --insecure + +# Mark first login complete +curl --silent -X PUT -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' \ + -d '{"value":"false"}' \ + "${RANCHER_URL}/v3/settings/first-login" --insecure + +# Set Rancher server URL +curl --silent -X PUT -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' \ + -d "{\"name\":\"server-url\",\"value\":\"${RANCHER_URL}\"}" \ + "${RANCHER_URL}/v3/settings/server-url" --insecure + +echo "✅ Rancher installation and configuration complete." diff --git a/resources/terraform/modules/ec2/install_rke2.sh b/resources/terraform/modules/ec2/install_rke2.sh index 3f64b9a..00f241d 100644 --- a/resources/terraform/modules/ec2/install_rke2.sh +++ b/resources/terraform/modules/ec2/install_rke2.sh @@ -1,47 +1,57 @@ #!/bin/bash -# Define default values -DEFAULT_RKE2_VERSION="v1.32.2+rke2r1" -DEFAULT_CERT_MANAGER_VERSION="v1.15.3" -DEFAULT_HELM_REPO_NAME="rancher" -DEFAULT_HELM_REPO_URL="https://releases.rancher.com/server-charts/latest" +set -euxo pipefail -# Get inputs or use defaults -RKE2_VERSION="${1:-$DEFAULT_RKE2_VERSION}" -CERT_MANAGER_VERSION="${2:-$DEFAULT_CERT_MANAGER_VERSION}" -HELM_REPO_URL="${3:-$DEFAULT_HELM_REPO_URL}" +RKE2_VERSION="${1}" +CERT_MANAGER_VERSION="${2}" +HELM_REPO_URL="${3}" echo "🚀 Installing RKE2 version: $RKE2_VERSION" echo "🔐 Installing Cert Manager version: $CERT_MANAGER_VERSION" -echo "📦 Using Helm repo URL: $HELM_REPO_URL" + +sudo apt-get update -qq && sudo apt-get install -y -qq jq curl +sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 && sudo chmod +x /usr/local/bin/yq + # Install RKE2 curl -sfL https://get.rke2.io | INSTALL_RKE2_VERSION=$RKE2_VERSION sh - -systemctl enable --now rke2-server.service -systemctl restart rke2-server +sudo systemctl enable --now rke2-server.service +sudo systemctl restart rke2-server -# Configure kubectl +# Wait a bit to ensure RKE2 starts up and generates kubeconfig +sleep 10 + +# Give permissions so Terraform can copy it +cat /etc/rancher/rke2/rke2.yaml +cp /etc/rancher/rke2/rke2.yaml /tmp/ +sudo chown ubuntu:ubuntu /tmp/rke2.yaml + +### 🔧 Patch kubeconfig with external IP +EXTERNAL_IP=$(curl -s ifconfig.me) +sudo sed -i "s/127.0.0.1/${EXTERNAL_IP}/" /tmp/rke2.yaml +yq e '.clusters[].cluster |= {"server": .server, "insecure-skip-tls-verify": true}' -i /tmp/rke2.yaml + +# Configure kubectl for current user (ubuntu) mkdir -p ~/.kube ln -sf /etc/rancher/rke2/rke2.yaml ~/.kube/config -ln -sf /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/ +ln -sf /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/kubectl # Install Helm -echo "📦 Installing Helm..." curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 -chmod 700 get_helm.sh +chmod +x get_helm.sh ./get_helm.sh rm -f get_helm.sh -# Add Rancher Helm repo (with default name 'rancher') -echo "📌 Adding Helm repo '$DEFAULT_HELM_REPO_NAME' -> $HELM_REPO_URL" -helm repo add "$DEFAULT_HELM_REPO_NAME" "$HELM_REPO_URL" +# Add cert-manager repo and install +helm repo add jetstack https://charts.jetstack.io helm repo update -# Install Cert Manager -echo "🔧 Installing Cert Manager version: $CERT_MANAGER_VERSION" +# Add Helm repo for Rancher +helm repo add rancher "$HELM_REPO_URL" +helm repo update + +# Install cert-manager kubectl apply -f "https://github.com/cert-manager/cert-manager/releases/download/$CERT_MANAGER_VERSION/cert-manager.yaml" -# Create Rancher namespace -kubectl create namespace cattle-system - -echo "✅ Installation complete! RKE2 and Rancher Helm repo is set up." +echo "✅ RKE2 and Cert Manager installed. Wait ~60 seconds before installing Rancher." +sleep 60 diff --git a/resources/terraform/modules/ec2/main.tf b/resources/terraform/modules/ec2/main.tf index 4d9b1f0..3b91f94 100644 --- a/resources/terraform/modules/ec2/main.tf +++ b/resources/terraform/modules/ec2/main.tf @@ -46,6 +46,7 @@ locals { resource "null_resource" "provision_rke2" { depends_on = [aws_instance.rke2_node] + # Transfer install_rke2.sh provisioner "file" { source = "${path.module}/install_rke2.sh" destination = "/home/ubuntu/install_rke2.sh" @@ -58,6 +59,19 @@ resource "null_resource" "provision_rke2" { } } + # Transfer install_rancher.sh + provisioner "file" { + source = "${path.module}/install_rancher.sh" + destination = "/home/ubuntu/install_rancher.sh" + + connection { + type = "ssh" + user = "ubuntu" + private_key = file(var.private_key_path) + host = local.rke2_host_ip + } + } + provisioner "file" { source = local_file.encrypted_config.filename destination = "/home/ubuntu/encryption-provider-config.yaml" @@ -70,10 +84,12 @@ resource "null_resource" "provision_rke2" { } } + # Run scripts sequentially provisioner "remote-exec" { inline = [ - "chmod +x /home/ubuntu/install_rke2.sh", - "sudo /home/ubuntu/install_rke2.sh ${var.rke2_version} ${var.cert_manager_version}" + "chmod +x /home/ubuntu/install_rke2.sh /home/ubuntu/install_rancher.sh", + "sudo -i bash /home/ubuntu/install_rke2.sh '${var.rke2_version}' '${var.cert_manager_version}' '${var.rancher_repo_url}'", + "sudo -i bash /home/ubuntu/install_rancher.sh '${var.rancher_version}' '${var.rancher_password}' '${var.rancher_repo_url}' '${var.install_rancher}'" ] connection { @@ -85,6 +101,27 @@ resource "null_resource" "provision_rke2" { } } +resource "null_resource" "copy_kubeconfig" { + depends_on = [null_resource.provision_rke2] + + provisioner "local-exec" { + command = "scp -i ${var.private_key_path} -o StrictHostKeyChecking=no ubuntu@${local.rke2_host_ip}:/tmp/rke2.yaml ./rke2-kubeconfig.yaml" + } +} + +resource "null_resource" "move_kubeconfig_local" { + depends_on = [null_resource.copy_kubeconfig] + + provisioner "local-exec" { + command = <