539 lines
14 KiB
Go
539 lines
14 KiB
Go
package tests
|
|
|
|
import (
|
|
"cmp"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
|
|
ec2 "github.com/aws/aws-sdk-go/service/ec2"
|
|
"github.com/google/go-github/v53/github"
|
|
aws "github.com/gruntwork-io/terratest/modules/aws"
|
|
g "github.com/gruntwork-io/terratest/modules/git"
|
|
"github.com/gruntwork-io/terratest/modules/random"
|
|
"github.com/gruntwork-io/terratest/modules/shell"
|
|
"github.com/gruntwork-io/terratest/modules/ssh"
|
|
"github.com/gruntwork-io/terratest/modules/terraform"
|
|
"golang.org/x/oauth2"
|
|
)
|
|
|
|
func GetRancherReleases() (string, string, string, error) {
|
|
releases, err := getReleases("rancher", "rancher")
|
|
if err != nil {
|
|
return "", "", "", err
|
|
}
|
|
filterPrerelease(&releases)
|
|
filterPrimeOnly(&releases)
|
|
versions := getVersionsFromReleases(&releases)
|
|
if len(versions) == 0 {
|
|
return "", "", "", errors.New("no eligible versions found")
|
|
}
|
|
zeroPadVersionNumbers(&versions)
|
|
sortVersions(&versions)
|
|
filterDuplicateMinors(&versions)
|
|
removeZeroPadding(&versions)
|
|
latest := versions[0]
|
|
stable := latest
|
|
lts := stable
|
|
if len(versions) > 1 {
|
|
stable = versions[1]
|
|
}
|
|
if len(versions) > 2 {
|
|
lts = versions[2]
|
|
}
|
|
return latest, stable, lts, nil
|
|
}
|
|
|
|
func GetRke2Releases() (string, string, string, error) {
|
|
releases, err := getReleases("rancher", "rke2")
|
|
if err != nil {
|
|
return "", "", "", err
|
|
}
|
|
filterPrerelease(&releases)
|
|
versions := getVersionsFromReleases(&releases)
|
|
if len(versions) == 0 {
|
|
return "", "", "", errors.New("no eligible versions found")
|
|
}
|
|
zeroPadVersionNumbers(&versions)
|
|
sortVersions(&versions)
|
|
filterDuplicateMinors(&versions)
|
|
removeZeroPadding(&versions)
|
|
latest := versions[0]
|
|
stable := latest
|
|
lts := stable
|
|
if len(versions) > 1 {
|
|
stable = versions[1]
|
|
}
|
|
if len(versions) > 2 {
|
|
lts = versions[2]
|
|
}
|
|
return latest, stable, lts, nil
|
|
}
|
|
|
|
func getReleases(org string, repo string) ([]*github.RepositoryRelease, error) {
|
|
githubToken := os.Getenv("GITHUB_TOKEN")
|
|
if githubToken == "" {
|
|
fmt.Println("GITHUB_TOKEN environment variable not set")
|
|
return nil, errors.New("GITHUB_TOKEN environment variable not set")
|
|
}
|
|
|
|
// Create a new OAuth2 token using the GitHub token
|
|
tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: githubToken})
|
|
tokenClient := oauth2.NewClient(context.Background(), tokenSource)
|
|
|
|
// Create a new GitHub client using the authenticated HTTP client
|
|
client := github.NewClient(tokenClient)
|
|
|
|
var releases []*github.RepositoryRelease
|
|
releases, _, err := client.Repositories.ListReleases(context.Background(), org, repo, &github.ListOptions{})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return releases, nil
|
|
}
|
|
|
|
func filterPrimeOnly(r *[]*github.RepositoryRelease) {
|
|
var fr []*github.RepositoryRelease
|
|
releases := *r
|
|
for i := 0; i < len(releases); i++ {
|
|
if len(releases[i].Assets) > 2 { // source zip and tar are always there
|
|
// prime only releases won't have artifacts
|
|
// so we only add releases with more than 2 artifacts
|
|
fr = append(fr, releases[i])
|
|
}
|
|
}
|
|
*r = fr
|
|
}
|
|
|
|
// this effectively removes release candidates as well as pending releases
|
|
func filterPrerelease(r *[]*github.RepositoryRelease) {
|
|
var fr []*github.RepositoryRelease
|
|
releases := *r
|
|
for i := 0; i < len(releases); i++ {
|
|
if !releases[i].GetPrerelease() {
|
|
fr = append(fr, releases[i])
|
|
}
|
|
}
|
|
*r = fr
|
|
}
|
|
|
|
func getVersionsFromReleases(r *[]*github.RepositoryRelease) []string {
|
|
var versions []string
|
|
releases := *r
|
|
for i := 0; i < len(releases); i++ {
|
|
versions = append(versions, *releases[i].TagName)
|
|
}
|
|
return versions
|
|
// [
|
|
// "v1.28.14+rke2r1",
|
|
// "v1.30.1+rke2r3",
|
|
// "v1.29.4+rke2r1",
|
|
// "v1.30.1+rke2r2",
|
|
// "v1.29.5+rke2r2",
|
|
// "v1.30.1+rke2r1",
|
|
// "v1.27.20+rke2r1",
|
|
// "v1.30.0+rke2r1",
|
|
// "v1.29.5+rke2r1",
|
|
// "v1.28.17+rke2r1",
|
|
// ]
|
|
}
|
|
|
|
func sortVersions(v *[]string) {
|
|
slices.SortFunc(*v, func(a, b string) int {
|
|
return cmp.Compare(b, a)
|
|
//[
|
|
// v1.4.1+rke2r3,
|
|
// v1.30.1+rke2r3,
|
|
// v1.30.1+rke2r2,
|
|
// v1.30.1+rke2r1,
|
|
// v1.30.0+rke2r1,
|
|
// v1.29.5+rke2r2,
|
|
// v1.29.5+rke2r1,
|
|
// v1.29.4+rke2r1,
|
|
// v1.28.17+rke2r1,
|
|
// v1.28.14+rke2r1,
|
|
// v1.27.20+rke2r1,
|
|
//]
|
|
})
|
|
}
|
|
|
|
func filterDuplicateMinors(v *[]string) { // assumes versions are sorted already
|
|
var fv []string
|
|
versions := *v
|
|
fv = append(fv, versions[0])
|
|
for i := 1; i < len(versions); i++ {
|
|
c := versions[i]
|
|
p := versions[i-1]
|
|
cp := strings.Split(c[1:], "+") //["1.30.1","rke2r3"]
|
|
pp := strings.Split(p[1:], "+") //["1.30.1","rke2r2"]
|
|
if cp[0] != pp[0] {
|
|
cpp := strings.Split(cp[0], ".") //["1","30","1]
|
|
ppp := strings.Split(pp[0], ".") //["1","30","1]
|
|
if cpp[1] != ppp[1] {
|
|
fv = append(fv, c)
|
|
//[
|
|
// v1.30.1+rke2r3,
|
|
// v1.29.5+rke2r2,
|
|
// v1.28.17+rke2r1,
|
|
// v1.27.20+rke2r1,
|
|
//]
|
|
}
|
|
}
|
|
}
|
|
*v = fv
|
|
}
|
|
|
|
func zeroPadVersionNumbers(v *[]string) {
|
|
var zv []string
|
|
versions := *v
|
|
for i := 0; i < len(versions); i++ {
|
|
vp := strings.Split(versions[i], "+") //["v1.3.1","rke2r3"] OR ["v2.5.4"] if no "+"
|
|
vpp := strings.Split(vp[0], ".") //["v1","3","1]
|
|
major := vpp[0] // assumes single digit major
|
|
minor := ""
|
|
trivial := ""
|
|
if len(vpp[1]) < 2 {
|
|
minor = fmt.Sprintf("0%s", vpp[1]) // assumes double digit versions
|
|
} else {
|
|
minor = vpp[1]
|
|
}
|
|
if len(vpp[2]) < 2 {
|
|
trivial = fmt.Sprintf("0%s", vpp[2]) // assumes double digit versions
|
|
} else {
|
|
trivial = vpp[2]
|
|
}
|
|
if len(vp) > 1 {
|
|
version := fmt.Sprintf("%s.%s.%s+%s", major, minor, trivial, vp[1]) //"v1.03.01+rke2r3"
|
|
zv = append(zv, version)
|
|
} else {
|
|
version := fmt.Sprintf("%s.%s.%s", major, minor, trivial) //"v1.03.01"
|
|
zv = append(zv, version)
|
|
}
|
|
}
|
|
*v = zv
|
|
}
|
|
|
|
func removeZeroPadding(v *[]string) {
|
|
var zv []string
|
|
versions := *v
|
|
for i := 0; i < len(versions); i++ {
|
|
vp := strings.Split(versions[i], "+") //["v1.03.01","rke2r3"] OR ["v2.05.04"] if no "+"
|
|
vpp := strings.Split(vp[0], ".") //["v1","03","01]
|
|
major := vpp[0] // assumes single digit major
|
|
minor := vpp[1]
|
|
trivial := vpp[2]
|
|
if minor[0] == '0' {
|
|
minor = minor[1:]
|
|
}
|
|
if trivial[0] == '0' {
|
|
trivial = trivial[1:]
|
|
}
|
|
if len(vp) > 1 {
|
|
version := fmt.Sprintf("%s.%s.%s+%s", major, minor, trivial, vp[1]) //"v1.3.1+rke2r3"
|
|
zv = append(zv, version)
|
|
} else {
|
|
version := fmt.Sprintf("%s.%s.%s", major, minor, trivial) //"v1.3.1"
|
|
zv = append(zv, version)
|
|
}
|
|
}
|
|
*v = zv
|
|
}
|
|
|
|
func CreateKeypair(t *testing.T, region string, owner string, id string) (*aws.Ec2Keypair, error) {
|
|
t.Log("Creating keypair...")
|
|
// Create an EC2 KeyPair that we can use for SSH access
|
|
keyPairName := fmt.Sprintf("terraform-ci-%s", id)
|
|
keyPair := aws.CreateAndImportEC2KeyPair(t, region, keyPairName)
|
|
|
|
// tag the key pair so we can find in the access module
|
|
client, err := aws.NewEc2ClientE(t, region)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
k := "key-name"
|
|
keyNameFilter := ec2.Filter{
|
|
Name: &k,
|
|
Values: []*string{&keyPairName},
|
|
}
|
|
input := &ec2.DescribeKeyPairsInput{
|
|
Filters: []*ec2.Filter{&keyNameFilter},
|
|
}
|
|
result, err := client.DescribeKeyPairs(input)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
err = aws.AddTagsToResourceE(t, region, *result.KeyPairs[0].KeyPairId, map[string]string{"Name": keyPairName, "Owner": owner})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Verify that the name and owner tags were placed properly
|
|
k = "tag:Name"
|
|
keyNameFilter = ec2.Filter{
|
|
Name: &k,
|
|
Values: []*string{&keyPairName},
|
|
}
|
|
input = &ec2.DescribeKeyPairsInput{
|
|
Filters: []*ec2.Filter{&keyNameFilter},
|
|
}
|
|
result, err = client.DescribeKeyPairs(input)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
k = "tag:Owner"
|
|
keyNameFilter = ec2.Filter{
|
|
Name: &k,
|
|
Values: []*string{&owner},
|
|
}
|
|
input = &ec2.DescribeKeyPairsInput{
|
|
Filters: []*ec2.Filter{&keyNameFilter},
|
|
}
|
|
result, err = client.DescribeKeyPairs(input)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return keyPair, nil
|
|
}
|
|
|
|
func GetRetryableTerraformErrors() map[string]string {
|
|
retryableTerraformErrors := map[string]string{
|
|
// The reason is unknown, but eventually these succeed after a few retries.
|
|
".*unable to verify signature.*": "Failed due to transient network error.",
|
|
".*unable to verify checksum.*": "Failed due to transient network error.",
|
|
".*no provider exists with the given name.*": "Failed due to transient network error.",
|
|
".*registry service is unreachable.*": "Failed due to transient network error.",
|
|
".*connection reset by peer.*": "Failed due to transient network error.",
|
|
".*TLS handshake timeout.*": "Failed due to transient network error.",
|
|
".*context deadline exceeded.*": "Failed due to kubernetes timeout, retrying.",
|
|
".*http2: client connection lost.*": "Failed due to transient network error.",
|
|
}
|
|
return retryableTerraformErrors
|
|
}
|
|
|
|
func SetAcmeServer() string {
|
|
acmeserver := os.Getenv("ACME_SERVER_URL")
|
|
if acmeserver == "" {
|
|
os.Setenv("ACME_SERVER_URL", "https://acme-staging-v02.api.letsencrypt.org/directory")
|
|
}
|
|
return acmeserver
|
|
}
|
|
|
|
func GetRegion() string {
|
|
region := os.Getenv("AWS_REGION")
|
|
if region == "" {
|
|
region = os.Getenv("AWS_DEFAULT_REGION")
|
|
}
|
|
if region == "" {
|
|
region = "us-west-2"
|
|
}
|
|
return region
|
|
}
|
|
|
|
func GetAwsAccessKey() string {
|
|
key := os.Getenv("AWS_ACCESS_KEY_ID")
|
|
if key == "" {
|
|
key = "FAKE123-ABC"
|
|
}
|
|
return key
|
|
}
|
|
|
|
func GetAwsSecretKey() string {
|
|
secret := os.Getenv("AWS_SECRET_ACCESS_KEY")
|
|
if secret == "" {
|
|
secret = "FAKE123-ABC"
|
|
}
|
|
return secret
|
|
}
|
|
|
|
func GetAwsSessionToken() string {
|
|
return os.Getenv("AWS_SESSION_TOKEN")
|
|
}
|
|
|
|
func GetId() string {
|
|
id := os.Getenv("IDENTIFIER")
|
|
if id == "" {
|
|
id = random.UniqueId()
|
|
}
|
|
id += "-" + random.UniqueId()
|
|
return id
|
|
}
|
|
|
|
func CreateTestDirectories(t *testing.T, id string) error {
|
|
gwd := g.GetRepoRoot(t)
|
|
fwd, err := filepath.Abs(gwd)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
paths := []string{
|
|
filepath.Join(fwd, "test/tests/data"),
|
|
filepath.Join(fwd, "test/tests/data", id),
|
|
filepath.Join(fwd, "test/tests/data", id, "backend"),
|
|
filepath.Join(fwd, "test/tests/data", id, "data"),
|
|
}
|
|
for _, path := range paths {
|
|
err = os.Mkdir(path, 0755)
|
|
if err != nil && !os.IsExist(err) {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func Teardown(t *testing.T, dataDir string, exampleDir string, options []*terraform.Options, keyPair *aws.Ec2Keypair, agent *ssh.SshAgent) {
|
|
directoryExists := true
|
|
_, err := os.Stat(dataDir)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
directoryExists = false
|
|
}
|
|
}
|
|
if directoryExists {
|
|
for _, option := range options {
|
|
t.Logf("Tearing down %v", option.TerraformDir)
|
|
jsonOptions, err := json.Marshal(option)
|
|
if err != nil {
|
|
t.Logf("Failed to marshal options for destroy log: %v", err)
|
|
}
|
|
fmt.Println(string(jsonOptions))
|
|
_, err = terraform.InitE(t, option)
|
|
if err != nil {
|
|
t.Logf("Failed to init for destroy: %v", err)
|
|
}
|
|
_, err = terraform.DestroyE(t, option)
|
|
if err != nil {
|
|
t.Logf("Failed to destroy: %v", err)
|
|
}
|
|
}
|
|
err = os.RemoveAll(dataDir)
|
|
if err != nil {
|
|
t.Logf("Failed to delete test data directory: %v", err)
|
|
}
|
|
}
|
|
agent.Stop()
|
|
err = aws.DeleteEC2KeyPairE(t, keyPair)
|
|
if err != nil {
|
|
t.Logf("Failed to destroy key pair: %v", err)
|
|
}
|
|
os.Remove(exampleDir + "/.terraform.lock.hcl")
|
|
}
|
|
|
|
func GetErrorLogs(t *testing.T, kubeconfigPath string) {
|
|
repoRoot, err := filepath.Abs(g.GetRepoRoot(t))
|
|
if err != nil {
|
|
t.Logf("Error getting git root directory: %v", err)
|
|
}
|
|
script, err := os.ReadFile(repoRoot + "/test/scripts/getLogs.sh")
|
|
if err != nil {
|
|
t.Logf("Error reading script: %v", err)
|
|
}
|
|
errorLogsScript := shell.Command{
|
|
Command: "bash",
|
|
Args: []string{"-c", string(script)},
|
|
Env: map[string]string{
|
|
"KUBECONFIG": kubeconfigPath,
|
|
},
|
|
}
|
|
out, err := shell.RunCommandAndGetOutputE(t, errorLogsScript)
|
|
if err != nil {
|
|
t.Logf("Error running script: %s", err)
|
|
}
|
|
t.Logf("Log script output: %s", out)
|
|
}
|
|
|
|
func CheckReady(t *testing.T, kubeconfigPath string) {
|
|
repoRoot, err := filepath.Abs(g.GetRepoRoot(t))
|
|
if err != nil {
|
|
t.Logf("Error getting git root directory: %v", err)
|
|
t.Fail()
|
|
}
|
|
script, err := os.ReadFile(repoRoot + "/test/scripts/readyNodes.sh")
|
|
if err != nil {
|
|
t.Logf("Error reading script: %v", err)
|
|
t.Fail()
|
|
}
|
|
readyScript := shell.Command{
|
|
Command: "bash",
|
|
Args: []string{"-c", string(script)},
|
|
Env: map[string]string{
|
|
"KUBECONFIG": kubeconfigPath,
|
|
},
|
|
}
|
|
out, err := shell.RunCommandAndGetOutputE(t, readyScript)
|
|
if err != nil {
|
|
t.Logf("Error running script: %s", err)
|
|
t.Fail()
|
|
}
|
|
t.Logf("Ready script output: %s", out)
|
|
}
|
|
|
|
func CheckRunning(t *testing.T, kubeconfigPath string) {
|
|
repoRoot, err := filepath.Abs(g.GetRepoRoot(t))
|
|
if err != nil {
|
|
t.Logf("Error getting git root directory: %v", err)
|
|
t.Fail()
|
|
}
|
|
script, err := os.ReadFile(repoRoot + "/test/scripts/runningPods.sh")
|
|
if err != nil {
|
|
t.Logf("Error reading script: %v", err)
|
|
t.Fail()
|
|
}
|
|
readyScript := shell.Command{
|
|
Command: "bash",
|
|
Args: []string{"-c", string(script)},
|
|
Env: map[string]string{
|
|
"KUBECONFIG": kubeconfigPath,
|
|
},
|
|
}
|
|
out, err := shell.RunCommandAndGetOutputE(t, readyScript)
|
|
if err != nil {
|
|
t.Logf("Error running script: %s", err)
|
|
t.Fail()
|
|
}
|
|
t.Logf("Ready script output: %s", out)
|
|
}
|
|
|
|
func CreateObjectStorageBackend(t *testing.T, testDir string, id string, owner string, region string) (*terraform.Options, error) {
|
|
repoRoot, err := filepath.Abs(g.GetRepoRoot(t))
|
|
if err != nil {
|
|
t.Fatalf("Error getting git root directory: %v", err)
|
|
}
|
|
exampleDir := repoRoot + "/examples/backend_s3"
|
|
|
|
terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{
|
|
TerraformDir: exampleDir,
|
|
// Variables to pass to our Terraform code using -var options
|
|
Vars: map[string]interface{}{
|
|
"identifier": id,
|
|
"owner": owner,
|
|
},
|
|
// Environment variables to set when running Terraform
|
|
EnvVars: map[string]string{
|
|
"AWS_DEFAULT_REGION": region,
|
|
"AWS_REGION": region,
|
|
"TF_DATA_DIR": testDir + "/backend",
|
|
"TF_IN_AUTOMATION": "1",
|
|
"TF_CLI_ARGS_plan": "-state=" + testDir + "/backend/tfstate",
|
|
"TF_CLI_ARGS_apply": "-state=" + testDir + "/backend/tfstate",
|
|
"TF_CLI_ARGS_destroy": "-state=" + testDir + "/backend/tfstate",
|
|
"TF_CLI_ARGS_output": "-state=" + testDir + "/backend/tfstate",
|
|
},
|
|
RetryableTerraformErrors: GetRetryableTerraformErrors(),
|
|
Reconfigure: true,
|
|
NoColor: true,
|
|
Upgrade: true,
|
|
})
|
|
|
|
_, err = terraform.InitAndApplyE(t, terraformOptions)
|
|
return terraformOptions, err
|
|
}
|