470 lines
13 KiB
Go
470 lines
13 KiB
Go
package tests
|
|
|
|
import (
|
|
"cmp"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
|
|
ec2 "github.com/aws/aws-sdk-go/service/ec2"
|
|
"github.com/google/go-github/v53/github"
|
|
aws "github.com/gruntwork-io/terratest/modules/aws"
|
|
g "github.com/gruntwork-io/terratest/modules/git"
|
|
"github.com/gruntwork-io/terratest/modules/random"
|
|
"github.com/gruntwork-io/terratest/modules/shell"
|
|
"github.com/gruntwork-io/terratest/modules/terraform"
|
|
"golang.org/x/oauth2"
|
|
)
|
|
|
|
func GetRancherReleases() (string, string, string, error) {
|
|
releases, err := getRancherReleases()
|
|
if err != nil {
|
|
return "", "", "", err
|
|
}
|
|
versions := filterPrerelease(releases)
|
|
if len(versions) == 0 {
|
|
return "", "", "", errors.New("no eligible versions found")
|
|
}
|
|
zeroPadVersionNumbers(&versions)
|
|
sortVersions(&versions)
|
|
filterDuplicateMinors(&versions)
|
|
removeZeroPadding(&versions)
|
|
latest := versions[0]
|
|
stable := latest
|
|
lts := stable
|
|
if len(versions) > 1 {
|
|
stable = versions[1]
|
|
}
|
|
if len(versions) > 2 {
|
|
lts = versions[2]
|
|
}
|
|
return latest, stable, lts, nil
|
|
}
|
|
|
|
func getRancherReleases() ([]*github.RepositoryRelease, error) {
|
|
githubToken := os.Getenv("GITHUB_TOKEN")
|
|
if githubToken == "" {
|
|
fmt.Println("GITHUB_TOKEN environment variable not set")
|
|
return nil, errors.New("GITHUB_TOKEN environment variable not set")
|
|
}
|
|
|
|
// Create a new OAuth2 token using the GitHub token
|
|
tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: githubToken})
|
|
tokenClient := oauth2.NewClient(context.Background(), tokenSource)
|
|
|
|
// Create a new GitHub client using the authenticated HTTP client
|
|
client := github.NewClient(tokenClient)
|
|
|
|
var releases []*github.RepositoryRelease
|
|
releases, _, err := client.Repositories.ListReleases(context.Background(), "rancher", "rancher", &github.ListOptions{})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return releases, nil
|
|
}
|
|
|
|
func GetRke2Releases() (string, string, string, error) {
|
|
releases, err := getRke2Releases()
|
|
if err != nil {
|
|
return "", "", "", err
|
|
}
|
|
versions := filterPrerelease(releases)
|
|
if len(versions) == 0 {
|
|
return "", "", "", errors.New("no eligible versions found")
|
|
}
|
|
zeroPadVersionNumbers(&versions)
|
|
sortVersions(&versions)
|
|
filterDuplicateMinors(&versions)
|
|
removeZeroPadding(&versions)
|
|
latest := versions[0]
|
|
stable := latest
|
|
lts := stable
|
|
if len(versions) > 1 {
|
|
stable = versions[1]
|
|
}
|
|
if len(versions) > 2 {
|
|
lts = versions[2]
|
|
}
|
|
return latest, stable, lts, nil
|
|
}
|
|
|
|
func getRke2Releases() ([]*github.RepositoryRelease, error) {
|
|
githubToken := os.Getenv("GITHUB_TOKEN")
|
|
if githubToken == "" {
|
|
fmt.Println("GITHUB_TOKEN environment variable not set")
|
|
return nil, errors.New("GITHUB_TOKEN environment variable not set")
|
|
}
|
|
|
|
// Create a new OAuth2 token using the GitHub token
|
|
tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: githubToken})
|
|
tokenClient := oauth2.NewClient(context.Background(), tokenSource)
|
|
|
|
// Create a new GitHub client using the authenticated HTTP client
|
|
client := github.NewClient(tokenClient)
|
|
|
|
var releases []*github.RepositoryRelease
|
|
releases, _, err := client.Repositories.ListReleases(context.Background(), "rancher", "rke2", &github.ListOptions{})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return releases, nil
|
|
}
|
|
|
|
func filterReleaseCandidate(v *[]string) {
|
|
var fv []string
|
|
versions := *v
|
|
for i := 1; i < len(versions); i++ {
|
|
if strings.Contains(versions[i], "-") != true {
|
|
fv = append(fv, versions[i])
|
|
}
|
|
}
|
|
*v = fv
|
|
}
|
|
|
|
func filterPrerelease(r []*github.RepositoryRelease) []string {
|
|
var versions []string
|
|
for _, release := range r {
|
|
version := release.GetTagName()
|
|
if !release.GetPrerelease() {
|
|
versions = append(versions, version)
|
|
// [
|
|
// "v1.28.14+rke2r1",
|
|
// "v1.30.1+rke2r3",
|
|
// "v1.29.4+rke2r1",
|
|
// "v1.30.1+rke2r2",
|
|
// "v1.29.5+rke2r2",
|
|
// "v1.30.1+rke2r1",
|
|
// "v1.27.20+rke2r1",
|
|
// "v1.30.0+rke2r1",
|
|
// "v1.29.5+rke2r1",
|
|
// "v1.28.17+rke2r1",
|
|
// ]
|
|
}
|
|
}
|
|
return versions
|
|
}
|
|
|
|
func sortVersions(v *[]string) {
|
|
slices.SortFunc(*v, func(a, b string) int {
|
|
return cmp.Compare(b, a)
|
|
//[
|
|
// v1.4.1+rke2r3,
|
|
// v1.30.1+rke2r3,
|
|
// v1.30.1+rke2r2,
|
|
// v1.30.1+rke2r1,
|
|
// v1.30.0+rke2r1,
|
|
// v1.29.5+rke2r2,
|
|
// v1.29.5+rke2r1,
|
|
// v1.29.4+rke2r1,
|
|
// v1.28.17+rke2r1,
|
|
// v1.28.14+rke2r1,
|
|
// v1.27.20+rke2r1,
|
|
//]
|
|
})
|
|
}
|
|
|
|
func filterDuplicateMinors(v *[]string) { // assumes versions are sorted already
|
|
var fv []string
|
|
versions := *v
|
|
fv = append(fv, versions[0])
|
|
for i := 1; i < len(versions); i++ {
|
|
c := versions[i]
|
|
p := versions[i-1]
|
|
cp := strings.Split(c[1:], "+") //["1.30.1","rke2r3"]
|
|
pp := strings.Split(p[1:], "+") //["1.30.1","rke2r2"]
|
|
if cp[0] != pp[0] {
|
|
cpp := strings.Split(cp[0], ".") //["1","30","1]
|
|
ppp := strings.Split(pp[0], ".") //["1","30","1]
|
|
if cpp[1] != ppp[1] {
|
|
fv = append(fv, c)
|
|
//[
|
|
// v1.30.1+rke2r3,
|
|
// v1.29.5+rke2r2,
|
|
// v1.28.17+rke2r1,
|
|
// v1.27.20+rke2r1,
|
|
//]
|
|
}
|
|
}
|
|
}
|
|
*v = fv
|
|
}
|
|
|
|
func zeroPadVersionNumbers(v *[]string) {
|
|
var zv []string
|
|
versions := *v
|
|
for i := 0; i < len(versions); i++ {
|
|
vp := strings.Split(versions[i], "+") //["v1.3.1","rke2r3"] OR ["v2.5.4"] if no "+"
|
|
vpp := strings.Split(vp[0], ".") //["v1","3","1]
|
|
major := vpp[0] // assumes single digit major
|
|
minor := ""
|
|
trivial := ""
|
|
if len(vpp[1]) < 2 {
|
|
minor = fmt.Sprintf("0%s", vpp[1]) // assumes double digit versions
|
|
} else {
|
|
minor = vpp[1]
|
|
}
|
|
if len(vpp[2]) < 2 {
|
|
trivial = fmt.Sprintf("0%s", vpp[2]) // assumes double digit versions
|
|
} else {
|
|
trivial = vpp[2]
|
|
}
|
|
if len(vp) > 1 {
|
|
version := fmt.Sprintf("%s.%s.%s+%s", major, minor, trivial, vp[1]) //"v1.03.01+rke2r3"
|
|
zv = append(zv, version)
|
|
} else {
|
|
version := fmt.Sprintf("%s.%s.%s", major, minor, trivial) //"v1.03.01"
|
|
zv = append(zv, version)
|
|
}
|
|
}
|
|
*v = zv
|
|
}
|
|
|
|
func removeZeroPadding (v *[]string) {
|
|
var zv []string
|
|
versions := *v
|
|
for i := 0; i < len(versions); i++ {
|
|
vp := strings.Split(versions[i], "+") //["v1.03.01","rke2r3"] OR ["v2.05.04"] if no "+"
|
|
vpp := strings.Split(vp[0], ".") //["v1","03","01]
|
|
major := vpp[0] // assumes single digit major
|
|
minor := vpp[1]
|
|
trivial := vpp[2]
|
|
if minor[0] == '0' {
|
|
minor = minor[1:]
|
|
}
|
|
if trivial[0] == '0' {
|
|
trivial = trivial[1:]
|
|
}
|
|
if len(vp) > 1 {
|
|
version := fmt.Sprintf("%s.%s.%s+%s", major, minor, trivial, vp[1]) //"v1.3.1+rke2r3"
|
|
zv = append(zv, version)
|
|
} else {
|
|
version := fmt.Sprintf("%s.%s.%s", major, minor, trivial) //"v1.3.1"
|
|
zv = append(zv, version)
|
|
}
|
|
}
|
|
*v = zv
|
|
}
|
|
|
|
func CreateKeypair(t *testing.T, region string, owner string, id string) (*aws.Ec2Keypair, error) {
|
|
t.Log("Creating keypair...")
|
|
// Create an EC2 KeyPair that we can use for SSH access
|
|
keyPairName := fmt.Sprintf("terraform-ci-%s", id)
|
|
keyPair := aws.CreateAndImportEC2KeyPair(t, region, keyPairName)
|
|
|
|
// tag the key pair so we can find in the access module
|
|
client, err := aws.NewEc2ClientE(t, region)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
k := "key-name"
|
|
keyNameFilter := ec2.Filter{
|
|
Name: &k,
|
|
Values: []*string{&keyPairName},
|
|
}
|
|
input := &ec2.DescribeKeyPairsInput{
|
|
Filters: []*ec2.Filter{&keyNameFilter},
|
|
}
|
|
result, err := client.DescribeKeyPairs(input)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
err = aws.AddTagsToResourceE(t, region, *result.KeyPairs[0].KeyPairId, map[string]string{"Name": keyPairName, "Owner": owner})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Verify that the name and owner tags were placed properly
|
|
k = "tag:Name"
|
|
keyNameFilter = ec2.Filter{
|
|
Name: &k,
|
|
Values: []*string{&keyPairName},
|
|
}
|
|
input = &ec2.DescribeKeyPairsInput{
|
|
Filters: []*ec2.Filter{&keyNameFilter},
|
|
}
|
|
result, err = client.DescribeKeyPairs(input)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
k = "tag:Owner"
|
|
keyNameFilter = ec2.Filter{
|
|
Name: &k,
|
|
Values: []*string{&owner},
|
|
}
|
|
input = &ec2.DescribeKeyPairsInput{
|
|
Filters: []*ec2.Filter{&keyNameFilter},
|
|
}
|
|
result, err = client.DescribeKeyPairs(input)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return keyPair, nil
|
|
}
|
|
|
|
func GetRetryableTerraformErrors() map[string]string {
|
|
retryableTerraformErrors := map[string]string{
|
|
// The reason is unknown, but eventually these succeed after a few retries.
|
|
".*unable to verify signature.*": "Failed due to transient network error.",
|
|
".*unable to verify checksum.*": "Failed due to transient network error.",
|
|
".*no provider exists with the given name.*": "Failed due to transient network error.",
|
|
".*registry service is unreachable.*": "Failed due to transient network error.",
|
|
".*connection reset by peer.*": "Failed due to transient network error.",
|
|
".*TLS handshake timeout.*": "Failed due to transient network error.",
|
|
".*context deadline exceeded.*": "Failed due to kubernetes timeout, retrying.",
|
|
".*http2: client connection lost.*": "Failed due to transient network error.",
|
|
}
|
|
return retryableTerraformErrors
|
|
}
|
|
|
|
func SetAcmeServer() string {
|
|
acmeserver := os.Getenv("ACME_SERVER_URL")
|
|
if acmeserver == "" {
|
|
os.Setenv("ACME_SERVER_URL", "https://acme-staging-v02.api.letsencrypt.org/directory")
|
|
}
|
|
return acmeserver
|
|
}
|
|
|
|
func GetRegion() string {
|
|
region := os.Getenv("AWS_REGION")
|
|
if region == "" {
|
|
region = os.Getenv("AWS_DEFAULT_REGION")
|
|
}
|
|
if region == "" {
|
|
region = "us-west-2"
|
|
}
|
|
return region
|
|
}
|
|
|
|
func GetAwsAccessKey() string {
|
|
key := os.Getenv("AWS_ACCESS_KEY_ID")
|
|
if key == "" {
|
|
key = "FAKE123-ABC"
|
|
}
|
|
return key
|
|
}
|
|
|
|
func GetAwsSecretKey() string {
|
|
secret := os.Getenv("AWS_SECRET_ACCESS_KEY")
|
|
if secret == "" {
|
|
secret = "FAKE123-ABC"
|
|
}
|
|
return secret
|
|
}
|
|
|
|
func GetAwsSessionToken() string {
|
|
return os.Getenv("AWS_SESSION_TOKEN")
|
|
}
|
|
|
|
func GetId() string {
|
|
id := os.Getenv("IDENTIFIER")
|
|
if id == "" {
|
|
id = random.UniqueId()
|
|
}
|
|
id += "-" + random.UniqueId()
|
|
return id
|
|
}
|
|
|
|
func CreateTestDirectories(t *testing.T, id string) error {
|
|
gwd := g.GetRepoRoot(t)
|
|
fwd, err := filepath.Abs(gwd)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
tdd := fwd + "/test/tests/data"
|
|
err = os.Mkdir(tdd, 0755)
|
|
if err != nil && !os.IsExist(err) {
|
|
return err
|
|
}
|
|
tdd = fwd + "/test/tests/data/" + id
|
|
err = os.Mkdir(tdd, 0755)
|
|
if err != nil && !os.IsExist(err) {
|
|
return err
|
|
}
|
|
tdd = fwd + "/test/tests/data/" + id + "/data"
|
|
err = os.Mkdir(tdd, 0755)
|
|
if err != nil && !os.IsExist(err) {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func Teardown(t *testing.T, directory string, options *terraform.Options, keyPair *aws.Ec2Keypair) {
|
|
directoryExists := true
|
|
_, err := os.Stat(directory)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
directoryExists = false
|
|
}
|
|
}
|
|
if directoryExists {
|
|
_, err := terraform.DestroyE(t, options)
|
|
if err != nil {
|
|
t.Logf("Failed to destroy: %v", err)
|
|
}
|
|
|
|
err = os.RemoveAll(directory)
|
|
if err != nil {
|
|
t.Logf("Failed to delete test data directory: %v", err)
|
|
}
|
|
}
|
|
aws.DeleteEC2KeyPair(t, keyPair)
|
|
}
|
|
|
|
func GetErrorLogs(t *testing.T, kubeconfigPath string) {
|
|
repoRoot, err := filepath.Abs(g.GetRepoRoot(t))
|
|
if err != nil {
|
|
t.Logf("Error getting git root directory: %v", err)
|
|
}
|
|
script, err := os.ReadFile(repoRoot + "/test/scripts/getLogs.sh")
|
|
if err != nil {
|
|
t.Logf("Error reading script: %v", err)
|
|
}
|
|
errorLogsScript := shell.Command{
|
|
Command: "bash",
|
|
Args: []string{"-c", string(script)},
|
|
Env: map[string]string{
|
|
"KUBECONFIG": kubeconfigPath,
|
|
},
|
|
}
|
|
out, err := shell.RunCommandAndGetOutputE(t, errorLogsScript)
|
|
if err != nil {
|
|
t.Logf("Error running script: %s", err)
|
|
}
|
|
t.Logf("Log script output: %s", out)
|
|
}
|
|
|
|
func CheckReady(t *testing.T, kubeconfigPath string) {
|
|
repoRoot, err := filepath.Abs(g.GetRepoRoot(t))
|
|
if err != nil {
|
|
t.Logf("Error getting git root directory: %v", err)
|
|
t.Fail()
|
|
}
|
|
script, err := os.ReadFile(repoRoot + "/test/scripts/readyNodes.sh")
|
|
if err != nil {
|
|
t.Logf("Error reading script: %v", err)
|
|
t.Fail()
|
|
}
|
|
readyScript := shell.Command{
|
|
Command: "bash",
|
|
Args: []string{"-c", string(script)},
|
|
Env: map[string]string{
|
|
"KUBECONFIG": kubeconfigPath,
|
|
},
|
|
}
|
|
out, err := shell.RunCommandAndGetOutputE(t, readyScript)
|
|
if err != nil {
|
|
t.Logf("Error running script: %s", err)
|
|
t.Fail()
|
|
}
|
|
t.Logf("Ready script output: %s", out)
|
|
}
|