mirror of https://github.com/kubernetes/kops.git
Use ephemeral S3 buckets for E2E tests (#17157)
* Use ephemeral S3 buckets for E2E tests Use S3 buckets created during the lifecycle of a test instead of a static one and provide the capability to make them read-only public. Signed-off-by: Arnaud Meukam <ameukam@gmail.com> * Improve ephemeral S3 buckets implementation for tests Signed-off-by: Marko Mudrinić <mudrinic.mare@gmail.com> Signed-off-by: Arnaud Meukam <ameukam@gmail.com> * Base S3 bucket name on ProwJob ID Signed-off-by: Marko Mudrinić <mudrinic.mare@gmail.com> --------- Signed-off-by: Arnaud Meukam <ameukam@gmail.com> Signed-off-by: Marko Mudrinić <mudrinic.mare@gmail.com> Co-authored-by: Marko Mudrinić <mudrinic.mare@gmail.com>
This commit is contained in:
Arnaud M.
GitHub
parent
4539d80a52
commit
06250623d6
|
|
@ -5,6 +5,10 @@ go 1.23.5
|
|||
replace k8s.io/kops => ../../.
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2 v1.31.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.38
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.63.2
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.31.2
|
||||
github.com/blang/semver/v4 v4.0.0
|
||||
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
|
||||
github.com/octago/sflags v0.2.0
|
||||
|
|
@ -66,9 +70,7 @@ require (
|
|||
github.com/aliyun/credentials-go v1.2.3 // indirect
|
||||
github.com/apparentlymart/go-cidr v1.1.0 // indirect
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.31.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.38 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.36 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 // indirect
|
||||
|
|
@ -82,10 +84,8 @@ require (
|
|||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.63.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.23.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.31.2 // indirect
|
||||
github.com/aws/smithy-go v1.21.0 // indirect
|
||||
github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20220228164355-396b2034c795 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
|
|
|
|||
|
|
@ -0,0 +1,194 @@
|
|||
/*
|
||||
Copyright 2024 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package aws
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/aws-sdk-go-v2/service/sts"
|
||||
"k8s.io/klog/v2"
|
||||
)
|
||||
|
||||
// defaultRegion is the region to query the AWS APIs through, this can be any AWS region is required even if we are not
|
||||
// running on AWS.
|
||||
const defaultRegion = "us-east-2"
|
||||
|
||||
// Client contains S3 and STS clients that are used to perform bucket and object actions.
|
||||
type Client struct {
|
||||
s3Client *s3.Client
|
||||
stsClient *sts.Client
|
||||
}
|
||||
|
||||
// NewAWSClient returns a new instance of awsClient configured to work in the default region (us-east-2).
|
||||
func NewClient(ctx context.Context) (*Client, error) {
|
||||
cfg, err := awsconfig.LoadDefaultConfig(ctx,
|
||||
awsconfig.WithRegion(defaultRegion))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("loading AWS config: %w", err)
|
||||
}
|
||||
|
||||
return &Client{
|
||||
s3Client: s3.NewFromConfig(cfg),
|
||||
stsClient: sts.NewFromConfig(cfg),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// BucketName constructs an unique bucket name using the AWS account ID in the default region (us-east-2).
|
||||
func (c Client) BucketName(ctx context.Context) (string, error) {
|
||||
// Construct the bucket name based on the ProwJob ID (if running in Prow) or AWS account ID (if running outside
|
||||
// Prow) and the current timestamp
|
||||
var identifier string
|
||||
if jobID := os.Getenv("PROW_JOB_ID"); len(jobID) >= 4 {
|
||||
identifier = jobID[:4]
|
||||
} else {
|
||||
callerIdentity, err := c.stsClient.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("building AWS STS presigned request: %w", err)
|
||||
}
|
||||
identifier = *callerIdentity.Account
|
||||
}
|
||||
timestamp := time.Now().Format("20060102150405")
|
||||
bucket := fmt.Sprintf("k8s-infra-kops-%s-%s", identifier, timestamp)
|
||||
|
||||
bucket = strings.ToLower(bucket)
|
||||
// Only allow lowercase letters, numbers, and hyphens
|
||||
bucket = regexp.MustCompile("[^a-z0-9-]").ReplaceAllString(bucket, "")
|
||||
|
||||
if len(bucket) > 63 {
|
||||
bucket = bucket[:63] // Max length is 63
|
||||
}
|
||||
|
||||
return bucket, nil
|
||||
}
|
||||
|
||||
// EnsureS3Bucket creates a new S3 bucket with the given name and public read permissions.
|
||||
func (c Client) EnsureS3Bucket(ctx context.Context, bucketName string, publicRead bool) error {
|
||||
bucketName = strings.TrimPrefix(bucketName, "s3://")
|
||||
_, err := c.s3Client.CreateBucket(ctx, &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
CreateBucketConfiguration: &types.CreateBucketConfiguration{
|
||||
LocationConstraint: defaultRegion,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
var exists *types.BucketAlreadyExists
|
||||
if errors.As(err, &exists) {
|
||||
klog.Infof("Bucket %s already exists\n", bucketName)
|
||||
} else {
|
||||
klog.Infof("Error creating bucket %s, err: %v\n", bucketName, err)
|
||||
}
|
||||
|
||||
return fmt.Errorf("creating bucket %s: %w", bucketName, err)
|
||||
}
|
||||
|
||||
// Wait for the bucket to be created
|
||||
err = s3.NewBucketExistsWaiter(c.s3Client).Wait(
|
||||
ctx, &s3.HeadBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
},
|
||||
time.Minute)
|
||||
if err != nil {
|
||||
klog.Infof("Failed attempt to wait for bucket %s to exist, err: %v", bucketName, err)
|
||||
|
||||
return fmt.Errorf("waiting for bucket %s to exist: %w", bucketName, err)
|
||||
}
|
||||
|
||||
klog.Infof("Bucket %s created successfully", bucketName)
|
||||
|
||||
if publicRead {
|
||||
err = c.setPublicReadPolicy(ctx, bucketName)
|
||||
if err != nil {
|
||||
klog.Errorf("Failed to set public read policy on bucket %s, err: %v", bucketName, err)
|
||||
|
||||
return fmt.Errorf("setting public read policy for bucket %s: %w", bucketName, err)
|
||||
}
|
||||
|
||||
klog.Infof("Public read policy set on bucket %s", bucketName)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteS3Bucket deletes a S3 bucket with the given name.
|
||||
func (c Client) DeleteS3Bucket(ctx context.Context, bucketName string) error {
|
||||
bucketName = strings.TrimPrefix(bucketName, "s3://")
|
||||
_, err := c.s3Client.DeleteBucket(ctx, &s3.DeleteBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
if err != nil {
|
||||
var noBucket *types.NoSuchBucket
|
||||
if errors.As(err, &noBucket) {
|
||||
klog.Infof("Bucket %s does not exits.", bucketName)
|
||||
|
||||
return nil
|
||||
} else {
|
||||
klog.Infof("Couldn't delete bucket %s, err: %v", bucketName, err)
|
||||
|
||||
return fmt.Errorf("deleting bucket %s: %w", bucketName, err)
|
||||
}
|
||||
}
|
||||
|
||||
err = s3.NewBucketNotExistsWaiter(c.s3Client).Wait(
|
||||
ctx, &s3.HeadBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
},
|
||||
time.Minute)
|
||||
if err != nil {
|
||||
klog.Infof("Failed attempt to wait for bucket %s to be deleted, err: %v", bucketName, err)
|
||||
|
||||
return fmt.Errorf("waiting for bucket %s to be deleted, err: %w", bucketName, err)
|
||||
}
|
||||
|
||||
klog.Infof("Bucket %s deleted", bucketName)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c Client) setPublicReadPolicy(ctx context.Context, bucketName string) error {
|
||||
policy := fmt.Sprintf(`{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "PublicReadGetObject",
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": "arn:aws:s3:::%s/*"
|
||||
}
|
||||
]
|
||||
}`, bucketName)
|
||||
|
||||
_, err := c.s3Client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Policy: aws.String(policy),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("putting bucket policy for %s: %w", bucketName, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -17,6 +17,7 @@ limitations under the License.
|
|||
package deployer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
|
|
@ -24,6 +25,7 @@ import (
|
|||
"strings"
|
||||
|
||||
"k8s.io/klog/v2"
|
||||
"k8s.io/kops/tests/e2e/kubetest2-kops/aws"
|
||||
"k8s.io/kops/tests/e2e/kubetest2-kops/gce"
|
||||
"k8s.io/kops/tests/e2e/pkg/target"
|
||||
"k8s.io/kops/tests/e2e/pkg/util"
|
||||
|
|
@ -51,6 +53,12 @@ func (d *deployer) initialize() error {
|
|||
|
||||
switch d.CloudProvider {
|
||||
case "aws":
|
||||
client, err := aws.NewClient(context.Background())
|
||||
if err != nil {
|
||||
return fmt.Errorf("init failed to build AWS client: %w", err)
|
||||
}
|
||||
d.aws = client
|
||||
|
||||
if d.SSHPrivateKeyPath == "" {
|
||||
d.SSHPrivateKeyPath = os.Getenv("AWS_SSH_PRIVATE_KEY_FILE")
|
||||
}
|
||||
|
|
@ -316,11 +324,21 @@ func defaultClusterName(cloudProvider string) (string, error) {
|
|||
// stateStore returns the kops state store to use
|
||||
// defaulting to values used in prow jobs
|
||||
func (d *deployer) stateStore() string {
|
||||
if d.stateStoreName != "" {
|
||||
return d.stateStoreName
|
||||
}
|
||||
ss := os.Getenv("KOPS_STATE_STORE")
|
||||
if ss == "" {
|
||||
switch d.CloudProvider {
|
||||
case "aws":
|
||||
ss = "s3://k8s-kops-prow"
|
||||
ctx := context.Background()
|
||||
bucketName, err := d.aws.BucketName(ctx)
|
||||
if err != nil {
|
||||
klog.Fatalf("Failed to generate bucket name: %v", err)
|
||||
return ""
|
||||
}
|
||||
d.createBucket = true
|
||||
ss = "s3://" + bucketName
|
||||
case "gce":
|
||||
d.createBucket = true
|
||||
ss = "gs://" + gce.GCSBucketName(d.GCPProject, "state")
|
||||
|
|
@ -328,11 +346,16 @@ func (d *deployer) stateStore() string {
|
|||
ss = "do://e2e-kops-space"
|
||||
}
|
||||
}
|
||||
|
||||
d.stateStoreName = ss
|
||||
return ss
|
||||
}
|
||||
|
||||
// discoveryStore returns the VFS path to use for public OIDC documents
|
||||
func (d *deployer) discoveryStore() string {
|
||||
if d.discoveryStoreName != "" {
|
||||
return d.discoveryStoreName
|
||||
}
|
||||
discovery := os.Getenv("KOPS_DISCOVERY_STORE")
|
||||
if discovery == "" {
|
||||
switch d.CloudProvider {
|
||||
|
|
@ -340,10 +363,14 @@ func (d *deployer) discoveryStore() string {
|
|||
discovery = "s3://k8s-kops-ci-prow"
|
||||
}
|
||||
}
|
||||
d.discoveryStoreName = discovery
|
||||
return discovery
|
||||
}
|
||||
|
||||
func (d *deployer) stagingStore() string {
|
||||
if d.stagingStoreName != "" {
|
||||
return d.stagingStoreName
|
||||
}
|
||||
sb := os.Getenv("KOPS_STAGING_BUCKET")
|
||||
if sb == "" {
|
||||
switch d.CloudProvider {
|
||||
|
|
@ -352,6 +379,7 @@ func (d *deployer) stagingStore() string {
|
|||
sb = "gs://" + gce.GCSBucketName(d.GCPProject, "staging")
|
||||
}
|
||||
}
|
||||
d.stagingStoreName = sb
|
||||
return sb
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import (
|
|||
"github.com/octago/sflags/gen/gpflag"
|
||||
"github.com/spf13/pflag"
|
||||
"k8s.io/klog/v2"
|
||||
"k8s.io/kops/tests/e2e/kubetest2-kops/aws"
|
||||
"k8s.io/kops/tests/e2e/kubetest2-kops/builder"
|
||||
"k8s.io/kops/tests/e2e/pkg/target"
|
||||
|
||||
|
|
@ -57,7 +58,6 @@ type deployer struct {
|
|||
CreateArgs string `flag:"create-args" desc:"Extra space-separated arguments passed to 'kops create cluster'"`
|
||||
KopsBinaryPath string `flag:"kops-binary-path" desc:"The path to kops executable used for testing"`
|
||||
KubernetesFeatureGates string `flag:"kubernetes-feature-gates" desc:"Feature Gates to enable on Kubernetes components"`
|
||||
createBucket bool `flag:"-"`
|
||||
|
||||
// ControlPlaneCount specifies the number of VMs in the control-plane.
|
||||
ControlPlaneCount int `flag:"control-plane-count" desc:"Number of control-plane instances"`
|
||||
|
|
@ -90,6 +90,13 @@ type deployer struct {
|
|||
manifestPath string
|
||||
terraform *target.Terraform
|
||||
|
||||
aws *aws.Client
|
||||
|
||||
createBucket bool
|
||||
stateStoreName string
|
||||
discoveryStoreName string
|
||||
stagingStoreName string
|
||||
|
||||
// boskos struct field will be non-nil when the deployer is
|
||||
// using boskos to acquire a GCP project
|
||||
boskos *client.Client
|
||||
|
|
@ -106,8 +113,10 @@ type deployer struct {
|
|||
var _ types.NewDeployer = New
|
||||
|
||||
// assert that deployer implements types.Deployer
|
||||
var _ types.Deployer = &deployer{}
|
||||
var _ types.DeployerWithPostTester = &deployer{}
|
||||
var (
|
||||
_ types.Deployer = &deployer{}
|
||||
_ types.DeployerWithPostTester = &deployer{}
|
||||
)
|
||||
|
||||
func (d *deployer) Provider() string {
|
||||
return Name
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ limitations under the License.
|
|||
package deployer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
|
|
@ -72,9 +73,17 @@ func (d *deployer) Down() error {
|
|||
return err
|
||||
}
|
||||
|
||||
if d.CloudProvider == "gce" && d.createBucket {
|
||||
gce.DeleteGCSBucket(d.stateStore(), d.GCPProject)
|
||||
gce.DeleteGCSBucket(d.stagingStore(), d.GCPProject)
|
||||
if d.createBucket {
|
||||
switch d.CloudProvider {
|
||||
case "aws":
|
||||
ctx := context.Background()
|
||||
if err := d.aws.DeleteS3Bucket(ctx, d.stateStore()); err != nil {
|
||||
return err
|
||||
}
|
||||
case "gce":
|
||||
gce.DeleteGCSBucket(d.stateStore(), d.GCPProject)
|
||||
gce.DeleteGCSBucket(d.stagingStore(), d.GCPProject)
|
||||
}
|
||||
}
|
||||
|
||||
if d.boskos != nil {
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ limitations under the License.
|
|||
package deployer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
osexec "os/exec"
|
||||
|
|
@ -61,9 +62,17 @@ func (d *deployer) Up() error {
|
|||
_ = d.Down()
|
||||
}
|
||||
|
||||
if d.CloudProvider == "gce" && d.createBucket {
|
||||
if err := gce.EnsureGCSBucket(d.stateStore(), d.GCPProject, false); err != nil {
|
||||
return err
|
||||
if d.createBucket {
|
||||
switch d.CloudProvider {
|
||||
case "aws":
|
||||
ctx := context.Background()
|
||||
if err := d.aws.EnsureS3Bucket(ctx, d.stateStore(), false); err != nil {
|
||||
return err
|
||||
}
|
||||
case "gce":
|
||||
if err := gce.EnsureGCSBucket(d.stateStore(), d.GCPProject, false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue