More logging around errors in s3 write path

Hopefully to shed some light on issues like #2108

util/pkg/vfs/s3context.go

@@ -94,6 +94,7 @@ func (s *S3Context) getRegionForBucket(bucket string) (string, error) {
 
 	// and fallback to brute-forcing if it fails
 	if err != nil {
+		glog.V(2).Infof("unable to get bucket location from region %q; scanning all regions: %v", awsRegion, err)
 		response, err = bruteforceBucketLocation(&awsRegion, request)
 	}
 
@@ -134,21 +135,22 @@ See also: https://docs.aws.amazon.com/goto/WebAPI/s3-2006-03-01/GetBucketLocatio
 */
 func bruteforceBucketLocation(region *string, request *s3.GetBucketLocationInput) (*s3.GetBucketLocationOutput, error) {
 	session, _ := session.NewSession(&aws.Config{Region: region})
-	regions, err := ec2.New(session).DescribeRegions(nil)
 
+	regions, err := ec2.New(session).DescribeRegions(nil)
 	if err != nil {
 		return nil, fmt.Errorf("Unable to list AWS regions: %v", err)
 	}
 
 	glog.V(2).Infof("Querying S3 for bucket location for %s", *request.Bucket)
 
-	out := make(chan *s3.GetBucketLocationOutput)
+	out := make(chan *s3.GetBucketLocationOutput, len(regions.Regions))
 	for _, region := range regions.Regions {
 		go func(regionName string) {
+			glog.V(8).Infof("Doing GetBucketLocation in %q", regionName)
 			s3Client := s3.New(session, &aws.Config{Region: aws.String(regionName)})
 			result, bucketError := s3Client.GetBucketLocation(request)
-
 			if bucketError == nil {
+				glog.V(8).Infof("GetBucketLocation succeeded in %q", regionName)
 				out <- result
 			}
 		}(*region.RegionName)

util/pkg/vfs/s3fs.go

@@ -109,22 +109,33 @@ func (p *S3Path) WriteFile(data []byte) error {
 
 	glog.V(4).Infof("Writing file %q", p)
 
+	// We always use server-side-encryption; it doesn't really cost us anything
+	sse := "AES256"
+
 	request := &s3.PutObjectInput{}
 	request.Body = bytes.NewReader(data)
 	request.Bucket = aws.String(p.bucket)
 	request.Key = aws.String(p.key)
-	request.ServerSideEncryption = aws.String("AES256")
+	request.ServerSideEncryption = aws.String(sse)
 
 	acl := os.Getenv("KOPS_STATE_S3_ACL")
+	acl = strings.TrimSpace(acl)
 	if acl != "" {
+		glog.Infof("Using KOPS_STATE_S3_ACL=%s", acl)
 		request.ACL = aws.String(acl)
 	}
 
 	// We don't need Content-MD5: https://github.com/aws/aws-sdk-go/issues/208
 
+	glog.V(8).Infof("Calling S3 PutObject Bucket=%q Key=%q SSE=%q ACL=%q BodyLen=%d", p.bucket, p.key, sse, acl, len(data))
+
 	_, err = client.PutObject(request)
 	if err != nil {
-		return fmt.Errorf("error writing %s: %v", p, err)
+		if acl != "" {
+			return fmt.Errorf("error writing %s (with ACL=%q): %v", p, acl, err)
+		} else {
+			return fmt.Errorf("error writing %s: %v", p, err)
+		}
 	}
 
 	return nil