diff --git a/upup/cmd/cloudup/main.go b/upup/cmd/cloudup/main.go index 3a547fe280..ffc7354ce3 100644 --- a/upup/cmd/cloudup/main.go +++ b/upup/cmd/cloudup/main.go @@ -74,7 +74,7 @@ func main() { os.Exit(1) } - statePath, err := fi.BuildVfsPath(*stateLocation) + statePath, err := vfs.Context.BuildVfsPath(*stateLocation) if err != nil { glog.Errorf("error building state location: %v", err) os.Exit(1) @@ -364,7 +364,7 @@ func (c *CreateClusterCmd) Run() error { if c.Config.KubernetesVersion == "" { stableURL := "https://storage.googleapis.com/kubernetes-release/release/stable.txt" - b, err := utils.ReadLocation(stableURL) + b, err := vfs.Context.ReadFile(stableURL) if err != nil { return fmt.Errorf("--kubernetes-version not specified, and unable to download latest version from %q: %v", stableURL, err) } diff --git a/upup/cmd/upup/root.go b/upup/cmd/upup/root.go index bbeab3b092..96f55436bc 100644 --- a/upup/cmd/upup/root.go +++ b/upup/cmd/upup/root.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" "k8s.io/kube-deploy/upup/pkg/fi" + "k8s.io/kube-deploy/upup/pkg/fi/vfs" ) type RootCmd struct { @@ -76,7 +77,7 @@ func (c *RootCmd) StateStore() (fi.StateStore, error) { return nil, fmt.Errorf("--state is required") } - statePath, err := fi.BuildVfsPath(c.stateLocation) + statePath, err := vfs.Context.BuildVfsPath(c.stateLocation) if err != nil { return nil, fmt.Errorf("error building state store path: %v", err) } diff --git a/upup/pkg/fi/nodeup/command.go b/upup/pkg/fi/nodeup/command.go index b81797f3ce..01b58234f4 100644 --- a/upup/pkg/fi/nodeup/command.go +++ b/upup/pkg/fi/nodeup/command.go @@ -8,6 +8,7 @@ import ( "k8s.io/kube-deploy/upup/pkg/fi/nodeup/cloudinit" "k8s.io/kube-deploy/upup/pkg/fi/nodeup/local" "k8s.io/kube-deploy/upup/pkg/fi/utils" + "k8s.io/kube-deploy/upup/pkg/fi/vfs" ) type NodeUpCommand struct { @@ -20,7 +21,7 @@ type NodeUpCommand struct { func (c *NodeUpCommand) Run(out io.Writer) error { if c.ConfigLocation != "" { - config, err := utils.ReadLocation(c.ConfigLocation) + config, err := vfs.Context.ReadFile(c.ConfigLocation) if err != nil { return fmt.Errorf("error loading configuration %q: %v", c.ConfigLocation, err) } diff --git a/upup/pkg/fi/nodeup/template_functions.go b/upup/pkg/fi/nodeup/template_functions.go index ca6ae0ebcc..46fc9d4b25 100644 --- a/upup/pkg/fi/nodeup/template_functions.go +++ b/upup/pkg/fi/nodeup/template_functions.go @@ -5,6 +5,7 @@ import ( "github.com/golang/glog" "k8s.io/kube-deploy/upup/pkg/fi" "text/template" + "k8s.io/kube-deploy/upup/pkg/fi/vfs" ) type templateFunctions struct { @@ -22,7 +23,7 @@ func buildTemplateFunctions(config *NodeConfig, dest template.FuncMap) error { if config.SecretStore != "" { glog.Infof("Building SecretStore at %q", config.SecretStore) - p, err := fi.BuildVfsPath(config.SecretStore) + p, err := vfs.Context.BuildVfsPath(config.SecretStore) if err != nil { return fmt.Errorf("error building secret store path: %v", err) } @@ -37,7 +38,7 @@ func buildTemplateFunctions(config *NodeConfig, dest template.FuncMap) error { if config.KeyStore != "" { glog.Infof("Building KeyStore at %q", config.KeyStore) - p, err := fi.BuildVfsPath(config.KeyStore) + p, err := vfs.Context.BuildVfsPath(config.KeyStore) if err != nil { return fmt.Errorf("error building key store path: %v", err) } diff --git a/upup/pkg/fi/statestore.go b/upup/pkg/fi/statestore.go index b5212e3f1f..cb0fc558f4 100644 --- a/upup/pkg/fi/statestore.go +++ b/upup/pkg/fi/statestore.go @@ -2,13 +2,8 @@ package fi import ( "fmt" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/golang/glog" "k8s.io/kube-deploy/upup/pkg/fi/utils" "k8s.io/kube-deploy/upup/pkg/fi/vfs" - "net/url" "os" "strings" ) @@ -99,51 +94,3 @@ func (s *VFSStateStore) WriteConfig(config interface{}) error { } return nil } - -func BuildVfsPath(p string) (vfs.Path, error) { - if strings.HasPrefix(p, "s3://") { - u, err := url.Parse(p) - if err != nil { - return nil, fmt.Errorf("invalid s3 path: %q", err) - } - - var region string - { - config := aws.NewConfig().WithRegion("us-east-1") - session := session.New() - s3Client := s3.New(session, config) - - bucket := strings.TrimSuffix(u.Host, "/") - request := &s3.GetBucketLocationInput{} - request.Bucket = aws.String(bucket) - - response, err := s3Client.GetBucketLocation(request) - if err != nil { - // TODO: Auto-create bucket? - return nil, fmt.Errorf("error getting location for S3 bucket %q: %v", bucket, err) - } - if response.LocationConstraint == nil { - // US Classic does not return a region - region = "us-east-1" - } else { - region = *response.LocationConstraint - // Another special case: "EU" can mean eu-west-1 - if region == "EU" { - region = "eu-west-1" - } - } - glog.V(2).Infof("Found bucket %q in region %q", bucket, region) - } - - { - config := aws.NewConfig().WithRegion(region) - session := session.New() - s3Client := s3.New(session, config) - - s3path := vfs.NewS3Path(s3Client, u.Host, u.Path) - return s3path, nil - } - } - - return nil, fmt.Errorf("unknown / unhandled path type: %q", p) -} diff --git a/upup/pkg/fi/utils/uris.go b/upup/pkg/fi/utils/uris.go deleted file mode 100644 index 9424fa6104..0000000000 --- a/upup/pkg/fi/utils/uris.go +++ /dev/null @@ -1,69 +0,0 @@ -package utils - -import ( - "fmt" - "io/ioutil" - "net/http" - "net/url" - "strings" -) - -func ReadLocation(location string) ([]byte, error) { - if !strings.Contains(location, "://") { - // Assume a simple file - v, err := ioutil.ReadFile(location) - if err != nil { - return nil, fmt.Errorf("error reading file %q: %v", location, err) - } - return v, nil - } - - u, err := url.Parse(location) - if err != nil { - return nil, fmt.Errorf("error parsing location %q - not a valid URI") - } - - var httpURL string - httpHeaders := make(map[string]string) - - switch u.Scheme { - case "metadata": - switch u.Host { - case "gce": - httpURL = "http://169.254.169.254/computeMetadata/v1/instance/attributes/" + u.Path - httpHeaders["Metadata-Flavor"] = "Google" - - case "aws": - httpURL = "http://169.254.169.254/latest/" + u.Path - - default: - return nil, fmt.Errorf("unknown metadata type: %q in %q", u.Host, location) - } - - case "http", "https": - httpURL = location - - default: - return nil, fmt.Errorf("unrecognized scheme for location %q") - } - - req, err := http.NewRequest("GET", httpURL, nil) - if err != nil { - return nil, err - } - for k, v := range httpHeaders { - req.Header.Add(k, v) - } - response, err := http.DefaultClient.Do(req) - if response != nil { - defer response.Body.Close() - } - if err != nil { - return nil, fmt.Errorf("error fetching %q: %v", httpURL, err) - } - body, err := ioutil.ReadAll(response.Body) - if err != nil { - return nil, fmt.Errorf("error reading response for %q: %v", httpURL, err) - } - return body, nil -} diff --git a/upup/pkg/fi/vfs/context.go b/upup/pkg/fi/vfs/context.go new file mode 100644 index 0000000000..13f9c16b34 --- /dev/null +++ b/upup/pkg/fi/vfs/context.go @@ -0,0 +1,142 @@ +package vfs + +import ( + "strings" + "io/ioutil" + "net/url" + "fmt" + "net/http" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/golang/glog" +) + +// VFSContext is a 'context' for VFS, that is normally a singleton +// but allows us to configure S3 credentials, for example +type VFSContext struct { + +} + +var Context VFSContext + +// ReadLocation reads a file from a vfs URL +// It supports additional schemes which don't (yet) have full VFS implementations: +// metadata: reads from instance metadata on GCE/AWS +// http / https: reads from HTTP +func (c*VFSContext) ReadFile(location string) ([]byte, error) { + if strings.Contains(location, "://") { + // Handle our special case schemas + u, err := url.Parse(location) + if err != nil { + return nil, fmt.Errorf("error parsing location %q - not a valid URI") + } + + switch u.Scheme { + case "metadata": + switch u.Host { + case "gce": + httpURL := "http://169.254.169.254/computeMetadata/v1/instance/attributes/" + u.Path + httpHeaders := make(map[string]string) + httpHeaders["Metadata-Flavor"] = "Google" + return c.readHttpLocation(httpURL, httpHeaders) + case "aws": + httpURL := "http://169.254.169.254/latest/" + u.Path + return c.readHttpLocation(httpURL, nil) + + default: + return nil, fmt.Errorf("unknown metadata type: %q in %q", u.Host, location) + } + + case "http", "https": + return c.readHttpLocation(location, nil) + } + + } + + p, err := c.BuildVfsPath(location) + if err != nil { + return nil, err + } + return p.ReadFile() +} + +func (c*VFSContext) BuildVfsPath(p string) (Path, error) { + if !strings.Contains(p, "://") { + return NewFSPath(p), nil + } + + if strings.HasPrefix(p, "s3://") { + return c.buildS3Path(p) + } + + return nil, fmt.Errorf("unknown / unhandled path type: %q", p) +} + +func (c*VFSContext) readHttpLocation(httpURL string, httpHeaders map[string]string) ([]byte, error) { + req, err := http.NewRequest("GET", httpURL, nil) + if err != nil { + return nil, err + } + for k, v := range httpHeaders { + req.Header.Add(k, v) + } + response, err := http.DefaultClient.Do(req) + if response != nil { + defer response.Body.Close() + } + if err != nil { + return nil, fmt.Errorf("error fetching %q: %v", httpURL, err) + } + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return nil, fmt.Errorf("error reading response for %q: %v", httpURL, err) + } + return body, nil +} + +func (c*VFSContext) buildS3Path(p string) (*S3Path, error) { + u, err := url.Parse(p) + if err != nil { + return nil, fmt.Errorf("invalid s3 path: %q", err) + } + + bucket := strings.TrimSuffix(u.Host, "/") + + var region string + { + // Probe to find correct region for bucket + // TODO: Caching (both of the client & of the bucket location) + config := aws.NewConfig().WithRegion("us-east-1") + session := session.New() + s3Client := s3.New(session, config) + + request := &s3.GetBucketLocationInput{} + request.Bucket = aws.String(bucket) + + response, err := s3Client.GetBucketLocation(request) + if err != nil { + // TODO: Auto-create bucket? + return nil, fmt.Errorf("error getting location for S3 bucket %q: %v", bucket, err) + } + if response.LocationConstraint == nil { + // US Classic does not return a region + region = "us-east-1" + } else { + region = *response.LocationConstraint + // Another special case: "EU" can mean eu-west-1 + if region == "EU" { + region = "eu-west-1" + } + } + glog.V(2).Infof("Found bucket %q in region %q", bucket, region) + } + + // TODO: Caching (of the S3 client) + config := aws.NewConfig().WithRegion(region) + session := session.New() + s3Client := s3.New(session, config) + + s3path := NewS3Path(s3Client, bucket, u.Path) + return s3path, nil +}