kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #130 from justinsb/upup_vfs_context 

upup: Add VFS context object, centralize usage
This commit is contained in:
Justin Santa Barbara 2016-06-23 10:30:09 -04:00 committed by GitHub
parent 1bc19ca0ed 75af0d089d
commit 0a6260f7c4
7 changed files with 151 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
upup/cmd/cloudup/main.go
View File

@ -74,7 +74,7 @@ func main() {
os.Exit(1)
}
statePath, err := fi.BuildVfsPath(*stateLocation)
statePath, err := vfs.Context.BuildVfsPath(*stateLocation)
if err != nil {
glog.Errorf("error building state location: %v", err)
os.Exit(1)
@ -364,7 +364,7 @@ func (c *CreateClusterCmd) Run() error {
if c.Config.KubernetesVersion == "" {
stableURL := "https://storage.googleapis.com/kubernetes-release/release/stable.txt"
b, err := utils.ReadLocation(stableURL)
b, err := vfs.Context.ReadFile(stableURL)
if err != nil {
return fmt.Errorf("--kubernetes-version not specified, and unable to download latest version from %q: %v", stableURL, err)
}

3
upup/cmd/upup/root.go
View File

@ -8,6 +8,7 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/viper"
"k8s.io/kube-deploy/upup/pkg/fi"
"k8s.io/kube-deploy/upup/pkg/fi/vfs"
)
type RootCmd struct {
@ -76,7 +77,7 @@ func (c *RootCmd) StateStore() (fi.StateStore, error) {
return nil, fmt.Errorf("--state is required")
}
statePath, err := fi.BuildVfsPath(c.stateLocation)
statePath, err := vfs.Context.BuildVfsPath(c.stateLocation)
if err != nil {
return nil, fmt.Errorf("error building state store path: %v", err)
}

3
upup/pkg/fi/nodeup/command.go
View File

@ -8,6 +8,7 @@ import (
"k8s.io/kube-deploy/upup/pkg/fi/nodeup/cloudinit"
"k8s.io/kube-deploy/upup/pkg/fi/nodeup/local"
"k8s.io/kube-deploy/upup/pkg/fi/utils"
"k8s.io/kube-deploy/upup/pkg/fi/vfs"
)
type NodeUpCommand struct {
@ -20,7 +21,7 @@ type NodeUpCommand struct {
func (c *NodeUpCommand) Run(out io.Writer) error {
if c.ConfigLocation != "" {
config, err := utils.ReadLocation(c.ConfigLocation)
config, err := vfs.Context.ReadFile(c.ConfigLocation)
if err != nil {
return fmt.Errorf("error loading configuration %q: %v", c.ConfigLocation, err)
}

5
upup/pkg/fi/nodeup/template_functions.go
View File

@ -5,6 +5,7 @@ import (
"github.com/golang/glog"
"k8s.io/kube-deploy/upup/pkg/fi"
"text/template"
"k8s.io/kube-deploy/upup/pkg/fi/vfs"
)
type templateFunctions struct {
@ -22,7 +23,7 @@ func buildTemplateFunctions(config *NodeConfig, dest template.FuncMap) error {
if config.SecretStore != "" {
glog.Infof("Building SecretStore at %q", config.SecretStore)
p, err := fi.BuildVfsPath(config.SecretStore)
p, err := vfs.Context.BuildVfsPath(config.SecretStore)
if err != nil {
return fmt.Errorf("error building secret store path: %v", err)
}
@ -37,7 +38,7 @@ func buildTemplateFunctions(config *NodeConfig, dest template.FuncMap) error {
if config.KeyStore != "" {
glog.Infof("Building KeyStore at %q", config.KeyStore)
p, err := fi.BuildVfsPath(config.KeyStore)
p, err := vfs.Context.BuildVfsPath(config.KeyStore)
if err != nil {
return fmt.Errorf("error building key store path: %v", err)
}

53
upup/pkg/fi/statestore.go
View File

@ -2,13 +2,8 @@ package fi
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/golang/glog"
"k8s.io/kube-deploy/upup/pkg/fi/utils"
"k8s.io/kube-deploy/upup/pkg/fi/vfs"
"net/url"
"os"
"strings"
)
@ -99,51 +94,3 @@ func (s *VFSStateStore) WriteConfig(config interface{}) error {
}
return nil
}
func BuildVfsPath(p string) (vfs.Path, error) {
if strings.HasPrefix(p, "s3://") {
u, err := url.Parse(p)
if err != nil {
return nil, fmt.Errorf("invalid s3 path: %q", err)
}
var region string
{
config := aws.NewConfig().WithRegion("us-east-1")
session := session.New()
s3Client := s3.New(session, config)
bucket := strings.TrimSuffix(u.Host, "/")
request := &s3.GetBucketLocationInput{}
request.Bucket = aws.String(bucket)
response, err := s3Client.GetBucketLocation(request)
if err != nil {
// TODO: Auto-create bucket?
return nil, fmt.Errorf("error getting location for S3 bucket %q: %v", bucket, err)
}
if response.LocationConstraint == nil {
// US Classic does not return a region
region = "us-east-1"
} else {
region = *response.LocationConstraint
// Another special case: "EU" can mean eu-west-1
if region == "EU" {
region = "eu-west-1"
}
}
glog.V(2).Infof("Found bucket %q in region %q", bucket, region)
}
{
config := aws.NewConfig().WithRegion(region)
session := session.New()
s3Client := s3.New(session, config)
s3path := vfs.NewS3Path(s3Client, u.Host, u.Path)
return s3path, nil
}
}
return nil, fmt.Errorf("unknown / unhandled path type: %q", p)
}

69
upup/pkg/fi/utils/uris.go
View File

@ -1,69 +0,0 @@
package utils
import (
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
)
func ReadLocation(location string) ([]byte, error) {
if !strings.Contains(location, "://") {
// Assume a simple file
v, err := ioutil.ReadFile(location)
if err != nil {
return nil, fmt.Errorf("error reading file %q: %v", location, err)
}
return v, nil
}
u, err := url.Parse(location)
if err != nil {
return nil, fmt.Errorf("error parsing location %q - not a valid URI")
}
var httpURL string
httpHeaders := make(map[string]string)
switch u.Scheme {
case "metadata":
switch u.Host {
case "gce":
httpURL = "http://169.254.169.254/computeMetadata/v1/instance/attributes/" + u.Path
httpHeaders["Metadata-Flavor"] = "Google"
case "aws":
httpURL = "http://169.254.169.254/latest/" + u.Path
default:
return nil, fmt.Errorf("unknown metadata type: %q in %q", u.Host, location)
}
case "http", "https":
httpURL = location
default:
return nil, fmt.Errorf("unrecognized scheme for location %q")
}
req, err := http.NewRequest("GET", httpURL, nil)
if err != nil {
return nil, err
}
for k, v := range httpHeaders {
req.Header.Add(k, v)
}
response, err := http.DefaultClient.Do(req)
if response != nil {
defer response.Body.Close()
}
if err != nil {
return nil, fmt.Errorf("error fetching %q: %v", httpURL, err)
}
body, err := ioutil.ReadAll(response.Body)
if err != nil {
return nil, fmt.Errorf("error reading response for %q: %v", httpURL, err)
}
return body, nil
}

142
upup/pkg/fi/vfs/context.go Normal file
View File

@ -0,0 +1,142 @@
package vfs
import (
"strings"
"io/ioutil"
"net/url"
"fmt"
"net/http"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/golang/glog"
)
// VFSContext is a 'context' for VFS, that is normally a singleton
// but allows us to configure S3 credentials, for example
type VFSContext struct {
}
var Context VFSContext
// ReadLocation reads a file from a vfs URL
// It supports additional schemes which don't (yet) have full VFS implementations:
// metadata: reads from instance metadata on GCE/AWS
// http / https: reads from HTTP
func (c*VFSContext) ReadFile(location string) ([]byte, error) {
if strings.Contains(location, "://") {
// Handle our special case schemas
u, err := url.Parse(location)
if err != nil {
return nil, fmt.Errorf("error parsing location %q - not a valid URI")
}
switch u.Scheme {
case "metadata":
switch u.Host {
case "gce":
httpURL := "http://169.254.169.254/computeMetadata/v1/instance/attributes/" + u.Path
httpHeaders := make(map[string]string)
httpHeaders["Metadata-Flavor"] = "Google"
return c.readHttpLocation(httpURL, httpHeaders)
case "aws":
httpURL := "http://169.254.169.254/latest/" + u.Path
return c.readHttpLocation(httpURL, nil)
default:
return nil, fmt.Errorf("unknown metadata type: %q in %q", u.Host, location)
}
case "http", "https":
return c.readHttpLocation(location, nil)
}
}
p, err := c.BuildVfsPath(location)
if err != nil {
return nil, err
}
return p.ReadFile()
}
func (c*VFSContext) BuildVfsPath(p string) (Path, error) {
if !strings.Contains(p, "://") {
return NewFSPath(p), nil
}
if strings.HasPrefix(p, "s3://") {
return c.buildS3Path(p)
}
return nil, fmt.Errorf("unknown / unhandled path type: %q", p)
}
func (c*VFSContext) readHttpLocation(httpURL string, httpHeaders map[string]string) ([]byte, error) {
req, err := http.NewRequest("GET", httpURL, nil)
if err != nil {
return nil, err
}
for k, v := range httpHeaders {
req.Header.Add(k, v)
}
response, err := http.DefaultClient.Do(req)
if response != nil {
defer response.Body.Close()
}
if err != nil {
return nil, fmt.Errorf("error fetching %q: %v", httpURL, err)
}
body, err := ioutil.ReadAll(response.Body)
if err != nil {
return nil, fmt.Errorf("error reading response for %q: %v", httpURL, err)
}
return body, nil
}
func (c*VFSContext) buildS3Path(p string) (*S3Path, error) {
u, err := url.Parse(p)
if err != nil {
return nil, fmt.Errorf("invalid s3 path: %q", err)
}
bucket := strings.TrimSuffix(u.Host, "/")
var region string
{
// Probe to find correct region for bucket
// TODO: Caching (both of the client & of the bucket location)
config := aws.NewConfig().WithRegion("us-east-1")
session := session.New()
s3Client := s3.New(session, config)
request := &s3.GetBucketLocationInput{}
request.Bucket = aws.String(bucket)
response, err := s3Client.GetBucketLocation(request)
if err != nil {
// TODO: Auto-create bucket?
return nil, fmt.Errorf("error getting location for S3 bucket %q: %v", bucket, err)
}
if response.LocationConstraint == nil {
// US Classic does not return a region
region = "us-east-1"
} else {
region = *response.LocationConstraint
// Another special case: "EU" can mean eu-west-1
if region == "EU" {
region = "eu-west-1"
}
}
glog.V(2).Infof("Found bucket %q in region %q", bucket, region)
}
// TODO: Caching (of the S3 client)
config := aws.NewConfig().WithRegion(region)
session := session.New()
s3Client := s3.New(session, config)
s3path := NewS3Path(s3Client, bucket, u.Path)
return s3path, nil
}