kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

First model -> tf test

This commit is contained in:
Justin Santa Barbara 2016-12-10 22:08:29 -05:00
parent ee24ca1acc
commit afd0c25abe
20 changed files with 896 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -76,9 +76,10 @@ codegen: kops-gobindata
PATH=${GOPATH_1ST}/bin:${PATH} go generate k8s.io/kops/upup/pkg/fi/fitasks
test:
go test k8s.io/kops/upup/pkg/... -args -v=1 -logtostderr
go test k8s.io/kops/pkg/... -args -v=1 -logtostderr
go test k8s.io/kops/upup/pkg/... -args -v=1 -logtostderr
go test k8s.io/kops/dns-controller/pkg/... -args -v=1 -logtostderr
go test k8s.io/kops/cmd/... -args -v=1 -logtostderr
crossbuild:
mkdir -p .build/dist/
@ -201,6 +202,7 @@ gofmt:
gofmt -w -s util/
gofmt -w -s upup/pkg/
gofmt -w -s pkg/
gofmt -w -s tests/
gofmt -w -s protokube/cmd
gofmt -w -s protokube/pkg
gofmt -w -s dns-controller/cmd

2
cloudmock/aws/mockroute53/api.go
View File

@ -17,10 +17,12 @@ limitations under the License.
package mockroute53
import (
"github.com/aws/aws-sdk-go/service/route53"
"github.com/aws/aws-sdk-go/service/route53/route53iface"
)
type MockRoute53 struct {
Zones []*route53.HostedZone
}
var _ route53iface.Route53API = &MockRoute53{}

40
cloudmock/aws/mockroute53/unimplemented.go
View File

@ -287,27 +287,7 @@ func (m *MockRoute53) ListHealthChecks(*route53.ListHealthChecksInput) (*route53
}
func (m *MockRoute53) ListHealthChecksPages(*route53.ListHealthChecksInput, func(*route53.ListHealthChecksOutput, bool) bool) error {
panic("MockRoute53 function not implemented")
return nil, nil
}
func (m *MockRoute53) ListHostedZonesRequest(*route53.ListHostedZonesInput) (*request.Request, *route53.ListHostedZonesOutput) {
panic("MockRoute53 function not implemented")
return nil, nil
}
func (m *MockRoute53) ListHostedZones(*route53.ListHostedZonesInput) (*route53.ListHostedZonesOutput, error) {
panic("MockRoute53 function not implemented")
return nil, nil
}
func (m *MockRoute53) ListHostedZonesPages(*route53.ListHostedZonesInput, func(*route53.ListHostedZonesOutput, bool) bool) error {
panic("MockRoute53 function not implemented")
return nil, nil
}
func (m *MockRoute53) ListHostedZonesByNameRequest(*route53.ListHostedZonesByNameInput) (*request.Request, *route53.ListHostedZonesByNameOutput) {
panic("MockRoute53 function not implemented")
return nil, nil
}
func (m *MockRoute53) ListHostedZonesByName(*route53.ListHostedZonesByNameInput) (*route53.ListHostedZonesByNameOutput, error) {
panic("MockRoute53 function not implemented")
return nil, nil
return nil
}
func (m *MockRoute53) ListResourceRecordSetsRequest(*route53.ListResourceRecordSetsInput) (*request.Request, *route53.ListResourceRecordSetsOutput) {
panic("MockRoute53 function not implemented")
@ -319,7 +299,7 @@ func (m *MockRoute53) ListResourceRecordSets(*route53.ListResourceRecordSetsInpu
}
func (m *MockRoute53) ListResourceRecordSetsPages(*route53.ListResourceRecordSetsInput, func(*route53.ListResourceRecordSetsOutput, bool) bool) error {
panic("MockRoute53 function not implemented")
return nil, nil
return nil
}
func (m *MockRoute53) ListReusableDelegationSetsRequest(*route53.ListReusableDelegationSetsInput) (*request.Request, *route53.ListReusableDelegationSetsOutput) {
panic("MockRoute53 function not implemented")
@ -385,6 +365,17 @@ func (m *MockRoute53) ListTrafficPolicyVersions(*route53.ListTrafficPolicyVersio
panic("MockRoute53 function not implemented")
return nil, nil
}
func (m *MockRoute53) TestDNSAnswerRequest(*route53.TestDNSAnswerInput) (*request.Request, *route53.TestDNSAnswerOutput) {
panic("MockRoute53 function not implemented")
return nil, nil
}
func (m *MockRoute53) TestDNSAnswer(*route53.TestDNSAnswerInput) (*route53.TestDNSAnswerOutput, error) {
panic("MockRoute53 function not implemented")
return nil, nil
}
func (m *MockRoute53) UpdateHealthCheckRequest(*route53.UpdateHealthCheckInput) (*request.Request, *route53.UpdateHealthCheckOutput) {
panic("MockRoute53 function not implemented")
return nil, nil
@ -417,3 +408,8 @@ func (m *MockRoute53) UpdateTrafficPolicyInstance(*route53.UpdateTrafficPolicyIn
panic("MockRoute53 function not implemented")
return nil, nil
}
func (m *MockRoute53) WaitUntilResourceRecordSetsChanged(*route53.GetChangeInput) error {
panic("MockRoute53 WaitUntilResourceRecordSetsChanged function not implemented")
return nil
}

44
cloudmock/aws/mockroute53/zones.go Normal file
View File

@ -0,0 +1,44 @@
package mockroute53
import (
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/service/route53"
"github.com/golang/glog"
)
type zoneInfo struct {
}
func (m *MockRoute53) ListHostedZonesRequest(*route53.ListHostedZonesInput) (*request.Request, *route53.ListHostedZonesOutput) {
panic("MockRoute53 function not implemented")
return nil, nil
}
func (m *MockRoute53) ListHostedZones(*route53.ListHostedZonesInput) (*route53.ListHostedZonesOutput, error) {
panic("MockRoute53 function not implemented")
return nil, nil
}
func (m *MockRoute53) ListHostedZonesPages(request *route53.ListHostedZonesInput, callback func(*route53.ListHostedZonesOutput, bool) bool) error {
glog.Infof("ListHostedZonesPages %v", request)
page := &route53.ListHostedZonesOutput{}
for _, zone := range m.Zones {
copy := *zone
page.HostedZones = append(page.HostedZones, &copy)
}
lastPage := true
callback(page, lastPage)
return nil
}
func (m *MockRoute53) ListHostedZonesByNameRequest(*route53.ListHostedZonesByNameInput) (*request.Request, *route53.ListHostedZonesByNameOutput) {
panic("MockRoute53 function not implemented")
return nil, nil
}
func (m *MockRoute53) ListHostedZonesByName(*route53.ListHostedZonesByNameInput) (*route53.ListHostedZonesByNameOutput, error) {
panic("MockRoute53 function not implemented")
return nil, nil
}

79
cmd/kops/create.go
View File

@ -20,13 +20,16 @@ import (
"fmt"
"io"
"bytes"
"github.com/golang/glog"
"github.com/spf13/cobra"
"k8s.io/kops/cmd/kops/util"
kopsapi "k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/apis/kops/v1alpha1"
"k8s.io/kops/util/pkg/vfs"
k8sapi "k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/errors"
"k8s.io/kubernetes/pkg/api/unversioned"
cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
"k8s.io/kubernetes/pkg/kubectl/resource"
)
@ -51,7 +54,7 @@ func NewCmdCreate(f *util.Factory, out io.Writer) *cobra.Command {
}
//cmdutil.CheckErr(ValidateArgs(cmd, args))
//cmdutil.CheckErr(cmdutil.ValidateOutputArgs(cmd))
cmdutil.CheckErr(RunCreate(f, cmd, out, options))
cmdutil.CheckErr(RunCreate(f, out, options))
},
}
@ -72,7 +75,7 @@ func NewCmdCreate(f *util.Factory, out io.Writer) *cobra.Command {
return cmd
}
func RunCreate(f *util.Factory, cmd *cobra.Command, out io.Writer, c *CreateOptions) error {
func RunCreate(f *util.Factory, out io.Writer, c *CreateOptions) error {
clientset, err := f.Clientset()
if err != nil {
return err
@ -89,46 +92,54 @@ func RunCreate(f *util.Factory, cmd *cobra.Command, out io.Writer, c *CreateOpti
return fmt.Errorf("error reading file %q: %v", f, err)
}
o, gvk, err := codec.Decode(contents, nil, nil)
if err != nil {
return fmt.Errorf("error parsing file %q: %v", f, err)
}
sections := bytes.Split(contents, []byte("\n---\n"))
switch v := o.(type) {
case *kopsapi.Federation:
_, err = clientset.Federations().Create(v)
for _, section := range sections {
defaults := &unversioned.GroupVersionKind{
Group: v1alpha1.SchemeGroupVersion.Group,
Version: v1alpha1.SchemeGroupVersion.Version,
}
o, gvk, err := codec.Decode(section, defaults, nil)
if err != nil {
if errors.IsAlreadyExists(err) {
return fmt.Errorf("federation %q already exists", v.Name)
}
return fmt.Errorf("error creating federation: %v", err)
return fmt.Errorf("error parsing file %q: %v", f, err)
}
case *kopsapi.Cluster:
_, err = clientset.Clusters().Create(v)
if err != nil {
if errors.IsAlreadyExists(err) {
return fmt.Errorf("cluster %q already exists", v.Name)
switch v := o.(type) {
case *kopsapi.Federation:
_, err = clientset.Federations().Create(v)
if err != nil {
if errors.IsAlreadyExists(err) {
return fmt.Errorf("federation %q already exists", v.Name)
}
return fmt.Errorf("error creating federation: %v", err)
}
return fmt.Errorf("error creating cluster: %v", err)
}
case *kopsapi.InstanceGroup:
clusterName := v.Labels[ClusterNameLabel]
if clusterName == "" {
return fmt.Errorf("must specify %q label with cluster name to create instanceGroup", ClusterNameLabel)
}
_, err = clientset.InstanceGroups(clusterName).Create(v)
if err != nil {
if errors.IsAlreadyExists(err) {
return fmt.Errorf("instanceGroup %q already exists", v.Name)
case *kopsapi.Cluster:
_, err = clientset.Clusters().Create(v)
if err != nil {
if errors.IsAlreadyExists(err) {
return fmt.Errorf("cluster %q already exists", v.Name)
}
return fmt.Errorf("error creating cluster: %v", err)
}
return fmt.Errorf("error creating instanceGroup: %v", err)
}
default:
glog.V(2).Infof("Type of object was %T", v)
return fmt.Errorf("Unhandled kind %q in %q", gvk, f)
case *kopsapi.InstanceGroup:
clusterName := v.Labels[ClusterNameLabel]
if clusterName == "" {
return fmt.Errorf("must specify %q label with cluster name to create instanceGroup", ClusterNameLabel)
}
_, err = clientset.InstanceGroups(clusterName).Create(v)
if err != nil {
if errors.IsAlreadyExists(err) {
return fmt.Errorf("instanceGroup %q already exists", v.Name)
}
return fmt.Errorf("error creating instanceGroup: %v", err)
}
default:
glog.V(2).Infof("Type of object was %T", v)
return fmt.Errorf("Unhandled kind %q in %q", gvk, f)
}
}
}

49
cmd/kops/create_secret_sshpublickey.go
View File

@ -27,7 +27,9 @@ import (
)
type CreateSecretPublickeyOptions struct {
Pubkey string
ClusterName string
Name string
PublicKeyPath string
}
func NewCmdCreateSecretPublicKey(f *util.Factory, out io.Writer) *cobra.Command {
@ -38,32 +40,43 @@ func NewCmdCreateSecretPublicKey(f *util.Factory, out io.Writer) *cobra.Command
Short: "Create SSH publickey",
Long: `Create SSH publickey.`,
Run: func(cmd *cobra.Command, args []string) {
err := RunCreateSecretPublicKey(f, cmd, args, os.Stdout, options)
if len(args) == 0 {
exitWithError(fmt.Errorf("syntax: NAME -i <PublicKeyPath>"))
}
if len(args) != 1 {
exitWithError(fmt.Errorf("syntax: NAME -i <PublicKeyPath>"))
}
options.Name = args[0]
err := rootCommand.ProcessArgs(args)
if err != nil {
exitWithError(err)
}
options.ClusterName = rootCommand.ClusterName()
err = RunCreateSecretPublicKey(f, os.Stdout, options)
if err != nil {
exitWithError(err)
}
},
}
cmd.Flags().StringVarP(&options.Pubkey, "pubkey", "i", "", "Path to SSH public key")
cmd.Flags().StringVarP(&options.PublicKeyPath, "pubkey", "i", "", "Path to SSH public key")
return cmd
}
func RunCreateSecretPublicKey(f *util.Factory, cmd *cobra.Command, args []string, out io.Writer, options *CreateSecretPublickeyOptions) error {
if len(args) == 0 {
return fmt.Errorf("syntax: NAME -i <PublicKeyPath>")
}
if len(args) != 1 {
return fmt.Errorf("syntax: NAME -i <PublicKeyPath>")
}
name := args[0]
if options.Pubkey == "" {
return fmt.Errorf("pubkey path is required (use -i)")
func RunCreateSecretPublicKey(f *util.Factory, out io.Writer, options *CreateSecretPublickeyOptions) error {
if options.PublicKeyPath == "" {
return fmt.Errorf("public key path is required (use -i)")
}
cluster, err := rootCommand.Cluster()
if options.Name == "" {
return fmt.Errorf("Name is required")
}
cluster, err := GetCluster(f, options.ClusterName)
if err != nil {
return err
}
@ -73,12 +86,12 @@ func RunCreateSecretPublicKey(f *util.Factory, cmd *cobra.Command, args []string
return err
}
data, err := ioutil.ReadFile(options.Pubkey)
data, err := ioutil.ReadFile(options.PublicKeyPath)
if err != nil {
return fmt.Errorf("error reading SSH public key %v: %v", options.Pubkey, err)
return fmt.Errorf("error reading SSH public key %v: %v", options.PublicKeyPath, err)
}
err = keyStore.AddSSHPublicKey(name, data)
err = keyStore.AddSSHPublicKey(options.Name, data)
if err != nil {
return fmt.Errorf("error adding SSH public key: %v", err)
}

233
cmd/kops/integration_test.go Normal file
View File

@ -0,0 +1,233 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"bytes"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/pem"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/route53"
"github.com/golang/glog"
"golang.org/x/crypto/ssh"
"io/ioutil"
"k8s.io/kops/cloudmock/aws/mockec2"
"k8s.io/kops/cloudmock/aws/mockroute53"
"k8s.io/kops/cmd/kops/util"
"k8s.io/kops/pkg/diff"
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
"k8s.io/kops/util/pkg/vfs"
"os"
"path"
"sort"
"strings"
"testing"
"time"
"reflect"
)
// TestMinimal runs the test on a minimum configuration, similar to kops create cluster minimal.example.com --zones us-west-1a
func TestMinimal(t *testing.T) {
runTest(t, "minimal.example.com", "../../tests/integration/minimal")
}
// TestMinimal_141 runs the test on a configuration from 1.4.1 release
func TestMinimal_141(t *testing.T) {
runTest(t, "minimal-141.example.com", "../../tests/integration/minimal-141")
}
func runTest(t *testing.T, clusterName string, srcDir string) {
var stdout bytes.Buffer
inputYAML := "in.yaml"
expectedTFPath := "kubernetes.tf"
factoryOptions := &util.FactoryOptions{}
factoryOptions.RegistryPath = "memfs://tests"
vfs.Context.ResetMemfsContext(true)
cloud := awsup.InstallMockAWSCloud("us-test-1", "abc")
mockEC2 := &mockec2.MockEC2{}
cloud.MockEC2 = mockEC2
mockRoute53 := &mockroute53.MockRoute53{}
cloud.MockRoute53 = mockRoute53
mockRoute53.Zones = append(mockRoute53.Zones, &route53.HostedZone{
Id: aws.String("123"),
Name: aws.String("example.com."),
})
mockEC2.Images = append(mockEC2.Images, &ec2.Image{
ImageId: aws.String("ami-12345"),
Name: aws.String("k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21"),
OwnerId: aws.String(awsup.WellKnownAccountKopeio),
})
tempDir, err := ioutil.TempDir("", "test")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer func() {
if os.Getenv("KEEP_TEMP_DIR") != "" {
glog.Infof("NOT removing temp directory, because KEEP_TEMP_DIR is set: %s", tempDir)
return
}
err := os.RemoveAll(tempDir)
if err != nil {
t.Fatalf("failed to remove temp dir %q: %v", tempDir, err)
}
}()
factory := util.NewFactory(factoryOptions)
{
options := &CreateOptions{}
options.Filenames = []string{path.Join(srcDir, inputYAML)}
err := RunCreate(factory, &stdout, options)
if err != nil {
t.Fatalf("error running %q create: %v", inputYAML, err)
}
}
{
//publicKeyPath := path.Join(tempDir, "id_rsa.pub")
//privateKeyPath := path.Join(tempDir, "id_rsa")
//
//if err := MakeSSHKeyPair(publicKeyPath, privateKeyPath); err != nil {
// t.Fatalf("error making SSH keypair: %v", err)
//}
options := &CreateSecretPublickeyOptions{}
options.ClusterName = clusterName
options.Name = "admin"
options.PublicKeyPath = path.Join(srcDir, "id_rsa.pub")
err := RunCreateSecretPublicKey(factory, &stdout, options)
if err != nil {
t.Fatalf("error running %q create: %v", inputYAML, err)
}
}
{
options := &UpdateClusterOptions{}
options.InitDefaults()
options.Target = "terraform"
options.OutDir = path.Join(tempDir, "out")
options.MaxTaskDuration = 30 * time.Second
err := RunUpdateCluster(factory, clusterName, &stdout, options)
if err != nil {
t.Fatalf("error running update cluster %q: %v", clusterName, err)
}
}
// Compare main files
{
files, err := ioutil.ReadDir(path.Join(tempDir, "out"))
if err != nil {
t.Fatalf("failed to read dir: %v", err)
}
var fileNames []string
for _, f := range files {
fileNames = append(fileNames, f.Name())
}
sort.Strings(fileNames)
actualFilenames := strings.Join(fileNames, ",")
expectedFilenames := "data,kubernetes.tf"
if actualFilenames != expectedFilenames {
t.Fatalf("unexpected files. actual=%q, expected=%q", actualFilenames, expectedFilenames)
}
actualTF, err := ioutil.ReadFile(path.Join(tempDir, "out", "kubernetes.tf"))
if err != nil {
t.Fatalf("unexpected error reading actual terraform output: %v", err)
}
expectedTF, err := ioutil.ReadFile(path.Join(srcDir, expectedTFPath))
if err != nil {
t.Fatalf("unexpected error reading expected terraform output: %v", err)
}
if !bytes.Equal(actualTF, expectedTF) {
diffString := diff.FormatDiff(string(expectedTF), string(actualTF))
t.Log("diff:\n", diffString)
t.Fatalf("terraform output differed from expected")
}
}
// Compare data files
{
files, err := ioutil.ReadDir(path.Join(tempDir, "out", "data"))
if err != nil {
t.Fatalf("failed to read data dir: %v", err)
}
var actualFilenames []string
for _, f := range files {
actualFilenames = append(actualFilenames, f.Name())
}
expectedFilenames := []string{
"aws_iam_role_masters." + clusterName + "_policy",
"aws_iam_role_nodes." + clusterName + "_policy",
"aws_iam_role_policy_masters." + clusterName + "_policy",
"aws_iam_role_policy_nodes." + clusterName + "_policy",
"aws_key_pair_kubernetes." + clusterName + "-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key",
"aws_launch_configuration_master-us-test-1a.masters." + clusterName + "_user_data",
"aws_launch_configuration_nodes." + clusterName + "_user_data",
}
sort.Strings(expectedFilenames)
if !reflect.DeepEqual(actualFilenames, expectedFilenames) {
t.Fatalf("unexpected data files. actual=%q, expected=%q", actualFilenames, expectedFilenames)
}
// TODO: any verification of data files?
}
}
func MakeSSHKeyPair(publicKeyPath string, privateKeyPath string) error {
privateKey, err := rsa.GenerateKey(rand.Reader, 1024)
if err != nil {
return err
}
var privateKeyBytes bytes.Buffer
privateKeyPEM := &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)}
if err := pem.Encode(&privateKeyBytes, privateKeyPEM); err != nil {
return err
}
if err := ioutil.WriteFile(privateKeyPath, privateKeyBytes.Bytes(), os.FileMode(0700)); err != nil {
return err
}
publicKey, err := ssh.NewPublicKey(&privateKey.PublicKey)
if err != nil {
return err
}
publicKeyBytes := ssh.MarshalAuthorizedKey(publicKey)
if err := ioutil.WriteFile(publicKeyPath, publicKeyBytes, os.FileMode(0744)); err != nil {
return err
}
return nil
}

6
cmd/kops/replace.go
View File

@ -31,12 +31,12 @@ import (
"k8s.io/kubernetes/pkg/kubectl/resource"
)
type UpdateOptions struct {
type ReplaceOptions struct {
resource.FilenameOptions
}
func NewCmdReplace(f *util.Factory, out io.Writer) *cobra.Command {
options := &UpdateOptions{}
options := &ReplaceOptions{}
cmd := &cobra.Command{
Use: "replace -f FILENAME",
@ -57,7 +57,7 @@ func NewCmdReplace(f *util.Factory, out io.Writer) *cobra.Command {
return cmd
}
func RunReplace(f *util.Factory, cmd *cobra.Command, out io.Writer, c *UpdateOptions) error {
func RunReplace(f *util.Factory, cmd *cobra.Command, out io.Writer, c *ReplaceOptions) error {
clientset, err := f.Clientset()
if err != nil {
return err

19
cmd/kops/root.go
View File

@ -32,6 +32,7 @@ import (
"k8s.io/kops/upup/pkg/kutil"
k8sapi "k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
"k8s.io/kubernetes/pkg/util/validation/field"
)
type Factory interface {
@ -221,16 +222,24 @@ func (c *RootCmd) Clientset() (simple.Clientset, error) {
}
func (c *RootCmd) Cluster() (*kopsapi.Cluster, error) {
clientset, err := c.Clientset()
if err != nil {
return nil, err
}
clusterName := c.ClusterName()
if clusterName == "" {
return nil, fmt.Errorf("--name is required")
}
return GetCluster(c.factory, clusterName)
}
func GetCluster(factory *util.Factory, clusterName string) (*kopsapi.Cluster, error) {
if clusterName == "" {
return nil, field.Required(field.NewPath("ClusterName"), "Cluster name is required")
}
clientset, err := factory.Clientset()
if err != nil {
return nil, err
}
cluster, err := clientset.Clusters().Get(clusterName)
if err != nil {
return nil, fmt.Errorf("error reading cluster configuration: %v", err)

63
cmd/kops/update_cluster.go
View File

@ -31,46 +31,60 @@ import (
"k8s.io/kops/upup/pkg/kutil"
"os"
"strings"
"time"
)
type UpdateClusterOptions struct {
Yes bool
Target string
Models string
OutDir string
SSHPublicKey string
Yes bool
Target string
Models string
OutDir string
SSHPublicKey string
MaxTaskDuration time.Duration
}
func (o *UpdateClusterOptions) InitDefaults() {
o.Yes = false
o.Target = "direct"
o.Models = strings.Join(cloudup.CloudupModels, ",")
o.SSHPublicKey = ""
o.OutDir = ""
o.MaxTaskDuration = cloudup.DefaultMaxTaskDuration
}
func NewCmdUpdateCluster(f *util.Factory, out io.Writer) *cobra.Command {
options := &UpdateClusterOptions{}
options.InitDefaults()
cmd := &cobra.Command{
Use: "cluster",
Short: "Update cluster",
Long: `Updates a k8s cluster.`,
Run: func(cmd *cobra.Command, args []string) {
err := RunUpdateCluster(f, cmd, args, os.Stdout, options)
err := rootCommand.ProcessArgs(args)
if err != nil {
exitWithError(err)
}
clusterName := rootCommand.ClusterName()
err = RunUpdateCluster(f, clusterName, os.Stdout, options)
if err != nil {
exitWithError(err)
}
},
}
cmd.Flags().BoolVar(&options.Yes, "yes", false, "Actually create cloud resources")
cmd.Flags().StringVar(&options.Target, "target", "direct", "Target - direct, terraform")
cmd.Flags().StringVar(&options.Models, "model", strings.Join(cloudup.CloudupModels, ","), "Models to apply (separate multiple models with commas)")
cmd.Flags().StringVar(&options.SSHPublicKey, "ssh-public-key", "", "SSH public key to use (deprecated: use kops create secret instead)")
cmd.Flags().StringVar(&options.OutDir, "out", "", "Path to write any local output")
cmd.Flags().BoolVar(&options.Yes, "yes", options.Yes, "Actually create cloud resources")
cmd.Flags().StringVar(&options.Target, "target", options.Target, "Target - direct, terraform")
cmd.Flags().StringVar(&options.Models, "model", options.Models, "Models to apply (separate multiple models with commas)")
cmd.Flags().StringVar(&options.SSHPublicKey, "ssh-public-key", options.SSHPublicKey, "SSH public key to use (deprecated: use kops create secret instead)")
cmd.Flags().StringVar(&options.OutDir, "out", options.OutDir, "Path to write any local output")
return cmd
}
func RunUpdateCluster(f *util.Factory, cmd *cobra.Command, args []string, out io.Writer, c *UpdateClusterOptions) error {
err := rootCommand.ProcessArgs(args)
if err != nil {
return err
}
func RunUpdateCluster(f *util.Factory, clusterName string, out io.Writer, c *UpdateClusterOptions) error {
isDryrun := false
targetName := c.Target
@ -94,7 +108,7 @@ func RunUpdateCluster(f *util.Factory, cmd *cobra.Command, args []string, out io
}
}
cluster, err := rootCommand.Cluster()
cluster, err := GetCluster(f, clusterName)
if err != nil {
return err
}
@ -129,12 +143,13 @@ func RunUpdateCluster(f *util.Factory, cmd *cobra.Command, args []string, out io
}
applyCmd := &cloudup.ApplyClusterCmd{
Cluster: cluster,
Models: strings.Split(c.Models, ","),
Clientset: clientset,
TargetName: targetName,
OutDir: c.OutDir,
DryRun: isDryrun,
Cluster: cluster,
Models: strings.Split(c.Models, ","),
Clientset: clientset,
TargetName: targetName,
OutDir: c.OutDir,
DryRun: isDryrun,
MaxTaskDuration: c.MaxTaskDuration,
}
err = applyCmd.Run()
if err != nil {

1
tests/integration/minimal/id_rsa.pub Executable file
View File

@ -0,0 +1 @@
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==

73
tests/integration/minimal/in.yaml Normal file
View File

@ -0,0 +1,73 @@
kind: Cluster
metadata:
creationTimestamp: "2016-12-10T22:42:27Z"
name: minimal.example.com
spec:
adminAccess:
- 0.0.0.0/0
channel: stable
cloudProvider: aws
configBase: memfs://clusters.example.com/minimal.example.com
etcdClusters:
- etcdMembers:
- name: us-test-1a
zone: us-test-1a
name: main
- etcdMembers:
- name: us-test-1a
zone: us-test-1a
name: events
kubernetesVersion: v1.4.6
masterInternalName: api.internal.minimal.example.com
masterPublicName: api.minimal.example.com
networkCIDR: 172.20.0.0/16
networking:
kubenet: {}
nonMasqueradeCIDR: 100.64.0.0/10
topology:
bastion:
idleTimeout: 120
machineType: t2.medium
masters: public
nodes: public
zones:
- cidr: 172.20.32.0/19
name: us-test-1a
---
kind: InstanceGroup
metadata:
creationTimestamp: "2016-12-10T22:42:28Z"
name: nodes
labels:
kops.k8s.io/cluster: minimal.example.com
spec:
associatePublicIp: true
image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21
machineType: t2.medium
maxSize: 2
minSize: 2
role: Node
zones:
- us-test-1a
---
kind: InstanceGroup
metadata:
creationTimestamp: "2016-12-10T22:42:28Z"
name: master-us-test-1a
labels:
kops.k8s.io/cluster: minimal.example.com
spec:
associatePublicIp: true
image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21
machineType: m3.medium
maxSize: 1
minSize: 1
role: Master
zones:
- us-test-1a

312
tests/integration/minimal/kubernetes.tf Normal file
View File

@ -0,0 +1,312 @@
resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" {
name = "master-us-test-1a.masters.minimal.example.com"
launch_configuration = "${aws_launch_configuration.master-us-test-1a-masters-minimal-example-com.id}"
max_size = 1
min_size = 1
vpc_zone_identifier = ["${aws_subnet.us-test-1a-minimal-example-com.id}"]
tag = {
key = "KubernetesCluster"
value = "minimal.example.com"
propagate_at_launch = true
}
tag = {
key = "Name"
value = "master-us-test-1a.masters.minimal.example.com"
propagate_at_launch = true
}
tag = {
key = "k8s.io/role/master"
value = "1"
propagate_at_launch = true
}
}
resource "aws_autoscaling_group" "nodes-minimal-example-com" {
name = "nodes.minimal.example.com"
launch_configuration = "${aws_launch_configuration.nodes-minimal-example-com.id}"
max_size = 2
min_size = 2
vpc_zone_identifier = ["${aws_subnet.us-test-1a-minimal-example-com.id}"]
tag = {
key = "KubernetesCluster"
value = "minimal.example.com"
propagate_at_launch = true
}
tag = {
key = "Name"
value = "nodes.minimal.example.com"
propagate_at_launch = true
}
tag = {
key = "k8s.io/role/node"
value = "1"
propagate_at_launch = true
}
}
resource "aws_ebs_volume" "us-test-1a-etcd-events-minimal-example-com" {
availability_zone = "us-test-1a"
size = 20
type = "gp2"
encrypted = false
tags = {
KubernetesCluster = "minimal.example.com"
Name = "us-test-1a.etcd-events.minimal.example.com"
"k8s.io/etcd/events" = "us-test-1a/us-test-1a"
"k8s.io/role/master" = "1"
}
}
resource "aws_ebs_volume" "us-test-1a-etcd-main-minimal-example-com" {
availability_zone = "us-test-1a"
size = 20
type = "gp2"
encrypted = false
tags = {
KubernetesCluster = "minimal.example.com"
Name = "us-test-1a.etcd-main.minimal.example.com"
"k8s.io/etcd/main" = "us-test-1a/us-test-1a"
"k8s.io/role/master" = "1"
}
}
resource "aws_iam_instance_profile" "masters-minimal-example-com" {
name = "masters.minimal.example.com"
roles = ["${aws_iam_role.masters-minimal-example-com.name}"]
}
resource "aws_iam_instance_profile" "nodes-minimal-example-com" {
name = "nodes.minimal.example.com"
roles = ["${aws_iam_role.nodes-minimal-example-com.name}"]
}
resource "aws_iam_role" "masters-minimal-example-com" {
name = "masters.minimal.example.com"
assume_role_policy = "${file("data/aws_iam_role_masters.minimal.example.com_policy")}"
}
resource "aws_iam_role" "nodes-minimal-example-com" {
name = "nodes.minimal.example.com"
assume_role_policy = "${file("data/aws_iam_role_nodes.minimal.example.com_policy")}"
}
resource "aws_iam_role_policy" "masters-minimal-example-com" {
name = "masters.minimal.example.com"
role = "${aws_iam_role.masters-minimal-example-com.name}"
policy = "${file("data/aws_iam_role_policy_masters.minimal.example.com_policy")}"
}
resource "aws_iam_role_policy" "nodes-minimal-example-com" {
name = "nodes.minimal.example.com"
role = "${aws_iam_role.nodes-minimal-example-com.name}"
policy = "${file("data/aws_iam_role_policy_nodes.minimal.example.com_policy")}"
}
resource "aws_internet_gateway" "minimal-example-com" {
vpc_id = "${aws_vpc.minimal-example-com.id}"
tags = {
KubernetesCluster = "minimal.example.com"
Name = "minimal.example.com"
}
}
resource "aws_key_pair" "kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" {
key_name = "kubernetes.minimal.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57"
public_key = "${file("data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key")}"
}
resource "aws_launch_configuration" "master-us-test-1a-masters-minimal-example-com" {
name_prefix = "master-us-test-1a.masters.minimal.example.com-"
image_id = "ami-12345"
instance_type = "m3.medium"
key_name = "${aws_key_pair.kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id}"
iam_instance_profile = "${aws_iam_instance_profile.masters-minimal-example-com.id}"
security_groups = ["${aws_security_group.masters-minimal-example-com.id}"]
associate_public_ip_address = true
user_data = "${file("data/aws_launch_configuration_master-us-test-1a.masters.minimal.example.com_user_data")}"
root_block_device = {
volume_type = "gp2"
volume_size = 20
delete_on_termination = true
}
ephemeral_block_device = {
device_name = "/dev/sdc"
virtual_name = "ephemeral0"
}
lifecycle = {
create_before_destroy = true
}
}
resource "aws_launch_configuration" "nodes-minimal-example-com" {
name_prefix = "nodes.minimal.example.com-"
image_id = "ami-12345"
instance_type = "t2.medium"
key_name = "${aws_key_pair.kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id}"
iam_instance_profile = "${aws_iam_instance_profile.nodes-minimal-example-com.id}"
security_groups = ["${aws_security_group.nodes-minimal-example-com.id}"]
associate_public_ip_address = true
user_data = "${file("data/aws_launch_configuration_nodes.minimal.example.com_user_data")}"
root_block_device = {
volume_type = "gp2"
volume_size = 20
delete_on_termination = true
}
lifecycle = {
create_before_destroy = true
}
}
resource "aws_route" "0-0-0-0--0" {
route_table_id = "${aws_route_table.minimal-example-com.id}"
destination_cidr_block = "0.0.0.0/0"
gateway_id = "${aws_internet_gateway.minimal-example-com.id}"
}
resource "aws_route_table" "minimal-example-com" {
vpc_id = "${aws_vpc.minimal-example-com.id}"
tags = {
KubernetesCluster = "minimal.example.com"
Name = "minimal.example.com"
}
}
resource "aws_route_table_association" "us-test-1a-minimal-example-com" {
subnet_id = "${aws_subnet.us-test-1a-minimal-example-com.id}"
route_table_id = "${aws_route_table.minimal-example-com.id}"
}
resource "aws_security_group" "masters-minimal-example-com" {
name = "masters.minimal.example.com"
vpc_id = "${aws_vpc.minimal-example-com.id}"
description = "Security group for masters"
tags = {
KubernetesCluster = "minimal.example.com"
Name = "masters.minimal.example.com"
}
}
resource "aws_security_group" "nodes-minimal-example-com" {
name = "nodes.minimal.example.com"
vpc_id = "${aws_vpc.minimal-example-com.id}"
description = "Security group for nodes"
tags = {
KubernetesCluster = "minimal.example.com"
Name = "nodes.minimal.example.com"
}
}
resource "aws_security_group_rule" "all-master-to-master" {
type = "ingress"
security_group_id = "${aws_security_group.masters-minimal-example-com.id}"
source_security_group_id = "${aws_security_group.masters-minimal-example-com.id}"
from_port = 0
to_port = 0
protocol = "-1"
}
resource "aws_security_group_rule" "all-master-to-node" {
type = "ingress"
security_group_id = "${aws_security_group.nodes-minimal-example-com.id}"
source_security_group_id = "${aws_security_group.masters-minimal-example-com.id}"
from_port = 0
to_port = 0
protocol = "-1"
}
resource "aws_security_group_rule" "all-node-to-master" {
type = "ingress"
security_group_id = "${aws_security_group.masters-minimal-example-com.id}"
source_security_group_id = "${aws_security_group.nodes-minimal-example-com.id}"
from_port = 0
to_port = 0
protocol = "-1"
}
resource "aws_security_group_rule" "all-node-to-node" {
type = "ingress"
security_group_id = "${aws_security_group.nodes-minimal-example-com.id}"
source_security_group_id = "${aws_security_group.nodes-minimal-example-com.id}"
from_port = 0
to_port = 0
protocol = "-1"
}
resource "aws_security_group_rule" "https-external-to-master-0" {
type = "ingress"
security_group_id = "${aws_security_group.masters-minimal-example-com.id}"
from_port = 443
to_port = 443
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
}
resource "aws_security_group_rule" "master-egress" {
type = "egress"
security_group_id = "${aws_security_group.masters-minimal-example-com.id}"
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
resource "aws_security_group_rule" "node-egress" {
type = "egress"
security_group_id = "${aws_security_group.nodes-minimal-example-com.id}"
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
resource "aws_security_group_rule" "ssh-external-to-master-0" {
type = "ingress"
security_group_id = "${aws_security_group.masters-minimal-example-com.id}"
from_port = 22
to_port = 22
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
}
resource "aws_security_group_rule" "ssh-external-to-node-0" {
type = "ingress"
security_group_id = "${aws_security_group.nodes-minimal-example-com.id}"
from_port = 22
to_port = 22
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
}
resource "aws_subnet" "us-test-1a-minimal-example-com" {
vpc_id = "${aws_vpc.minimal-example-com.id}"
cidr_block = "172.20.32.0/19"
availability_zone = "us-test-1a"
tags = {
KubernetesCluster = "minimal.example.com"
Name = "us-test-1a.minimal.example.com"
}
}
resource "aws_vpc" "minimal-example-com" {
cidr_block = "172.20.0.0/16"
enable_dns_hostnames = true
enable_dns_support = true
tags = {
KubernetesCluster = "minimal.example.com"
Name = "minimal.example.com"
}
}
resource "aws_vpc_dhcp_options" "minimal-example-com" {
domain_name = "us-test-1.compute.internal"
domain_name_servers = ["AmazonProvidedDNS"]
tags = {
KubernetesCluster = "minimal.example.com"
Name = "minimal.example.com"
}
}
resource "aws_vpc_dhcp_options_association" "minimal-example-com" {
vpc_id = "${aws_vpc.minimal-example-com.id}"
dhcp_options_id = "${aws_vpc_dhcp_options.minimal-example-com.id}"
}

9
upup/pkg/fi/cloudup/apply_cluster.go
View File

@ -45,7 +45,7 @@ const (
NodeUpVersion = "1.4.1"
)
const MaxTaskDuration = 10 * time.Minute
const DefaultMaxTaskDuration = 10 * time.Minute
var CloudupModels = []string{"config", "proto", "cloudup"}
@ -79,9 +79,14 @@ type ApplyClusterCmd struct {
// DryRun is true if this is only a dry run
DryRun bool
MaxTaskDuration time.Duration
}
func (c *ApplyClusterCmd) Run() error {
if c.MaxTaskDuration == 0 {
c.MaxTaskDuration = DefaultMaxTaskDuration
}
if c.InstanceGroups == nil {
list, err := c.Clientset.InstanceGroups(c.Cluster.Name).List(k8sapi.ListOptions{})
@ -563,7 +568,7 @@ func (c *ApplyClusterCmd) Run() error {
}
defer context.Close()
err = context.RunTasks(MaxTaskDuration)
err = context.RunTasks(c.MaxTaskDuration)
if err != nil {
return fmt.Errorf("error running tasks: %v", err)
}

15
upup/pkg/fi/cloudup/awsup/aws_cloud.go
View File

@ -27,6 +27,7 @@ import (
"github.com/aws/aws-sdk-go/service/elb"
"github.com/aws/aws-sdk-go/service/iam"
"github.com/aws/aws-sdk-go/service/route53"
"github.com/aws/aws-sdk-go/service/route53/route53iface"
"github.com/golang/glog"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kubernetes/federation/pkg/dnsprovider"
@ -55,6 +56,8 @@ const DeleteTagsLogInterval = 10 // this is in "retry intervals"
const TagClusterName = "KubernetesCluster"
const WellKnownAccountKopeio = "383156758163"
type AWSCloud interface {
fi.Cloud
@ -64,7 +67,7 @@ type AWSCloud interface {
IAM() *iam.IAM
ELB() *elb.ELB
Autoscaling() *autoscaling.AutoScaling
Route53() *route53.Route53
Route53() route53iface.Route53API
// TODO: Document and rationalize these tags/filters methods
AddTags(name *string, tags map[string]string)
@ -543,6 +546,10 @@ func (t *awsCloudImplementation) DescribeVPC(vpcID string) (*ec2.Vpc, error) {
// owner/name in which case we find the image with the specified name, owned by owner
// name in which case we find the image with the specified name, with the current owner
func (c *awsCloudImplementation) ResolveImage(name string) (*ec2.Image, error) {
return resolveImage(c.ec2, name)
}
func resolveImage(ec2Client ec2iface.EC2API, name string) (*ec2.Image, error) {
// TODO: Cache this result during a single execution (we get called multiple times)
glog.V(2).Infof("Calling DescribeImages to resolve name %q", name)
request := &ec2.DescribeImagesInput{}
@ -563,7 +570,7 @@ func (c *awsCloudImplementation) ResolveImage(name string) (*ec2.Image, error) {
// Check for well known owner aliases
switch owner {
case "kope.io":
owner = "383156758163"
owner = WellKnownAccountKopeio
case "redhat.com":
owner = "309956199498"
}
@ -575,7 +582,7 @@ func (c *awsCloudImplementation) ResolveImage(name string) (*ec2.Image, error) {
}
}
response, err := c.EC2().DescribeImages(request)
response, err := ec2Client.DescribeImages(request)
if err != nil {
return nil, fmt.Errorf("error listing images: %v", err)
}
@ -664,6 +671,6 @@ func (c *awsCloudImplementation) Autoscaling() *autoscaling.AutoScaling {
return c.autoscaling
}
func (c *awsCloudImplementation) Route53() *route53.Route53 {
func (c *awsCloudImplementation) Route53() route53iface.Route53API {
return c.route53
}

31
upup/pkg/fi/cloudup/awsup/mock_aws_cloud.go
View File

@ -24,10 +24,11 @@ import (
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
"github.com/aws/aws-sdk-go/service/elb"
"github.com/aws/aws-sdk-go/service/iam"
"github.com/aws/aws-sdk-go/service/route53"
"github.com/aws/aws-sdk-go/service/route53/route53iface"
"github.com/golang/glog"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kubernetes/federation/pkg/dnsprovider"
dnsproviderroute53 "k8s.io/kubernetes/federation/pkg/dnsprovider/providers/aws/route53"
)
type MockAWSCloud struct {
@ -40,12 +41,13 @@ type MockAWSCloud struct {
var _ fi.Cloud = (*MockAWSCloud)(nil)
func InstallMockAWSCloud(region string, zoneLetters string) {
func InstallMockAWSCloud(region string, zoneLetters string) *MockAWSCloud {
i := BuildMockAWSCloud(region, zoneLetters)
awsCloudInstances[region] = i
allRegions = []*ec2.Region{
{RegionName: aws.String(region)},
}
return i
}
func BuildMockAWSCloud(region string, zoneLetters string) *MockAWSCloud {
@ -63,7 +65,8 @@ func BuildMockAWSCloud(region string, zoneLetters string) *MockAWSCloud {
}
type MockCloud struct {
MockEC2 ec2iface.EC2API
MockEC2 ec2iface.EC2API
MockRoute53 route53iface.Route53API
}
func (c *MockCloud) ProviderID() fi.CloudProviderID {
@ -71,7 +74,10 @@ func (c *MockCloud) ProviderID() fi.CloudProviderID {
}
func (c *MockCloud) DNS() (dnsprovider.Interface, error) {
return nil, fmt.Errorf("MockCloud DNS not implemented")
if c.MockRoute53 == nil {
return nil, fmt.Errorf("MockRoute53 not set")
}
return dnsproviderroute53.NewInterfaceWithStub(c.MockRoute53), nil
}
func (c *MockAWSCloud) Region() string {
@ -83,7 +89,12 @@ func (c *MockAWSCloud) DescribeAvailabilityZones() ([]*ec2.AvailabilityZone, err
}
func (c *MockAWSCloud) AddTags(name *string, tags map[string]string) {
glog.Fatalf("MockAWSCloud AddTags not implemented")
if name != nil {
tags["Name"] = *name
}
for k, v := range c.tags {
tags[k] = v
}
}
func (c *MockAWSCloud) BuildFilters(name *string) []*ec2.Filter {
@ -128,7 +139,7 @@ func (c *MockAWSCloud) DescribeVPC(vpcID string) (*ec2.Vpc, error) {
}
func (c *MockAWSCloud) ResolveImage(name string) (*ec2.Image, error) {
return nil, fmt.Errorf("MockAWSCloud ResolveImage not implemented")
return resolveImage(c.MockEC2, name)
}
func (c *MockAWSCloud) WithTags(tags map[string]string) AWSCloud {
@ -160,7 +171,9 @@ func (c *MockAWSCloud) Autoscaling() *autoscaling.AutoScaling {
return nil
}
func (c *MockAWSCloud) Route53() *route53.Route53 {
glog.Fatalf("MockAWSCloud Route53 not implemented")
return nil
func (c *MockAWSCloud) Route53() route53iface.Route53API {
if c.MockRoute53 == nil {
glog.Fatalf("MockRoute53 not set")
}
return c.MockRoute53
}

3
upup/pkg/fi/cloudup/iam_builder.go
View File

@ -222,6 +222,9 @@ func (b *IAMPolicyBuilder) BuildAWSIAMPolicy() (*IAMPolicy, error) {
iamPrefix + ":s3:::" + s3Path.Bucket(),
},
})
} else if _, ok := vfsPath.(*vfs.MemFSPath); ok {
// Tests -ignore - nothing we can do in terms of IAM policy
glog.Warningf("ignoring memfs path %q for IAM policy builder", vfsPath)
} else {
// We could implement this approach, but it seems better to get all clouds using cluster-readable storage
return nil, fmt.Errorf("path is not cluster readable: %v", root)

27
util/pkg/vfs/context.go
View File

@ -28,7 +28,8 @@ import (
// VFSContext is a 'context' for VFS, that is normally a singleton
// but allows us to configure S3 credentials, for example
type VFSContext struct {
s3Context *S3Context
s3Context *S3Context
memfsContext *MemFSContext
}
var Context = VFSContext{
@ -85,6 +86,10 @@ func (c *VFSContext) BuildVfsPath(p string) (Path, error) {
return c.buildS3Path(p)
}
if strings.HasPrefix(p, "memfs://") {
return c.buildMemFSPath(p)
}
return nil, fmt.Errorf("unknown / unhandled path type: %q", p)
}
@ -129,3 +134,23 @@ func (c *VFSContext) buildS3Path(p string) (*S3Path, error) {
s3path := NewS3Path(c.s3Context, bucket, u.Path)
return s3path, nil
}
func (c *VFSContext) buildMemFSPath(p string) (*MemFSPath, error) {
if !strings.HasPrefix(p, "memfs://") {
return nil, fmt.Errorf("memfs path not recognized: %q", p)
}
location := strings.TrimPrefix(p, "memfs://")
if c.memfsContext == nil {
// We only initialize this in unit tests etc
return nil, fmt.Errorf("memfs context not initialized")
}
fspath := NewMemFSPath(c.memfsContext, location)
return fspath, nil
}
func (c *VFSContext) ResetMemfsContext(clusterReadable bool) {
c.memfsContext = NewMemFSContext()
if clusterReadable {
c.memfsContext.MarkClusterReadable()
}
}

7
util/pkg/vfs/memfs.go
View File

@ -20,12 +20,14 @@ import (
"os"
"path"
"strings"
"sync"
)
type MemFSPath struct {
context *MemFSContext
location string
mutex sync.Mutex
contents []byte
children map[string]*MemFSPath
}
@ -62,6 +64,9 @@ func NewMemFSPath(context *MemFSContext, location string) *MemFSPath {
}
func (p *MemFSPath) Join(relativePath ...string) Path {
p.mutex.Lock()
defer p.mutex.Unlock()
joined := path.Join(relativePath...)
tokens := strings.Split(joined, "/")
current := p
@ -130,7 +135,7 @@ func (p *MemFSPath) Base() string {
}
func (p *MemFSPath) Path() string {
return p.location
return "memfs://" + p.location
}
func (p *MemFSPath) String() string {

3
util/pkg/vfs/vfs.go
View File

@ -96,6 +96,9 @@ func IsClusterReadable(p Path) bool {
case *FSPath:
return false
case *MemFSPath:
return false
default:
glog.Fatalf("IsClusterReadable not implemented for type %T", p)
return false