Simplify how we build addon manifests, support image redirection

pkg/assets/builder.go

@@ -0,0 +1,54 @@
+package assets
+
+import (
+	"fmt"
+	"github.com/golang/glog"
+	"k8s.io/kops/pkg/kubemanifest"
+	"os"
+	"strings"
+)
+
+// AssetBuilder discovers and remaps assets
+type AssetBuilder struct {
+}
+
+func NewAssetBuilder() *AssetBuilder {
+	return &AssetBuilder{}
+}
+
+func (a *AssetBuilder) RemapManifest(data []byte) ([]byte, error) {
+	manifests, err := kubemanifest.LoadManifestsFrom(data)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, manifest := range manifests {
+		err := manifest.RemapImages(a.remapImage)
+		if err != nil {
+			return nil, fmt.Errorf("error remapping images: %v", err)
+		}
+		y, err := manifest.ToYAML()
+		if err != nil {
+			return nil, fmt.Errorf("error re-marshalling manifest: %v", err)
+		}
+
+		glog.Infof("manifest: %v", string(y))
+	}
+
+	return data, nil
+}
+
+func (a *AssetBuilder) remapImage(image string) (string, error) {
+	if strings.HasPrefix(image, "kope/dns-controller:") {
+		// To use user-defined DNS Controller:
+		// 1. DOCKER_REGISTRY=[your docker hub repo] make dns-controller-push
+		// 2. export DNSCONTROLLER_IMAGE=[your docker hub repo]
+		// 3. make kops and create/apply cluster
+		override := os.Getenv("DNSCONTROLLER_IMAGE")
+		if override != "" {
+			return override, nil
+		}
+	}
+
+	return image, nil
+}

pkg/assets/templates.go

@@ -0,0 +1,130 @@
+package assets
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/golang/glog"
+	"io"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/util/pkg/vfs"
+	"os"
+	"strings"
+	"text/template"
+)
+
+type Templates struct {
+	cluster           *kops.Cluster
+	resources         map[string]fi.Resource
+	TemplateFunctions template.FuncMap
+}
+
+func LoadTemplates(cluster *kops.Cluster, base vfs.Path) (*Templates, error) {
+	t := &Templates{
+		cluster:           cluster,
+		resources:         make(map[string]fi.Resource),
+		TemplateFunctions: make(template.FuncMap),
+	}
+	err := t.loadFrom(base)
+	if err != nil {
+		return nil, err
+	}
+	return t, nil
+}
+
+func (t *Templates) Find(key string) fi.Resource {
+	return t.resources[key]
+}
+
+func (t *Templates) loadFrom(base vfs.Path) error {
+	files, err := base.ReadTree()
+	if err != nil {
+		return fmt.Errorf("error reading from %s", base)
+	}
+
+	for _, f := range files {
+		contents, err := f.ReadFile()
+		if err != nil {
+			if os.IsNotExist(err) {
+				// This is just an annoyance of gobindata - we can't tell the difference between files & directories.  Ignore.
+				continue
+			}
+			return fmt.Errorf("error reading %s: %v", f, err)
+		}
+
+		key, err := vfs.RelativePath(base, f)
+		if err != nil {
+			return fmt.Errorf("error getting relative path for %s", f)
+		}
+
+		var resource fi.Resource
+		if strings.HasSuffix(key, ".template") {
+			key = strings.TrimSuffix(key, ".template")
+			glog.V(6).Infof("loading (templated) resource %q", key)
+
+			resource = &templateResource{
+				template: string(contents),
+				loader:   t,
+				key:      key,
+			}
+		} else {
+			glog.V(6).Infof("loading resource %q", key)
+			resource = fi.NewBytesResource(contents)
+
+		}
+
+		t.resources[key] = resource
+	}
+	return nil
+}
+
+func (l *Templates) executeTemplate(key string, d string) (string, error) {
+	t := template.New(key)
+
+	funcMap := make(template.FuncMap)
+	//funcMap["Args"] = func() []string {
+	//	return args
+	//}
+	//funcMap["RenderResource"] = func(resourceName string, args []string) (string, error) {
+	//	return l.renderResource(resourceName, args)
+	//}
+	for k, fn := range l.TemplateFunctions {
+		funcMap[k] = fn
+	}
+	t.Funcs(funcMap)
+
+	t.Option("missingkey=zero")
+
+	spec := l.cluster.Spec
+
+	_, err := t.Parse(d)
+	if err != nil {
+		return "", fmt.Errorf("error parsing template %q: %v", key, err)
+	}
+
+	var buffer bytes.Buffer
+	err = t.ExecuteTemplate(&buffer, key, spec)
+	if err != nil {
+		return "", fmt.Errorf("error executing template %q: %v", key, err)
+	}
+
+	return buffer.String(), nil
+}
+
+type templateResource struct {
+	key      string
+	loader   *Templates
+	template string
+}
+
+var _ fi.Resource = &templateResource{}
+
+func (a *templateResource) Open() (io.Reader, error) {
+	var err error
+	result, err := a.loader.executeTemplate(a.key, a.template)
+	if err != nil {
+		return nil, fmt.Errorf("error executing resource template %q: %v", a.key, err)
+	}
+	reader := bytes.NewReader([]byte(result))
+	return reader, nil
+}

pkg/kubemanifest/images.go

@@ -0,0 +1,53 @@
+package kubemanifest
+
+import (
+	"fmt"
+	"github.com/golang/glog"
+	"strings"
+)
+
+type ImageRemapFunction func(image string) (string, error)
+
+func (m *Manifest) RemapImages(mapper ImageRemapFunction) error {
+	visitor := &imageRemapVisitor{
+		mapper: mapper,
+	}
+	err := m.accept(visitor)
+	if err != nil {
+		return err
+	}
+	//if changed {
+	//	// invalidate cached rendering
+	//	m.bytes = nil
+	//}
+	return nil
+}
+
+type imageRemapVisitor struct {
+	visitorBase
+	mapper ImageRemapFunction
+}
+
+func (m *imageRemapVisitor) VisitString(path []string, v string, mutator func(string)) error {
+	n := len(path)
+	if n < 1 || path[n-1] != "image" {
+		return nil
+	}
+
+	// Deployments look like spec.template.spec.containers.[2].image
+	if n < 3 || path[n-3] != "containers" {
+		glog.Warningf("Skipping likely image field: %s", strings.Join(path, "."))
+		return nil
+	}
+
+	image := v
+	glog.V(4).Infof("Consider image for re-mapping: %q", image)
+	remapped, err := m.mapper(v)
+	if err != nil {
+		return fmt.Errorf("error remapping image %q: %v", image, err)
+	}
+	if remapped != image {
+		mutator(remapped)
+	}
+	return nil
+}

pkg/kubemanifest/manifest.go

@@ -0,0 +1,55 @@
+package kubemanifest
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/ghodss/yaml"
+	"github.com/golang/glog"
+)
+
+type Manifest struct {
+	//bytes []byte
+	data map[string]interface{}
+}
+
+func LoadManifestsFrom(contents []byte) ([]*Manifest, error) {
+	var manifests []*Manifest
+
+	// TODO: Support more separators?
+	sections := bytes.Split(contents, []byte("\n---\n"))
+
+	for _, section := range sections {
+		data := make(map[string]interface{})
+		err := yaml.Unmarshal(section, &data)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing yaml: %v", err)
+		}
+
+		manifest := &Manifest{
+			//bytes: section,
+			data: data,
+		}
+		manifests = append(manifests, manifest)
+	}
+
+	return manifests, nil
+}
+
+func (m *Manifest) ToYAML() ([]byte, error) {
+	//if m.bytes == nil {
+	b, err := yaml.Marshal(m.data)
+	if err != nil {
+		return nil, fmt.Errorf("error marshalling manifest to yaml: %v", err)
+	}
+	//	m.bytes = b
+	//}
+	//return m.bytes, nil
+	return b, nil
+}
+
+func (m *Manifest) accept(visitor Visitor) error {
+	err := visit(visitor, m.data, []string{}, func(v interface{}) {
+		glog.Fatalf("cannot mutate top-level data")
+	})
+	return err
+}

pkg/kubemanifest/visitor.go

@@ -0,0 +1,92 @@
+package kubemanifest
+
+import (
+	"fmt"
+	"github.com/golang/glog"
+	"strings"
+)
+
+type visitorBase struct {
+}
+
+func (m *visitorBase) VisitString(path []string, v string, mutator func(string)) error {
+	glog.Infof("string value at %s: %s", strings.Join(path, "."), v)
+	return nil
+}
+
+func (m *visitorBase) VisitBool(path []string, v bool, mutator func(bool)) error {
+	glog.Infof("string value at %s: %s", strings.Join(path, "."), v)
+	return nil
+}
+
+func (m *visitorBase) VisitFloat64(path []string, v float64, mutator func(float64)) error {
+	glog.Infof("float64 value at %s: %s", strings.Join(path, "."), v)
+	return nil
+}
+
+type Visitor interface {
+	VisitBool(path []string, v bool, mutator func(bool)) error
+	VisitString(path []string, v string, mutator func(string)) error
+	VisitFloat64(path []string, v float64, mutator func(float64)) error
+}
+
+func visit(visitor Visitor, data interface{}, path []string, mutator func(interface{})) error {
+	switch data.(type) {
+	case string:
+		err := visitor.VisitString(path, data.(string), func(v string) {
+			mutator(v)
+		})
+		if err != nil {
+			return err
+		}
+
+	case bool:
+		err := visitor.VisitBool(path, data.(bool), func(v bool) {
+			mutator(v)
+		})
+		if err != nil {
+			return err
+		}
+
+	case float64:
+		err := visitor.VisitFloat64(path, data.(float64), func(v float64) {
+			mutator(v)
+		})
+		if err != nil {
+			return err
+		}
+
+	case map[string]interface{}:
+		m := data.(map[string]interface{})
+		for k, v := range m {
+			path = append(path, k)
+
+			err := visit(visitor, v, path, func(v interface{}) {
+				m[k] = v
+			})
+			if err != nil {
+				return err
+			}
+			path = path[:len(path)-1]
+		}
+
+	case []interface{}:
+		s := data.([]interface{})
+		for i, v := range s {
+			path = append(path, fmt.Sprintf("[%d]", i))
+
+			err := visit(visitor, v, path, func(v interface{}) {
+				s[i] = v
+			})
+			if err != nil {
+				return err
+			}
+			path = path[:len(path)-1]
+		}
+
+	default:
+		return fmt.Errorf("unhandled type in manifest: %T", data)
+	}
+
+	return nil
+}

upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/pre-k8s-1.6.yaml.template

@@ -28,7 +28,7 @@ spec:
       hostNetwork: true
       containers:
       - name: dns-controller
-        image: {{ DnsControllerImage }}:1.7.0
+        image: kope/dns-controller:1.7.0
         command:
 {{ range $arg := DnsControllerArgv }}
         - "{{ $arg }}"

upup/pkg/fi/cloudup/apply_cluster.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/kops/pkg/apis/kops/util"
 	"k8s.io/kops/pkg/apis/kops/validation"
 	"k8s.io/kops/pkg/apis/nodeup"
+	"k8s.io/kops/pkg/assets"
 	"k8s.io/kops/pkg/client/simple"
 	"k8s.io/kops/pkg/dns"
 	"k8s.io/kops/pkg/featureflag"
@@ -39,6 +40,7 @@ import (
 	"k8s.io/kops/pkg/model/components"
 	"k8s.io/kops/pkg/model/gcemodel"
 	"k8s.io/kops/pkg/model/vspheremodel"
+	"k8s.io/kops/upup/models"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awstasks"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
@@ -418,6 +420,8 @@ func (c *ApplyClusterCmd) Run() error {
 	l.WorkDir = c.OutDir
 	l.ModelStore = modelStore
 
+	assetBuilder := assets.NewAssetBuilder()
+
 	var fileModels []string
 	for _, m := range c.Models {
 		switch m {
@@ -425,8 +429,18 @@ func (c *ApplyClusterCmd) Run() error {
 		// No proto code options; no file model
 
 		case "cloudup":
+			templates, err := assets.LoadTemplates(cluster, models.NewAssetPath("cloudup/resources"))
+			if err != nil {
+				return fmt.Errorf("error loading templates: %v", err)
+			}
+			tf.AddTo(templates.TemplateFunctions)
+
 			l.Builders = append(l.Builders,
-				&BootstrapChannelBuilder{cluster: cluster},
+				&BootstrapChannelBuilder{
+					cluster:      cluster,
+					templates:    templates,
+					assetBuilder: assetBuilder,
+				},
 			)
 
 			switch kops.CloudProviderID(cluster.Spec.CloudProvider) {

upup/pkg/fi/cloudup/bootstrapchannelbuilder.go

@@ -21,6 +21,7 @@ import (
 
 	channelsapi "k8s.io/kops/channels/pkg/api"
 	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/assets"
 	"k8s.io/kops/pkg/featureflag"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/fitasks"
@@ -28,7 +29,9 @@ import (
 )
 
 type BootstrapChannelBuilder struct {
-	cluster *kops.Cluster
+	cluster      *kops.Cluster
+	templates    *assets.Templates
+	assetBuilder *assets.AssetBuilder
 }
 
 var _ fi.ModelBuilder = &BootstrapChannelBuilder{}
@@ -56,10 +59,26 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error {
 
 	for key, manifest := range manifests {
 		name := b.cluster.ObjectMeta.Name + "-addons-" + key
+
+		manifestResource := b.templates.Find(manifest)
+		if manifestResource == nil {
+			return fmt.Errorf("unable to find manifest %s", manifest)
+		}
+
+		manifestBytes, err := fi.ResourceAsBytes(manifestResource)
+		if err != nil {
+			return fmt.Errorf("error reading manifest %s: %v", manifest, err)
+		}
+
+		manifestBytes, err = b.assetBuilder.RemapManifest(manifestBytes)
+		if err != nil {
+			return fmt.Errorf("error remapping manifest %s: %v", manifest, err)
+		}
+
 		tasks[name] = &fitasks.ManagedFile{
 			Name:     fi.String(name),
 			Location: fi.String(manifest),
-			Contents: &fi.ResourceHolder{Name: manifest},
+			Contents: fi.WrapResource(fi.NewBytesResource(manifestBytes)),
 		}
 	}
 

upup/pkg/fi/cloudup/template_functions.go

@@ -30,7 +30,6 @@ package cloudup
 import (
 	"encoding/base64"
 	"fmt"
-	"os"
 	"strings"
 	"text/template"
 
@@ -98,8 +97,6 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap) {
 
 	// TODO: Only for GCE?
 	dest["EncodeGCELabel"] = gce.EncodeGCELabel
-
-	dest["DnsControllerImage"] = tf.DnsControllerImage
 }
 
 // SharedVPC is a simple helper function which makes the templates for a shared VPC clearer
@@ -171,19 +168,6 @@ func (tf *TemplateFunctions) DnsControllerArgv() ([]string, error) {
 	return argv, nil
 }
 
-// To use user-defined DNS Controller:
-// 1. DOCKER_REGISTRY=[your docker hub repo] make dns-controller-push
-// 2. export DNSCONTROLLER_IMAGE=[your docker hub repo]
-// 3. make kops and create/apply cluster
-func (tf *TemplateFunctions) DnsControllerImage() (string, error) {
-	image := os.Getenv("DNSCONTROLLER_IMAGE")
-	if image == "" {
-		return "kope/dns-controller", nil
-	} else {
-		return image, nil
-	}
-}
-
 func (tf *TemplateFunctions) ExternalDnsArgv() ([]string, error) {
 	var argv []string