Merge pull request #274 from mengqiy/kinflate_tree

Kinflate use manifest tree builder

cmd/kinflate/longerDemo/instances/production.md

@@ -22,7 +22,7 @@ objectLabels:
 objectAnnotations:
   note: Hello, I am production!
 
-resources:
+packages:
 - ..
 
 patches:

cmd/kinflate/longerDemo/instances/staging.md

@@ -22,7 +22,7 @@ objectLabels:
 objectAnnotations:
   note: Hello, I am staging!
 
-resources:
+packages:
 - ..
 
 patches:

pkg/kinflate/commands/inflate.go

@@ -23,7 +23,8 @@ import (
 
 	"github.com/spf13/cobra"
 
-	outil "k8s.io/kubectl/pkg/kinflate"
+	"k8s.io/kubectl/pkg/kinflate/tree"
+	"k8s.io/kubectl/pkg/kinflate/types"
 	kutil "k8s.io/kubectl/pkg/kinflate/util"
 )
 
@@ -80,11 +81,20 @@ func (o *inflateOptions) Complete(cmd *cobra.Command, args []string) error {
 
 // RunKinflate runs inflate command (do real work).
 func (o *inflateOptions) RunKinflate(out, errOut io.Writer) error {
-	m, err := outil.LoadFromManifestPath(o.manifestPath)
+	// Build a tree of ManifestData.
+	root, err := tree.LoadManifestDataFromPath(o.manifestPath)
 	if err != nil {
 		return err
 	}
-	res, err := kutil.Encode(m)
+
+	// Do the transformation for the tree.
+	err = root.Inflate()
+	if err != nil {
+		return err
+	}
+
+	// Output the objects.
+	res, err := kutil.Encode(types.KObject(root.Resources))
 	if err != nil {
 		return err
 	}

pkg/kinflate/examples/simple/instances/exampleinstance/Kube-manifest.yaml

@@ -15,8 +15,8 @@ objectLabels:
   repo: test-infra
 objectAnnotations:
   note: This is a test annotation
-resources:
+packages:
-- ../../package
+- ../../package/
 #These are strategic merge patch overlays in the form of API resources
 patches:
 - deployment/deployment.yaml

pkg/kinflate/mergemap/merge.go

@@ -1,36 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package mergemap
-
-import (
-	"fmt"
-
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/kubectl/pkg/kinflate/types"
-)
-
-// Merge will merge all the entries in m2 to m1.
-func Merge(m1, m2 map[types.GroupVersionKindName]*unstructured.Unstructured,
-) error {
-	for gvkn, obj := range m2 {
-		if _, found := m1[gvkn]; found {
-			return fmt.Errorf("there is already an entry: %q", gvkn)
-		}
-		m1[gvkn] = obj
-	}
-	return nil
-}

pkg/kinflate/transformers/default.go

@@ -1,60 +0,0 @@
-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package transformers
-
-import (
-	manifest "k8s.io/kubectl/pkg/apis/manifest/v1alpha1"
-)
-
-// DefaultTransformer generates 4 transformers:
-// 1) name prefix 2) apply labels 3) apply annotations 4) update name reference
-func DefaultTransformer(m *manifest.Manifest) (Transformer, error) {
-	transformers := []Transformer{}
-
-	npt, err := NewDefaultingNamePrefixTransformer(m.NamePrefix)
-	if err != nil {
-		return nil, err
-	}
-	if npt != nil {
-		transformers = append(transformers, npt)
-	}
-
-	lt, err := NewDefaultingLabelsMapTransformer(m.ObjectLabels)
-	if err != nil {
-		return nil, err
-	}
-	if lt != nil {
-		transformers = append(transformers, lt)
-	}
-
-	at, err := NewDefaultingAnnotationsMapTransformer(m.ObjectAnnotations)
-	if err != nil {
-		return nil, err
-	}
-	if at != nil {
-		transformers = append(transformers, at)
-	}
-
-	nrt, err := NewDefaultingNameReferenceTransformer()
-	if err != nil {
-		return nil, err
-	}
-	if nrt != nil {
-		transformers = append(transformers, nrt)
-	}
-	return NewMultiTransformer(transformers), nil
-}

pkg/kinflate/transformers/overlay.go

@@ -0,0 +1,81 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package transformers
+
+import (
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
+	"k8s.io/kubectl/pkg/kinflate/types"
+	"k8s.io/kubectl/pkg/scheme"
+)
+
+// OverlayTransformer contains a map of overlay objects
+type OverlayTransformer struct {
+	overlay types.KObject
+}
+
+var _ Transformer = &OverlayTransformer{}
+
+// NewOverlayTransformer constructs a OverlayTransformer.
+func NewOverlayTransformer(overlay types.KObject) (Transformer, error) {
+	if len(overlay) == 0 {
+		return nil, nil
+	}
+	return &OverlayTransformer{overlay}, nil
+}
+
+// Transform apply the overlay on top of the base resources.
+func (o *OverlayTransformer) Transform(baseResourceMap types.KObject) error {
+	// Strategic merge the resources exist in both base and overlay.
+	for gvkn, base := range baseResourceMap {
+		// Merge overlay with base resource.
+		if overlay, found := o.overlay[gvkn]; found {
+			versionedObj, err := scheme.Scheme.New(gvkn.GVK)
+			if err != nil {
+				switch {
+				case runtime.IsNotRegisteredError(err):
+					return fmt.Errorf("failed to find schema for %#v (which may be a CRD type): %v", gvkn.GVK, err)
+				default:
+					return err
+				}
+			}
+			// TODO: Change this to use the new Merge package.
+			// Store the name of the base object, because this name may have been munged.
+			// Apply this name to the StrategicMergePatched object.
+			baseName := base.GetName()
+			merged, err := strategicpatch.StrategicMergeMapPatch(
+				base.UnstructuredContent(),
+				overlay.UnstructuredContent(),
+				versionedObj)
+			if err != nil {
+				return err
+			}
+			base.SetName(baseName)
+			baseResourceMap[gvkn].Object = merged
+			delete(o.overlay, gvkn)
+		}
+	}
+	// If there are resources in overlay that are not defined in base, just add it to base.
+	if len(o.overlay) > 0 {
+		for gvkn, jsonObj := range o.overlay {
+			baseResourceMap[gvkn] = jsonObj
+		}
+	}
+	return nil
+}

pkg/kinflate/transformers/overlay_test.go

@@ -0,0 +1,152 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package transformers
+
+import (
+	"reflect"
+	"testing"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/kubectl/pkg/kinflate/types"
+)
+
+func makeBaseDeployment() *unstructured.Unstructured {
+	return &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "apps/v1",
+			"kind":       "Deployment",
+			"metadata": map[string]interface{}{
+				"name": "deploy1",
+			},
+			"spec": map[string]interface{}{
+				"template": map[string]interface{}{
+					"metadata": map[string]interface{}{
+						"labels": map[string]interface{}{
+							"old-label": "old-value",
+						},
+					},
+					"spec": map[string]interface{}{
+						"containers": []interface{}{
+							map[string]interface{}{
+								"name":  "nginx",
+								"image": "nginx",
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func makeOverlayDeployment() *unstructured.Unstructured {
+	return &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "apps/v1",
+			"kind":       "Deployment",
+			"metadata": map[string]interface{}{
+				"name": "deploy1",
+			},
+			"spec": map[string]interface{}{
+				"template": map[string]interface{}{
+					"metadata": map[string]interface{}{
+						"labels": map[string]interface{}{
+							"another-label": "foo",
+						},
+					},
+					"spec": map[string]interface{}{
+						"containers": []interface{}{
+							map[string]interface{}{
+								"name":  "nginx",
+								"image": "nginx:latest",
+								"env": []interface{}{
+									map[string]interface{}{
+										"name":  "SOMEENV",
+										"value": "BAR",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func makeMergedDeployment() *unstructured.Unstructured {
+	return &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "apps/v1",
+			"kind":       "Deployment",
+			"metadata": map[string]interface{}{
+				"name": "deploy1",
+			},
+			"spec": map[string]interface{}{
+				"template": map[string]interface{}{
+					"metadata": map[string]interface{}{
+						"labels": map[string]interface{}{
+							"old-label":     "old-value",
+							"another-label": "foo",
+						},
+					},
+					"spec": map[string]interface{}{
+						"containers": []interface{}{
+							map[string]interface{}{
+								"name":  "nginx",
+								"image": "nginx:latest",
+								"env": []interface{}{
+									map[string]interface{}{
+										"name":  "SOMEENV",
+										"value": "BAR",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func makeTestKObject(genDeployment func() *unstructured.Unstructured) types.KObject {
+	return types.KObject{
+		{
+			GVK:  schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"},
+			Name: "deploy1",
+		}: genDeployment(),
+	}
+}
+
+func TestOverlayRun(t *testing.T) {
+	base := makeTestKObject(makeBaseDeployment)
+	lt, err := NewOverlayTransformer(makeTestKObject(makeOverlayDeployment))
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	err = lt.Transform(base)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	expected := makeTestKObject(makeMergedDeployment)
+	if !reflect.DeepEqual(base, expected) {
+		err = compareMap(base, expected)
+		t.Fatalf("actual doesn't match expected: %v", err)
+	}
+}

pkg/kinflate/tree/builder.go

@@ -24,45 +24,43 @@ import (
 
 	manifest "k8s.io/kubectl/pkg/apis/manifest/v1alpha1"
 	cutil "k8s.io/kubectl/pkg/kinflate/configmapandsecret"
-	"k8s.io/kubectl/pkg/kinflate/mergemap"
 	"k8s.io/kubectl/pkg/kinflate/types"
 	kutil "k8s.io/kubectl/pkg/kinflate/util"
 	"k8s.io/kubectl/pkg/kinflate/util/fs"
 )
 
-// LoadManifestNodeFromPath takes a path to a Kube-manifest.yaml or a dir that has a Kube-manifest.yaml.
+// LoadManifestDataFromPath takes a path to a Kube-manifest.yaml or a dir that has a Kube-manifest.yaml.
-// It returns a tree of ManifestNode.
+// It returns a tree of ManifestData.
-func LoadManifestNodeFromPath(path string) (*ManifestNode, error) {
+func LoadManifestDataFromPath(path string) (*ManifestData, error) {
-	return loadManifestNodeFromPath(path)
+	return loadManifestDataFromPath(path)
 }
 
-// loadManifestNodeFromPath make a ManifestNode from path
+// loadManifestDataFromPath make a ManifestData from path
-func loadManifestNodeFromPath(path string) (*ManifestNode, error) {
+func loadManifestDataFromPath(path string) (*ManifestData, error) {
 	m, err := loadManifestFileFromPath(path)
 	if err != nil {
 		return nil, err
 	}
-	return manifestToManifestNode(m)
+	return manifestToManifestData(m)
 }
 
-// manifestToManifestNode make a ManifestNode given an Manifest object
+// manifestToManifestData make a ManifestData given an Manifest object
-func manifestToManifestNode(m *manifest.Manifest) (*ManifestNode, error) {
+func manifestToManifestData(m *manifest.Manifest) (*ManifestData, error) {
-	mnode := &ManifestNode{}
+	mdata, err := loadManifestDataFromManifestFileAndResources(m)
-	var err error
-	mnode.data, err = loadManifestDataFromManifestFileAndResources(m)
 	if err != nil {
 		return nil, err
 	}
 
-	mnode.children = []*ManifestNode{}
+	pkgs := []*ManifestData{}
 	for _, pkg := range m.Packages {
-		child, err := loadManifestNodeFromPath(pkg)
+		pkgNode, err := loadManifestDataFromPath(pkg)
 		if err != nil {
 			return nil, err
 		}
-		mnode.children = append(mnode.children, child)
+		pkgs = append(pkgs, pkgNode)
 	}
-	return mnode, nil
+	mdata.Packages = pkgs
+	return mdata, nil
 }
 
 // loadManifestFileFromPath loads a manifest object from file.
@@ -75,40 +73,40 @@ func loadManifestFileFromPath(filename string) (*manifest.Manifest, error) {
 	return m, err
 }
 
-func loadManifestDataFromManifestFileAndResources(m *manifest.Manifest) (*manifestData, error) {
+func loadManifestDataFromManifestFileAndResources(m *manifest.Manifest) (*ManifestData, error) {
-	mdata := &manifestData{}
+	mdata := &ManifestData{}
 	var err error
-	mdata.name = m.Name
+	mdata.Name = m.Name
-	mdata.namePrefix = namePrefixType(m.NamePrefix)
+	mdata.NamePrefix = NamePrefixType(m.NamePrefix)
-	mdata.objectLabels = m.ObjectLabels
+	mdata.ObjectLabels = m.ObjectLabels
-	mdata.objectAnnotations = m.ObjectAnnotations
+	mdata.ObjectAnnotations = m.ObjectAnnotations
 
 	res, err := loadKObjectFromPaths(m.Resources)
 	if err != nil {
 		return nil, err
 	}
-	mdata.resources = resourcesType(res)
+	mdata.Resources = ResourcesType(res)
 
 	pat, err := loadKObjectFromPaths(m.Patches)
 	if err != nil {
 		return nil, err
 	}
-	mdata.patches = patchesType(pat)
+	mdata.Patches = PatchesType(pat)
 
 	cms, err := cutil.MakeConfigMapsKObject(m.Configmaps)
 	if err != nil {
 		return nil, err
 	}
-	mdata.configmaps = configmapsType(cms)
+	mdata.Configmaps = ConfigmapsType(cms)
 
 	sec, err := cutil.MakeGenericSecretsKObject(m.GenericSecrets)
 	if err != nil {
 		return nil, err
 	}
-	mdata.secrets = secretsType(sec)
+	mdata.Secrets = SecretsType(sec)
 
 	TLS, err := cutil.MakeTLSSecretsKObject(m.TLSSecrets)
-	err = mergemap.Merge(mdata.secrets, TLS)
+	err = types.Merge(mdata.Secrets, TLS)
 	if err != nil {
 		return nil, err
 	}

pkg/kinflate/tree/builder_test.go

@@ -24,7 +24,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	manifest "k8s.io/kubectl/pkg/apis/manifest/v1alpha1"
-	"k8s.io/kubectl/pkg/kinflate/mergemap"
 	"k8s.io/kubectl/pkg/kinflate/types"
 )
 
@@ -77,13 +76,14 @@ func makeMapOfPodWithImageName(imageName string) types.KObject {
 	}
 }
 
-func makeManifestData(name string) *manifestData {
+func makeManifestData(name string) *ManifestData {
-	return &manifestData{
+	return &ManifestData{
-		name:       name,
+		Name:       name,
-		resources:  resourcesType(types.KObject{}),
+		Resources:  ResourcesType(types.KObject{}),
-		patches:    patchesType(types.KObject{}),
+		Patches:    PatchesType(types.KObject{}),
-		configmaps: configmapsType(types.KObject{}),
+		Configmaps: ConfigmapsType(types.KObject{}),
-		secrets:    secretsType(types.KObject{}),
+		Secrets:    SecretsType(types.KObject{}),
+		Packages:   []*ManifestData{},
 	}
 }
 
@@ -211,7 +211,7 @@ func TestPathsToMap(t *testing.T) {
 
 	mapOfConfigMap := makeMapOfConfigMap()
 	mapOfPod := makeMapOfPod()
-	err := mergemap.Merge(mapOfPod, mapOfConfigMap)
+	err := types.Merge(mapOfPod, mapOfConfigMap)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -264,7 +264,7 @@ func TestPathsToMap(t *testing.T) {
 func TestManifestToManifestData(t *testing.T) {
 	mapOfConfigMap := makeMapOfConfigMap()
 	mapOfPod := makeMapOfPod()
-	err := mergemap.Merge(mapOfPod, mapOfConfigMap)
+	err := types.Merge(mapOfPod, mapOfConfigMap)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -290,15 +290,15 @@ func TestManifestToManifestData(t *testing.T) {
 		},
 	}
 
-	expectedMd := &manifestData{
+	expectedMd := &ManifestData{
-		name:              "test-manifest",
+		Name:              "test-manifest",
-		namePrefix:        "someprefix-",
+		NamePrefix:        "someprefix-",
-		objectLabels:      map[string]string{"foo": "bar"},
+		ObjectLabels:      map[string]string{"foo": "bar"},
-		objectAnnotations: map[string]string{"note": "This is an annotation."},
+		ObjectAnnotations: map[string]string{"note": "This is an annotation."},
-		resources:         resourcesType(mergedMap),
+		Resources:         ResourcesType(mergedMap),
-		patches:           patchesType(makeMapOfPodWithImageName("nginx:latest")),
+		Patches:           PatchesType(makeMapOfPodWithImageName("nginx:latest")),
-		configmaps:        configmapsType(types.KObject{}),
+		Configmaps:        ConfigmapsType(types.KObject{}),
-		secrets:           secretsType(types.KObject{}),
+		Secrets:           SecretsType(types.KObject{}),
 	}
 
 	actual, err := loadManifestDataFromManifestFileAndResources(m)
@@ -311,32 +311,18 @@ func TestManifestToManifestData(t *testing.T) {
 	}
 }
 
-func TestMakeManifestNode(t *testing.T) {
+func TestLoadManifestDataFromPath(t *testing.T) {
-	expected := &ManifestNode{
+	grandparent := makeManifestData("grandparent")
-		data: makeManifestData("grandparent"),
+	parent1 := makeManifestData("parent1")
-		children: []*ManifestNode{
+	parent2 := makeManifestData("parent2")
-			{
+	child1 := makeManifestData("child1")
-				data: makeManifestData("parent1"),
+	child2 := makeManifestData("child2")
-				children: []*ManifestNode{
+	grandparent.Packages = []*ManifestData{parent1, parent2}
-					{
+	parent1.Packages = []*ManifestData{child1}
-						data:     makeManifestData("child1"),
+	parent2.Packages = []*ManifestData{child2}
-						children: []*ManifestNode{},
-					},
-				},
-			},
-			{
-				data: makeManifestData("parent2"),
-				children: []*ManifestNode{
-					{
-						data:     makeManifestData("child2"),
-						children: []*ManifestNode{},
-					},
-				},
-			},
-		},
-	}
 
-	actual, err := loadManifestNodeFromPath("testdata/hierarchy")
+	expected := grandparent
+	actual, err := loadManifestDataFromPath("testdata/hierarchy")
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}

pkg/kinflate/tree/default.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tree
+
+import (
+	"k8s.io/kubectl/pkg/kinflate/transformers"
+	"k8s.io/kubectl/pkg/kinflate/types"
+)
+
+// DefaultTransformer generates the following transformers:
+// 1) apply overlay
+// 2) name prefix
+// 3) apply labels
+// 4) apply annotations
+// 5) update name reference
+func DefaultTransformer(m *ManifestData) (transformers.Transformer, error) {
+	ts := []transformers.Transformer{}
+
+	ot, err := transformers.NewOverlayTransformer(types.KObject(m.Patches))
+	if err != nil {
+		return nil, err
+	}
+	if ot != nil {
+		ts = append(ts, ot)
+	}
+
+	npt, err := transformers.NewDefaultingNamePrefixTransformer(string(m.NamePrefix))
+	if err != nil {
+		return nil, err
+	}
+	if npt != nil {
+		ts = append(ts, npt)
+	}
+
+	lt, err := transformers.NewDefaultingLabelsMapTransformer(m.ObjectLabels)
+	if err != nil {
+		return nil, err
+	}
+	if lt != nil {
+		ts = append(ts, lt)
+	}
+
+	at, err := transformers.NewDefaultingAnnotationsMapTransformer(m.ObjectAnnotations)
+	if err != nil {
+		return nil, err
+	}
+	if at != nil {
+		ts = append(ts, at)
+	}
+
+	nrt, err := transformers.NewDefaultingNameReferenceTransformer()
+	if err != nil {
+		return nil, err
+	}
+	if nrt != nil {
+		ts = append(ts, nrt)
+	}
+	return transformers.NewMultiTransformer(ts), nil
+}

pkg/kinflate/tree/node.go

@@ -20,39 +20,70 @@ import (
 	"k8s.io/kubectl/pkg/kinflate/types"
 )
 
-type namePrefixType string
+type NamePrefixType string
 
-type objectLabelsType map[string]string
+type ObjectLabelsType map[string]string
 
-type objectAnnotationsType map[string]string
+type ObjectAnnotationsType map[string]string
 
-type resourcesType types.KObject
+type ResourcesType types.KObject
 
-type patchesType types.KObject
+type PatchesType types.KObject
 
-type configmapsType types.KObject
+type ConfigmapsType types.KObject
 
-type secretsType types.KObject
+type SecretsType types.KObject
 
-// ManifestNode groups (possibly empty) manifest data with a (possibly empty)
+type PackagesType []*ManifestData
-// set of manifest nodes.
+
-// data in one node may refer to data in other nodes.
+// ManifestData contains all the objects loaded from the filesystem according to
+// the Manifest Object.
+// Data in one node may refer to data in other nodes.
 // The node is invalid if it requires data it cannot find.
 // A node is either a base app, or a patch to the base, or a patch to a patch to the base, etc.
-type ManifestNode struct {
+type ManifestData struct {
-	data     *manifestData
+	// Name of the manifest
-	children []*ManifestNode
+	Name string
+
+	NamePrefix        NamePrefixType
+	ObjectLabels      ObjectLabelsType
+	ObjectAnnotations ObjectAnnotationsType
+	Resources         ResourcesType
+	Patches           PatchesType
+	Configmaps        ConfigmapsType
+	Secrets           SecretsType
+
+	Packages PackagesType
 }
 
-// manifestData contains all the objects loaded from the filesystem according to
+func (md *ManifestData) allResources() error {
-// the Manifest Object.
+	err := types.Merge(md.Resources, md.Configmaps)
-type manifestData struct {
+	if err != nil {
-	name              string
+		return err
-	namePrefix        namePrefixType
+	}
-	objectLabels      objectLabelsType
+	return types.Merge(md.Resources, md.Secrets)
-	objectAnnotations objectAnnotationsType
+}
-	resources         resourcesType
+
-	patches           patchesType
+// Inflate will recursively do the transformation on all the nodes below.
-	configmaps        configmapsType
+func (md *ManifestData) Inflate() error {
-	secrets           secretsType
+	for _, pkg := range md.Packages {
+		err := pkg.Inflate()
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, pkg := range md.Packages {
+		err := types.Merge(md.Resources, pkg.Resources)
+		if err != nil {
+			return err
+		}
+	}
+
+	err := md.allResources()
+	if err != nil {
+		return err
+	}
+	t, err := DefaultTransformer(md)
+	return t.Transform(types.KObject(md.Resources))
 }

pkg/kinflate/types/util.go

@@ -17,6 +17,9 @@ limitations under the License.
 package types
 
 import (
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 )
 
@@ -48,3 +51,15 @@ func SelectByGVK(in schema.GroupVersionKind, selector *schema.GroupVersionKind)
 	}
 	return true
 }
+
+// Merge will merge all the entries in m2 to m1.
+func Merge(m1, m2 map[GroupVersionKindName]*unstructured.Unstructured,
+) error {
+	for gvkn, obj := range m2 {
+		if _, found := m1[gvkn]; found {
+			return fmt.Errorf("there is already an entry: %q", gvkn)
+		}
+		m1[gvkn] = obj
+	}
+	return nil
+}

pkg/kinflate/util.go

@@ -1,276 +0,0 @@
-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package kinflate
-
-import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path"
-	"path/filepath"
-	"strings"
-
-	"k8s.io/apimachinery/pkg/api/meta"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/util/strategicpatch"
-	manifest "k8s.io/kubectl/pkg/apis/manifest/v1alpha1"
-	cutil "k8s.io/kubectl/pkg/kinflate/configmapandsecret"
-	"k8s.io/kubectl/pkg/kinflate/constants"
-	"k8s.io/kubectl/pkg/kinflate/mergemap"
-	"k8s.io/kubectl/pkg/kinflate/transformers"
-	"k8s.io/kubectl/pkg/kinflate/types"
-	kutil "k8s.io/kubectl/pkg/kinflate/util"
-	"k8s.io/kubectl/pkg/scheme"
-)
-
-func populateMap(m types.KObject, obj *unstructured.Unstructured, newName string) error {
-	accessor, err := meta.Accessor(obj)
-	if err != nil {
-		return err
-	}
-	oldName := accessor.GetName()
-	gvk := obj.GetObjectKind().GroupVersionKind()
-	gvkn := types.GroupVersionKindName{GVK: gvk, Name: oldName}
-
-	if _, found := m[gvkn]; found {
-		return fmt.Errorf("cannot use a duplicate name %q for %s", oldName, gvk)
-	}
-	accessor.SetName(newName)
-	m[gvkn] = obj
-	return nil
-}
-
-func populateConfigMapAndSecretMap(manifest *manifest.Manifest, m types.KObject) error {
-	configmaps, err := cutil.MakeConfigMapsKObject(manifest.Configmaps)
-	if err != nil {
-		return err
-	}
-	err = mergemap.Merge(m, configmaps)
-	if err != nil {
-		return err
-	}
-
-	genericSecrets, err := cutil.MakeGenericSecretsKObject(manifest.GenericSecrets)
-	if err != nil {
-		return err
-	}
-	err = mergemap.Merge(m, genericSecrets)
-	if err != nil {
-		return err
-	}
-
-	TLSSecrets, err := cutil.MakeTLSSecretsKObject(manifest.TLSSecrets)
-	if err != nil {
-		return err
-	}
-	return mergemap.Merge(m, TLSSecrets)
-}
-
-func populateResourceMap(files []string,
-	m types.KObject) error {
-	for _, file := range files {
-		err := pathToMap(file, m)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// LoadFromManifestPath loads the manifest from the given path.
-// It returns a map of resources defined in the manifest file.
-func LoadFromManifestPath(mPath string,
-) (types.KObject, error) {
-	f, err := os.Stat(mPath)
-	if err != nil {
-		return nil, err
-	}
-	if f.IsDir() {
-		mPath = path.Join(mPath, constants.KubeManifestFileName)
-	} else {
-		if !strings.HasSuffix(mPath, constants.KubeManifestFileName) {
-			return nil, fmt.Errorf("expecting file: %q, but got: %q", constants.KubeManifestFileName, mPath)
-		}
-	}
-	manifest, err := (&kutil.ManifestLoader{}).Read(mPath)
-	if err != nil {
-		return nil, err
-	}
-	return ManifestToMap(manifest)
-}
-
-func pathToMap(path string, into types.KObject) error {
-	f, err := os.Stat(path)
-	if err != nil {
-		return err
-	}
-	if into == nil {
-		into = types.KObject{}
-	}
-	switch mode := f.Mode(); {
-	case mode.IsDir():
-		err = dirToMap(path, into)
-	case mode.IsRegular():
-		err = fileToMap(path, into)
-	}
-	return err
-}
-
-func fileToMap(filename string, into types.KObject) error {
-	f, err := os.Stat(filename)
-	if f.IsDir() {
-		return fmt.Errorf("%q is NOT expected to be an dir", filename)
-	}
-	content, err := ioutil.ReadFile(filename)
-	if err != nil {
-		return err
-	}
-
-	_, err = kutil.Decode(content, into)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-// dirToMap tries to find Kube-manifest.yaml first in a dir.
-// If not found, traverse all the file in the dir.
-func dirToMap(dirname string, into types.KObject) error {
-	if into == nil {
-		into = types.KObject{}
-	}
-	f, err := os.Stat(dirname)
-	if !f.IsDir() {
-		return fmt.Errorf("%q is expected to be an dir", dirname)
-	}
-
-	kubeManifestFileAbsName := path.Join(dirname, constants.KubeManifestFileName)
-	_, err = os.Stat(kubeManifestFileAbsName)
-	switch {
-	case err != nil && !os.IsNotExist(err):
-		return err
-	case err == nil:
-		manifest, err := (&kutil.ManifestLoader{}).Read(kubeManifestFileAbsName)
-		if err != nil {
-			return err
-		}
-		_, err = manifestToMap(manifest, into)
-		if err != nil {
-			return err
-		}
-	case err != nil && os.IsNotExist(err):
-		files, err := ioutil.ReadDir(dirname)
-		if err != nil {
-			return err
-		}
-
-		for _, file := range files {
-			err = pathToMap(path.Join(dirname, file.Name()), into)
-		}
-
-		var e error
-		filepath.Walk(dirname, func(path string, _ os.FileInfo, err error) error {
-			if err != nil {
-				e = err
-				return err
-			}
-			err = fileToMap(path, into)
-			return nil
-		})
-	}
-	return nil
-}
-
-// ManifestToMap takes a manifest and recursively finds all instances of Kube-manifest,
-// reads them and merges them all in a map of resources.
-func ManifestToMap(m *manifest.Manifest,
-) (types.KObject, error) {
-	return manifestToMap(m, nil)
-}
-
-// manifestToMap takes a manifest and recursively finds all instances of Kube-manifest,
-// reads them and merges them all into `into`.
-func manifestToMap(m *manifest.Manifest,
-	into types.KObject,
-) (types.KObject, error) {
-	baseResourceMap := types.KObject{}
-	if into != nil {
-		baseResourceMap = into
-	}
-	err := populateResourceMap(m.Resources, baseResourceMap)
-	if err != nil {
-		return nil, err
-	}
-
-	overlayResouceMap := types.KObject{}
-	err = populateResourceMap(m.Patches, overlayResouceMap)
-	if err != nil {
-		return nil, err
-	}
-
-	// Strategic merge the resources exist in both base and overlay.
-	for gvkn, base := range baseResourceMap {
-		// Merge overlay with base resource.
-		if overlay, found := overlayResouceMap[gvkn]; found {
-			versionedObj, err := scheme.Scheme.New(gvkn.GVK)
-			if err != nil {
-				switch {
-				case runtime.IsNotRegisteredError(err):
-					return nil, fmt.Errorf("CRD and TPR are not supported now: %v", err)
-				default:
-					return nil, err
-				}
-			}
-			// Store the name of the base object, because this name may have been munged.
-			// Apply this name to the StrategicMergePatched object.
-			baseName := base.GetName()
-			merged, err := strategicpatch.StrategicMergeMapPatch(
-				base.UnstructuredContent(),
-				overlay.UnstructuredContent(),
-				versionedObj)
-			if err != nil {
-				return nil, err
-			}
-			base.SetName(baseName)
-			baseResourceMap[gvkn].Object = merged
-			delete(overlayResouceMap, gvkn)
-		}
-	}
-
-	// If there are resources in overlay that are not defined in base, just add it to base.
-	if len(overlayResouceMap) > 0 {
-		for gvkn, jsonObj := range overlayResouceMap {
-			baseResourceMap[gvkn] = jsonObj
-		}
-	}
-
-	err = populateConfigMapAndSecretMap(m, baseResourceMap)
-	if err != nil {
-		return nil, err
-	}
-
-	t, err := transformers.DefaultTransformer(m)
-	if err != nil {
-		return nil, err
-	}
-	err = t.Transform(baseResourceMap)
-	if err != nil {
-		return nil, err
-	}
-	return baseResourceMap, nil
-}

pkg/kinflate/util_test.go

@@ -1,197 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package kinflate
-
-import (
-	"encoding/base64"
-	"reflect"
-	"strings"
-	"testing"
-
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	manifest "k8s.io/kubectl/pkg/apis/manifest/v1alpha1"
-	"k8s.io/kubectl/pkg/kinflate/types"
-)
-
-func makeUnstructuredEnvConfigMap(name string) *unstructured.Unstructured {
-	return &unstructured.Unstructured{
-		Object: map[string]interface{}{
-			"apiVersion": "v1",
-			"kind":       "ConfigMap",
-			"metadata": map[string]interface{}{
-				"name":              name,
-				"creationTimestamp": nil,
-			},
-			"data": map[string]interface{}{
-				"DB_USERNAME": "admin",
-				"DB_PASSWORD": "somepw",
-			},
-		},
-	}
-}
-
-func makeUnstructuredEnvSecret(name string) *unstructured.Unstructured {
-	return &unstructured.Unstructured{
-		Object: map[string]interface{}{
-			"apiVersion": "v1",
-			"kind":       "Secret",
-			"metadata": map[string]interface{}{
-				"name":              name,
-				"creationTimestamp": nil,
-			},
-			"type": string(corev1.SecretTypeOpaque),
-			"data": map[string]interface{}{
-				"DB_USERNAME": base64.StdEncoding.EncodeToString([]byte("admin")),
-				"DB_PASSWORD": base64.StdEncoding.EncodeToString([]byte("somepw")),
-			},
-		},
-	}
-}
-
-func makeUnstructuredTLSSecret(name string) *unstructured.Unstructured {
-	return &unstructured.Unstructured{
-		Object: map[string]interface{}{
-			"apiVersion": "v1",
-			"kind":       "Secret",
-			"metadata": map[string]interface{}{
-				"name":              name,
-				"creationTimestamp": nil,
-			},
-			"type": string(corev1.SecretTypeTLS),
-			"data": map[string]interface{}{
-				"tls.key": base64.StdEncoding.EncodeToString([]byte(`-----BEGIN RSA PRIVATE KEY-----
-MIIBOwIBAAJBANLJhPHhITqQbPklG3ibCVxwGMRfp/v4XqhfdQHdcVfHap6NQ5Wo
-k/4xIA+ui35/MmNartNuC+BdZ1tMuVCPFZcCAwEAAQJAEJ2N+zsR0Xn8/Q6twa4G
-6OB1M1WO+k+ztnX/1SvNeWu8D6GImtupLTYgjZcHufykj09jiHmjHx8u8ZZB/o1N
-MQIhAPW+eyZo7ay3lMz1V01WVjNKK9QSn1MJlb06h/LuYv9FAiEA25WPedKgVyCW
-SmUwbPw8fnTcpqDWE3yTO3vKcebqMSsCIBF3UmVue8YU3jybC3NxuXq3wNm34R8T
-xVLHwDXh/6NJAiEAl2oHGGLz64BuAfjKrqwz7qMYr9HCLIe/YsoWq/olzScCIQDi
-D2lWusoe2/nEqfDVVWGWlyJ7yOmqaVm/iNUN9B2N2g==
------END RSA PRIVATE KEY-----
-`)),
-				"tls.crt": base64.StdEncoding.EncodeToString([]byte(`-----BEGIN CERTIFICATE-----
-MIIB0zCCAX2gAwIBAgIJAI/M7BYjwB+uMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV
-BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
-aWRnaXRzIFB0eSBMdGQwHhcNMTIwOTEyMjE1MjAyWhcNMTUwOTEyMjE1MjAyWjBF
-MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50
-ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANLJ
-hPHhITqQbPklG3ibCVxwGMRfp/v4XqhfdQHdcVfHap6NQ5Wok/4xIA+ui35/MmNa
-rtNuC+BdZ1tMuVCPFZcCAwEAAaNQME4wHQYDVR0OBBYEFJvKs8RfJaXTH08W+SGv
-zQyKn0H8MB8GA1UdIwQYMBaAFJvKs8RfJaXTH08W+SGvzQyKn0H8MAwGA1UdEwQF
-MAMBAf8wDQYJKoZIhvcNAQEFBQADQQBJlffJHybjDGxRMqaRmDhX0+6v02TUKZsW
-r5QuVbpQhH6u+0UgcW0jp9QwpxoPTLTWGXEWBBBurxFwiCBhkQ+V
------END CERTIFICATE-----
-`)),
-			},
-		},
-	}
-}
-func TestPopulateMap(t *testing.T) {
-	expectedMap := types.KObject{
-		{
-			GVK: schema.GroupVersionKind{
-				Version: "v1",
-				Kind:    "ConfigMap",
-			},
-			Name: "envConfigMap",
-		}: makeUnstructuredEnvConfigMap("newNameConfigMap"),
-		{
-			GVK: schema.GroupVersionKind{
-				Version: "v1",
-				Kind:    "Secret",
-			},
-			Name: "envSecret",
-		}: makeUnstructuredEnvSecret("newNameSecret"),
-		{
-			GVK: schema.GroupVersionKind{
-				Version: "v1",
-				Kind:    "Secret",
-			},
-			Name: "tlsSecret",
-		}: makeUnstructuredTLSSecret("newNameTLSSecret"),
-	}
-
-	m := types.KObject{}
-	err := populateMap(m, makeUnstructuredEnvConfigMap("envConfigMap"), "newNameConfigMap")
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-	err = populateMap(m, makeUnstructuredEnvSecret("envSecret"), "newNameSecret")
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-	err = populateMap(m, makeUnstructuredTLSSecret("tlsSecret"), "newNameTLSSecret")
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-
-	if !reflect.DeepEqual(m, expectedMap) {
-		t.Fatalf("%#v\ndoesn't match expected\n%#v\n", m, expectedMap)
-	}
-
-	err = populateMap(m, makeUnstructuredEnvSecret("envSecret"), "newNameSecret")
-	if err == nil || !strings.Contains(err.Error(), "duplicate name") {
-		t.Fatalf("expected error to contain %q, but got: %v", "duplicate name", err)
-	}
-}
-
-func TestPopulateMapOfConfigMapAndSecret(t *testing.T) {
-	m := types.KObject{}
-	manifest := &manifest.Manifest{
-		Configmaps: []manifest.ConfigMap{
-			{
-				Name: "envConfigMap",
-				DataSources: manifest.DataSources{
-					EnvSource: "examples/simple/instances/exampleinstance/configmap/app.env",
-				},
-			},
-		},
-		GenericSecrets: []manifest.GenericSecret{
-			{
-				Name: "envSecret",
-				DataSources: manifest.DataSources{
-					EnvSource: "examples/simple/instances/exampleinstance/configmap/app.env",
-				},
-			},
-		},
-	}
-	expectedMap := types.KObject{
-		{
-			GVK: schema.GroupVersionKind{
-				Version: "v1",
-				Kind:    "ConfigMap",
-			},
-			Name: "envConfigMap",
-		}: makeUnstructuredEnvConfigMap("envConfigMap-d2c89bt4kk"),
-		{
-			GVK: schema.GroupVersionKind{
-				Version: "v1",
-				Kind:    "Secret",
-			},
-			Name: "envSecret",
-		}: makeUnstructuredEnvSecret("envSecret-684h2mm268"),
-	}
-	err := populateConfigMapAndSecretMap(manifest, m)
-	if err != nil {
-		t.Fatalf("unexpected erorr: %v", err)
-	}
-	if !reflect.DeepEqual(m, expectedMap) {
-		t.Fatalf("%#v\ndoesn't match expected\n%#v\n", m, expectedMap)
-	}
-}