add tree builder

pkg/kinflate/tree/builder.go

@@ -0,0 +1,200 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tree
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/ghodss/yaml"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	manifest "k8s.io/kubectl/pkg/apis/manifest/v1alpha1"
+	"k8s.io/kubectl/pkg/kinflate/adjustpath"
+	cutil "k8s.io/kubectl/pkg/kinflate/configmapandsecret"
+	"k8s.io/kubectl/pkg/kinflate/constants"
+	"k8s.io/kubectl/pkg/kinflate/gvkn"
+	"k8s.io/kubectl/pkg/kinflate/mergemap"
+	kutil "k8s.io/kubectl/pkg/kinflate/util"
+)
+
+// BuildManifestTree takes a path to a Kube-manifest.yaml or a dir that has a Kube-manifest.yaml.
+// It returns a tree of ManifestNode.
+func BuildManifestTree(path string) (*ManifestNode, error) {
+	return manifestPathToManifestNode(path)
+}
+
+func manifestPathToManifestNode(path string) (*ManifestNode, error) {
+	path, err := validateManifestPath(path)
+	if err != nil {
+		return nil, err
+	}
+	m, err := manifestPathToManifest(path)
+	if err != nil {
+		return nil, err
+	}
+	return manifestToManifestNode(m)
+}
+
+func manifestToManifestNode(m *manifest.Manifest) (*ManifestNode, error) {
+	mnode := &ManifestNode{}
+	var err error
+	mnode.Data, err = manifestToManifestData(m)
+	if err != nil {
+		return nil, err
+	}
+
+	mnode.Children = []*ManifestNode{}
+	for _, pkg := range m.Packages {
+		child, err := manifestPathToManifestNode(pkg)
+		if err != nil {
+			return nil, err
+		}
+		mnode.Children = append(mnode.Children, child)
+	}
+	return mnode, nil
+}
+
+// manifestPathToManifest loads a manifest file and parse it in to the Manifest object.
+func manifestPathToManifest(filename string) (*manifest.Manifest, error) {
+	bytes, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	var m manifest.Manifest
+	// TODO: support json
+	err = yaml.Unmarshal(bytes, &m)
+	if err != nil {
+		return nil, err
+	}
+	dir, _ := path.Split(filename)
+	adjustpath.AdjustPathsForManifest(&m, []string{dir})
+	return &m, err
+}
+
+// validateManifestPath loads the manifest from the given path.
+// It returns ManifestData and an potential error.
+func validateManifestPath(mPath string) (string, error) {
+	f, err := os.Stat(mPath)
+	if err != nil {
+		return "", err
+	}
+	if f.IsDir() {
+		mPath = path.Join(mPath, constants.KubeManifestFileName)
+		_, err = os.Stat(mPath)
+		if err != nil {
+			return "", err
+		}
+	} else {
+		if !strings.HasSuffix(mPath, constants.KubeManifestFileName) {
+			return "", fmt.Errorf("expecting file: %q, but got: %q", constants.KubeManifestFileName, mPath)
+		}
+	}
+	return mPath, nil
+}
+
+func manifestToManifestData(m *manifest.Manifest) (*ManifestData, error) {
+	mdata := &ManifestData{}
+	var err error
+	mdata.Name = m.Name
+	mdata.NamePrefix = NamePrefixType(m.NamePrefix)
+	mdata.ObjectLabels = m.ObjectLabels
+	mdata.ObjectAnnotations = m.ObjectAnnotations
+	mdata.Resources, err = pathsToMap(m.Resources)
+	if err != nil {
+		return nil, err
+	}
+	mdata.Patches, err = pathsToMap(m.Patches)
+	if err != nil {
+		return nil, err
+	}
+	mdata.Configmaps, err = cutil.MakeMapOfConfigMap(m)
+	if err != nil {
+		return nil, err
+	}
+	mdata.Secrets, err = cutil.MakeMapOfGenericSecret(m)
+	if err != nil {
+		return nil, err
+	}
+	TLSSecrets, err := cutil.MakeMapOfTLSSecret(m)
+	err = mergemap.Merge(mdata.Secrets, TLSSecrets)
+	if err != nil {
+		return nil, err
+	}
+	return mdata, nil
+}
+
+func pathsToMap(paths []string) (map[gvkn.GroupVersionKindName]*unstructured.Unstructured, error) {
+	res := map[gvkn.GroupVersionKindName]*unstructured.Unstructured{}
+	for _, path := range paths {
+		err := pathToMap(path, res)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return res, nil
+}
+
+func pathToMap(path string, into map[gvkn.GroupVersionKindName]*unstructured.Unstructured) error {
+	_, err := os.Stat(path)
+	if err != nil {
+		return err
+	}
+	if into == nil {
+		into = map[gvkn.GroupVersionKindName]*unstructured.Unstructured{}
+	}
+
+	var e error
+	filepath.Walk(path, func(filepath string, info os.FileInfo, err error) error {
+		if err != nil {
+			e = err
+			return err
+		}
+		// Skip all the dir
+		if info.IsDir() {
+			return nil
+		}
+
+		err = fileToMap(filepath, into)
+		return nil
+	})
+	return e
+}
+
+func fileToMap(filename string, into map[gvkn.GroupVersionKindName]*unstructured.Unstructured) error {
+	f, err := os.Stat(filename)
+	if err != nil {
+		return err
+	}
+	if f.IsDir() {
+		return fmt.Errorf("%q is NOT expected to be an dir", filename)
+	}
+	content, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return err
+	}
+
+	_, err = kutil.Decode(content, into)
+	if err != nil {
+		return err
+	}
+	return nil
+}

pkg/kinflate/tree/builder_test.go

@@ -0,0 +1,394 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tree
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	manifest "k8s.io/kubectl/pkg/apis/manifest/v1alpha1"
+	"k8s.io/kubectl/pkg/kinflate/gvkn"
+	"k8s.io/kubectl/pkg/kinflate/mergemap"
+)
+
+func makeMapOfConfigMap() map[gvkn.GroupVersionKindName]*unstructured.Unstructured {
+	return map[gvkn.GroupVersionKindName]*unstructured.Unstructured{
+		{
+			GVK:  schema.GroupVersionKind{Version: "v1", Kind: "ConfigMap"},
+			Name: "cm1",
+		}: {
+			Object: map[string]interface{}{
+				"apiVersion": "v1",
+				"kind":       "ConfigMap",
+				"metadata": map[string]interface{}{
+					"name": "cm1",
+				},
+				"data": map[string]interface{}{
+					"foo": "bar",
+				},
+			},
+		},
+	}
+}
+
+func makeMapOfPod() map[gvkn.GroupVersionKindName]*unstructured.Unstructured {
+	return makeMapOfPodWithImageName("nginx")
+}
+
+func makeMapOfPodWithImageName(imageName string) map[gvkn.GroupVersionKindName]*unstructured.Unstructured {
+	return map[gvkn.GroupVersionKindName]*unstructured.Unstructured{
+		{
+			GVK:  schema.GroupVersionKind{Version: "v1", Kind: "Pod"},
+			Name: "pod1",
+		}: {
+			Object: map[string]interface{}{
+				"apiVersion": "v1",
+				"kind":       "Pod",
+				"metadata": map[string]interface{}{
+					"name": "pod1",
+				},
+				"spec": map[string]interface{}{
+					"containers": []interface{}{
+						map[string]interface{}{
+							"name":  "nginx",
+							"image": imageName,
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func makeManifestData(name string) *ManifestData {
+	return &ManifestData{
+		Name:       name,
+		Resources:  map[gvkn.GroupVersionKindName]*unstructured.Unstructured{},
+		Patches:    map[gvkn.GroupVersionKindName]*unstructured.Unstructured{},
+		Configmaps: map[gvkn.GroupVersionKindName]*unstructured.Unstructured{},
+		Secrets:    map[gvkn.GroupVersionKindName]*unstructured.Unstructured{},
+	}
+}
+
+func TestValidateManifestPath(t *testing.T) {
+	type testcase struct {
+		filename  string
+		expectErr bool
+		errorStr  string
+	}
+
+	testcases := []testcase{
+		{
+			filename:  "testdata/valid/",
+			expectErr: false,
+		},
+		{
+			filename:  "testdata/valid/Kube-manifest.yaml",
+			expectErr: false,
+		},
+		{
+			filename:  "does-not-exist",
+			expectErr: true,
+			errorStr:  "no such file or directory",
+		},
+		{
+			filename:  "testdata/invalid/",
+			expectErr: true,
+			errorStr:  "no such file or directory",
+		},
+	}
+
+	for _, tc := range testcases {
+		_, err := validateManifestPath(tc.filename)
+		if err == nil {
+			if tc.expectErr {
+				t.Errorf("filename: %q, expect an error containing %q, but didn't get an error", tc.filename, tc.errorStr)
+			}
+		} else {
+			if tc.expectErr {
+				if !strings.Contains(err.Error(), tc.errorStr) {
+					t.Errorf("filename: %q, expect an error containing %q, but got %v", tc.filename, tc.errorStr, err)
+				}
+			} else {
+				t.Errorf("unexpected error: %v", err)
+			}
+		}
+	}
+}
+
+func TestFileToMap(t *testing.T) {
+	type testcase struct {
+		filename  string
+		expected  map[gvkn.GroupVersionKindName]*unstructured.Unstructured
+		expectErr bool
+		errorStr  string
+	}
+
+	testcases := []testcase{
+		{
+			filename: "testdata/valid/cm/configmap.yaml",
+			expected: map[gvkn.GroupVersionKindName]*unstructured.Unstructured{
+				{
+					GVK:  schema.GroupVersionKind{Version: "v1", Kind: "ConfigMap"},
+					Name: "cm1",
+				}: {
+					Object: map[string]interface{}{
+						"apiVersion": "v1",
+						"kind":       "ConfigMap",
+						"metadata": map[string]interface{}{
+							"name": "cm1",
+						},
+						"data": map[string]interface{}{
+							"foo": "bar",
+						},
+					},
+				},
+			},
+			expectErr: false,
+		},
+		{
+			filename:  "testdata/valid/cm/",
+			expectErr: true,
+			errorStr:  "NOT expected to be an dir",
+		},
+		{
+			filename:  "does-not-exist",
+			expectErr: true,
+			errorStr:  "no such file or directory",
+		},
+	}
+
+	for _, tc := range testcases {
+		actual := map[gvkn.GroupVersionKindName]*unstructured.Unstructured{}
+		err := fileToMap(tc.filename, actual)
+		if err == nil {
+			if tc.expectErr {
+				t.Errorf("filename: %q, expect an error containing %q, but didn't get an error", tc.filename, tc.errorStr)
+			}
+			if !reflect.DeepEqual(actual, tc.expected) {
+				t.Errorf("filename: %q, expect %v, but got %v", tc.filename, tc.expected, actual)
+			}
+		} else {
+			if tc.expectErr {
+				if !strings.Contains(err.Error(), tc.errorStr) {
+					t.Errorf("filename: %q, expect an error containing %q, but got %v", tc.filename, tc.errorStr, err)
+				}
+			} else {
+				t.Errorf("unexpected error: %v", err)
+			}
+		}
+	}
+}
+
+func TestPathToMap(t *testing.T) {
+	type testcase struct {
+		filename  string
+		expected  map[gvkn.GroupVersionKindName]*unstructured.Unstructured
+		expectErr bool
+		errorStr  string
+	}
+
+	expectedMap := makeMapOfConfigMap()
+
+	testcases := []testcase{
+		{
+			filename:  "testdata/valid/cm/configmap.yaml",
+			expected:  expectedMap,
+			expectErr: false,
+		},
+		{
+			filename:  "testdata/valid/cm/",
+			expected:  expectedMap,
+			expectErr: false,
+		},
+		{
+			filename:  "does-not-exist",
+			expectErr: true,
+			errorStr:  "no such file or directory",
+		},
+	}
+
+	for _, tc := range testcases {
+		actual := map[gvkn.GroupVersionKindName]*unstructured.Unstructured{}
+		err := pathToMap(tc.filename, actual)
+		if err == nil {
+			if tc.expectErr {
+				t.Errorf("filename: %q, expect an error containing %q, but didn't get an error", tc.filename, tc.errorStr)
+			}
+			if !reflect.DeepEqual(actual, tc.expected) {
+				t.Errorf("filename: %q, expect %v, but got %v", tc.filename, tc.expected, actual)
+			}
+		} else {
+			if tc.expectErr {
+				if !strings.Contains(err.Error(), tc.errorStr) {
+					t.Errorf("filename: %q, expect an error containing %q, but got %v", tc.filename, tc.errorStr, err)
+				}
+			} else {
+				t.Errorf("unexpected error: %v", err)
+			}
+		}
+	}
+}
+
+func TestPathsToMap(t *testing.T) {
+	type testcase struct {
+		filenames []string
+		expected  map[gvkn.GroupVersionKindName]*unstructured.Unstructured
+		expectErr bool
+		errorStr  string
+	}
+
+	mapOfConfigMap := makeMapOfConfigMap()
+	mapOfPod := makeMapOfPod()
+	err := mergemap.Merge(mapOfPod, mapOfConfigMap)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	mergedMap := mapOfPod
+
+	testcases := []testcase{
+		{
+			filenames: []string{"testdata/valid/cm/"},
+			expected:  mapOfConfigMap,
+			expectErr: false,
+		},
+		{
+			filenames: []string{"testdata/valid/pod.yaml"},
+			expected:  makeMapOfPod(),
+			expectErr: false,
+		},
+		{
+			filenames: []string{"testdata/valid/cm/", "testdata/valid/pod.yaml"},
+			expected:  mergedMap,
+			expectErr: false,
+		},
+		{
+			filenames: []string{"does-not-exist"},
+			expectErr: true,
+			errorStr:  "no such file or directory",
+		},
+	}
+
+	for _, tc := range testcases {
+		actual, err := pathsToMap(tc.filenames)
+		if err == nil {
+			if tc.expectErr {
+				t.Errorf("filenames: %q, expect an error containing %q, but didn't get an error", tc.filenames, tc.errorStr)
+			}
+			if !reflect.DeepEqual(actual, tc.expected) {
+				t.Errorf("filenames: %q, expect %v, but got %v", tc.filenames, tc.expected, actual)
+			}
+		} else {
+			if tc.expectErr {
+				if !strings.Contains(err.Error(), tc.errorStr) {
+					t.Errorf("filenames: %q, expect an error containing %q, but got %v", tc.filenames, tc.errorStr, err)
+				}
+			} else {
+				t.Errorf("unexpected error: %v", err)
+			}
+		}
+	}
+}
+
+func TestManifestToManifestData(t *testing.T) {
+	mapOfConfigMap := makeMapOfConfigMap()
+	mapOfPod := makeMapOfPod()
+	err := mergemap.Merge(mapOfPod, mapOfConfigMap)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	mergedMap := mapOfPod
+
+	m := &manifest.Manifest{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-manifest",
+		},
+		NamePrefix: "someprefix-",
+		ObjectLabels: map[string]string{
+			"foo": "bar",
+		},
+		ObjectAnnotations: map[string]string{
+			"note": "This is an annotation.",
+		},
+		Resources: []string{
+			"testdata/valid/cm/",
+			"testdata/valid/pod.yaml",
+		},
+		Patches: []string{
+			"testdata/valid/patch.yaml",
+		},
+	}
+
+	expectedMd := &ManifestData{
+		Name:              "test-manifest",
+		NamePrefix:        "someprefix-",
+		ObjectLabels:      map[string]string{"foo": "bar"},
+		ObjectAnnotations: map[string]string{"note": "This is an annotation."},
+		Resources:         mergedMap,
+		Patches:           makeMapOfPodWithImageName("nginx:latest"),
+		Configmaps:        map[gvkn.GroupVersionKindName]*unstructured.Unstructured{},
+		Secrets:           map[gvkn.GroupVersionKindName]*unstructured.Unstructured{},
+	}
+
+	actual, err := manifestToManifestData(m)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if !reflect.DeepEqual(actual, expectedMd) {
+		t.Errorf("expect:\n%#v\nbut got:\n%#v", expectedMd, actual)
+	}
+}
+
+func TestManifestPathToManifestNode(t *testing.T) {
+	expected := &ManifestNode{
+		Data: makeManifestData("grandparent"),
+		Children: []*ManifestNode{
+			{
+				Data: makeManifestData("parent1"),
+				Children: []*ManifestNode{
+					{
+						Data:     makeManifestData("child1"),
+						Children: []*ManifestNode{},
+					},
+				},
+			},
+			{
+				Data: makeManifestData("parent2"),
+				Children: []*ManifestNode{
+					{
+						Data:     makeManifestData("child2"),
+						Children: []*ManifestNode{},
+					},
+				},
+			},
+		},
+	}
+
+	actual, err := manifestPathToManifestNode("testdata/hierarchy")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if !reflect.DeepEqual(actual, expected) {
+		t.Fatalf("expect:\n%#v\nbut got:\n%#v", expected, actual)
+	}
+}

pkg/kinflate/tree/node.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tree
+
+import (
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/kubectl/pkg/kinflate/gvkn"
+)
+
+type NamePrefixType string
+
+type ObjectLabelsType map[string]string
+
+type ObjectAnnotationsType map[string]string
+
+type ResourcesType map[gvkn.GroupVersionKindName]*unstructured.Unstructured
+
+type PatchesType map[gvkn.GroupVersionKindName]*unstructured.Unstructured
+
+type ConfigmapsType map[gvkn.GroupVersionKindName]*unstructured.Unstructured
+
+type SecretsType map[gvkn.GroupVersionKindName]*unstructured.Unstructured
+
+// ManifestNode is the node for building the manifest tree.
+// Children points to the packages defined on this node's Manifest.
+type ManifestNode struct {
+	Data     *ManifestData
+	Children []*ManifestNode
+}
+
+// ManifestData contains all the objects loaded from the filesystem according to
+// the Manifest Object.
+type ManifestData struct {
+	Name              string
+	NamePrefix        NamePrefixType
+	ObjectLabels      ObjectLabelsType
+	ObjectAnnotations ObjectAnnotationsType
+	Resources         ResourcesType
+	Patches           PatchesType
+	Configmaps        ConfigmapsType
+	Secrets           SecretsType
+}

pkg/kinflate/tree/testdata/hierarchy/Kube-manifest.yaml

@@ -0,0 +1,7 @@
+apiVersion: manifest.k8s.io/v1alpha1
+kind: Manifest
+metadata:
+  name: grandparent
+packages:
+  - parent1/
+  - parent2/

pkg/kinflate/tree/testdata/hierarchy/parent1/Kube-manifest.yaml

@@ -0,0 +1,6 @@
+apiVersion: manifest.k8s.io/v1alpha1
+kind: Manifest
+metadata:
+  name: parent1
+packages:
+  - child1/

pkg/kinflate/tree/testdata/hierarchy/parent1/child1/Kube-manifest.yaml

@@ -0,0 +1,4 @@
+apiVersion: manifest.k8s.io/v1alpha1
+kind: Manifest
+metadata:
+  name: child1

pkg/kinflate/tree/testdata/hierarchy/parent2/Kube-manifest.yaml

@@ -0,0 +1,6 @@
+apiVersion: manifest.k8s.io/v1alpha1
+kind: Manifest
+metadata:
+  name: parent2
+packages:
+  - child2/

pkg/kinflate/tree/testdata/hierarchy/parent2/child2/Kube-manifest.yaml

@@ -0,0 +1,4 @@
+apiVersion: manifest.k8s.io/v1alpha1
+kind: Manifest
+metadata:
+  name: child2

pkg/kinflate/tree/testdata/valid/Kube-manifest.yaml

@@ -0,0 +1,14 @@
+apiVersion: manifest.k8s.io/v1alpha1
+kind: Manifest
+metadata:
+  name: valid-app
+namePrefix: someprefix-
+objectLabels:
+  foo: bar
+objectAnnotations:
+  baseAnno: This is an annotation.
+resources:
+  - deployment.yaml
+  - cm/configmap.yaml
+patches:
+  - patch.yaml

pkg/kinflate/tree/testdata/valid/cm/configmap.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+data:
+  foo: bar
+kind: ConfigMap
+metadata:
+  name: cm1

pkg/kinflate/tree/testdata/valid/patch.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod1
+spec:
+  containers:
+    - name: nginx
+      image: nginx:latest

pkg/kinflate/tree/testdata/valid/pod.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod1
+spec:
+  containers:
+    - name: nginx
+      image: nginx