Extend the test harness to incorporate parameters and negative testing (#79)

* extend the test harness to incorporate parameters and negative testing

* functioning tests with parameters

* separated the actual vs the expected

docs/TestFramework.md

@@ -0,0 +1,11 @@
+
+1. we want to version the generated golang test cases that include the resources embedded in the golang code (what the hack/gen-test-target.sh is doing)
+2. the generated code is a known, passing test case that is used to compare with PR changes.
+3. if the author of the PR is making changes- they should regen the test case
+4. we should also do a `kustomize build | kubectl apply --validate --dry-run -f -` so kubectl can check on the indentation in the yaml, bogus values, schema checks. It will validate syntax and parameters and values.
+
+
+5. gotest close to package (manifests)
+6. golang is not brittle
+
+

hack/gen-test-target.sh

@@ -22,9 +22,16 @@ gen-target-start() {
   fname=/manifests${dir##*/manifests}
   target=$(kebab-case-2-PascalCase $(get-target-name $1))
 
-  echo 'package kustomize_test'
+  echo 'package tests_test'
   echo ''
   echo 'import ('
+  echo '  "sigs.k8s.io/kustomize/k8sdeps/kunstruct"'
+  echo '  "sigs.k8s.io/kustomize/k8sdeps/transformer"'
+  echo '  "sigs.k8s.io/kustomize/pkg/fs"'
+  echo '  "sigs.k8s.io/kustomize/pkg/loader"'
+  echo '  "sigs.k8s.io/kustomize/pkg/resmap"'
+  echo '  "sigs.k8s.io/kustomize/pkg/resource"'
+  echo '  "sigs.k8s.io/kustomize/pkg/target"'
   echo '  "testing"'
   echo ')'
   echo ''
@@ -95,22 +102,6 @@ gen-target-resource() {
   echo '`)'
 }
 
-gen-expected-start() {
-  echo '  th.assertActualEqualsExpected(m, `'
-}
-
-gen-expected-end() {
-  echo '`)'
-}
-
-gen-expected() {
-  gen-expected-start
-  cd $1
-  kustomize build
-  cd - > /dev/null
-  gen-expected-end
-}
-
 gen-test-case() {
   local base=$(get-target-name $1) dir=$(get-target $1) target fname
   fname=/manifests${dir##*/manifests}/$(get-target-dirname $1)
@@ -125,7 +116,24 @@ gen-test-case() {
   echo '  if err != nil {'
   echo '    t.Fatalf("Err: %v", err)'
   echo '  }'
-  gen-expected $1
+  echo '  targetPath := "'$1'"'
+
+  echo '  fsys := fs.MakeRealFS()'
+  echo '    _loader, loaderErr := loader.NewLoader(targetPath, fsys)'
+  echo '    if loaderErr != nil {'
+  echo '      t.Fatalf("could not load kustomize loader: %v", loaderErr)'
+  echo '    }'
+  echo '    rf := resmap.NewFactory(resource.NewFactory(kunstruct.NewKunstructuredFactoryImpl()))'
+  echo '    kt, err := target.NewKustTarget(_loader, rf, transformer.NewFactoryImpl())'
+  echo '    if err != nil {'
+  echo '      th.t.Fatalf("Unexpected construction error %v", err)'
+  echo '    }'
+  echo '  n, err := kt.MakeCustomizedResMap()'
+  echo '  if err != nil {'
+  echo '    t.Fatalf("Err: %v", err)'
+  echo '  }'
+  echo '  expected, err := n.EncodeAsYaml()'
+  echo '  th.assertActualEqualsExpected(m, string(expected))'
   echo '}'
 }
 

hack/gen-test-targets.sh

@@ -15,7 +15,9 @@ rm -f $(ls tests/*_test.go | grep -v kusttestharness_test.go)
 for i in $(find * -type d -exec sh -c '(ls -p "{}"|grep />/dev/null)||echo "{}"' \; | egrep -v 'docs|tests|hack'); do
   rootdir=$(pwd)
   absdir=$rootdir/$i
-  testname=$(get-target-name $absdir)_test.go
-  echo generating $testname from $absdir
-  ./hack/gen-test-target.sh $absdir > tests/$testname
+  if [[ ! $absdir  =~ overlays/test$ ]]; then
+    testname=$(get-target-name $absdir)_test.go
+    echo generating $testname from /manifests/${absdir#*manifests/}
+    ./hack/gen-test-target.sh $absdir > tests/$testname
+  fi
 done

hack/kustomize/plugin/kfdef.apps.kubeflow.org/v1alpha1/KfDef

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+cat $1
+

hack/utils.sh

@@ -75,7 +75,7 @@ get-target() {
 # get-target-name will return tf-job-operator-base
 #
 # Given the path /manifests/tf-training/tf-job-operator/overlays/namespaced-gangscheduled
-# get-target-name will return tf-job-operator-namespaced-gangscheduled
+# get-target-name will return tf-job-operator-overlays-namespaced-gangscheduled
 #
 get-target-name() {
   local b=$(basename $1)
@@ -84,7 +84,9 @@ get-target-name() {
       echo $(basename $(dirname $1))-$b
       ;;
     *)
-      echo $(basename $(dirname $(dirname $1)))-$b
+      overlaydir=$(dirname $1)
+      overlay=$(basename $overlaydir)
+      echo $(basename $(dirname $overlaydir))-$overlay-$b
       ;;
   esac
 }

profiles/overlays/debug/params.yaml

@@ -1,2 +1,3 @@
 varReference:
 - path: spec/template/spec/containers/image
+  kind: Deployment

profiles/overlays/test/app_test.yaml

@@ -0,0 +1,23 @@
+apiVersion: kfdef.apps.kubeflow.org/v1alpha1
+kind: KfDef
+metadata:
+  name: plugin-test
+spec:
+  appdir: .
+  componentParams:
+    profiles:
+    - name: overlay
+      value: debug
+    - name: overlay
+      value: devices
+  components:
+  - profiles
+  manifestsRepo: /Users/kdkasrav/plugin-test/.cache/manifests/pull/31/head
+  packageManager: kustomize@pull/31
+  packages:
+  - profiles
+  repo: /Users/kdkasrav/plugin-test/.cache/kubeflow/master/kubeflow
+  useBasicAuth: false
+  useIstio: true
+  version: master
+status: {}

profiles/overlays/test/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+bases:
+- ../../base
+generators:
+- app_test.yaml

tests/Makefile

@@ -12,11 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-GOLANG_VERSION ?= 1.12
+GOLANG_VERSION ?= 1.12.4
 GOPATH ?= $(HOME)/go
 
 # Comma seperated items within {} for more than one file
-EXCLUDE ?= {webhook-base_test.go,jupyter-base_test.go}
+EXCLUDE ?= 
 
 export GO111MODULE = on
 export GO = go
@@ -24,10 +24,13 @@ export GO = go
 all: test
 
 generate: clean
-	@cd .. && hack/gen-test-targets.sh && [ -n "$(EXCLUDE)" ] && rm -f tests/$(EXCLUDE) && rm -f tests/webhook-base_test.go && rm -f tests/bootstrap-base_test.go
+	@cd .. && hack/gen-test-targets.sh && [ -n "$(EXCLUDE)" ] && rm -f tests/$(EXCLUDE) || echo done
 
-test: generate
+test: 
 	@GO111MODULE=on $(GO) test -v .
 
 clean:
 	@rm -f $$(ls *_test.go | egrep -v 'kusttestharness_test.go|go.mod|go.sum')
+
+run-plugin:
+	cd .. && XDG_CONFIG_HOME=$$(pwd)/hack kustomize --enable_alpha_goplugins_accept_panic_risk build profiles/overlays/test

tests/kusttestharness_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package kustomize_test
+package tests_test
 
 // A collection of utilities used in target tests.
 

tests/scripts/run-tests.sh

@@ -25,4 +25,4 @@ cd tests
 # the tests depend on kustomize
 export PATH=${GOPATH}/bin:/usr/local/go/bin:${PATH}
 go get sigs.k8s.io/kustomize
-make all
+make test