Add unit test to make sure helloworld samples README consistent with source (#1102)

* unclean working version of unit testing

* fix after merging

* rename test file

* remove debugging message

* based on PR comments

test/e2e/main.go

@@ -0,0 +1,17 @@
+/*
+Copyright 2019 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2etest

test/e2e/sampleapp_test.go

@@ -19,7 +19,6 @@ limitations under the License.
 package e2etest
 
 import (
-	"strings"
 	"testing"
 
 	"github.com/knative/docs/test/sampleapp"
@@ -38,12 +37,7 @@ func TestSampleApp(t *testing.T) {
 		t.Fatalf("Failed reading config file %s: '%v'", configFile, err)
 	}
 
-	whitelist := make(map[string]bool)
+	whitelist := test.GetWhitelistedLanguages()
-	if "" != test.Flags.Languages {
-		for _, l := range strings.Split(test.Flags.Languages, ",") {
-			whitelist[l] = true
-		}
-	}
 	for _, lc := range lcs.Languages {
 		if _, ok := whitelist[lc.Language]; len(whitelist) > 0 && !ok {
 			continue

test/flag.go

@@ -20,6 +20,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strings"
 )
 
 // Flags holds the command line flags or defaults for settings in the user's environment.
@@ -61,3 +62,14 @@ func initializeFlags() *EnvironmentFlags {
 func ImagePath(name string) string {
 	return fmt.Sprintf("%s/%s:%s", Flags.DockerRepo, name, Flags.Tag)
 }
+
+// GetWhitelistedLanguages is a helper function to return a map of whitelisted languages based on Languages filter
+func GetWhitelistedLanguages() map[string]bool {
+	whitelist := make(map[string]bool)
+	if "" != Flags.Languages {
+		for _, l := range strings.Split(Flags.Languages, ",") {
+			whitelist[l] = true
+		}
+	}
+	return whitelist
+}

test/sampleconsistency/main.go

@@ -0,0 +1,17 @@
+/*
+Copyright 2019 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sampleconsistency

test/sampleconsistency/sampleconsistency_test.go

@@ -0,0 +1,120 @@
+/*
+Copyright 2019 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sampleconsistency
+
+import (
+	"bufio"
+	"os"
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/knative/docs/test"
+	"github.com/knative/docs/test/sampleapp"
+)
+
+const (
+	configFile = "../sampleapp/config.yaml"
+)
+
+func readlines(t *testing.T, filename string) []string {
+	var res []string
+	f, err := os.Open(filename)
+	if nil != err {
+		t.Fatalf("Failed opening file '%s': '%v'", filename, err)
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		res = append(res, scanner.Text())
+	}
+	if err = scanner.Err(); nil != err {
+		t.Fatalf("Failed reading file '%s': '%v'", filename, err)
+	}
+	return res
+}
+
+func normalize(si string) string {
+	return strings.Trim(si, " `\t\r\n")
+}
+
+func checkContains(t *testing.T, rl []string, src string) {
+	sl := readlines(t, src)
+	ir := 0
+	is := 0
+	best := -1
+	// Scans rl(README lines) for entire block matching sl(source lines)
+	// Pointer ir keeps on moving, and pointer moves only when there is a line match, otherwise back to 0.
+	// A match is found if pointer is moved to end.
+	for ir < len(rl) && is < len(sl) {
+		nr := normalize(rl[ir])
+		ns := normalize(sl[is])
+
+		if "" == ns {
+			is++
+			if is > best {
+				best = is
+			}
+			continue
+		}
+		if "" == nr {
+			ir++
+			continue
+		}
+		if nr != ns {
+			is = 0
+		} else {
+			is++
+			if is > best {
+				best = is
+			}
+		}
+		ir++
+	}
+	if is < len(sl) && best < len(sl) && best != -1 {
+		t.Fatalf("README.md file is missing line %d ('%s') from file '%s'", best, sl[best], src)
+	}
+}
+
+func checkDoc(t *testing.T, lc sampleapp.LanguageConfig) {
+	readme := path.Join(lc.SrcDir, "README.md")
+	rl := readlines(t, readme)
+	for _, f := range lc.Copies {
+		src := path.Join(lc.SrcDir, f)
+		checkContains(t, rl, src)
+	}
+}
+
+// TestDocSrc checks content of README.md files, and ensures that the real code of the samples
+// is properly embedded in the docs.
+func TestDocSrc(t *testing.T) {
+	lcs, err := sampleapp.GetConfigs(configFile)
+	if nil != err {
+		t.Fatalf("Failed reading config file %s: '%v'", configFile, err)
+	}
+
+	whitelist := test.GetWhitelistedLanguages()
+	for _, lc := range lcs.Languages {
+		if _, ok := whitelist[lc.Language]; len(whitelist) > 0 && !ok {
+			continue
+		}
+		lc.UseDefaultIfNotProvided()
+		t.Run(lc.Language, func(t *testing.T) {
+			checkDoc(t, lc)
+		})
+	}
+}