Update Helloworld unit test error message to make it actionable (#1812)

* Update Helloworld unit test error message to make it actionable

Update error message with detailed explanation so that developers know how to debug if unit test ever failed.
Fix a bug in unit test.
Update all READMEs so that they are valid

* Feedback updates

docs/serving/samples/hello-world/helloworld-nodejs/README.md

@@ -54,19 +54,19 @@ cd knative-docs/docs/serving/samples/hello-world/helloworld-nodejs
 1. Create a new file named `index.js` and paste the following code:
 
    ```js
-   const express = require("express");
+   const express = require('express');
    const app = express();
 
-   app.get("/", (req, res) => {
-     console.log("Hello world received a request.");
+   app.get('/', (req, res) => {
+     console.log('Hello world received a request.');
 
-     const target = process.env.TARGET || "World";
+     const target = process.env.TARGET || 'World';
      res.send(`Hello ${target}!`);
    });
 
    const port = process.env.PORT || 8080;
    app.listen(port, () => {
-     console.log("Hello world listening on port", port);
+     console.log('Hello world listening on port', port);
    });
    ```
 

docs/serving/samples/hello-world/helloworld-python/Dockerfile

@@ -1,4 +1,4 @@
-# Use the official Python image.
+# Use the official lightweight Python image.
 # https://hub.docker.com/_/python
 FROM python:3.7-slim
 

test/sampleapp/config.go

@@ -32,6 +32,16 @@ const (
 	defaultWorkDir              = "helloworld-%s_tmp"
 	defaultAppName              = "helloworld-%s"
 	defaultYamlImagePlaceHolder = "docker.io/{username}/helloworld-%s"
+
+	// ActionMsg serves as documentation purpose, which will be referenced for
+	// clearly displaying error messages.
+	ActionMsg = "All files required for running sample apps are checked " +
+		"against README.md, the content of source files should be identical with what's " +
+		"in README.md file, the list of the files to be verified is the same set of files " +
+		"used for running sample apps, they are configured in `/test/sampleapp/config.yaml`. " +
+		"If an exception is needed the file can be configured to be copied as a separate step " +
+		"in `PreCommand` such as: " +
+		"https://github.com/knative/docs/blob/65f7b402fee7f94dfbd9e4512ef3beed7b85de66/test/sampleapp/config.yaml#L4"
 )
 
 // AllConfigs contains all LanguageConfig

test/sampleapp/config.yaml

@@ -10,8 +10,10 @@ languages:
       - "Dockerfile"
   - language: "go"
     expectedOutput: "Hello Go Sample v1!"
+    preCommands:
+      - exec: "cp"
+        args: "../../docs/serving/samples/hello-world/helloworld-go/go.mod helloworld-go_tmp/go.mod"
     copies:
-      - "go.mod"
       - "helloworld.go"
       - "service.yaml"
       - "Dockerfile"
@@ -24,6 +26,9 @@ languages:
       - "Dockerfile"
   - language: "nodejs"
     expectedOutput: "Hello Node.js Sample v1!"
+    preCommands:
+      - exec: "cp"
+        args: "../../docs/serving/samples/hello-world/helloworld-nodejs/package-lock.json helloworld-nodejs_tmp/package-lock.json"
     copies:
       - "index.js"
       - "package.json"
@@ -43,10 +48,12 @@ languages:
       - "Dockerfile"
   - language: "ruby"
     expectedOutput: "Hello Ruby Sample v1!"
+    preCommands:
+      - exec: "cp"
+        args: "../../docs/serving/samples/hello-world/helloworld-ruby/Gemfile.lock helloworld-ruby_tmp/Gemfile.lock"
     copies:
       - "app.rb"
       - "Gemfile"
-      - "Gemfile.lock"
       - "service.yaml"
       - "Dockerfile"
   - language: "shell"

test/sampleconsistency/sampleconsistency_test.go

@@ -57,16 +57,19 @@ func checkContains(t *testing.T, rl []string, src string) {
 	ir := 0
 	is := 0
 	best := -1
-	// Scans rl(README lines) for entire block matching sl(source lines)
-	// Pointer ir keeps on moving, and pointer moves only when there is a line match, otherwise back to 0.
-	// A match is found if pointer is moved to end.
+	// Scans rl(README lines) for entire block matching sl(source lines).
+	// Pointer ir: keeps on moving no matter what.
+	// Pointer is: moves only when there is a line match, otherwise back to 0. A
+	//   match is found if pointer is moved to end.
+	// best: tracks where the best match is on source lines, it's always one
+	//   more line ahead of real match
 	for ir < len(rl) && is < len(sl) {
 		nr := normalize(rl[ir])
 		ns := normalize(sl[is])
 
 		if "" == ns {
 			is++
-			if is > best {
+			if is > best { // Consider it a match if it's empty line
 				best = is
 			}
 			continue
@@ -75,7 +78,7 @@ func checkContains(t *testing.T, rl []string, src string) {
 			ir++
 			continue
 		}
-		if nr != ns {
+		if nr != ns { // Start over if a non-match is found
 			is = 0
 		} else {
 			is++
@@ -85,8 +88,13 @@ func checkContains(t *testing.T, rl []string, src string) {
 		}
 		ir++
 	}
-	if is < len(sl) && best < len(sl) && best != -1 {
-		t.Fatalf("README.md file is missing line %d ('%s') from file '%s'", best, sl[best], src)
+
+	if best == -1 {
+		// missing line is line 0
+		best = 0
+	}
+	if best != len(sl) {
+		t.Fatalf("README.md file is missing line %d ('%s') from file '%s'\nAdditional info:\n%s", best, sl[best], src, sampleapp.ActionMsg)
 	}
 }