CI - added tests for relative links [DO NOT MERGE] (#1052)

* CI - added tests for relative links

Signed-off-by: Adrien Duermael <adrien@duermael.com>

* fixes to check relative links properly

Signed-off-by: Adrien Duermael <adrien@duermael.com>

* /engine/extend/plugins/ -> /engine/extend/legacy_plugins/

Signed-off-by: Adrien Duermael <adrien@duermael.com>

* do not build /tests folder with Jekyll…

Signed-off-by: Adrien Duermael <adrien@duermael.com>

* list all problematic urls in each file

don’t stop at first error encountered

Signed-off-by: Adrien Duermael <adrien@duermael.com>

* removed “stack tasks” from menu (toc.yaml)

Signed-off-by: Adrien Duermael <adrien@duermael.com>

* fixed broken links

Signed-off-by: Adrien Duermael <adrien@duermael.com>

* fixed broken link in docker-for-mac/osxfs.md

Signed-off-by: Adrien Duermael <adrien@duermael.com>

* fixed broken links in /index.html

Signed-off-by: Adrien Duermael <adrien@duermael.com>

Dockerfile

@@ -29,6 +29,7 @@ RUN svn co https://github.com/docker/docker/branches/$ENGINE_BRANCH/docs/referen
  && svn co https://github.com/docker/distribution/branches/$DISTRIBUTION_BRANCH/docs/spec allv/registry/spec \
  && wget -O allv/registry/configuration.md https://raw.githubusercontent.com/docker/distribution/$DISTRIBUTION_BRANCH/docs/configuration.md \
  && rm -rf allv/apidocs/cloud-api-source \
+ && rm -rf allv/tests \
  && jekyll build -s allv -d allvbuild \
  && find allvbuild/engine/reference -type f -name '*.html' -print0 | xargs -0 sed -i 's#href="https://docs.docker.com/#href="/#g' \
  && find allvbuild/engine/extend -type f -name '*.html' -print0 | xargs -0 sed -i 's#href="https://docs.docker.com/#href="/#g' \

_data/toc.yaml

@@ -271,7 +271,7 @@ toc:
       title: Managed plugin system
     - path: /engine/extend/plugins_authorization/
       title: Access authorization plugin
-    - path: /engine/extend/plugins/
+    - path: /engine/extend/legacy_plugins/
       title: Extending Engine with plugins
     - path: /engine/extend/plugins_network/
       title: Docker network driver plugins
@@ -435,8 +435,6 @@ toc:
         title: stack rm
       - path: /engine/reference/commandline/stack_services/
         title: stack services
-      - path: /engine/reference/commandline/stack_tasks/
-        title: stack tasks
       - path: /engine/reference/commandline/start/
         title: start
       - path: /engine/reference/commandline/stats/

_includes/content/typical_docker_workflow.md

@@ -11,10 +11,10 @@
    - Build, then run your containers on a virtual host via [Docker Machine](machine/overview.md) as you develop.
 2. Configure [networking](engine/tutorials/networkingcontainers.md) and
    [storage](engine/tutorials/dockervolumes.md) for your solution, if needed.
-3. Upload builds to a registry ([ours](engine/tutorials/dockerrepos.md), [yours](docker-trusted-registry/index.md), or your cloud provider's), to collaborate with your team.
+3. Upload builds to a registry ([ours](engine/tutorials/dockerrepos.md), [yours](/datacenter/dtr/2.0/index.md), or your cloud provider's), to collaborate with your team.
 4. If you're gonna need to scale your solution across multiple hosts (VMs or physical machines), [plan
    for how you'll set up your Swarm cluster](engine/swarm/key-concepts.md) and [scale it to meet demand](engine/swarm/swarm-tutorial/index.md).
-   - Note: Use [Universal Control Plane](ucp/overview.md) and you can manage your
+   - Note: Use [Universal Control Plane](/datacenter/ucp/1.1/overview.md) and you can manage your
      Swarm cluster using a friendly UI!
 5. Finally, deploy to your preferred
-   cloud provider (or, for redundancy, *multiple* cloud providers) with [Docker Cloud](docker-cloud/overview.md). Or, use [Docker Datacenter](https://www.docker.com/products/docker-datacenter), and deploy to your own on-premise hardware.
+   cloud provider (or, for redundancy, *multiple* cloud providers) with [Docker Cloud](/docker-cloud/index.md). Or, use [Docker Datacenter](https://www.docker.com/products/docker-datacenter), and deploy to your own on-premise hardware.

docker-for-mac/osxfs.md

@@ -200,9 +200,7 @@ associated issues and on reducing the file system data path latency. This
 requires significant analysis of file system traces and speculative development
 of system improvements to try to address specific performance issues. Perhaps
 surprisingly, application workload can have a huge effect on performance. As an
-example, here are two different use cases contributed on the [forum topic]([File
-access in mounted volumes extremely
-slow](https://forums.docker.com/t/file-access-in-mounted-volumes-extremely-slow-cpu-bound/)))
+example, here are two different use cases contributed on the [forum topic](https://forums.docker.com/t/file-access-in-mounted-volumes-extremely-slow-cpu-bound/)
 and how their performance differs and suffers due to latency, caching, and
 coherence:
 

index.md

@@ -66,11 +66,11 @@ title: Docker Documentation
 <ul class="items widthcol2 media">
 <li>
 <div class="media_image">
-	<a href="/docker-hub/overview/"><img src="/images/icon-hub@2X.png" alt="Docker Hub"></a>
+	<a href="/docker-hub/index.md"><img src="/images/icon-hub@2X.png" alt="Docker Hub"></a>
 </div>
 	<div class="media_content">
 	<div data-mh="mh_docker_projects">
-	<h3><a href="/docker-hub/overview/">Docker Hub</a></h3>
+	<h3><a href="/docker-hub/index.md">Docker Hub</a></h3>
 		<p>
     A hosted registry service for managing and building images.</p>
 	</div>
@@ -78,11 +78,11 @@ title: Docker Documentation
 </li>
 <li>
 <div class="media_image">
-	<a href="/docker-cloud/overview/"><img src="/images/icon-cloud@2X.png" alt="Docker Cloud"></a>
+	<a href="/docker-cloud/index.md"><img src="/images/icon-cloud@2X.png" alt="Docker Cloud"></a>
 </div>
 	<div class="media_content">
 	<div data-mh="mh_docker_projects">
-	<h3><a href="/docker-cloud/overview/">Docker Cloud</a></h3>
+	<h3><a href="/docker-cloud/index.md">Docker Cloud</a></h3>
 		<p>
     A hosted service for building, testing, and deploying Docker images to your hosts.</p>
 	</div>

swarm/install-w-machine.md

@@ -239,7 +239,7 @@ your swarm, and start an image on your swarm.
     $ docker run -it ubuntu bash
     ```
 
-    For more examples and ideas, visit the [User Guide](/userguide/).
+    For more examples and ideas, visit the [User Guide](/engine/userguide/intro/).
 
 5.  Use the `docker ps` command to find out which node the container ran on.
 

swarm/networking.md

@@ -12,7 +12,7 @@ container networks that span multiple Docker hosts.
 
 Before using Swarm with a custom network, read through the conceptual
 information in [Docker container
-networking](/engine/userguide/networking/dockernetworks/).
+networking](/engine/userguide/networking/).
 You should also have walked through the [Get started with multi-host
 networking](/engine/userguide/networking/get-started-overlay/)
 example.

tests/src/validator/html_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"golang.org/x/net/html"
+	"net/url"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -14,15 +15,16 @@ import (
 
 var countLinks = 0
 var countImages = 0
+var htmlContentRootPath = "/usr/src/app/allvbuild"
 
 // TestURLs tests if we're not using absolute paths for URLs
 // when pointing to local pages.
 func TestURLs(t *testing.T) {
 	count := 0
 
-	filepath.Walk("/usr/src/app/allvbuild", func(path string, info os.FileInfo, err error) error {
+	filepath.Walk(htmlContentRootPath, func(path string, info os.FileInfo, err error) error {
 
-		relPath := strings.TrimPrefix(path, "/usr/src/app/allvbuild")
+		relPath := strings.TrimPrefix(path, htmlContentRootPath)
 
 		isArchive, err := regexp.MatchString(`^/v[0-9]+\.[0-9]+/.*`, relPath)
 		if err != nil {
@@ -48,9 +50,9 @@ func TestURLs(t *testing.T) {
 
 		count++
 
-		err = testURLs(htmlBytes)
+		err = testURLs(htmlBytes, path)
 		if err != nil {
-			t.Error(err.Error(), "-", relPath)
+			t.Error(relPath + err.Error())
 		}
 		return nil
 	})
@@ -62,23 +64,27 @@ func TestURLs(t *testing.T) {
 
 // testURLs tests if we're not using absolute paths for URLs
 // when pointing to local pages.
-func testURLs(htmlBytes []byte) error {
+func testURLs(htmlBytes []byte, htmlPath string) error {
 
 	reader := bytes.NewReader(htmlBytes)
 
 	z := html.NewTokenizer(reader)
 
-	for {
+	urlErrors := ""
+	// fmt.Println("urlErrors:", urlErrors)
+	done := false
+
+	for !done {
 		tt := z.Next()
 
 		switch tt {
 		case html.ErrorToken:
 			// End of the document, we're done
-			return nil
+			done = true
 		case html.StartTagToken:
 			t := z.Token()
 
-			url := ""
+			urlStr := ""
 
 			// check tag types
 			switch t.Data {
@@ -89,29 +95,92 @@ func testURLs(htmlBytes []byte) error {
 				if !ok {
 					break
 				}
-				url = href
+				urlStr = href
 
 			case "img":
 				countImages++
 				ok, src := getSrc(t)
 				if !ok {
-					return errors.New("img with no src: " + t.String())
+					urlErrors += "\nimg with no src: " + t.String()
+					break
 				}
-				url = src
+				urlStr = src
 			}
 
 			// there's an url to test!
-			if url != "" {
-				if strings.HasPrefix(url, "http://docs.docker.com") || strings.HasPrefix(url, "https://docs.docker.com") {
-					return errors.New("found absolute link: " + t.String())
+			if urlStr != "" {
+				u, err := url.Parse(urlStr)
+				if err != nil {
+					urlErrors += "\ncan't parse url: " + t.String()
+					break
+					// return errors.New("can't parse url: " + t.String())
+				}
+				// test with github.com
+				if u.Scheme != "" && u.Host == "docs.docker.com" {
+					urlErrors += "\nabsolute: " + t.String()
+					break
+				}
+
+				// relative link
+				if u.Scheme == "" {
+
+					resourcePath := ""
+					resourcePathIsAbs := false
+
+					if filepath.IsAbs(u.Path) {
+						resourcePath = filepath.Join(htmlContentRootPath, mdToHtmlPath(u.Path))
+						resourcePathIsAbs = true
+					} else {
+						resourcePath = filepath.Join(filepath.Dir(htmlPath), mdToHtmlPath(u.Path))
+					}
+
+					if _, err := os.Stat(resourcePath); os.IsNotExist(err) {
+
+						fail := true
+
+						// index.html could mean there's a corresponding index.md meaning built the correct path
+						// but Jekyll actually creates index.html files for all md files.
+						// foo.md -> foo/index.html
+						// it does this to prettify urls, content of foo.md would then be rendered here:
+						// http://domain.com/foo/ (instead of http://domain.com/foo.html)
+						// so if there's an error, let's see if index.md exists, otherwise retry from parent folder
+						// (only if the resource path is not absolute)
+						if !resourcePathIsAbs && filepath.Base(htmlPath) == "index.html" {
+							// retry from parent folder
+							resourcePath = filepath.Join(filepath.Dir(htmlPath), "..", mdToHtmlPath(u.Path))
+							if _, err := os.Stat(resourcePath); err == nil {
+								fail = false
+							}
+						}
+
+						if fail {
+							urlErrors += "\nbroken: " + t.String()
+							break
+						}
+					}
 				}
 			}
 		}
 	}
 
+	// fmt.Println("urlErrors:", urlErrors)
+	if urlErrors != "" {
+		return errors.New(urlErrors)
+	}
 	return nil
 }
 
+func mdToHtmlPath(mdPath string) string {
+	if strings.HasSuffix(mdPath, ".md") == false {
+		// file is not a markdown, don't change anything
+		return mdPath
+	}
+	if strings.HasSuffix(mdPath, "index.md") {
+		return strings.TrimSuffix(mdPath, "md") + "html"
+	}
+	return strings.TrimSuffix(mdPath, ".md") + "/index.html"
+}
+
 // helpers
 
 func getHref(t html.Token) (ok bool, href string) {