add llms.txt to repo (#22298)

## Description
- Adds .md versions of manuals and guide pages
- Adds site-wide llms.txt file to footer (this is essentially a sitemap
that LLMs can crawl)
- Adds options for llms/AI agents to better crawl docs, users to ask
about docs in Docs AI
- Preview:
https://deploy-preview-22298--docsdocker.netlify.app/admin/organization/onboard/

## Related issues or tickets
https://docker.atlassian.net/browse/ENGDOCS-2454

## Reviews

<!-- Notes for reviewers here -->
<!-- List applicable reviews (optionally @tag reviewers) -->

- [ ] Technical review
- [ ] Editorial review
- [ ] Product review

hugo.yaml

@@ -74,8 +74,22 @@ outputFormats:
     isPlainText: true
     mediaType: "text/plain"
     notAlternative: true
+  # Markdown for LLMs, see layouts/_default/single.markdown.md
+  Markdown:
+    baseName: index
+    mediaType: "text/markdown"
+    isPlainText: true
+    isHTML: false
+    permalinkable: false
+  # llms.txt
+  llms:
+    baseName: llms
+    isPlainText: true
+    mediaType: "text/plain"
+    notAlternative: true
+    permalinkable: false
 
-# Enable custom output formats for the home page only
+# Enable custom output formats
 # (only generate the custom output files once)
 outputs:
   home:
@@ -83,6 +97,13 @@ outputs:
     - redirects
     - metadata
     - robots
+    - llms
+  page:
+    - html
+    - Markdown
+  section:
+    - html
+    - Markdown
 
 languages:
   en:

hugo_stats.json

@@ -128,6 +128,7 @@
       "With-systemd-Highly-recommended",
       "Without-packages",
       "Without-systemd",
+      "[display:none]",
       "absolute",
       "aspect-video",
       "bake-action",
@@ -219,6 +220,7 @@
       "dark:hover:bg-blue-dark",
       "dark:hover:bg-blue-dark-500",
       "dark:hover:bg-gray-dark-200",
+      "dark:hover:bg-gray-dark-400",
       "dark:hover:bg-gray-dark-500",
       "dark:hover:text-blue-dark",
       "dark:prose-invert",
@@ -284,6 +286,8 @@
       "grid-cols-1",
       "group",
       "group-hover:block'",
+      "group-open:[display:block]",
+      "group-open:rotate-180",
       "h-16",
       "h-2",
       "h-32",
@@ -318,6 +322,7 @@
       "icon-sm",
       "icon-svg",
       "inline",
+      "inline-block",
       "inline-flex",
       "inset-0",
       "invertible",
@@ -329,7 +334,9 @@
       "justify-center",
       "justify-end",
       "justify-evenly",
+      "leading-none",
       "leading-snug",
+      "leading-tight",
       "left-0",
       "lg:block",
       "lg:flex",
@@ -384,11 +391,13 @@
       "ml-2",
       "ml-3",
       "ml-4",
+      "ml-auto",
       "mt-1",
       "mt-2",
       "mt-20",
       "mt-4",
       "mt-8",
+      "mt-[2px]",
       "mx-auto",
       "my-0",
       "my-1",
@@ -401,6 +410,7 @@
       "open-kapa-widget",
       "openSUSE-and-SLES",
       "origin-bottom-right",
+      "origin-top-right",
       "ot-sdk-show-settings",
       "outline-none",
       "overflow-clip",
@@ -465,6 +475,7 @@
       "self-start",
       "shadow",
       "shadow-lg",
+      "shadow-md",
       "sm:block",
       "sm:flex",
       "sm:flex-row",
@@ -514,10 +525,13 @@
       "top-6",
       "top-full",
       "transition",
+      "transition-colors",
+      "transition-transform",
       "truncate",
       "underline-offset-2",
       "uppercase",
       "w-2",
+      "w-56",
       "w-8",
       "w-[1200px]",
       "w-[32px]",

layouts/_default/index.llms.txt

@@ -0,0 +1,11 @@
+{{- $pages := .Site.RegularPages -}}
+{{- $sorted := sort $pages "RelPermalink" -}}
+{{- $grouped := $sorted.GroupBy "Section" -}}
+
+# Docker Documentation
+
+{{ range $grouped }}
+## {{ humanize .Key }}
+{{ range .Pages }}
+- [{{ .Title }}]({{ .Permalink }}){{ end }}
+{{ end -}}

layouts/_default/list.markdown.md

@@ -0,0 +1,7 @@
+{{ .Title }}
+
+{{ .RawContent }}
+
+{{ range .Pages }}
+- [{{ .Title }}](https://docs.docker.com{{ .RelPermalink }})
+{{ end }}

layouts/_default/single.markdown.md

@@ -0,0 +1,3 @@
+{{ .Title }}
+
+{{ .RawContent }}

layouts/partials/aside.html

@@ -12,4 +12,4 @@
     </div>
   {{- end }}
 {{- end }}
-</aside>
+</aside>

layouts/partials/content-default.html

@@ -1,7 +1,12 @@
-<div class="flex gap-8 w-full">
+<div class="flex w-full gap-8">
-  <article class="prose min-w-0 flex-[2_2_0%] max-w-4xl dark:prose-invert">
+  <article class="prose min-w-0 max-w-4xl flex-[2_2_0%] dark:prose-invert">
     {{ partial "breadcrumbs.html" . }}
-    <h1 data-pagefind-weight="10" class="scroll-mt-36">{{ .Title }}</h1>
+    <h1 data-pagefind-weight="10" class="flex scroll-mt-36 items-center">
+      <span>{{ .Title }}</span>
+        <span class="ml-auto">
+          {{ partial "md-dropdown.html" . }}
+        </span>
+    </h1>
 
     <div class="block lg:hidden">
       {{ partialCached "pagemeta.html" . . }}
@@ -9,7 +14,7 @@
     </div>
     {{ .Content }}
   </article>
-  <div class="hidden flex-1 min-w-52 lg:block -mr-8 -mt-8">
+  <div class="-mr-8 -mt-8 hidden min-w-52 flex-1 lg:block">
     {{ partial "aside.html" . }}
   </div>
 </div>

layouts/partials/footer.html

@@ -10,6 +10,7 @@
       {{- with .GetPage "/contribute" }}
         <a class="underline-offset-2 hover:underline" href="{{ .Permalink }}">{{ .LinkTitle }}</a>
       {{- end }}
+      <a href="{{ "llms.txt" | relURL }}">Read llms.txt</a>
     </div>
     <hr class="text-divider-light dark:text-divider-dark" />
     <div class="grid lg:grid-cols-3 place-items-center gap-8 grid-cols-1">

layouts/partials/md-dropdown.html

@@ -0,0 +1,119 @@
+<details id="markdownDropdown" class="group relative z-10 inline-block" data-heap-id="markdown-dropdown">
+    <summary
+      class="inline-flex cursor-pointer items-center gap-2 rounded border border-gray-light-200 bg-gray-light-200 px-4 py-2 text-base font-semibold text-black transition-colors hover:bg-gray-light-300 dark:border-gray-dark-200 dark:bg-gray-dark-300 dark:text-white dark:hover:bg-gray-dark-400"
+      data-heap-id="markdown-dropdown-toggle"
+    >
+      <span>Page options</span>
+      <span class="icon-svg transition-transform group-open:rotate-180">
+        {{ partialCached "icon" "arrow_drop_down" "arrow_drop_down" }}
+      </span>
+        <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 9l-7 7-7-7" />
+      </svg>
+    </summary>
+
+    <!-- Dropdown menu -->
+    <div
+      class="absolute right-0 z-50 mt-2 w-56 origin-top-right rounded border border-gray-light-200 bg-gray-light-200 p-2 text-sm text-black shadow-md [display:none] group-open:[display:block] dark:border-gray-dark-200 dark:bg-gray-dark-300 dark:text-white"
+      data-heap-id="markdown-dropdown-menu"
+    >
+      <button
+        onclick="copyMarkdown()"
+        data-heap-id="copy-markdown-button"
+        class="flex w-full items-start gap-2 rounded px-2 py-2 text-left transition-colors hover:bg-gray-light-300 dark:hover:bg-gray-dark-400"
+      >
+        <span class="icon-svg mt-[2px] text-base leading-none">
+          {{ partial "icon" "content_copy" }}
+        </span>
+        <span class="icon-svg hidden mt-[2px] text-base leading-none">
+          {{ partial "icon" "check_circle" }}
+        </span>
+        <div class="leading-tight">
+          <div class="text-base">Copy page as Markdown for LLMs</div>
+        </div>
+      </button>
+
+      <button
+        onclick="viewPlainText()"
+        data-heap-id="view-markdown-button"
+        class="flex w-full items-start gap-2 rounded px-2 py-2 text-left transition-colors hover:bg-gray-light-300 dark:hover:bg-gray-dark-400"
+      >
+        <span class="icon-svg mt-[2px] text-base leading-none">
+          {{ partial "icon" "description" }}
+        </span>
+        <div class="leading-tight">
+          <div class="text-base">View page as plain text</div>
+        </div>
+      </button>
+
+      <button
+        onclick="openInDocsAI()"
+        data-heap-id="search-docs-ai-button"
+        class="flex w-full items-start gap-2 rounded px-2 py-2 text-left transition-colors hover:bg-gray-light-300 dark:hover:bg-gray-dark-400"
+      >
+        <span class="icon-svg mt-[2px] text-base leading-none">
+          {{ partial "icon" "search" }}
+        </span>
+        <div class="leading-tight">
+          <div class="text-base">Ask questions with Docs AI</div>
+        </div>
+      </button>
+    </div>
+  </details>
+
+  <script>
+    function getCurrentPlaintextUrl() {
+      const url = window.location.href.split("#")[0].replace(/\/$/, "");
+      return `${url}/index.md`;
+    }
+
+    function copyMarkdown() {
+      fetch(getCurrentPlaintextUrl())
+        .then((response) => response.text())
+        .then((text) => {
+          navigator.clipboard.writeText(text).then(() => {
+            const button = document.querySelector('[data-heap-id="copy-markdown-button"]');
+            if (!button) return;
+
+            const icons = button.querySelectorAll(".icon-svg");
+            const copyIcon = icons[0];
+            const checkIcon = icons[1];
+
+            copyIcon.classList.add("hidden");
+            checkIcon.classList.remove("hidden");
+
+            setTimeout(() => {
+              copyIcon.classList.remove("hidden");
+              checkIcon.classList.add("hidden");
+            }, 2000);
+          });
+        })
+        .catch((err) => {
+          console.error("Error copying markdown:", err);
+        });
+    }
+
+    function viewPlainText() {
+      window.open(getCurrentPlaintextUrl(), "_blank");
+    }
+
+    function openInDocsAI() {
+      const kapaButton = document.querySelector(".open-kapa-widget");
+      if (kapaButton) {
+        kapaButton.click();
+      } else {
+        alert("Couldn't find Docs AI.");
+      }
+    }
+
+    document.addEventListener("click", function (event) {
+      const dropdown = document.getElementById("markdownDropdown");
+
+      if (!dropdown) return;
+
+      const isClickInside = dropdown.contains(event.target);
+
+      if (!isClickInside && dropdown.hasAttribute("open")) {
+        dropdown.removeAttribute("open");
+      }
+    });
+  </script>

layouts/partials/sidebar/sections.html

@@ -104,4 +104,4 @@
   {{- with .Params.sidebar.badge }}
     <span>{{- partial "components/badge.html" (dict "color" .color "content" .text) }}</span>
   {{- end }}
-{{ end }}
+{{ end }}