Add robots.txt (#652)

2021-02-16 10:06:00 -05:00 · 2021-02-16 10:06:00 -05:00 · 4d859cf2ea
parent adf2c6720d
commit 4d859cf2ea
6 changed files with 12 additions and 0 deletions
--- a/config.yaml
+++ b/config.yaml
@ -7,6 +7,7 @@ pygmentsCodeFences: true
 theme: ['docsy']

 enableGitInfo: true
+enableRobotsTXT: true

 contentDir: 'content/en'
 defaultContentLanguage: 'en'
--- a/content/en/docs/languages/csharp/daily-builds.md
+++ b/content/en/docs/languages/csharp/daily-builds.md
@ -1,5 +1,6 @@
 ---
 title: Daily builds
+robots: noindex, nofollow
 weight: 90
 # Note: this is a placeholder page. The URL to this page redirects elsewhere.
 ---
--- a/content/en/docs/languages/php/daily-builds.md
+++ b/content/en/docs/languages/php/daily-builds.md
@ -1,5 +1,6 @@
 ---
 title: Daily builds
+robots: noindex, nofollow
 weight: 90
 # Note: this is a placeholder page. The URL to this page redirects elsewhere.
 ---
--- a/content/en/docs/languages/python/daily-builds.md
+++ b/content/en/docs/languages/python/daily-builds.md
@ -1,5 +1,6 @@
 ---
 title: Daily builds
+robots: noindex, nofollow
 weight: 90
 # Note: this is a placeholder page. The URL to this page redirects elsewhere.
 ---
--- a/content/en/docs/languages/ruby/daily-builds.md
+++ b/content/en/docs/languages/ruby/daily-builds.md
@ -1,5 +1,6 @@
 ---
 title: Daily builds
+robots: noindex, nofollow
 weight: 90
 # Note: this is a placeholder page. The URL to this page redirects elsewhere.
 ---
--- a/layouts/robots.txt
+++ b/layouts/robots.txt
@ -0,0 +1,7 @@
+User-agent: *
+Disallow: /get_grpcurl
+{{ range .Pages -}}
+{{ if in (lower .Params.robots) "noindex" -}}
+Disallow: {{ .RelPermalink }}
+{{ end -}}
+{{ end -}}