Improve snippet logic. (#5205)

- Support snippets that specify the body syntax and output
syntax of the snippet.

- Snippets with bash syntax triggered an incorrect error message.

- No error message was produced for a misnamed snippet

- Convert a security task to use snippets to populate its
many preformatted blocks.

Makefile.core.mk

@@ -60,4 +60,7 @@ update_ref_docs:
 update_operator_yamls:
 	@scripts/grab_operator_yamls.sh
 
+update_examples:
+	@scripts/grab_examples.sh
+
 include common/Makefile.common.mk

content/en/about/contribute/creating-and-editing-pages/index.md

@@ -525,6 +525,13 @@ which renders as:
 
 {{< text_import file="test/snippet_example.txt" syntax="plain" snippet="SNIP1" >}}
 
+Within a text file, snippets can indicate the syntax of the snippet content and, for bash syntax, can
+include the syntax of the output. For example:
+
+{{< text plain >}}
+$snippet MySnippetFile.txt syntax="bash" outputis="json"
+{{< /text >}}
+
 ## Glossary terms
 
 When first introducing a specialized Istio term in a page, it is desirable to annotate the term as being in the glossary. This

content/en/docs/tasks/security/authz-http/index.md

@@ -36,9 +36,7 @@ The app presents the reviews in a round robin style: red stars, black stars, or
 
 Run the following command to enable Istio authorization for the `default` namespace:
 
-{{< text bash >}}
+{{< text_import file="examples/TestAuthorizationForHTTPServices.txt" snippet="enabling_istio_authorization.sh" >}}
-$ kubectl apply -f @samples/bookinfo/platform/kube/rbac/rbac-config-ON.yaml@
-{{< /text >}}
 
 Point your browser at the Bookinfo `productpage` (`http://$GATEWAY_URL/productpage`). Now you should see
 `"RBAC: access denied"`. This is because Istio authorization is "deny by default", which means that you need to
@@ -61,9 +59,7 @@ is accessible by services in the same namespace (i.e., `default`) and services i
 
 Run the following command to create a namespace-level access control policy:
 
-{{< text bash >}}
+{{< text_import file="examples/TestAuthorizationForHTTPServices.txt" snippet="enforcing_namespace_level_access_control_apply.sh" >}}
-$ kubectl apply -f @samples/bookinfo/platform/kube/rbac/namespace-policy.yaml@
-{{< /text >}}
 
 Once applied, the policy has the following effects:
 
@@ -73,39 +69,11 @@ set to one of the values `productpage`, `details`, `reviews`, or `ratings`. Note
 constraint specifying that
 the services must have one of the listed `app` labels.
 
-    {{< text yaml >}}
+    {{< text_import file="examples/TestAuthorizationForHTTPServices.txt" snippet="enforcing_namespace_level_access_control_service_viewer.yaml" >}}
-    apiVersion: "rbac.istio.io/v1alpha1"
-    kind: ServiceRole
-    metadata:
-      name: service-viewer
-      namespace: default
-    spec:
-      rules:
-      - services: ["*"]
-        methods: ["GET"]
-        constraints:
-        - key: "destination.labels[app]"
-          values: ["productpage", "details", "reviews", "ratings"]
-    {{< /text >}}
 
-*   Creates a `ServiceRoleBinding` that assign the `service-viewer` role to all services in the `istio-system` and `default` namespaces.
+*   Creates a `ServiceRoleBinding` that assigns the `service-viewer` role to all services in the `istio-system` and `default` namespaces.
 
-    {{< text yaml >}}
+    {{< text_import file="examples/TestAuthorizationForHTTPServices.txt" snippet="enforcing_namespace_level_access_control_bind_service_viewer.yaml" >}}
-    apiVersion: "rbac.istio.io/v1alpha1"
-    kind: ServiceRoleBinding
-    metadata:
-      name: bind-service-viewer
-      namespace: default
-    spec:
-      subjects:
-      - properties:
-          source.namespace: "istio-system"
-      - properties:
-          source.namespace: "default"
-      roleRef:
-        kind: ServiceRole
-        name: "service-viewer"
-    {{< /text >}}
 
 You can expect to see output similar to the following:
 
@@ -145,42 +113,18 @@ In this step, we will create a policy that allows external requests to access th
 
 Run the following command:
 
-{{< text bash >}}
+{{< text_import file="examples/TestAuthorizationForHTTPServices.txt" snippet="enforcing_service_level_access_control_step1_apply.sh" >}}
-$ kubectl apply -f @samples/bookinfo/platform/kube/rbac/productpage-policy.yaml@
-{{< /text >}}
 
 Once applied, the policy has the following effects:
 
 *   Creates a `ServiceRole` `productpage-viewer` which allows read access to the `productpage` service.
 
-    {{< text yaml >}}
+    {{< text_import file="examples/TestAuthorizationForHTTPServices.txt" snippet="enforcing_service_level_access_control_step1_productpage_viewer.yaml" >}}
-    apiVersion: "rbac.istio.io/v1alpha1"
-    kind: ServiceRole
-    metadata:
-      name: productpage-viewer
-      namespace: default
-    spec:
-      rules:
-      - services: ["productpage.default.svc.cluster.local"]
-        methods: ["GET"]
-    {{< /text >}}
 
 *   Creates a `ServiceRoleBinding` `bind-productpage-viewer` which assigns the `productpage-viewer` role to all
 users and services.
 
-    {{< text yaml >}}
+    {{< text_import file="examples/TestAuthorizationForHTTPServices.txt" snippet="enforcing_service_level_access_control_step1_bind_productpage_viewer.yaml" >}}
-    apiVersion: "rbac.istio.io/v1alpha1"
-    kind: ServiceRoleBinding
-    metadata:
-      name: bind-productpage-viewer
-      namespace: default
-    spec:
-      subjects:
-      - user: "*"
-      roleRef:
-        kind: ServiceRole
-        name: "productpage-viewer"
-    {{< /text >}}
 
 Point your browser at the Bookinfo `productpage` (`http://$GATEWAY_URL/productpage`). Now you should see the "Bookinfo Sample"
 page. But there are errors `Error fetching product details` and `Error fetching product reviews` on the page. These errors
@@ -199,42 +143,18 @@ We will create a policy to allow the `productpage` service to access the `detail
 
 Run the following command:
 
-{{< text bash >}}
+{{< text_import file="examples/TestAuthorizationForHTTPServices.txt" snippet="enforcing_service_level_access_control_step2_apply.sh" >}}
-$ kubectl apply -f @samples/bookinfo/platform/kube/rbac/details-reviews-policy.yaml@
-{{< /text >}}
 
 Once applied, the policy has the following effects:
 
 *   Creates a `ServiceRole` `details-reviews-viewer` which allows access to the `details` and `reviews` services.
 
-    {{< text yaml >}}
+    {{< text_import file="examples/TestAuthorizationForHTTPServices.txt" snippet="enforcing_service_level_access_control_step2_details_reviews_viewer.yaml" >}}
-    apiVersion: "rbac.istio.io/v1alpha1"
-    kind: ServiceRole
-    metadata:
-      name: details-reviews-viewer
-      namespace: default
-    spec:
-      rules:
-      - services: ["details.default.svc.cluster.local", "reviews.default.svc.cluster.local"]
-        methods: ["GET"]
-    {{< /text >}}
 
 *   Creates a `ServiceRoleBinding` `bind-details-reviews` which assigns the `details-reviews-viewer` role to the
 `cluster.local/ns/default/sa/bookinfo-productpage` service account (representing the `productpage` service).
 
-    {{< text yaml >}}
+    {{< text_import file="examples/TestAuthorizationForHTTPServices.txt" snippet="enforcing_service_level_access_control_step2_bind_details_reviews.yaml" >}}
-    apiVersion: "rbac.istio.io/v1alpha1"
-    kind: ServiceRoleBinding
-    metadata:
-      name: bind-details-reviews
-      namespace: default
-    spec:
-      subjects:
-      - user: "cluster.local/ns/default/sa/bookinfo-productpage"
-      roleRef:
-        kind: ServiceRole
-        name: "details-reviews-viewer"
-    {{< /text >}}
 
 Point your browser at the Bookinfo `productpage` (`http://$GATEWAY_URL/productpage`). Now you should see the "Bookinfo Sample"
 page with "Book Details" on the lower left part, and "Book Reviews" on the lower right part. However, in the "Book Reviews" section,
@@ -254,42 +174,18 @@ service account is the authenticated identify for the `reviews` service.
 
 Run the following command to create a policy that allows the `reviews` service to access the `ratings` service.
 
-{{< text bash >}}
+{{< text_import file="examples/TestAuthorizationForHTTPServices.txt" snippet="enforcing_service_level_access_control_step3_apply.sh" >}}
-$ kubectl apply -f @samples/bookinfo/platform/kube/rbac/ratings-policy.yaml@
-{{< /text >}}
 
 Once applied, the policy has the following effects:
 
 *   Creates a `ServiceRole` `ratings-viewer` which allows access to the `ratings` service.
 
-    {{< text yaml >}}
+    {{< text_import file="examples/TestAuthorizationForHTTPServices.txt" snippet="enforcing_service_level_access_control_step3_ratings_viewer.yaml" >}}
-    apiVersion: "rbac.istio.io/v1alpha1"
-    kind: ServiceRole
-    metadata:
-      name: ratings-viewer
-      namespace: default
-    spec:
-      rules:
-      - services: ["ratings.default.svc.cluster.local"]
-        methods: ["GET"]
-    {{< /text >}}
 
 *   Creates a `ServiceRoleBinding` `bind-ratings` which assigns `ratings-viewer` role to the
 `cluster.local/ns/default/sa/bookinfo-reviews` service account, which represents the `reviews` service.
 
-    {{< text yaml >}}
+    {{< text_import file="examples/TestAuthorizationForHTTPServices.txt" snippet="enforcing_service_level_access_control_step3_bind_ratings.yaml" >}}
-    apiVersion: "rbac.istio.io/v1alpha1"
-    kind: ServiceRoleBinding
-    metadata:
-      name: bind-ratings
-      namespace: default
-    spec:
-      subjects:
-      - user: "cluster.local/ns/default/sa/bookinfo-reviews"
-      roleRef:
-        kind: ServiceRole
-        name: "ratings-viewer"
-    {{< /text >}}
 
 Point your browser at the Bookinfo `productpage` (`http://$GATEWAY_URL/productpage`). Now you should see
 the "black" and "red" ratings in the "Book Reviews" section.
@@ -302,21 +198,12 @@ There may be some delays due to caching and other propagation overhead.
 
 *   Remove Istio authorization policy configuration:
 
-    {{< text bash >}}
+    {{< text_import file="examples/TestAuthorizationForHTTPServices.txt" snippet="remove_istio_authorization_policy.sh" >}}
-    $ kubectl delete -f @samples/bookinfo/platform/kube/rbac/ratings-policy.yaml@
-    $ kubectl delete -f @samples/bookinfo/platform/kube/rbac/details-reviews-policy.yaml@
-    $ kubectl delete -f @samples/bookinfo/platform/kube/rbac/productpage-policy.yaml@
-    {{< /text >}}
 
     Alternatively, you can delete all `ServiceRole` and `ServiceRoleBinding` resources by running the following commands:
 
-    {{< text bash >}}
+    {{< text_import file="examples/TestAuthorizationForHTTPServices.txt" snippet="remove_istio_authorization_policy_alternative.sh" >}}
-    $ kubectl delete servicerole --all
-    $ kubectl delete servicerolebinding --all
-    {{< /text >}}
 
 *   Disable Istio authorization:
 
-    {{< text bash >}}
+    {{< text_import file="examples/TestAuthorizationForHTTPServices.txt" snippet="disabling_istio_authorization.sh" >}}
-    $ kubectl delete -f @samples/bookinfo/platform/kube/rbac/rbac-config-ON.yaml@
-    {{< /text >}}

examples/TestAuthorizationForHTTPServices.txt

@@ -0,0 +1,153 @@
+$snippet enabling_istio_authorization.sh syntax="bash"
+$ kubectl apply -f @samples/bookinfo/platform/kube/rbac/rbac-config-ON.yaml@
+$endsnippet
+
+$snippet enforcing_namespace_level_access_control_apply.sh syntax="bash"
+$ kubectl apply -f @samples/bookinfo/platform/kube/rbac/namespace-policy.yaml@
+$endsnippet
+
+$snippet enforcing_namespace_level_access_control_apply.sh_output
+servicerole.rbac.istio.io/service-viewer created
+servicerolebinding.rbac.istio.io/bind-service-viewer created
+$endsnippet
+
+$snippet enforcing_namespace_level_access_control_delete.sh syntax="bash"
+$ kubectl delete -f @samples/bookinfo/platform/kube/rbac/namespace-policy.yaml@
+$endsnippet
+
+$snippet enforcing_namespace_level_access_control_service_viewer.yaml syntax="yaml"
+apiVersion: "rbac.istio.io/v1alpha1"
+kind: ServiceRole
+metadata:
+  name: service-viewer
+  namespace: default
+spec:
+  rules:
+  - services: ["*"]
+    methods: ["GET"]
+    constraints:
+    - key: "destination.labels[app]"
+      values: ["productpage", "details", "reviews", "ratings"]
+$endsnippet
+
+$snippet enforcing_namespace_level_access_control_bind_service_viewer.yaml syntax="yaml"
+apiVersion: "rbac.istio.io/v1alpha1"
+kind: ServiceRoleBinding
+metadata:
+  name: bind-service-viewer
+  namespace: default
+spec:
+  subjects:
+  - properties:
+      source.namespace: "istio-system"
+  - properties:
+      source.namespace: "default"
+  roleRef:
+    kind: ServiceRole
+    name: "service-viewer"
+$endsnippet
+
+$snippet enforcing_service_level_access_control_step1_apply.sh syntax="bash"
+$ kubectl apply -f @samples/bookinfo/platform/kube/rbac/productpage-policy.yaml@
+$endsnippet
+
+$snippet enforcing_service_level_access_control_step1_productpage_viewer.yaml syntax="yaml"
+apiVersion: "rbac.istio.io/v1alpha1"
+kind: ServiceRole
+metadata:
+  name: productpage-viewer
+  namespace: default
+spec:
+  rules:
+  - services: ["productpage.default.svc.cluster.local"]
+    methods: ["GET"]
+$endsnippet
+
+$snippet enforcing_service_level_access_control_step1_bind_productpage_viewer.yaml syntax="yaml"
+apiVersion: "rbac.istio.io/v1alpha1"
+kind: ServiceRoleBinding
+metadata:
+  name: bind-productpage-viewer
+  namespace: default
+spec:
+  subjects:
+  - user: "*"
+  roleRef:
+    kind: ServiceRole
+    name: "productpage-viewer"
+$endsnippet
+
+$snippet enforcing_service_level_access_control_step2_apply.sh syntax="bash"
+$ kubectl apply -f @samples/bookinfo/platform/kube/rbac/details-reviews-policy.yaml@
+$endsnippet
+
+$snippet enforcing_service_level_access_control_step2_details_reviews_viewer.yaml syntax="yaml"
+apiVersion: "rbac.istio.io/v1alpha1"
+kind: ServiceRole
+metadata:
+  name: details-reviews-viewer
+  namespace: default
+spec:
+  rules:
+  - services: ["details.default.svc.cluster.local", "reviews.default.svc.cluster.local"]
+    methods: ["GET"]
+$endsnippet
+
+$snippet enforcing_service_level_access_control_step2_bind_details_reviews.yaml syntax="yaml"
+apiVersion: "rbac.istio.io/v1alpha1"
+kind: ServiceRoleBinding
+metadata:
+  name: bind-details-reviews
+  namespace: default
+spec:
+  subjects:
+  - user: "cluster.local/ns/default/sa/bookinfo-productpage"
+  roleRef:
+    kind: ServiceRole
+    name: "details-reviews-viewer"
+$endsnippet
+
+$snippet enforcing_service_level_access_control_step3_apply.sh syntax="bash"
+$ kubectl apply -f @samples/bookinfo/platform/kube/rbac/ratings-policy.yaml@
+$endsnippet
+
+$snippet enforcing_service_level_access_control_step3_ratings_viewer.yaml syntax="yaml"
+apiVersion: "rbac.istio.io/v1alpha1"
+kind: ServiceRole
+metadata:
+  name: ratings-viewer
+  namespace: default
+spec:
+  rules:
+  - services: ["ratings.default.svc.cluster.local"]
+    methods: ["GET"]
+$endsnippet
+
+$snippet enforcing_service_level_access_control_step3_bind_ratings.yaml syntax="yaml"
+apiVersion: "rbac.istio.io/v1alpha1"
+kind: ServiceRoleBinding
+metadata:
+  name: bind-ratings
+  namespace: default
+spec:
+  subjects:
+  - user: "cluster.local/ns/default/sa/bookinfo-reviews"
+  roleRef:
+    kind: ServiceRole
+    name: "ratings-viewer"
+$endsnippet
+
+$snippet remove_istio_authorization_policy.sh syntax="bash"
+$ kubectl delete -f @samples/bookinfo/platform/kube/rbac/ratings-policy.yaml@
+$ kubectl delete -f @samples/bookinfo/platform/kube/rbac/details-reviews-policy.yaml@
+$ kubectl delete -f @samples/bookinfo/platform/kube/rbac/productpage-policy.yaml@
+$endsnippet
+
+$snippet remove_istio_authorization_policy_alternative.sh syntax="bash"
+$ kubectl delete servicerole --all
+$ kubectl delete servicerolebinding --all
+$endsnippet
+
+$snippet disabling_istio_authorization.sh syntax="bash"
+$ kubectl delete -f @samples/bookinfo/platform/kube/rbac/rbac-config-ON.yaml@
+$endsnippet

layouts/partials/code_block.html

@@ -17,7 +17,7 @@
     {{- $expand_links = .Get "expandlinks" }}
 {{- end -}}
 
-{{- if not $syntax -}}
+{{- if not (or $syntax $snippet) -}}
     {{- errorf "Text block does not specify a syntax (%s)" .Position -}}
     {{- $syntax = "plain" -}}
 {{- end -}}
@@ -30,8 +30,12 @@
         {{- errorf "Can't combine inner content with a url attribute (%s)" .Position -}}
     {{- end -}}
     {{- if not $download_as -}}
-        {{- $tmp := split $url "/" | last 1 -}}
+        {{- if $snippet }}
-        {{- $download_as = index $tmp 0 -}}
+            {{ $download_as = $snippet }}
+        {{- else -}}
+            {{- $tmp := split $url "/" | last 1 -}}
+            {{- $download_as = index $tmp 0 -}}
+        {{- end -}}
     {{- end -}}
 {{- else -}}
     {{- if $file -}}
@@ -40,8 +44,12 @@
             {{- errorf "Can't combine url or inner content with a file attribute (%s)" .Position -}}
         {{- end -}}
         {{- if not $download_as -}}
-            {{- $tmp := split $file "/" | last 1 -}}
+            {{- if $snippet }}
-            {{- $download_as = index $tmp 0 -}}
+                {{ $download_as = $snippet }}
+            {{- else -}}
+                {{- $tmp := split $file "/" | last 1 -}}
+                {{- $download_as = index $tmp 0 -}}
+            {{- end -}}
         {{- end -}}
     {{- else -}}
         {{- if $snippet -}}
@@ -54,20 +62,51 @@
         {{- errorf "Text blocks need to not be indented, or indented by a multiple of 4 spaces (%s)" .Page.Position -}}
     {{- end -}}
 
+    {{- if $snippet -}}
+        {{- $pattern := printf "(?msU).*\\$snippet %s +syntax=\"(.+)\" +outputis=\"(.+)\" *$\\n(.*)(?-s)\\n^.*\\$endsnippet *$(?s-U).*" $snippet -}}
+        {{- $match := findRE $pattern $text -}}
+        {{- if eq (len $match) 0 -}}
+
+            {{- $pattern := printf "(?msU).*\\$snippet %s +syntax=\"(.+)\" *$\\n(.*)(?-s)\\n^.*\\$endsnippet *$(?s-U).*" $snippet -}}
+            {{- $match = findRE $pattern $text -}}
+            {{- if eq (len $match) 0 -}}
+
+                {{- $pattern := printf "(?msU).*\\$snippet %s *$\\n(.*)(?-s)\\n^.*\\$endsnippet *$(?s-U).*" $snippet -}}
+                {{- $match = findRE $pattern $text -}}
+                {{- if eq (len $match) 0 -}}
+                    {{- errorf "Could not find snippet %s (%s)" $snippet .Position -}}
+                {{- else -}}
+                    {{- $text = replaceRE $pattern "$1" $text -}}
+                {{- end -}}
+            {{- else -}}
+                {{- if not $syntax -}}
+                    {{- $syntax = replaceRE $pattern "$1" $text -}}
+                {{- end -}}
+
+                {{- $text = replaceRE $pattern "$2" $text -}}
+            {{- end -}}
+        {{- else -}}
+            {{- if not $syntax -}}
+                {{- $syntax = replaceRE $pattern "$1" $text -}}
+            {{- end -}}
+
+            {{- if not $output_is -}}
+                {{- $output_is = replaceRE $pattern "$2" $text -}}
+            {{- end -}}
+
+            {{- $text = replaceRE $pattern "$3" $text -}}
+        {{- end -}}
+    {{- end -}}
+
     {{- if eq $syntax "bash" -}}
         {{- if not (hasPrefix $text "$") -}}
             {{- errorf "Text block specifies a bash syntax, but the first line of the block does not start with $ (%s)" .Position -}}
         {{- end -}}
     {{- else -}}
         {{- if $output_is -}}
-            {{- errorf "Only text blocks with a bash syntax can use the outputis attribute (%s)" .Position -}}
+            {{- errorf "Only text blocks with a bash syntax can use outputis (%s)" .Position -}}
         {{- end -}}
     {{- end -}}
-
-    {{- if $snippet -}}
-        {{- $pattern := printf "(?msU)(.*\\$snippet %s.*$\\n)(.*)(?-s)(\\n^.*\\$endsnippet.*$)(?s-U)(.*)" $snippet -}}
-        {{- $text = replaceRE $pattern "$2" $text -}}
-    {{- end -}}
 {{- end -}}
 
 {{- $attrs := printf "data-expandlinks='%s' " $expand_links -}}

scripts/grab_examples.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# Copyright Istio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# PLACEHOLDER
+
+gsutil -m rsync -r gs://istio-prow/pr-logs/pull/istio_istio/18015/integ-istioio-k8s-tests_istio/427/artifacts/security-690d44c99a924fefb88bca/TestAuthorizationForHTTPServices snips