Automated test for Querying metrics from prometheus (#7966)

* automated test for querying metrics from Prometheus

* automated test for querying metrics from Prometheus

* remove IfNotPresent from tests/test.mk

* remove request timeout

* fix linter(shellcheck) issues

* address code review comments

* address review comments

content/en/docs/tasks/observability/metrics/querying-metrics/index.md

@@ -7,7 +7,7 @@ aliases:
     - /docs/tasks/telemetry/querying-metrics/
     - /docs/tasks/telemetry/metrics/querying-metrics/
 owner: istio/wg-policies-and-telemetry-maintainers
-test: no
+test: yes
 ---
 
 This task shows you how to query for Istio Metrics using Prometheus. As part of
@@ -40,7 +40,7 @@ the example application throughout this task.
     browser or issue the following command:
 
     {{< text bash >}}
-    $ curl http://$GATEWAY_URL/productpage
+    $ curl "http://$GATEWAY_URL/productpage"
     {{< /text >}}
 
     {{< tip >}}

content/en/docs/tasks/observability/metrics/querying-metrics/snips.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+# shellcheck disable=SC2034,SC2153,SC2155,SC2164
+
+# Copyright Istio Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+####################################################################################################
+# WARNING: THIS IS AN AUTO-GENERATED FILE, DO NOT EDIT. PLEASE MODIFY THE ORIGINAL MARKDOWN FILE:
+#          docs/tasks/observability/metrics/querying-metrics/index.md
+####################################################################################################
+
+snip_querying_istio_metrics_1() {
+kubectl -n istio-system get svc prometheus
+}
+
+! read -r -d '' snip_querying_istio_metrics_1_out <<\ENDSNIP
+NAME         TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)    AGE
+prometheus   ClusterIP   10.109.160.254   <none>        9090/TCP   4m
+ENDSNIP
+
+snip_querying_istio_metrics_2() {
+curl "http://$GATEWAY_URL/productpage"
+}
+
+snip_querying_istio_metrics_3() {
+istioctl dashboard prometheus
+}
+
+! read -r -d '' snip_querying_istio_metrics_4 <<\ENDSNIP
+istio_requests_total
+ENDSNIP
+
+! read -r -d '' snip_querying_istio_metrics_5 <<\ENDSNIP
+istio_requests_total{destination_service="productpage.default.svc.cluster.local"}
+ENDSNIP
+
+! read -r -d '' snip_querying_istio_metrics_6 <<\ENDSNIP
+istio_requests_total{destination_service="reviews.default.svc.cluster.local", destination_version="v3"}
+ENDSNIP
+
+! read -r -d '' snip_querying_istio_metrics_7 <<\ENDSNIP
+rate(istio_requests_total{destination_service=~"productpage.*", response_code="200"}[5m])
+ENDSNIP
+
+snip_cleanup_1() {
+killall istioctl
+}

content/en/docs/tasks/observability/metrics/querying-metrics/test.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1090,SC2154,SC2155
+
+# Copyright Istio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -u
+set -o pipefail
+
+source "tests/util/samples.sh"
+
+# @setup profile=demo
+kubectl label namespace default istio-injection=enabled --overwrite
+
+# Install Bookinfo sample
+startup_bookinfo_sample
+
+# Install Prometheus
+kubectl apply -f samples/addons/prometheus.yaml -n istio-system
+_wait_for_deployment istio-system prometheus
+
+_verify_like snip_querying_istio_metrics_1 "$snip_querying_istio_metrics_1_out"
+
+# Fire a couple of requests
+_set_ingress_environment_variables
+export GATEWAY_URL="$INGRESS_HOST:$INGRESS_PORT"
+for _ in {1..50}; do
+    snip_querying_istio_metrics_2 > /dev/null
+done
+
+# Now check Prometheus dashboard for the metric. It should be present
+function urlencode() {
+    python3 -c "import urllib.parse; print(urllib.parse.quote('''$1'''))"
+}
+
+function query_prometheus() {
+    local prometheus_api_root="localhost:9090/api/v1"
+    local encoded_query=$(urlencode "$1")
+    curl -sg -m 3.0 "http://$prometheus_api_root/query?query=$encoded_query"
+}
+
+function query_total_requests() {
+    query_prometheus "$snip_querying_istio_metrics_4" | jq '.data.result[0].metric.__name__'
+}
+
+function query_requests_to_productpage() {
+    query_prometheus "$snip_querying_istio_metrics_5" | jq '.data.result[0].metric.destination_service'
+}
+
+function query_requests_to_reviews_v3() {
+    query_prometheus "$snip_querying_istio_metrics_6" | jq '.data.result[0].metric.destination_service,.data.result[0].metric.destination_version'
+}
+
+function query_rate_of_requests_to_productpage_5m() {
+    query_prometheus "$snip_querying_istio_metrics_7" | jq '.data.result[0].metric.destination_service'
+}
+
+# Prometheus living inside cluster is not accessible from outside.
+# So we need some sort of port forwarding mechanism
+snip_querying_istio_metrics_3 &
+
+_verify_contains query_total_requests '"istio_requests_total"'
+_verify_contains query_requests_to_productpage '"productpage.default.svc.cluster.local"'
+_verify_contains query_requests_to_reviews_v3 '"reviews.default.svc.cluster.local"'
+_verify_contains query_requests_to_reviews_v3 '"v3"'
+_verify_contains query_rate_of_requests_to_productpage_5m '"productpage.default.svc.cluster.local"'
+pgrep istioctl | xargs kill
+
+# @cleanup
+set +e
+pgrep istioctl | xargs kill
+kubectl delete -f samples/addons/prometheus.yaml -n istio-system
+cleanup_bookinfo_sample