istio.io/archive/v0.1/docs/tasks/metrics-logs.html

<!DOCTYPE html><html lang="en" itemscope itemtype="https://schema.org/WebPage" style="overflow-y: scroll;"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1"><meta name="title" content="Collecting Metrics and Logs"><meta name="og:title" content="Collecting Metrics and Logs"><meta name="og:image" content="/v0.1/img/logo.png"/><meta name="description" content="This task shows you how to configure Mixer to collect metrics and logs from Envoy instances."><meta name="og:description" content="This task shows you how to configure Mixer to collect metrics and logs from Envoy instances."><title>Istioldie 0.1 / Collecting Metrics and Logs</title><script> window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date; ga('create', 'UA-98480406-2', 'auto'); ga('send', 'pageview'); </script> <script async src='https://www.google-analytics.com/analytics.js'></script><link href='https://fonts.googleapis.com/css?family=Roboto:400,100,100italic,300,300italic,400italic,500,500italic,700,700italic,900,900italic' rel='stylesheet' type='text/css'><link rel="alternate" type="application/rss+xml" title="Istio Blog RSS" href="/v0.1/feed.xml"><link rel="apple-touch-icon" href="/v0.1/favicons/apple-touch-icon.png" sizes="180x180"><link rel="icon" type="image/png" href="/v0.1/favicons/android-chrome-96x96.png" sizes="96x96" ><link rel="icon" type="image/png" href="/v0.1/favicons/favicon-32x32.png" sizes="32x32"><link rel="icon" type="image/png" href="/v0.1/favicons/favicon-16x16.png" sizes="16x16"><link rel="manifest" href="/v0.1/favicons/manifest.json"><link rel="mask-icon" href="/v0.1/favicons/safari-pinned-tab.svg" color="#2DA6B0"><meta name="msapplication-TileColor" content="#ffffff"><meta name="msapplication-TileImage" content="/v0.1/favicons/mstile-150x150.png"><link href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css" rel="stylesheet"><link rel="stylesheet" href="/v0.1/css/all.css"><link rel="stylesheet" href="/v0.1/css/prism.css"><link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/font-awesome/4.4.0/css/font-awesome.min.css"> <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script></head><body class="language-unknown"><div class="nav-hero-container" style="z-index: 200000;"><nav id="header-nav" class="navbar navbar-inverse" role="navigation"><div class="container"><div class="row"><div class="col-md-11 nofloat center-block "><div class="navbar-header"> <button type="button" class="hamburger navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar-collapse-1" aria-expanded="false"> <span class="sr-only">Toggle navigation</span> <span class="icon-bar"></span> <span class="icon-bar"></span> <span class="icon-bar"></span> </button> <a class="navbar-brand" href="/v0.1/"><div> <img src="/v0.1/img/logo.png" alt="Istio" width="36px" height="54px"/> <span class="brand-name">Istioldie 0.1</span></div></a></div><div class="collapse navbar-collapse" id="navbar-collapse-1"><ul class="nav navbar-nav navbar-right"><li><a href="/v0.1/about/" >About</a></li><li><a href="/v0.1/docs/" class='current'>Docs</a></li><li><a href="/v0.1/blog/" >Blog</a></li><li><a href="/v0.1/community/" >Community</a></li><li><a href="/v0.1/faq/" >FAQ</a></li><li class="dropdown"> <a class="dropdown-toggle" data-toggle="dropdown" href=""> <i class='fa fa-lg fa-cog'></i> <span class="caret"></span> </a><ul class="dropdown-menu"><h6 class="dropdown-header">Other versions of this site</h6><li> <a href="https://istio.io">Current Release</a></li><li> <a href="https://preliminary.istio.io">Next Release</a></li><li> <a href="https://archive.istio.io">Older Releases</a></li></ul></li><li><form name="cse" id="searchbox_demo" class="navbar-form navbar-right" role="search"> <input type="hidden" name="cx" value="013699703217164175118:iwwf17ikgf4" /> <input type="hidden" name="ie" value="utf-8" /> <input type="hidden" name="hl" value="en" /><div class="form-group"><div class="input-group"> <input name="q" class="form-control" type="text" size="30" /><div class="input-group-addon"> <span class="btn-search glyphicon glyphicon-search"></span></div></div></div></form> <script type="text/javascript" src="https://www.google.com/cse/brand?form=searchbox_demo"></script></li></ul></div></div></div></div></nav></div><div class="container"><div class="row"><div class="col-md-11 nofloat center-block" style="margin-top: 3px;"><ul class="col-sm-10 nav nav-tabs"><li role="presentation" ><a href="/v0.1/docs/index.html">Welcome</a></li><li role="presentation" ><a href="/v0.1/docs/concepts/index.html">Concepts</a></li><li role="presentation" class='active'><a href="/v0.1/docs/tasks/index.html">Tasks</a></li><li role="presentation" ><a href="/v0.1/docs/samples/index.html">Samples</a></li><li role="presentation" ><a href="/v0.1/docs/reference/index.html">Reference</a></li></ul></div></div></div><script src="/v0.1/js/navtree.js"></script><div class="container docs"><div class="row"><div class="col-md-11 nofloat center-block"><div class="row"><div id="sidebar-container" class="col-sm-3"><ul class="doc-side-nav"><li><h5 class='doc-side-nav-title'>Tasks</h5></li><script type="text/javascript"> var docs = []; docs.push({path: [ "basic-access-control.md", ], url: "/docs/tasks/basic-access-control.html", title: "Enabling Simple Access Control", order: 90, overview: "This task shows how to use Istio to control access to a service."}); docs.push({path: [ "egress.md", ], url: "/docs/tasks/egress.html", title: "Enabling Egress Traffic", order: 40, overview: "Describes how to configure Istio to route traffic from services in the mesh to external services."}); docs.push({path: [ "fault-injection.md", ], url: "/docs/tasks/fault-injection.html", title: "Fault Injection", order: 60, overview: "This task shows how to inject delays and test the resiliency of your application."}); docs.push({path: [ "index.md", ], url: "/docs/tasks/index.html", title: "Tasks", order: 20, overview: "Tasks show you how to do a single specific targeted activity with the Istio system."}); docs.push({path: [ "ingress.md", ], url: "/docs/tasks/ingress.html", title: "Enabling Ingress Traffic", order: 30, overview: "Describes how to configure Istio to expose a service outside of the service mesh."}); docs.push({path: [ "installing-istio.md", ], url: "/docs/tasks/installing-istio.html", title: "Installing Istio", order: 10, overview: "This task shows you how to setup the Istio service mesh."}); docs.push({path: [ "integrating-services-into-istio.md", ], url: "/docs/tasks/integrating-services-into-istio.html", title: "Integrating Services into the Mesh", order: 20, overview: "This task shows you how to integrate your applications with the Istio service mesh."}); docs.push({path: [ "istio-auth.md", ], url: "/docs/tasks/istio-auth.html", title: "Testing Istio Auth", order: 100, overview: "This task shows you how to verify and test Istio-Auth."}); docs.push({path: [ "metrics-logs.md", ], url: "/docs/tasks/metrics-logs.html", title: "Collecting Metrics and Logs", order: 110, overview: "This task shows you how to configure Mixer to collect metrics and logs from Envoy instances."}); docs.push({path: [ "rate-limiting.md", ], url: "/docs/tasks/rate-limiting.html", title: "Enabling Rate Limits", order: 80, overview: "This task shows you how to use Istio to dynamically limit the traffic to a service."}); docs.push({path: [ "request-routing.md", ], url: "/docs/tasks/request-routing.html", title: "Configuring Request Routing", order: 50, overview: "This task shows you how to configure dynamic request routing based on weights and HTTP headers."}); docs.push({path: [ "request-timeouts.md", ], url: "/docs/tasks/request-timeouts.html", title: "Setting Request Timeouts", order: 70, overview: "This task shows you how to setup request timeouts in Envoy using Istio."}); docs.push({path: [ "zipkin-tracing.md", ], url: "/docs/tasks/zipkin-tracing.html", title: "Distributed Request Tracing", order: 120, overview: "How to configure the proxies to send tracing requests to Zipkin"}); genNavBarTree(docs) </script></ul></div><div id="tab-container" class="col-xs-1 tab-neg-margin pull-left"> <a id="sidebar-tab" class="glyphicon glyphicon-chevron-left" href="javascript:void 0;"></a></div><div id="content-container" class="thin-left-border col-sm-9 markdown"><div id="toc" class="toc"></div><div id="doc-content"><h1>Collecting Metrics and Logs</h1><p>This task shows how to configure Mixer to automatically gather telemetry for a service within a cluster. At the end of this task, a new metric and a new log stream will be enabled for calls to a specific service within your cluster.</p><p>The <a href="/v0.1/docs/samples/bookinfo.html">BookInfo</a> sample application is used as the example application throughout this task.</p><h2 id="before-you-begin">Before you begin</h2><ul><li><p><a href="./installing-istio.html">Install Istio</a> in your kubernetes cluster and deploy an application.</p></li><li><p>Configure your environment to support calling <code>istioctl mixer</code>. This may require setting up port-forwarding for the Mixer Config API as described in the <a href="/v0.1/docs/reference/commands/istioctl.html#istioctl-mixer">reference docs</a> for <code>istioctl mixer</code>.</p></li><li><p>Configure your environment to support accessing the Istio dashboard, as described in the <a href="./installing-istio.html">Installation Guide</a>. This requires installing the optional add-ons (<a href="https://prometheus.io">Prometheus</a> and <a href="https://grafana.com/">Grafana</a>), as well as verifying access to the dashboard. The Istio dashboard will be used to verify task success.</p></li></ul><h2 id="collecting-new-telemetry-data">Collecting new telemetry data</h2><ol><li><p>Create a new YAML file to hold configuration for the new metric and log stream that Istio will generate and collect automatically.</p><p>Save the following as <code>new_rule.yaml</code>:</p><pre data-src="https://raw.githubusercontent.com/istio/istio/release-0.1/samples/apps/bookinfo/mixer-rule-additional-telemetry.yaml"></pre></li><li><p>Pick a target service for the new rule.</p><p>If using the BookInfo sample, select <code>reviews.default.svc.cluster.local</code>. A fully-qualified domain name for the service is required in the following steps.</p></li><li><p>Validate that the selected service has no service-specific rules already applied.</p><pre><code class="language-bash">istioctl mixer rule get reviews.default.svc.cluster.local reviews.default.svc.cluster.local
</code></pre><p>The expected output is:</p><pre><code class="language-bash">Error: the server could not find the requested resource
</code></pre><p>If your selected service has service-specific rules, update <code>new_rule.yaml</code> to include the existing rules appropriately. Append the rule from <code>new_rule.yaml</code> to the existing <code>rules</code> block and save the updated content back over <code>new_rule.yaml</code>.</p></li><li><p>Push the new configuration to Mixer for a specific service.</p><pre><code class="language-bash">istioctl mixer rule create reviews.default.svc.cluster.local reviews.default.svc.cluster.local -f new_rule.yaml
</code></pre></li><li><p>Send traffic to that service.</p><p>For the BookInfo sample, visit <code>http://$GATEWAY_URL/productpage</code> in your web browser or issue the following command:</p><pre><code class="language-bash">curl http://$GATEWAY_URL/productpage
</code></pre><p>For purposes of this task, please refresh the page several times or issue the curl command a few times to generate traffic.</p></li><li><p>Verify that the new metric is being collected.</p><p>Setup port-forwarding for Grafana:</p><pre><code class="language-bash">kubectl port-forward $(kubectl get pod -l app=grafana -o jsonpath='{.items[0].metadata.name}') 3000:3000 &amp;
</code></pre><p>Then open the Istio dashboard in a web browser: <a href="http://localhost:3000/dashboard/db/istio-dashboard">http://localhost:3000/dashboard/db/istio-dashboard</a></p><p>One of the rows in the dashboard will be named “reviews”. If that row is not visible, please refresh the dashboard page. The “reviews” row contains a graph entitled “Response Size by Source And Version”. The graph displays a breakdown of the distribution of Response Sizes returned by the “reviews” service.</p><p>The request from the previous step is reflected in the graphs. This looks similar to:</p><figure><img style="max-width: 100%;" src="./img/dashboard_response_size.png" alt="Istio Dashboard for Reviews Service" title="Istio Dashboard for Reviews Service" /><figcaption>Istio Dashboard for Reviews Service</figcaption></figure></li><li><p>Verify that the logs stream has been created and is being populated for requests.</p><p>Search through the logs for the Mixer pod as follows:</p><pre><code class="language-bash">kubectl logs $(kubectl get pods -l istio=mixer -o jsonpath='{.items[0].metadata.name}') | grep \"combined_log\"
</code></pre><p>The expected output is similar to:</p><pre><code class="language-json">{"logName":"combined_log","labels":{"referer":"","responseSize":871,"timestamp":"2017-04-29T02:11:54.989466058Z","url":"/reviews","userAgent":"python-requests/2.11.1"},"textPayload":"- - - [29/Apr/2017:02:11:54 +0000] \"- /reviews -\" - 871 - python-requests/2.11.1"}
</code></pre></li></ol><h2 id="understanding-the-new-telemetry-rule">Understanding the new telemetry rule</h2><p>In this task, you added a new rule for a service within your cluster. The new rule instructed Mixer to automatically generate and report a new metric and a new log stream for all traffic going to a specific service.</p><p>The new rule was comprised of a new <code>aspect</code> definitions. These <code>aspect</code> definitions were for the aspect kind of <code>metrics</code> and <code>access-logs</code>.</p><h3 id="understanding-the-rules-metrics-aspect">Understanding the rule’s metrics aspect</h3><p>The <code>metrics</code> aspect directs Mixer to report metrics to the <code>prometheus</code> adapter. The adapter <code>params</code> tell Mixer <em>how</em> to generate metric values for any given request, based on the attributes reported by Envoy (and generated by Mixer itself).</p><p>The schema for the metric came from a predefined metric <code>descriptor</code> known to Mixer. In this task, the descriptor used was <code>response_size</code>. The <code>response_size</code> metric descriptor uses buckets to record a distribution of values, making it easier for the backend metrics systems to provide summary statistics for a bunch of requests in aggregate (as is often desirable when looking at response sizes).</p><p>The new rule instructs Mixer to generate values for the metric based on the values of the attribute <code>response.size</code>. A default values of <code>0</code> was added, in case Envoy does not report the values as expected.</p><p>A set of dimensions were also configured for the metric value, via the <code>labels</code> chunks of configuration. For the new metric, the dimensions were <code>source</code>, <code>target</code>, <code>service</code>, <code>version</code>, <code>method</code>, and <code>response_code</code>.</p><p>Dimensions provide a way to slice, aggregate, and analyze metric data according to different needs and directions of inquiry. For instance, it may be desirable to only consider response sizes for non-error responses when troubleshooting the rollout of a new application version.</p><p>The new rule instructs Mixer to populate values for these dimensions based on attribute values. For instance, for the <code>service</code> dimension, the new rule requests that the value be taken from the <code>target.labels["app"]</code> attribute. If that attribute value is not populated, the rule instructs Mixer to use a default value of <code>"unknown"</code>.</p><p>At the moment, it is not possible to programmatically generate new metric descriptors for use within Mixer. As a result, all new metric configurations must use one of the predefined metrics descriptors: <code>request_count</code>, <code>request_duration</code>, <code>request_size</code>, and <code>response_size</code>.</p><p>Work is ongoing to extend the Mixer Config API to add support for creating new descriptors.</p><h3 id="understanding-the-rules-access_logs-aspect">Understanding the rule’s access_logs aspect</h3><p>The <code>access-logs</code> aspect directs Mixer to send access logs to the <code>default</code> adapter (typically, <code>stdioLogger</code>). The adapter <code>params</code> tell Mixer <em>how</em> to generate the access logs for incoming requests based on attributes reported by Envoy.</p><p>The <code>logName</code> parameter is used by Mixer to identify a logs stream. In this task, the log name <code>combined_log</code> was used to identify the log stream amidst the rest of the Mixer logging output. This name should be used to uniquely identify log streams to various logging backends.</p><p>The <code>log</code> section of the rule describes the shape of the access log that Mixer will generate when the rule is applied. In this task, the pre-configured definition for an access log named <code>accesslog.combined</code> was used. It is based on the well-known <a href="https://httpd.apache.org/docs/1.3/logs.html#combined">Combined Log Format</a>.</p><p>Access logs use a template to generate a plaintext log from a set of named arguments. The template is defined in the configured <code>descriptor</code> for the aspect. In this task, the template used is defined in the descriptor named <code>accesslog.combined</code>. The set of inputs to the <code>template_expressions</code> is fixed in the descriptor and cannot be altered in aspect configuration.</p><p>The <code>template_expressions</code> describe how to translate attribute values into the named arguments for the template processing. For example, the value for <code>userAgent</code> is to be derived directly from the value for the attribute <code>request.headers["user-agent"]</code>.</p><p>Mixer supports structured log generation in addition to plaintext logs. In this task, a set of <code>labels</code> to populate for structured log generation was configured. These <code>labels</code> are populated from attribute values according to attribute expressions, in exactly the same manner as the <code>template_expressions</code>.</p><p>While it is common practice to include the same set of arguments in the <code>labels</code> as in the <code>template_expressions</code>, this is not required. Mixer will generate the <code>labels</code> completely independently of the <code>template_expressions</code>.</p><p>As with metric descriptors, it is not currently possible to programmatically generate new access logs descriptors. Work is ongoing to extend the Mixer Config API to add support for creating new descriptors.</p><h2 id="whats-next">What’s next</h2><ul><li><p>Learn more about <a href="/v0.1/docs/concepts/policy-and-control/mixer.html">Mixer</a> and <a href="/v0.1/docs/concepts/policy-and-control/mixer-config.html">Mixer Config</a>.</p></li><li><p>Discover the full <a href="/v0.1/docs/reference/config/mixer/attribute-vocabulary.html">Attribute Vocabulary</a>.</p></li><li><p>Read the reference guide to <a href="/v0.1/docs/reference/writing-config.html">Writing Config</a>.</p></li><li><p>If you are not planning to explore any follow-on tasks, refer to the <a href="/v0.1/docs/samples/bookinfo.html#cleanup">BookInfo cleanup</a> instructions to shutdown the application and cleanup the associated rules.</p></li></ul></div></div></div></div></div></div><script src="/v0.1/js/sidemenu.js"></script><footer><div class="container"><div class="row"><div class="col-md-2"></div><div class="col-md-3 col-sm-4 col-xs-12 center-block"><ul class="toggle"><p class="header">Docs</p><li><a href="/v0.1/docs/">Welcome</a></li><li><a href="/v0.1/docs/concepts">Concepts</a></li><li><a href="/v0.1/docs/tasks">Tasks</a></li><li><a href="/v0.1/docs/samples">Samples</a></li><li><a href="/v0.1/docs/reference">Reference</a></li></ul></div><hr class="footer-sections" /><div class="col-md-3 col-sm-4 col-xs-12 center-block"><ul class="toggle"><p class="header">Resources</p><li><a href="/v0.1/faq">Frequently Asked Questions</a></li><li><a href="/v0.1/troubleshooting">Troubleshooting Guide</a></li><li><a href="/v0.1/bugs">Report a Bug</a></li><li><a href="https://github.com/istio/istio.github.io/issues/new?title=Issue with _docs/tasks/metrics-logs.md">Report a Doc Issue</a></li><li><a href="https://github.com/istio/istio.github.io/edit/master/_docs/tasks/metrics-logs.md">Edit This Page on GitHub</a></li></ul></div><hr class="footer-sections" /><div class="col-md-3 col-sm-4 col-xs-12 center-block"><ul class="toggle"><p class="header">Community</p><li><a href="https://groups.google.com/forum/#!forum/istio-users" target="_blank"><span class="group">User</span></a> | <a href="https://groups.google.com/forum/#!forum/istio-dev" target="_blank">Dev Mailing Lists</a></li><li><a href="https://twitter.com/IstioMesh" target="_blank"><span class="twitter">Twitter</span></a></li><li><a href="https://github.com/istio/istio" target="_blank"><span class="github">GitHub</span></a></li></ul></div><div class="col-md-1"></div></div><div class="row"><p class="description small text-center"> Copyright &copy; 2017 Istio Authors<br> Istio 0.1<br> Archived on 20-Jul-2017</p></div></div></footer><script src="https://cdnjs.cloudflare.com/ajax/libs/jquery-validate/1.15.0/jquery.validate.min.js"></script> <script src="/v0.1/js/jquery.form.min.js"></script> <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/js/bootstrap.min.js"></script> <script src="/v0.1/js/slick.min.js"></script> <script src="/v0.1/js/jquery.visible.min.js"></script> <script src="/v0.1/js/common.js" type="text/javascript" charset="utf-8"></script> <script src="/v0.1/js/buttons.js"></script> <script src="/v0.1/js/search.js"></script> <script src="/v0.1/js/prism.js"></script></body></html>