istio.io/archive/v0.8/help/troubleshooting/index.html

985 lines
53 KiB
HTML

<!DOCTYPE html>
<html lang="en" itemscope itemtype="https://schema.org/WebPage">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<meta name="theme-color" content="#466BB0"/>
<meta name="title" content="Troubleshooting Guide">
<meta name="description" content="Practical advice on practical problems with Istio">
<meta name="og:title" content="Troubleshooting Guide">
<meta name="og:description" content="Practical advice on practical problems with Istio">
<meta name="og:url" content="/v0.8/help/troubleshooting/">
<meta name="og.site_name" content="Istio">
<title>Istioldie 0.8 / Troubleshooting Guide</title>
<script>
window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
ga('create', 'UA-98480406-2', 'auto');
ga('send', 'pageview');
</script>
<script async src='https://www.google-analytics.com/analytics.js'></script>
<script>
var branchName = "release-0.8";
</script>
<link rel="alternate" type="application/rss+xml" title="Istio Blog" href="/v0.8/feed.xml">
<link rel="shortcut icon" href="/v0.8/favicons/favicon.ico" >
<link rel="apple-touch-icon" href="/v0.8/favicons/apple-touch-icon-180x180.png" sizes="180x180">
<link rel="icon" type="image/png" href="/v0.8/favicons/favicon-16x16.png" sizes="16x16">
<link rel="icon" type="image/png" href="/v0.8/favicons/favicon-32x32.png" sizes="32x32">
<link rel="icon" type="image/png" href="/v0.8/favicons/android-36x36.png" sizes="36x36">
<link rel="icon" type="image/png" href="/v0.8/favicons/android-48x48.png" sizes="48x48">
<link rel="icon" type="image/png" href="/v0.8/favicons/android-72x72.png" sizes="72x72">
<link rel="icon" type="image/png" href="/v0.8/favicons/android-96x196.png" sizes="96x196">
<link rel="icon" type="image/png" href="/v0.8/favicons/android-144x144.png" sizes="144x144">
<link rel="icon" type="image/png" href="/v0.8/favicons/android-192x192.png" sizes="192x192">
<link rel="manifest" href="/v0.8/manifest.json">
<meta name="apple-mobile-web-app-title" content="Istio">
<meta name="application-name" content="Istio">
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:400,100,100italic,300,300italic,400italic,500,500italic,700,700italic,900,900italic">
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.6/css/all.css">
<link rel="stylesheet" href="/v0.8/css/light_theme_archive.css" title="light">
<link rel="alternate stylesheet" href="/v0.8/css/dark_theme_archive.css" title="dark">
<script src="/v0.8/js/styleSwitcher.min.js"></script>
</head>
<body class="language-unknown">
<header>
<nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark justify-content-between">
<a class="navbar-brand" href="/v0.8/">
<span class="logo"><svg viewBox="0 0 300 300">
<circle cx="150" cy="150" r="150" stroke-width="2" />
<polygon points="65,240 225,240 125,270"/>
<polygon points="65,230 125,220 125,110"/>
<polygon points="135,220 225,230 135,30"/>
</svg>
</span>
<span class="brand-name">Istioldie 0.8</span>
</a>
<button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarCollapse" aria-controls="navbarCollapse" aria-expanded="false" aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
<div class="collapse navbar-collapse justify-content-end" id="navbarCollapse">
<ul id="navbar-links" class="navbar-nav active">
<li class="nav-item">
<a class="nav-link " href="/v0.8/docs/">Docs</a>
</li>
<li class="nav-item">
<a class="nav-link " href="/v0.8/blog/2018/egress-monitoring-access-control/">Blog</a>
</li>
<li class="nav-item">
<a class="nav-link active" href="/v0.8/help/">Help</a>
</li>
<li class="nav-item">
<a class="nav-link " href="/v0.8/community/">Community</a>
</li>
<li class="nav-item">
<a class="nav-link " href="/v0.8/about/">About</a>
</li>
<li class="nav-item dropdown" id="gearDropdown" style="white-space: nowrap">
<a href="" class="nav-link" data-toggle="dropdown" aria-label="Tools" aria-haspopup="true" aria-expanded="false">
<i style="width: 1em" class='fa fa-lg fa-cog'></i>
</a>
<div class="dropdown-menu dropdown-menu-right" aria-labelledby="gearDropdown">
<a class="dropdown-item" id="light-theme-item" href="" onclick="setActiveStyleSheet('light');return false;">Light Theme</a>
<a class="dropdown-item" id="dark-theme-item" href="" onclick="setActiveStyleSheet('dark');return false;">Dark Theme</a>
<div class="dropdown-divider"></div>
<h6 class="dropdown-header">Other versions of this site</h6>
<a href="https://istio.io" class="dropdown-item">Current Release</a>
<a href="https://preliminary.istio.io" class="dropdown-item">Next Release</a>
<a href="https://archive.istio.io" class="dropdown-item">Older Releases</a>
</div>
</li>
<li class="nav-item">
<a id="search_show" class="nav-link" href="" aria-label="Search"><i style="width: 1em" class="fa fa-lg fa-search"></i></a>
</li>
</ul>
<form name="cse" id="search_form" class="form-inline mr-sm-2" role="search">
<input type="hidden" name="cx" value="013699703217164175118:iwwf17ikgf4" />
<input type="hidden" name="ie" value="utf-8" />
<input type="hidden" name="hl" value="en" />
<input type="hidden" id="search_page_url" value="/v0.8/search.html" />
<input id="search_textbox" class="form-control" name="q" type="text" aria-label="Search this site"/>
<button id="search_close" type="reset" aria-label="Cancel Search"><i class="far fa-lg fa-times-circle"></i></button>
</form>
</div>
</nav>
</header>
<div class="container-fluid">
<div class="row row-offcanvas">
<div class="col-0 col-md-3 col-xl-2 sidebar-offcanvas">
<nav class="sidebar d-print-none">
<div class="spacer"></div>
<div class="directory" role="tablist">
<div class="card">
<div class="card-header" role="tab">
<div title="A bunch of resources to help you deploy, configure and use Istio.">
Help!
</div>
</div>
<div role="tabpanel" aria-labelledby="header0">
<div class="card-body">
<ul class="tree">
<li class="sublist">
<label class='tree-toggle'>
<i class='fa fa-lg fa-caret-right'></i>
<a title="Frequently Asked Questions about Istio." href="/v0.8/help/faq/">FAQ </a>
</label>
<ul class="tree collapse">
<li>
<a title="General Q &amp; A" href="/v0.8/help/faq/general/">General</a>
</li>
<li>
<a title="Setup Q &amp; A" href="/v0.8/help/faq/setup/">Setup</a>
</li>
<li>
<a title="Security Q &amp; A" href="/v0.8/help/faq/security/">Security</a>
</li>
<li>
<a title="Mixer Q &amp; A" href="/v0.8/help/faq/mixer/">Mixer</a>
</li>
<li>
<a title="Traffic Management Q &amp; A" href="/v0.8/help/faq/traffic-management/">Traffic Management</a>
</li>
</ul>
</li>
<li>
<a title="A glossary of common Istio terms." href="/v0.8/help/glossary/">Glossary</a>
</li>
<li>
<a title="What to do about bugs" href="/v0.8/help/bugs/">Reporting Bugs</a>
</li>
<li>
<span class="current" title="Practical advice on practical problems with Istio">Troubleshooting Guide</span>
</li>
</ul>
</div>
</div>
</div>
</div>
</nav>
</div>
<div class="col-12 col-md-9 col-xl-10">
<p class="d-md-none">
<label class="sidebar-toggler" data-toggle="offcanvas">
<i class="fa fa-sign-out-alt"></i>
</label>
</p>
<main aria-labelledby="title">
<h1 id="title">Troubleshooting Guide</h1>
<nav class="toc-inlined d-xl-none d-print-none" style="display:block!important" >
<div class="directory" role="directory">
<nav id="InlinedTableOfContents">
<ul>
<li><a href="#route-rules-have-no-effect-on-my-application">Route rules have no effect on my application</a></li>
<li><a href="#verifying-connectivity-to-istio-pilot">Verifying connectivity to Istio Pilot</a></li>
<li><a href="#no-traces-appearing-in-zipkin-when-running-istio-locally-on-mac">No traces appearing in Zipkin when running Istio locally on Mac</a></li>
<li><a href="#envoy-wont-connect-to-my-http10-service">Envoy won't connect to my HTTP/1.0 service</a></li>
<li><a href="#no-grafana-output-when-connecting-from-a-local-web-client-to-istio-remotely-hosted">No Grafana output when connecting from a local web client to Istio remotely hosted</a></li>
<li><a href="#where-are-the-metrics-for-my-service">Where are the metrics for my service?</a>
<ul>
<li><a href="#verify-mixer-is-receiving-report-calls">Verify Mixer is receiving Report calls</a></li>
<li><a href="#verify-mixer-metrics-configuration-exists">Verify Mixer metrics configuration exists</a></li>
<li><a href="#verify-mixer-is-sending-metric-instances-to-the-prometheus-adapter">Verify Mixer is sending metric instances to the Prometheus adapter</a></li>
<li><a href="#verify-prometheus-configuration">Verify Prometheus configuration</a></li>
</ul>
</li>
<li><a href="#how-can-i-debug-issues-with-the-service-mesh">How can I debug issues with the service mesh?</a>
<ul>
<li><a href="#with-istioctl">With istioctl</a></li>
<li><a href="#with-gdb">With GDB</a></li>
<li><a href="#with-tcpdump">With Tcpdump</a></li>
</ul>
</li>
<li><a href="#envoy-is-crashing-under-load">Envoy is crashing under load</a></li>
<li><a href="#headless-tcp-services-losing-connection-from-istiofied-containers">Headless TCP services losing connection from Istiofied containers</a></li>
<li><a href="#envoy-process-high-cpu-usage">Envoy Process High CPU Usage</a></li>
<li><a href="#kubernetes-webhook-setup-script-files-are-missing-from-05-release-package">Kubernetes webhook setup script files are missing from 0.5 release package</a></li>
<li><a href="#automatic-sidecar-injection-will-fail-if-the-kube-apiserver-has-proxy-settings">Automatic sidecar injection will fail if the kube-apiserver has proxy settings</a></li>
</ul>
</nav>
</div>
</nav>
<p>Oh no! You're having trouble? Below is a list of solutions to common problems.</p>
<h2 id="route-rules-have-no-effect-on-my-application">Route rules have no effect on my application</h2>
<p>If route rules are working perfectly for the <a href="/v0.8/docs/guides/bookinfo/">Bookinfo</a> sample,
but similar version routing rules have no effect on your own application, it may be that
your Kubernetes services need to be changed slightly.</p>
<p>Kubernetes services must adhere to certain restrictions in order to take advantage of
Istio's L7 routing features.
Refer to the <a href="/v0.8/docs/setup/kubernetes/sidecar-injection/#pod-spec-requirements">sidecar injection documentation</a>
for details.</p>
<h2 id="verifying-connectivity-to-istio-pilot">Verifying connectivity to Istio Pilot</h2>
<p>Verifying connectivity to Pilot is a useful troubleshooting step. Every proxy container in the service mesh should be able to communicate with Pilot. This can be accomplished in a few simple steps:</p>
<ol>
<li>
<p>Get the name of the Istio Ingress pod:</p>
<pre><code class="language-command" data-lang="command">$ INGRESS_POD_NAME=$(kubectl get po -n istio-system | grep ingress\- | awk '{print$1}')
</code></pre></li>
<li>
<p>Exec into the Istio Ingress pod:</p>
<pre><code class="language-command" data-lang="command">$ kubectl exec -it $INGRESS_POD_NAME -n istio-system /bin/bash
</code></pre></li>
<li>
<p>Test connectivity to Pilot using cURL. The following example cURL's the v1 registration API using default Pilot configuration parameters and mTLS enabled:</p>
<pre><code class="language-command" data-lang="command">$ curl -k --cert /etc/certs/cert-chain.pem --cacert /etc/certs/root-cert.pem --key /etc/certs/key.pem https://istio-pilot:15003/v1/registration
</code></pre><p>If mTLS is disabled:</p>
<pre><code class="language-command" data-lang="command">$ curl http://istio-pilot:15003/v1/registration
</code></pre></li>
</ol>
<p>You should receive a response listing the &ldquo;service-key&rdquo; and &ldquo;hosts&rdquo; for each service in the mesh.</p>
<h2 id="no-traces-appearing-in-zipkin-when-running-istio-locally-on-mac">No traces appearing in Zipkin when running Istio locally on Mac</h2>
<p>Istio is installed and everything seems to be working except there are no traces showing up in Zipkin when there
should be.</p>
<p>This may be caused by a known <a href="https://github.com/docker/for-mac/issues/1260">Docker issue</a> where the time inside
containers may skew significantly from the time on the host machine. If this is the case,
when you select a very long date range in Zipkin you will see the traces appearing as much as several days too early.</p>
<p>You can also confirm this problem by comparing the date inside a docker container to outside:</p>
<pre><code class="language-command" data-lang="command">$ docker run --entrypoint date gcr.io/istio-testing/ubuntu-16-04-slave:latest
Sun Jun 11 11:44:18 UTC 2017
</code></pre><pre><code class="language-command" data-lang="command">$ date -u
Thu Jun 15 02:25:42 UTC 2017
</code></pre><p>To fix the problem, you'll need to shutdown and then restart Docker before reinstalling Istio.</p>
<h2 id="envoy-wont-connect-to-my-http10-service">Envoy won't connect to my HTTP/1.0 service</h2>
<p>Envoy requires HTTP/1.1 or HTTP/2 traffic for upstream services. For example, when using <a href="https://www.nginx.com/">NGINX</a> for serving traffic behind Envoy, you
will need to set the <a href="https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_http_version">proxy_http_version</a> directive in your NGINX config to be &ldquo;1.1&rdquo;, since the NGINX default is 1.0</p>
<p>Example config:</p>
<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-plain" data-lang="plain">upstream http_backend {
server 127.0.0.1:8080;
keepalive 16;
}
server {
...
location /http/ {
proxy_pass http://http_backend;
proxy_http_version 1.1;
proxy_set_header Connection &#34;&#34;;
...
}
}
</code></pre></div><h2 id="no-grafana-output-when-connecting-from-a-local-web-client-to-istio-remotely-hosted">No Grafana output when connecting from a local web client to Istio remotely hosted</h2>
<p>Validate the client and server date and time match.</p>
<p>The time of the web client (e.g. Chrome) affects the output from Grafana. A simple solution
to this problem is to verify a time synchronization service is running correctly within the
Kubernetes cluster and the web client machine also is correctly using a time synchronization
service. Some common time synchronization systems are NTP and Chrony. This is especially
problematic is engineering labs with firewalls. In these scenarios, NTP may not be configured
properly to point at the lab-based NTP services.</p>
<h2 id="where-are-the-metrics-for-my-service">Where are the metrics for my service?</h2>
<p>The expected flow of metrics is:</p>
<ol>
<li>Envoy reports attributes to Mixer in batch (asynchronously from requests)</li>
<li>Mixer translates the attributes from Mixer into instances based on
operator-provided configuration.</li>
<li>The instances are handed to Mixer adapters for processing and backend storage.</li>
<li>The backend storage systems record metrics data.</li>
</ol>
<p>The default installations of Mixer ship with a <a href="https://prometheus.io/">Prometheus</a>
adapter, as well as configuration for generating a basic set of metric
values and sending them to the Prometheus adapter. The
<a href="/v0.8/docs/tasks/telemetry/querying-metrics/#about-the-prometheus-add-on">Prometheus add-on</a>
also supplies configuration for an instance of Prometheus to scrape
Mixer for metrics.</p>
<p>If you do not see the expected metrics in the Istio Dashboard and/or via
Prometheus queries, there may be an issue at any of the steps in the flow
listed above. Below is a set of instructions to troubleshoot each of
those steps.</p>
<h3 id="verify-mixer-is-receiving-report-calls">Verify Mixer is receiving Report calls</h3>
<p>Mixer generates metrics for monitoring the behavior of Mixer itself.
Check these metrics.</p>
<ol>
<li>
<p>Establish a connection to the Mixer self-monitoring endpoint.</p>
<p>In Kubernetes environments, execute the following command:</p>
<pre><code class="language-command" data-lang="command">$ kubectl -n istio-system port-forward &lt;mixer pod&gt; 9093 &amp;
</code></pre></li>
<li>
<p>Verify successful report calls.</p>
<p>On the <a href="http://localhost:9093/metrics">Mixer self-monitoring endpoint</a>,
search for <code>grpc_server_handled_total</code>.</p>
<p>You should see something like:</p>
<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-plain" data-lang="plain">grpc_server_handled_total{grpc_code=&#34;OK&#34;,grpc_method=&#34;Report&#34;,grpc_service=&#34;istio.mixer.v1.Mixer&#34;,grpc_type=&#34;unary&#34;} 68
</code></pre></div></li>
</ol>
<p>If you do not see any data for <code>grpc_server_handled_total</code> with a
<code>grpc_method=&quot;Report&quot;</code>, then Mixer is not being called by Envoy to report
telemetry. In this case, ensure that the services have been properly
integrated into the mesh (either by via
<a href="/v0.8/docs/setup/kubernetes/sidecar-injection/#automatic-sidecar-injection">automatic</a>
or <a href="/v0.8/docs/setup/kubernetes/sidecar-injection/#manual-sidecar-injection">manual</a> sidecar injection).</p>
<h3 id="verify-mixer-metrics-configuration-exists">Verify Mixer metrics configuration exists</h3>
<ol>
<li>
<p>Verify Mixer rules exist.</p>
<p>In Kubernetes environments, issue the following command:</p>
<pre><code class="language-command" data-lang="command">$ kubectl get rules --all-namespaces
NAMESPACE NAME KIND
istio-system promhttp rule.v1alpha2.config.istio.io
istio-system promtcp rule.v1alpha2.config.istio.io
istio-system stdio rule.v1alpha2.config.istio.io
</code></pre><p>If you do not see anything named <code>promhttp</code> or <code>promtcp</code>, then there is
no Mixer configuration for sending metric instances to a Prometheus adapter.
You will need to supply configuration for rules that connect Mixer metric
instances to a Prometheus handler.</p>
</li>
</ol>
<!-- raw HTML omitted -->
<ol>
<li>
<p>Verify Prometheus handler config exists.</p>
<p>In Kubernetes environments, issue the following command:</p>
<pre><code class="language-command" data-lang="command">$ kubectl get prometheuses.config.istio.io --all-namespaces
NAMESPACE NAME KIND
istio-system handler prometheus.v1alpha2.config.istio.io
</code></pre><p>If there are no prometheus handlers configured, you will need to reconfigure
Mixer with the appropriate handler configuration.</p>
</li>
</ol>
<!-- raw HTML omitted -->
<ol>
<li>
<p>Verify Mixer metric instances config exists.</p>
<p>In Kubernetes environments, issue the following command:</p>
<pre><code class="language-command" data-lang="command">$ kubectl get metrics.config.istio.io --all-namespaces
NAMESPACE NAME KIND
istio-system requestcount metric.v1alpha2.config.istio.io
istio-system requestduration metric.v1alpha2.config.istio.io
istio-system requestsize metric.v1alpha2.config.istio.io
istio-system responsesize metric.v1alpha2.config.istio.io
istio-system stackdriverrequestcount metric.v1alpha2.config.istio.io
istio-system stackdriverrequestduration metric.v1alpha2.config.istio.io
istio-system stackdriverrequestsize metric.v1alpha2.config.istio.io
istio-system stackdriverresponsesize metric.v1alpha2.config.istio.io
istio-system tcpbytereceived metric.v1alpha2.config.istio.io
istio-system tcpbytesent metric.v1alpha2.config.istio.io
</code></pre><p>If there are no metric instances configured, you will need to reconfigure
Mixer with the appropriate instance configuration.</p>
</li>
</ol>
<!-- raw HTML omitted -->
<ol>
<li>
<p>Verify Mixer configuration resolution is working for your service.</p>
<ol>
<li>
<p>Establish a connection to the Mixer self-monitoring endpoint.</p>
<p>Setup a <code>port-forward</code> to the Mixer self-monitoring port as described in
<a href="#verify-mixer-is-receiving-report-calls">Verify Mixer is receiving Report calls</a>.</p>
</li>
<li>
<p>On the <a href="http://localhost:9093/metrics">Mixer self-monitoring port</a>, search
for <code>mixer_config_resolve_count</code>.</p>
<p>You should find something like:</p>
<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-plain" data-lang="plain">mixer_config_resolve_count{error=&#34;false&#34;,target=&#34;details.default.svc.cluster.local&#34;} 56
mixer_config_resolve_count{error=&#34;false&#34;,target=&#34;ingress.istio-system.svc.cluster.local&#34;} 67
mixer_config_resolve_count{error=&#34;false&#34;,target=&#34;mongodb.default.svc.cluster.local&#34;} 18
mixer_config_resolve_count{error=&#34;false&#34;,target=&#34;productpage.default.svc.cluster.local&#34;} 59
mixer_config_resolve_count{error=&#34;false&#34;,target=&#34;ratings.default.svc.cluster.local&#34;} 26
mixer_config_resolve_count{error=&#34;false&#34;,target=&#34;reviews.default.svc.cluster.local&#34;} 54
</code></pre></div></li>
<li>
<p>Validate that there are values for <code>mixer_config_resolve_count</code> where
<code>target=&quot;&lt;your service&gt;&quot;</code> and <code>error=&quot;false&quot;</code>.</p>
<p>If there are only instances where <code>error=&quot;true&quot;</code> where <code>target=&lt;your service&gt;</code>,
there is likely an issue with Mixer configuration for your service. Logs
information is needed to further debug.</p>
<p>In Kubernetes environments, retrieve the Mixer logs via:</p>
<pre><code class="language-command" data-lang="command">$ kubectl -n istio-system logs &lt;mixer pod&gt; mixer
</code></pre><p>Look for errors related to your configuration or your service in the
returned logs.</p>
</li>
</ol>
</li>
</ol>
<p>More on viewing Mixer configuration can be found <a href="/v0.8/help/faq/mixer/#mixer-self-monitoring">here</a></p>
<h3 id="verify-mixer-is-sending-metric-instances-to-the-prometheus-adapter">Verify Mixer is sending metric instances to the Prometheus adapter</h3>
<ol>
<li>
<p>Establish a connection to the Mixer self-monitoring endpoint.</p>
<p>Setup a <code>port-forward</code> to the Mixer self-monitoring port as described in
<a href="#verify-mixer-is-receiving-report-calls">Verify Mixer is receiving Report calls</a>.</p>
</li>
<li>
<p>On the <a href="http://localhost:9093/metrics">Mixer self-monitoring port</a>, search
for <code>mixer_adapter_dispatch_count</code>.</p>
<p>You should find something like:</p>
<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-plain" data-lang="plain">mixer_adapter_dispatch_count{adapter=&#34;prometheus&#34;,error=&#34;false&#34;,handler=&#34;handler.prometheus.istio-system&#34;,meshFunction=&#34;metric&#34;,response_code=&#34;OK&#34;} 114
mixer_adapter_dispatch_count{adapter=&#34;prometheus&#34;,error=&#34;true&#34;,handler=&#34;handler.prometheus.default&#34;,meshFunction=&#34;metric&#34;,response_code=&#34;INTERNAL&#34;} 4
mixer_adapter_dispatch_count{adapter=&#34;stdio&#34;,error=&#34;false&#34;,handler=&#34;handler.stdio.istio-system&#34;,meshFunction=&#34;logentry&#34;,response_code=&#34;OK&#34;} 104
</code></pre></div></li>
<li>
<p>Validate that there are values for <code>mixer_adapter_dispatch_count</code> where
<code>adapter=&quot;prometheus&quot;</code> and <code>error=&quot;false&quot;</code>.</p>
<p>If there are are no recorded dispatches to the Prometheus adapter, there
is likely a configuration issue. Please see
<a href="#verify-mixer-metrics-configuration-exists">Verify Mixer metrics configuration exists</a>.</p>
<p>If dispatches to the Prometheus adapter are reporting errors, check the
Mixer logs to determine the source of the error. Most likely, there is a
configuration issue for the handler listed in <code>mixer_adapter_dispatch_count</code>.</p>
<p>In Kubernetes environment, check the Mixer logs via:</p>
<pre><code class="language-command" data-lang="command">$ kubectl -n istio-system logs &lt;mixer pod&gt; mixer
</code></pre><p>Filter for lines including something like <code>Report 0 returned with: INTERNAL (1 error occurred:</code> (with some surrounding context) to find more information
regarding Report dispatch failures.</p>
</li>
</ol>
<h3 id="verify-prometheus-configuration">Verify Prometheus configuration</h3>
<ol>
<li>
<p>Connect to the Prometheus UI and verify that it can successfully
scrape Mixer.</p>
<p>In Kubernetes environments, setup port-forwarding as follows:</p>
<pre><code class="language-command" data-lang="command">$ kubectl -n istio-system port-forward $(kubectl -n istio-system get pod -l app=prometheus -o jsonpath='{.items[0].metadata.name}') 9090:9090 &amp;
</code></pre></li>
<li>
<p>Visit <a href="http://localhost:9090/targets">http://localhost:9090/targets</a> and confirm that the target <code>istio-mesh</code> has a status of <strong>UP</strong>.</p>
</li>
<li>
<p>Visit <a href="http://localhost:9090/config">http://localhost:9090/config</a> and confirm that an entry exists that looks like:</p>
<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-yaml" data-lang="yaml"> - job_name: <span style="color:#e6db74">&#39;istio-mesh&#39;</span>
<span style="color:#75715e"># Override the global default and scrape targets from this job every 5 seconds.</span>
scrape_interval: 5s
<span style="color:#75715e"># metrics_path defaults to &#39;/metrics&#39;</span>
<span style="color:#75715e"># scheme defaults to &#39;http&#39;.</span>
static_configs:
- targets: [<span style="color:#e6db74">&#39;istio-mixer.istio-system:42422&#39;</span>]
</code></pre></div></li>
</ol>
<h2 id="how-can-i-debug-issues-with-the-service-mesh">How can I debug issues with the service mesh?</h2>
<h3 id="with-istioctl">With istioctl</h3>
<p>Istioctl allows you to inspect the current xDS of a given Envoy from its admin interface (locally) or from Pilot using the <code>proxy-config</code> or <code>pc</code> command.</p>
<p>For example, to retrieve the configured clusters in an Envoy via the admin interface run the following command:</p>
<pre><code class="language-command" data-lang="command">$ istioctl proxy-config endpoint &lt;pod-name&gt; clusters
</code></pre><p>To retrieve endpoints for a given pod in the application namespace from Pilot run the following command:</p>
<pre><code class="language-command" data-lang="command">$ istioctl proxy-config pilot -n application &lt;pod-name&gt; eds
</code></pre><p>The <code>proxy-config</code> command also allows you to retrieve the state of the entire mesh from Pilot using the following command:</p>
<pre><code class="language-command" data-lang="command">$ istioctl proxy-config pilot mesh ads
</code></pre><h3 id="with-gdb">With GDB</h3>
<p>To debug Istio with <code>gdb</code>, you will need to run the debug images of Envoy / Mixer / Pilot. A recent <code>gdb</code> and the golang extensions (for Mixer/Pilot or other golang components) is required.</p>
<ol>
<li>
<p><code>kubectl exec -it PODNAME -c [proxy | mixer | pilot]</code></p>
</li>
<li>
<p>Find process ID: ps ax</p>
</li>
<li>
<p>gdb -p PID binary</p>
</li>
<li>
<p>For go: info goroutines, goroutine x bt</p>
</li>
</ol>
<h3 id="with-tcpdump">With Tcpdump</h3>
<p>Tcpdump doesn't work in the sidecar pod - the container doesn't run as root. However any other container in the same pod will see all the packets, since the network namespace is shared. <code>iptables</code> will also see the pod-wide config.</p>
<p>Communication between Envoy and the app happens on 127.0.0.1, and is not encrypted.</p>
<h2 id="envoy-is-crashing-under-load">Envoy is crashing under load</h2>
<p>Check your <code>ulimit -a</code>. Many systems have a 1024 open file descriptor limit by default which will cause Envoy to assert and crash with:</p>
<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-plain" data-lang="plain">[2017-05-17 03:00:52.735][14236][critical][assert] assert failure: fd_ != -1: external/envoy/source/common/network/connection_impl.cc:58
</code></pre></div><p>Make sure to raise your ulimit. Example: <code>ulimit -n 16384</code></p>
<h2 id="headless-tcp-services-losing-connection-from-istiofied-containers">Headless TCP services losing connection from Istiofied containers</h2>
<p>If <code>istio-citadel</code> is deployed, Envoy is restarted every 15 minutes to refresh certificates.
This causes the disconnection of TCP streams or long-running connections between services.</p>
<p>You should build resilience into your application for this type of
disconnect, but if you still want to prevent the disconnects from
happening, you will need to disable mTLS and the <code>istio-citadel</code> deployment.</p>
<p>First, edit your istio config to disable mTLS</p>
<pre><code class="language-command" data-lang="command">$ kubectl edit configmap -n istio-system istio
$ kubectl delete pods -n istio-system -l istio=pilot
</code></pre><p>Next, scale down the <code>istio-citadel</code> deployment to disable Envoy restarts.</p>
<pre><code class="language-command" data-lang="command">$ kubectl scale --replicas=0 deploy/istio-citadel -n istio-system
</code></pre><p>This should stop Istio from restarting Envoy and disconnecting TCP connections.</p>
<h2 id="envoy-process-high-cpu-usage">Envoy Process High CPU Usage</h2>
<p>For larger clusters, the default configuration that comes with Istio
refreshes the Envoy configuration every 1 second. This can cause high
CPU usage, even when Envoy isn't doing anything. In order to bring the
CPU usage down for larger deployments, increase the refresh interval for
Envoy to something higher, like 30 seconds.</p>
<pre><code class="language-command" data-lang="command">$ kubectl edit configmap -n istio-system istio
$ kubectl delete pods -n istio-system -l istio=pilot
</code></pre><p>Also make sure to reinject the sidecar into all of your pods, as
their configuration needs to be updated as well.</p>
<p>Afterwards, you should see CPU usage fall back to 0-1% while idling.
Make sure to tune these values for your specific deployment.</p>
<p><em>Warning:</em>: Changes created by routing rules will take up to 2x refresh interval to propagate to the sidecars.
While the larger refresh interval will reduce CPU usage, updates caused by routing rules may cause a period
of HTTP 404s (up to 2x the refresh interval) until the Envoy sidecars get all relevant configuration.</p>
<h2 id="kubernetes-webhook-setup-script-files-are-missing-from-05-release-package">Kubernetes webhook setup script files are missing from 0.5 release package</h2>
<blockquote>
<p>The 0.5.0 and 0.5.1 releases are missing scripts to provision webhook certificates. Download the missing files
from <a href="https://raw.githubusercontent.com/istio/istio/release-0.7/install/kubernetes/webhook-create-signed-cert.sh">here</a> and <a href="https://raw.githubusercontent.com/istio/istio/release-0.7/install/kubernetes/webhook-patch-ca-bundle.sh">here</a>. Subsequent releases (&gt; 0.5.1) include these files.</p>
</blockquote>
<h2 id="automatic-sidecar-injection-will-fail-if-the-kube-apiserver-has-proxy-settings">Automatic sidecar injection will fail if the kube-apiserver has proxy settings</h2>
<p>This was tested on 0.5.0 with the additional files required as referenced in the above issue. When the Kube-apiserver included
proxy settings such as:</p>
<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-yaml" data-lang="yaml">env:
- name: http_proxy
value: http://proxy-wsa.esl.foo.com:<span style="color:#ae81ff">80</span>
- name: https_proxy
value: http://proxy-wsa.esl.foo.com:<span style="color:#ae81ff">80</span>
- name: no_proxy
value: <span style="color:#ae81ff">127.0</span><span style="color:#ae81ff">.0</span><span style="color:#ae81ff">.1</span>,localhost,dockerhub.foo.com,devhub-docker.foo.com,<span style="color:#ae81ff">10.84</span><span style="color:#ae81ff">.100</span><span style="color:#ae81ff">.125</span>,<span style="color:#ae81ff">10.84</span><span style="color:#ae81ff">.100</span><span style="color:#ae81ff">.126</span>,<span style="color:#ae81ff">10.84</span><span style="color:#ae81ff">.100</span><span style="color:#ae81ff">.127</span>
</code></pre></div><p>The sidecar injection would fail. The only related failure logs was in the kube-apiserver log:</p>
<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-plain" data-lang="plain">W0227 21:51:03.156818 1 admission.go:257] Failed calling webhook, failing open sidecar-injector.istio.io: failed calling admission webhook &#34;sidecar-injector.istio.io&#34;: Post https://istio-sidecar-injector.istio-system.svc:443/inject: Service Unavailable
</code></pre></div><p>Make sure both pod and service CIDRs are not proxied according to *_proxy variables. Check the kube-apiserver files and logs to verify the configuration and whether any requests are being proxied.</p>
<p>A workaround is to remove the proxy settings from the kube-apiserver manifest and restart the server or use a later version of kubernetes.</p>
<p>An issue was filed in kubernetes related to this and has since been closed. <a href="https://github.com/kubernetes/kubeadm/issues/666">https://github.com/kubernetes/kubeadm/issues/666</a>
<a href="https://github.com/kubernetes/kubernetes/pull/58698#discussion_r163879443">https://github.com/kubernetes/kubernetes/pull/58698#discussion_r163879443</a></p>
</main>
<div class="container-fluid d-print-none">
<br/><hr/><br/>
<div class="row">
<div class="col-6">
<a title="What to do about bugs" href="/v0.8/help/bugs/"><i class="fa fa-arrow-left"></i> Reporting Bugs</a>
</div>
<div class="col-6" style="text-align: right">
</div>
</div>
</div>
<div class="d-none d-print-block" aria-hidden="true">
<h2>Links</h2>
<ol id="endnotes"></ol>
</div>
</div>
</div>
</div>
<footer class="d-print-none container-fluid">
<div class="row">
<div class="col-6 col-lg-4" role="navigation">
<div class="container-fluid">
<div class="row">
<div class="icon">
<span>istio-users@</span>
<a title="Join the istio-users@ mailing list to participate in discussions and get help troubleshooting problems"
href="https://groups.google.com/forum/#!forum/istio-users" aria-label="istio-users mailing list">
<svg viewBox="0 0 490 490">
<path d="M480,410.248H10c-5.523,0-10-4.477-10-10V89.752c0-5.523,4.477-10,10-10h470c5.522,0,10,4.477,10,10v310.495
C490,405.771,485.522,410.248,480,410.248z M20,390.248h450V99.752H20V390.248z"/>
<path d="M245,286.131c-2.083,0-4.167-0.649-5.931-1.948L48.64,143.929c-4.446-3.275-5.396-9.535-2.121-13.982
c3.275-4.447,9.535-5.396,13.982-2.121L245,263.712l184.5-135.886c4.447-3.274,10.709-2.326,13.982,2.121
c3.275,4.447,2.325,10.707-2.121,13.982L250.931,284.183C249.167,285.482,247.083,286.131,245,286.131z"/>
</svg>
</a>
</div>
<div class="icon">
<span>twitter</span>
<a title="Follow us on Twitter to get the latest news"
href="https://twitter.com/IstioMesh" aria-label="Twitter">
<svg viewBox="0 0 310 310">
<path d="M302.973,57.388c-4.87,2.16-9.877,3.983-14.993,5.463c6.057-6.85,10.675-14.91,13.494-23.73
c0.632-1.977-0.023-4.141-1.648-5.434c-1.623-1.294-3.878-1.449-5.665-0.39c-10.865,6.444-22.587,11.075-34.878,13.783
c-12.381-12.098-29.197-18.983-46.581-18.983c-36.695,0-66.549,29.853-66.549,66.547c0,2.89,0.183,5.764,0.545,8.598
C101.163,99.244,58.83,76.863,29.76,41.204c-1.036-1.271-2.632-1.956-4.266-1.825c-1.635,0.128-3.104,1.05-3.93,2.467
c-5.896,10.117-9.013,21.688-9.013,33.461c0,16.035,5.725,31.249,15.838,43.137c-3.075-1.065-6.059-2.396-8.907-3.977
c-1.529-0.851-3.395-0.838-4.914,0.033c-1.52,0.871-2.473,2.473-2.513,4.224c-0.007,0.295-0.007,0.59-0.007,0.889
c0,23.935,12.882,45.484,32.577,57.229c-1.692-0.169-3.383-0.414-5.063-0.735c-1.732-0.331-3.513,0.276-4.681,1.597
c-1.17,1.32-1.557,3.16-1.018,4.84c7.29,22.76,26.059,39.501,48.749,44.605c-18.819,11.787-40.34,17.961-62.932,17.961
c-4.714,0-9.455-0.277-14.095-0.826c-2.305-0.274-4.509,1.087-5.294,3.279c-0.785,2.193,0.047,4.638,2.008,5.895
c29.023,18.609,62.582,28.445,97.047,28.445c67.754,0,110.139-31.95,133.764-58.753c29.46-33.421,46.356-77.658,46.356-121.367
c0-1.826-0.028-3.67-0.084-5.508c11.623-8.757,21.63-19.355,29.773-31.536c1.237-1.85,1.103-4.295-0.33-5.998
C307.394,57.037,305.009,56.486,302.973,57.388z"/>
</svg>
</a>
</div>
<div class="icon">
<span>stack overflow</span>
<a title="Stack Overflow is where you can ask questions and find curated answers on deploying, configuring, and using Istio"
href="https://stackoverflow.com/questions/tagged/istio" aria-label="Stack Overflow">
<svg viewBox="0 0 120 120">
<polygon points="84.4,93.8 84.4,70.6 92.1,70.6 92.1,101.5 22.6,101.5 22.6,70.6 30.3,70.6 30.3,93.8 "/>
<path d="M38.8,68.4l37.8,7.9l1.6-7.6l-37.8-7.9L38.8,68.4z M43.8,50.4l35,16.3l3.2-7l-35-16.4L43.8,50.4z M53.5,33.2
l29.7,24.7l4.9-5.9L58.4,27.3L53.5,33.2z M72.7,14.9l-6.2,4.6l23,31l6.2-4.6L72.7,14.9z M38,86h38.6v-7.7H38V86z"/>
</svg>
</a>
</div>
<div class="icon">
<span>rocket chat</span>
<a title="Interactively chat with members of the Istio community."
href="https://istio.rocket.chat" aria-label="Rocket Chat">
<svg viewBox="0 0 512 512">
<path d="M496.293,255.338c0-24.103-7.21-47.215-21.437-68.699c-12.771-19.288-30.666-36.362-53.184-50.745
c-43.474-27.771-100.612-43.065-160.885-43.065c-20.131,0-39.974,1.702-59.222,5.072c-11.942-11.176-25.919-21.233-40.712-29.187
c-79.026-38.298-144.561-0.9-144.561-0.9s60.931,50.053,51.023,93.93c-27.259,27.041-42.033,59.646-42.033,93.594
c0,0.108,0.005,0.216,0.006,0.324c-0.001,0.108-0.006,0.216-0.006,0.324c0,33.949,14.774,66.554,42.033,93.595
c9.907,43.874-51.023,93.93-51.023,93.93s65.535,37.397,144.561-0.901c14.792-7.953,28.77-18.01,40.712-29.188
c19.249,3.372,39.091,5.072,59.222,5.072c60.272,0,117.411-15.294,160.885-43.064c22.518-14.383,40.412-31.457,53.184-50.742
c14.227-21.487,21.437-44.599,21.437-68.702c0-0.107-0.006-0.216-0.006-0.324C496.287,255.554,496.293,255.446,496.293,255.338z
M260.882,387.763c-25.367,0-49.66-2.932-72.107-8.282c-22.81,27.443-72.993,65.596-121.742,53.26
c15.857-17.031,39.352-45.81,34.32-93.207c-29.218-22.738-46.759-51.832-46.759-83.541c0-72.776,92.36-131.769,206.288-131.769
c113.928,0,206.288,58.993,206.288,131.769C467.17,328.765,374.81,387.763,260.882,387.763z M288.283,255.991
c0,15.133-12.27,27.403-27.4,27.403c-15.134,0-27.402-12.271-27.402-27.403s12.268-27.401,27.402-27.401
C276.014,228.59,288.283,240.858,288.283,255.991z M356.163,228.59c-15.133,0-27.4,12.268-27.4,27.401s12.268,27.403,27.4,27.403
c15.134,0,27.399-12.271,27.399-27.403S371.297,228.59,356.163,228.59z M165.601,228.59c-15.133,0-27.4,12.268-27.4,27.401
s12.268,27.403,27.4,27.403c15.134,0,27.401-12.271,27.401-27.403S180.735,228.59,165.601,228.59z"/>
</svg>
</a>
</div>
</div>
<div class="tag row d-none d-lg-flex">
for users
</div>
</div>
</div>
<div class="col-6 col-lg-4">
<p class="text-center copyright" role="contentinfo">
Istio
Archive
0.8<br>&copy; 2018 Istio Authors, <a href="https://policies.google.com/privacy">Privacy Policy</a><br>
Archived on July 31, 2018
</p>
</div>
<div class="col-6 col-lg-4 d-none d-lg-flex" role="navigation">
<div class="container-fluid">
<div class="row justify-content-end">
<div class="icon">
<span>istio-dev@</span>
<a title="Join the istio-dev@ mailing list to discuss development issues around the Istio project"
href="https://groups.google.com/forum/#!forum/istio-dev" aria-label="istio-dev mailing list">
<svg viewBox="0 0 490 490">
<path d="M480,410.248H10c-5.523,0-10-4.477-10-10V89.752c0-5.523,4.477-10,10-10h470c5.522,0,10,4.477,10,10v310.495
C490,405.771,485.522,410.248,480,410.248z M20,390.248h450V99.752H20V390.248z"/>
<path d="M245,286.131c-2.083,0-4.167-0.649-5.931-1.948L48.64,143.929c-4.446-3.275-5.396-9.535-2.121-13.982
c3.275-4.447,9.535-5.396,13.982-2.121L245,263.712l184.5-135.886c4.447-3.274,10.709-2.326,13.982,2.121
c3.275,4.447,2.325,10.707-2.121,13.982L250.931,284.183C249.167,285.482,247.083,286.131,245,286.131z"/>
</svg>
</a>
</div>
<div class="icon">
<span>github</span>
<a title="GitHub is where development takes place on Istio code"
href="https://github.com/istio/community" aria-label="GitHub">
<svg viewBox="0 0 478.165 478.165">
<path d="M349.22,55.768c6.136,14.046,10.241,37.556,4.224,54.69
c24.426,20.999,33.073,71.904,21.079,113.704c35.006,2.73,76.666-1.235,103.642,9.484c-25.183-3.248-59.651-9.563-91.987-7.431
c-6.136,0.458-15.361-0.239-14.903,8.408c37.735,3.008,75.092,6.117,105.894,15.779c-30.702-4.981-67.74-12.552-105.894-13.668
c-15.54,30.921-47.239,46.262-90.991,49.49c4.682,10.261,13.847,14.066,15.879,30.702c3.267,24.406-4.881,60.328,3.208,76.686
c4.064,7.89,10.579,8.009,14.863,14.604c-10.699,12.871-37.257-1.395-40.186-14.604c-5.14-22.852,7.89-58.256-6.415-73.737
c0.996,24.865-5.718,59.85,0.996,82.145c2.789,8.806,10.659,12.113,8.647,20.063c-49.809,5.08-28.989-64.373-37.177-105.356
c-7.471,0.697-4.204,11.197-4.224,15.76c-0.199,40.106,8.189,94.836-34.846,89.556c-1.315-8.348,5.838-11.217,8.467-19.007
c7.91-22.434-1.454-56.045,2.112-83.161c-16.417,12.512,1.793,55.666-8.428,77.961c-5.838,12.671-24.785,18.27-39.19,12.651
c1.873-9.464,11.695-7.989,15.879-16.875c5.818-12.452,0.02-30.244,2.092-48.494c-30.423,6.097-53.993-0.877-65.608-20.023
c-5.12-8.507-6.356-18.708-12.632-26.219c-6.117-7.551-16.098-8.507-19.087-18.808c37.755-9.185,39.17,38.771,73.06,39.807
c10.44,0.418,15.799-2.909,25.402-5.16c2.749-12.113,8.428-21.039,16.875-27.494c-42.078-5.658-76.865-18.788-93.023-50.466
c-38.293,1.893-73.339,7.013-105.894,14.843c29.547-10.679,65.807-14.604,104.778-15.819c-2.351-13.807-22.434-10.022-34.866-9.543
C47.677,227.17,18.449,230.138,0,233.645c26.817-9.543,64.233-8.348,100.454-8.428c-11.038-34.767-7.232-90.014,17.015-110.615
c-6.854-17.254-4.722-45.346,4.184-58.834c27.036,1.175,43.374,12.891,60.388,24.247c21.019-6.017,43.035-9.045,71.904-7.451
c12.133,0.677,24.705,6.097,33.731,5.32c8.906-0.877,18.728-10.898,27.534-14.843C326.507,58.099,336.17,56.206,349.22,55.768z"/>
</svg>
</a>
</div>
<div class="icon">
<span>drive</span>
<a title="Access our team drive if you'd like to take a look at the Istio technical design documents"
href="https://groups.google.com/forum/#!forum/istio-team-drive-access" aria-label="team drive">
<svg viewBox="0 0 207.027 207.027">
<path d="M69.866,15.557L0,138.919l28.732,52.552l143.288-0.029l35.008-59.588L136.39,15.735L69.866,15.557z M17.166,139.046
L74.268,38.205L91.21,67.783L33.24,168.447L17.166,139.046z M99.841,82.851l23.805,41.558l-47.732-0.006L99.841,82.851z
M163.434,176.443l-117.332,0.024l21.53-37.065l64.606,0.008l0.067,0.119l52.865-0.085L163.434,176.443z M140.932,124.411
L90.157,35.767l-2.966-5.178l40.751,0.121l57.003,93.706L140.932,124.411z"/>
</svg>
</a>
</div>
<div class="icon">
<span>working groups</span>
<a title="If you'd like to contribute to the Istio project, consider participating in our working groups"
href="https://github.com/istio/community/blob/master/WORKING-GROUPS.md" aria-label="working groups">
<svg viewBox="0 -45 439.833 439.833">
<polygon points="246.048,195.833 299.966,235.085 319.497,227.296 276.278,195.833"/>
<polygon points="193.786,195.833 163.556,195.833 120.33,227.3 139.862,235.089"/>
<path d="M219.927,11.558c-23.854,0-37.057,12.362-36.814,36.182c0.348,32.623,14.211,52.414,36.814,52.068
c0,0,36.802,1.492,36.802-52.068C256.729,23.918,244.294,11.558,219.927,11.558z"/>
<path d="M285.017,124.567l-36.77-14.659l-8.608-7.256c-2.274-1.922-5.636-1.78-7.741,0.317l-11.973,11.904l-12.008-11.907
c-2.109-2.094-5.465-2.229-7.736-0.313l-8.611,7.256l-36.77,14.661c-11.842,4.715-11.83,46.647-12.848,50.497h155.93
C296.866,171.228,296.862,129.28,285.017,124.567z"/>
<path d="M77.976,228.568c0,0,36.801,1.492,36.801-52.068c0-23.82-12.434-36.182-36.801-36.182
c-23.854,0-37.057,12.362-36.814,36.182C41.509,209.124,55.372,228.915,77.976,228.568z"/>
<path d="M143.065,253.329l-36.77-14.658l-8.609-7.256c-2.275-1.923-5.635-1.781-7.742,0.315l-11.971,11.904l-12.008-11.908
c-2.109-2.094-5.465-2.229-7.736-0.312l-8.611,7.256l-36.77,14.66C1.006,258.045,1.018,299.977,0,303.827h155.93
C154.915,299.988,154.911,258.042,143.065,253.329z"/>
<path d="M361.878,228.568c0,0,36.801,1.492,36.801-52.068c0-23.82-12.434-36.182-36.801-36.182
c-23.854,0-37.057,12.362-36.812,36.182C325.411,209.124,339.274,228.915,361.878,228.568z"/>
<path d="M426.968,253.329l-36.77-14.658l-8.609-7.256c-2.273-1.923-5.635-1.781-7.742,0.315l-11.971,11.904l-12.008-11.908
c-2.109-2.094-5.465-2.229-7.736-0.312l-8.61,7.256l-36.771,14.66c-11.842,4.715-11.83,46.646-12.848,50.497h155.93
C438.817,299.988,438.812,258.042,426.968,253.329z"/>
</svg>
</a>
</div>
<div class="icon">
<span>slack</span>
<a title="Interactively discuss development issues with the Istio community on Slack (invitation-only)"
href="https://istio.slack.com" aria-label="slack">
<svg viewBox="0 0 31.444 31.443">
<path d="M31.202,16.369c-0.62-1.388-2.249-2.011-3.637-1.391l-1.325,0.594l-3.396-7.591l1.325-0.592
c1.388-0.622,2.01-2.25,1.389-3.637c-0.62-1.389-2.248-2.012-3.637-1.39l-1.324,0.593l-0.593-1.326
c-0.621-1.388-2.249-2.009-3.637-1.388c-1.388,0.62-2.009,2.247-1.389,3.637l0.593,1.325L7.98,8.598L7.388,7.273
c-0.621-1.39-2.249-2.009-3.637-1.39C2.363,6.504,1.742,8.132,2.362,9.52l0.592,1.324L1.63,11.438
c-1.388,0.621-2.01,2.247-1.389,3.636c0.62,1.388,2.249,2.01,3.637,1.39l1.325-0.594l3.394,7.592l-1.325,0.592
c-1.388,0.621-2.009,2.25-1.389,3.637c0.621,1.389,2.249,2.011,3.637,1.391l1.324-0.593l0.593,1.325
c0.621,1.389,2.249,2.01,3.637,1.389c1.387-0.62,2.009-2.248,1.388-3.636l-0.591-1.326l7.591-3.394l0.592,1.321
c0.621,1.391,2.248,2.013,3.637,1.392c1.388-0.619,2.01-2.248,1.389-3.637l-0.592-1.324l1.323-0.594
C31.201,19.384,31.823,17.757,31.202,16.369z M13.623,21.215l-3.395-7.593l7.591-3.394l3.395,7.591L13.623,21.215z"/>
</svg>
</a>
</div>
</div>
<div class="tag row justify-content-end text-right">
for developers
</div>
</div>
</div>
</div>
</footer>
<div class="d-xl-none d-print-none">
<button id="scroll-to-top" aria-hidden="true" onclick="scrollToTop()" title="Back to top"><i class="fa fa-lg fa-arrow-up"></i></button>
</div>
<script src="https://code.jquery.com/jquery-3.2.1.slim.min.js" integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN" crossorigin="anonymous"></script>
<script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/1.7.1/clipboard.min.js"></script>
<script src="https://www.google.com/cse/brand?form=search_form"></script>
<script src="/v0.8/js/all.min.js" data-manual></script>
</body>
</html>