mirror of https://github.com/istio/istio.io.git
89 lines
36 KiB
HTML
89 lines
36 KiB
HTML
<!DOCTYPE html><html lang="en" itemscope itemtype="https://schema.org/WebPage"><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><meta name="theme-color" content="#466BB0"/><meta name="title" content="Troubleshooting Guide"><meta name="description" content="Practical advice on practical problems with Istio"><meta name="og:title" content="Troubleshooting Guide"><meta name="og:description" content="Practical advice on practical problems with Istio"><meta name="og:url" content="/help/troubleshooting.html"><meta name="og.site_name" content="Istio"><title>Istioldie 0.6 / Troubleshooting Guide</title><script> window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date; ga('create', 'UA-98480406-2', 'auto'); ga('send', 'pageview'); </script> <script async src='https://www.google-analytics.com/analytics.js'></script><link rel="alternate" type="application/rss+xml" title="Istio Blog RSS" href="/v0.6/feed.xml"><link rel="shortcut icon" href="/v0.6/favicons/favicon.ico" ><link rel="apple-touch-icon" href="/v0.6/favicons/apple-touch-icon-180x180.png" sizes="180x180"><link rel="icon" type="image/png" href="/v0.6/favicons/favicon-16x16.png" sizes="16x16"><link rel="icon" type="image/png" href="/v0.6/favicons/favicon-32x32.png" sizes="32x32"><link rel="icon" type="image/png" href="/v0.6/favicons/android-36x36.png" sizes="36x36"><link rel="icon" type="image/png" href="/v0.6/favicons/android-48x48.png" sizes="48x48"><link rel="icon" type="image/png" href="/v0.6/favicons/android-72x72.png" sizes="72x72"><link rel="icon" type="image/png" href="/v0.6/favicons/android-96x196.png" sizes="96x196"><link rel="icon" type="image/png" href="/v0.6/favicons/android-144x144.png" sizes="144x144"><link rel="icon" type="image/png" href="/v0.6/favicons/android-192x192.png" sizes="192x192"><link rel="manifest" href="/v0.6/manifest.json"><meta name="apple-mobile-web-app-title" content="Istio"><meta name="application-name" content="Istio"><link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:400,100,100italic,300,300italic,400italic,500,500italic,700,700italic,900,900italic"><link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous"><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.6/css/all.css"><link rel="stylesheet" href="/v0.6/css/light_theme.css" title="light"><link rel="alternate stylesheet" href="/v0.6/css/dark_theme.css" title="dark"> <script src="/v0.6/js/styleSwitcher.min.js"></script></head><body class="language-unknown theme-unknown"><header role="banner"><nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark"> <a class="navbar-brand" href="/v0.6/" style="visibility: visible"> <img class="logo" src="/v0.6/img/istio-logo.svg" alt="Istio Logo"/> <span class="brand-name">Istioldie 0.6</span> </a> <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarCollapse" aria-controls="navbarCollapse" aria-expanded="false" aria-label="Toggle navigation"> <span class="navbar-toggler-icon"></span> </button><div class="collapse navbar-collapse justify-content-end" id="navbarCollapse"><ul class="navbar-nav"><li class="nav-item"> <a class="nav-link " href="/v0.6/about/intro.html">About</a></li><li class="nav-item"> <a class="nav-link " href="/v0.6/blog/2018/traffic-mirroring.html">Blog</a></li><li class="nav-item"> <a class="nav-link " href="/v0.6/docs/">Docs</a></li><li class="nav-item"> <a class="nav-link active" href="/v0.6/help/">Help</a></li><li class="nav-item"> <a class="nav-link " href="/v0.6/community.html">Community</a></li><li class="nav-item dropdown" id="gearDropdown" style="white-space: nowrap"> <a href="" class="nav-link dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false"> <i style="width: 1em" class='fa fa-lg fa-cog'></i> </a><ul class="dropdown-menu" aria-labelledby="gearDropdown"><h6 class="dropdown-header">Other versions of this site</h6><li> <a href="https://istio.io">Current Release</a></li><li> <a href="https://preliminary.istio.io">Next Release</a></li><li> <a href="https://archive.istio.io">Older Releases</a></li><li class="dropdown-divider"></li><li> <i class='fa fa-check light'></i> <a href="" onclick="setActiveStyleSheet('light');return false;">Light Theme</a></li><li> <i class='fa fa-check dark'></i> <a href="" onclick="setActiveStyleSheet('dark');return false;">Dark Theme</a></li></ul></li></ul><form name="cse" id="searchbox" class="form-inline justify-content-end" role="search"> <input type="hidden" name="cx" value="013699703217164175118:iwwf17ikgf4" /> <input type="hidden" name="ie" value="utf-8" /> <input type="hidden" name="hl" value="en" /><div class="input-group"> <input name="q" class="form-control search-box" type="text" size="30" /> <button class="btn btn-search input-group-addon my-2 my-sm-0 fa fa-search" type="submit"></button></div></form></div></nav></header><div class="container-fluid"><div class="row row-offcanvas row-offcanvas-left"><div class="col-6 col-md-3 col-xl-2 sidebar-offcanvas"><nav class="sidebar"><div class="spacer"></div><div class="directory" role="tablist"><div class="card"><div class="card-header" role="tab" id="header0"><div title="A bunch of resources to help you deploy, configure and use Istio."> Help!</div></div><div id="collapse0" class="collapse show" data-parent="#sidebar" role="tabpanel" aria-labelledby="header0"><div class="card-body"><ul class="tree"><li class="sublist"> <label class='tree-toggle'> <i class='fa fa-lg fa-caret-right'></i> <a class="" title="Frequently Asked Questions about Istio." href="/v0.6/help/faq">FAQ</a> </label><ul class="tree collapse"><li> <a title="General Q&A" href="/v0.6/help/faq/general.html">General</a></li><li> <a title="Setup Q&A" href="/v0.6/help/faq/setup.html">Setup</a></li><li> <a title="Security Q&A" href="/v0.6/help/faq/security.html">Security</a></li><li> <a title="Mixer Q&A" href="/v0.6/help/faq/mixer.html">Mixer</a></li><li> <a title="Traffic Management Q&A" href="/v0.6/help/faq/traffic-management.html">Traffic Management</a></li></ul></li><li> <a title="A glossary of common Istio terms." href="/v0.6/help/glossary.html">Glossary</a></li><li> <a title="What to do about bugs" href="/v0.6/help/bugs.html">Reporting Bugs</a></li><li> <span class="current" title="Practical advice on practical problems with Istio">Troubleshooting Guide</span></li></ul></div></div></div></div></nav></div><div class="col-12 col-md-9 col-lg-6 col-xl-7"><p class="d-md-none"> <label class="sidebar-toggler" data-toggle="offcanvas"> <i class="fa fa-chevron-right"></i> </label></p><main role="main"><h1>Troubleshooting Guide</h1><p>Oh no! You’re having trouble? Below is a list of solutions to common problems.</p><h2 id="verifying-connectivity-to-istio-pilot">Verifying connectivity to Istio Pilot</h2><p>Verifying connectivity to Pilot is a useful troubleshooting step. Every proxy container in the service mesh should be able to communicate with Pilot. This can be accomplished in a few simple steps:</p><ol><li>Get the name of the Istio Ingress pod:<div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="nv">INGRESS_POD_NAME</span><span class="o">=</span><span class="k">$(</span>kubectl get po <span class="nt">-n</span> istio-system | <span class="nb">grep </span>ingress | awk <span class="s1">'{print$1}'</span><span class="k">)</span>
|
||
</code></pre></div></div></li><li>Exec into the Istio Ingress pod:<div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>kubectl <span class="nb">exec</span> <span class="nt">-it</span> <span class="nv">$INGRESS_POD_NAME</span> <span class="nt">-n</span> istio-system /bin/bash
|
||
</code></pre></div></div></li><li>Unless you installed Istio using the debug proxy image (<code class="highlighter-rouge">istioctl kube-inject --debug=true</code>), you need to install curl.<div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>apt-get update <span class="o">&&</span> apt-get install <span class="nt">-y</span> curl
|
||
</code></pre></div></div></li><li>Test connectivity to Pilot using cURL. The following example cURL’s the v1 registration API using default Pilot configuration parameters and mTLS enabled:<div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>curl <span class="nt">-k</span> <span class="nt">--cert</span> /etc/certs/cert-chain.pem <span class="nt">--cacert</span> /etc/certs/root-cert.pem <span class="nt">--key</span> /etc/certs/key.pem https://istio-pilot:15003/v1/registration
|
||
</code></pre></div></div></li></ol><p>If mTLS is disabled:</p><div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>curl http://istio-pilot:15003/v1/registration
|
||
</code></pre></div></div><p>You should receive a response listing the “service-key” and “hosts” for each service in the mesh.</p><h2 id="no-traces-appearing-in-zipkin-when-running-istio-locally-on-mac">No traces appearing in Zipkin when running Istio locally on Mac</h2><p>Istio is installed and everything seems to be working except there are no traces showing up in Zipkin when there should be.</p><p>This may be caused by a known <a href="https://github.com/docker/for-mac/issues/1260">Docker issue</a> where the time inside containers may skew significantly from the time on the host machine. If this is the case, when you select a very long date range in Zipkin you will see the traces appearing as much as several days too early.</p><p>You can also confirm this problem by comparing the date inside a docker container to outside:</p><div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>docker run <span class="nt">--entrypoint</span> date gcr.io/istio-testing/ubuntu-16-04-slave:latest
|
||
Sun Jun 11 11:44:18 UTC 2017
|
||
date <span class="nt">-u</span>
|
||
Thu Jun 15 02:25:42 UTC 2017
|
||
</code></pre></div></div><p>To fix the problem, you’ll need to shutdown and then restart Docker before reinstalling Istio.</p><h2 id="envoy-wont-connect-to-my-http10-service">Envoy won’t connect to my HTTP/1.0 service</h2><p>Envoy requires HTTP/1.1 or HTTP/2 traffic for upstream services. For example, when using <a href="https://www.nginx.com/">NGINX</a> for serving traffic behind Envoy, you will need to set the <a href="https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_http_version">proxy_http_version</a> directive in your NGINX config to be “1.1”, since the NGINX default is 1.0</p><p>Example config:</p><div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>upstream http_backend {
|
||
server 127.0.0.1:8080;
|
||
|
||
keepalive 16;
|
||
}
|
||
|
||
server {
|
||
...
|
||
|
||
location /http/ {
|
||
proxy_pass http://http_backend;
|
||
proxy_http_version 1.1;
|
||
proxy_set_header Connection "";
|
||
...
|
||
}
|
||
}
|
||
</code></pre></div></div><h2 id="no-grafana-output-when-connecting-from-a-local-web-client-to-istio-remotely-hosted">No grafana output when connecting from a local web client to Istio remotely hosted</h2><p>Validate the client and server date and time match.</p><p>The time of the web client (e.g. Chrome) affects the output from Grafana. A simple solution to this problem is to verify a time synchronization service is running correctly within the Kubernetes cluster and the web client machine also is correctly using a time synchronization service. Some common time synchronization systems are NTP and Chrony. This is especially problematic is engineering labs with firewalls. In these scenarios, NTP may not be configured properly to point at the lab-based NTP services.</p><h2 id="where-are-the-metrics-for-my-service">Where are the metrics for my service?</h2><p>The expected flow of metrics is:</p><ol><li>Envoy reports attributes to Mixer in batch (asynchronously from requests)</li><li>Mixer translates the attributes from Mixer into instances based on operator-provided configuration.</li><li>The instances are handed to Mixer adapters for processing and backend storage.</li><li>The backend storage systems record metrics data.</li></ol><p>The default installations of Mixer ship with a <a href="https://prometheus.io/">Prometheus</a> adapter, as well as configuration for generating a basic set of metric values and sending them to the Prometheus adapter. The <a href="/v0.6/docs/tasks/telemetry/querying-metrics.html#about-the-prometheus-add-on">Prometheus add-on</a> also supplies configuration for an instance of Prometheus to scrape Mixer for metrics.</p><p>If you do not see the expected metrics in the Istio Dashboard and/or via Prometheus queries, there may be an issue at any of the steps in the flow listed above. Below is a set of instructions to troubleshoot each of those steps.</p><h3 id="verify-mixer-is-receiving-report-calls">Verify Mixer is receiving Report calls</h3><p>Mixer generates metrics for monitoring the behavior of Mixer itself. Check these metrics.</p><ol><li><p>Establish a connection to the Mixer self-monitoring endpoint.</p><p>In Kubernetes environments, execute the following command:</p><div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>kubectl <span class="nt">-n</span> istio-system port-forward <mixer pod> 9093 &
|
||
</code></pre></div></div></li><li><p>Verify successful report calls.</p><p>On the <a href="http://localhost:9093/metrics">Mixer self-monitoring endpoint</a>, search for <code class="highlighter-rouge">grpc_server_handled_total</code>.</p><p>You should see something like:</p><div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>grpc_server_handled_total{grpc_code="OK",grpc_method="Report",grpc_service="istio.mixer.v1.Mixer",grpc_type="unary"} 68
|
||
</code></pre></div></div></li></ol><p>If you do not see any data for <code class="highlighter-rouge">grpc_server_handled_total</code> with a <code class="highlighter-rouge">grpc_method="Report"</code>, then Mixer is not being called by Envoy to report telemetry. In this case, ensure that the services have been properly integrated into the mesh (either by via <a href="/v0.6/docs/setup/kubernetes/sidecar-injection.html#automatic-sidecar-injection">automatic</a> or <a href="/v0.6/docs/setup/kubernetes/sidecar-injection.html#manual-sidecar-injection">manual</a> sidecar injection).</p><h3 id="verify-mixer-metrics-configuration-exists">Verify Mixer metrics configuration exists</h3><ol><li><p>Verify Mixer rules exist.</p><p>In Kubernetes environments, issue the following command:</p><div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>kubectl get rules <span class="nt">--all-namespaces</span>
|
||
</code></pre></div></div><p>With the default configuration, you should see something like:</p><div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>NAMESPACE NAME KIND
|
||
istio-system promhttp rule.v1alpha2.config.istio.io
|
||
istio-system promtcp rule.v1alpha2.config.istio.io
|
||
istio-system stdio rule.v1alpha2.config.istio.io
|
||
</code></pre></div></div><p>If you do not see anything named <code class="highlighter-rouge">promhttp</code> or <code class="highlighter-rouge">promtcp</code>, then there is no Mixer configuration for sending metric instances to a Prometheus adapter. You will need to supply configuration for rules that connect Mixer metric instances to a Prometheus handler.</p></li><li><p>Verify Prometheus handler config exists.</p><p>In Kubernetes environments, issue the following command:</p><div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>kubectl get prometheuses.config.istio.io <span class="nt">--all-namespaces</span>
|
||
</code></pre></div></div><p>The expected output is:</p><div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>NAMESPACE NAME KIND
|
||
istio-system handler prometheus.v1alpha2.config.istio.io
|
||
</code></pre></div></div><p>If there are no prometheus handlers configured, you will need to reconfigure Mixer with the appropriate handler configuration.</p></li><li><p>Verify Mixer metric instances config exists.</p><p>In Kubernetes environments, issue the following command:</p><div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>kubectl get metrics.config.istio.io <span class="nt">--all-namespaces</span>
|
||
</code></pre></div></div><p>The expected output is:</p><div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>NAMESPACE NAME KIND
|
||
istio-system requestcount metric.v1alpha2.config.istio.io
|
||
istio-system requestduration metric.v1alpha2.config.istio.io
|
||
istio-system requestsize metric.v1alpha2.config.istio.io
|
||
istio-system responsesize metric.v1alpha2.config.istio.io
|
||
istio-system stackdriverrequestcount metric.v1alpha2.config.istio.io
|
||
istio-system stackdriverrequestduration metric.v1alpha2.config.istio.io
|
||
istio-system stackdriverrequestsize metric.v1alpha2.config.istio.io
|
||
istio-system stackdriverresponsesize metric.v1alpha2.config.istio.io
|
||
istio-system tcpbytereceived metric.v1alpha2.config.istio.io
|
||
istio-system tcpbytesent metric.v1alpha2.config.istio.io
|
||
</code></pre></div></div><p>If there are no metric instances configured, you will need to reconfigure Mixer with the appropriate instance configuration.</p></li><li><p>Verify Mixer configuration resolution is working for your service.</p><ol><li><p>Establish a connection to the Mixer self-monitoring endpoint.</p><p>Setup a <code class="highlighter-rouge">port-forward</code> to the Mixer self-monitoring port as described in <a href="#verify-mixer-is-receiving-report-calls">Verify Mixer is receiving Report calls</a>.</p></li><li><p>On the <a href="http://localhost:9093/metrics">Mixer self-monitoring port</a>, search for <code class="highlighter-rouge">mixer_config_resolve_count</code>.</p><p>You should find something like:</p><div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>mixer_config_resolve_count{error="false",target="details.default.svc.cluster.local"} 56
|
||
mixer_config_resolve_count{error="false",target="ingress.istio-system.svc.cluster.local"} 67
|
||
mixer_config_resolve_count{error="false",target="mongodb.default.svc.cluster.local"} 18
|
||
mixer_config_resolve_count{error="false",target="productpage.default.svc.cluster.local"} 59
|
||
mixer_config_resolve_count{error="false",target="ratings.default.svc.cluster.local"} 26
|
||
mixer_config_resolve_count{error="false",target="reviews.default.svc.cluster.local"} 54
|
||
</code></pre></div></div></li><li><p>Validate that there are values for <code class="highlighter-rouge">mixer_config_resolve_count</code> where <code class="highlighter-rouge">target="<your service>"</code> and <code class="highlighter-rouge">error="false"</code>.</p><p>If there are only instances where <code class="highlighter-rouge">error="true"</code> where <code class="highlighter-rouge">target=<your service></code>, there is likely an issue with Mixer configuration for your service. Logs information is needed to further debug.</p><p>In Kubernetes environments, retrieve the Mixer logs via:</p><div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>kubectl <span class="nt">-n</span> istio-system logs <mixer pod> mixer
|
||
</code></pre></div></div><p>Look for errors related to your configuration or your service in the returned logs.</p></li></ol></li></ol><p>More on viewing Mixer configuration can be found <a href="/v0.6/help/faq/mixer.html#mixer-self-monitoring">here</a></p><h3 id="verify-mixer-is-sending-metric-instances-to-the-prometheus-adapter">Verify Mixer is sending metric instances to the Prometheus adapter</h3><ol><li><p>Establish a connection to the Mixer self-monitoring endpoint.</p><p>Setup a <code class="highlighter-rouge">port-forward</code> to the Mixer self-monitoring port as described in <a href="#verify-mixer-is-receiving-report-calls">Verify Mixer is receiving Report calls</a>.</p></li><li><p>On the <a href="http://localhost:9093/metrics">Mixer self-monitoring port</a>, search for <code class="highlighter-rouge">mixer_adapter_dispatch_count</code>.</p><p>You should find something like:</p><div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>mixer_adapter_dispatch_count{adapter="prometheus",error="false",handler="handler.prometheus.istio-system",meshFunction="metric",response_code="OK"} 114
|
||
mixer_adapter_dispatch_count{adapter="prometheus",error="true",handler="handler.prometheus.default",meshFunction="metric",response_code="INTERNAL"} 4
|
||
mixer_adapter_dispatch_count{adapter="stdio",error="false",handler="handler.stdio.istio-system",meshFunction="logentry",response_code="OK"} 104
|
||
</code></pre></div></div></li><li><p>Validate that there are values for <code class="highlighter-rouge">mixer_adapter_dispatch_count</code> where <code class="highlighter-rouge">adapter="prometheus"</code> and <code class="highlighter-rouge">error="false"</code>.</p><p>If there are are no recorded dispatches to the Prometheus adapter, there is likely a configuration issue. Please see <a href="#verify-mixer-metrics-configuration-exists">Verify Mixer metrics configuration exists</a>.</p><p>If dispatches to the Prometheus adapter are reporting errors, check the Mixer logs to determine the source of the error. Most likely, there is a configuration issue for the handler listed in <code class="highlighter-rouge">mixer_adapter_dispatch_count</code>.</p><p>In Kubernetes environment, check the Mixer logs via:</p><div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>kubectl <span class="nt">-n</span> istio-system logs <mixer pod> mixer
|
||
</code></pre></div></div><p>Filter for lines including something like <code class="highlighter-rouge">Report 0 returned with: INTERNAL (1 error occurred:</code> (with some surrounding context) to find more information regarding Report dispatch failures.</p></li></ol><h3 id="verify-prometheus-configuration">Verify Prometheus configuration</h3><ol><li><p>Connect to the Prometheus UI and verify that it can successfully scrape Mixer.</p><p>In Kubernetes environments, setup port-forwarding as follows:</p><div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>kubectl <span class="nt">-n</span> istio-system port-forward <span class="k">$(</span>kubectl <span class="nt">-n</span> istio-system get pod <span class="nt">-l</span> <span class="nv">app</span><span class="o">=</span>prometheus <span class="nt">-o</span> <span class="nv">jsonpath</span><span class="o">=</span><span class="s1">'{.items[0].metadata.name}'</span><span class="k">)</span> 9090:9090 &
|
||
</code></pre></div></div></li><li><p>Visit <a href="http://localhost:9090/config">http://localhost:9090/config</a>.</p><p>Confirm that an entry exists that looks like:</p><div class="language-yaml highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="pi">-</span> <span class="na">job_name</span><span class="pi">:</span> <span class="s1">'</span><span class="s">istio-mesh'</span>
|
||
<span class="c1"># Override the global default and scrape targets from this job every 5 seconds.</span>
|
||
<span class="na">scrape_interval</span><span class="pi">:</span> <span class="s">5s</span>
|
||
<span class="c1"># metrics_path defaults to '/metrics'</span>
|
||
<span class="c1"># scheme defaults to 'http'.</span>
|
||
<span class="na">static_configs</span><span class="pi">:</span>
|
||
<span class="pi">-</span> <span class="na">targets</span><span class="pi">:</span> <span class="pi">[</span><span class="s1">'</span><span class="s">istio-mixer.istio-system:42422'</span><span class="pi">]</span>
|
||
</code></pre></div></div></li><li><p>Visit <a href="http://localhost:9090/targets">http://localhost:9090/targets</a>.</p><p>Confirm that target <code class="highlighter-rouge">istio-mesh</code> has a status of <strong>UP</strong>.</p></li></ol><h2 id="how-can-i-debug-issues-with-the-service-mesh">How can I debug issues with the service mesh?</h2><h3 id="with-gdb">With <a href="https://www.gnu.org/software/gdb/">GDB</a></h3><p>To debug Istio with <code class="highlighter-rouge">gdb</code>, you will need to run the debug images of Envoy / Mixer / Pilot. A recent <code class="highlighter-rouge">gdb</code> and the golang extensions (for Mixer/Pilot or other golang components) is required.</p><ol><li><code class="highlighter-rouge">kubectl exec -it PODNAME -c [proxy | mixer | pilot]</code></li><li>Find process ID: ps ax</li><li>gdb -p PID binary</li><li>For go: info goroutines, goroutine x bt</li></ol><h3 id="with-tcpdump">With <a href="https://www.tcpdump.org/tcpdump_man.html">Tcpdump</a></h3><p>Tcpdump doesn’t work in the sidecar pod - the container doesn’t run as root. However any other container in the same pod will see all the packets, since the network namespace is shared. <code class="highlighter-rouge">iptables</code> will also see the pod-wide config.</p><p>Communication between Envoy and the app happens on 127.0.0.1, and is not encrypted.</p><h2 id="envoy-is-crashing-under-load">Envoy is crashing under load</h2><p>Check your <code class="highlighter-rouge">ulimit -a</code>. Many systems have a 1024 open file descriptor limit by default which will cause Envoy to assert and crash with:</p><div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="o">[</span>2017-05-17 03:00:52.735][14236][critical][assert] assert failure: fd_ <span class="o">!=</span> <span class="nt">-1</span>: external/envoy/source/common/network/connection_impl.cc:58
|
||
</code></pre></div></div><p>Make sure to raise your ulimit. Example: <code class="highlighter-rouge">ulimit -n 16384</code></p><h2 id="headless-tcp-services-losing-connection-from-istiofied-containers">Headless TCP Services Losing Connection from Istiofied Containers</h2><p>If <code class="highlighter-rouge">istio-ca</code> is deployed, Envoy is restarted every 15 minutes to refresh certificates. This causes the disconnection of TCP streams or long-running connections between services.</p><p>You should build resilience into your application for this type of disconnect, but if you still want to prevent the disconnects from happening, you will need to disable mTLS and the <code class="highlighter-rouge">istio-ca</code> deployment.</p><p>First, edit your istio config to disable mTLS</p><div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code># comment out or uncomment out authPolicy: MUTUAL_TLS to toggle mTLS and then
|
||
kubectl edit configmap -n istio-system istio
|
||
|
||
# restart pilot and wait a few minutes
|
||
kubectl delete pods -n istio-system -l istio=pilot
|
||
</code></pre></div></div><p>Next, scale down the <code class="highlighter-rouge">istio-ca</code> deployment to disable Envoy restarts.</p><div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>kubectl scale --replicas=0 deploy/istio-ca -n istio-system
|
||
</code></pre></div></div><p>This should stop istio from restarting Envoy and disconnecting TCP connections.</p><h2 id="envoy-process-high-cpu-usage">Envoy Process High CPU Usage</h2><p>For larger clusters, the default configuration that comes with Istio refreshes the Envoy configuration every 1 second. This can cause high CPU usage, even when Envoy isn’t doing anything. In order to bring the CPU usage down for larger deployments, increase the refresh interval for Envoy to something higher, like 30 seconds.</p><div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code># increase the field rdsRefreshDelay in the mesh and defaultConfig section
|
||
# set the refresh interval to 30s
|
||
kubectl edit configmap -n istio-system istio
|
||
|
||
# restart pilot and wait a few minutes
|
||
kubectl delete pods -n istio-system -l istio=pilot
|
||
</code></pre></div></div><p>Also make sure to reinject the sidecar into all of your pods, as their configuration needs to be updated as well.</p><p>Afterwards, you should see CPU usage fall back to 0-1% while idling. Make sure to tune these values for your specific deployment.</p><p><em>Warning:</em>: Changes created by routing rules will take up to 2x refresh interval to propagate to the sidecars. While the larger refresh interval will reduce CPU usage, updates caused by routing rules may cause a period of HTTP 404s (upto 2x the refresh interval) until the Envoy sidecars get all relevant configuration.</p><h2 id="kubernetes-webhook-setup-script-files-are-missing-from-05-release-package">Kubernetes webhook setup script files are missing from 0.5 release package</h2><p>NOTE: The 0.5.0 and 0.5.1 releases are missing scripts to provision webhook certificates. Download the missing files from <a href="https://raw.githubusercontent.com/istio/istio/master/install/kubernetes/webhook-create-signed-cert.sh">here</a> and <a href="https://raw.githubusercontent.com/istio/istio/master/install/kubernetes/webhook-patch-ca-bundle.sh">here</a>. Subsequent releases (> 0.5.1) should include these missing files.</p><h2 id="automatic-sidecar-injection-will-fail-if-the-kube-apiserver-has-proxy-settings">Automatic sidecar injection will fail if the kube-apiserver has proxy settings</h2><p>This was tested on 0.5.0 with the additional files required as referenced in the above issue. When the Kube-apiserver included proxy settings such as:</p><div class="language-yaml highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="na">env</span><span class="pi">:</span>
|
||
<span class="pi">-</span> <span class="na">name</span><span class="pi">:</span> <span class="s">http_proxy</span>
|
||
<span class="na">value</span><span class="pi">:</span> <span class="s">http://proxy-wsa.esl.foo.com:80</span>
|
||
<span class="pi">-</span> <span class="na">name</span><span class="pi">:</span> <span class="s">https_proxy</span>
|
||
<span class="na">value</span><span class="pi">:</span> <span class="s">http://proxy-wsa.esl.foo.com:80</span>
|
||
<span class="pi">-</span> <span class="na">name</span><span class="pi">:</span> <span class="s">no_proxy</span>
|
||
<span class="na">value</span><span class="pi">:</span> <span class="s">127.0.0.1,localhost,dockerhub.foo.com,devhub-docker.foo.com,10.84.100.125,10.84.100.126,10.84.100.127</span>
|
||
</code></pre></div></div><p>The sidecar injection would fail. The only related failure logs was in the kube-apiserver log:</p><div class="language-bash highlighter-rouge"><div class="highlight"><pre class="highlight"><code>W0227 21:51:03.156818 1 admission.go:257] Failed calling webhook, failing open sidecar-injector.istio.io: failed calling admission webhook <span class="s2">"sidecar-injector.istio.io"</span>: Post https://istio-sidecar-injector.istio-system.svc:443/inject: Service Unavailable
|
||
</code></pre></div></div><p>Make sure both pod and service CIDRs are not proxied according to *_proxy variables. Check the kube-apiserver files and logs to verify the configuration and whether any requests are being proxied.</p><p>A workaround is to remove the proxy settings from the kube-apiserver manifest and restart the server or use a later version of kubernetes.</p><p>An issue was filed in kubernetes related to this and has since been closed. <a href="https://github.com/kubernetes/kubeadm/issues/666">https://github.com/kubernetes/kubeadm/issues/666</a> <a href="https://github.com/kubernetes/kubernetes/pull/58698#discussion_r163879443">https://github.com/kubernetes/kubernetes/pull/58698#discussion_r163879443</a></p></main></div><div class="col-12 col-md-3 d-none d-lg-block"><nav class="toc"><div class="spacer"></div><div class="directory" role="directory"><ul><li><a href="#verifying-connectivity-to-istio-pilot">Verifying connectivity to Istio Pilot</a></li><li><a href="#no-traces-appearing-in-zipkin-when-running-istio-locally-on-mac">No traces appearing in Zipkin when running Istio locally on Mac</a></li><li><a href="#envoy-wont-connect-to-my-http10-service">Envoy won’t connect to my HTTP/1.0 service</a></li><li><a href="#no-grafana-output-when-connecting-from-a-local-web-client-to-istio-remotely-hosted">No grafana output when connecting from a local web client to Istio remotely hosted</a></li><li><a href="#where-are-the-metrics-for-my-service">Where are the metrics for my service?</a><ul><li><a href="#verify-mixer-is-receiving-report-calls">Verify Mixer is receiving Report calls</a></li><li><a href="#verify-mixer-metrics-configuration-exists">Verify Mixer metrics configuration exists</a></li><li><a href="#verify-mixer-is-sending-metric-instances-to-the-prometheus-adapter">Verify Mixer is sending metric instances to the Prometheus adapter</a></li><li><a href="#verify-prometheus-configuration">Verify Prometheus configuration</a></li></ul></li><li><a href="#how-can-i-debug-issues-with-the-service-mesh">How can I debug issues with the service mesh?</a><ul><li><a href="#with-gdb">With GDB</a></li><li><a href="#with-tcpdump">With Tcpdump</a></li></ul></li><li><a href="#envoy-is-crashing-under-load">Envoy is crashing under load</a></li><li><a href="#headless-tcp-services-losing-connection-from-istiofied-containers">Headless TCP Services Losing Connection from Istiofied Containers</a></li><li><a href="#envoy-process-high-cpu-usage">Envoy Process High CPU Usage</a></li><li><a href="#kubernetes-webhook-setup-script-files-are-missing-from-05-release-package">Kubernetes webhook setup script files are missing from 0.5 release package</a></li><li><a href="#automatic-sidecar-injection-will-fail-if-the-kube-apiserver-has-proxy-settings">Automatic sidecar injection will fail if the kube-apiserver has proxy settings</a></li></ul></div></nav></div></div></div><div class="footer"><footer><div class="container-fluid"><div class="row"><div class="col-sm-2"></div><nav class=" col-12 col-sm-3" role="navigation"><ul class="first"><li><a class="header" href="/v0.6/docs/">Docs</a></li><li><a href="/v0.6/docs/concepts/">Concepts</a></li><li><a href="/v0.6/docs/setup/">Setup</a></li><li><a href="/v0.6/docs/tasks/">Tasks</a></li><li><a href="/v0.6/docs/guides/">Guides</a></li><li><a href="/v0.6/docs/reference/">Reference</a></li></ul></nav><nav class="col-12 col-sm-3" role="navigation"><ul><li><a class="header" href="/v0.6/help/">Help</a></li><li><a href="/v0.6/help/faq/index.html">FAQ</a></li><li><a href="/v0.6/help/glossary.html">Glossary</a></li><li><a href="/v0.6/help/troubleshooting.html">Troubleshooting</a></li><li><a href="/v0.6/help/bugs.html">Report Bugs</a></li><li><a href="https://github.com/istio/istio.github.io/issues/new?title=Issue with _help/troubleshooting.md" target="_blank" rel="noopener">Doc Bugs & Gaps</a></li><li><a href="https://github.com/istio/istio.github.io/edit/master/_help/troubleshooting.md" target="_blank" rel="noopener">Edit This Page</a></li></ul></nav><nav class="col-12 col-sm-3" role="navigation"><ul><li><a class="header" href="/v0.6/community.html">Community</a></li><li> <a href="https://groups.google.com/forum/#!forum/istio-users" target="_blank" rel="noopener">User</a> | <a href="https://groups.google.com/forum/#!forum/istio-dev" target="_blank" rel="noopener">Dev Mailing Lists</a></li><li><a href="https://twitter.com/IstioMesh" target="_blank" rel="noopener">Twitter</a></li><li><a href="https://stackoverflow.com/questions/tagged/istio" target="_blank" rel="noopener">Stack Overflow</a></li><li><a href="https://github.com/istio/community/" target="_blank" rel="noopener">GitHub</a></li><li><a href="https://github.com/istio/community/blob/master/WORKING-GROUPS.md" target="_blank" rel="noopener">Working Groups</a></li></ul></nav></div><div class="row"><div class="col-12"><p class="description text-center" role="contentinfo"> Istio Archive 0.6, Copyright © 2018 Istio Authors<br> Archived on 02-Apr-2018</p></div></div></div></footer></div><script src="https://code.jquery.com/jquery-3.2.1.slim.min.js" integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN" crossorigin="anonymous"></script> <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js" integrity="sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q" crossorigin="anonymous"></script> <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script> <script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/1.7.1/clipboard.min.js"></script> <script src="https://www.google.com/cse/brand?form=searchbox"></script> <script src="/v0.6/js/misc.min.js"></script></body></html>
|