mirror of https://github.com/istio/istio.io.git
157 lines
30 KiB
HTML
157 lines
30 KiB
HTML
<!doctype html><html lang=en itemscope itemtype=https://schema.org/WebPage><head><meta charset=utf-8><meta http-equiv=x-ua-compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=theme-color content=#466BB0><meta name=title content="Troubleshooting Networking Issues"><meta name=description content="Describes common networking issues and how to recognize and avoid them."><meta name=keywords content=microservices,services,mesh><meta property=og:title content="Troubleshooting Networking Issues"><meta property=og:type content=website><meta property=og:description content="Describes common networking issues and how to recognize and avoid them."><meta property=og:url content=/v1.1/help/ops/traffic-management/troubleshooting/><meta property=og:image content=/v1.1/img/istio-whitelogo-bluebackground-framed.svg><meta property=og:image:alt content="Istio Logo"><meta property=og:image:width content=112><meta property=og:image:height content=150><meta property=og:site_name content=Istio><meta name=twitter:card content=summary><meta name=twitter:site content=@IstioMesh><title>Istioldie 1.1 / Troubleshooting Networking Issues</title><script async src="https://www.googletagmanager.com/gtag/js?id=UA-98480406-2"></script><script>window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments);}
|
||
gtag('js',new Date());gtag('config','UA-98480406-2');</script><link rel=alternate type=application/rss+xml title="Istio Blog" href=/v1.1/feed.xml><link rel="shortcut icon" href=/v1.1/favicons/favicon.ico><link rel=apple-touch-icon href=/v1.1/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/v1.1/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/v1.1/favicons/favicon-32x32.png sizes=32x32><link rel=icon type=image/png href=/v1.1/favicons/android-36x36.png sizes=36x36><link rel=icon type=image/png href=/v1.1/favicons/android-48x48.png sizes=48x48><link rel=icon type=image/png href=/v1.1/favicons/android-72x72.png sizes=72x72><link rel=icon type=image/png href=/v1.1/favicons/android-96x96.png sizes=96xW96><link rel=icon type=image/png href=/v1.1/favicons/android-144x144.png sizes=144x144><link rel=icon type=image/png href=/v1.1/favicons/android-192x192.png sizes=192x192><link rel=manifest href=/v1.1/manifest.json><meta name=apple-mobile-web-app-title content=Istio><meta name=application-name content=Istio><link rel=stylesheet href="https://fonts.googleapis.com/css?family=Work+Sans:400|Chivo:400|Work+Sans:500,300,600,300italic,400italic,500italic,600italic|Chivo:500,300,600,300italic,400italic,500italic,600italic"><link rel=stylesheet href=/v1.1/css/all.css></head><body class="language-unknown archive-site"><script src=/v1.1/js/themes_init.min.js></script><script>const branchName="release-1.1";const docTitle="Troubleshooting Networking Issues";const iconFile="\/v1.1/img/icons.svg";const buttonCopy='Copy to clipboard';const buttonPrint='Print';const buttonDownload='Download';</script><script src="https://www.google.com/cse/brand?form=search-form" defer></script><script src=/v1.1/js/all.min.js data-manual defer></script><header><nav><a id=brand href=/v1.1/><span class=logo><svg viewBox="0 0 300 300"><circle cx="150" cy="150" r="146" stroke-width="2" /><path d="M65 240H225L125 270z"/><path d="M65 230l60-10V110z"/><path d="M135 220l90 10L135 30z"/></svg></span><span class=name>Istioldie 1.1</span></a><div id=hamburger><svg class="icon"><use xlink:href="/v1.1/img/icons.svg#hamburger"/></svg></div><div id=header-links><a title="Learn how to deploy, use, and operate Istio." href=/v1.1/docs/>Docs</a>
|
||
<a title="Posts about using Istio." href=/v1.1/blog/2019/announcing-1.1.9/>Blog</a>
|
||
<span title="A bunch of resources to help you deploy, configure and use Istio.">Help</span>
|
||
<a title="Get a bit more in-depth info about the Istio project." href=/v1.1/about/>About</a><div class=menu><button id=gearDropdownButton class=menu-trigger title="Options and settings" aria-label="Options and Settings" aria-controls=gearDropdownContent><svg class="icon"><use xlink:href="/v1.1/img/icons.svg#gear"/></svg></button><div id=gearDropdownContent class=menu-content aria-labelledby=gearDropdownButton role=menu><a tabindex=-1 role=menuitem lang=en id=switch-lang-en class=active>English</a>
|
||
<a tabindex=-1 role=menuitem lang=zh id=switch-lang-zh>中文</a><div role=separator></div><a tabindex=-1 role=menuitem class=active id=light-theme-item>Light Theme</a>
|
||
<a tabindex=-1 role=menuitem id=dark-theme-item>Dark Theme</a><div role=separator></div><a tabindex=-1 role=menuitem id=syntax-coloring-item>Color Examples</a><div role=separator></div><h6>Other versions of this site</h6><a tabindex=-1 role=menuitem onclick="navigateToUrlOrRoot('https://istio.io/help\/ops\/traffic-management\/troubleshooting\/');return false;">Current Release</a>
|
||
<a tabindex=-1 role=menuitem onclick="navigateToUrlOrRoot('https://preliminary.istio.io/help\/ops\/traffic-management\/troubleshooting\/');return false;">Next Release</a>
|
||
<a tabindex=-1 role=menuitem href=https://archive.istio.io>Older Releases</a></div></div><button id=search-show title="Search this site" aria-label=Search><svg class="icon"><use xlink:href="/v1.1/img/icons.svg#magnifier"/></svg></button></div><form id=search-form name=cse role=search><input type=hidden name=cx value=013699703217164175118:iwwf17ikgf4>
|
||
<input type=hidden name=ie value=utf-8>
|
||
<input type=hidden name=hl value=en>
|
||
<input type=hidden id=search-page-url value=/v1.1/search.html>
|
||
<input id=search-textbox class=form-control name=q type=search aria-label="Search this site">
|
||
<button id=search-close title="Cancel search" type=reset aria-label="Cancel search"><svg class="icon"><use xlink:href="/v1.1/img/icons.svg#cancel-x"/></svg></button></form></nav></header><main class=primary><div id=sidebar-container class="sidebar-container sidebar-offcanvas"><nav id=sidebar aria-label="Section Navigation"><div class=directory><div class=card><div id=header0 class=header title="A bunch of resources to help you deploy, configure and use Istio."><svg class="icon"><use xlink:href="/v1.1/img/icons.svg#help"/></svg>Need Help?</div><div class="body default" aria-labelledby=header0><ul role=tree aria-expanded=true aria-labelledby=header0><li role=treeitem aria-label="Operations Guide"><button class=show aria-hidden=true></button><a title="Hints, tips, tricks about running an Istio mesh." href=/v1.1/help/ops/>Operations Guide</a><ul role=group aria-expanded=true><li role=none><a role=treeitem title="Describes how to use component-level logging to get insights into a running component's behavior." href=/v1.1/help/ops/component-logging/>Component Logging</a></li><li role=none><a role=treeitem title="Describes how to use ControlZ to get insight into individual running components." href=/v1.1/help/ops/controlz/>Component Introspection</a></li><li role=none><a role=treeitem title="How to do low-level debugging of Istio components." href=/v1.1/help/ops/component-debugging/>Component Debugging</a></li><li role=treeitem aria-label="Traffic Management"><button class=show aria-hidden=true></button><a title="Helps you manage the networking aspects of a running mesh." href=/v1.1/help/ops/traffic-management/>Traffic Management</a><ul role=group aria-expanded=true class=leaf-section><li role=none><a role=treeitem title="An introduction to Istio networking operational aspects." href=/v1.1/help/ops/traffic-management/introduction/>Introduction to Network Operations</a></li><li role=none><a role=treeitem title="Provides specific deployment and configuration guidelines." href=/v1.1/help/ops/traffic-management/deploy-guidelines/>Deployment and Configuration Guidelines</a></li><li role=none><span role=treeitem class=current title="Describes common networking issues and how to recognize and avoid them.">Troubleshooting Networking Issues</span></li><li role=none><a role=treeitem title="Describes tools and techniques to diagnose Envoy configuration issues related to traffic management." href=/v1.1/help/ops/traffic-management/proxy-cmd/>Debugging Envoy and Pilot</a></li><li role=none><a role=treeitem title="Information on how to enable and understand Locality Load Balancing." href=/v1.1/help/ops/traffic-management/locality-load-balancing/>Locality Load Balancing</a></li></ul></li><li role=treeitem aria-label=Security><button aria-hidden=true></button><a title="Helps you manage the security aspects of a running mesh." href=/v1.1/help/ops/security/>Security</a><ul role=group aria-expanded=false class=leaf-section><li role=none><a role=treeitem title="Demonstrates how to debug authorization." href=/v1.1/help/ops/security/debugging-authorization/>Debugging Authorization</a></li><li role=none><a role=treeitem title="What to do if Citadel is not behaving properly." href=/v1.1/help/ops/security/repairing-citadel/>Repairing Citadel</a></li><li role=none><a role=treeitem title="What to do if you suspect problems with Istio keys and certificates." href=/v1.1/help/ops/security/keys-and-certs/>Keys and Certificates</a></li><li role=none><a role=treeitem title="What to do if mutual TLS authentication isn't working." href=/v1.1/help/ops/security/mutual-tls/>Mutual TLS</a></li><li role=none><a role=treeitem title="Authorization is enabled, but requests make it through anyway." href=/v1.1/help/ops/security/authorization-permissive/>Authorization Too Permissive</a></li><li role=none><a role=treeitem title="Authorization is enabled and no requests make it through to the service." href=/v1.1/help/ops/security/authorization-restrictive/>Authorization Too Restrictive</a></li><li role=none><a role=treeitem title="What to do if end-user authentication doesn't work." href=/v1.1/help/ops/security/end-user-auth/>End User Authentication</a></li><li role=none><a role=treeitem title="Learn how to extend the lifetime of the Istio self-signed root certificate." href=/v1.1/help/ops/security/root-transition/>Extending Self-Signed Certificate Lifetime</a></li></ul></li><li role=treeitem aria-label=Telemetry><button aria-hidden=true></button><a title="Helps you manage telemetry collection and visualization in a running mesh." href=/v1.1/help/ops/telemetry/>Telemetry</a><ul role=group aria-expanded=false class=leaf-section><li role=none><a role=treeitem title="Diagnose problems where metrics are not being collected." href=/v1.1/help/ops/telemetry/missing-metrics/>Missing Metrics</a></li><li role=none><a role=treeitem title="Dealing with Grafana issues." href=/v1.1/help/ops/telemetry/grafana/>Grafana</a></li><li role=none><a role=treeitem title="Fine-grained control of Envoy statistics." href=/v1.1/help/ops/telemetry/envoy-stats/>Envoy Statistics</a></li></ul></li><li role=treeitem aria-label="Installation and Setup"><button aria-hidden=true></button><a title="Helps you diagnose and repair Istio installations." href=/v1.1/help/ops/setup/>Installation and Setup</a><ul role=group aria-expanded=false class=leaf-section><li role=none><a role=treeitem title="Provides a general overview of Istio's use of Kubernetes webhooks and the related issues that can arise." href=/v1.1/help/ops/setup/webhook/>Dynamic Admission Webhooks Overview</a></li><li role=none><a role=treeitem title="Describes Istio's use of Kubernetes webhooks for server-side configuration validation." href=/v1.1/help/ops/setup/validation/>Configuration Validation Webhook</a></li><li role=none><a role=treeitem title="Describes Istio's use of Kubernetes webhooks for automatic sidecar injection." href=/v1.1/help/ops/setup/injection/>Sidecar Injection Webhook</a></li><li role=none><a role=treeitem title="Describes how to check which capabilities are allowed for your pods." href=/v1.1/help/ops/setup/required-pod-capabilities/>Required Pod Capabilities</a></li><li role=none><a role=treeitem title="Shows how to do health checking for Istio services." href=/v1.1/help/ops/setup/app-health-check/>Health Checking of Istio Services</a></li></ul></li><li role=none><a role=treeitem title="Advice on tackling common problems with Istio." href=/v1.1/help/ops/misc/>Miscellaneous</a></li></ul></li><li role=treeitem aria-label=FAQ><button aria-hidden=true></button><a title="Frequently Asked Questions about Istio." href=/v1.1/help/faq/>FAQ</a><ul role=group aria-expanded=false class=leaf-section><li role=none><a role=treeitem title="General Q & A." href=/v1.1/help/faq/general/>General</a></li><li role=none><a role=treeitem title="Setup Q & A." href=/v1.1/help/faq/setup/>Setup</a></li><li role=none><a role=treeitem title="Security Q & A." href=/v1.1/help/faq/security/>Security</a></li><li role=none><a role=treeitem title="Mixer Q & A." href=/v1.1/help/faq/mixer/>Mixer</a></li><li role=none><a role=treeitem title="Metrics and Logs Q & A." href=/v1.1/help/faq/metrics-and-logs/>Metrics and Logs</a></li><li role=none><a role=treeitem title="Distributed Tracing Q & A." href=/v1.1/help/faq/distributed-tracing/>Distributed Tracing</a></li><li role=none><a role=treeitem title="Traffic Management Q & A." href=/v1.1/help/faq/traffic-management/>Traffic Management</a></li></ul></li><li role=none><a role=treeitem title="A glossary of common Istio terms." href=/v1.1/help/glossary/>Glossary</a></li></ul></div></div></div></nav></div><div class=article-container><button tabindex=-1 id=sidebar-toggler title="Toggle the navigation bar"><svg class="icon"><use xlink:href="/v1.1/img/icons.svg#pull"/></svg></button><nav aria-label=Breadcrumb><ol><li><a href=/v1.1/ title="Connect, secure, control, and observe services.">Istio</a></li><li><a href=/v1.1/help/ title="A bunch of resources to help you deploy, configure and use Istio.">Help</a></li><li><a href=/v1.1/help/ops/ title="Hints, tips, tricks about running an Istio mesh.">Operations Guide</a></li><li><a href=/v1.1/help/ops/traffic-management/ title="Helps you manage the networking aspects of a running mesh.">Traffic Management</a></li><li>Troubleshooting Networking Issues</li></ol></nav><article aria-labelledby=title><div class=title-area><div><h1 id=title>Troubleshooting Networking Issues</h1><p class=byline><span title="1152 words"><svg class="icon"><use xlink:href="/v1.1/img/icons.svg#clock"/></svg><span> </span>6 minute read</span></p></div></div><nav class=toc-inlined aria-label="Table of Contents"><div><hr><ol><li role=none aria-label="Requests are rejected by Envoy"><a href=#requests-are-rejected-by-envoy>Requests are rejected by Envoy</a><li role=none aria-label="Route rules don’t seem to affect traffic flow"><a href=#route-rules-don-t-seem-to-affect-traffic-flow>Route rules don’t seem to affect traffic flow</a><li role=none aria-label="503 errors after setting destination rule"><a href=#503-errors-after-setting-destination-rule>503 errors after setting destination rule</a><li role=none aria-label="Route rules have no effect on ingress gateway requests"><a href=#route-rules-have-no-effect-on-ingress-gateway-requests>Route rules have no effect on ingress gateway requests</a><li role=none aria-label="Headless TCP services losing connection"><a href=#headless-tcp-services-losing-connection>Headless TCP services losing connection</a><li role=none aria-label="Envoy is crashing under load"><a href=#envoy-is-crashing-under-load>Envoy is crashing under load</a><li role=none aria-label="Envoy won’t connect to my HTTP/1.0 service"><a href=#envoy-won-t-connect-to-my-http-1-0-service>Envoy won’t connect to my HTTP/1.0 service</a></ol><hr></div></nav><p>This section describes common problems and tools and techniques to address issues related to traffic management.</p><h2 id=requests-are-rejected-by-envoy>Requests are rejected by Envoy</h2><p>Requests may be rejected for various reasons. The best way to understand why requests are being rejected is
|
||
by inspecting Envoy’s access logs. By default, access logs are output to the standard output of the container.
|
||
Run the following command to see the log:</p><pre><code class=language-bash data-expandlinks=true>$ kubectl logs PODNAME -c istio-proxy -n NAMESPACE
|
||
</code></pre><p>In the default access log format, Envoy response flags and Mixer policy status are located after the response code,
|
||
if you are using a custom log format, make sure to include <code>%RESPONSE_FLAGS%</code> and <code>%DYNAMIC_METADATA(istio.mixer:status)%</code>.</p><p>Refer to the <a href=https://www.envoyproxy.io/docs/envoy/latest/configuration/access_log#config-access-log-format-response-flags>Envoy response flags</a>
|
||
for details of response flags.</p><p>Common response flags are:</p><ul><li><code>NR</code>: No route configured, check your <code>DestinationRule</code> or <code>VirtualService</code>.</li><li><code>UO</code>: Upstream overflow with circuit breaking, check your circuit breaker configuration in <code>DestinationRule</code>.</li><li><code>UF</code>: Failed to connect to upstream, if you’re using Istio authentication, check for a
|
||
<a href=#503-errors-after-setting-destination-rule>mutual TLS configuration conflict</a>.</li></ul><p>A request is rejected by Mixer if the response flag is <code>UAEX</code> and the Mixer policy status is not <code>-</code>.</p><p>Common Mixer policy statuses are:</p><ul><li><code>UNAVAILABLE</code>: Envoy cannot connect to Mixer and the policy is configured to fail close.</li><li><code>UNAUTHENTICATED</code>: The request is rejected by Mixer authentication.</li><li><code>PERMISSION_DENIED</code>: The request is rejected by Mixer authorization.</li><li><code>RESOURCE_EXHAUSTED</code>: The request is rejected by Mixer quota.</li><li><code>INTERNAL</code>: The request is rejected due to Mixer internal error.</li></ul><h2 id=route-rules-don-t-seem-to-affect-traffic-flow>Route rules don’t seem to affect traffic flow</h2><p>With the current Envoy sidecar implementation, up to 100 requests may be required for weighted
|
||
version distribution to be observed.</p><p>If route rules are working perfectly for the <a href=/v1.1/docs/examples/bookinfo/>Bookinfo</a> sample,
|
||
but similar version routing rules have no effect on your own application, it may be that
|
||
your Kubernetes services need to be changed slightly.
|
||
Kubernetes services must adhere to certain restrictions in order to take advantage of
|
||
Istio’s L7 routing features.
|
||
Refer to the <a href=/v1.1/docs/setup/kubernetes/prepare/requirements/>Requirements for Pods and Services</a>
|
||
for details.</p><p>Another potential issue is that the route rules may simply be slow to take effect.
|
||
The Istio implementation on Kubernetes utilizes an eventually consistent
|
||
algorithm to ensure all Envoy sidecars have the correct configuration
|
||
including all route rules. A configuration change will take some time
|
||
to propagate to all the sidecars. With large deployments the
|
||
propagation will take longer and there may be a lag time on the
|
||
order of seconds.</p><h2 id=503-errors-after-setting-destination-rule>503 errors after setting destination rule</h2><p>If requests to a service immediately start generating HTTP 503 errors after you applied a <code>DestinationRule</code>
|
||
and the errors continue until you remove or revert the <code>DestinationRule</code>, then the <code>DestinationRule</code> is probably
|
||
causing a TLS conflict for the service.</p><p>For example, if you configure mutual TLS in the cluster globally, the <code>DestinationRule</code> must include the following <code>trafficPolicy</code>:</p><pre><code class=language-yaml data-expandlinks=true>trafficPolicy:
|
||
tls:
|
||
mode: ISTIO_MUTUAL
|
||
</code></pre><p>Otherwise, the mode defaults to <code>DISABLE</code> causing client proxy sidecars to make plain HTTP requests
|
||
instead of TLS encrypted requests. Thus, the requests conflict with the server proxy because the server proxy expects
|
||
encrypted requests.</p><p>To confirm there is a conflict, check whether the <code>STATUS</code> field in the output of the <code>istioctl authn tls-check</code> command
|
||
is set to <code>CONFLICT</code> for your service. For example:</p><pre><code class=language-bash data-expandlinks=true>$ istioctl authn tls-check httpbin.default.svc.cluster.local
|
||
HOST:PORT STATUS SERVER CLIENT AUTHN POLICY DESTINATION RULE
|
||
httpbin.default.svc.cluster.local:8000 CONFLICT mTLS HTTP default/ httpbin/default
|
||
</code></pre><p>Whenever you apply a <code>DestinationRule</code>, ensure the <code>trafficPolicy</code> TLS mode matches the global server configuration.</p><h2 id=route-rules-have-no-effect-on-ingress-gateway-requests>Route rules have no effect on ingress gateway requests</h2><p>Let’s assume you are using an ingress <code>Gateway</code> and corresponding <code>VirtualService</code> to access an internal service.
|
||
For example, your <code>VirtualService</code> looks something like this:</p><pre><code class=language-yaml data-expandlinks=true>apiVersion: networking.istio.io/v1alpha3
|
||
kind: VirtualService
|
||
metadata:
|
||
name: myapp
|
||
spec:
|
||
hosts:
|
||
- "myapp.com" # or maybe "*" if you are testing without DNS using the ingress-gateway IP (e.g., http://1.2.3.4/hello)
|
||
gateways:
|
||
- myapp-gateway
|
||
http:
|
||
- match:
|
||
- uri:
|
||
prefix: /hello
|
||
route:
|
||
- destination:
|
||
host: helloworld.default.svc.cluster.local
|
||
- match:
|
||
...
|
||
</code></pre><p>You also have a <code>VirtualService</code> which routes traffic for the helloworld service to a particular subset:</p><pre><code class=language-yaml data-expandlinks=true>apiVersion: networking.istio.io/v1alpha3
|
||
kind: VirtualService
|
||
metadata:
|
||
name: helloworld
|
||
spec:
|
||
hosts:
|
||
- helloworld.default.svc.cluster.local
|
||
http:
|
||
- route:
|
||
- destination:
|
||
host: helloworld.default.svc.cluster.local
|
||
subset: v1
|
||
</code></pre><p>In this situation you will notice that requests to the helloworld service via the ingress gateway will
|
||
not be directed to subset v1 but instead will continue to use default round-robin routing.</p><p>The ingress requests are using the gateway host (e.g., <code>myapp.com</code>)
|
||
which will activate the rules in the myapp <code>VirtualService</code> that routes to any endpoint of the helloworld service.
|
||
Only internal requests with the host <code>helloworld.default.svc.cluster.local</code> will use the
|
||
helloworld <code>VirtualService</code> which directs traffic exclusively to subset v1.</p><p>To control the traffic from the gateway, you need to also include the subset rule in the myapp <code>VirtualService</code>:</p><pre><code class=language-yaml data-expandlinks=true>apiVersion: networking.istio.io/v1alpha3
|
||
kind: VirtualService
|
||
metadata:
|
||
name: myapp
|
||
spec:
|
||
hosts:
|
||
- "myapp.com" # or maybe "*" if you are testing without DNS using the ingress-gateway IP (e.g., http://1.2.3.4/hello)
|
||
gateways:
|
||
- myapp-gateway
|
||
http:
|
||
- match:
|
||
- uri:
|
||
prefix: /hello
|
||
route:
|
||
- destination:
|
||
host: helloworld.default.svc.cluster.local
|
||
subset: v1
|
||
- match:
|
||
...
|
||
</code></pre><p>Alternatively, you can combine both <code>VirtualServices</code> into one unit if possible:</p><pre><code class=language-yaml data-expandlinks=true>apiVersion: networking.istio.io/v1alpha3
|
||
kind: VirtualService
|
||
metadata:
|
||
name: myapp
|
||
spec:
|
||
hosts:
|
||
- myapp.com # cannot use "*" here since this is being combined with the mesh services
|
||
- helloworld.default.svc.cluster.local
|
||
gateways:
|
||
- mesh # applies internally as well as externally
|
||
- myapp-gateway
|
||
http:
|
||
- match:
|
||
- uri:
|
||
prefix: /hello
|
||
gateways:
|
||
- myapp-gateway #restricts this rule to apply only to ingress gateway
|
||
route:
|
||
- destination:
|
||
host: helloworld.default.svc.cluster.local
|
||
subset: v1
|
||
- match:
|
||
- gateways:
|
||
- mesh # applies to all services inside the mesh
|
||
route:
|
||
- destination:
|
||
host: helloworld.default.svc.cluster.local
|
||
subset: v1
|
||
</code></pre><h2 id=headless-tcp-services-losing-connection>Headless TCP services losing connection</h2><p>If <code>istio-citadel</code> is deployed, Envoy is restarted every 45 days to refresh certificates.
|
||
This causes the disconnection of TCP streams or long-running connections between services.</p><p>You should build resilience into your application for this type of
|
||
disconnect, but if you still want to prevent the disconnects from
|
||
happening, you will need to disable mutual TLS and the <code>istio-citadel</code> deployment.</p><p>First, edit your <code>istio</code> configuration to disable mutual TLS:</p><pre><code class=language-bash data-expandlinks=true>$ kubectl edit configmap -n istio-system istio
|
||
$ kubectl delete pods -n istio-system -l istio=pilot
|
||
</code></pre><p>Next, scale down the <code>istio-citadel</code> deployment to disable Envoy restarts:</p><pre><code class=language-bash data-expandlinks=true>$ kubectl scale --replicas=0 deploy/istio-citadel -n istio-system
|
||
</code></pre><p>This should stop Istio from restarting Envoy and disconnecting TCP connections.</p><h2 id=envoy-is-crashing-under-load>Envoy is crashing under load</h2><p>Check your <code>ulimit -a</code>. Many systems have a 1024 open file descriptor limit by default which will cause Envoy to assert and crash with:</p><pre><code class=language-plain data-expandlinks=true>[2017-05-17 03:00:52.735][14236][critical][assert] assert failure: fd_ != -1: external/envoy/source/common/network/connection_impl.cc:58
|
||
</code></pre><p>Make sure to raise your ulimit. Example: <code>ulimit -n 16384</code></p><h2 id=envoy-won-t-connect-to-my-http-1-0-service>Envoy won’t connect to my HTTP/1.0 service</h2><p>Envoy requires <code>HTTP/1.1</code> or <code>HTTP/2</code> traffic for upstream services. For example, when using <a href=https://www.nginx.com/>NGINX</a> for serving traffic behind Envoy, you
|
||
will need to set the <a href=https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_http_version>proxy_http_version</a> directive in your NGINX configuration to be “1.1”, since the NGINX default is 1.0.</p><p>Example configuration:</p><pre><code class=language-plain data-expandlinks=true>upstream http_backend {
|
||
server 127.0.0.1:8080;
|
||
|
||
keepalive 16;
|
||
}
|
||
|
||
server {
|
||
...
|
||
|
||
location /http/ {
|
||
proxy_pass http://http_backend;
|
||
proxy_http_version 1.1;
|
||
proxy_set_header Connection "";
|
||
...
|
||
}
|
||
}
|
||
</code></pre></article><nav class=pagenav><div class=left><a title="Provides specific deployment and configuration guidelines." href=/v1.1/help/ops/traffic-management/deploy-guidelines/><svg class="icon"><use xlink:href="/v1.1/img/icons.svg#left-arrow"/></svg>Deployment and Configuration Guidelines</a></div><div class=right><a title="Describes tools and techniques to diagnose Envoy configuration issues related to traffic management." href=/v1.1/help/ops/traffic-management/proxy-cmd/>Debugging Envoy and Pilot<svg class="icon"><use xlink:href="/v1.1/img/icons.svg#right-arrow"/></svg></a></div></nav><div id=endnotes-container aria-hidden=true><h2>Links</h2><ol id=endnotes></ol></div></div><div class=toc-container><nav class=toc aria-label="Table of Contents"><div id=toc><ol><li role=none aria-label="Requests are rejected by Envoy"><a href=#requests-are-rejected-by-envoy>Requests are rejected by Envoy</a><li role=none aria-label="Route rules don’t seem to affect traffic flow"><a href=#route-rules-don-t-seem-to-affect-traffic-flow>Route rules don’t seem to affect traffic flow</a><li role=none aria-label="503 errors after setting destination rule"><a href=#503-errors-after-setting-destination-rule>503 errors after setting destination rule</a><li role=none aria-label="Route rules have no effect on ingress gateway requests"><a href=#route-rules-have-no-effect-on-ingress-gateway-requests>Route rules have no effect on ingress gateway requests</a><li role=none aria-label="Headless TCP services losing connection"><a href=#headless-tcp-services-losing-connection>Headless TCP services losing connection</a><li role=none aria-label="Envoy is crashing under load"><a href=#envoy-is-crashing-under-load>Envoy is crashing under load</a><li role=none aria-label="Envoy won’t connect to my HTTP/1.0 service"><a href=#envoy-won-t-connect-to-my-http-1-0-service>Envoy won’t connect to my HTTP/1.0 service</a></ol></div></nav></div></main><footer><div class=user-links><a class=channel title="Go download Istio 1.1.9 now" href=https://github.com/istio/istio/releases/tag/1.1.9 aria-label="Download Istio"><span>download</span><svg class="icon"><use xlink:href="/v1.1/img/icons.svg#download"/></svg>
|
||
</a><a class=channel title="Join the Istio discussion board to participate in discussions and get help troubleshooting problems" href=https://discuss.istio.io aria-label="Istio discussion board"><span>discuss</span><svg class="icon"><use xlink:href="/v1.1/img/icons.svg#discourse"/></svg></a>
|
||
<a class=channel title="Stack Overflow is where you can ask questions and find curated answers on deploying, configuring, and using Istio" href=https://stackoverflow.com/questions/tagged/istio aria-label="Stack Overflow"><span>stack overflow</span><svg class="icon"><use xlink:href="/v1.1/img/icons.svg#stackoverflow"/></svg></a>
|
||
<a class=channel title="Follow us on Twitter to get the latest news" href=https://twitter.com/IstioMesh aria-label=Twitter><span>twitter</span><svg class="icon"><use xlink:href="/v1.1/img/icons.svg#twitter"/></svg></a><div class=tag>for everyone</div></div><div class=info><p class=copyright>Istio Archive
|
||
1.1.9<br>© 2019 Istio Authors, <a href=https://policies.google.com/privacy>Privacy Policy</a><br>Archived on June 18, 2019</p></div><div class=dev-links><a class=channel title="GitHub is where development takes place on Istio code" href=https://github.com/istio/community aria-label=GitHub><span>github</span><svg class="icon"><use xlink:href="/v1.1/img/icons.svg#github"/></svg></a>
|
||
<a class=channel title="Interactively discuss issues with the Istio community on Slack" href=https://istio.slack.com aria-label=slack><span>slack</span><svg class="icon"><use xlink:href="/v1.1/img/icons.svg#slack"/></svg></a>
|
||
<a class=channel title="Access our team drive if you'd like to take a look at the Istio technical design documents" href=https://groups.google.com/forum/#!forum/istio-team-drive-access aria-label="team drive"><span>drive</span><svg class="icon"><use xlink:href="/v1.1/img/icons.svg#drive"/></svg></a>
|
||
<a class=channel title="If you'd like to contribute to the Istio project, consider participating in our working groups" href=https://github.com/istio/community/blob/master/WORKING-GROUPS.md aria-label="working groups"><span>working groups</span><svg class="icon"><use xlink:href="/v1.1/img/icons.svg#working-groups"/></svg></a><div class=tag>for developers</div></div></footer><div id=scroll-to-top-container aria-hidden=true><button id=scroll-to-top title="Back to top"><svg class="icon"><use xlink:href="/v1.1/img/icons.svg#top"/></svg></button></div></body></html> |