mirror of https://github.com/istio/istio.io.git
298 lines
37 KiB
HTML
298 lines
37 KiB
HTML
<!doctype html><html lang=en itemscope itemtype=https://schema.org/WebPage><head><meta charset=utf-8><meta http-equiv=x-ua-compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=theme-color content=#466BB0><meta name=title content="Monitoring blocked and passthrough external service traffic"><meta name=description content="How can you use Istio to monitor blocked and passthrough external traffic."><meta name=author content="Neeraj Poddar (Aspen Mesh)"><meta name=keywords content=microservices,services,mesh,monitoring,blackhole,passthrough><meta property=og:title content="Monitoring blocked and passthrough external service traffic"><meta property=og:type content=website><meta property=og:description content="How can you use Istio to monitor blocked and passthrough external traffic."><meta property=og:url content=/v1.3/blog/2019/monitoring-external-service-traffic/><meta property=og:image content=/v1.3/img/istio-whitelogo-bluebackground-framed.svg><meta property=og:image:alt content="Istio Logo"><meta property=og:image:width content=112><meta property=og:image:height content=150><meta property=og:site_name content=Istio><meta name=twitter:card content=summary><meta name=twitter:site content=@IstioMesh><title>Istioldie 1.3 / Monitoring blocked and passthrough external service traffic</title><script async src="https://www.googletagmanager.com/gtag/js?id=UA-98480406-2"></script><script>window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments);}
|
|
gtag('js',new Date());gtag('config','UA-98480406-2');</script><link rel=alternate type=application/rss+xml title="Istio Blog" href=/v1.3/blog/feed.xml><link rel=alternate type=application/rss+xml title="Istio News" href=/v1.3/news/feed.xml><link rel=alternate type=application/rss+xml title="Istio Blog and News" href=/v1.3/feed.xml><link rel="shortcut icon" href=/v1.3/favicons/favicon.ico><link rel=apple-touch-icon href=/v1.3/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/v1.3/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/v1.3/favicons/favicon-32x32.png sizes=32x32><link rel=icon type=image/png href=/v1.3/favicons/android-36x36.png sizes=36x36><link rel=icon type=image/png href=/v1.3/favicons/android-48x48.png sizes=48x48><link rel=icon type=image/png href=/v1.3/favicons/android-72x72.png sizes=72x72><link rel=icon type=image/png href=/v1.3/favicons/android-96x96.png sizes=96xW96><link rel=icon type=image/png href=/v1.3/favicons/android-144x144.png sizes=144x144><link rel=icon type=image/png href=/v1.3/favicons/android-192x192.png sizes=192x192><link rel=manifest href=/v1.3/manifest.json><meta name=apple-mobile-web-app-title content=Istio><meta name=application-name content=Istio><link rel=stylesheet href="https://fonts.googleapis.com/css?family=Work+Sans:400|Chivo:400|Work+Sans:500,300,600,300italic,400italic,500italic,600italic|Chivo:500,300,600,300italic,400italic,500italic,600italic"><link rel=stylesheet href=/v1.3/css/all.css><script src=/v1.3/js/themes_init.min.js></script></head><body class="language-unknown archive-site"><script>const branchName="release-1.3";const docTitle="Monitoring blocked and passthrough external service traffic";const iconFile="\/v1.3/img/icons.svg";const buttonCopy='Copy to clipboard';const buttonPrint='Print';const buttonDownload='Download';</script><script src="https://www.google.com/cse/brand?form=search-form" defer></script><script src=/v1.3/js/all.min.js data-manual defer></script><header><nav><a id=brand href=/v1.3/><span class=logo><svg viewBox="0 0 300 300"><circle cx="150" cy="150" r="146" stroke-width="2" /><path d="M65 240H225L125 270z"/><path d="M65 230l60-10V110z"/><path d="M135 220l90 10L135 30z"/></svg></span><span class=name>Istioldie 1.3</span></a><div id=hamburger><svg class="icon"><use xlink:href="/v1.3/img/icons.svg#hamburger"/></svg></div><div id=header-links><a title="Learn how to deploy, use, and operate Istio." href=/v1.3/docs/>Docs</a>
|
|
<span title="Posts about using Istio.">Blog</span>
|
|
<a title="Timely news about the Istio project." href=/v1.3/news/2019/announcing-1.2-eol/>News</a>
|
|
<a title="Frequently Asked Questions about Istio." href=/v1.3/faq/>FAQ</a>
|
|
<a title="Get a bit more in-depth info about the Istio project." href=/v1.3/about/>About</a><div class=menu><button id=gearDropdownButton class=menu-trigger title="Options and settings" aria-label="Options and Settings" aria-controls=gearDropdownContent><svg class="icon"><use xlink:href="/v1.3/img/icons.svg#gear"/></svg></button><div id=gearDropdownContent class=menu-content aria-labelledby=gearDropdownButton role=menu><a tabindex=-1 role=menuitem class=active id=light-theme-item>Light Theme</a>
|
|
<a tabindex=-1 role=menuitem id=dark-theme-item>Dark Theme</a><div role=separator></div><a tabindex=-1 role=menuitem id=syntax-coloring-item>Color Examples</a><div role=separator></div><h6>Other versions of this site</h6><a tabindex=-1 role=menuitem onclick="navigateToUrlOrRoot('https://istio.io/blog\/2019\/monitoring-external-service-traffic\/');return false;">Current Release</a>
|
|
<a tabindex=-1 role=menuitem onclick="navigateToUrlOrRoot('https://preliminary.istio.io/blog\/2019\/monitoring-external-service-traffic\/');return false;">Next Release</a>
|
|
<a tabindex=-1 role=menuitem href=https://archive.istio.io>Older Releases</a></div></div><button id=search-show title="Search this site" aria-label=Search><svg class="icon"><use xlink:href="/v1.3/img/icons.svg#magnifier"/></svg></button></div><form id=search-form name=cse role=search><input type=hidden name=cx value=013699703217164175118:iwwf17ikgf4>
|
|
<input type=hidden name=ie value=utf-8>
|
|
<input type=hidden name=hl value=en>
|
|
<input type=hidden id=search-page-url value=/v1.3/search>
|
|
<input id=search-textbox class=form-control name=q type=search aria-label="Search this site">
|
|
<button id=search-close title="Cancel search" type=reset aria-label="Cancel search"><svg class="icon"><use xlink:href="/v1.3/img/icons.svg#cancel-x"/></svg></button></form></nav></header><main class=primary><div id=sidebar-container class="sidebar-container sidebar-offcanvas"><nav id=sidebar aria-label="Section Navigation"><div class=directory><div class=card><button class="header dynamic" id=card0 title="Blog posts for 2019." aria-controls=card0-body><svg class="icon"><use xlink:href="/v1.3/img/icons.svg#blog"/></svg>2019 Posts</button><div class="body default" aria-labelledby=card0 role=region id=card0-body><ul role=tree aria-expanded=true class=leaf-section aria-labelledby=card0><li role=none><a role=treeitem title="Configure Istio ingress gateway to act as a proxy for external services." href=/v1.3/blog/2019/proxy/>Istio as a Proxy for External Services</a></li><li role=none><span role=treeitem class=current title="How can you use Istio to monitor blocked and passthrough external traffic.">Monitoring blocked and passthrough external service traffic</span></li><li role=none><a role=treeitem title="Using Istio to secure multi-cloud Kubernetes applications with zero code changes." href=/v1.3/blog/2019/app-identity-and-access-adapter/>App Identity and Access Adapter</a></li><li role=none><a role=treeitem title="Demonstrates a Mixer out-of-process adapter which implements the Knative scale-from-zero logic." href=/v1.3/blog/2019/knative-activator-adapter/>Mixer out-of-process adapter for Knative</a></li><li role=none><a role=treeitem title="Taking advantage of Kubernetes trustworthy JWTs to issue certificates for workload instances more securely." href=/v1.3/blog/2019/trustworthy-jwt-sds/>Change in Secret Discovery Service in Istio 1.3</a></li><li role=none><a role=treeitem title="The design principles behind Istio's APIs and how those APIs are evolving." href=/v1.3/blog/2019/evolving-istios-apis/>The Evolution of Istio's APIs</a></li><li role=none><a role=treeitem title="Comparison of alternative solutions to control egress traffic including performance considerations." href=/v1.3/blog/2019/egress-traffic-control-in-istio-part-3/>Secure Control of Egress Traffic in Istio, part 3</a></li><li role=none><a role=treeitem title="Use Istio Egress Traffic Control to prevent attacks involving egress traffic." href=/v1.3/blog/2019/egress-traffic-control-in-istio-part-2/>Secure Control of Egress Traffic in Istio, part 2</a></li><li role=none><a role=treeitem title="Tools and guidance for evaluating Istio's data plane performance." href=/v1.3/blog/2019/performance-best-practices/>Best Practices: Benchmarking Service Mesh Performance</a></li><li role=none><a role=treeitem title="Learn how to extend the lifetime of Istio self-signed root certificate." href=/v1.3/blog/2019/root-transition/>Extending Istio Self-Signed Root Certificate Lifetime</a></li><li role=none><a role=treeitem title="Attacks involving egress traffic and requirements for egress traffic control." href=/v1.3/blog/2019/egress-traffic-control-in-istio-part-1/>Secure Control of Egress Traffic in Istio, part 1</a></li><li role=none><a role=treeitem title="An overview of Istio 1.1 performance." href=/v1.3/blog/2019/istio1.1_perf/>Architecting Istio 1.1 for Performance</a></li><li role=none><a role=treeitem title="Configuring Istio route rules in a multicluster service mesh." href=/v1.3/blog/2019/multicluster-version-routing/>Version Routing in a Multicluster Service Mesh</a></li><li role=none><a role=treeitem title="Announces the new Istio blog policy." href=/v1.3/blog/2019/sail-the-blog/>Sail the Blog!</a></li><li role=none><a role=treeitem title="De-mystify how Istio manages to plugin its data-plane components into an existing deployment." href=/v1.3/blog/2019/data-plane-setup/>Demystifying Istio's Sidecar Injection Model</a></li><li role=none><a role=treeitem title="Verifies the performance impact of adding an egress gateway." href=/v1.3/blog/2019/egress-performance/>Egress Gateway Performance Investigation</a></li><li role=none><a role=treeitem title="Addressing application startup ordering and startup latency using AppSwitch." href=/v1.3/blog/2019/appswitch/>Sidestepping Dependency Ordering with AppSwitch</a></li><li role=none><a role=treeitem title="Describes how to deploy a custom ingress gateway using cert-manager manually." href=/v1.3/blog/2019/custom-ingress-gateway/>Deploy a Custom Ingress Gateway Using Cert-Manager</a></li></ul></div></div><div class=card><button class="header dynamic" id=card1 title="Blog posts for 2018." aria-controls=card1-body><svg class="icon"><use xlink:href="/v1.3/img/icons.svg#blog"/></svg>2018 Posts</button><div class=body aria-labelledby=card1 role=region id=card1-body><ul role=tree aria-expanded=true class=leaf-section aria-labelledby=card1><li role=none><a role=treeitem title="How to use Istio for traffic management without deploying sidecar proxies." href=/v1.3/blog/2018/incremental-traffic-management/>Incremental Istio Part 1, Traffic Management</a></li><li role=none><a role=treeitem title="Describes a simple scenario based on Istio's Bookinfo example." href=/v1.3/blog/2018/egress-mongo/>Consuming External MongoDB Services</a></li><li role=none><a role=treeitem title="Istio hosting an all day Twitch stream to celebrate the 1.0 release." href=/v1.3/blog/2018/istio-twitch-stream/>All Day Istio Twitch Stream</a></li><li role=none><a role=treeitem title="How HP is building its next-generation footwear personalization platform on Istio." href=/v1.3/blog/2018/hp/>Istio a Game Changer for HP's FitStation Platform</a></li><li role=none><a role=treeitem title="Automatic application onboarding and latency optimizations using AppSwitch." href=/v1.3/blog/2018/delayering-istio/>Delayering Istio with AppSwitch</a></li><li role=none><a role=treeitem title="Describe Istio's authorization feature and how to use it in various use cases." href=/v1.3/blog/2018/istio-authorization/>Micro-Segmentation with Istio Authorization</a></li><li role=none><a role=treeitem title="How to export Istio Access Logs to different sinks like BigQuery, GCS, Pub/Sub through Stackdriver." href=/v1.3/blog/2018/export-logs-through-stackdriver/>Exporting Logs to BigQuery, GCS, Pub/Sub through Stackdriver</a></li><li role=none><a role=treeitem title="Describes how to configure Istio for monitoring and access policies of HTTP egress traffic." href=/v1.3/blog/2018/egress-monitoring-access-control/>Monitoring and Access Policies for HTTP Egress Traffic</a></li><li role=none><a role=treeitem title="Introduction, motivation and design principles for the Istio v1alpha3 routing API." href=/v1.3/blog/2018/v1alpha3-routing/>Introducing the Istio v1alpha3 routing API</a></li><li role=none><a role=treeitem title="Describes how to configure Istio ingress with a network load balancer on AWS." href=/v1.3/blog/2018/aws-nlb/>Configuring Istio Ingress with AWS NLB</a></li><li role=none><a role=treeitem title="Using Kubernetes namespaces and RBAC to create an Istio soft multi-tenancy environment." href=/v1.3/blog/2018/soft-multitenancy/>Istio Soft Multi-Tenancy Support</a></li><li role=none><a role=treeitem title="An introduction to safer, lower-risk deployments and release to production." href=/v1.3/blog/2018/traffic-mirroring/>Traffic Mirroring with Istio for Testing in Production</a></li><li role=none><a role=treeitem title="Describes a simple scenario based on Istio's Bookinfo example." href=/v1.3/blog/2018/egress-tcp/>Consuming External TCP Services</a></li><li role=none><a role=treeitem title="Describes a simple scenario based on Istio's Bookinfo example." href=/v1.3/blog/2018/egress-https/>Consuming External Web Services</a></li></ul></div></div><div class=card><button class="header dynamic" id=card2 title="Blog posts for 2017." aria-controls=card2-body><svg class="icon"><use xlink:href="/v1.3/img/icons.svg#blog"/></svg>2017 Posts</button><div class=body aria-labelledby=card2 role=region id=card2-body><ul role=tree aria-expanded=true class=leaf-section aria-labelledby=card2><li role=none><a role=treeitem title="Improving availability and reducing latency." href=/v1.3/blog/2017/mixer-spof-myth/>Mixer and the SPOF Myth</a></li><li role=none><a role=treeitem title="Provides an overview of Mixer's plug-in architecture." href=/v1.3/blog/2017/adapter-model/>Mixer Adapter Model</a></li><li role=none><a role=treeitem title="How Kubernetes Network Policy relates to Istio policy." href=/v1.3/blog/2017/0.1-using-network-policy/>Using Network Policy with Istio</a></li><li role=none><a role=treeitem title="Using Istio to create autoscaled canary deployments." href=/v1.3/blog/2017/0.1-canary/>Canary Deployments using Istio</a></li><li role=none><a role=treeitem title="Istio Auth 0.1 announcement." href=/v1.3/blog/2017/0.1-auth/>Using Istio to Improve End-to-End Security</a></li></ul></div></div></div></nav></div><div class=article-container><button tabindex=-1 id=sidebar-toggler title="Toggle the navigation bar"><svg class="icon"><use xlink:href="/v1.3/img/icons.svg#pull"/></svg></button><nav aria-label=Breadcrumb><ol><li><a href=/v1.3/ title="Connect, secure, control, and observe services.">Istio</a></li><li><a href=/v1.3/blog/ title="Posts about using Istio.">Blog</a></li><li><a href=/v1.3/blog/2019/ title="Blog posts for 2019.">2019 Posts</a></li><li>Monitoring blocked and passthrough external service traffic</li></ol></nav><article aria-labelledby=title><div class=title-area><div><h1 id=title>Monitoring blocked and passthrough external service traffic</h1><p class=byline><span>By</span>
|
|
<span class=attribution>Neeraj Poddar (Aspen Mesh)</span><span> | </span><span><svg class="icon"><use xlink:href="/v1.3/img/icons.svg#calendar"/></svg><span> </span>September 28, 2019</span><span> | </span><span title="1575 words"><svg class="icon"><use xlink:href="/v1.3/img/icons.svg#clock"/></svg><span> </span>8 minute read</span></p></div></div><nav class=toc-inlined aria-label="Table of Contents"><div><hr><ol><li role=none aria-label="External and internal services"><a href=#external-and-internal-services>External and internal services</a><li role=none aria-label="What are BlackHole and Passthrough clusters?"><a href=#what-are-blackhole-and-passthrough-clusters>What are BlackHole and Passthrough clusters?</a><li role=none aria-label="Using the augmented metrics"><a href=#using-the-augmented-metrics>Using the augmented metrics</a><ol><li role=none aria-label="PassthroughCluster metrics"><a href=#passthroughcluster-metrics>PassthroughCluster metrics</a><li role=none aria-label="BlackHoleCluster metrics"><a href=#blackholecluster-metrics>BlackHoleCluster metrics</a></ol></li><li role=none aria-label="See also"><a href=#see-also>See also</a></li></ol><hr></div></nav><p>Understanding, controlling and securing your external service access is one
|
|
of the key benefits that you get from a service mesh like Istio. From a security
|
|
and operations point of view, it is critical to monitor what external service traffic
|
|
is getting blocked as they might surface possible misconfigurations or a
|
|
security vulnerability if an application is attempting to communicate with a
|
|
service that it should not be allowed to. Similarly, if you currently have a
|
|
policy of allowing any external service access, it is beneficial to monitor
|
|
the traffic so you can incrementally add explicit Istio configuration to allow
|
|
access and better security your cluster. In either case, having visibility into this
|
|
traffic via telemetry is quite helpful as it enables you to create alerts and
|
|
dashboards, and better reason about your security posture. This was a highly
|
|
requested feature by production users of Istio and we are excited that the
|
|
support for this was added in release 1.3.</p><p>To implement this, the Istio <a href=/v1.3/docs/reference/config/policy-and-telemetry/metrics>default
|
|
metrics</a> are augmented with
|
|
explicit labels to capture blocked and passthrough external service traffic.
|
|
This blog will cover how you can use these augmented metrics to monitor all
|
|
external service traffic.</p><p>The Istio control plane configures the sidecar proxy with
|
|
predefined clusters called BlackHoleCluster and Passthrough which block or
|
|
allow all traffic respectively. To understand these clusters, let’s start with
|
|
what external and internal services mean in the context of Istio service mesh.</p><h2 id=external-and-internal-services>External and internal services</h2><p>Internal services are defined as services which are part of your platform
|
|
and are considered to be in the mesh. For internal services, Istio control
|
|
plane provides all the required configuration to the sidecars by default.
|
|
For example, in Kubernetes clusters, Istio configures the sidecars for all
|
|
Kubernetes services to preserve the default Kubernetes behavior of all
|
|
services being able to communicate with other.</p><p>External services are services which are not part of your platform i.e. services
|
|
which are outside of the mesh. For external services, Istio provides two
|
|
options, first to block all external service access (enabled by setting
|
|
<code>global.outboundTrafficPolicy.mode</code> to <code>REGISTRY_ONLY</code>) and
|
|
second to allow all access to external service (enabled by setting
|
|
<code>global.outboundTrafficPolicy.mode</code> to <code>ALLOW_ANY</code>). The default option for this
|
|
setting (as of Istio 1.3) is to allow all external service access. This
|
|
option can be configured via <a href=/v1.3/docs/reference/config/istio.mesh.v1alpha1/#MeshConfig-OutboundTrafficPolicy-Mode>mesh configuration</a>.</p><p>This is where the BlackHole and Passthrough clusters are used.</p><h2 id=what-are-blackhole-and-passthrough-clusters>What are BlackHole and Passthrough clusters?</h2><ul><li><strong>BlackHoleCluster</strong> - The BlackHoleCluster is a virtual cluster created
|
|
in the Envoy configuration when <code>global.outboundTrafficPolicy.mode</code> is set to
|
|
<code>REGISTRY_ONLY</code>. In this mode, all traffic to external service is blocked unless
|
|
<a href=/v1.3/docs/reference/config/networking/v1alpha3/service-entry>service entries</a>
|
|
are explicitly added for each service. To implement this, the default virtual
|
|
outbound listener at <code>0.0.0.0:15001</code> which uses
|
|
<a href=https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/service_discovery#original-destination>original destination</a>
|
|
is setup as a TCP Proxy with the BlackHoleCluster as the static cluster.
|
|
The configuration for the BlackHoleCluster looks like this:</li></ul><pre><code class=language-json data-expandlinks=true data-repo=istio>{
|
|
"name": "BlackHoleCluster",
|
|
"type": "STATIC",
|
|
"connectTimeout": "10s"
|
|
}
|
|
</code></pre><p>As you can see, this cluster is static with no endpoints so all the traffic
|
|
will be dropped. Additionally, Istio creates unique listeners for every
|
|
port/protocol combination of platform services which gets hit instead of the
|
|
virtual listener if the request is made to an external service on the same port.
|
|
In that case, the route configuration of every virtual route in Envoy is augmented to
|
|
add the BlackHoleCluster like this:</p><pre><code class=language-json data-expandlinks=true data-repo=istio>{
|
|
"name": "block_all",
|
|
"domains": [
|
|
"*"
|
|
],
|
|
"routes": [
|
|
{
|
|
"match": {
|
|
"prefix": "/"
|
|
},
|
|
"directResponse": {
|
|
"status": 502
|
|
}
|
|
}
|
|
]
|
|
}
|
|
</code></pre><p>The route is setup as <a href=https://www.envoyproxy.io/docs/envoy/latest/api-v2/api/v2/route/route.proto#envoy-api-msg-route-directresponseaction>direct response</a>
|
|
with <code>502</code> response code which means if no other routes match the Envoy proxy
|
|
will directly return a <code>502</code> HTTP status code.</p><ul><li><strong>PassthroughCluster</strong> - The PassthroughCluster is a virtual cluster created
|
|
in the Envoy configuration when <code>global.outboundTrafficPolicy.mode</code> is set to
|
|
<code>ALLOW_ANY</code>. In this mode, all traffic to any external service external is allowed.
|
|
To implement this, the default virtual outbound listener at <code>0.0.0.0:15001</code>
|
|
which uses <code>SO_ORIGINAL_DST</code>, is setup as a TCP Proxy with the PassthroughCluster
|
|
as the static cluster.
|
|
The configuration for the PassthroughCluster looks like this:</li></ul><pre><code class=language-json data-expandlinks=true data-repo=istio>{
|
|
"name": "PassthroughCluster",
|
|
"type": "ORIGINAL_DST",
|
|
"connectTimeout": "10s",
|
|
"lbPolicy": "ORIGINAL_DST_LB",
|
|
"circuitBreakers": {
|
|
"thresholds": [
|
|
{
|
|
"maxConnections": 102400,
|
|
"maxRetries": 1024
|
|
}
|
|
]
|
|
}
|
|
}
|
|
</code></pre><p>This cluster uses the <a href=https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/service_discovery#original-destination>original destination load balancing</a>
|
|
policy which configures Envoy to send the traffic to the
|
|
original destination i.e. passthrough.</p><p>Similar to the BlackHoleCluster, for every port/protocol based listener the
|
|
virtual route configuration is augmented to add the PassthroughCluster as the
|
|
default route:</p><pre><code class=language-json data-expandlinks=true data-repo=istio>{
|
|
"name": "allow_any",
|
|
"domains": [
|
|
"*"
|
|
],
|
|
"routes": [
|
|
{
|
|
"match": {
|
|
"prefix": "/"
|
|
},
|
|
"route": {
|
|
"cluster": "PassthroughCluster"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
</code></pre><p>Prior to Istio 1.3, there were no metrics reported or if metrics were reported
|
|
there were no explicit labels set when traffic hit these clusters, resulting in
|
|
lack of visibility in traffic flowing through the mesh.</p><p>The next section covers how to take advantage of this enhancement as the metrics
|
|
and labels emitted are conditional on whether the virtual outbound or explicit port/protocol
|
|
listener is being hit.</p><h2 id=using-the-augmented-metrics>Using the augmented metrics</h2><p>To capture all external service traffic in either of the cases (BlackHole or
|
|
Passthrough), you will need to monitor <code>istio_requests_total</code> and
|
|
<code>istio_tcp_connections_closed_total</code> metrics. Depending upon the Envoy listener
|
|
type i.e. TCP proxy or HTTP proxy that gets invoked, one of these metrics
|
|
will be incremented.</p><p>Additionally, in case of a TCP proxy listener in order to see the IP address of
|
|
the external service that is blocked or allowed via BlackHole or Passthrough
|
|
cluster, you will need to add the <code>destination_ip</code> label to the
|
|
<code>istio_tcp_connections_closed_total</code> metric. In this scenario, the host name of
|
|
the external service is not captured. This label is not added by default and can
|
|
be easily added by augmenting the Istio configuration for attribute generation
|
|
and Prometheus handler. You should be careful about cardinality explosion in
|
|
time series if you have many services with non-stable IP addresses.</p><h3 id=passthroughcluster-metrics>PassthroughCluster metrics</h3><p>This section explains the metrics and the labels emitted based on the listener
|
|
type invoked in Envoy.</p><ul><li>HTTP proxy listener: This happens when the port of the external service is
|
|
same as one of the service ports defined in the cluster. In this scenario,
|
|
when the PassthroughCluster is hit, <code>istio_requests_total</code> will get increased
|
|
like this:</li></ul><pre><code class=language-json data-expandlinks=true data-repo=istio>{
|
|
"metric": {
|
|
"__name__": "istio_requests_total",
|
|
"connection_security_policy": "unknown",
|
|
"destination_app": "unknown",
|
|
"destination_principal": "unknown",
|
|
"destination_service": "httpbin.org",
|
|
"destination_service_name": "PassthroughCluster",
|
|
"destination_service_namespace": "unknown",
|
|
"destination_version": "unknown",
|
|
"destination_workload": "unknown",
|
|
"destination_workload_namespace": "unknown",
|
|
"instance": "100.96.2.183:42422",
|
|
"job": "istio-mesh",
|
|
"permissive_response_code": "none",
|
|
"permissive_response_policyid": "none",
|
|
"reporter": "source",
|
|
"request_protocol": "http",
|
|
"response_code": "200",
|
|
"response_flags": "-",
|
|
"source_app": "sleep",
|
|
"source_principal": "unknown",
|
|
"source_version": "unknown",
|
|
"source_workload": "sleep",
|
|
"source_workload_namespace": "default"
|
|
},
|
|
"value": [
|
|
1567033080.282,
|
|
"1"
|
|
]
|
|
}
|
|
</code></pre><p>Note that the <code>destination_service_name</code> label is set to PassthroughCluster to
|
|
indicate that this cluster was hit and the <code>destination_service</code> is set to the
|
|
host of the external service.</p><ul><li>TCP proxy virtual listener - If the external service port doesn’t map to any
|
|
HTTP based service ports within the cluster, this listener is invoked and
|
|
<code>istio_tcp_connections_closed_total</code> is the metric that will be increased:</li></ul><pre><code class=language-json data-expandlinks=true data-repo=istio>{
|
|
"status": "success",
|
|
"data": {
|
|
"resultType": "vector",
|
|
"result": [
|
|
{
|
|
"metric": {
|
|
"__name__": "istio_tcp_connections_closed_total",
|
|
"connection_security_policy": "unknown",
|
|
"destination_app": "unknown",
|
|
"destination_ip": "52.22.188.80",
|
|
"destination_principal": "unknown",
|
|
"destination_service": "unknown",
|
|
"destination_service_name": "PassthroughCluster",
|
|
"destination_service_namespace": "unknown",
|
|
"destination_version": "unknown",
|
|
"destination_workload": "unknown",
|
|
"destination_workload_namespace": "unknown",
|
|
"instance": "100.96.2.183:42422",
|
|
"job": "istio-mesh",
|
|
"reporter": "source",
|
|
"response_flags": "-",
|
|
"source_app": "sleep",
|
|
"source_principal": "unknown",
|
|
"source_version": "unknown",
|
|
"source_workload": "sleep",
|
|
"source_workload_namespace": "default"
|
|
},
|
|
"value": [
|
|
1567033761.879,
|
|
"1"
|
|
]
|
|
}
|
|
]
|
|
}
|
|
}
|
|
</code></pre><p>In this case, <code>destination_service_name</code> is set to PassthroughCluster and
|
|
the <code>destination_ip</code> is set to the IP address of the external service.
|
|
The <code>destination_ip</code> label can be used to do a reverse DNS lookup and
|
|
get the host name of the external service. As this cluster is passthrough,
|
|
other TCP related metrics like <code>istio_tcp_connections_opened_total</code>,
|
|
<code>istio_tcp_received_bytes_total</code> and <code>istio_tcp_sent_bytes_total</code> are also
|
|
updated.</p><h3 id=blackholecluster-metrics>BlackHoleCluster metrics</h3><p>Similar to the PassthroughCluster, this section explains the metrics and the
|
|
labels emitted based on the listener type invoked in Envoy.</p><ul><li>HTTP proxy listener: This happens when the port of the external service is same
|
|
as one of the service ports defined in the cluster.
|
|
In this scenario, when the BlackHoleCluster is hit,
|
|
<code>istio_requests_total</code> will get increased like this:</li></ul><pre><code class=language-json data-expandlinks=true data-repo=istio>{
|
|
"metric": {
|
|
"__name__": "istio_requests_total",
|
|
"connection_security_policy": "unknown",
|
|
"destination_app": "unknown",
|
|
"destination_principal": "unknown",
|
|
"destination_service": "httpbin.org",
|
|
"destination_service_name": "BlackHoleCluster",
|
|
"destination_service_namespace": "unknown",
|
|
"destination_version": "unknown",
|
|
"destination_workload": "unknown",
|
|
"destination_workload_namespace": "unknown",
|
|
"instance": "100.96.2.183:42422",
|
|
"job": "istio-mesh",
|
|
"permissive_response_code": "none",
|
|
"permissive_response_policyid": "none",
|
|
"reporter": "source",
|
|
"request_protocol": "http",
|
|
"response_code": "502",
|
|
"response_flags": "-",
|
|
"source_app": "sleep",
|
|
"source_principal": "unknown",
|
|
"source_version": "unknown",
|
|
"source_workload": "sleep",
|
|
"source_workload_namespace": "default"
|
|
},
|
|
"value": [
|
|
1567034251.717,
|
|
"1"
|
|
]
|
|
}
|
|
</code></pre><p>Note the <code>destination_service_name</code> label is set to BlackHoleCluster and the
|
|
<code>destination_service</code> to the host name of the external service. The response
|
|
code should always be <code>502</code> in this case.</p><ul><li>TCP proxy virtual listener - If the external service port doesn’t map to any
|
|
HTTP based service ports within the cluster, this listener is invoked and
|
|
<code>istio_tcp_connections_closed_total</code> is the metric that will be increased:</li></ul><pre><code class=language-json data-expandlinks=true data-repo=istio>{
|
|
"metric": {
|
|
"__name__": "istio_tcp_connections_closed_total",
|
|
"connection_security_policy": "unknown",
|
|
"destination_app": "unknown",
|
|
"destination_ip": "52.22.188.80",
|
|
"destination_principal": "unknown",
|
|
"destination_service": "unknown",
|
|
"destination_service_name": "BlackHoleCluster",
|
|
"destination_service_namespace": "unknown",
|
|
"destination_version": "unknown",
|
|
"destination_workload": "unknown",
|
|
"destination_workload_namespace": "unknown",
|
|
"instance": "100.96.2.183:42422",
|
|
"job": "istio-mesh",
|
|
"reporter": "source",
|
|
"response_flags": "-",
|
|
"source_app": "sleep",
|
|
"source_principal": "unknown",
|
|
"source_version": "unknown",
|
|
"source_workload": "sleep",
|
|
"source_workload_namespace": "default"
|
|
},
|
|
"value": [
|
|
1567034481.03,
|
|
"1"
|
|
]
|
|
}
|
|
</code></pre><p>Note the <code>destination_ip</code> label represents the IP address of the external
|
|
service and the <code>destination_service_name</code> is set to BlackHoleCluster
|
|
to indicate that this traffic was blocked by the mesh. Is is interesting to
|
|
note that for the BlackHole cluster case, other TCP related metrics like
|
|
<code>istio_tcp_connections_opened_total</code> are not increased as there’s no
|
|
connection that is ever established.</p><p>Monitoring these metrics can help operators easily understand all the external
|
|
services consumed by the applications in their cluster.</p><nav id=see-also><h2>See also</h2><div class=see-also><div class=entry><p class=link><a data-skipendnotes=true href=/v1.3/blog/2018/egress-monitoring-access-control/>Monitoring and Access Policies for HTTP Egress Traffic</a></p><p class=desc>Describes how to configure Istio for monitoring and access policies of HTTP egress traffic.</p></div></div></nav></article><nav class=pagenav><div class=left><a title="Configure Istio ingress gateway to act as a proxy for external services." href=/v1.3/blog/2019/proxy/><svg class="icon"><use xlink:href="/v1.3/img/icons.svg#left-arrow"/></svg>Istio as a Proxy for External Services</a></div><div class=right><a title="Using Istio to secure multi-cloud Kubernetes applications with zero code changes." href=/v1.3/blog/2019/app-identity-and-access-adapter/>App Identity and Access Adapter<svg class="icon"><use xlink:href="/v1.3/img/icons.svg#right-arrow"/></svg></a></div></nav><div id=endnotes-container aria-hidden=true><h2>Links</h2><ol id=endnotes></ol></div></div><div class=toc-container><nav class=toc aria-label="Table of Contents"><div id=toc><ol><li role=none aria-label="External and internal services"><a href=#external-and-internal-services>External and internal services</a><li role=none aria-label="What are BlackHole and Passthrough clusters?"><a href=#what-are-blackhole-and-passthrough-clusters>What are BlackHole and Passthrough clusters?</a><li role=none aria-label="Using the augmented metrics"><a href=#using-the-augmented-metrics>Using the augmented metrics</a><ol><li role=none aria-label="PassthroughCluster metrics"><a href=#passthroughcluster-metrics>PassthroughCluster metrics</a><li role=none aria-label="BlackHoleCluster metrics"><a href=#blackholecluster-metrics>BlackHoleCluster metrics</a></ol></li><li role=none aria-label="See also"><a href=#see-also>See also</a></li></ol></div></nav></div></main><footer><div class=user-links><a class=channel title="Go download Istio 1.3.5 now" href=/v1.3/docs/setup#downloading-the-release aria-label="Download Istio"><span>download</span><svg class="icon"><use xlink:href="/v1.3/img/icons.svg#download"/></svg>
|
|
</a><a class=channel title="Join the Istio discussion board to participate in discussions and get help troubleshooting problems" href=https://discuss.istio.io aria-label="Istio discussion board"><span>discuss</span><svg class="icon"><use xlink:href="/v1.3/img/icons.svg#discourse"/></svg></a>
|
|
<a class=channel title="Stack Overflow is where you can ask questions and find curated answers on deploying, configuring, and using Istio" href=https://stackoverflow.com/questions/tagged/istio aria-label="Stack Overflow"><span>stack overflow</span><svg class="icon"><use xlink:href="/v1.3/img/icons.svg#stackoverflow"/></svg></a>
|
|
<a class=channel title="Interactively discuss issues with the Istio community on Slack" href=https://istio.slack.com aria-label=slack><span>slack</span><svg class="icon"><use xlink:href="/v1.3/img/icons.svg#slack"/></svg></a>
|
|
<a class=channel title="Follow us on Twitter to get the latest news" href=https://twitter.com/IstioMesh aria-label=Twitter><span>twitter</span><svg class="icon"><use xlink:href="/v1.3/img/icons.svg#twitter"/></svg></a><div class=tag>for everyone</div></div><div class=info><p class=copyright>Istio Archive
|
|
1.3.5<br>© 2019 Istio Authors, <a href=https://policies.google.com/privacy>Privacy Policy</a><br>Archived on November 14, 2019</p></div><div class=dev-links><a class=channel title="GitHub is where development takes place on Istio code" href=https://github.com/istio/community aria-label=GitHub><span>github</span><svg class="icon"><use xlink:href="/v1.3/img/icons.svg#github"/></svg></a>
|
|
<a class=channel title="Access our team drive if you'd like to take a look at the Istio technical design documents" href=https://groups.google.com/forum/#!forum/istio-team-drive-access aria-label="team drive"><span>drive</span><svg class="icon"><use xlink:href="/v1.3/img/icons.svg#drive"/></svg></a>
|
|
<a class=channel title="If you'd like to contribute to the Istio project, consider participating in our working groups" href=https://github.com/istio/community/blob/master/WORKING-GROUPS.md aria-label="working groups"><span>working groups</span><svg class="icon"><use xlink:href="/v1.3/img/icons.svg#working-groups"/></svg></a><div class=tag>for developers</div></div></footer><div id=scroll-to-top-container aria-hidden=true><button id=scroll-to-top title="Back to top"><svg class="icon"><use xlink:href="/v1.3/img/icons.svg#top"/></svg></button></div></body></html> |