istio.io/archive/v1.9/blog/2019/appswitch/index.html

<!doctype html><html lang=en itemscope itemtype=https://schema.org/WebPage><head><meta charset=utf-8><meta http-equiv=x-ua-compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=theme-color content="#466BB0"><meta name=title content="Sidestepping Dependency Ordering with AppSwitch"><meta name=description content="Addressing application startup ordering and startup latency using AppSwitch."><meta name=author content="Dinesh Subhraveti (AppOrbit and Columbia University)"><meta name=keywords content="microservices,services,mesh,appswitch,performance"><meta property="og:title" content="Sidestepping Dependency Ordering with AppSwitch"><meta property="og:type" content="website"><meta property="og:description" content="Addressing application startup ordering and startup latency using AppSwitch."><meta property="og:url" content="/v1.9/blog/2019/appswitch/"><meta property="og:image" content="/v1.9/img/istio-whitelogo-bluebackground-framed.svg"><meta property="og:image:alt" content="Istio Logo"><meta property="og:image:width" content="112"><meta property="og:image:height" content="150"><meta property="og:site_name" content="Istio"><meta name=twitter:card content="summary"><meta name=twitter:site content="@IstioMesh"><title>Istioldie 1.9 / Sidestepping Dependency Ordering with AppSwitch</title><script async src="https://www.googletagmanager.com/gtag/js?id=UA-98480406-2"></script><script>window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments);}
gtag('js',new Date());gtag('config','UA-98480406-2');</script><link rel=alternate type=application/rss+xml title="Istio Blog" href=/v1.9/blog/feed.xml><link rel=alternate type=application/rss+xml title="Istio News" href=/v1.9/news/feed.xml><link rel=alternate type=application/rss+xml title="Istio Blog and News" href=/v1.9/feed.xml><link rel="shortcut icon" href=/v1.9/favicons/favicon.ico><link rel=apple-touch-icon href=/v1.9/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/v1.9/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/v1.9/favicons/favicon-32x32.png sizes=32x32><link rel=icon type=image/png href=/v1.9/favicons/android-36x36.png sizes=36x36><link rel=icon type=image/png href=/v1.9/favicons/android-48x48.png sizes=48x48><link rel=icon type=image/png href=/v1.9/favicons/android-72x72.png sizes=72x72><link rel=icon type=image/png href=/v1.9/favicons/android-96x96.png sizes=96xW96><link rel=icon type=image/png href=/v1.9/favicons/android-144x144.png sizes=144x144><link rel=icon type=image/png href=/v1.9/favicons/android-192x192.png sizes=192x192><link rel=manifest href=/v1.9/manifest.json><meta name=apple-mobile-web-app-title content="Istio"><meta name=application-name content="Istio"><link rel=stylesheet href="https://fonts.googleapis.com/css?family=Work+Sans:400|Chivo:400|Work+Sans:500,300,600,300italic,400italic,500italic,600italic|Chivo:500,300,600,300italic,400italic,500italic,600italic"><link rel=stylesheet href=/v1.9/css/all.css><script src=/v1.9/js/themes_init.min.js></script></head><body class="language-unknown archive-site"><script>const branchName="release-1.9";const docTitle="Sidestepping Dependency Ordering with AppSwitch";const iconFile="\/v1.9/img/icons.svg";const buttonCopy='Copy to clipboard';const buttonPrint='Print';const buttonDownload='Download';</script><script src="https://www.google.com/cse/brand?form=search-form" defer></script><script src=/v1.9/js/all.min.js data-manual defer></script><header><nav><a id=brand href=/v1.9/><span class=logo><svg viewBox="0 0 300 300"><circle cx="150" cy="150" r="146" stroke-width="2"/><polygon points="65 240 225 240 125 270"/><polygon points="65 230 125 220 125 110"/><polygon points="135 220 225 230 135 30"/></svg></span><span class=name>Istioldie 1.9</span></a><div id=hamburger><svg class="icon hamburger"><use xlink:href="/v1.9/img/icons.svg#hamburger"/></svg></div><div id=header-links><a title="Learn how to deploy, use, and operate Istio." href=/v1.9/docs/>Docs</a>
<a class=current title="Posts about using Istio." href=/v1.9/blog/2021/>Blog<i class=dot data-prefix=/blog></i></a>
<a title="Timely news about the Istio project." href=/v1.9/news/>News<i class=dot data-prefix=/news></i></a>
<a title="Frequently Asked Questions about Istio." href=/v1.9/faq/>FAQ</a>
<a title="Get a bit more in-depth info about the Istio project." href=/v1.9/about/>About</a><div class=menu><button id=gearDropdownButton class=menu-trigger title="Options and settings" aria-label="Options and Settings" aria-controls=gearDropdownContent><svg class="icon gear"><use xlink:href="/v1.9/img/icons.svg#gear"/></svg></button><div id=gearDropdownContent class=menu-content aria-labelledby=gearDropdownButton role=menu><a tabindex=-1 role=menuitem lang=en id=switch-lang-en class=active>English</a>
<a tabindex=-1 role=menuitem lang=zh id=switch-lang-zh>中文</a><div role=separator></div><a tabindex=-1 role=menuitem class=active id=light-theme-item>Light Theme</a>
<a tabindex=-1 role=menuitem id=dark-theme-item>Dark Theme</a><div role=separator></div><a tabindex=-1 role=menuitem id=syntax-coloring-item>Color Examples</a><div role=separator></div><h6>Other versions of this site</h6><a tabindex=-1 role=menuitem onclick="navigateToUrlOrRoot('https://istio.io/blog\/2019\/appswitch\/');return false;">Current Release</a>
<a tabindex=-1 role=menuitem onclick="navigateToUrlOrRoot('https://preliminary.istio.io/blog\/2019\/appswitch\/');return false;">Next Release</a>
<a tabindex=-1 role=menuitem href=https://istio.io/archive>Older Releases</a></div></div><button id=search-show title="Search this site" aria-label=Search><svg class="icon magnifier"><use xlink:href="/v1.9/img/icons.svg#magnifier"/></svg></button></div><form id=search-form name=cse role=search><input type=hidden name=cx value=002184991200833970123:iwwf17ikgf4>
<input type=hidden name=ie value=utf-8>
<input type=hidden name=hl value=en>
<input type=hidden id=search-page-url value=/v1.9/search>
<input id=search-textbox class=form-control name=q type=search aria-label="Search this site">
<button id=search-close title="Cancel search" type=reset aria-label="Cancel search"><svg class="icon cancel-x"><use xlink:href="/v1.9/img/icons.svg#cancel-x"/></svg></button></form></nav></header><div class=banner-container></div><main class=primary><div id=sidebar-container class="sidebar-container sidebar-offcanvas"><nav id=sidebar aria-label="Section Navigation"><div class=directory><div class=card><button class="header dynamic" id=card0 title="Blog posts for 2021." aria-controls=card0-body><svg class="icon blog"><use xlink:href="/v1.9/img/icons.svg#blog"/></svg>2021 Posts</button><div class=body aria-labelledby=card0 role=region id=card0-body><ul role=tree aria-expanded=true class=leaf-section aria-labelledby=card0><li role=none><a role=treeitem title="The Product Security working group announces Patch Tuesdays, how 0-days and embargoes are handled, updates to the security best practices page and the notification of the early disclosure list (May 11, 2021)" href=/v1.9/blog/2021/patch-tuesdays/>Updates to how Istio security releases are handled: Patch Tuesday, embargoes, and 0-days</a></li><li role=none><a role=treeitem title="Learn how to use discovery selectors and how they intersect with Sidecar resources (April 30, 2021)" href=/v1.9/blog/2021/discovery-selectors/>Use discovery selectors to configure namespaces for your Istio service mesh</a></li><li role=none><a role=treeitem title="Understanding the upcoming changes to Istio networking, how they may impact your cluster, and what action to take (April 15, 2021)" href=/v1.9/blog/2021/upcoming-networking-changes/>Upcoming networking changes in Istio 1.10</a></li><li role=none><a role=treeitem title="An update on Envoy and Istio's WebAssembly-based extensibility effort (March 5, 2021)" href=/v1.9/blog/2021/wasm-progress/>Istio and Envoy WebAssembly Extensibility, One Year On</a></li><li role=none><a role=treeitem title="A tutorial to help customers migrate from the deprecated v1alpha1 security policy to the supported v1beta1 version (March 3, 2021)" href=/v1.9/blog/2021/migrate-alpha-policy/>Migrate pre-Istio 1.4 Alpha security policy to the current APIs</a></li><li role=none><a role=treeitem title="Understanding the benefits Istio brings, even when no configuration is used (February 25, 2021)" href=/v1.9/blog/2021/zero-config-istio/>Zero Configuration Istio</a></li><li role=none><a role=treeitem title="Learn about sessions, panels, workshops and more on the IstioCon website (February 16, 2021)" href=/v1.9/blog/2021/istiocon-2021-program/>IstioCon 2021: Schedule Is Live!</a></li><li role=none><a role=treeitem title="AuthorizationPolicy now supports CUSTOM action to delegate the authorization to external system (February 9, 2021)" href=/v1.9/blog/2021/better-external-authz/>Better External Authorization</a></li></ul></div></div><div class=card><button class="header dynamic" id=card1 title="Blog posts for 2020." aria-controls=card1-body><svg class="icon blog"><use xlink:href="/v1.9/img/icons.svg#blog"/></svg>2020 Posts</button><div class=body aria-labelledby=card1 role=region id=card1-body><ul role=tree aria-expanded=true class=leaf-section aria-labelledby=card1><li role=none><a role=treeitem title="Deploy multiple Istio egress gateways independently to have fine-grained control of egress communication from the mesh (December 16, 2020)" href=/v1.9/blog/2020/proxying-legacy-services-using-egress-gateways/>Proxying legacy services using Istio egress gateways</a></li><li role=none><a role=treeitem title="How to enable proxy protocol on AWS NLB and Istio ingress gateway (December 11, 2020)" href=/v1.9/blog/2020/show-source-ip/>Proxy protocol on AWS NLB and Istio ingress gateway</a></li><li role=none><a role=treeitem title="The inaugural conference for Istio will take place at the end of February (December 8, 2020)" href=/v1.9/blog/2020/istiocon-2021/>Join us for the first IstioCon in 2021!</a></li><li role=none><a role=treeitem title="How to ensure your clusters are not impacted by Docker Hub rate limiting (December 7, 2020)" href=/v1.9/blog/2020/docker-rate-limit/>Handling Docker Hub rate limiting</a></li><li role=none><a role=treeitem title="Workload Local DNS resolution to simplify VM integration, multicluster, and more (November 12, 2020)" href=/v1.9/blog/2020/dns-proxy/>Expanding into New Frontiers - Smart DNS Proxying in Istio</a></li><li role=none><a role=treeitem title="Announcing the four newest Istio Steering Committee members (September 29, 2020)" href=/v1.9/blog/2020/steering-election-results/>2020 Steering Committee Election Results</a></li><li role=none><a role=treeitem title="The effect of security policies on latency of requests (September 15, 2020)" href=/v1.9/blog/2020/large-scale-security-policy-performance-tests/>Large Scale Security Policy Performance Tests</a></li><li role=none><a role=treeitem title="A new deployment model for Istio (August 27, 2020)" href=/v1.9/blog/2020/new-deployment-model/>Deploying Istio Control Planes Outside the Mesh</a></li><li role=none><a role=treeitem title="The Istio Steering Committee is now in part proportionally allocated to companies based on contribution, and in part elected by community members (August 24, 2020)" href=/v1.9/blog/2020/steering-changes/>Introducing the new Istio steering committee</a></li><li role=none><a role=treeitem title="An alternative sidecar proxy for Istio (July 28, 2020)" href=/v1.9/blog/2020/mosn-proxy/>Using MOSN with Istio: an alternative data plane</a></li><li role=none><a role=treeitem title="An update on trademarks and project governance (July 8, 2020)" href=/v1.9/blog/2020/open-usage/>Open and neutral: transferring our trademarks to the Open Usage Commons</a></li><li role=none><a role=treeitem title="A new way to manage installation of telemetry addons (June 4, 2020)" href=/v1.9/blog/2020/addon-rework/>Reworking our Addon Integrations</a></li><li role=none><a role=treeitem title="Describing the new functionality of Workload Entries (May 21, 2020)" href=/v1.9/blog/2020/workload-entry/>Introducing Workload Entries</a></li><li role=none><a role=treeitem title="Simplifying Istio upgrades by offering safe canary deployments of the control plane (May 19, 2020)" href=/v1.9/blog/2020/multiple-control-planes/>Safely Upgrade Istio using a Canary Control Plane Deployment</a></li><li role=none><a role=treeitem title="Configure the IBM Cloud Kubernetes Service Application Load Balancer to direct traffic to the Istio Ingress gateway with mutual TLS (May 15, 2020)" href=/v1.9/blog/2020/alb-ingress-gateway-iks/>Direct encrypted traffic from IBM Cloud Kubernetes Service Ingress to Istio Ingress Gateway</a></li><li role=none><a role=treeitem title="Community partner tooling of Wasm for Istio by Solo.io (March 25, 2020)" href=/v1.9/blog/2020/wasmhub-istio/>Extended and Improved WebAssemblyHub to Bring the Power of WebAssembly to Envoy and Istio</a></li><li role=none><a role=treeitem title="A mechanism to acquire and share an application certificate and key through mounted files (March 25, 2020)" href=/v1.9/blog/2020/proxy-cert/>Provision a certificate and key for an application without sidecars</a></li><li role=none><a role=treeitem title="Istiod consolidates the Istio control plane components into a single binary (March 19, 2020)" href=/v1.9/blog/2020/istiod/>Introducing istiod: simplifying the control plane</a></li><li role=none><a role=treeitem title="Configuring Wasm extensions for Envoy and Istio declaratively (March 16, 2020)" href=/v1.9/blog/2020/deploy-wasm-declarative/>Declarative WebAssembly deployment for Istio</a></li><li role=none><a role=treeitem title="The future of Istio extensibility using WASM (March 5, 2020)" href=/v1.9/blog/2020/wasm-announce/>Redefining extensibility in proxies - introducing WebAssembly to Envoy and Istio</a></li><li role=none><a role=treeitem title="A vision statement and roadmap for Istio in 2020 (March 3, 2020)" href=/v1.9/blog/2020/tradewinds-2020/>Istio in 2020 - Following the Trade Winds</a></li><li role=none><a role=treeitem title="A more secure way to manage secrets (February 20, 2020)" href=/v1.9/blog/2020/istio-agent/>Remove cross-pod unix domain sockets</a></li><li role=none><a role=treeitem title="Automating Istio configuration for Istio deployments (clusters) that work as a single mesh (January 5, 2020)" href=/v1.9/blog/2020/multi-cluster-mesh-automation/>Multicluster Istio configuration and service discovery using Admiral</a></li></ul></div></div><div class=card><button class="header dynamic" id=card2 title="Blog posts for 2019." aria-controls=card2-body><svg class="icon blog"><use xlink:href="/v1.9/img/icons.svg#blog"/></svg>2019 Posts</button><div class="body default" aria-labelledby=card2 role=region id=card2-body><ul role=tree aria-expanded=true class=leaf-section aria-labelledby=card2><li role=none><a role=treeitem title="Provision and manage DNS certificates in Istio (November 14, 2019)" href=/v1.9/blog/2019/dns-cert/>DNS Certificate Management</a></li><li role=none><a role=treeitem title="Analyze your Istio configuration to detect potential issues and get general insights (November 14, 2019)" href=/v1.9/blog/2019/introducing-istioctl-analyze/>Introducing istioctl analyze</a></li><li role=none><a role=treeitem title="Introduction to Istio's new operator-based installation and control plane management feature (November 14, 2019)" href=/v1.9/blog/2019/introducing-istio-operator/>Introducing the Istio Operator</a></li><li role=none><a role=treeitem title="Introduction, motivation and design principles for the Istio v1beta1 Authorization Policy (November 14, 2019)" href=/v1.9/blog/2019/v1beta1-authorization-policy/>Introducing the Istio v1beta1 Authorization Policy</a></li><li role=none><a role=treeitem title="Getting programmatic access to Istio resources (November 14, 2019)" href=/v1.9/blog/2019/announcing-istio-client-go/>Announcing Istio client-go</a></li><li role=none><a role=treeitem title="A more secure way to manage Istio webhooks (November 14, 2019)" href=/v1.9/blog/2019/webhook/>Secure Webhook Management</a></li><li role=none><a role=treeitem title="Configure Istio ingress gateway to act as a proxy for external services (October 15, 2019)" href=/v1.9/blog/2019/proxy/>Istio as a Proxy for External Services</a></li><li role=none><a role=treeitem title="Deploy environments that require isolation into separate meshes and enable inter-mesh communication by mesh federation (October 2, 2019)" href=/v1.9/blog/2019/isolated-clusters/>Multi-Mesh Deployments for Isolation and Boundary Protection</a></li><li role=none><a role=treeitem title="How can you use Istio to monitor blocked and passthrough external traffic (September 28, 2019)" href=/v1.9/blog/2019/monitoring-external-service-traffic/>Monitoring Blocked and Passthrough External Service Traffic</a></li><li role=none><a role=treeitem title="Using Istio to secure multi-cloud Kubernetes applications with zero code changes (September 18, 2019)" href=/v1.9/blog/2019/app-identity-and-access-adapter/>App Identity and Access Adapter</a></li><li role=none><a role=treeitem title="Demonstrates a Mixer out-of-process adapter which implements the Knative scale-from-zero logic (September 18, 2019)" href=/v1.9/blog/2019/knative-activator-adapter/>Mixer Adapter for Knative</a></li><li role=none><a role=treeitem title="Taking advantage of Kubernetes trustworthy JWTs to issue certificates for workload instances more securely (September 10, 2019)" href=/v1.9/blog/2019/trustworthy-jwt-sds/>Change in Secret Discovery Service in Istio 1.3</a></li><li role=none><a role=treeitem title="The design principles behind Istio's APIs and how those APIs are evolving (August 5, 2019)" href=/v1.9/blog/2019/evolving-istios-apis/>The Evolution of Istio's APIs</a></li><li role=none><a role=treeitem title="Comparison of alternative solutions to control egress traffic including performance considerations (July 22, 2019)" href=/v1.9/blog/2019/egress-traffic-control-in-istio-part-3/>Secure Control of Egress Traffic in Istio, part 3</a></li><li role=none><a role=treeitem title="Use Istio Egress Traffic Control to prevent attacks involving egress traffic (July 10, 2019)" href=/v1.9/blog/2019/egress-traffic-control-in-istio-part-2/>Secure Control of Egress Traffic in Istio, part 2</a></li><li role=none><a role=treeitem title="Tools and guidance for evaluating Istio's data plane performance (July 9, 2019)" href=/v1.9/blog/2019/performance-best-practices/>Best Practices: Benchmarking Service Mesh Performance</a></li><li role=none><a role=treeitem title="Learn how to extend the lifetime of Istio self-signed root certificate (June 7, 2019)" href=/v1.9/blog/2019/root-transition/>Extending Istio Self-Signed Root Certificate Lifetime</a></li><li role=none><a role=treeitem title="Attacks involving egress traffic and requirements for egress traffic control (May 22, 2019)" href=/v1.9/blog/2019/egress-traffic-control-in-istio-part-1/>Secure Control of Egress Traffic in Istio, part 1</a></li><li role=none><a role=treeitem title="An overview of Istio 1.1 performance (March 19, 2019)" href=/v1.9/blog/2019/istio1.1_perf/>Architecting Istio 1.1 for Performance</a></li><li role=none><a role=treeitem title="Configuring Istio route rules in a multicluster service mesh (February 7, 2019)" href=/v1.9/blog/2019/multicluster-version-routing/>Version Routing in a Multicluster Service Mesh</a></li><li role=none><a role=treeitem title="Announces the new Istio blog policy (February 5, 2019)" href=/v1.9/blog/2019/sail-the-blog/>Sail the Blog!</a></li><li role=none><a role=treeitem title="De-mystify how Istio manages to plugin its data-plane components into an existing deployment (January 31, 2019)" href=/v1.9/blog/2019/data-plane-setup/>Demystifying Istio's Sidecar Injection Model</a></li><li role=none><a role=treeitem title="Verifies the performance impact of adding an egress gateway (January 31, 2019)" href=/v1.9/blog/2019/egress-performance/>Egress Gateway Performance Investigation</a></li><li role=none><span role=treeitem class=current title="Addressing application startup ordering and startup latency using AppSwitch (January 14, 2019)">Sidestepping Dependency Ordering with AppSwitch</span></li><li role=none><a role=treeitem title="Describes how to deploy a custom ingress gateway using cert-manager manually (January 10, 2019)" href=/v1.9/blog/2019/custom-ingress-gateway/>Deploy a Custom Ingress Gateway Using Cert-Manager</a></li><li role=none><a role=treeitem title="Istio has a new discussion board (January 10, 2019)" href=/v1.9/blog/2019/announcing-discuss.istio.io/>Announcing discuss.istio.io</a></li></ul></div></div><div class=card><button class="header dynamic" id=card3 title="Blog posts for 2018." aria-controls=card3-body><svg class="icon blog"><use xlink:href="/v1.9/img/icons.svg#blog"/></svg>2018 Posts</button><div class=body aria-labelledby=card3 role=region id=card3-body><ul role=tree aria-expanded=true class=leaf-section aria-labelledby=card3><li role=none><a role=treeitem title="How to use Istio for traffic management without deploying sidecar proxies (November 21, 2018)" href=/v1.9/blog/2018/incremental-traffic-management/>Incremental Istio Part 1, Traffic Management</a></li><li role=none><a role=treeitem title="Describes a simple scenario based on Istio's Bookinfo example (November 16, 2018)" href=/v1.9/blog/2018/egress-mongo/>Consuming External MongoDB Services</a></li><li role=none><a role=treeitem title="Istio hosting an all day Twitch stream to celebrate the 1.0 release (August 3, 2018)" href=/v1.9/blog/2018/istio-twitch-stream/>All Day Istio Twitch Stream</a></li><li role=none><a role=treeitem title="How HP is building its next-generation footwear personalization platform on Istio (July 31, 2018)" href=/v1.9/blog/2018/hp/>Istio a Game Changer for HP's FitStation Platform</a></li><li role=none><a role=treeitem title="Automatic application onboarding and latency optimizations using AppSwitch (July 30, 2018)" href=/v1.9/blog/2018/delayering-istio/>Delayering Istio with AppSwitch</a></li><li role=none><a role=treeitem title="Describe Istio's authorization feature and how to use it in various use cases (July 20, 2018)" href=/v1.9/blog/2018/istio-authorization/>Micro-Segmentation with Istio Authorization</a></li><li role=none><a role=treeitem title="How to export Istio Access Logs to different sinks like BigQuery, GCS, Pub/Sub through Stackdriver (July 9, 2018)" href=/v1.9/blog/2018/export-logs-through-stackdriver/>Exporting Logs to BigQuery, GCS, Pub/Sub through Stackdriver</a></li><li role=none><a role=treeitem title="Describes how to configure Istio for monitoring and access policies of HTTP egress traffic (June 22, 2018)" href=/v1.9/blog/2018/egress-monitoring-access-control/>Monitoring and Access Policies for HTTP Egress Traffic</a></li><li role=none><a role=treeitem title="Introduction, motivation and design principles for the Istio v1alpha3 routing API (April 25, 2018)" href=/v1.9/blog/2018/v1alpha3-routing/>Introducing the Istio v1alpha3 routing API</a></li><li role=none><a role=treeitem title="Describes how to configure Istio ingress with a network load balancer on AWS (April 20, 2018)" href=/v1.9/blog/2018/aws-nlb/>Configuring Istio Ingress with AWS NLB</a></li><li role=none><a role=treeitem title="Using Kubernetes namespaces and RBAC to create an Istio soft multi-tenancy environment (April 19, 2018)" href=/v1.9/blog/2018/soft-multitenancy/>Istio Soft Multi-Tenancy Support</a></li><li role=none><a role=treeitem title="An introduction to safer, lower-risk deployments and release to production (February 8, 2018)" href=/v1.9/blog/2018/traffic-mirroring/>Traffic Mirroring with Istio for Testing in Production</a></li><li role=none><a role=treeitem title="Describes a simple scenario based on Istio's Bookinfo example (February 6, 2018)" href=/v1.9/blog/2018/egress-tcp/>Consuming External TCP Services</a></li><li role=none><a role=treeitem title="Describes a simple scenario based on Istio's Bookinfo example (January 31, 2018)" href=/v1.9/blog/2018/egress-https/>Consuming External Web Services</a></li></ul></div></div><div class=card><button class="header dynamic" id=card4 title="Blog posts for 2017." aria-controls=card4-body><svg class="icon blog"><use xlink:href="/v1.9/img/icons.svg#blog"/></svg>2017 Posts</button><div class=body aria-labelledby=card4 role=region id=card4-body><ul role=tree aria-expanded=true class=leaf-section aria-labelledby=card4><li role=none><a role=treeitem title="Improving availability and reducing latency (December 7, 2017)" href=/v1.9/blog/2017/mixer-spof-myth/>Mixer and the SPOF Myth</a></li><li role=none><a role=treeitem title="Provides an overview of Mixer's plug-in architecture (November 3, 2017)" href=/v1.9/blog/2017/adapter-model/>Mixer Adapter Model</a></li><li role=none><a role=treeitem title="How Kubernetes Network Policy relates to Istio policy (August 10, 2017)" href=/v1.9/blog/2017/0.1-using-network-policy/>Using Network Policy with Istio</a></li><li role=none><a role=treeitem title="Using Istio to create autoscaled canary deployments (June 14, 2017)" href=/v1.9/blog/2017/0.1-canary/>Canary Deployments using Istio</a></li><li role=none><a role=treeitem title="Istio Authentication 0.1 announcement (May 25, 2017)" href=/v1.9/blog/2017/0.1-auth/>Using Istio to Improve End-to-End Security</a></li></ul></div></div></div></nav></div><div class=article-container><button tabindex=-1 id=sidebar-toggler title="Toggle the navigation bar"><svg class="icon pull"><use xlink:href="/v1.9/img/icons.svg#pull"/></svg></button><nav aria-label=Breadcrumb><ol><li><a href=/v1.9/ title="Connect, secure, control, and observe services.">Istio</a></li><li><a href=/v1.9/blog/ title="Posts about using Istio.">Blog</a></li><li><a href=/v1.9/blog/2019/ title="Blog posts for 2019.">2019 Posts</a></li><li>Sidestepping Dependency Ordering with AppSwitch</li></ol></nav><article aria-labelledby=title><div class=title-area><div style=width:100%><h1 id=title>Sidestepping Dependency Ordering with AppSwitch</h1><p class=byline><span>By</span>
<span class=attribution>Dinesh Subhraveti (AppOrbit and Columbia University)</span><span> | </span><span><svg class="icon calendar"><use xlink:href="/v1.9/img/icons.svg#calendar"/></svg><span>&nbsp;</span>January 14, 2019</span><span> | </span><span title="2749 words"><svg class="icon clock"><use xlink:href="/v1.9/img/icons.svg#clock"/></svg><span>&nbsp;</span>13 minute read</span>
<span>&nbsp;</span>
<span></span></p></div></div><nav class=toc-inlined aria-label="Table of Contents"><div><hr><ol><li role=none aria-label="Dependency ordering problem"><a href=#dependency-ordering-problem>Dependency ordering problem</a><ol><li role=none aria-label="Example scenario: IBM WebSphere ND"><a href=#example-scenario-ibm-websphere-nd>Example scenario: IBM WebSphere ND</a><li role=none aria-label="Sidecar dependency in Istio"><a href=#sidecar-dependency-in-istio>Sidecar dependency in Istio</a></ol></li><li role=none aria-label="Dependency ordering with AppSwitch"><a href=#dependency-ordering-with-appswitch>Dependency ordering with AppSwitch</a><ol><li role=none aria-label="AppSwitch model and constructs"><a href=#appswitch-model-and-constructs>AppSwitch model and constructs</a><li role=none aria-label="Decoupling services from their references"><a href=#decoupling-services-from-their-references>Decoupling services from their references</a><li role=none aria-label="Non-blocking requests"><a href=#non-blocking-requests>Non-blocking requests</a><li role=none aria-label="Application timeouts"><a href=#application-timeouts>Application timeouts</a><li role=none aria-label="Wildcard service references for sidecar dependency"><a href=#wildcard-service-references-for-sidecar-dependency>Wildcard service references for sidecar dependency</a></ol></li><li role=none aria-label=Summary><a href=#summary>Summary</a><li role=none aria-label=Acknowledgements><a href=#acknowledgements>Acknowledgements</a><li role=none aria-label="See also"><a href=#see-also>See also</a></li></ol><hr></div></nav><div><aside class="callout warning"><div class=type><svg class="large-icon"><use xlink:href="/v1.9/img/icons.svg#callout-warning"/></svg></div><div class=content>This blog post was written assuming Istio 1, so some of this content may now be outdated.</div></aside></div><p>We are going through an interesting cycle of application decomposition and recomposition. While the microservice paradigm is driving monolithic applications to be broken into separate individual services, the service mesh approach is helping them to be connected back together into well-structured applications. As such, microservices are logically separate but not independent. They are usually closely interdependent and taking them apart introduces many new concerns such as need for mutual authentication between services. Istio directly addresses most of those issues.</p><h2 id=dependency-ordering-problem>Dependency ordering problem</h2><p>An issue that arises due to application decomposition and one that Istio doesn’t address is dependency ordering &ndash; bringing up individual services of an application in an order that guarantees that the application as a whole comes up quickly and correctly. In a monolithic application, with all its components built-in, dependency ordering between the components is enforced by internal locking mechanisms. But with individual services potentially scattered across the cluster in a service mesh, starting a service first requires checking that the services it depends on are up and available.</p><p>Dependency ordering is deceptively nuanced with a host of interrelated problems. Ordering individual services requires having the dependency graph of the services so that they can be brought up starting from leaf nodes back to the root nodes. It is not easy to construct such a graph and keep it updated over time as interdependencies evolve with the behavior of the application. Even if the dependency graph is somehow provided, enforcing the ordering itself is not easy. Simply starting the services in the specified order obviously won’t do. A service may have started but not be ready to accept connections yet. This is the problem with docker-compose&rsquo;s <code>depends-on</code> tag, for example.</p><p>Apart from introducing sufficiently long sleeps between service startups, a common pattern that is often used is to check for readiness of dependencies before starting a service. In Kubernetes, this could be done with a wait script as part of the init container of the pod. However that means that the entire application would be held up until all its dependencies come alive. Sometimes applications spend several minutes initializing themselves on startup before making their first outbound connection. Not allowing a service to start at all adds substantial overhead to overall startup time of the application. Also, the strategy of waiting on the init container won&rsquo;t work for the case of multiple interdependent services within the same pod.</p><h3 id=example-scenario-ibm-websphere-nd>Example scenario: IBM WebSphere ND</h3><p>Let us consider IBM WebSphere ND &ndash; a widely deployed application middleware &ndash; to grok these problems more closely. It is a fairly complex framework in itself and consists of a central component called deployment manager (<code>dmgr</code>) that manages a set of node instances. It uses UDP to negotiate cluster membership among the nodes and requires that deployment manager is up and operational before any of the node instances can come up and join the cluster.</p><p>Why are we talking about a traditional application in the modern cloud-native context? It turns out that there are significant gains to be had by enabling them to run on the Kubernetes and Istio platforms. Essentially it&rsquo;s a part of the modernization journey that allows running traditional apps alongside green-field apps on the same modern platform to facilitate interoperation between the two. In fact, WebSphere ND is a demanding application. It expects a consistent network environment with specific network interface attributes etc. AppSwitch is equipped to take care of those requirements. For the purpose of this blog however, I&rsquo;ll focus on the dependency ordering requirement and how AppSwitch addresses it.</p><p>Simply deploying <code>dmgr</code> and node instances as pods on a Kubernetes cluster does not work. <code>dmgr</code> and the node instances happen to have a lengthy initialization process that can take several minutes. If they are all co-scheduled, the application typically ends up in a funny state. When a node instance comes up and finds that <code>dmgr</code> is missing, it would take an alternate startup path. Instead, if it had exited immediately, Kubernetes crash-loop would have taken over and perhaps the application would have come up. But even in that case, it turns out that a timely startup is not guaranteed.</p><p>One <code>dmgr</code> along with its node instances is a basic deployment configuration for WebSphere ND. Applications like IBM Business Process Manager that are built on top of WebSphere ND running in production environments include several other services. In those configurations, there could be a chain of interdependencies. Depending on the applications hosted by the node instances, there may be an ordering requirement among them as well. With long service initialization times and crash-loop restarts, there is little chance for the application to start in any reasonable length of time.</p><h3 id=sidecar-dependency-in-istio>Sidecar dependency in Istio</h3><p>Istio itself is affected by a version of the dependency ordering problem. Since connections into and out of a service running under Istio are redirected through its sidecar proxy, an implicit dependency is created between the application service and its sidecar. Unless the sidecar is fully operational, all requests from and to the service get dropped.</p><h2 id=dependency-ordering-with-appswitch>Dependency ordering with AppSwitch</h2><p>So how do we go about addressing these issues? One way is to defer it to the applications and say that they are supposed to be &ldquo;well behaved&rdquo; and implement appropriate logic to make themselves immune to startup order issues. However, many applications (especially traditional ones) either timeout or deadlock if misordered. Even for new applications, implementing one off logic for each service is substantial additional burden that is best avoided. Service mesh needs to provide adequate support around these problems. After all, factoring out common patterns into an underlying framework is really the point of service mesh.</p><p><a href=http://appswitch.io>AppSwitch</a> explicitly addresses dependency ordering. It sits on the control path of the application’s network interactions between clients and services in a cluster and knows precisely when a service becomes a client by making the <code>connect</code> call and when a particular service becomes ready to accept connections by making the <code>listen</code> call. It&rsquo;s <em>service router</em> component disseminates information about these events across the cluster and arbitrates interactions among clients and servers. That is how AppSwitch implements functionality such as load balancing and isolation in a simple and efficient manner. Leveraging the same strategic location of the application&rsquo;s network control path, it is conceivable that the <code>connect</code> and <code>listen</code> calls made by those services can be lined up at a finer granularity rather than coarsely sequencing entire services as per a dependency graph. That would effectively solve the multilevel dependency problem and speedup application startup.</p><p>But that still requires a dependency graph. A number of products and tools exist to help with discovering service dependencies. But they are typically based on passive monitoring of network traffic and cannot provide the information beforehand for any arbitrary application. Network level obfuscation due to encryption and tunneling also makes them unreliable. The burden of discovering and specifying the dependencies ultimately falls to the developer or the operator of the application. As it is, even consistency checking a dependency specification is itself quite complex and any way to avoid requiring a dependency graph would be most desirable.</p><p>The point of a dependency graph is to know which clients depend on a particular service so that those clients can then be made to wait for the respective service to become live. But does it really matter which specific clients? Ultimately one tautology that always holds is that all clients of a service have an implicit dependency on the service. That’s what AppSwitch leverages to get around the requirement. In fact, that sidesteps dependency ordering altogether. All services of the application can be co-scheduled without regard to any startup order. Interdependencies among them automatically work themselves out at the granularity of individual requests and responses, resulting in quick and correct application startups.</p><h3 id=appswitch-model-and-constructs>AppSwitch model and constructs</h3><p>Now that we have a conceptual understanding of AppSwitch’s high-level approach, let’s look at the constructs involved. But first a quick summary of the usage model is in order. Even though it is written for a different context, reviewing my earlier <a href=/v1.9/blog/2018/delayering-istio/>blog</a> on this topic would be useful as well. For completeness, let me also note AppSwitch doesn’t bother with non-network dependencies. For example it may be possible for two services to interact using IPC mechanisms or through the shared file system. Processes with deep ties like that are typically part of the same service anyway and don’t require framework’s intervention for ordering.</p><p>At its core, AppSwitch is built on a mechanism that allows instrumenting the BSD socket API and other related calls like <code>fcntl</code> and <code>ioctl</code> that deal with sockets. As interesting as the details of its implementation are, it’s going to distract us from the main topic, so I’d just summarize the key properties that distinguish it from other implementations. (1) It’s fast. It uses a combination of <code>seccomp</code> filtering and binary instrumentation to aggressively limit intervening with application’s normal execution. AppSwitch is particularly suited for service mesh and application networking use cases given that it implements those features without ever having to actually touch the data. In contrast, network level approaches incur per-packet cost. Take a look at this <a href=/v1.9/blog/2018/delayering-istio/>blog</a> for some of the performance measurements. (2) It doesn’t require any kernel support, kernel module or a patch and works on standard distro kernels (3) It can run as regular user (no root). In fact, the mechanism can even make it possible to run <a href=https://linuxpiter.com/en/materials/2478>Docker daemon without root</a> by removing root requirement to network containers (4) It doesn’t require any changes to the applications whatsoever and works for any type of application &ndash; from WebSphere ND and SAP to custom C apps to statically linked Go apps. Only requirement at this point is Linux/x86.</p><h3 id=decoupling-services-from-their-references>Decoupling services from their references</h3><p>AppSwitch is built on the fundamental premise that applications should be decoupled from their references. The identity of applications is traditionally derived from the identity of the host on which they run. However, applications and hosts are very different objects that need to be referenced independently. Detailed discussion around this topic along with a conceptual foundation of AppSwitch is presented in this <a href=https://arxiv.org/abs/1711.02294>research paper</a>.</p><p>The central AppSwitch construct that achieves the decoupling between services objects and their identities is <em>service reference</em> (<em>reference</em>, for short). AppSwitch implements service references based on the API instrumentation mechanism outlined above. A service reference consists of an IP:port pair (and optionally a DNS name) and a label-selector that selects the service represented by the reference and the clients to which this reference applies. A reference supports a few key properties. (1) It can be named independently of the name of the object it refers to. That is, a service may be listening on an IP and port but a reference allows that service to be reached on any other IP and port chosen by the user. This is what allows AppSwitch to run traditional applications captured from their source environments with static IP configurations to run on Kubernetes by providing them with necessary IP addresses and ports regardless of the target network environment. (2) It remains unchanged even if the location of the target service changes. A reference automatically redirects itself as its label-selector now resolves to the new instance of the service (3) Most important for this discussion, a reference remains valid even as the target service is coming up.</p><p>To facilitate discovering services that can be accessed through service references, AppSwitch provides an <em>auto-curated service registry</em>. The registry is automatically kept up to date as services come and go across the cluster based on the network API that AppSwitch tracks. Each entry in the registry consists of the IP and port where the respective service is bound. Along with that, it includes a set of labels indicating the application to which this service belongs, the IP and port that the application passed through the socket API when creating the service, the IP and port where AppSwitch actually bound the service on the underlying host on behalf of the application etc. In addition, applications created under AppSwitch carry a set of labels passed by the user that describe the application together with a few default system labels indicating the user that created the application and the host where the application is running etc. These labels are all available to be expressed in the label-selector carried by a service reference. A service in the registry can be made accessible to clients by creating a service reference. A client would then be able to reach the service at the reference’s name (IP:port). Now let’s look at how AppSwitch guarantees that the reference remains valid even when the target service has not yet come up.</p><h3 id=non-blocking-requests>Non-blocking requests</h3><p>AppSwitch leverages the semantics of the BSD socket API to ensure that service references appear valid from the perspective of clients as corresponding services come up. When a client makes a blocking connect call to another service that has not yet come up, AppSwitch blocks the call for a certain time waiting for the target service to become live. Since it is known that the target service is a part of the application and is expected to come up shortly, making the client block rather than returning an error such as <code>ECONNREFUSED</code> prevents the application from failing to start. If the service doesn’t come up within time, an error is returned to the application so that framework-level mechanisms like Kubernetes crash-loop can kick in.</p><p>If the client request is marked as non-blocking, AppSwitch handles that by returning <code>EAGAIN</code> to inform the application to retry rather than give up. Once again, that is in-line with the semantics of socket API and prevents failures due to startup races. AppSwitch essentially enables the retry logic already built into applications in support of the BSD socket API to be transparently repurposed for dependency ordering.</p><h3 id=application-timeouts>Application timeouts</h3><p>What if the application times out based on its own internal timer? Truth be told, AppSwitch can also fake application’s perception of time if wanted but that would be overstepping and actually unnecessary. Application decides and knows best how long it should wait and it’s not appropriate for AppSwitch to mess with that. Application timeouts are conservatively long and if the target service still hasn’t come up in time, it is unlikely to be a dependency ordering issue. There must be something else going on that should not be masked.</p><h3 id=wildcard-service-references-for-sidecar-dependency>Wildcard service references for sidecar dependency</h3><p>Service references can be used to address the Istio sidecar dependency issue mentioned earlier. AppSwitch allows the IP:port specified as part of a service reference to be a wildcard. That is, the service reference IP address can be a netmask indicating the IP address range to be captured. If the label selector of the service reference points to the sidecar service, then all outgoing connections of any application for which this service reference is applied, will be transparently redirected to the sidecar. And of course, the service reference remains valid while sidecar is still coming up and the race is removed.</p><p>Using service references for sidecar dependency ordering also implicitly redirects application’s connections to the sidecar without requiring iptables and attendant privilege issues. Essentially it works as if the application is directly making connections to the sidecar rather than the target destination, leaving the sidecar in charge of what to do. AppSwitch would interject metadata about the original destination etc. into the data stream of the connection using the proxy protocol that the sidecar could decode before passing the connection through to the application. Some of these details were discussed <a href=/v1.9/blog/2018/delayering-istio/>here</a>. That takes care of outbound connections but what about incoming connections? With all services and their sidecars running under AppSwitch, any incoming connections that would have come from remote nodes would be redirected to their respective remote sidecars. So nothing special to do about incoming connections.</p><h2 id=summary>Summary</h2><p>Dependency ordering is a pesky problem. This is mostly due to lack of access to fine-grain application-level events around inter-service interactions. Addressing this problem would have normally required applications to implement their own internal logic. But AppSwitch makes those internal application events to be instrumented without requiring application changes. AppSwitch then leverages the ubiquitous support for the BSD socket API to sidestep the requirement of ordering dependencies.</p><h2 id=acknowledgements>Acknowledgements</h2><p>Thanks to Eric Herness and team for their insights and support with IBM WebSphere and BPM products as we modernized them onto the Kubernetes platform and to Mandar Jog, Martin Taillefer and Shriram Rajagopalan for reviewing early drafts of this blog.</p><nav id=see-also><h2>See also</h2><div class=see-also><div class=entry><p class=link><a data-skipendnotes=true href=/v1.9/blog/2018/delayering-istio/>Delayering Istio with AppSwitch</a></p><p class=desc>Automatic application onboarding and latency optimizations using AppSwitch.</p></div><div class=entry><p class=link><a data-skipendnotes=true href=/v1.9/blog/2020/large-scale-security-policy-performance-tests/>Large Scale Security Policy Performance Tests</a></p><p class=desc>The effect of security policies on latency of requests.</p></div><div class=entry><p class=link><a data-skipendnotes=true href=/v1.9/blog/2020/wasmhub-istio/>Extended and Improved WebAssemblyHub to Bring the Power of WebAssembly to Envoy and Istio</a></p><p class=desc>Community partner tooling of Wasm for Istio by Solo.io.</p></div><div class=entry><p class=link><a data-skipendnotes=true href=/v1.9/blog/2020/wasm-announce/>Redefining extensibility in proxies - introducing WebAssembly to Envoy and Istio</a></p><p class=desc>The future of Istio extensibility using WASM.</p></div><div class=entry><p class=link><a data-skipendnotes=true href=/v1.9/blog/2020/tradewinds-2020/>Istio in 2020 - Following the Trade Winds</a></p><p class=desc>A vision statement and roadmap for Istio in 2020.</p></div><div class=entry><p class=link><a data-skipendnotes=true href=/v1.9/blog/2019/performance-best-practices/>Best Practices: Benchmarking Service Mesh Performance</a></p><p class=desc>Tools and guidance for evaluating Istio's data plane performance.</p></div></div></nav></article><nav class=pagenav><div class=left><a title="Verifies the performance impact of adding an egress gateway." href=/v1.9/blog/2019/egress-performance/><svg class="icon left-arrow"><use xlink:href="/v1.9/img/icons.svg#left-arrow"/></svg>Egress Gateway Performance Investigation</a></div><div class=right><a title="Istio has a new discussion board." href=/v1.9/blog/2019/announcing-discuss.istio.io/>Announcing discuss.istio.io<svg class="icon right-arrow"><use xlink:href="/v1.9/img/icons.svg#right-arrow"/></svg></a></div></nav><div id=feedback><div id=feedback-initial>Was this information useful?<br><button class="btn feedback" onclick="sendFeedback('en',1)">Yes</button>
<button class="btn feedback" onclick="sendFeedback('en',0)">No</button></div><div id=feedback-comment>Do you have any suggestions for improvement?<br><br><input id=feedback-textbox type=text placeholder="Help us improve..." data-lang=en></div><div id=feedback-thankyou>Thanks for your feedback!</div></div><div id=endnotes-container aria-hidden=true><h2>Links</h2><ol id=endnotes></ol></div></div><div class=toc-container><nav class=toc aria-label="Table of Contents"><div id=toc><ol><li role=none aria-label="Dependency ordering problem"><a href=#dependency-ordering-problem>Dependency ordering problem</a><ol><li role=none aria-label="Example scenario: IBM WebSphere ND"><a href=#example-scenario-ibm-websphere-nd>Example scenario: IBM WebSphere ND</a><li role=none aria-label="Sidecar dependency in Istio"><a href=#sidecar-dependency-in-istio>Sidecar dependency in Istio</a></ol></li><li role=none aria-label="Dependency ordering with AppSwitch"><a href=#dependency-ordering-with-appswitch>Dependency ordering with AppSwitch</a><ol><li role=none aria-label="AppSwitch model and constructs"><a href=#appswitch-model-and-constructs>AppSwitch model and constructs</a><li role=none aria-label="Decoupling services from their references"><a href=#decoupling-services-from-their-references>Decoupling services from their references</a><li role=none aria-label="Non-blocking requests"><a href=#non-blocking-requests>Non-blocking requests</a><li role=none aria-label="Application timeouts"><a href=#application-timeouts>Application timeouts</a><li role=none aria-label="Wildcard service references for sidecar dependency"><a href=#wildcard-service-references-for-sidecar-dependency>Wildcard service references for sidecar dependency</a></ol></li><li role=none aria-label=Summary><a href=#summary>Summary</a><li role=none aria-label=Acknowledgements><a href=#acknowledgements>Acknowledgements</a><li role=none aria-label="See also"><a href=#see-also>See also</a></li></ol></div></nav></div></main><footer><div class=user-links><a class=channel title="Go download Istio 1.9.5 now" href=/v1.9/docs/setup/getting-started/#download aria-label="Download Istio"><span>download</span><svg class="icon download"><use xlink:href="/v1.9/img/icons.svg#download"/></svg>
</a><a class=channel title="Join the Istio discussion board to participate in discussions and get help troubleshooting problems" href=https://discuss.istio.io aria-label="Istio discussion board"><span>discuss</span><svg class="icon discourse"><use xlink:href="/v1.9/img/icons.svg#discourse"/></svg></a>
<a class=channel title="Stack Overflow is where you can ask questions and find curated answers on deploying, configuring, and using Istio" href=https://stackoverflow.com/questions/tagged/istio aria-label="Stack Overflow"><span>stack overflow</span><svg class="icon stackoverflow"><use xlink:href="/v1.9/img/icons.svg#stackoverflow"/></svg></a>
<a class=channel title="Interactively discuss issues with the Istio community on Slack" href=https://slack.istio.io aria-label=slack><span>slack</span><svg class="icon slack"><use xlink:href="/v1.9/img/icons.svg#slack"/></svg></a>
<a class=channel title="Follow us on Twitter to get the latest news" href=https://twitter.com/IstioMesh aria-label=Twitter><span>twitter</span><svg class="icon twitter"><use xlink:href="/v1.9/img/icons.svg#twitter"/></svg></a><div class=tag>for everyone</div></div><div class=info><p class=copyright>Istio Archive
1.9.5<br>&copy; 2020 Istio Authors, <a href=https://policies.google.com/privacy>Privacy Policy</a><br>Archived on May 18, 2021</p></div><div class=dev-links><a class=channel title="GitHub is where development takes place on Istio code" href=https://github.com/istio/community aria-label=GitHub><span>github</span><svg class="icon github"><use xlink:href="/v1.9/img/icons.svg#github"/></svg></a>
<a class=channel title="Access our team drive if you'd like to take a look at the Istio technical design documents" href=https://groups.google.com/forum/#!forum/istio-team-drive-access aria-label="team drive"><span>drive</span><svg class="icon drive"><use xlink:href="/v1.9/img/icons.svg#drive"/></svg></a>
<a class=channel title="If you'd like to contribute to the Istio project, consider participating in our working groups" href=https://github.com/istio/community/blob/master/WORKING-GROUPS.md aria-label="working groups"><span>working groups</span><svg class="icon working-groups"><use xlink:href="/v1.9/img/icons.svg#working-groups"/></svg></a><div class=tag>for developers</div></div></footer><div id=scroll-to-top-container aria-hidden=true><button id=scroll-to-top title="Back to top"><svg class="icon top"><use xlink:href="/v1.9/img/icons.svg#top"/></svg></button></div></body></html>