istio.io/archive/v0.8/blog/2018/soft-multitenancy/index.html

1094 lines
54 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

<!DOCTYPE html>
<html lang="en" itemscope itemtype="https://schema.org/WebPage">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<meta name="theme-color" content="#466BB0"/>
<meta name="title" content="Istio Soft Multi-tenancy Support">
<meta name="description" content="Using Kubernetes namespace and RBAC to create an Istio soft multi-tenancy environment">
<meta name="og:title" content="Istio Soft Multi-tenancy Support">
<meta name="og:description" content="Using Kubernetes namespace and RBAC to create an Istio soft multi-tenancy environment">
<meta name="og:url" content="/v0.8/blog/2018/soft-multitenancy/">
<meta name="og.site_name" content="Istio">
<title>Istioldie 0.8 / Istio Soft Multi-tenancy Support</title>
<script>
window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
ga('create', 'UA-98480406-2', 'auto');
ga('send', 'pageview');
</script>
<script async src='https://www.google-analytics.com/analytics.js'></script>
<script>
var branchName = "release-0.8";
</script>
<link rel="alternate" type="application/rss+xml" title="Istio Blog" href="/v0.8/feed.xml">
<link rel="shortcut icon" href="/v0.8/favicons/favicon.ico" >
<link rel="apple-touch-icon" href="/v0.8/favicons/apple-touch-icon-180x180.png" sizes="180x180">
<link rel="icon" type="image/png" href="/v0.8/favicons/favicon-16x16.png" sizes="16x16">
<link rel="icon" type="image/png" href="/v0.8/favicons/favicon-32x32.png" sizes="32x32">
<link rel="icon" type="image/png" href="/v0.8/favicons/android-36x36.png" sizes="36x36">
<link rel="icon" type="image/png" href="/v0.8/favicons/android-48x48.png" sizes="48x48">
<link rel="icon" type="image/png" href="/v0.8/favicons/android-72x72.png" sizes="72x72">
<link rel="icon" type="image/png" href="/v0.8/favicons/android-96x196.png" sizes="96x196">
<link rel="icon" type="image/png" href="/v0.8/favicons/android-144x144.png" sizes="144x144">
<link rel="icon" type="image/png" href="/v0.8/favicons/android-192x192.png" sizes="192x192">
<link rel="manifest" href="/v0.8/manifest.json">
<meta name="apple-mobile-web-app-title" content="Istio">
<meta name="application-name" content="Istio">
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:400,100,100italic,300,300italic,400italic,500,500italic,700,700italic,900,900italic">
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.6/css/all.css">
<link rel="stylesheet" href="/v0.8/css/light_theme_archive.css" title="light">
<link rel="alternate stylesheet" href="/v0.8/css/dark_theme_archive.css" title="dark">
<script src="/v0.8/js/styleSwitcher.min.js"></script>
</head>
<body class="language-unknown">
<header>
<nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark justify-content-between">
<a class="navbar-brand" href="/v0.8/">
<span class="logo"><svg viewBox="0 0 300 300">
<circle cx="150" cy="150" r="150" stroke-width="2" />
<polygon points="65,240 225,240 125,270"/>
<polygon points="65,230 125,220 125,110"/>
<polygon points="135,220 225,230 135,30"/>
</svg>
</span>
<span class="brand-name">Istioldie 0.8</span>
</a>
<button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarCollapse" aria-controls="navbarCollapse" aria-expanded="false" aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
<div class="collapse navbar-collapse justify-content-end" id="navbarCollapse">
<ul id="navbar-links" class="navbar-nav active">
<li class="nav-item">
<a class="nav-link " href="/v0.8/docs/">Docs</a>
</li>
<li class="nav-item">
<a class="nav-link active" href="/v0.8/blog/2018/egress-monitoring-access-control/">Blog</a>
</li>
<li class="nav-item">
<a class="nav-link " href="/v0.8/help/">Help</a>
</li>
<li class="nav-item">
<a class="nav-link " href="/v0.8/community/">Community</a>
</li>
<li class="nav-item">
<a class="nav-link " href="/v0.8/about/">About</a>
</li>
<li class="nav-item dropdown" id="gearDropdown" style="white-space: nowrap">
<a href="" class="nav-link" data-toggle="dropdown" aria-label="Tools" aria-haspopup="true" aria-expanded="false">
<i style="width: 1em" class='fa fa-lg fa-cog'></i>
</a>
<div class="dropdown-menu dropdown-menu-right" aria-labelledby="gearDropdown">
<a class="dropdown-item" id="light-theme-item" href="" onclick="setActiveStyleSheet('light');return false;">Light Theme</a>
<a class="dropdown-item" id="dark-theme-item" href="" onclick="setActiveStyleSheet('dark');return false;">Dark Theme</a>
<div class="dropdown-divider"></div>
<h6 class="dropdown-header">Other versions of this site</h6>
<a href="https://istio.io" class="dropdown-item">Current Release</a>
<a href="https://preliminary.istio.io" class="dropdown-item">Next Release</a>
<a href="https://archive.istio.io" class="dropdown-item">Older Releases</a>
</div>
</li>
<li class="nav-item">
<a id="search_show" class="nav-link" href="" aria-label="Search"><i style="width: 1em" class="fa fa-lg fa-search"></i></a>
</li>
</ul>
<form name="cse" id="search_form" class="form-inline mr-sm-2" role="search">
<input type="hidden" name="cx" value="013699703217164175118:iwwf17ikgf4" />
<input type="hidden" name="ie" value="utf-8" />
<input type="hidden" name="hl" value="en" />
<input type="hidden" id="search_page_url" value="/v0.8/search.html" />
<input id="search_textbox" class="form-control" name="q" type="text" aria-label="Search this site"/>
<button id="search_close" type="reset" aria-label="Cancel Search"><i class="far fa-lg fa-times-circle"></i></button>
</form>
</div>
</nav>
</header>
<div class="blog">
<div class="container-fluid">
<div class="row row-offcanvas">
<div class="col-0 col-md-3 col-xl-2 sidebar-offcanvas">
<nav class="sidebar d-print-none">
<div class="spacer"></div>
<div class="directory" role="tablist">
<div class="card">
<div class="card-header" role="tab" id="header0">
<a data-toggle="collapse" href="#collapse0" title="Blog posts for 2018" role="button" aria-controls="collapse0">
<div>
2018 Posts
</div>
</a>
</div>
<div id="collapse0" class="collapse show" data-parent="#sidebar" role="tabpanel" aria-labelledby="header0">
<div class="card-body">
<ul class="tree">
<li>
<a title="Describes how to configure Istio for monitoring and access policies of HTTP egress traffic." href="/v0.8/blog/2018/egress-monitoring-access-control/">Monitoring and Access Policies for HTTP Egress Traffic</a>
</li>
<li>
<a title="Introduction, motivation and design principles for the Istio v1alpha3 routing API." href="/v0.8/blog/2018/v1alpha3-routing/">Introducing the Istio v1alpha3 routing API</a>
</li>
<li>
<a title="Describes how to configure Istio ingress with a network load balancer on AWS" href="/v0.8/blog/2018/aws-nlb/">Configuring Istio Ingress with AWS NLB</a>
</li>
<li>
<span class="current" title="Using Kubernetes namespace and RBAC to create an Istio soft multi-tenancy environment">Istio Soft Multi-tenancy Support</span>
</li>
<li>
<a title="An introduction to safer, lower-risk deployments and release to production" href="/v0.8/blog/2018/traffic-mirroring/">Traffic Mirroring with Istio for Testing in Production</a>
</li>
<li>
<a title="Describes a simple scenario based on Istio Bookinfo sample" href="/v0.8/blog/2018/egress-tcp/">Consuming External TCP Services</a>
</li>
<li>
<a title="Describes a simple scenario based on Istio Bookinfo sample" href="/v0.8/blog/2018/egress-https/">Consuming External Web Services</a>
</li>
</ul>
</div>
</div>
</div>
<div class="card">
<div class="card-header" role="tab" id="header1">
<a data-toggle="collapse" href="#collapse1" title="Blog posts for 2017" role="button" aria-controls="collapse1">
<div>
2017 Posts
</div>
</a>
</div>
<div id="collapse1" class="collapse" data-parent="#sidebar" role="tabpanel" aria-labelledby="header1">
<div class="card-body">
<ul class="tree">
<li>
<a title="Improving availability and reducing latency" href="/v0.8/blog/2017/mixer-spof-myth/">Mixer and the SPOF Myth</a>
</li>
<li>
<a title="Provides an overview of the Mixer plug-in architecture" href="/v0.8/blog/2017/adapter-model/">Mixer Adapter Model</a>
</li>
<li>
<a title="Istio 0.2 announcement" href="/v0.8/blog/2017/0.2-announcement/">Announcing Istio 0.2</a>
</li>
<li>
<a title="How Kubernetes Network Policy relates to Istio policy" href="/v0.8/blog/2017/0.1-using-network-policy/">Using Network Policy with Istio</a>
</li>
<li>
<a title="Using Istio to create autoscaled canary deployments" href="/v0.8/blog/2017/0.1-canary/">Canary Deployments using Istio</a>
</li>
<li>
<a title="Istio Auth 0.1 announcement" href="/v0.8/blog/2017/0.1-auth/">Using Istio to Improve End-to-End Security</a>
</li>
<li>
<a title="Istio 0.1 announcement" href="/v0.8/blog/2017/0.1-announcement/">Introducing Istio</a>
</li>
</ul>
</div>
</div>
</div>
</div>
</nav>
</div>
<div class="col-12 col-md-9 col-xl-8">
<p class="d-md-none">
<label class="sidebar-toggler" data-toggle="offcanvas">
<i class="fa fa-sign-out-alt"></i>
</label>
</p>
<main aria-labelledby="title">
<h1 id="title">Istio Soft Multi-tenancy Support</h1>
<p class="subtitle">Using multiple Istio control planes and RBAC to create multi-tenancy</p>
<p class="byline">
By <span class="attribution">John Joyce and Rich Curran</span> /
<span class="publish_date">April 19, 2018</span>
</p>
<nav class="toc-inlined d-xl-none d-print-none" >
<div class="directory" role="directory">
<nav id="InlinedTableOfContents">
<ul>
<li><a href="#soft-multi-tenancy">Soft multi-tenancy</a></li>
<li><a href="#deployment">Deployment</a>
<ul>
<li><a href="#multiple-istio-control-planes">Multiple Istio control planes</a></li>
<li><a href="#split-common-and-namespace-specific-resources">Split common and namespace specific resources</a></li>
<li><a href="#kubernetes-rbac-for-istio-control-plane-resources">Kubernetes RBAC for Istio control plane resources</a></li>
<li><a href="#watching-specific-namespaces-for-service-discovery">Watching specific namespaces for service discovery</a></li>
<li><a href="#deploying-the-tenant-application-in-a-namespace">Deploying the tenant application in a namespace</a></li>
<li><a href="#using-istioctl-in-a-multi-tenant-environment">Using istioctl in a multi-tenant environment</a></li>
<li><a href="#test-results">Test results</a></li>
</ul>
</li>
<li><a href="#conclusion">Conclusion</a></li>
<li><a href="#issues">Issues</a></li>
<li><a href="#challenges-with-other-multi-tenancy-models">Challenges with other multi-tenancy models</a></li>
<li><a href="#future-work">Future work</a></li>
<li><a href="#references">References</a></li>
</ul>
</nav>
</div>
</nav>
<p>Multi-tenancy is commonly used in many environments across many different applications,
but the implementation details and functionality provided on a per tenant basis does not
follow one model in all environments. The <a href="https://github.com/kubernetes/community/blob/master/wg-multitenancy/README.md">Kubernetes multi-tenancy working group</a>
is working to define the multi-tenant use cases and functionality that should be available
within Kubernetes. However, from their work so far it is clear that only &ldquo;soft multi-tenancy&rdquo;
is possible due to the inability to fully protect against malicious containers or workloads
gaining access to other tenant's pods or kernel resources.</p>
<h2 id="soft-multi-tenancy">Soft multi-tenancy</h2>
<p>For this blog, &ldquo;soft multi-tenancy&rdquo; is defined as having a single Kubernetes control plane
with multiple Istio control planes and multiple meshes, one control plane and one mesh
per tenant. The cluster administrator gets control and visibility across all the Istio
control planes, while the tenant administrator only gets control of a specific Istio
instance. Separation between the tenants is provided by Kubernetes namespaces and RBAC.</p>
<p>One use case for this deployment model is a shared corporate infrastructure where malicious
actions are not expected, but a clean separation of the tenants is still required.</p>
<p>Potential future Istio multi-tenant deployment models are described at the bottom of this
blog.</p>
<blockquote>
<p>Note: This blog is a high-level description of how to deploy Istio in a
limited multi-tenancy environment. The <a href="/v0.8/docs/">docs</a> section will be updated
when official multi-tenancy support is provided.</p>
</blockquote>
<h2 id="deployment">Deployment</h2>
<h3 id="multiple-istio-control-planes">Multiple Istio control planes</h3>
<p>Deploying multiple Istio control planes starts by replacing all <code>namespace</code> references
in a manifest file with the desired namespace. Using istio.yaml as an example, if two tenant
level Istio control planes are required; the first can use the istio.yaml default name of
<em>istio-system</em> and a second control plane can be created by generating a new yaml file with
a different namespace. As an example, the following command creates a yaml file with
the Istio namespace of <em>istio-system1</em>.</p>
<pre><code class="language-command" data-lang="command">$ cat istio.yaml | sed s/istio-system/istio-system1/g &gt; istio-system1.yaml
</code></pre><p>The istio yaml file contains the details of the Istio control plane deployment, including the
pods that make up the control plane (mixer, pilot, ingress, CA). Deploying the two Istio
control plane yaml files:</p>
<pre><code class="language-command" data-lang="command">$ kubectl apply -f @install/kubernetes/istio.yaml@
$ kubectl apply -f @install/kubernetes/istio-system1.yaml@
</code></pre><p>Results in two Istio control planes running in two namespaces.</p>
<pre><code class="language-command" data-lang="command">$ kubectl get pods --all-namespaces
NAMESPACE NAME READY STATUS RESTARTS AGE
istio-system istio-ca-ffbb75c6f-98w6x 1/1 Running 0 15d
istio-system istio-ingress-68d65fc5c6-dnvfl 1/1 Running 0 15d
istio-system istio-mixer-5b9f8dffb5-8875r 3/3 Running 0 15d
istio-system istio-pilot-678fc976c8-b8tv6 2/2 Running 0 15d
istio-system1 istio-ca-5f496fdbcd-lqhlk 1/1 Running 0 15d
istio-system1 istio-ingress-68d65fc5c6-2vldg 1/1 Running 0 15d
istio-system1 istio-mixer-7d4f7b9968-66z44 3/3 Running 0 15d
istio-system1 istio-pilot-5bb6b7669c-779vb 2/2 Running 0 15d
</code></pre><p>The Istio <a href="/v0.8/docs/setup/kubernetes/sidecar-injection/">sidecar</a> and
<a href="/v0.8/docs/tasks/telemetry/">addons</a>, if required, manifests must also
be deployed to match the configured <code>namespace</code> in use by the tenant's Istio control plane.</p>
<p>The execution of these two yaml files is the responsibility of the cluster
administrator, not the tenant level administrator. Additional RBAC restrictions will also
need to be configured and applied by the cluster administrator, limiting the tenant
administrator to only the assigned namespace.</p>
<h3 id="split-common-and-namespace-specific-resources">Split common and namespace specific resources</h3>
<p>The manifest files in the Istio repositories create both common resources that would
be used by all Istio control planes as well as resources that are replicated per control
plane. Although it is a simple matter to deploy multiple control planes by replacing the
<em>istio-system</em> namespace references as described above, a better approach is to split the
manifests into a common part that is deployed once for all tenants and a tenant
specific part. For the <a href="https://kubernetes.io/docs/concepts/api-extension/custom-resources/#customresourcedefinitions">Custom Resource Definitions</a>, the roles and the role
bindings should be separated out from the provided Istio manifests. Additionally, the
roles and role bindings in the provided Istio manifests are probably unsuitable for a
multi-tenant environment and should be modified or augmented as described in the next
section.</p>
<h3 id="kubernetes-rbac-for-istio-control-plane-resources">Kubernetes RBAC for Istio control plane resources</h3>
<p>To restrict a tenant administrator to a single Istio namespace, the cluster
administrator would create a manifest containing, at a minimum, a <code>Role</code> and <code>RoleBinding</code>
similar to the one below. In this example, a tenant administrator named <em>sales-admin</em>
is limited to the namespace <em>istio-system1</em>. A completed manifest would contain many
more <code>apiGroups</code> under the <code>Role</code> providing resource access to the tenant administrator.</p>
<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-yaml" data-lang="yaml">kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
namespace: istio-system1
name: ns-access-for-sales-admin-istio-system1
rules:
- apiGroups: [<span style="color:#e6db74">&#34;&#34;</span>] <span style="color:#75715e"># &#34;&#34; indicates the core API group</span>
resources: [<span style="color:#e6db74">&#34;*&#34;</span>]
verbs: [<span style="color:#e6db74">&#34;*&#34;</span>]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: access-all-istio-system1
namespace: istio-system1
subjects:
- kind: User
name: sales-admin
apiGroup: rbac.authorization.k8s.io
roleRef:
kind: Role
name: ns-access-for-sales-admin-istio-system1
apiGroup: rbac.authorization.k8s.io
</code></pre></div><h3 id="watching-specific-namespaces-for-service-discovery">Watching specific namespaces for service discovery</h3>
<p>In addition to creating RBAC rules limiting the tenant administrator's access to a specific
Istio control plane, the Istio manifest must be updated to specify the application namespace
that Pilot should watch for creation of its xDS cache. This is done by starting the Pilot
component with the additional command line arguments <code>--appNamespace, ns-1</code>. Where <em>ns-1</em>
is the namespace that the tenants application will be deployed in. An example snippet from
the istio-system1.yaml file is included below.</p>
<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-yaml" data-lang="yaml">apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: istio-pilot
namespace: istio-system1
annotations:
sidecar.istio.io/inject: <span style="color:#e6db74">&#34;false&#34;</span>
spec:
replicas: <span style="color:#ae81ff">1</span>
template:
metadata:
labels:
istio: pilot
spec:
serviceAccountName: istio-pilot-service-account
containers:
- name: discovery
image: docker.io/&lt;user ID&gt;/pilot:&lt;tag<span style="color:#e6db74">&gt;
</span><span style="color:#e6db74"> </span><span style="color:#e6db74"> </span><span style="color:#e6db74">imagePullPolicy: IfNotPresent</span>
args: [<span style="color:#e6db74">&#34;discovery&#34;</span>, <span style="color:#e6db74">&#34;-v&#34;</span>, <span style="color:#e6db74">&#34;2&#34;</span>, <span style="color:#e6db74">&#34;--admission-service&#34;</span>, <span style="color:#e6db74">&#34;istio-pilot&#34;</span>, <span style="color:#e6db74">&#34;--appNamespace&#34;</span>, <span style="color:#e6db74">&#34;ns-1&#34;</span>]
ports:
- containerPort: <span style="color:#ae81ff">8080</span>
- containerPort: <span style="color:#ae81ff">443</span>
</code></pre></div><h3 id="deploying-the-tenant-application-in-a-namespace">Deploying the tenant application in a namespace</h3>
<p>Now that the cluster administrator has created the tenant's namespace (ex. <em>istio-system1</em>) and
Pilot's service discovery has been configured to watch for a specific application
namespace (ex. <em>ns-1</em>), create the application manifests to deploy in that tenant's specific
namespace. For example:</p>
<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-yaml" data-lang="yaml">apiVersion: v1
kind: Namespace
metadata:
name: ns<span style="color:#ae81ff">-1</span>
</code></pre></div><p>And add the namespace reference to each resource type included in the application's manifest
file. For example:</p>
<div class="highlight"><pre style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-yaml" data-lang="yaml">apiVersion: v1
kind: Service
metadata:
name: details
labels:
app: details
namespace: ns<span style="color:#ae81ff">-1</span>
</code></pre></div><p>Although not shown, the application namespaces will also have RBAC settings limiting access
to certain resources. These RBAC settings could be set by the cluster administrator and/or
the tenant administrator.</p>
<h3 id="using-istioctl-in-a-multi-tenant-environment">Using <code>istioctl</code> in a multi-tenant environment</h3>
<p>When defining <a href="/v0.8/docs/reference/config/istio.routing.v1alpha1/#RouteRule">route rules</a>
or <a href="/v0.8/docs/reference/config/istio.routing.v1alpha1/#DestinationPolicy">destination policies</a>,
it is necessary to ensure that the <code>istioctl</code> command is scoped to
the namespace the Istio control plane is running in to ensure the resource is created
in the proper namespace. Additionally, the rule itself must be scoped to the tenant's namespace
so that it will be applied properly to that tenant's mesh. The <em>-i</em> option is used to create
(or get or describe) the rule in the namespace that the Istio control plane is deployed in.
The <em>-n</em> option will scope the rule to the tenant's mesh and should be set to the namespace that
the tenant's app is deployed in. Note that the <em>-n</em> option can be skipped on the command line if
the .yaml file for the resource scopes it properly instead.</p>
<p>For example, the following command would be required to add a route rule to the <em>istio-system1</em>
namespace:</p>
<pre><code class="language-command" data-lang="command">$ istioctl i istio-system1 create -n ns-1 -f route_rule_v2.yaml
</code></pre><p>And can be displayed using the command:</p>
<pre><code class="language-command" data-lang="command">$ istioctl -i istio-system1 -n ns-1 get routerule
NAME KIND NAMESPACE
details-Default RouteRule.v1alpha2.config.istio.io ns-1
productpage-default RouteRule.v1alpha2.config.istio.io ns-1
ratings-default RouteRule.v1alpha2.config.istio.io ns-1
reviews-default RouteRule.v1alpha2.config.istio.io ns-1
</code></pre><p>See the <a href="/v0.8/blog/2018/soft-multitenancy/#multiple-istio-control-planes">Multiple Istio control planes</a> section of this document for more details on <code>namespace</code> requirements in a
multi-tenant environment.</p>
<h3 id="test-results">Test results</h3>
<p>Following the instructions above, a cluster administrator can create an environment limiting,
via RBAC and namespaces, what a tenant administrator can deploy.</p>
<p>After deployment, accessing the Istio control plane pods assigned to a specific tenant
administrator is permitted:</p>
<pre><code class="language-command" data-lang="command">$ kubectl get pods -n istio-system
NAME READY STATUS RESTARTS AGE
grafana-78d649479f-8pqk9 1/1 Running 0 1d
istio-ca-ffbb75c6f-98w6x 1/1 Running 0 1d
istio-ingress-68d65fc5c6-dnvfl 1/1 Running 0 1d
istio-mixer-5b9f8dffb5-8875r 3/3 Running 0 1d
istio-pilot-678fc976c8-b8tv6 2/2 Running 0 1d
istio-sidecar-injector-7587bd559d-5tgk6 1/1 Running 0 1d
prometheus-cf8456855-hdcq7 1/1 Running 0 1d
servicegraph-75ff8f7c95-wcjs7 1/1 Running 0 1d
</code></pre><p>However, accessing all the cluster's pods is not permitted:</p>
<pre><code class="language-command" data-lang="command">$ kubectl get pods --all-namespaces
Error from server (Forbidden): pods is forbidden: User &quot;dev-admin&quot; cannot list pods at the cluster scope
</code></pre><p>And neither is accessing another tenant's namespace:</p>
<pre><code class="language-command" data-lang="command">$ kubectl get pods -n istio-system1
Error from server (Forbidden): pods is forbidden: User &quot;dev-admin&quot; cannot list pods in the namespace &quot;istio-system1&quot;
</code></pre><p>The tenant administrator can deploy applications in the application namespace configured for
that tenant. As an example, updating the <a href="/v0.8/docs/guides/bookinfo/">Bookinfo</a>
manifests and then deploying under the tenant's application namespace of <em>ns-0</em>, listing the
pods in use by this tenant's namespace is permitted:</p>
<pre><code class="language-command" data-lang="command">$ kubectl get pods -n ns-0
NAME READY STATUS RESTARTS AGE
details-v1-64b86cd49-b7rkr 2/2 Running 0 1d
productpage-v1-84f77f8747-rf2mt 2/2 Running 0 1d
ratings-v1-5f46655b57-5b4c5 2/2 Running 0 1d
reviews-v1-ff6bdb95b-pm5lb 2/2 Running 0 1d
reviews-v2-5799558d68-b989t 2/2 Running 0 1d
reviews-v3-58ff7d665b-lw5j9 2/2 Running 0 1d
</code></pre><p>But accessing another tenant's application namespace is not:</p>
<pre><code class="language-command" data-lang="command">$ kubectl get pods -n ns-1
Error from server (Forbidden): pods is forbidden: User &quot;dev-admin&quot; cannot list pods in the namespace &quot;ns-1&quot;
</code></pre><p>If the <a href="/v0.8/docs/tasks/telemetry/">addon tools</a>, example
<a href="/v0.8/docs/tasks/telemetry//querying-metrics/">prometheus</a>, are deployed
(also limited by an Istio <code>namespace</code>) the statistical results returned would represent only
that traffic seen from that tenant's application namespace.</p>
<h2 id="conclusion">Conclusion</h2>
<p>The evaluation performed indicates Istio has sufficient capabilities and security to meet a
small number of multi-tenant use cases. It also shows that Istio and Kubernetes <strong>cannot</strong>
provide sufficient capabilities and security for other use cases, especially those use
cases that require complete security and isolation between untrusted tenants. The improvements
required to reach a more secure model of security and isolation require work in container
technology, ex. Kubernetes, rather than improvements in Istio capabilities.</p>
<h2 id="issues">Issues</h2>
<ul>
<li>The CA (Certificate Authority) and mixer Istio pod logs from one tenant's Istio control
plane (ex. <em>istio-system</em> <code>namespace</code>) contained &lsquo;info&rsquo; messages from a second tenant's
Istio control plane (ex <em>istio-system1</em> <code>namespace</code>).</li>
</ul>
<h2 id="challenges-with-other-multi-tenancy-models">Challenges with other multi-tenancy models</h2>
<p>Other multi-tenancy deployment models were considered:</p>
<ol>
<li>
<p>A single mesh with multiple applications, one for each tenant on the mesh. The cluster
administrator gets control and visibility mesh wide and across all applications, while the
tenant administrator only gets control of a specific application.</p>
</li>
<li>
<p>A single Istio control plane with multiple meshes, one mesh per tenant. The cluster
administrator gets control and visibility across the entire Istio control plane and all
meshes, while the tenant administrator only gets control of a specific mesh.</p>
</li>
<li>
<p>A single cloud environment (cluster controlled), but multiple Kubernetes control planes
(tenant controlled).</p>
</li>
</ol>
<p>These options either can't be properly supported without code changes or don't fully
address the use cases.</p>
<p>Current Istio capabilities are poorly suited to support the first model as it lacks
sufficient RBAC capabilities to support cluster versus tenant operations. Additionally,
having multiple tenants under one mesh is too insecure with the current mesh model and the
way Istio drives configuration to the envoy proxies.</p>
<p>Regarding the second option, the current Istio paradigm assumes a single mesh per Istio control
plane. The needed changes to support this model are substantial. They would require
finer grained scoping of resources and security domains based on namespaces, as well as,
additional Istio RBAC changes. This model will likely be addressed by future work, but not
currently possible.</p>
<p>The third model doesnt satisfy most use cases, as most cluster administrators prefer
a common Kubernetes control plane which they provide as a
<a href="https://en.wikipedia.org/wiki/Platform_as_a_service">PaaS</a> to their tenants.</p>
<h2 id="future-work">Future work</h2>
<p>Allowing a single Istio control plane to control multiple meshes would be an obvious next
feature. An additional improvement is to provide a single mesh that can host different
tenants with some level of isolation and security between the tenants. This could be done
by partitioning within a single control plane using the same logical notion of namespace as
Kubernetes. A <a href="https://docs.google.com/document/d/14Hb07gSrfVt5KX9qNi7FzzGwB_6WBpAnDpPG6QEEd9Q">document</a>
has been started within the Istio community to define additional use cases and the
Istio functionality required to support those use cases.</p>
<h2 id="references">References</h2>
<ul>
<li>Video on Kubernetes multi-tenancy support, <a href="https://www.youtube.com/watch?v=ahwCkJGItkU">Multi-Tenancy Support &amp; Security Modeling with RBAC and Namespaces</a>, and the <a href="https://schd.ws/hosted_files/kccncna17/21/Multi-tenancy%20Support%20%26%20Security%20Modeling%20with%20RBAC%20and%20Namespaces.pdf">supporting slide deck</a>.</li>
<li>Kubecon talk on security that discusses Kubernetes support for &ldquo;Cooperative soft multi-tenancy&rdquo;, <a href="https://www.youtube.com/watch?v=YRR-kZub0cA">Building for Trust: How to Secure Your Kubernetes</a>.</li>
<li>Kubernetes documentation on <a href="https://kubernetes.io/docs/admin/authorization/rbac/">RBAC</a> and <a href="https://kubernetes.io/docs/tasks/administer-cluster/namespaces-walkthrough/">namespaces</a>.</li>
<li>Kubecon slide deck on <a href="https://schd.ws/hosted_files/kccncna17/a9/kubecon-multitenancy.pdf">Multi-tenancy Deep Dive</a>.</li>
<li>Google document on <a href="https://docs.google.com/document/d/15w1_fesSUZHv-vwjiYa9vN_uyc--PySRoLKTuDhimjc/edit#heading=h.3dawx97e3hz6">Multi-tenancy models for Kubernetes</a>. (Requires permission)</li>
<li>Cloud Foundry WIP document, <a href="https://docs.google.com/document/d/14Hb07gSrfVt5KX9qNi7FzzGwB_6WBpAnDpPG6QEEd9Q">Multi-cloud and Multi-tenancy</a></li>
<li><a href="https://docs.google.com/document/d/12F183NIRAwj2hprx-a-51ByLeNqbJxK16X06vwH5OWE/edit#heading=h.x0f9qplja3q">Istio Auto Multi-Tenancy 101</a></li>
</ul>
</main>
<div class="container-fluid d-print-none">
<br/><hr/><br/>
<div class="row">
<div class="col-6">
<a title="Describes how to configure Istio ingress with a network load balancer on AWS" href="/v0.8/blog/2018/aws-nlb/"><i class="fa fa-arrow-left"></i> Configuring Istio Ingress with AWS NLB</a>
</div>
<div class="col-6" style="text-align: right">
<a title="An introduction to safer, lower-risk deployments and release to production" href="/v0.8/blog/2018/traffic-mirroring/">Traffic Mirroring with Istio for Testing in Production <i class="fa fa-arrow-right"></i></a>
</div>
</div>
</div>
<div class="d-none d-print-block" aria-hidden="true">
<h2>Links</h2>
<ol id="endnotes"></ol>
</div>
</div>
<div class="col-12 col-md-2 d-none d-xl-block d-print-none">
<nav class="toc">
<div class="spacer"></div>
<div id="toc" class="directory" role="directory">
<nav id="TableOfContents">
<ul>
<li><a href="#soft-multi-tenancy">Soft multi-tenancy</a></li>
<li><a href="#deployment">Deployment</a>
<ul>
<li><a href="#multiple-istio-control-planes">Multiple Istio control planes</a></li>
<li><a href="#split-common-and-namespace-specific-resources">Split common and namespace specific resources</a></li>
<li><a href="#kubernetes-rbac-for-istio-control-plane-resources">Kubernetes RBAC for Istio control plane resources</a></li>
<li><a href="#watching-specific-namespaces-for-service-discovery">Watching specific namespaces for service discovery</a></li>
<li><a href="#deploying-the-tenant-application-in-a-namespace">Deploying the tenant application in a namespace</a></li>
<li><a href="#using-istioctl-in-a-multi-tenant-environment">Using istioctl in a multi-tenant environment</a></li>
<li><a href="#test-results">Test results</a></li>
</ul>
</li>
<li><a href="#conclusion">Conclusion</a></li>
<li><a href="#issues">Issues</a></li>
<li><a href="#challenges-with-other-multi-tenancy-models">Challenges with other multi-tenancy models</a></li>
<li><a href="#future-work">Future work</a></li>
<li><a href="#references">References</a></li>
</ul>
</nav>
</div>
</nav>
</div>
</div>
</div>
</div>
<footer class="d-print-none container-fluid">
<div class="row">
<div class="col-6 col-lg-4" role="navigation">
<div class="container-fluid">
<div class="row">
<div class="icon">
<span>istio-users@</span>
<a title="Join the istio-users@ mailing list to participate in discussions and get help troubleshooting problems"
href="https://groups.google.com/forum/#!forum/istio-users" aria-label="istio-users mailing list">
<svg viewBox="0 0 490 490">
<path d="M480,410.248H10c-5.523,0-10-4.477-10-10V89.752c0-5.523,4.477-10,10-10h470c5.522,0,10,4.477,10,10v310.495
C490,405.771,485.522,410.248,480,410.248z M20,390.248h450V99.752H20V390.248z"/>
<path d="M245,286.131c-2.083,0-4.167-0.649-5.931-1.948L48.64,143.929c-4.446-3.275-5.396-9.535-2.121-13.982
c3.275-4.447,9.535-5.396,13.982-2.121L245,263.712l184.5-135.886c4.447-3.274,10.709-2.326,13.982,2.121
c3.275,4.447,2.325,10.707-2.121,13.982L250.931,284.183C249.167,285.482,247.083,286.131,245,286.131z"/>
</svg>
</a>
</div>
<div class="icon">
<span>twitter</span>
<a title="Follow us on Twitter to get the latest news"
href="https://twitter.com/IstioMesh" aria-label="Twitter">
<svg viewBox="0 0 310 310">
<path d="M302.973,57.388c-4.87,2.16-9.877,3.983-14.993,5.463c6.057-6.85,10.675-14.91,13.494-23.73
c0.632-1.977-0.023-4.141-1.648-5.434c-1.623-1.294-3.878-1.449-5.665-0.39c-10.865,6.444-22.587,11.075-34.878,13.783
c-12.381-12.098-29.197-18.983-46.581-18.983c-36.695,0-66.549,29.853-66.549,66.547c0,2.89,0.183,5.764,0.545,8.598
C101.163,99.244,58.83,76.863,29.76,41.204c-1.036-1.271-2.632-1.956-4.266-1.825c-1.635,0.128-3.104,1.05-3.93,2.467
c-5.896,10.117-9.013,21.688-9.013,33.461c0,16.035,5.725,31.249,15.838,43.137c-3.075-1.065-6.059-2.396-8.907-3.977
c-1.529-0.851-3.395-0.838-4.914,0.033c-1.52,0.871-2.473,2.473-2.513,4.224c-0.007,0.295-0.007,0.59-0.007,0.889
c0,23.935,12.882,45.484,32.577,57.229c-1.692-0.169-3.383-0.414-5.063-0.735c-1.732-0.331-3.513,0.276-4.681,1.597
c-1.17,1.32-1.557,3.16-1.018,4.84c7.29,22.76,26.059,39.501,48.749,44.605c-18.819,11.787-40.34,17.961-62.932,17.961
c-4.714,0-9.455-0.277-14.095-0.826c-2.305-0.274-4.509,1.087-5.294,3.279c-0.785,2.193,0.047,4.638,2.008,5.895
c29.023,18.609,62.582,28.445,97.047,28.445c67.754,0,110.139-31.95,133.764-58.753c29.46-33.421,46.356-77.658,46.356-121.367
c0-1.826-0.028-3.67-0.084-5.508c11.623-8.757,21.63-19.355,29.773-31.536c1.237-1.85,1.103-4.295-0.33-5.998
C307.394,57.037,305.009,56.486,302.973,57.388z"/>
</svg>
</a>
</div>
<div class="icon">
<span>stack overflow</span>
<a title="Stack Overflow is where you can ask questions and find curated answers on deploying, configuring, and using Istio"
href="https://stackoverflow.com/questions/tagged/istio" aria-label="Stack Overflow">
<svg viewBox="0 0 120 120">
<polygon points="84.4,93.8 84.4,70.6 92.1,70.6 92.1,101.5 22.6,101.5 22.6,70.6 30.3,70.6 30.3,93.8 "/>
<path d="M38.8,68.4l37.8,7.9l1.6-7.6l-37.8-7.9L38.8,68.4z M43.8,50.4l35,16.3l3.2-7l-35-16.4L43.8,50.4z M53.5,33.2
l29.7,24.7l4.9-5.9L58.4,27.3L53.5,33.2z M72.7,14.9l-6.2,4.6l23,31l6.2-4.6L72.7,14.9z M38,86h38.6v-7.7H38V86z"/>
</svg>
</a>
</div>
<div class="icon">
<span>rocket chat</span>
<a title="Interactively chat with members of the Istio community."
href="https://istio.rocket.chat" aria-label="Rocket Chat">
<svg viewBox="0 0 512 512">
<path d="M496.293,255.338c0-24.103-7.21-47.215-21.437-68.699c-12.771-19.288-30.666-36.362-53.184-50.745
c-43.474-27.771-100.612-43.065-160.885-43.065c-20.131,0-39.974,1.702-59.222,5.072c-11.942-11.176-25.919-21.233-40.712-29.187
c-79.026-38.298-144.561-0.9-144.561-0.9s60.931,50.053,51.023,93.93c-27.259,27.041-42.033,59.646-42.033,93.594
c0,0.108,0.005,0.216,0.006,0.324c-0.001,0.108-0.006,0.216-0.006,0.324c0,33.949,14.774,66.554,42.033,93.595
c9.907,43.874-51.023,93.93-51.023,93.93s65.535,37.397,144.561-0.901c14.792-7.953,28.77-18.01,40.712-29.188
c19.249,3.372,39.091,5.072,59.222,5.072c60.272,0,117.411-15.294,160.885-43.064c22.518-14.383,40.412-31.457,53.184-50.742
c14.227-21.487,21.437-44.599,21.437-68.702c0-0.107-0.006-0.216-0.006-0.324C496.287,255.554,496.293,255.446,496.293,255.338z
M260.882,387.763c-25.367,0-49.66-2.932-72.107-8.282c-22.81,27.443-72.993,65.596-121.742,53.26
c15.857-17.031,39.352-45.81,34.32-93.207c-29.218-22.738-46.759-51.832-46.759-83.541c0-72.776,92.36-131.769,206.288-131.769
c113.928,0,206.288,58.993,206.288,131.769C467.17,328.765,374.81,387.763,260.882,387.763z M288.283,255.991
c0,15.133-12.27,27.403-27.4,27.403c-15.134,0-27.402-12.271-27.402-27.403s12.268-27.401,27.402-27.401
C276.014,228.59,288.283,240.858,288.283,255.991z M356.163,228.59c-15.133,0-27.4,12.268-27.4,27.401s12.268,27.403,27.4,27.403
c15.134,0,27.399-12.271,27.399-27.403S371.297,228.59,356.163,228.59z M165.601,228.59c-15.133,0-27.4,12.268-27.4,27.401
s12.268,27.403,27.4,27.403c15.134,0,27.401-12.271,27.401-27.403S180.735,228.59,165.601,228.59z"/>
</svg>
</a>
</div>
</div>
<div class="tag row d-none d-lg-flex">
for users
</div>
</div>
</div>
<div class="col-6 col-lg-4">
<p class="text-center copyright" role="contentinfo">
Istio
Archive
0.8<br>&copy; 2018 Istio Authors, <a href="https://policies.google.com/privacy">Privacy Policy</a><br>
Archived on July 31, 2018
</p>
</div>
<div class="col-6 col-lg-4 d-none d-lg-flex" role="navigation">
<div class="container-fluid">
<div class="row justify-content-end">
<div class="icon">
<span>istio-dev@</span>
<a title="Join the istio-dev@ mailing list to discuss development issues around the Istio project"
href="https://groups.google.com/forum/#!forum/istio-dev" aria-label="istio-dev mailing list">
<svg viewBox="0 0 490 490">
<path d="M480,410.248H10c-5.523,0-10-4.477-10-10V89.752c0-5.523,4.477-10,10-10h470c5.522,0,10,4.477,10,10v310.495
C490,405.771,485.522,410.248,480,410.248z M20,390.248h450V99.752H20V390.248z"/>
<path d="M245,286.131c-2.083,0-4.167-0.649-5.931-1.948L48.64,143.929c-4.446-3.275-5.396-9.535-2.121-13.982
c3.275-4.447,9.535-5.396,13.982-2.121L245,263.712l184.5-135.886c4.447-3.274,10.709-2.326,13.982,2.121
c3.275,4.447,2.325,10.707-2.121,13.982L250.931,284.183C249.167,285.482,247.083,286.131,245,286.131z"/>
</svg>
</a>
</div>
<div class="icon">
<span>github</span>
<a title="GitHub is where development takes place on Istio code"
href="https://github.com/istio/community" aria-label="GitHub">
<svg viewBox="0 0 478.165 478.165">
<path d="M349.22,55.768c6.136,14.046,10.241,37.556,4.224,54.69
c24.426,20.999,33.073,71.904,21.079,113.704c35.006,2.73,76.666-1.235,103.642,9.484c-25.183-3.248-59.651-9.563-91.987-7.431
c-6.136,0.458-15.361-0.239-14.903,8.408c37.735,3.008,75.092,6.117,105.894,15.779c-30.702-4.981-67.74-12.552-105.894-13.668
c-15.54,30.921-47.239,46.262-90.991,49.49c4.682,10.261,13.847,14.066,15.879,30.702c3.267,24.406-4.881,60.328,3.208,76.686
c4.064,7.89,10.579,8.009,14.863,14.604c-10.699,12.871-37.257-1.395-40.186-14.604c-5.14-22.852,7.89-58.256-6.415-73.737
c0.996,24.865-5.718,59.85,0.996,82.145c2.789,8.806,10.659,12.113,8.647,20.063c-49.809,5.08-28.989-64.373-37.177-105.356
c-7.471,0.697-4.204,11.197-4.224,15.76c-0.199,40.106,8.189,94.836-34.846,89.556c-1.315-8.348,5.838-11.217,8.467-19.007
c7.91-22.434-1.454-56.045,2.112-83.161c-16.417,12.512,1.793,55.666-8.428,77.961c-5.838,12.671-24.785,18.27-39.19,12.651
c1.873-9.464,11.695-7.989,15.879-16.875c5.818-12.452,0.02-30.244,2.092-48.494c-30.423,6.097-53.993-0.877-65.608-20.023
c-5.12-8.507-6.356-18.708-12.632-26.219c-6.117-7.551-16.098-8.507-19.087-18.808c37.755-9.185,39.17,38.771,73.06,39.807
c10.44,0.418,15.799-2.909,25.402-5.16c2.749-12.113,8.428-21.039,16.875-27.494c-42.078-5.658-76.865-18.788-93.023-50.466
c-38.293,1.893-73.339,7.013-105.894,14.843c29.547-10.679,65.807-14.604,104.778-15.819c-2.351-13.807-22.434-10.022-34.866-9.543
C47.677,227.17,18.449,230.138,0,233.645c26.817-9.543,64.233-8.348,100.454-8.428c-11.038-34.767-7.232-90.014,17.015-110.615
c-6.854-17.254-4.722-45.346,4.184-58.834c27.036,1.175,43.374,12.891,60.388,24.247c21.019-6.017,43.035-9.045,71.904-7.451
c12.133,0.677,24.705,6.097,33.731,5.32c8.906-0.877,18.728-10.898,27.534-14.843C326.507,58.099,336.17,56.206,349.22,55.768z"/>
</svg>
</a>
</div>
<div class="icon">
<span>drive</span>
<a title="Access our team drive if you'd like to take a look at the Istio technical design documents"
href="https://groups.google.com/forum/#!forum/istio-team-drive-access" aria-label="team drive">
<svg viewBox="0 0 207.027 207.027">
<path d="M69.866,15.557L0,138.919l28.732,52.552l143.288-0.029l35.008-59.588L136.39,15.735L69.866,15.557z M17.166,139.046
L74.268,38.205L91.21,67.783L33.24,168.447L17.166,139.046z M99.841,82.851l23.805,41.558l-47.732-0.006L99.841,82.851z
M163.434,176.443l-117.332,0.024l21.53-37.065l64.606,0.008l0.067,0.119l52.865-0.085L163.434,176.443z M140.932,124.411
L90.157,35.767l-2.966-5.178l40.751,0.121l57.003,93.706L140.932,124.411z"/>
</svg>
</a>
</div>
<div class="icon">
<span>working groups</span>
<a title="If you'd like to contribute to the Istio project, consider participating in our working groups"
href="https://github.com/istio/community/blob/master/WORKING-GROUPS.md" aria-label="working groups">
<svg viewBox="0 -45 439.833 439.833">
<polygon points="246.048,195.833 299.966,235.085 319.497,227.296 276.278,195.833"/>
<polygon points="193.786,195.833 163.556,195.833 120.33,227.3 139.862,235.089"/>
<path d="M219.927,11.558c-23.854,0-37.057,12.362-36.814,36.182c0.348,32.623,14.211,52.414,36.814,52.068
c0,0,36.802,1.492,36.802-52.068C256.729,23.918,244.294,11.558,219.927,11.558z"/>
<path d="M285.017,124.567l-36.77-14.659l-8.608-7.256c-2.274-1.922-5.636-1.78-7.741,0.317l-11.973,11.904l-12.008-11.907
c-2.109-2.094-5.465-2.229-7.736-0.313l-8.611,7.256l-36.77,14.661c-11.842,4.715-11.83,46.647-12.848,50.497h155.93
C296.866,171.228,296.862,129.28,285.017,124.567z"/>
<path d="M77.976,228.568c0,0,36.801,1.492,36.801-52.068c0-23.82-12.434-36.182-36.801-36.182
c-23.854,0-37.057,12.362-36.814,36.182C41.509,209.124,55.372,228.915,77.976,228.568z"/>
<path d="M143.065,253.329l-36.77-14.658l-8.609-7.256c-2.275-1.923-5.635-1.781-7.742,0.315l-11.971,11.904l-12.008-11.908
c-2.109-2.094-5.465-2.229-7.736-0.312l-8.611,7.256l-36.77,14.66C1.006,258.045,1.018,299.977,0,303.827h155.93
C154.915,299.988,154.911,258.042,143.065,253.329z"/>
<path d="M361.878,228.568c0,0,36.801,1.492,36.801-52.068c0-23.82-12.434-36.182-36.801-36.182
c-23.854,0-37.057,12.362-36.812,36.182C325.411,209.124,339.274,228.915,361.878,228.568z"/>
<path d="M426.968,253.329l-36.77-14.658l-8.609-7.256c-2.273-1.923-5.635-1.781-7.742,0.315l-11.971,11.904l-12.008-11.908
c-2.109-2.094-5.465-2.229-7.736-0.312l-8.61,7.256l-36.771,14.66c-11.842,4.715-11.83,46.646-12.848,50.497h155.93
C438.817,299.988,438.812,258.042,426.968,253.329z"/>
</svg>
</a>
</div>
<div class="icon">
<span>slack</span>
<a title="Interactively discuss development issues with the Istio community on Slack (invitation-only)"
href="https://istio.slack.com" aria-label="slack">
<svg viewBox="0 0 31.444 31.443">
<path d="M31.202,16.369c-0.62-1.388-2.249-2.011-3.637-1.391l-1.325,0.594l-3.396-7.591l1.325-0.592
c1.388-0.622,2.01-2.25,1.389-3.637c-0.62-1.389-2.248-2.012-3.637-1.39l-1.324,0.593l-0.593-1.326
c-0.621-1.388-2.249-2.009-3.637-1.388c-1.388,0.62-2.009,2.247-1.389,3.637l0.593,1.325L7.98,8.598L7.388,7.273
c-0.621-1.39-2.249-2.009-3.637-1.39C2.363,6.504,1.742,8.132,2.362,9.52l0.592,1.324L1.63,11.438
c-1.388,0.621-2.01,2.247-1.389,3.636c0.62,1.388,2.249,2.01,3.637,1.39l1.325-0.594l3.394,7.592l-1.325,0.592
c-1.388,0.621-2.009,2.25-1.389,3.637c0.621,1.389,2.249,2.011,3.637,1.391l1.324-0.593l0.593,1.325
c0.621,1.389,2.249,2.01,3.637,1.389c1.387-0.62,2.009-2.248,1.388-3.636l-0.591-1.326l7.591-3.394l0.592,1.321
c0.621,1.391,2.248,2.013,3.637,1.392c1.388-0.619,2.01-2.248,1.389-3.637l-0.592-1.324l1.323-0.594
C31.201,19.384,31.823,17.757,31.202,16.369z M13.623,21.215l-3.395-7.593l7.591-3.394l3.395,7.591L13.623,21.215z"/>
</svg>
</a>
</div>
</div>
<div class="tag row justify-content-end text-right">
for developers
</div>
</div>
</div>
</div>
</footer>
<div class="d-xl-none d-print-none">
<button id="scroll-to-top" aria-hidden="true" onclick="scrollToTop()" title="Back to top"><i class="fa fa-lg fa-arrow-up"></i></button>
</div>
<script src="https://code.jquery.com/jquery-3.2.1.slim.min.js" integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN" crossorigin="anonymous"></script>
<script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/1.7.1/clipboard.min.js"></script>
<script src="https://www.google.com/cse/brand?form=search_form"></script>
<script src="/v0.8/js/all.min.js" data-manual></script>
</body>
</html>