mirror of https://github.com/istio/istio.io.git
163 lines
36 KiB
HTML
163 lines
36 KiB
HTML
<!doctype html><html lang=en itemscope itemtype=https://schema.org/WebPage><head><meta charset=utf-8><meta http-equiv=x-ua-compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=theme-color content="#466BB0"><meta name=title content="Exporting Logs to BigQuery, GCS, Pub/Sub through Stackdriver"><meta name=description content="How to export Istio Access Logs to different sinks like BigQuery, GCS, Pub/Sub through Stackdriver."><meta name=author content="Nupur Garg and Douglas Reid"><meta name=keywords content="microservices,services,mesh"><meta property="og:title" content="Exporting Logs to BigQuery, GCS, Pub/Sub through Stackdriver"><meta property="og:type" content="website"><meta property="og:description" content="How to export Istio Access Logs to different sinks like BigQuery, GCS, Pub/Sub through Stackdriver."><meta property="og:url" content="/v1.0/blog/2018/export-logs-through-stackdriver/"><meta property="og:image" content="/v1.0/img/istio-logo-blue-background.svg"><meta property="og:image:alt" content="Istio Logo"><meta property="og:image:width" content="112"><meta property="og:image:height" content="150"><meta property="og:site_name" content="Istio"><meta name=twitter:card content="summary"><meta name=twitter:site content="@IstioMesh"><title>Istioldie 1.0 / Exporting Logs to BigQuery, GCS, Pub/Sub through Stackdriver</title><script async src="https://www.googletagmanager.com/gtag/js?id=UA-98480406-2"></script><script>window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments);}
|
||
gtag('js',new Date());gtag('config','UA-98480406-2');</script><script>var branchName="release-1.0";var docTitle="Exporting Logs to BigQuery, GCS, Pub\/Sub through Stackdriver";</script><link rel=alternate type=application/rss+xml title="Istio Blog" href=/v1.0/feed.xml><link rel="shortcut icon" href=/v1.0/favicons/favicon.ico><link rel=apple-touch-icon href=/v1.0/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/v1.0/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/v1.0/favicons/favicon-32x32.png sizes=32x32><link rel=icon type=image/png href=/v1.0/favicons/android-36x36.png sizes=36x36><link rel=icon type=image/png href=/v1.0/favicons/android-48x48.png sizes=48x48><link rel=icon type=image/png href=/v1.0/favicons/android-72x72.png sizes=72x72><link rel=icon type=image/png href=/v1.0/favicons/android-96x196.png sizes=96x196><link rel=icon type=image/png href=/v1.0/favicons/android-144x144.png sizes=144x144><link rel=icon type=image/png href=/v1.0/favicons/android-192x192.png sizes=192x192><link rel=manifest href=/v1.0/manifest.json><meta name=apple-mobile-web-app-title content="Istio"><meta name=application-name content="Istio"><link rel=stylesheet href="https://fonts.googleapis.com/css?family=Chivo:400,100,100italic,300,300italic,400italic,500,500italic,700,700italic,900,900italic"><link rel=stylesheet href="https://fonts.googleapis.com/css?family=Work Sans:400,100,100italic,300,300italic,400italic,500,500italic,700,700italic,900,900italic"><link rel=stylesheet href=https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css integrity=sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm crossorigin=anonymous><link rel=stylesheet href=https://use.fontawesome.com/releases/v5.0.6/css/all.css><link rel=stylesheet href=/v1.0/css/light_theme_archive.css title=light><link rel="alternate stylesheet" href=/v1.0/css/dark_theme_archive.css title=dark><script src=/v1.0/js/styleSwitcher.min.js></script></head><body class=language-unknown><header><nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark justify-content-between"><a class=navbar-brand href=/v1.0/><span class=logo><svg viewBox="0 0 300 300"><circle cx="150" cy="150" r="150" stroke-width="2" /><polygon points="65,240 225,240 125,270"/><polygon points="65,230 125,220 125,110"/><polygon points="135,220 225,230 135,30"/></svg></span><span class=brand-name>Istioldie 1.0</span></a>
|
||
<button class=navbar-toggler type=button data-toggle=collapse data-target=#navbarCollapse aria-controls=navbarCollapse aria-expanded=false aria-label="Toggle navigation">
|
||
<span class=navbar-toggler-icon></span></button><div class="collapse navbar-collapse justify-content-end" id=navbarCollapse><ul id=navbar-links class="navbar-nav active"><li class=nav-item><a class=nav-link title="Learn how to deploy, use, and operate Istio." href=/v1.0/docs/>Docs</a></li><li class=nav-item><a class="nav-link active" title="Posts about using Istio." href=/v1.0/blog/2019/announcing-1.0.6/>Blog</a></li><li class=nav-item><a class=nav-link title="A bunch of resources to help you deploy, configure and use Istio." href=/v1.0/help/>Help</a></li><li class=nav-item><a class=nav-link title="Get a bit more in-depth info about the Istio project." href=/v1.0/about/>About</a></li><li class="nav-item dropdown" id=gearDropdown style=white-space:nowrap><a title="Options and Settings" href class=nav-link data-toggle=dropdown aria-label=Tools aria-haspopup=true aria-expanded=false><i style=width:1em class="fa fa-lg fa-cog"></i></a><div class="dropdown-menu dropdown-menu-right" aria-labelledby=gearDropdown><a class=dropdown-item id=light-theme-item href onclick="setActiveStyleSheet('light');return false;">Light Theme</a>
|
||
<a class=dropdown-item id=dark-theme-item href onclick="setActiveStyleSheet('dark');return false;">Dark Theme</a><div class=dropdown-divider></div><h6 class=dropdown-header>Other versions of this site</h6><a href=https://istio.io class=dropdown-item>Current Release</a>
|
||
<a href=https://preliminary.istio.io class=dropdown-item>Next Release</a>
|
||
<a href=https://archive.istio.io class=dropdown-item>Older Releases</a></div></li><li class=nav-item><a id=search_show class=nav-link href title="Search istio.io" aria-label=Search><i style=width:1em class="fa fa-lg fa-search"></i></a></li></ul><form name=cse id=search_form class="form-inline mr-sm-2" role=search><input type=hidden name=cx value=013699703217164175118:iwwf17ikgf4>
|
||
<input type=hidden name=ie value=utf-8>
|
||
<input type=hidden name=hl value=en>
|
||
<input type=hidden id=search_page_url value=/v1.0/search.html>
|
||
<input id=search_textbox class=form-control name=q type=text aria-label="Search this site">
|
||
<button id=search_close type=reset aria-label="Cancel Search"><i class="far fa-lg fa-times-circle"></i></button></form></div></nav></header><div class=blog><div class=container-fluid><div class="row row-offcanvas"><div class="col-0 col-md-3 col-xl-2 sidebar-offcanvas"><nav class="sidebar d-print-none"><div class=spacer></div><div class=directory role=tablist><div class=card><div class=card-header role=tab id=header0><a data-toggle=collapse href=#collapse0 title="Blog posts for 2019." role=button aria-controls=collapse0><div><img src=/v1.0/img/blog.svg alt=Icon class=page_icon>
|
||
2019 Posts</div></a></div><div id=collapse0 class=collapse data-parent=#sidebar role=tabpanel aria-labelledby=header0><div class=card-body><ul class=tree><li><a title="Istio 1.0.6 patch release." href=/v1.0/blog/2019/announcing-1.0.6/>Announcing Istio 1.0.6</a></li><li><a title="Addressing application startup ordering and startup latency using AppSwitch." href=/v1.0/blog/2019/appswitch/>Sidestepping Dependency Ordering with AppSwitch</a></li><li><a title="Describes how to deploy a custom ingress gateway using cert-manager manually." href=/v1.0/blog/2019/custom-ingress-gateway/>Deploy a custom ingress gateway using cert-manager</a></li><li><a title="Istio has a new discussion board." href=/v1.0/blog/2019/announcing-discuss.istio.io/>Announcing discuss.istio.io</a></li></ul></div></div></div><div class=card><div class=card-header role=tab id=header1><a data-toggle=collapse href=#collapse1 title="Blog posts for 2018." role=button aria-controls=collapse1><div><img src=/v1.0/img/blog.svg alt=Icon class=page_icon>
|
||
2018 Posts</div></a></div><div id=collapse1 class="collapse show" data-parent=#sidebar role=tabpanel aria-labelledby=header1><div class=card-body><ul class=tree><li><a title="Istio 1.0.5 patch release." href=/v1.0/blog/2018/announcing-1.0.5/>Announcing Istio 1.0.5</a></li><li><a title="How to use Istio for traffic management without deploying sidecar proxies." href=/v1.0/blog/2018/incremental-traffic-management/>Incremental Istio Part 1, Traffic Management</a></li><li><a title="Istio 1.0.4 patch release." href=/v1.0/blog/2018/announcing-1.0.4/>Announcing Istio 1.0.4</a></li><li><a title="Istio 1.0.3 patch release." href=/v1.0/blog/2018/announcing-1.0.3/>Announcing Istio 1.0.3</a></li><li><a title="Istio 1.0.2 patch release." href=/v1.0/blog/2018/announcing-1.0.2/>Announcing Istio 1.0.2</a></li><li><a title="Istio 1.0.1 patch release." href=/v1.0/blog/2018/announcing-1.0.1/>Announcing Istio 1.0.1</a></li><li><a title="Istio hosting an all day Twitch stream to celebrate the 1.0 release." href=/v1.0/blog/2018/istio-twitch-stream/>All Day Istio Twitch Stream</a></li><li><a title="How HP is building its next-generation footwear personalization platform on Istio." href=/v1.0/blog/2018/hp/>Istio a Game Changer for HP's FitStation Platform</a></li><li><a title="Istio is ready for production use with its 1.0 release." href=/v1.0/blog/2018/announcing-1.0/>Announcing Istio 1.0</a></li><li><a title="Automatic application onboarding and latency optimizations using AppSwitch." href=/v1.0/blog/2018/delayering-istio/delayering-istio/>Delayering Istio with AppSwitch</a></li><li><a title="Describe Istio's authorization feature and how to use it in various use cases." href=/v1.0/blog/2018/istio-authorization/>Micro-Segmentation with Istio Authorization</a></li><li><span class=current title="How to export Istio Access Logs to different sinks like BigQuery, GCS, Pub/Sub through Stackdriver.">Exporting Logs to BigQuery, GCS, Pub/Sub through Stackdriver</span></li><li><a title="Introduction, motivation and design principles for the Istio v1alpha3 routing API." href=/v1.0/blog/2018/v1alpha3-routing/>Introducing the Istio v1alpha3 routing API</a></li><li><a title="Describes how to configure Istio ingress with a network load balancer on AWS." href=/v1.0/blog/2018/aws-nlb/>Configuring Istio Ingress with AWS NLB</a></li><li><a title="Using Kubernetes namespaces and RBAC to create an Istio soft multi-tenancy environment." href=/v1.0/blog/2018/soft-multitenancy/>Istio Soft Multi-tenancy Support</a></li><li><a title="An introduction to safer, lower-risk deployments and release to production." href=/v1.0/blog/2018/traffic-mirroring/>Traffic Mirroring with Istio for Testing in Production</a></li><li><a title="Describes a simple scenario based on Istio's Bookinfo example." href=/v1.0/blog/2018/egress-tcp/>Consuming External TCP Services</a></li><li><a title="Describes a simple scenario based on Istio's Bookinfo example." href=/v1.0/blog/2018/egress-https/>Consuming External Web Services</a></li></ul></div></div></div><div class=card><div class=card-header role=tab id=header2><a data-toggle=collapse href=#collapse2 title="Blog posts for 2017." role=button aria-controls=collapse2><div><img src=/v1.0/img/blog.svg alt=Icon class=page_icon>
|
||
2017 Posts</div></a></div><div id=collapse2 class=collapse data-parent=#sidebar role=tabpanel aria-labelledby=header2><div class=card-body><ul class=tree><li><a title="Improving availability and reducing latency." href=/v1.0/blog/2017/mixer-spof-myth/>Mixer and the SPOF Myth</a></li><li><a title="Provides an overview of Mixer's plug-in architecture." href=/v1.0/blog/2017/adapter-model/>Mixer Adapter Model</a></li><li><a title="Istio 0.2 announcement." href=/v1.0/blog/2017/0.2-announcement/>Announcing Istio 0.2</a></li><li><a title="How Kubernetes Network Policy relates to Istio policy." href=/v1.0/blog/2017/0.1-using-network-policy/>Using Network Policy with Istio</a></li><li><a title="Using Istio to create autoscaled canary deployments." href=/v1.0/blog/2017/0.1-canary/>Canary Deployments using Istio</a></li><li><a title="Istio Auth 0.1 announcement." href=/v1.0/blog/2017/0.1-auth/>Using Istio to Improve End-to-End Security</a></li><li><a title="Istio 0.1 announcement." href=/v1.0/blog/2017/0.1-announcement/>Introducing Istio</a></li></ul></div></div></div></div></nav></div><div class="col-12 col-md-9 col-xl-8"><p class=d-md-none><label class=sidebar-toggler data-toggle=offcanvas><i class="fa fa-sign-out-alt"></i></label></p><main aria-labelledby=title><div class=pagenav><p><a href=/v1.0/blog/2018/ title="Blog posts for 2018."><i style=transform:scaleX(-1) class="fa fa-level-up-alt"></i> 2018 Posts</a></p></div><h1 id=title>Exporting Logs to BigQuery, GCS, Pub/Sub through Stackdriver</h1><p class=byline>By <span class=attribution>Nupur Garg and Douglas Reid</span>
|
||
/
|
||
<span class=publish_date>July 9, 2018</span></p><nav class="toc-inlined d-xl-none d-print-none"><hr><div class=directory role=directory><nav id=InlinedTableOfContents><ul><li><a href=#before-you-begin>Before you begin</a></li><li><a href=#configuring-istio-to-export-logs>Configuring Istio to export logs</a></li><ul><li><a href=#setting-up-various-log-sinks>Setting up various log sinks</a></li><ul><li><a href=#bigquery>BigQuery</a></li><li><a href=#google-cloud-storage-gcs>Google Cloud Storage (GCS)</a></li><li><a href=#google-cloud-pubsub>Google Cloud Pub/Sub</a></li></ul><li><a href=#setting-up-stackdriver>Setting up Stackdriver</a></li></ul><li><a href=#understanding-what-happened>Understanding what happened</a></li><li><a href=#cleanup>Cleanup</a></li><li><a href=#availability-of-logs-in-export-sinks>Availability of logs in export sinks</a></li></ul></nav></div><hr></nav><p>This post shows how to direct Istio logs to <a href=https://cloud.google.com/stackdriver/>Stackdriver</a>
|
||
and export those logs to various configured sinks such as such as
|
||
<a href=https://cloud.google.com/bigquery/>BigQuery</a>, <a href=https://cloud.google.com/storage/>Google Cloud Storage(GCS)</a>
|
||
or <a href=https://cloud.google.com/pubsub/>Cloud Pub/Sub</a>. At the end of this post you can perform
|
||
analytics on Istio data from your favorite places such as BigQuery, GCS or Cloud Pub/Sub.</p><p>The <a href=/v1.0/docs/examples/bookinfo/>Bookinfo</a> sample application is used as the example
|
||
application throughout this task.</p><h2 id=before-you-begin>Before you begin</h2><p><a href=/v1.0/docs/setup/>Install Istio</a> in your cluster and deploy an application.</p><h2 id=configuring-istio-to-export-logs>Configuring Istio to export logs</h2><p>Istio exports logs using the <code>logentry</code> <a href=/v1.0/docs/reference/config/policy-and-telemetry/templates/logentry>template</a>.
|
||
This specifies all the variables that are available for analysis. It
|
||
contains information like source service, destination service, auth
|
||
metrics (coming..) among others. Following is a diagram of the pipeline:</p><figure style=width:75%><div class=wrapper-with-intrinsic-ratio style=padding-bottom:75%><a class=not-for-endnotes href=/v1.0/blog/2018/export-logs-through-stackdriver/./istio-analytics-using-stackdriver.png><img class=element-to-stretch src=/v1.0/blog/2018/export-logs-through-stackdriver/./istio-analytics-using-stackdriver.png alt="Diagram of exporting logs from Istio to Stackdriver for analysis" title="Diagram of exporting logs from Istio to Stackdriver for analysis"></a></div><figcaption>Diagram of exporting logs from Istio to Stackdriver for analysis</figcaption></figure><p>Istio supports exporting logs to Stackdriver which can in turn be configured to export
|
||
logs to your favorite sink like BigQuery, Pub/Sub or GCS. Please follow the steps
|
||
below to setup your favorite sink for exporting logs first and then Stackdriver
|
||
in Istio.</p><h3 id=setting-up-various-log-sinks>Setting up various log sinks</h3><p>Common setup for all sinks:</p><ol><li>Enable <a href=https://cloud.google.com/monitoring/api/enable-api>Stackdriver Monitoring API</a> for the project.</li><li>Make sure <code>principalEmail</code> that would be setting up the sink has write access to the project and Logging Admin role permissions.</li><li>Make sure the <code>GOOGLE_APPLICATION_CREDENTIALS</code> environment variable is set. Please follow instructions <a href=https://cloud.google.com/docs/authentication/getting-started>here</a> to set it up.</li></ol><h4 id=bigquery>BigQuery</h4><ol><li><a href=https://cloud.google.com/bigquery/docs/datasets>Create a BigQuery dataset</a> as a destination for the logs export.</li><li>Record the ID of the dataset. It will be needed to configure the Stackdriver handler.
|
||
It would be of the form <code>bigquery.googleapis.com/projects/[PROJECT_ID]/datasets/[DATASET_ID]</code></li><li>Give <a href=https://cloud.google.com/logging/docs/api/tasks/exporting-logs#writing_to_the_destination>sink’s writer identity</a>: <a href=mailto:%60cloud-logs@system.gserviceaccount.com>`cloud-logs@system.gserviceaccount.com</a>` BigQuery Data Editor role in IAM.</li><li>If using <a href=/v1.0/docs/setup/kubernetes/platform-setup/gke/>Google Kubernetes Engine</a>, make sure <code>bigquery</code> <a href=https://cloud.google.com/sdk/gcloud/reference/container/clusters/create>Scope</a> is enabled on the cluster.</li></ol><h4 id=google-cloud-storage-gcs>Google Cloud Storage (GCS)</h4><ol><li><a href=https://cloud.google.com/storage/docs/creating-buckets>Create a GCS bucket</a> where you would like logs to get exported in GCS.</li><li>Recode the ID of the bucket. It will be needed to configure Stackdriver.
|
||
It would be of the form <code>storage.googleapis.com/[BUCKET_ID]</code></li><li>Give <a href=https://cloud.google.com/logging/docs/api/tasks/exporting-logs#writing_to_the_destination>sink’s writer identity</a>: <a href=mailto:%60cloud-logs@system.gserviceaccount.com>`cloud-logs@system.gserviceaccount.com</a>` Storage Object Creator role in IAM.</li></ol><h4 id=google-cloud-pubsub>Google Cloud Pub/Sub</h4><ol><li><a href=https://cloud.google.com/pubsub/docs/admin>Create a topic</a> where you would like logs to get exported in Google Cloud Pub/Sub.</li><li>Recode the ID of the topic. It will be needed to configure Stackdriver.
|
||
It would be of the form <code>pubsub.googleapis.com/projects/[PROJECT_ID]/topics/[TOPIC_ID]</code></li><li>Give <a href=https://cloud.google.com/logging/docs/api/tasks/exporting-logs#writing_to_the_destination>sink’s writer identity</a>: <a href=mailto:%60cloud-logs@system.gserviceaccount.com>`cloud-logs@system.gserviceaccount.com</a>` Pub/Sub Publisher role in IAM.</li><li>If using <a href=/v1.0/docs/setup/kubernetes/platform-setup/gke/>Google Kubernetes Engine</a>, make sure <code>pubsub</code> <a href=https://cloud.google.com/sdk/gcloud/reference/container/clusters/create>Scope</a> is enabled on the cluster.</li></ol><h3 id=setting-up-stackdriver>Setting up Stackdriver</h3><p>A Stackdriver handler must be created to export data to Stackdriver. The configuration for
|
||
a Stackdriver handler is described <a href=/v1.0/docs/reference/config/policy-and-telemetry/adapters/stackdriver/>here</a>.</p><ol><li><p>Save the following yaml file as <code>stackdriver.yaml</code>. Replace <code><project_id>, <sink_id>, <sink_destination>, <log_filter></code> with their specific values.</p><pre><code class=language-yaml>apiVersion: "config.istio.io/v1alpha2"
|
||
kind: stackdriver
|
||
metadata:
|
||
name: handler
|
||
namespace: istio-system
|
||
spec:
|
||
# We'll use the default value from the adapter, once per minute, so we don't need to supply a value.
|
||
# pushInterval: 1m
|
||
# Must be supplied for the Stackdriver adapter to work
|
||
project_id: "<project_id>"
|
||
# One of the following must be set; the preferred method is `appCredentials`, which corresponds to
|
||
# Google Application Default Credentials.
|
||
# If none is provided we default to app credentials.
|
||
# appCredentials:
|
||
# apiKey:
|
||
# serviceAccountPath:
|
||
# Describes how to map Istio logs into Stackdriver.
|
||
logInfo:
|
||
accesslog.logentry.istio-system:
|
||
payloadTemplate: '{{or (.sourceIp) "-"}} - {{or (.sourceUser) "-"}} [{{or (.timestamp.Format "02/Jan/2006:15:04:05 -0700") "-"}}] "{{or (.method) "-"}} {{or (.url) "-"}} {{or (.protocol) "-"}}" {{or (.responseCode) "-"}} {{or (.responseSize) "-"}}'
|
||
httpMapping:
|
||
url: url
|
||
status: responseCode
|
||
requestSize: requestSize
|
||
responseSize: responseSize
|
||
latency: latency
|
||
localIp: sourceIp
|
||
remoteIp: destinationIp
|
||
method: method
|
||
userAgent: userAgent
|
||
referer: referer
|
||
labelNames:
|
||
- sourceIp
|
||
- destinationIp
|
||
- sourceService
|
||
- sourceUser
|
||
- sourceNamespace
|
||
- destinationIp
|
||
- destinationService
|
||
- destinationNamespace
|
||
- apiClaims
|
||
- apiKey
|
||
- protocol
|
||
- method
|
||
- url
|
||
- responseCode
|
||
- responseSize
|
||
- requestSize
|
||
- latency
|
||
- connectionMtls
|
||
- userAgent
|
||
- responseTimestamp
|
||
- receivedBytes
|
||
- sentBytes
|
||
- referer
|
||
sinkInfo:
|
||
id: '<sink_id>'
|
||
destination: '<sink_destination>'
|
||
filter: '<log_filter>'
|
||
---
|
||
apiVersion: "config.istio.io/v1alpha2"
|
||
kind: rule
|
||
metadata:
|
||
name: stackdriver
|
||
namespace: istio-system
|
||
spec:
|
||
match: "true" # If omitted match is true.
|
||
actions:
|
||
- handler: handler.stackdriver
|
||
instances:
|
||
- accesslog.logentry
|
||
---</code></pre></li><li><p>Push the configuration</p><pre><code class=language-command>$ kubectl apply -f stackdriver.yaml
|
||
stackdriver "handler" created
|
||
rule "stackdriver" created
|
||
logentry "stackdriverglobalmr" created
|
||
metric "stackdriverrequestcount" created
|
||
metric "stackdriverrequestduration" created
|
||
metric "stackdriverrequestsize" created
|
||
metric "stackdriverresponsesize" created</code></pre></li><li><p>Send traffic to the sample application.</p><p>For the Bookinfo sample, visit <code>http://$GATEWAY_URL/productpage</code> in your web
|
||
browser or issue the following command:</p><pre><code class=language-command>$ curl http://$GATEWAY_URL/productpage</code></pre></li><li><p>Verify that logs are flowing through Stackdriver to the configured sink.</p><ul><li>Stackdriver: Navigate to the <a href=https://pantheon.corp.google.com/logs/viewer>Stackdriver Logs
|
||
Viewer</a> for your project
|
||
and look under “GKE Container” -> “Cluster Name” -> “Namespace Id” for
|
||
Istio Access logs.</li><li>BigQuery: Navigate to the <a href=https://bigquery.cloud.google.com/>BigQuery
|
||
Interface</a> for your project and you
|
||
should find a table with prefix <code>accesslog_logentry_istio</code> in your sink
|
||
dataset.</li><li>GCS: Navigate to the <a href=https://pantheon.corp.google.com/storage/browser/>Storage
|
||
Browser</a> for your
|
||
project and you should find a bucket named
|
||
<code>accesslog.logentry.istio-system</code> in your sink bucket.</li><li>Pub/Sub: Navigate to the <a href=https://pantheon.corp.google.com/cloudpubsub/topicList>Pub/Sub
|
||
TopicList</a> for
|
||
your project and you should find a topic for <code>accesslog</code> in your sink
|
||
topic.</li></ul></li></ol><h2 id=understanding-what-happened>Understanding what happened</h2><p><code>Stackdriver.yaml</code> file above configured Istio to send accesslogs to
|
||
Stackdriver and then added a sink configuration where these logs could be
|
||
exported. In detail as follows:</p><ol><li><p>Added a handler of kind <code>stackdriver</code></p><pre><code class=language-yaml>apiVersion: "config.istio.io/v1alpha2"
|
||
kind: stackdriver
|
||
metadata:
|
||
name: handler
|
||
namespace: <your defined namespace></code></pre></li><li><p>Added logInfo in spec</p><pre><code class=language-yaml>spec:
|
||
logInfo: accesslog.logentry.istio-system:
|
||
labelNames:
|
||
- sourceIp
|
||
- destinationIp
|
||
...
|
||
...
|
||
sinkInfo:
|
||
id: '<sink_id>'
|
||
destination: '<sink_destination>'
|
||
filter: '<log_filter>'</code></pre><p>In the above configuration sinkInfo contains information about the sink where you want
|
||
the logs to get exported to. For more information on how this gets filled for different sinks please refer
|
||
<a href=https://cloud.google.com/logging/docs/export/#sink-terms>here</a>.</p></li><li><p>Added a rule for Stackdriver</p><pre><code class=language-yaml>apiVersion: "config.istio.io/v1alpha2"
|
||
kind: rule
|
||
metadata:
|
||
name: stackdriver
|
||
namespace: istio-system spec:
|
||
match: "true" # If omitted match is true
|
||
actions:
|
||
- handler: handler.stackdriver
|
||
instances:
|
||
- accesslog.logentry</code></pre></li></ol><h2 id=cleanup>Cleanup</h2><ul><li><p>Remove the new Stackdriver configuration:</p><pre><code class=language-command>$ kubectl delete -f stackdriver.yaml</code></pre></li><li><p>If you are not planning to explore any follow-on tasks, refer to the
|
||
<a href=/v1.0/docs/examples/bookinfo/#cleanup>Bookinfo cleanup</a> instructions to shutdown
|
||
the application.</p></li></ul><h2 id=availability-of-logs-in-export-sinks>Availability of logs in export sinks</h2><p>Export to BigQuery is within minutes (we see it to be almost instant), GCS can
|
||
have a delay of 2 to 12 hours and Pub/Sub is almost immediately.</p></main><div class="container-fluid d-print-none"><br><div class=row><div class="col-6 pagenav"><p><a title="Describe Istio's authorization feature and how to use it in various use cases." href=/v1.0/blog/2018/istio-authorization/><i class="fa fa-long-arrow-alt-left"></i>Micro-Segmentation with Istio Authorization</a></p></div><div class="col-6 pagenav" style=text-align:right><p><a title="Introduction, motivation and design principles for the Istio v1alpha3 routing API." href=/v1.0/blog/2018/v1alpha3-routing/>Introducing the Istio v1alpha3 routing API
|
||
<i class="fa fa-long-arrow-alt-right"></i></a></p></div></div></div><div class="d-none d-print-block" aria-hidden=true><h2>Links</h2><ol id=endnotes></ol></div></div><div class="col-12 col-md-2 d-none d-xl-block d-print-none"><nav class=toc><div class=spacer></div><div id=toc class=directory role=directory><nav id=TableOfContents><ul><li><a href=#before-you-begin>Before you begin</a></li><li><a href=#configuring-istio-to-export-logs>Configuring Istio to export logs</a></li><ul><li><a href=#setting-up-various-log-sinks>Setting up various log sinks</a></li><ul><li><a href=#bigquery>BigQuery</a></li><li><a href=#google-cloud-storage-gcs>Google Cloud Storage (GCS)</a></li><li><a href=#google-cloud-pubsub>Google Cloud Pub/Sub</a></li></ul><li><a href=#setting-up-stackdriver>Setting up Stackdriver</a></li></ul><li><a href=#understanding-what-happened>Understanding what happened</a></li><li><a href=#cleanup>Cleanup</a></li><li><a href=#availability-of-logs-in-export-sinks>Availability of logs in export sinks</a></li></ul></nav></div></nav></div></div></div></div><footer class="d-print-none container-fluid"><div class=row><div class="col-5 col-lg-4" role=navigation><div class=container-fluid><div class=row><div class=icon><span>discuss</span>
|
||
<a title="Join the Istio discussion board to participate in discussions and get help troubleshooting problems" href=https://discuss.istio.io aria-label="Istio discussion board"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path d="M225.9 32C103.3 32 0 130.5.0 252.1.0 256 .1 480 .1 480l225.8-.2c122.7.0 222.1-102.3 222.1-223.9S348.6 32 225.9 32zM224 384c-19.4.0-37.9-4.3-54.4-12.1L88.5 392l22.9-75c-9.8-18.1-15.4-38.9-15.4-61 0-70.7 57.3-128 128-128s128 57.3 128 128-57.3 128-128 128z" /></svg></a></div><div class=icon><span>slack</span>
|
||
<a title="Interactively discuss issues with the Istio community on Slack" href=https://istio.slack.com aria-label=slack><svg viewBox="0 0 31.444 31.443"><path d="M31.202 16.369c-.62-1.388-2.249-2.011-3.637-1.391l-1.325.594-3.396-7.591 1.325-.592c1.388-.622 2.01-2.25 1.389-3.637-.62-1.389-2.248-2.012-3.637-1.39l-1.324.593-.593-1.326c-.621-1.388-2.249-2.009-3.637-1.388-1.388.62-2.009 2.247-1.389 3.637l.593 1.325L7.98 8.598 7.388 7.273c-.621-1.39-2.249-2.009-3.637-1.39C2.363 6.504 1.742 8.132 2.362 9.52l.592 1.324L1.63 11.438c-1.388.621-2.01 2.247-1.389 3.636.62 1.388 2.249 2.01 3.637 1.39l1.325-.594 3.394 7.592-1.325.592c-1.388.621-2.009 2.25-1.389 3.637.621 1.389 2.249 2.011 3.637 1.391l1.324-.593.593 1.325c.621 1.389 2.249 2.01 3.637 1.389 1.387-.62 2.009-2.248 1.388-3.636l-.591-1.326 7.591-3.394.592 1.321c.621 1.391 2.248 2.013 3.637 1.392 1.388-.619 2.01-2.248 1.389-3.637l-.592-1.324 1.323-.594C31.201 19.384 31.823 17.757 31.202 16.369zM13.623 21.215l-3.395-7.593 7.591-3.394 3.395 7.591L13.623 21.215z"/></svg></a></div><div class=icon><span>twitter</span>
|
||
<a title="Follow us on Twitter to get the latest news" href=https://twitter.com/IstioMesh aria-label=Twitter><svg viewBox="0 0 310 310"><path d="M302.973 57.388c-4.87 2.16-9.877 3.983-14.993 5.463 6.057-6.85 10.675-14.91 13.494-23.73.632-1.977-.023-4.141-1.648-5.434-1.623-1.294-3.878-1.449-5.665-.39-10.865 6.444-22.587 11.075-34.878 13.783-12.381-12.098-29.197-18.983-46.581-18.983-36.695.0-66.549 29.853-66.549 66.547.0 2.89.183 5.764.545 8.598C101.163 99.244 58.83 76.863 29.76 41.204c-1.036-1.271-2.632-1.956-4.266-1.825-1.635.128-3.104 1.05-3.93 2.467-5.896 10.117-9.013 21.688-9.013 33.461.0 16.035 5.725 31.249 15.838 43.137-3.075-1.065-6.059-2.396-8.907-3.977-1.529-.851-3.395-.838-4.914.033-1.52.871-2.473 2.473-2.513 4.224-.007.295-.007.59-.007.889.0 23.935 12.882 45.484 32.577 57.229-1.692-.169-3.383-.414-5.063-.735-1.732-.331-3.513.276-4.681 1.597-1.17 1.32-1.557 3.16-1.018 4.84 7.29 22.76 26.059 39.501 48.749 44.605-18.819 11.787-40.34 17.961-62.932 17.961-4.714.0-9.455-.277-14.095-.826-2.305-.274-4.509 1.087-5.294 3.279-.785 2.193.047 4.638 2.008 5.895 29.023 18.609 62.582 28.445 97.047 28.445 67.754.0 110.139-31.95 133.764-58.753 29.46-33.421 46.356-77.658 46.356-121.367.0-1.826-.028-3.67-.084-5.508 11.623-8.757 21.63-19.355 29.773-31.536 1.237-1.85 1.103-4.295-.33-5.998C307.394 57.037 305.009 56.486 302.973 57.388z"/></svg></a></div><div class=icon><span>stack overflow</span>
|
||
<a title="Stack Overflow is where you can ask questions and find curated answers on deploying, configuring, and using Istio" href=https://stackoverflow.com/questions/tagged/istio aria-label="Stack Overflow"><svg viewBox="0 0 120 120"><polygon points="84.4,93.8 84.4,70.6 92.1,70.6 92.1,101.5 22.6,101.5 22.6,70.6 30.3,70.6 30.3,93.8"/><path d="M38.8 68.4l37.8 7.9 1.6-7.6-37.8-7.9L38.8 68.4zM43.8 50.4l35 16.3 3.2-7-35-16.4L43.8 50.4zM53.5 33.2l29.7 24.7 4.9-5.9L58.4 27.3 53.5 33.2zM72.7 14.9l-6.2 4.6 23 31 6.2-4.6-23-31zM38 86h38.6v-7.7H38V86z"/></svg></a></div></div><div class="tag row d-none d-lg-flex">for everyone</div></div></div><div class="col-7 col-lg-4"><p class="text-center copyright" role=contentinfo>Istio
|
||
Archive
|
||
1.0<br>© 2019 Istio Authors, <a href=https://policies.google.com/privacy>Privacy Policy</a><br>Archived on March 19, 2019</p></div><div class="col-6 col-lg-4 d-none d-lg-flex" role=navigation><div class=container-fluid><div class="row justify-content-end"><div class=icon><span>github</span>
|
||
<a title="GitHub is where development takes place on Istio code" href=https://github.com/istio/community aria-label=GitHub><svg viewBox="0 0 478.165 478.165"><path d="M349.22 55.768c6.136 14.046 10.241 37.556 4.224 54.69 24.426 20.999 33.073 71.904 21.079 113.704 35.006 2.73 76.666-1.235 103.642 9.484-25.183-3.248-59.651-9.563-91.987-7.431-6.136.458-15.361-.239-14.903 8.408 37.735 3.008 75.092 6.117 105.894 15.779-30.702-4.981-67.74-12.552-105.894-13.668-15.54 30.921-47.239 46.262-90.991 49.49 4.682 10.261 13.847 14.066 15.879 30.702 3.267 24.406-4.881 60.328 3.208 76.686 4.064 7.89 10.579 8.009 14.863 14.604-10.699 12.871-37.257-1.395-40.186-14.604-5.14-22.852 7.89-58.256-6.415-73.737.996 24.865-5.718 59.85.996 82.145 2.789 8.806 10.659 12.113 8.647 20.063-49.809 5.08-28.989-64.373-37.177-105.356-7.471.697-4.204 11.197-4.224 15.76-.199 40.106 8.189 94.836-34.846 89.556-1.315-8.348 5.838-11.217 8.467-19.007 7.91-22.434-1.454-56.045 2.112-83.161-16.417 12.512 1.793 55.666-8.428 77.961-5.838 12.671-24.785 18.27-39.19 12.651 1.873-9.464 11.695-7.989 15.879-16.875 5.818-12.452.02-30.244 2.092-48.494-30.423 6.097-53.993-.877-65.608-20.023-5.12-8.507-6.356-18.708-12.632-26.219-6.117-7.551-16.098-8.507-19.087-18.808 37.755-9.185 39.17 38.771 73.06 39.807 10.44.418 15.799-2.909 25.402-5.16 2.749-12.113 8.428-21.039 16.875-27.494-42.078-5.658-76.865-18.788-93.023-50.466-38.293 1.893-73.339 7.013-105.894 14.843 29.547-10.679 65.807-14.604 104.778-15.819-2.351-13.807-22.434-10.022-34.866-9.543C47.677 227.17 18.449 230.138.0 233.645c26.817-9.543 64.233-8.348 100.454-8.428-11.038-34.767-7.232-90.014 17.015-110.615-6.854-17.254-4.722-45.346 4.184-58.834 27.036 1.175 43.374 12.891 60.388 24.247 21.019-6.017 43.035-9.045 71.904-7.451 12.133.677 24.705 6.097 33.731 5.32 8.906-.877 18.728-10.898 27.534-14.843C326.507 58.099 336.17 56.206 349.22 55.768z"/></svg></a></div><div class=icon><span>drive</span>
|
||
<a title="Access our team drive if you'd like to take a look at the Istio technical design documents" href=https://groups.google.com/forum/#!forum/istio-team-drive-access aria-label="team drive"><svg viewBox="0 0 207.027 207.027"><path d="M69.866 15.557.0 138.919l28.732 52.552 143.288-.029 35.008-59.588L136.39 15.735 69.866 15.557zM17.166 139.046 74.268 38.205 91.21 67.783 33.24 168.447 17.166 139.046zM99.841 82.851l23.805 41.558-47.732-.006L99.841 82.851zM163.434 176.443l-117.332.024 21.53-37.065 64.606.008.067.119 52.865-.085L163.434 176.443zM140.932 124.411 90.157 35.767l-2.966-5.178 40.751.121 57.003 93.706L140.932 124.411z"/></svg></a></div><div class=icon><span>working groups</span>
|
||
<a title="If you'd like to contribute to the Istio project, consider participating in our working groups" href=https://github.com/istio/community/blob/master/WORKING-GROUPS.md aria-label="working groups"><svg viewBox="0 -45 439.833 439.833"><polygon points="246.048,195.833 299.966,235.085 319.497,227.296 276.278,195.833"/><polygon points="193.786,195.833 163.556,195.833 120.33,227.3 139.862,235.089"/><path d="M219.927 11.558c-23.854.0-37.057 12.362-36.814 36.182.348 32.623 14.211 52.414 36.814 52.068.0.0 36.802 1.492 36.802-52.068C256.729 23.918 244.294 11.558 219.927 11.558z"/><path d="M285.017 124.567l-36.77-14.659-8.608-7.256c-2.274-1.922-5.636-1.78-7.741.317l-11.973 11.904-12.008-11.907c-2.109-2.094-5.465-2.229-7.736-.313l-8.611 7.256-36.77 14.661c-11.842 4.715-11.83 46.647-12.848 50.497h155.93C296.866 171.228 296.862 129.28 285.017 124.567z"/><path d="M77.976 228.568s36.801 1.492 36.801-52.068c0-23.82-12.434-36.182-36.801-36.182-23.854.0-37.057 12.362-36.814 36.182C41.509 209.124 55.372 228.915 77.976 228.568z"/><path d="M143.065 253.329l-36.77-14.658-8.609-7.256c-2.275-1.923-5.635-1.781-7.742.315l-11.971 11.904-12.008-11.908c-2.109-2.094-5.465-2.229-7.736-.312l-8.611 7.256-36.77 14.66C1.006 258.045 1.018 299.977.0 303.827h155.93C154.915 299.988 154.911 258.042 143.065 253.329z"/><path d="M361.878 228.568s36.801 1.492 36.801-52.068c0-23.82-12.434-36.182-36.801-36.182-23.854.0-37.057 12.362-36.812 36.182C325.411 209.124 339.274 228.915 361.878 228.568z"/><path d="M426.968 253.329l-36.77-14.658-8.609-7.256c-2.273-1.923-5.635-1.781-7.742.315l-11.971 11.904-12.008-11.908c-2.109-2.094-5.465-2.229-7.736-.312l-8.61 7.256-36.771 14.66c-11.842 4.715-11.83 46.646-12.848 50.497h155.93C438.817 299.988 438.812 258.042 426.968 253.329z"/></svg></a></div></div><div class="tag row justify-content-end text-right">for developers</div></div></div></div></footer><div class="d-xl-none d-print-none"><button id=scroll-to-top aria-hidden=true onclick=scrollToTop() title="Back to top"><i class="fa fa-lg fa-arrow-up"></i></button></div><script src=https://code.jquery.com/jquery-3.2.1.slim.min.js integrity=sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN crossorigin=anonymous></script><script src=https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js integrity=sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl crossorigin=anonymous></script><script src=https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/1.7.1/clipboard.min.js></script><script src="https://www.google.com/cse/brand?form=search_form"></script><script src=/v1.0/js/all.min.js data-manual></script></body></html> |