mirror of https://github.com/istio/istio.io.git
170 lines
44 KiB
HTML
170 lines
44 KiB
HTML
<!doctype html><html lang=en itemscope itemtype=https://schema.org/WebPage><head><meta charset=utf-8><meta http-equiv=x-ua-compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=theme-color content="#466BB0"><meta name=title content="Exporting Logs to BigQuery, GCS, Pub/Sub through Stackdriver"><meta name=description content="How to export Istio Access Logs to different sinks like BigQuery, GCS, Pub/Sub through Stackdriver."><meta name=author content="Nupur Garg and Douglas Reid"><meta name=keywords content="microservices,services,mesh"><meta property="og:title" content="Exporting Logs to BigQuery, GCS, Pub/Sub through Stackdriver"><meta property="og:type" content="website"><meta property="og:description" content="How to export Istio Access Logs to different sinks like BigQuery, GCS, Pub/Sub through Stackdriver."><meta property="og:url" content="/v1.9/blog/2018/export-logs-through-stackdriver/"><meta property="og:image" content="/v1.9/img/istio-whitelogo-bluebackground-framed.svg"><meta property="og:image:alt" content="Istio Logo"><meta property="og:image:width" content="112"><meta property="og:image:height" content="150"><meta property="og:site_name" content="Istio"><meta name=twitter:card content="summary"><meta name=twitter:site content="@IstioMesh"><title>Istioldie 1.9 / Exporting Logs to BigQuery, GCS, Pub/Sub through Stackdriver</title><script async src="https://www.googletagmanager.com/gtag/js?id=UA-98480406-2"></script><script>window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments);}
|
||
gtag('js',new Date());gtag('config','UA-98480406-2');</script><link rel=alternate type=application/rss+xml title="Istio Blog" href=/v1.9/blog/feed.xml><link rel=alternate type=application/rss+xml title="Istio News" href=/v1.9/news/feed.xml><link rel=alternate type=application/rss+xml title="Istio Blog and News" href=/v1.9/feed.xml><link rel="shortcut icon" href=/v1.9/favicons/favicon.ico><link rel=apple-touch-icon href=/v1.9/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/v1.9/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/v1.9/favicons/favicon-32x32.png sizes=32x32><link rel=icon type=image/png href=/v1.9/favicons/android-36x36.png sizes=36x36><link rel=icon type=image/png href=/v1.9/favicons/android-48x48.png sizes=48x48><link rel=icon type=image/png href=/v1.9/favicons/android-72x72.png sizes=72x72><link rel=icon type=image/png href=/v1.9/favicons/android-96x96.png sizes=96xW96><link rel=icon type=image/png href=/v1.9/favicons/android-144x144.png sizes=144x144><link rel=icon type=image/png href=/v1.9/favicons/android-192x192.png sizes=192x192><link rel=manifest href=/v1.9/manifest.json><meta name=apple-mobile-web-app-title content="Istio"><meta name=application-name content="Istio"><link rel=stylesheet href="https://fonts.googleapis.com/css?family=Work+Sans:400|Chivo:400|Work+Sans:500,300,600,300italic,400italic,500italic,600italic|Chivo:500,300,600,300italic,400italic,500italic,600italic"><link rel=stylesheet href=/v1.9/css/all.css><script src=/v1.9/js/themes_init.min.js></script></head><body class="language-unknown archive-site"><script>const branchName="release-1.9";const docTitle="Exporting Logs to BigQuery, GCS, Pub\/Sub through Stackdriver";const iconFile="\/v1.9/img/icons.svg";const buttonCopy='Copy to clipboard';const buttonPrint='Print';const buttonDownload='Download';</script><script src="https://www.google.com/cse/brand?form=search-form" defer></script><script src=/v1.9/js/all.min.js data-manual defer></script><header><nav><a id=brand href=/v1.9/><span class=logo><svg viewBox="0 0 300 300"><circle cx="150" cy="150" r="146" stroke-width="2"/><polygon points="65 240 225 240 125 270"/><polygon points="65 230 125 220 125 110"/><polygon points="135 220 225 230 135 30"/></svg></span><span class=name>Istioldie 1.9</span></a><div id=hamburger><svg class="icon hamburger"><use xlink:href="/v1.9/img/icons.svg#hamburger"/></svg></div><div id=header-links><a title="Learn how to deploy, use, and operate Istio." href=/v1.9/docs/>Docs</a>
|
||
<a class=current title="Posts about using Istio." href=/v1.9/blog/2021/>Blog<i class=dot data-prefix=/blog></i></a>
|
||
<a title="Timely news about the Istio project." href=/v1.9/news/>News<i class=dot data-prefix=/news></i></a>
|
||
<a title="Frequently Asked Questions about Istio." href=/v1.9/faq/>FAQ</a>
|
||
<a title="Get a bit more in-depth info about the Istio project." href=/v1.9/about/>About</a><div class=menu><button id=gearDropdownButton class=menu-trigger title="Options and settings" aria-label="Options and Settings" aria-controls=gearDropdownContent><svg class="icon gear"><use xlink:href="/v1.9/img/icons.svg#gear"/></svg></button><div id=gearDropdownContent class=menu-content aria-labelledby=gearDropdownButton role=menu><a tabindex=-1 role=menuitem lang=en id=switch-lang-en class=active>English</a>
|
||
<a tabindex=-1 role=menuitem lang=zh id=switch-lang-zh>中文</a><div role=separator></div><a tabindex=-1 role=menuitem class=active id=light-theme-item>Light Theme</a>
|
||
<a tabindex=-1 role=menuitem id=dark-theme-item>Dark Theme</a><div role=separator></div><a tabindex=-1 role=menuitem id=syntax-coloring-item>Color Examples</a><div role=separator></div><h6>Other versions of this site</h6><a tabindex=-1 role=menuitem onclick="navigateToUrlOrRoot('https://istio.io/blog\/2018\/export-logs-through-stackdriver\/');return false;">Current Release</a>
|
||
<a tabindex=-1 role=menuitem onclick="navigateToUrlOrRoot('https://preliminary.istio.io/blog\/2018\/export-logs-through-stackdriver\/');return false;">Next Release</a>
|
||
<a tabindex=-1 role=menuitem href=https://istio.io/archive>Older Releases</a></div></div><button id=search-show title="Search this site" aria-label=Search><svg class="icon magnifier"><use xlink:href="/v1.9/img/icons.svg#magnifier"/></svg></button></div><form id=search-form name=cse role=search><input type=hidden name=cx value=002184991200833970123:iwwf17ikgf4>
|
||
<input type=hidden name=ie value=utf-8>
|
||
<input type=hidden name=hl value=en>
|
||
<input type=hidden id=search-page-url value=/v1.9/search>
|
||
<input id=search-textbox class=form-control name=q type=search aria-label="Search this site">
|
||
<button id=search-close title="Cancel search" type=reset aria-label="Cancel search"><svg class="icon cancel-x"><use xlink:href="/v1.9/img/icons.svg#cancel-x"/></svg></button></form></nav></header><div class=banner-container></div><main class=primary><div id=sidebar-container class="sidebar-container sidebar-offcanvas"><nav id=sidebar aria-label="Section Navigation"><div class=directory><div class=card><button class="header dynamic" id=card0 title="Blog posts for 2021." aria-controls=card0-body><svg class="icon blog"><use xlink:href="/v1.9/img/icons.svg#blog"/></svg>2021 Posts</button><div class=body aria-labelledby=card0 role=region id=card0-body><ul role=tree aria-expanded=true class=leaf-section aria-labelledby=card0><li role=none><a role=treeitem title="The Product Security working group announces Patch Tuesdays, how 0-days and embargoes are handled, updates to the security best practices page and the notification of the early disclosure list (May 11, 2021)" href=/v1.9/blog/2021/patch-tuesdays/>Updates to how Istio security releases are handled: Patch Tuesday, embargoes, and 0-days</a></li><li role=none><a role=treeitem title="Learn how to use discovery selectors and how they intersect with Sidecar resources (April 30, 2021)" href=/v1.9/blog/2021/discovery-selectors/>Use discovery selectors to configure namespaces for your Istio service mesh</a></li><li role=none><a role=treeitem title="Understanding the upcoming changes to Istio networking, how they may impact your cluster, and what action to take (April 15, 2021)" href=/v1.9/blog/2021/upcoming-networking-changes/>Upcoming networking changes in Istio 1.10</a></li><li role=none><a role=treeitem title="An update on Envoy and Istio's WebAssembly-based extensibility effort (March 5, 2021)" href=/v1.9/blog/2021/wasm-progress/>Istio and Envoy WebAssembly Extensibility, One Year On</a></li><li role=none><a role=treeitem title="A tutorial to help customers migrate from the deprecated v1alpha1 security policy to the supported v1beta1 version (March 3, 2021)" href=/v1.9/blog/2021/migrate-alpha-policy/>Migrate pre-Istio 1.4 Alpha security policy to the current APIs</a></li><li role=none><a role=treeitem title="Understanding the benefits Istio brings, even when no configuration is used (February 25, 2021)" href=/v1.9/blog/2021/zero-config-istio/>Zero Configuration Istio</a></li><li role=none><a role=treeitem title="Learn about sessions, panels, workshops and more on the IstioCon website (February 16, 2021)" href=/v1.9/blog/2021/istiocon-2021-program/>IstioCon 2021: Schedule Is Live!</a></li><li role=none><a role=treeitem title="AuthorizationPolicy now supports CUSTOM action to delegate the authorization to external system (February 9, 2021)" href=/v1.9/blog/2021/better-external-authz/>Better External Authorization</a></li></ul></div></div><div class=card><button class="header dynamic" id=card1 title="Blog posts for 2020." aria-controls=card1-body><svg class="icon blog"><use xlink:href="/v1.9/img/icons.svg#blog"/></svg>2020 Posts</button><div class=body aria-labelledby=card1 role=region id=card1-body><ul role=tree aria-expanded=true class=leaf-section aria-labelledby=card1><li role=none><a role=treeitem title="Deploy multiple Istio egress gateways independently to have fine-grained control of egress communication from the mesh (December 16, 2020)" href=/v1.9/blog/2020/proxying-legacy-services-using-egress-gateways/>Proxying legacy services using Istio egress gateways</a></li><li role=none><a role=treeitem title="How to enable proxy protocol on AWS NLB and Istio ingress gateway (December 11, 2020)" href=/v1.9/blog/2020/show-source-ip/>Proxy protocol on AWS NLB and Istio ingress gateway</a></li><li role=none><a role=treeitem title="The inaugural conference for Istio will take place at the end of February (December 8, 2020)" href=/v1.9/blog/2020/istiocon-2021/>Join us for the first IstioCon in 2021!</a></li><li role=none><a role=treeitem title="How to ensure your clusters are not impacted by Docker Hub rate limiting (December 7, 2020)" href=/v1.9/blog/2020/docker-rate-limit/>Handling Docker Hub rate limiting</a></li><li role=none><a role=treeitem title="Workload Local DNS resolution to simplify VM integration, multicluster, and more (November 12, 2020)" href=/v1.9/blog/2020/dns-proxy/>Expanding into New Frontiers - Smart DNS Proxying in Istio</a></li><li role=none><a role=treeitem title="Announcing the four newest Istio Steering Committee members (September 29, 2020)" href=/v1.9/blog/2020/steering-election-results/>2020 Steering Committee Election Results</a></li><li role=none><a role=treeitem title="The effect of security policies on latency of requests (September 15, 2020)" href=/v1.9/blog/2020/large-scale-security-policy-performance-tests/>Large Scale Security Policy Performance Tests</a></li><li role=none><a role=treeitem title="A new deployment model for Istio (August 27, 2020)" href=/v1.9/blog/2020/new-deployment-model/>Deploying Istio Control Planes Outside the Mesh</a></li><li role=none><a role=treeitem title="The Istio Steering Committee is now in part proportionally allocated to companies based on contribution, and in part elected by community members (August 24, 2020)" href=/v1.9/blog/2020/steering-changes/>Introducing the new Istio steering committee</a></li><li role=none><a role=treeitem title="An alternative sidecar proxy for Istio (July 28, 2020)" href=/v1.9/blog/2020/mosn-proxy/>Using MOSN with Istio: an alternative data plane</a></li><li role=none><a role=treeitem title="An update on trademarks and project governance (July 8, 2020)" href=/v1.9/blog/2020/open-usage/>Open and neutral: transferring our trademarks to the Open Usage Commons</a></li><li role=none><a role=treeitem title="A new way to manage installation of telemetry addons (June 4, 2020)" href=/v1.9/blog/2020/addon-rework/>Reworking our Addon Integrations</a></li><li role=none><a role=treeitem title="Describing the new functionality of Workload Entries (May 21, 2020)" href=/v1.9/blog/2020/workload-entry/>Introducing Workload Entries</a></li><li role=none><a role=treeitem title="Simplifying Istio upgrades by offering safe canary deployments of the control plane (May 19, 2020)" href=/v1.9/blog/2020/multiple-control-planes/>Safely Upgrade Istio using a Canary Control Plane Deployment</a></li><li role=none><a role=treeitem title="Configure the IBM Cloud Kubernetes Service Application Load Balancer to direct traffic to the Istio Ingress gateway with mutual TLS (May 15, 2020)" href=/v1.9/blog/2020/alb-ingress-gateway-iks/>Direct encrypted traffic from IBM Cloud Kubernetes Service Ingress to Istio Ingress Gateway</a></li><li role=none><a role=treeitem title="Community partner tooling of Wasm for Istio by Solo.io (March 25, 2020)" href=/v1.9/blog/2020/wasmhub-istio/>Extended and Improved WebAssemblyHub to Bring the Power of WebAssembly to Envoy and Istio</a></li><li role=none><a role=treeitem title="A mechanism to acquire and share an application certificate and key through mounted files (March 25, 2020)" href=/v1.9/blog/2020/proxy-cert/>Provision a certificate and key for an application without sidecars</a></li><li role=none><a role=treeitem title="Istiod consolidates the Istio control plane components into a single binary (March 19, 2020)" href=/v1.9/blog/2020/istiod/>Introducing istiod: simplifying the control plane</a></li><li role=none><a role=treeitem title="Configuring Wasm extensions for Envoy and Istio declaratively (March 16, 2020)" href=/v1.9/blog/2020/deploy-wasm-declarative/>Declarative WebAssembly deployment for Istio</a></li><li role=none><a role=treeitem title="The future of Istio extensibility using WASM (March 5, 2020)" href=/v1.9/blog/2020/wasm-announce/>Redefining extensibility in proxies - introducing WebAssembly to Envoy and Istio</a></li><li role=none><a role=treeitem title="A vision statement and roadmap for Istio in 2020 (March 3, 2020)" href=/v1.9/blog/2020/tradewinds-2020/>Istio in 2020 - Following the Trade Winds</a></li><li role=none><a role=treeitem title="A more secure way to manage secrets (February 20, 2020)" href=/v1.9/blog/2020/istio-agent/>Remove cross-pod unix domain sockets</a></li><li role=none><a role=treeitem title="Automating Istio configuration for Istio deployments (clusters) that work as a single mesh (January 5, 2020)" href=/v1.9/blog/2020/multi-cluster-mesh-automation/>Multicluster Istio configuration and service discovery using Admiral</a></li></ul></div></div><div class=card><button class="header dynamic" id=card2 title="Blog posts for 2019." aria-controls=card2-body><svg class="icon blog"><use xlink:href="/v1.9/img/icons.svg#blog"/></svg>2019 Posts</button><div class=body aria-labelledby=card2 role=region id=card2-body><ul role=tree aria-expanded=true class=leaf-section aria-labelledby=card2><li role=none><a role=treeitem title="Provision and manage DNS certificates in Istio (November 14, 2019)" href=/v1.9/blog/2019/dns-cert/>DNS Certificate Management</a></li><li role=none><a role=treeitem title="Analyze your Istio configuration to detect potential issues and get general insights (November 14, 2019)" href=/v1.9/blog/2019/introducing-istioctl-analyze/>Introducing istioctl analyze</a></li><li role=none><a role=treeitem title="Introduction to Istio's new operator-based installation and control plane management feature (November 14, 2019)" href=/v1.9/blog/2019/introducing-istio-operator/>Introducing the Istio Operator</a></li><li role=none><a role=treeitem title="Introduction, motivation and design principles for the Istio v1beta1 Authorization Policy (November 14, 2019)" href=/v1.9/blog/2019/v1beta1-authorization-policy/>Introducing the Istio v1beta1 Authorization Policy</a></li><li role=none><a role=treeitem title="Getting programmatic access to Istio resources (November 14, 2019)" href=/v1.9/blog/2019/announcing-istio-client-go/>Announcing Istio client-go</a></li><li role=none><a role=treeitem title="A more secure way to manage Istio webhooks (November 14, 2019)" href=/v1.9/blog/2019/webhook/>Secure Webhook Management</a></li><li role=none><a role=treeitem title="Configure Istio ingress gateway to act as a proxy for external services (October 15, 2019)" href=/v1.9/blog/2019/proxy/>Istio as a Proxy for External Services</a></li><li role=none><a role=treeitem title="Deploy environments that require isolation into separate meshes and enable inter-mesh communication by mesh federation (October 2, 2019)" href=/v1.9/blog/2019/isolated-clusters/>Multi-Mesh Deployments for Isolation and Boundary Protection</a></li><li role=none><a role=treeitem title="How can you use Istio to monitor blocked and passthrough external traffic (September 28, 2019)" href=/v1.9/blog/2019/monitoring-external-service-traffic/>Monitoring Blocked and Passthrough External Service Traffic</a></li><li role=none><a role=treeitem title="Using Istio to secure multi-cloud Kubernetes applications with zero code changes (September 18, 2019)" href=/v1.9/blog/2019/app-identity-and-access-adapter/>App Identity and Access Adapter</a></li><li role=none><a role=treeitem title="Demonstrates a Mixer out-of-process adapter which implements the Knative scale-from-zero logic (September 18, 2019)" href=/v1.9/blog/2019/knative-activator-adapter/>Mixer Adapter for Knative</a></li><li role=none><a role=treeitem title="Taking advantage of Kubernetes trustworthy JWTs to issue certificates for workload instances more securely (September 10, 2019)" href=/v1.9/blog/2019/trustworthy-jwt-sds/>Change in Secret Discovery Service in Istio 1.3</a></li><li role=none><a role=treeitem title="The design principles behind Istio's APIs and how those APIs are evolving (August 5, 2019)" href=/v1.9/blog/2019/evolving-istios-apis/>The Evolution of Istio's APIs</a></li><li role=none><a role=treeitem title="Comparison of alternative solutions to control egress traffic including performance considerations (July 22, 2019)" href=/v1.9/blog/2019/egress-traffic-control-in-istio-part-3/>Secure Control of Egress Traffic in Istio, part 3</a></li><li role=none><a role=treeitem title="Use Istio Egress Traffic Control to prevent attacks involving egress traffic (July 10, 2019)" href=/v1.9/blog/2019/egress-traffic-control-in-istio-part-2/>Secure Control of Egress Traffic in Istio, part 2</a></li><li role=none><a role=treeitem title="Tools and guidance for evaluating Istio's data plane performance (July 9, 2019)" href=/v1.9/blog/2019/performance-best-practices/>Best Practices: Benchmarking Service Mesh Performance</a></li><li role=none><a role=treeitem title="Learn how to extend the lifetime of Istio self-signed root certificate (June 7, 2019)" href=/v1.9/blog/2019/root-transition/>Extending Istio Self-Signed Root Certificate Lifetime</a></li><li role=none><a role=treeitem title="Attacks involving egress traffic and requirements for egress traffic control (May 22, 2019)" href=/v1.9/blog/2019/egress-traffic-control-in-istio-part-1/>Secure Control of Egress Traffic in Istio, part 1</a></li><li role=none><a role=treeitem title="An overview of Istio 1.1 performance (March 19, 2019)" href=/v1.9/blog/2019/istio1.1_perf/>Architecting Istio 1.1 for Performance</a></li><li role=none><a role=treeitem title="Configuring Istio route rules in a multicluster service mesh (February 7, 2019)" href=/v1.9/blog/2019/multicluster-version-routing/>Version Routing in a Multicluster Service Mesh</a></li><li role=none><a role=treeitem title="Announces the new Istio blog policy (February 5, 2019)" href=/v1.9/blog/2019/sail-the-blog/>Sail the Blog!</a></li><li role=none><a role=treeitem title="De-mystify how Istio manages to plugin its data-plane components into an existing deployment (January 31, 2019)" href=/v1.9/blog/2019/data-plane-setup/>Demystifying Istio's Sidecar Injection Model</a></li><li role=none><a role=treeitem title="Verifies the performance impact of adding an egress gateway (January 31, 2019)" href=/v1.9/blog/2019/egress-performance/>Egress Gateway Performance Investigation</a></li><li role=none><a role=treeitem title="Addressing application startup ordering and startup latency using AppSwitch (January 14, 2019)" href=/v1.9/blog/2019/appswitch/>Sidestepping Dependency Ordering with AppSwitch</a></li><li role=none><a role=treeitem title="Describes how to deploy a custom ingress gateway using cert-manager manually (January 10, 2019)" href=/v1.9/blog/2019/custom-ingress-gateway/>Deploy a Custom Ingress Gateway Using Cert-Manager</a></li><li role=none><a role=treeitem title="Istio has a new discussion board (January 10, 2019)" href=/v1.9/blog/2019/announcing-discuss.istio.io/>Announcing discuss.istio.io</a></li></ul></div></div><div class=card><button class="header dynamic" id=card3 title="Blog posts for 2018." aria-controls=card3-body><svg class="icon blog"><use xlink:href="/v1.9/img/icons.svg#blog"/></svg>2018 Posts</button><div class="body default" aria-labelledby=card3 role=region id=card3-body><ul role=tree aria-expanded=true class=leaf-section aria-labelledby=card3><li role=none><a role=treeitem title="How to use Istio for traffic management without deploying sidecar proxies (November 21, 2018)" href=/v1.9/blog/2018/incremental-traffic-management/>Incremental Istio Part 1, Traffic Management</a></li><li role=none><a role=treeitem title="Describes a simple scenario based on Istio's Bookinfo example (November 16, 2018)" href=/v1.9/blog/2018/egress-mongo/>Consuming External MongoDB Services</a></li><li role=none><a role=treeitem title="Istio hosting an all day Twitch stream to celebrate the 1.0 release (August 3, 2018)" href=/v1.9/blog/2018/istio-twitch-stream/>All Day Istio Twitch Stream</a></li><li role=none><a role=treeitem title="How HP is building its next-generation footwear personalization platform on Istio (July 31, 2018)" href=/v1.9/blog/2018/hp/>Istio a Game Changer for HP's FitStation Platform</a></li><li role=none><a role=treeitem title="Automatic application onboarding and latency optimizations using AppSwitch (July 30, 2018)" href=/v1.9/blog/2018/delayering-istio/>Delayering Istio with AppSwitch</a></li><li role=none><a role=treeitem title="Describe Istio's authorization feature and how to use it in various use cases (July 20, 2018)" href=/v1.9/blog/2018/istio-authorization/>Micro-Segmentation with Istio Authorization</a></li><li role=none><span role=treeitem class=current title="How to export Istio Access Logs to different sinks like BigQuery, GCS, Pub/Sub through Stackdriver (July 9, 2018)">Exporting Logs to BigQuery, GCS, Pub/Sub through Stackdriver</span></li><li role=none><a role=treeitem title="Describes how to configure Istio for monitoring and access policies of HTTP egress traffic (June 22, 2018)" href=/v1.9/blog/2018/egress-monitoring-access-control/>Monitoring and Access Policies for HTTP Egress Traffic</a></li><li role=none><a role=treeitem title="Introduction, motivation and design principles for the Istio v1alpha3 routing API (April 25, 2018)" href=/v1.9/blog/2018/v1alpha3-routing/>Introducing the Istio v1alpha3 routing API</a></li><li role=none><a role=treeitem title="Describes how to configure Istio ingress with a network load balancer on AWS (April 20, 2018)" href=/v1.9/blog/2018/aws-nlb/>Configuring Istio Ingress with AWS NLB</a></li><li role=none><a role=treeitem title="Using Kubernetes namespaces and RBAC to create an Istio soft multi-tenancy environment (April 19, 2018)" href=/v1.9/blog/2018/soft-multitenancy/>Istio Soft Multi-Tenancy Support</a></li><li role=none><a role=treeitem title="An introduction to safer, lower-risk deployments and release to production (February 8, 2018)" href=/v1.9/blog/2018/traffic-mirroring/>Traffic Mirroring with Istio for Testing in Production</a></li><li role=none><a role=treeitem title="Describes a simple scenario based on Istio's Bookinfo example (February 6, 2018)" href=/v1.9/blog/2018/egress-tcp/>Consuming External TCP Services</a></li><li role=none><a role=treeitem title="Describes a simple scenario based on Istio's Bookinfo example (January 31, 2018)" href=/v1.9/blog/2018/egress-https/>Consuming External Web Services</a></li></ul></div></div><div class=card><button class="header dynamic" id=card4 title="Blog posts for 2017." aria-controls=card4-body><svg class="icon blog"><use xlink:href="/v1.9/img/icons.svg#blog"/></svg>2017 Posts</button><div class=body aria-labelledby=card4 role=region id=card4-body><ul role=tree aria-expanded=true class=leaf-section aria-labelledby=card4><li role=none><a role=treeitem title="Improving availability and reducing latency (December 7, 2017)" href=/v1.9/blog/2017/mixer-spof-myth/>Mixer and the SPOF Myth</a></li><li role=none><a role=treeitem title="Provides an overview of Mixer's plug-in architecture (November 3, 2017)" href=/v1.9/blog/2017/adapter-model/>Mixer Adapter Model</a></li><li role=none><a role=treeitem title="How Kubernetes Network Policy relates to Istio policy (August 10, 2017)" href=/v1.9/blog/2017/0.1-using-network-policy/>Using Network Policy with Istio</a></li><li role=none><a role=treeitem title="Using Istio to create autoscaled canary deployments (June 14, 2017)" href=/v1.9/blog/2017/0.1-canary/>Canary Deployments using Istio</a></li><li role=none><a role=treeitem title="Istio Authentication 0.1 announcement (May 25, 2017)" href=/v1.9/blog/2017/0.1-auth/>Using Istio to Improve End-to-End Security</a></li></ul></div></div></div></nav></div><div class=article-container><button tabindex=-1 id=sidebar-toggler title="Toggle the navigation bar"><svg class="icon pull"><use xlink:href="/v1.9/img/icons.svg#pull"/></svg></button><nav aria-label=Breadcrumb><ol><li><a href=/v1.9/ title="Connect, secure, control, and observe services.">Istio</a></li><li><a href=/v1.9/blog/ title="Posts about using Istio.">Blog</a></li><li><a href=/v1.9/blog/2018/ title="Blog posts for 2018.">2018 Posts</a></li><li>Exporting Logs to BigQuery, GCS, Pub/Sub through Stackdriver</li></ol></nav><article aria-labelledby=title><div class=title-area><div style=width:100%><h1 id=title>Exporting Logs to BigQuery, GCS, Pub/Sub through Stackdriver</h1><p class=byline><span>By</span>
|
||
<span class=attribution>Nupur Garg and Douglas Reid</span><span> | </span><span><svg class="icon calendar"><use xlink:href="/v1.9/img/icons.svg#calendar"/></svg><span> </span>July 9, 2018</span><span> | </span><span title="1007 words"><svg class="icon clock"><use xlink:href="/v1.9/img/icons.svg#clock"/></svg><span> </span>5 minute read</span>
|
||
<span> </span>
|
||
<span></span></p></div></div><nav class=toc-inlined aria-label="Table of Contents"><div><hr><ol><li role=none aria-label="Before you begin"><a href=#before-you-begin>Before you begin</a><li role=none aria-label="Configuring Istio to export logs"><a href=#configuring-istio-to-export-logs>Configuring Istio to export logs</a><ol><li role=none aria-label="Setting up various log sinks"><a href=#setting-up-various-log-sinks>Setting up various log sinks</a><ol><li role=none aria-label=BigQuery><a href=#bigquery>BigQuery</a><li role=none aria-label="Google Cloud Storage (GCS)"><a href=#google-cloud-storage-gcs>Google Cloud Storage (GCS)</a><li role=none aria-label="Google Cloud Pub/Sub"><a href=#google-cloud-pub-sub>Google Cloud Pub/Sub</a></ol></li><li role=none aria-label="Setting up Stackdriver"><a href=#setting-up-stackdriver>Setting up Stackdriver</a></ol></li><li role=none aria-label="Understanding what happened"><a href=#understanding-what-happened>Understanding what happened</a><li role=none aria-label=Cleanup><a href=#cleanup>Cleanup</a><li role=none aria-label="Availability of logs in export sinks"><a href=#availability-of-logs-in-export-sinks>Availability of logs in export sinks</a></ol><hr></div></nav><div><aside class="callout warning"><div class=type><svg class="large-icon"><use xlink:href="/v1.9/img/icons.svg#callout-warning"/></svg></div><div class=content>This blog post was written assuming Istio 0.8, so some of this content may now be outdated.</div></aside></div><p>This post shows how to direct Istio logs to <a href=https://cloud.google.com/stackdriver/>Stackdriver</a>
|
||
and export those logs to various configured sinks such as such as
|
||
<a href=https://cloud.google.com/bigquery/>BigQuery</a>, <a href=https://cloud.google.com/storage/>Google Cloud Storage</a>
|
||
or <a href=https://cloud.google.com/pubsub/>Cloud Pub/Sub</a>. At the end of this post you can perform
|
||
analytics on Istio data from your favorite places such as BigQuery, GCS or Cloud Pub/Sub.</p><p>The <a href=/v1.9/docs/examples/bookinfo/>Bookinfo</a> sample application is used as the example
|
||
application throughout this task.</p><h2 id=before-you-begin>Before you begin</h2><p><a href=/v1.9/docs/setup/>Install Istio</a> in your cluster and deploy an application.</p><h2 id=configuring-istio-to-export-logs>Configuring Istio to export logs</h2><p>Istio exports logs using the <code>logentry</code> <a href=https://istio.io/v1.6/docs/reference/config/policy-and-telemetry/templates/logentry>template</a>.
|
||
This specifies all the variables that are available for analysis. It
|
||
contains information like source service, destination service, auth
|
||
metrics (coming..) among others. Following is a diagram of the pipeline:</p><figure style=width:75%><div class=wrapper-with-intrinsic-ratio style=padding-bottom:75%><a data-skipendnotes=true href=/v1.9/blog/2018/export-logs-through-stackdriver/istio-analytics-using-stackdriver.png title="Exporting logs from Istio to Stackdriver for analysis"><img class=element-to-stretch src=/v1.9/blog/2018/export-logs-through-stackdriver/istio-analytics-using-stackdriver.png alt="Exporting logs from Istio to Stackdriver for analysis"></a></div><figcaption>Exporting logs from Istio to Stackdriver for analysis</figcaption></figure><p>Istio supports exporting logs to Stackdriver which can in turn be configured to export
|
||
logs to your favorite sink like BigQuery, Pub/Sub or GCS. Please follow the steps
|
||
below to setup your favorite sink for exporting logs first and then Stackdriver
|
||
in Istio.</p><h3 id=setting-up-various-log-sinks>Setting up various log sinks</h3><p>Common setup for all sinks:</p><ol><li>Enable <a href=https://cloud.google.com/monitoring/api/enable-api>Stackdriver Monitoring API</a> for the project.</li><li>Make sure <code>principalEmail</code> that would be setting up the sink has write access to the project and Logging Admin role permissions.</li><li>Make sure the <code>GOOGLE_APPLICATION_CREDENTIALS</code> environment variable is set. Please follow instructions <a href=https://cloud.google.com/docs/authentication/getting-started>here</a> to set it up.</li></ol><h4 id=bigquery>BigQuery</h4><ol><li><a href=https://cloud.google.com/bigquery/docs/datasets>Create a BigQuery dataset</a> as a destination for the logs export.</li><li>Record the ID of the dataset. It will be needed to configure the Stackdriver handler.
|
||
It would be of the form <code>bigquery.googleapis.com/projects/[PROJECT_ID]/datasets/[DATASET_ID]</code></li><li>Give <a href=https://cloud.google.com/logging/docs/api/tasks/exporting-logs#writing_to_the_destination>sink’s writer identity</a>: <code>cloud-logs@system.gserviceaccount.com</code> BigQuery Data Editor role in IAM.</li><li>If using <a href=/v1.9/docs/setup/platform-setup/gke/>Google Kubernetes Engine</a>, make sure <code>bigquery</code> <a href=https://cloud.google.com/sdk/gcloud/reference/container/clusters/create>Scope</a> is enabled on the cluster.</li></ol><h4 id=google-cloud-storage-gcs>Google Cloud Storage (GCS)</h4><ol><li><a href=https://cloud.google.com/storage/docs/creating-buckets>Create a GCS bucket</a> where you would like logs to get exported in GCS.</li><li>Recode the ID of the bucket. It will be needed to configure Stackdriver.
|
||
It would be of the form <code>storage.googleapis.com/[BUCKET_ID]</code></li><li>Give <a href=https://cloud.google.com/logging/docs/api/tasks/exporting-logs#writing_to_the_destination>sink’s writer identity</a>: <code>cloud-logs@system.gserviceaccount.com</code> Storage Object Creator role in IAM.</li></ol><h4 id=google-cloud-pub-sub>Google Cloud Pub/Sub</h4><ol><li><a href=https://cloud.google.com/pubsub/docs/admin>Create a topic</a> where you would like logs to get exported in Google Cloud Pub/Sub.</li><li>Recode the ID of the topic. It will be needed to configure Stackdriver.
|
||
It would be of the form <code>pubsub.googleapis.com/projects/[PROJECT_ID]/topics/[TOPIC_ID]</code></li><li>Give <a href=https://cloud.google.com/logging/docs/api/tasks/exporting-logs#writing_to_the_destination>sink’s writer identity</a>: <code>cloud-logs@system.gserviceaccount.com</code> Pub/Sub Publisher role in IAM.</li><li>If using <a href=/v1.9/docs/setup/platform-setup/gke/>Google Kubernetes Engine</a>, make sure <code>pubsub</code> <a href=https://cloud.google.com/sdk/gcloud/reference/container/clusters/create>Scope</a> is enabled on the cluster.</li></ol><h3 id=setting-up-stackdriver>Setting up Stackdriver</h3><p>A Stackdriver handler must be created to export data to Stackdriver. The configuration for
|
||
a Stackdriver handler is described <a href=https://istio.io/v1.6/docs/reference/config/policy-and-telemetry/adapters/stackdriver/>here</a>.</p><ol><li><p>Save the following yaml file as <code>stackdriver.yaml</code>. Replace <code><project_id>,
|
||
<sink_id>, <sink_destination>, <log_filter></code> with their specific values.</p><pre><code class=language-yaml data-expandlinks=true data-repo=istio>apiVersion: "config.istio.io/v1alpha2"
|
||
kind: stackdriver
|
||
metadata:
|
||
name: handler
|
||
namespace: istio-system
|
||
spec:
|
||
# We'll use the default value from the adapter, once per minute, so we don't need to supply a value.
|
||
# pushInterval: 1m
|
||
# Must be supplied for the Stackdriver adapter to work
|
||
project_id: "<project_id>"
|
||
# One of the following must be set; the preferred method is `appCredentials`, which corresponds to
|
||
# Google Application Default Credentials.
|
||
# If none is provided we default to app credentials.
|
||
# appCredentials:
|
||
# apiKey:
|
||
# serviceAccountPath:
|
||
# Describes how to map Istio logs into Stackdriver.
|
||
logInfo:
|
||
accesslog.logentry.istio-system:
|
||
payloadTemplate: '{{or (.sourceIp) "-"}} - {{or (.sourceUser) "-"}} [{{or (.timestamp.Format "02/Jan/2006:15:04:05 -0700") "-"}}] "{{or (.method) "-"}} {{or (.url) "-"}} {{or (.protocol) "-"}}" {{or (.responseCode) "-"}} {{or (.responseSize) "-"}}'
|
||
httpMapping:
|
||
url: url
|
||
status: responseCode
|
||
requestSize: requestSize
|
||
responseSize: responseSize
|
||
latency: latency
|
||
localIp: sourceIp
|
||
remoteIp: destinationIp
|
||
method: method
|
||
userAgent: userAgent
|
||
referer: referer
|
||
labelNames:
|
||
- sourceIp
|
||
- destinationIp
|
||
- sourceService
|
||
- sourceUser
|
||
- sourceNamespace
|
||
- destinationIp
|
||
- destinationService
|
||
- destinationNamespace
|
||
- apiClaims
|
||
- apiKey
|
||
- protocol
|
||
- method
|
||
- url
|
||
- responseCode
|
||
- responseSize
|
||
- requestSize
|
||
- latency
|
||
- connectionMtls
|
||
- userAgent
|
||
- responseTimestamp
|
||
- receivedBytes
|
||
- sentBytes
|
||
- referer
|
||
sinkInfo:
|
||
id: '<sink_id>'
|
||
destination: '<sink_destination>'
|
||
filter: '<log_filter>'
|
||
---
|
||
apiVersion: "config.istio.io/v1alpha2"
|
||
kind: rule
|
||
metadata:
|
||
name: stackdriver
|
||
namespace: istio-system
|
||
spec:
|
||
match: "true" # If omitted match is true.
|
||
actions:
|
||
- handler: handler.stackdriver
|
||
instances:
|
||
- accesslog.logentry
|
||
---
|
||
</code></pre></li><li><p>Push the configuration</p><pre><code class=language-bash data-expandlinks=true data-repo=istio>$ kubectl apply -f stackdriver.yaml
|
||
stackdriver "handler" created
|
||
rule "stackdriver" created
|
||
logentry "stackdriverglobalmr" created
|
||
metric "stackdriverrequestcount" created
|
||
metric "stackdriverrequestduration" created
|
||
metric "stackdriverrequestsize" created
|
||
metric "stackdriverresponsesize" created
|
||
</code></pre></li><li><p>Send traffic to the sample application.</p><p>For the Bookinfo sample, visit <code>http://$GATEWAY_URL/productpage</code> in your web
|
||
browser or issue the following command:</p><pre><code class=language-bash data-expandlinks=true data-repo=istio>$ curl http://$GATEWAY_URL/productpage
|
||
</code></pre></li><li><p>Verify that logs are flowing through Stackdriver to the configured sink.</p><ul><li>Stackdriver: Navigate to the <a href=https://pantheon.corp.google.com/logs/viewer>Stackdriver Logs
|
||
Viewer</a> for your project
|
||
and look under “GKE Container” -> “Cluster Name” -> “Namespace Id” for
|
||
Istio Access logs.</li><li>BigQuery: Navigate to the <a href=https://bigquery.cloud.google.com/>BigQuery
|
||
Interface</a> for your project and you
|
||
should find a table with prefix <code>accesslog_logentry_istio</code> in your sink
|
||
dataset.</li><li>GCS: Navigate to the <a href=https://pantheon.corp.google.com/storage/browser/>Storage
|
||
Browser</a> for your
|
||
project and you should find a bucket named
|
||
<code>accesslog.logentry.istio-system</code> in your sink bucket.</li><li>Pub/Sub: Navigate to the <a href=https://pantheon.corp.google.com/cloudpubsub/topicList>Pub/Sub
|
||
Topic List</a> for
|
||
your project and you should find a topic for <code>accesslog</code> in your sink
|
||
topic.</li></ul></li></ol><h2 id=understanding-what-happened>Understanding what happened</h2><p><code>Stackdriver.yaml</code> file above configured Istio to send access logs to
|
||
Stackdriver and then added a sink configuration where these logs could be
|
||
exported. In detail as follows:</p><ol><li><p>Added a handler of kind <code>stackdriver</code></p><pre><code class=language-yaml data-expandlinks=true data-repo=istio>apiVersion: "config.istio.io/v1alpha2"
|
||
kind: stackdriver
|
||
metadata:
|
||
name: handler
|
||
namespace: <your defined namespace>
|
||
</code></pre></li><li><p>Added <code>logInfo</code> in spec</p><pre><code class=language-yaml data-expandlinks=true data-repo=istio>spec:
|
||
logInfo: accesslog.logentry.istio-system:
|
||
labelNames:
|
||
- sourceIp
|
||
- destinationIp
|
||
...
|
||
...
|
||
sinkInfo:
|
||
id: '<sink_id>'
|
||
destination: '<sink_destination>'
|
||
filter: '<log_filter>'
|
||
</code></pre><p>In the above configuration sinkInfo contains information about the sink where you want
|
||
the logs to get exported to. For more information on how this gets filled for different sinks please refer
|
||
<a href=https://cloud.google.com/logging/docs/export/#sink-terms>here</a>.</p></li><li><p>Added a rule for Stackdriver</p><pre><code class=language-yaml data-expandlinks=true data-repo=istio>apiVersion: "config.istio.io/v1alpha2"
|
||
kind: rule
|
||
metadata:
|
||
name: stackdriver
|
||
namespace: istio-system spec:
|
||
match: "true" # If omitted match is true
|
||
actions:
|
||
- handler: handler.stackdriver
|
||
instances:
|
||
- accesslog.logentry
|
||
</code></pre></li></ol><h2 id=cleanup>Cleanup</h2><ul><li><p>Remove the new Stackdriver configuration:</p><pre><code class=language-bash data-expandlinks=true data-repo=istio>$ kubectl delete -f stackdriver.yaml
|
||
</code></pre></li><li><p>If you are not planning to explore any follow-on tasks, refer to the
|
||
<a href=/v1.9/docs/examples/bookinfo/#cleanup>Bookinfo cleanup</a> instructions to shutdown
|
||
the application.</p></li></ul><h2 id=availability-of-logs-in-export-sinks>Availability of logs in export sinks</h2><p>Export to BigQuery is within minutes (we see it to be almost instant), GCS can
|
||
have a delay of 2 to 12 hours and Pub/Sub is almost immediately.</p></article><nav class=pagenav><div class=left><a title="Describe Istio's authorization feature and how to use it in various use cases." href=/v1.9/blog/2018/istio-authorization/><svg class="icon left-arrow"><use xlink:href="/v1.9/img/icons.svg#left-arrow"/></svg>Micro-Segmentation with Istio Authorization</a></div><div class=right><a title="Describes how to configure Istio for monitoring and access policies of HTTP egress traffic." href=/v1.9/blog/2018/egress-monitoring-access-control/>Monitoring and Access Policies for HTTP Egress Traffic<svg class="icon right-arrow"><use xlink:href="/v1.9/img/icons.svg#right-arrow"/></svg></a></div></nav><div id=feedback><div id=feedback-initial>Was this information useful?<br><button class="btn feedback" onclick="sendFeedback('en',1)">Yes</button>
|
||
<button class="btn feedback" onclick="sendFeedback('en',0)">No</button></div><div id=feedback-comment>Do you have any suggestions for improvement?<br><br><input id=feedback-textbox type=text placeholder="Help us improve..." data-lang=en></div><div id=feedback-thankyou>Thanks for your feedback!</div></div><div id=endnotes-container aria-hidden=true><h2>Links</h2><ol id=endnotes></ol></div></div><div class=toc-container><nav class=toc aria-label="Table of Contents"><div id=toc><ol><li role=none aria-label="Before you begin"><a href=#before-you-begin>Before you begin</a><li role=none aria-label="Configuring Istio to export logs"><a href=#configuring-istio-to-export-logs>Configuring Istio to export logs</a><ol><li role=none aria-label="Setting up various log sinks"><a href=#setting-up-various-log-sinks>Setting up various log sinks</a><ol><li role=none aria-label=BigQuery><a href=#bigquery>BigQuery</a><li role=none aria-label="Google Cloud Storage (GCS)"><a href=#google-cloud-storage-gcs>Google Cloud Storage (GCS)</a><li role=none aria-label="Google Cloud Pub/Sub"><a href=#google-cloud-pub-sub>Google Cloud Pub/Sub</a></ol></li><li role=none aria-label="Setting up Stackdriver"><a href=#setting-up-stackdriver>Setting up Stackdriver</a></ol></li><li role=none aria-label="Understanding what happened"><a href=#understanding-what-happened>Understanding what happened</a><li role=none aria-label=Cleanup><a href=#cleanup>Cleanup</a><li role=none aria-label="Availability of logs in export sinks"><a href=#availability-of-logs-in-export-sinks>Availability of logs in export sinks</a></ol></div></nav></div></main><footer><div class=user-links><a class=channel title="Go download Istio 1.9.5 now" href=/v1.9/docs/setup/getting-started/#download aria-label="Download Istio"><span>download</span><svg class="icon download"><use xlink:href="/v1.9/img/icons.svg#download"/></svg>
|
||
</a><a class=channel title="Join the Istio discussion board to participate in discussions and get help troubleshooting problems" href=https://discuss.istio.io aria-label="Istio discussion board"><span>discuss</span><svg class="icon discourse"><use xlink:href="/v1.9/img/icons.svg#discourse"/></svg></a>
|
||
<a class=channel title="Stack Overflow is where you can ask questions and find curated answers on deploying, configuring, and using Istio" href=https://stackoverflow.com/questions/tagged/istio aria-label="Stack Overflow"><span>stack overflow</span><svg class="icon stackoverflow"><use xlink:href="/v1.9/img/icons.svg#stackoverflow"/></svg></a>
|
||
<a class=channel title="Interactively discuss issues with the Istio community on Slack" href=https://slack.istio.io aria-label=slack><span>slack</span><svg class="icon slack"><use xlink:href="/v1.9/img/icons.svg#slack"/></svg></a>
|
||
<a class=channel title="Follow us on Twitter to get the latest news" href=https://twitter.com/IstioMesh aria-label=Twitter><span>twitter</span><svg class="icon twitter"><use xlink:href="/v1.9/img/icons.svg#twitter"/></svg></a><div class=tag>for everyone</div></div><div class=info><p class=copyright>Istio Archive
|
||
1.9.5<br>© 2020 Istio Authors, <a href=https://policies.google.com/privacy>Privacy Policy</a><br>Archived on May 18, 2021</p></div><div class=dev-links><a class=channel title="GitHub is where development takes place on Istio code" href=https://github.com/istio/community aria-label=GitHub><span>github</span><svg class="icon github"><use xlink:href="/v1.9/img/icons.svg#github"/></svg></a>
|
||
<a class=channel title="Access our team drive if you'd like to take a look at the Istio technical design documents" href=https://groups.google.com/forum/#!forum/istio-team-drive-access aria-label="team drive"><span>drive</span><svg class="icon drive"><use xlink:href="/v1.9/img/icons.svg#drive"/></svg></a>
|
||
<a class=channel title="If you'd like to contribute to the Istio project, consider participating in our working groups" href=https://github.com/istio/community/blob/master/WORKING-GROUPS.md aria-label="working groups"><span>working groups</span><svg class="icon working-groups"><use xlink:href="/v1.9/img/icons.svg#working-groups"/></svg></a><div class=tag>for developers</div></div></footer><div id=scroll-to-top-container aria-hidden=true><button id=scroll-to-top title="Back to top"><svg class="icon top"><use xlink:href="/v1.9/img/icons.svg#top"/></svg></button></div></body></html> |