ingress-nginx/troubleshooting/index.html

1578 lines
49 KiB
HTML

<!DOCTYPE html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta http-equiv="x-ua-compatible" content="ie=edge">
<meta name="lang:clipboard.copy" content="Copy to clipboard">
<meta name="lang:clipboard.copied" content="Copied to clipboard">
<meta name="lang:search.language" content="en">
<meta name="lang:search.pipeline.stopwords" content="True">
<meta name="lang:search.pipeline.trimmer" content="True">
<meta name="lang:search.result.none" content="No matching documents">
<meta name="lang:search.result.one" content="1 matching document">
<meta name="lang:search.result.other" content="# matching documents">
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../assets/images/favicon.png">
<meta name="generator" content="mkdocs-0.17.5, mkdocs-material-2.9.4">
<title>Troubleshooting - NGINX Ingress Controller</title>
<link rel="stylesheet" href="../assets/stylesheets/application.451f80e5.css">
<link rel="stylesheet" href="../assets/stylesheets/application-palette.22915126.css">
<meta name="theme-color" content="#009688">
<script src="../assets/javascripts/modernizr.1aa3b519.js"></script>
<link href="https://fonts.gstatic.com" rel="preconnect" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700|Roboto+Mono">
<style>body,input{font-family:"Roboto","Helvetica Neue",Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono","Courier New",Courier,monospace}</style>
<link rel="stylesheet" href="../assets/fonts/material-icons.css">
<link rel="stylesheet" href="../extra.css">
</head>
<body dir="ltr" data-md-color-primary="teal" data-md-color-accent="green">
<svg class="md-svg">
<defs>
<svg xmlns="http://www.w3.org/2000/svg" width="416" height="448"
viewBox="0 0 416 448" id="__github">
<path fill="currentColor" d="M160 304q0 10-3.125 20.5t-10.75 19-18.125
8.5-18.125-8.5-10.75-19-3.125-20.5 3.125-20.5 10.75-19 18.125-8.5
18.125 8.5 10.75 19 3.125 20.5zM320 304q0 10-3.125 20.5t-10.75
19-18.125 8.5-18.125-8.5-10.75-19-3.125-20.5 3.125-20.5 10.75-19
18.125-8.5 18.125 8.5 10.75 19 3.125 20.5zM360
304q0-30-17.25-51t-46.75-21q-10.25 0-48.75 5.25-17.75 2.75-39.25
2.75t-39.25-2.75q-38-5.25-48.75-5.25-29.5 0-46.75 21t-17.25 51q0 22 8
38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0
37.25-1.75t35-7.375 30.5-15 20.25-25.75 8-38.375zM416 260q0 51.75-15.25
82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5-41.75
1.125q-19.5 0-35.5-0.75t-36.875-3.125-38.125-7.5-34.25-12.875-30.25-20.25-21.5-28.75q-15.5-30.75-15.5-82.75
0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25
30.875q36.75-8.75 77.25-8.75 37 0 70 8 26.25-20.5
46.75-30.25t47.25-9.75q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34
99.5z" />
</svg>
</defs>
</svg>
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" data-md-component="overlay" for="__drawer"></label>
<a href="#troubleshooting" tabindex="1" class="md-skip">
Skip to content
</a>
<header class="md-header" data-md-component="header">
<nav class="md-header-nav md-grid">
<div class="md-flex">
<div class="md-flex__cell md-flex__cell--shrink">
<a href=".." title="NGINX Ingress Controller" class="md-header-nav__button md-logo">
<i class="md-icon">public</i>
</a>
</div>
<div class="md-flex__cell md-flex__cell--shrink">
<label class="md-icon md-icon--menu md-header-nav__button" for="__drawer"></label>
</div>
<div class="md-flex__cell md-flex__cell--stretch">
<div class="md-flex__ellipsis md-header-nav__title" data-md-component="title">
<span class="md-header-nav__topic">
NGINX Ingress Controller
</span>
<span class="md-header-nav__topic">
Troubleshooting
</span>
</div>
</div>
<div class="md-flex__cell md-flex__cell--shrink">
<label class="md-icon md-icon--search md-header-nav__button" for="__search"></label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="query" data-md-state="active">
<label class="md-icon md-search__icon" for="__search"></label>
<button type="reset" class="md-icon md-search__icon" data-md-component="reset" tabindex="-1">
&#xE5CD;
</button>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" data-md-scrollfix>
<div class="md-search-result" data-md-component="result">
<div class="md-search-result__meta">
Type to start searching
</div>
<ol class="md-search-result__list"></ol>
</div>
</div>
</div>
</div>
</div>
</div>
<div class="md-flex__cell md-flex__cell--shrink">
<div class="md-header-nav__source">
<a href="https://github.com/kubernetes/ingress-nginx/" title="Go to repository" class="md-source" data-md-source="github">
<div class="md-source__icon">
<svg viewBox="0 0 24 24" width="24" height="24">
<use xlink:href="#__github" width="24" height="24"></use>
</svg>
</div>
<div class="md-source__repository">
kubernetes/ingress-nginx
</div>
</a>
</div>
</div>
</div>
</nav>
</header>
<div class="md-container">
<nav class="md-tabs" data-md-component="tabs">
<div class="md-tabs__inner md-grid">
<ul class="md-tabs__list">
<li class="md-tabs__item">
<a href=".." title="Welcome" class="md-tabs__link md-tabs__link--active">
Welcome
</a>
</li>
<li class="md-tabs__item">
<a href="../deploy/" title="Deployment" class="md-tabs__link">
Deployment
</a>
</li>
<li class="md-tabs__item">
<a href="../user-guide/nginx-configuration/" title="User guide" class="md-tabs__link">
User guide
</a>
</li>
<li class="md-tabs__item">
<a href="../examples/" title="Examples" class="md-tabs__link">
Examples
</a>
</li>
</ul>
</div>
</nav>
<main class="md-main">
<div class="md-main__inner md-grid" data-md-component="container">
<div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary" data-md-level="0">
<label class="md-nav__title md-nav__title--site" for="__drawer">
<a href=".." title="NGINX Ingress Controller" class="md-nav__button md-logo">
<i class="md-icon">public</i>
</a>
NGINX Ingress Controller
</label>
<div class="md-nav__source">
<a href="https://github.com/kubernetes/ingress-nginx/" title="Go to repository" class="md-source" data-md-source="github">
<div class="md-source__icon">
<svg viewBox="0 0 24 24" width="24" height="24">
<use xlink:href="#__github" width="24" height="24"></use>
</svg>
</div>
<div class="md-source__repository">
kubernetes/ingress-nginx
</div>
</a>
</div>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href=".." title="Welcome" class="md-nav__link">
Welcome
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-2" type="checkbox" id="nav-2">
<label class="md-nav__link" for="nav-2">
Deployment
</label>
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
<label class="md-nav__title" for="nav-2">
Deployment
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../deploy/" title="Installation Guide" class="md-nav__link">
Installation Guide
</a>
</li>
<li class="md-nav__item">
<a href="../deploy/rbac/" title="Role Based Access Control (RBAC)" class="md-nav__link">
Role Based Access Control (RBAC)
</a>
</li>
<li class="md-nav__item">
<a href="../deploy/upgrade/" title="Upgrading" class="md-nav__link">
Upgrading
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-3" type="checkbox" id="nav-3">
<label class="md-nav__link" for="nav-3">
User guide
</label>
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
<label class="md-nav__title" for="nav-3">
User guide
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--nested">
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-3-1" type="checkbox" id="nav-3-1">
<label class="md-nav__link" for="nav-3-1">
NGINX Configuration
</label>
<nav class="md-nav" data-md-component="collapsible" data-md-level="2">
<label class="md-nav__title" for="nav-3-1">
NGINX Configuration
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../user-guide/nginx-configuration/" title="NGINX Configuration" class="md-nav__link">
NGINX Configuration
</a>
</li>
<li class="md-nav__item">
<a href="../user-guide/nginx-configuration/annotations/" title="Annotations" class="md-nav__link">
Annotations
</a>
</li>
<li class="md-nav__item">
<a href="../user-guide/nginx-configuration/configmap/" title="ConfigMaps" class="md-nav__link">
ConfigMaps
</a>
</li>
<li class="md-nav__item">
<a href="../user-guide/nginx-configuration/custom-template/" title="Custom NGINX template" class="md-nav__link">
Custom NGINX template
</a>
</li>
<li class="md-nav__item">
<a href="../user-guide/nginx-configuration/log-format/" title="Log format" class="md-nav__link">
Log format
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../user-guide/cli-arguments/" title="Command line arguments" class="md-nav__link">
Command line arguments
</a>
</li>
<li class="md-nav__item">
<a href="../user-guide/custom-errors/" title="Custom errors" class="md-nav__link">
Custom errors
</a>
</li>
<li class="md-nav__item">
<a href="../user-guide/default-backend/" title="Default backend" class="md-nav__link">
Default backend
</a>
</li>
<li class="md-nav__item">
<a href="../user-guide/exposing-tcp-udp-services/" title="Exposing TCP and UDP services" class="md-nav__link">
Exposing TCP and UDP services
</a>
</li>
<li class="md-nav__item">
<a href="../user-guide/external-articles/" title="External Articles" class="md-nav__link">
External Articles
</a>
</li>
<li class="md-nav__item">
<a href="../user-guide/miscellaneous/" title="Miscellaneous" class="md-nav__link">
Miscellaneous
</a>
</li>
<li class="md-nav__item">
<a href="../user-guide/monitoring/" title="Prometheus and Grafana installation" class="md-nav__link">
Prometheus and Grafana installation
</a>
</li>
<li class="md-nav__item">
<a href="../user-guide/multiple-ingress/" title="Multiple Ingress controllers" class="md-nav__link">
Multiple Ingress controllers
</a>
</li>
<li class="md-nav__item">
<a href="../user-guide/tls/" title="TLS/HTTPS" class="md-nav__link">
TLS/HTTPS
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-3-11" type="checkbox" id="nav-3-11">
<label class="md-nav__link" for="nav-3-11">
Third party addons
</label>
<nav class="md-nav" data-md-component="collapsible" data-md-level="2">
<label class="md-nav__title" for="nav-3-11">
Third party addons
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../user-guide/third-party-addons/modsecurity/" title="ModSecurity Web Application Firewall" class="md-nav__link">
ModSecurity Web Application Firewall
</a>
</li>
<li class="md-nav__item">
<a href="../user-guide/third-party-addons/opentracing/" title="OpenTracing" class="md-nav__link">
OpenTracing
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-4" type="checkbox" id="nav-4">
<label class="md-nav__link" for="nav-4">
Examples
</label>
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
<label class="md-nav__title" for="nav-4">
Examples
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../examples/" title="Ingress examples" class="md-nav__link">
Ingress examples
</a>
</li>
<li class="md-nav__item">
<a href="../examples/PREREQUISITES/" title="Prerequisites" class="md-nav__link">
Prerequisites
</a>
</li>
<li class="md-nav__item">
<a href="../examples/affinity/cookie/README/" title="Sticky Session" class="md-nav__link">
Sticky Session
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-4-4" type="checkbox" id="nav-4-4">
<label class="md-nav__link" for="nav-4-4">
Auth
</label>
<nav class="md-nav" data-md-component="collapsible" data-md-level="2">
<label class="md-nav__title" for="nav-4-4">
Auth
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../examples/auth/basic/README/" title="Basic Authentication" class="md-nav__link">
Basic Authentication
</a>
</li>
<li class="md-nav__item">
<a href="../examples/auth/client-certs/README/" title="Client Certificate Authentication" class="md-nav__link">
Client Certificate Authentication
</a>
</li>
<li class="md-nav__item">
<a href="../examples/auth/external-auth/README/" title="External Basic Authentication" class="md-nav__link">
External Basic Authentication
</a>
</li>
<li class="md-nav__item">
<a href="../examples/auth/oauth-external-auth/README/" title="External OAUTH Authentication" class="md-nav__link">
External OAUTH Authentication
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-4-5" type="checkbox" id="nav-4-5">
<label class="md-nav__link" for="nav-4-5">
Customization
</label>
<nav class="md-nav" data-md-component="collapsible" data-md-level="2">
<label class="md-nav__title" for="nav-4-5">
Customization
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../examples/customization/configuration-snippets/README/" title="Configuration Snippets" class="md-nav__link">
Configuration Snippets
</a>
</li>
<li class="md-nav__item">
<a href="../examples/customization/custom-configuration/README/" title="Custom Configuration" class="md-nav__link">
Custom Configuration
</a>
</li>
<li class="md-nav__item">
<a href="../examples/customization/custom-errors/README/" title="Custom Errors" class="md-nav__link">
Custom Errors
</a>
</li>
<li class="md-nav__item">
<a href="../examples/customization/custom-headers/README/" title="Custom Headers" class="md-nav__link">
Custom Headers
</a>
</li>
<li class="md-nav__item">
<a href="../examples/customization/custom-upstream-check/README/" title="Custom Upstream server checks" class="md-nav__link">
Custom Upstream server checks
</a>
</li>
<li class="md-nav__item">
<a href="../examples/customization/external-auth-headers/README/" title="External authentication, authentication service response headers propagation" class="md-nav__link">
External authentication, authentication service response headers propagation
</a>
</li>
<li class="md-nav__item">
<a href="../examples/customization/ssl-dh-param/README/" title="Custom DH parameters for perfect forward secrecy" class="md-nav__link">
Custom DH parameters for perfect forward secrecy
</a>
</li>
<li class="md-nav__item">
<a href="../examples/customization/sysctl/README/" title="Sysctl tuning" class="md-nav__link">
Sysctl tuning
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../examples/docker-registry/README/" title="Docker registry" class="md-nav__link">
Docker registry
</a>
</li>
<li class="md-nav__item">
<a href="../examples/grpc/README/" title="gRPC" class="md-nav__link">
gRPC
</a>
</li>
<li class="md-nav__item">
<a href="../examples/multi-tls/README/" title="Multi TLS certificate termination" class="md-nav__link">
Multi TLS certificate termination
</a>
</li>
<li class="md-nav__item">
<a href="../examples/rewrite/README/" title="Rewrite" class="md-nav__link">
Rewrite
</a>
</li>
<li class="md-nav__item">
<a href="../examples/static-ip/README/" title="Static IPs" class="md-nav__link">
Static IPs
</a>
</li>
<li class="md-nav__item">
<a href="../examples/tls-termination/README/" title="TLS termination" class="md-nav__link">
TLS termination
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../development/" title="Developing for NGINX Ingress Controller" class="md-nav__link">
Developing for NGINX Ingress Controller
</a>
</li>
<li class="md-nav__item">
<a href="../how-it-works/" title="How it works" class="md-nav__link">
How it works
</a>
</li>
<li class="md-nav__item md-nav__item--active">
<input class="md-toggle md-nav__toggle" data-md-toggle="toc" type="checkbox" id="__toc">
<label class="md-nav__link md-nav__link--active" for="__toc">
Troubleshooting
</label>
<a href="./" title="Troubleshooting" class="md-nav__link md-nav__link--active">
Troubleshooting
</a>
<nav class="md-nav md-nav--secondary">
<label class="md-nav__title" for="__toc">Table of contents</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#ingress-controller-logs-and-events" title="Ingress-Controller Logs and Events" class="md-nav__link">
Ingress-Controller Logs and Events
</a>
</li>
<li class="md-nav__item">
<a href="#debug-logging" title="Debug Logging" class="md-nav__link">
Debug Logging
</a>
</li>
<li class="md-nav__item">
<a href="#authentication-to-the-kubernetes-api-server" title="Authentication to the Kubernetes API Server" class="md-nav__link">
Authentication to the Kubernetes API Server
</a>
<nav class="md-nav">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#service-account" title="Service Account" class="md-nav__link">
Service Account
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#kube-config" title="Kube-Config" class="md-nav__link">
Kube-Config
</a>
</li>
<li class="md-nav__item">
<a href="#using-gdb-with-nginx" title="Using GDB with Nginx" class="md-nav__link">
Using GDB with Nginx
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary">
<label class="md-nav__title" for="__toc">Table of contents</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#ingress-controller-logs-and-events" title="Ingress-Controller Logs and Events" class="md-nav__link">
Ingress-Controller Logs and Events
</a>
</li>
<li class="md-nav__item">
<a href="#debug-logging" title="Debug Logging" class="md-nav__link">
Debug Logging
</a>
</li>
<li class="md-nav__item">
<a href="#authentication-to-the-kubernetes-api-server" title="Authentication to the Kubernetes API Server" class="md-nav__link">
Authentication to the Kubernetes API Server
</a>
<nav class="md-nav">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#service-account" title="Service Account" class="md-nav__link">
Service Account
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#kube-config" title="Kube-Config" class="md-nav__link">
Kube-Config
</a>
</li>
<li class="md-nav__item">
<a href="#using-gdb-with-nginx" title="Using GDB with Nginx" class="md-nav__link">
Using GDB with Nginx
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content">
<article class="md-content__inner md-typeset">
<a href="https://github.com/kubernetes/ingress-nginx/edit/master/docs/troubleshooting.md" title="Edit this page" class="md-icon md-content__icon">&#xE3C9;</a>
<!--
-----------------NOTICE------------------------
This file is referenced in code as
https://github.com/kubernetes/ingress-nginx/blob/master/docs/troubleshooting.md
Do not move it without providing redirects.
-----------------------------------------------
-->
<h1 id="troubleshooting">Troubleshooting<a class="headerlink" href="#troubleshooting" title="Permanent link">&para;</a></h1>
<h2 id="ingress-controller-logs-and-events">Ingress-Controller Logs and Events<a class="headerlink" href="#ingress-controller-logs-and-events" title="Permanent link">&para;</a></h2>
<p>There are many ways to troubleshoot the ingress-controller. The following are basic troubleshooting
methods to obtain more information.</p>
<p>Check the Ingress Resource Events</p>
<div class="codehilite"><pre><span></span><span class="gp">$</span> kubectl get ing -n &lt;namespace-of-ingress-resource&gt;
<span class="go">NAME HOSTS ADDRESS PORTS AGE</span>
<span class="go">cafe-ingress cafe.com 10.0.2.15 80 25s</span>
<span class="gp">$</span> kubectl describe ing &lt;ingress-resource-name&gt; -n &lt;namespace-of-ingress-resource&gt;
<span class="go">Name: cafe-ingress</span>
<span class="go">Namespace: default</span>
<span class="go">Address: 10.0.2.15</span>
<span class="go">Default backend: default-http-backend:80 (172.17.0.5:8080)</span>
<span class="go">Rules:</span>
<span class="go"> Host Path Backends</span>
<span class="go"> ---- ---- --------</span>
<span class="go"> cafe.com</span>
<span class="go"> /tea tea-svc:80 (&lt;none&gt;)</span>
<span class="go"> /coffee coffee-svc:80 (&lt;none&gt;)</span>
<span class="go">Annotations:</span>
<span class="go"> kubectl.kubernetes.io/last-applied-configuration: {&quot;apiVersion&quot;:&quot;extensions/v1beta1&quot;,&quot;kind&quot;:&quot;Ingress&quot;,&quot;metadata&quot;:{&quot;annotations&quot;:{},&quot;name&quot;:&quot;cafe-ingress&quot;,&quot;namespace&quot;:&quot;default&quot;,&quot;selfLink&quot;:&quot;/apis/extensions/v1beta1/namespaces/default/ingresses/cafe-ingress&quot;},&quot;spec&quot;:{&quot;rules&quot;:[{&quot;host&quot;:&quot;cafe.com&quot;,&quot;http&quot;:{&quot;paths&quot;:[{&quot;backend&quot;:{&quot;serviceName&quot;:&quot;tea-svc&quot;,&quot;servicePort&quot;:80},&quot;path&quot;:&quot;/tea&quot;},{&quot;backend&quot;:{&quot;serviceName&quot;:&quot;coffee-svc&quot;,&quot;servicePort&quot;:80},&quot;path&quot;:&quot;/coffee&quot;}]}}]},&quot;status&quot;:{&quot;loadBalancer&quot;:{&quot;ingress&quot;:[{&quot;ip&quot;:&quot;169.48.142.110&quot;}]}}}</span>
<span class="go">Events:</span>
<span class="go"> Type Reason Age From Message</span>
<span class="go"> ---- ------ ---- ---- -------</span>
<span class="go"> Normal CREATE 1m nginx-ingress-controller Ingress default/cafe-ingress</span>
<span class="go"> Normal UPDATE 58s nginx-ingress-controller Ingress default/cafe-ingress</span>
</pre></div>
<p>Check the Ingress Controller Logs</p>
<div class="codehilite"><pre><span></span><span class="gp">$</span> kubectl get pods -n &lt;namespace-of-ingress-controller&gt;
<span class="go">NAME READY STATUS RESTARTS AGE</span>
<span class="go">nginx-ingress-controller-67956bf89d-fv58j 1/1 Running 0 1m</span>
<span class="gp">$</span> kubectl logs -n &lt;namespace&gt; nginx-ingress-controller-67956bf89d-fv58j
<span class="go">-------------------------------------------------------------------------------</span>
<span class="go">NGINX Ingress controller</span>
<span class="go"> Release: 0.14.0</span>
<span class="go"> Build: git-734361d</span>
<span class="go"> Repository: https://github.com/kubernetes/ingress-nginx</span>
<span class="go">-------------------------------------------------------------------------------</span>
<span class="go">....</span>
</pre></div>
<p>Check the Nginx Configuration</p>
<div class="codehilite"><pre><span></span><span class="gp">$</span> kubectl get pods -n &lt;namespace-of-ingress-controller&gt;
<span class="go">NAME READY STATUS RESTARTS AGE</span>
<span class="go">nginx-ingress-controller-67956bf89d-fv58j 1/1 Running 0 1m</span>
<span class="gp">$</span> kubectl <span class="nb">exec</span> -it -n &lt;namespace-of-ingress-controller&gt; nginx-ingress-controller-67956bf89d-fv58j cat /etc/nginx/nginx.conf
<span class="go">daemon off;</span>
<span class="go">worker_processes 2;</span>
<span class="go">pid /run/nginx.pid;</span>
<span class="go">worker_rlimit_nofile 523264;</span>
<span class="go">worker_shutdown_timeout 10s;</span>
<span class="go">events {</span>
<span class="go"> multi_accept on;</span>
<span class="go"> worker_connections 16384;</span>
<span class="go"> use epoll;</span>
<span class="go">}</span>
<span class="go">http {</span>
<span class="go">....</span>
</pre></div>
<p>Check if used Services Exist</p>
<div class="codehilite"><pre><span></span><span class="gp">$</span> kubectl get svc --all-namespaces
<span class="go">NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE</span>
<span class="go">default coffee-svc ClusterIP 10.106.154.35 &lt;none&gt; 80/TCP 18m</span>
<span class="go">default kubernetes ClusterIP 10.96.0.1 &lt;none&gt; 443/TCP 30m</span>
<span class="go">default tea-svc ClusterIP 10.104.172.12 &lt;none&gt; 80/TCP 18m</span>
<span class="go">kube-system default-http-backend NodePort 10.108.189.236 &lt;none&gt; 80:30001/TCP 30m</span>
<span class="go">kube-system kube-dns ClusterIP 10.96.0.10 &lt;none&gt; 53/UDP,53/TCP 30m</span>
<span class="go">kube-system kubernetes-dashboard NodePort 10.103.128.17 &lt;none&gt; 80:30000/TCP 30m</span>
</pre></div>
<h2 id="debug-logging">Debug Logging<a class="headerlink" href="#debug-logging" title="Permanent link">&para;</a></h2>
<p>Using the flag <code class="codehilite">--v=XX</code> it is possible to increase the level of logging. This is performed by editing
the deployment.</p>
<div class="codehilite"><pre><span></span><span class="gp">$</span> kubectl get deploy -n &lt;namespace-of-ingress-controller&gt;
<span class="go">NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE</span>
<span class="go">default-http-backend 1 1 1 1 35m</span>
<span class="go">nginx-ingress-controller 1 1 1 1 35m</span>
<span class="gp">$</span> kubectl edit deploy -n &lt;namespace-of-ingress-controller&gt; nginx-ingress-controller
<span class="gp">#</span> Add --v<span class="o">=</span>X to <span class="s2">&quot;- args&quot;</span>, where X is an integer
</pre></div>
<ul>
<li><code class="codehilite">--v=2</code> shows details using <code class="codehilite">diff</code> about the changes in the configuration in nginx</li>
<li><code class="codehilite">--v=3</code> shows details about the service, Ingress rule, endpoint changes and it dumps the nginx configuration in JSON format</li>
<li><code class="codehilite">--v=5</code> configures NGINX in <a href="http://nginx.org/en/docs/debugging_log.html">debug mode</a></li>
</ul>
<h2 id="authentication-to-the-kubernetes-api-server">Authentication to the Kubernetes API Server<a class="headerlink" href="#authentication-to-the-kubernetes-api-server" title="Permanent link">&para;</a></h2>
<p>A number of components are involved in the authentication process and the first step is to narrow
down the source of the problem, namely whether it is a problem with service authentication or
with the kubeconfig file.</p>
<p>Both authentications must work:</p>
<div class="codehilite"><pre><span></span>+-------------+ service +------------+
| | authentication | |
+ apiserver +&lt;-------------------+ ingress |
| | | controller |
+-------------+ +------------+
</pre></div>
<p><strong>Service authentication</strong></p>
<p>The Ingress controller needs information from apiserver. Therefore, authentication is required, which can be achieved in two different ways:</p>
<ol>
<li>
<p><em>Service Account:</em> This is recommended, because nothing has to be configured. The Ingress controller will use information provided by the system to communicate with the API server. See 'Service Account' section for details.</p>
</li>
<li>
<p><em>Kubeconfig file:</em> In some Kubernetes environments service accounts are not available. In this case a manual configuration is required. The Ingress controller binary can be started with the <code class="codehilite">--kubeconfig</code> flag. The value of the flag is a path to a file specifying how to connect to the API server. Using the <code class="codehilite">--kubeconfig</code> does not requires the flag <code class="codehilite">--apiserver-host</code>.
The format of the file is identical to <code class="codehilite">~/.kube/config</code> which is used by kubectl to connect to the API server. See 'kubeconfig' section for details.</p>
</li>
<li>
<p><em>Using the flag <code class="codehilite">--apiserver-host</code>:</em> Using this flag <code class="codehilite">--apiserver-host=http://localhost:8080</code> it is possible to specify an unsecured API server or reach a remote kubernetes cluster using <a href="https://kubernetes.io/docs/user-guide/kubectl/kubectl_proxy/">kubectl proxy</a>.
Please do not use this approach in production.</p>
</li>
</ol>
<p>In the diagram below you can see the full authentication flow with all options, starting with the browser
on the lower left hand side.</p>
<div class="codehilite"><pre><span></span>Kubernetes Workstation
+---------------------------------------------------+ +------------------+
| | | |
| +-----------+ apiserver +------------+ | | +------------+ |
| | | proxy | | | | | | |
| | apiserver | | ingress | | | | ingress | |
| | | | controller | | | | controller | |
| | | | | | | | | |
| | | | | | | | | |
| | | service account/ | | | | | | |
| | | kubeconfig | | | | | | |
| | +&lt;-------------------+ | | | | | |
| | | | | | | | | |
| +------+----+ kubeconfig +------+-----+ | | +------+-----+ |
| |&lt;--------------------------------------------------------| |
| | | |
+---------------------------------------------------+ +------------------+
</pre></div>
<h3 id="service-account">Service Account<a class="headerlink" href="#service-account" title="Permanent link">&para;</a></h3>
<p>If using a service account to connect to the API server, Dashboard expects the file
<code class="codehilite">/var/run/secrets/kubernetes.io/serviceaccount/token</code> to be present. It provides a secret
token that is required to authenticate with the API server.</p>
<p>Verify with the following commands:</p>
<div class="codehilite"><pre><span></span><span class="gp">#</span> start a container that contains curl
<span class="gp">$</span> kubectl run <span class="nb">test</span> --image<span class="o">=</span>tutum/curl -- sleep <span class="m">10000</span>
<span class="gp">#</span> check that container is running
<span class="gp">$</span> kubectl get pods
<span class="go">NAME READY STATUS RESTARTS AGE</span>
<span class="go">test-701078429-s5kca 1/1 Running 0 16s</span>
<span class="gp">#</span> check <span class="k">if</span> secret exists
<span class="gp">$</span> kubectl <span class="nb">exec</span> test-701078429-s5kca ls /var/run/secrets/kubernetes.io/serviceaccount/
<span class="go">ca.crt</span>
<span class="go">namespace</span>
<span class="go">token</span>
<span class="gp">#</span> get service IP of master
<span class="gp">$</span> kubectl get services
<span class="go">NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE</span>
<span class="go">kubernetes 10.0.0.1 &lt;none&gt; 443/TCP 1d</span>
<span class="gp">#</span> check base connectivity from cluster inside
<span class="gp">$</span> kubectl <span class="nb">exec</span> test-701078429-s5kca -- curl -k https://10.0.0.1
<span class="go">Unauthorized</span>
<span class="gp">#</span> connect using tokens
<span class="gp">$</span> <span class="nv">TOKEN_VALUE</span><span class="o">=</span><span class="k">$(</span>kubectl <span class="nb">exec</span> test-701078429-s5kca -- cat /var/run/secrets/kubernetes.io/serviceaccount/token<span class="k">)</span>
<span class="gp">$</span> <span class="nb">echo</span> <span class="nv">$TOKEN_VALUE</span>
<span class="go">eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3Mi....9A</span>
<span class="gp">$</span> kubectl <span class="nb">exec</span> test-701078429-s5kca -- curl --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt -H <span class="s2">&quot;Authorization: Bearer </span><span class="nv">$TOKEN_VALUE</span><span class="s2">&quot;</span> https://10.0.0.1
<span class="go">{</span>
<span class="go"> &quot;paths&quot;: [</span>
<span class="go"> &quot;/api&quot;,</span>
<span class="go"> &quot;/api/v1&quot;,</span>
<span class="go"> &quot;/apis&quot;,</span>
<span class="go"> &quot;/apis/apps&quot;,</span>
<span class="go"> &quot;/apis/apps/v1alpha1&quot;,</span>
<span class="go"> &quot;/apis/authentication.k8s.io&quot;,</span>
<span class="go"> &quot;/apis/authentication.k8s.io/v1beta1&quot;,</span>
<span class="go"> &quot;/apis/authorization.k8s.io&quot;,</span>
<span class="go"> &quot;/apis/authorization.k8s.io/v1beta1&quot;,</span>
<span class="go"> &quot;/apis/autoscaling&quot;,</span>
<span class="go"> &quot;/apis/autoscaling/v1&quot;,</span>
<span class="go"> &quot;/apis/batch&quot;,</span>
<span class="go"> &quot;/apis/batch/v1&quot;,</span>
<span class="go"> &quot;/apis/batch/v2alpha1&quot;,</span>
<span class="go"> &quot;/apis/certificates.k8s.io&quot;,</span>
<span class="go"> &quot;/apis/certificates.k8s.io/v1alpha1&quot;,</span>
<span class="go"> &quot;/apis/extensions&quot;,</span>
<span class="go"> &quot;/apis/extensions/v1beta1&quot;,</span>
<span class="go"> &quot;/apis/policy&quot;,</span>
<span class="go"> &quot;/apis/policy/v1alpha1&quot;,</span>
<span class="go"> &quot;/apis/rbac.authorization.k8s.io&quot;,</span>
<span class="go"> &quot;/apis/rbac.authorization.k8s.io/v1alpha1&quot;,</span>
<span class="go"> &quot;/apis/storage.k8s.io&quot;,</span>
<span class="go"> &quot;/apis/storage.k8s.io/v1beta1&quot;,</span>
<span class="go"> &quot;/healthz&quot;,</span>
<span class="go"> &quot;/healthz/ping&quot;,</span>
<span class="go"> &quot;/logs&quot;,</span>
<span class="go"> &quot;/metrics&quot;,</span>
<span class="go"> &quot;/swaggerapi/&quot;,</span>
<span class="go"> &quot;/ui/&quot;,</span>
<span class="go"> &quot;/version&quot;</span>
<span class="go"> ]</span>
<span class="go">}</span>
</pre></div>
<p>If it is not working, there are two possible reasons:</p>
<ol>
<li>
<p>The contents of the tokens are invalid. Find the secret name with <code class="codehilite">kubectl get secrets | grep service-account</code> and
delete it with <code class="codehilite">kubectl delete secret &lt;name&gt;</code>. It will automatically be recreated.</p>
</li>
<li>
<p>You have a non-standard Kubernetes installation and the file containing the token may not be present.
The API server will mount a volume containing this file, but only if the API server is configured to use
the ServiceAccount admission controller.
If you experience this error, verify that your API server is using the ServiceAccount admission controller.
If you are configuring the API server by hand, you can set this with the <code class="codehilite">--admission-control</code> parameter.</p>
<blockquote>
<p>Note that you should use other admission controllers as well. Before configuring this option, you should read about admission controllers.</p>
</blockquote>
</li>
</ol>
<p>More information:</p>
<ul>
<li><a href="http://kubernetes.io/docs/user-guide/service-accounts/">User Guide: Service Accounts</a></li>
<li><a href="http://kubernetes.io/docs/admin/service-accounts-admin/">Cluster Administrator Guide: Managing Service Accounts</a></li>
</ul>
<h2 id="kube-config">Kube-Config<a class="headerlink" href="#kube-config" title="Permanent link">&para;</a></h2>
<p>If you want to use a kubeconfig file for authentication, follow the <a href="../deploy/">deploy procedure</a> and
add the flag <code class="codehilite">--kubeconfig=/etc/kubernetes/kubeconfig.yaml</code> to the args section of the deployment.</p>
<h2 id="using-gdb-with-nginx">Using GDB with Nginx<a class="headerlink" href="#using-gdb-with-nginx" title="Permanent link">&para;</a></h2>
<p><a href="https://www.gnu.org/software/gdb/">Gdb</a> can be used to with nginx to perform a configuration
dump. This allows us to see which configuration is being used, as well as older configurations.</p>
<p>Note: The below is based on the nginx <a href="https://docs.nginx.com/nginx/admin-guide/monitoring/debugging/#dumping-nginx-configuration-from-a-running-process">documentation</a>.</p>
<ol>
<li>SSH into the worker</li>
</ol>
<div class="codehilite"><pre><span></span><span class="gp">$</span> ssh user@workerIP
</pre></div>
<ol>
<li>Obtain the Docker Container Running nginx</li>
</ol>
<div class="codehilite"><pre><span></span><span class="gp">$</span> docker ps <span class="p">|</span> grep nginx-ingress-controller
<span class="go">CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES</span>
<span class="go">d9e1d243156a quay.io/kubernetes-ingress-controller/nginx-ingress-controller &quot;/usr/bin/dumb-init …&quot; 19 minutes ago Up 19 minutes k8s_nginx-ingress-controller_nginx-ingress-controller-67956bf89d-mqxzt_kube-system_079f31ec-aa37-11e8-ad39-080027a227db_0</span>
</pre></div>
<ol>
<li>Exec into the container</li>
</ol>
<div class="codehilite"><pre><span></span><span class="gp">$</span> docker <span class="nb">exec</span> -it --user<span class="o">=</span><span class="m">0</span> --privileged d9e1d243156a bash
</pre></div>
<ol>
<li>Make sure nginx is running in <code class="codehilite">--with-debug</code></li>
</ol>
<div class="codehilite"><pre><span></span><span class="gp">$</span> nginx -V <span class="m">2</span>&gt;<span class="p">&amp;</span><span class="m">1</span> <span class="p">|</span> grep -- <span class="s1">&#39;--with-debug&#39;</span>
</pre></div>
<ol>
<li>Get list of processes running on container</li>
</ol>
<div class="codehilite"><pre><span></span><span class="gp">$</span> ps -ef
<span class="go">UID PID PPID C STIME TTY TIME CMD</span>
<span class="go">root 1 0 0 20:23 ? 00:00:00 /usr/bin/dumb-init /nginx-ingres</span>
<span class="go">root 5 1 0 20:23 ? 00:00:05 /nginx-ingress-controller --defa</span>
<span class="go">root 21 5 0 20:23 ? 00:00:00 nginx: master process /usr/sbin/</span>
<span class="go">nobody 106 21 0 20:23 ? 00:00:00 nginx: worker process</span>
<span class="go">nobody 107 21 0 20:23 ? 00:00:00 nginx: worker process</span>
<span class="go">root 172 0 0 20:43 pts/0 00:00:00 bash</span>
</pre></div>
<ol>
<li>Attach gdb to the nginx master process</li>
</ol>
<div class="codehilite"><pre><span></span><span class="gp">$</span> gdb -p <span class="m">21</span>
<span class="go">....</span>
<span class="go">Attaching to process 21</span>
<span class="go">Reading symbols from /usr/sbin/nginx...done.</span>
<span class="go">....</span>
<span class="go">(gdb)</span>
</pre></div>
<ol>
<li>Copy and paste the following:</li>
</ol>
<div class="codehilite"><pre><span></span><span class="go">set $cd = ngx_cycle-&gt;config_dump</span>
<span class="go">set $nelts = $cd.nelts</span>
<span class="go">set $elts = (ngx_conf_dump_t*)($cd.elts)</span>
<span class="go">while ($nelts-- &gt; 0)</span>
<span class="go">set $name = $elts[$nelts]-&gt;name.data</span>
<span class="go">printf &quot;Dumping %s to nginx_conf.txt\n&quot;, $name</span>
<span class="go">append memory nginx_conf.txt \</span>
<span class="gp"> $</span>elts<span class="o">[</span><span class="nv">$nelts</span><span class="o">]</span>-&gt;buffer.start <span class="nv">$elts</span><span class="o">[</span><span class="nv">$nelts</span><span class="o">]</span>-&gt;buffer.end
<span class="go">end</span>
</pre></div>
<ol>
<li>
<p>Quit GDB by pressing CTRL+D</p>
</li>
<li>
<p>Open nginx_conf.txt</p>
</li>
</ol>
<div class="codehilite"><pre><span></span><span class="go">cat nginx_conf.txt</span>
</pre></div>
</article>
</div>
</div>
</main>
<footer class="md-footer">
<div class="md-footer-nav">
<nav class="md-footer-nav__inner md-grid">
<a href="../how-it-works/" title="How it works" class="md-flex md-footer-nav__link md-footer-nav__link--prev" rel="prev">
<div class="md-flex__cell md-flex__cell--shrink">
<i class="md-icon md-icon--arrow-back md-footer-nav__button"></i>
</div>
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
<span class="md-flex__ellipsis">
<span class="md-footer-nav__direction">
Previous
</span>
How it works
</span>
</div>
</a>
</nav>
</div>
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-footer-copyright">
powered by
<a href="https://www.mkdocs.org">MkDocs</a>
and
<a href="https://squidfunk.github.io/mkdocs-material/">
Material for MkDocs</a>
</div>
</div>
</div>
</footer>
</div>
<script src="../assets/javascripts/application.30f6b8b1.js"></script>
<script>app.initialize({version:"0.17.5",url:{base:".."}})</script>
<script>!function(e,a,t,n,o,c,i){e.GoogleAnalyticsObject=o,e.ga=e.ga||function(){(e.ga.q=e.ga.q||[]).push(arguments)},e.ga.l=1*new Date,c=a.createElement(t),i=a.getElementsByTagName(t)[0],c.async=1,c.src="https://www.google-analytics.com/analytics.js",i.parentNode.insertBefore(c,i)}(window,document,"script",0,"ga"),ga("create","UA-118407822-1","kubernetes.github.io"),ga("set","anonymizeIp",!0),ga("send","pageview");var links=document.getElementsByTagName("a");if(Array.prototype.map.call(links,function(e){e.host!=document.location.host&&e.addEventListener("click",function(){var a=e.getAttribute("data-md-action")||"follow";ga("send","event","outbound",a,e.href)})}),document.forms.search){var query=document.forms.search.query;query.addEventListener("blur",function(){if(this.value){var e=document.location.pathname;ga("send","pageview",e+"?q="+this.value)}})}</script>
</body>
</html>