Merge pull request #1752 from ssabo/master

add exit code

docs/pod-metrics.md

@@ -21,6 +21,7 @@
 | kube_pod_container_status_terminated | Gauge | Describes whether the container is currently in terminated state | |`container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
 | kube_pod_container_status_terminated_reason | Gauge | Describes the reason the container is currently in terminated state | |`container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `reason`=&lt;container-terminated-reason&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |
 | kube_pod_container_status_last_terminated_reason | Gauge | Describes the last reason the container was in terminated state | |`container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `reason`=&lt;last-terminated-reason&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |
+| kube_pod_container_status_last_terminated_exitcode | Gauge | Describes the exit code for the last container in terminated state. |  | `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |
 | kube_pod_container_status_ready | Gauge | Describes whether the containers readiness check succeeded | |`container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
 | kube_pod_container_status_restarts_total | Counter | The number of container restarts per container | | `container`=&lt;container-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `pod`=&lt;pod-name&gt; <br> `uid`=&lt;pod-uid&gt; | STABLE | - |
 | kube_pod_container_resource_requests | Gauge | The number of requested request resource by a container | `cpu`=&lt;core&gt; <br> `memory`=&lt;bytes&gt; |`resource`=&lt;resource-name&gt; <br> `unit`=&lt;resource-unit&gt; <br> `container`=&lt;container-name&gt; <br> `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `node`=&lt; node-name&gt; <br> `uid`=&lt;pod-uid&gt; | EXPERIMENTAL | - |

internal/store/pod.go

@@ -47,6 +47,7 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat
 		createPodContainerResourceRequestsFamilyGenerator(),
 		createPodContainerStateStartedFamilyGenerator(),
 		createPodContainerStatusLastTerminatedReasonFamilyGenerator(),
+		createPodContainerStatusLastTerminatedExitCodeFamilyGenerator(),
 		createPodContainerStatusReadyFamilyGenerator(),
 		createPodContainerStatusRestartsTotalFamilyGenerator(),
 		createPodContainerStatusRunningFamilyGenerator(),
@@ -335,6 +336,31 @@ func createPodContainerStatusLastTerminatedReasonFamilyGenerator() generator.Fam
 	)
 }
 
+func createPodContainerStatusLastTerminatedExitCodeFamilyGenerator() generator.FamilyGenerator {
+	return *generator.NewFamilyGenerator(
+		"kube_pod_container_status_last_terminated_exitcode",
+		"Describes the exit code for the last container in terminated state.",
+		metric.Gauge,
+		"",
+		wrapPodFunc(func(p *v1.Pod) *metric.Family {
+			ms := make([]*metric.Metric, 0, len(p.Status.ContainerStatuses))
+			for _, cs := range p.Status.ContainerStatuses {
+				if cs.LastTerminationState.Terminated != nil {
+					ms = append(ms, &metric.Metric{
+						LabelKeys:   []string{"container"},
+						LabelValues: []string{cs.Name},
+						Value:       float64(cs.LastTerminationState.Terminated.ExitCode),
+					})
+				}
+			}
+
+			return &metric.Family{
+				Metrics: ms,
+			}
+		}),
+	)
+}
+
 func createPodContainerStatusReadyFamilyGenerator() generator.FamilyGenerator {
 	return *generator.NewFamilyGenerator(
 		"kube_pod_container_status_ready",

internal/store/pod_test.go

@@ -573,7 +573,8 @@ func TestPodStore(t *testing.T) {
 									StartedAt: metav1.Time{
 										Time: time.Unix(1501777018, 0),
 									},
-									Reason: "OOMKilled",
+									Reason:   "OOMKilled",
+									ExitCode: 137,
 								},
 							},
 						},
@@ -648,12 +649,14 @@ func TestPodStore(t *testing.T) {
 				},
 			},
 			Want: `
+				# HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state.
 				# HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state.
 				# HELP kube_pod_container_status_running Describes whether the container is currently in running state.
 				# HELP kube_pod_container_status_terminated Describes whether the container is currently in terminated state.
 				# HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state.
 				# HELP kube_pod_container_status_waiting Describes whether the container is currently in waiting state.
 				# HELP kube_pod_container_status_waiting_reason Describes the reason the container is currently in waiting state.
+				# TYPE kube_pod_container_status_last_terminated_exitcode gauge
 				# TYPE kube_pod_container_status_last_terminated_reason gauge
 				# TYPE kube_pod_container_status_running gauge
 				# TYPE kube_pod_container_status_terminated gauge
@@ -678,6 +681,10 @@ func TestPodStore(t *testing.T) {
 				"kube_pod_container_status_last_terminated_reason",
 				"kube_pod_container_status_last_terminated_reason",
 				"kube_pod_container_status_last_terminated_reason",
+				"kube_pod_container_status_last_terminated_exitcode",
+				"kube_pod_container_status_last_terminated_exitcode",
+				"kube_pod_container_status_last_terminated_exitcode",
+				"kube_pod_container_status_last_terminated_exitcode",
 			},
 		},
 		{
@@ -709,7 +716,8 @@ func TestPodStore(t *testing.T) {
 							},
 							LastTerminationState: v1.ContainerState{
 								Terminated: &v1.ContainerStateTerminated{
-									Reason: "OOMKilled",
+									Reason:   "OOMKilled",
+									ExitCode: 137,
 								},
 							},
 						},
@@ -718,6 +726,7 @@ func TestPodStore(t *testing.T) {
 			},
 			Want: `
 				# HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state.
+				# HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state.
 				# HELP kube_pod_container_status_running Describes whether the container is currently in running state.
 				# HELP kube_pod_container_status_terminated Describes whether the container is currently in terminated state.
 				# HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state.
@@ -725,6 +734,7 @@ func TestPodStore(t *testing.T) {
 				# HELP kube_pod_container_status_waiting_reason Describes the reason the container is currently in waiting state.
 				# HELP kube_pod_container_state_started Start time in unix timestamp for a pod container.
 				# TYPE kube_pod_container_status_last_terminated_reason gauge
+				# TYPE kube_pod_container_status_last_terminated_exitcode gauge
 				# TYPE kube_pod_container_status_running gauge
 				# TYPE kube_pod_container_status_terminated gauge
 				# TYPE kube_pod_container_status_terminated_reason gauge
@@ -736,9 +746,11 @@ func TestPodStore(t *testing.T) {
 				kube_pod_container_status_terminated{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 0
 				kube_pod_container_status_waiting{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 0
 				kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns6",pod="pod6",reason="OOMKilled",uid="uid6"} 1
+				kube_pod_container_status_last_terminated_exitcode{container="container7",namespace="ns6",pod="pod6",uid="uid6"} 137
 			`,
 			MetricNames: []string{
 				"kube_pod_container_status_last_terminated_reason",
+				"kube_pod_container_status_last_terminated_exitcode",
 				"kube_pod_container_status_running",
 				"kube_pod_container_state_started",
 				"kube_pod_container_status_terminated",
@@ -774,7 +786,8 @@ func TestPodStore(t *testing.T) {
 							},
 							LastTerminationState: v1.ContainerState{
 								Terminated: &v1.ContainerStateTerminated{
-									Reason: "DeadlineExceeded",
+									Reason:   "DeadlineExceeded",
+									ExitCode: 143,
 								},
 							},
 						},
@@ -782,6 +795,7 @@ func TestPodStore(t *testing.T) {
 				},
 			},
 			Want: `
+				# HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state.
 				# HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state.
 				# HELP kube_pod_container_status_running Describes whether the container is currently in running state.
 				# HELP kube_pod_container_state_started Start time in unix timestamp for a pod container.
@@ -789,6 +803,7 @@ func TestPodStore(t *testing.T) {
 				# HELP kube_pod_container_status_terminated_reason Describes the reason the container is currently in terminated state.
 				# HELP kube_pod_container_status_waiting Describes whether the container is currently in waiting state.
 				# HELP kube_pod_container_status_waiting_reason Describes the reason the container is currently in waiting state.
+				# TYPE kube_pod_container_status_last_terminated_exitcode gauge
 				# TYPE kube_pod_container_status_last_terminated_reason gauge
 				# TYPE kube_pod_container_status_running gauge
 				# TYPE kube_pod_container_state_started gauge
@@ -797,6 +812,7 @@ func TestPodStore(t *testing.T) {
 				# TYPE kube_pod_container_status_waiting gauge
 				# TYPE kube_pod_container_status_waiting_reason gauge
 				kube_pod_container_state_started{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 1.501777018e+09
+				kube_pod_container_status_last_terminated_exitcode{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 143
 				kube_pod_container_status_last_terminated_reason{container="container7",namespace="ns7",pod="pod7",reason="DeadlineExceeded",uid="uid7"} 1
 				kube_pod_container_status_running{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 1
 				kube_pod_container_status_terminated{container="container7",namespace="ns7",pod="pod7",uid="uid7"} 0
@@ -809,6 +825,7 @@ func TestPodStore(t *testing.T) {
 				"kube_pod_container_status_terminated_reason",
 				"kube_pod_container_status_waiting",
 				"kube_pod_container_status_last_terminated_reason",
+				"kube_pod_container_status_last_terminated_exitcode",
 			},
 		},
 		{
@@ -2019,7 +2036,8 @@ func BenchmarkPodStore(b *testing.B) {
 					},
 					LastTerminationState: v1.ContainerState{
 						Terminated: &v1.ContainerStateTerminated{
-							Reason: "OOMKilled",
+							Reason:   "OOMKilled",
+							ExitCode: 137,
 						},
 					},
 				},
@@ -2035,7 +2053,8 @@ func BenchmarkPodStore(b *testing.B) {
 					},
 					LastTerminationState: v1.ContainerState{
 						Terminated: &v1.ContainerStateTerminated{
-							Reason: "OOMKilled",
+							Reason:   "OOMKilled",
+							ExitCode: 137,
 						},
 					},
 				},
@@ -2051,7 +2070,8 @@ func BenchmarkPodStore(b *testing.B) {
 					},
 					LastTerminationState: v1.ContainerState{
 						Terminated: &v1.ContainerStateTerminated{
-							Reason: "OOMKilled",
+							Reason:   "OOMKilled",
+							ExitCode: 137,
 						},
 					},
 				},
@@ -2059,7 +2079,7 @@ func BenchmarkPodStore(b *testing.B) {
 		},
 	}
 
-	expectedFamilies := 46
+	expectedFamilies := 47
 	for n := 0; n < b.N; n++ {
 		families := f(pod)
 		if len(families) != expectedFamilies {

pkg/app/server_test.go

@@ -196,6 +196,7 @@ func TestFullScrapeCycle(t *testing.T) {
 # HELP kube_pod_container_resource_limits The number of requested limit resource by a container.
 # HELP kube_pod_container_resource_requests The number of requested request resource by a container.
 # HELP kube_pod_container_state_started Start time in unix timestamp for a pod container.
+# HELP kube_pod_container_status_last_terminated_exitcode Describes the exit code for the last container in terminated state.
 # HELP kube_pod_container_status_last_terminated_reason Describes the last reason the container was in terminated state.
 # HELP kube_pod_container_status_ready Describes whether the containers readiness check succeeded.
 # HELP kube_pod_container_status_restarts_total The number of container restarts per container.
@@ -241,6 +242,7 @@ func TestFullScrapeCycle(t *testing.T) {
 # TYPE kube_pod_container_resource_limits gauge
 # TYPE kube_pod_container_resource_requests gauge
 # TYPE kube_pod_container_state_started gauge
+# TYPE kube_pod_container_status_last_terminated_exitcode gauge
 # TYPE kube_pod_container_status_last_terminated_reason gauge
 # TYPE kube_pod_container_status_ready gauge
 # TYPE kube_pod_container_status_restarts_total counter
@@ -297,6 +299,7 @@ kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",
 kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",node="node1",resource="storage",unit="byte"} 4e+08
 kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2",node="node1",resource="cpu",unit="core"} 0.3
 kube_pod_container_resource_requests{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2",node="node1",resource="memory",unit="byte"} 2e+08
+kube_pod_container_status_last_terminated_exitcode{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 137
 kube_pod_container_status_last_terminated_reason{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1",reason="OOMKilled"} 1
 kube_pod_container_status_ready{namespace="default",pod="pod0",uid="abc-0",container="pod1_con1"} 0
 kube_pod_container_status_ready{namespace="default",pod="pod0",uid="abc-0",container="pod1_con2"} 0
@@ -794,7 +797,8 @@ func pod(client *fake.Clientset, index int) error {
 					},
 					LastTerminationState: v1.ContainerState{
 						Terminated: &v1.ContainerStateTerminated{
-							Reason: "OOMKilled",
+							Reason:   "OOMKilled",
+							ExitCode: 137,
 						},
 					},
 				},