Make it possible to enable Prometheus metrics for Cilium

k8s/crds/kops.k8s.io_clusters.yaml

@@ -2547,6 +2547,8 @@ spec:
                       items:
                         type: string
                       type: array
+                    agentPrometheusPort:
+                      type: integer
                     allowLocalhost:
                       type: string
                     autoDirectNodeRoutes:
@@ -2593,6 +2595,8 @@ spec:
                       type: boolean
                     enablePolicy:
                       type: string
+                    enablePrometheusMetrics:
+                      type: boolean
                     enableTracing:
                       type: boolean
                     enableipv4:

pkg/apis/kops/networking.go

@@ -200,6 +200,7 @@ type CiliumNetworkingSpec struct {
 
 	AccessLog                string            `json:"accessLog,omitempty"`
 	AgentLabels              []string          `json:"agentLabels,omitempty"`
+	AgentPrometheusPort      int               `json:"agentPrometheusPort,omitempty"`
 	AllowLocalhost           string            `json:"allowLocalhost,omitempty"`
 	AutoIpv6NodeRoutes       bool              `json:"autoIpv6NodeRoutes,omitempty"`
 	BPFRoot                  string            `json:"bpfRoot,omitempty"`
@@ -213,6 +214,7 @@ type CiliumNetworkingSpec struct {
 	DisableK8sServices       bool              `json:"disableK8sServices,omitempty"`
 	EnablePolicy             string            `json:"enablePolicy,omitempty"`
 	EnableTracing            bool              `json:"enableTracing,omitempty"`
+	EnablePrometheusMetrics  bool              `json:"enablePrometheusMetrics,omitempty"`
 	EnvoyLog                 string            `json:"envoyLog,omitempty"`
 	Ipv4ClusterCIDRMaskSize  int               `json:"ipv4ClusterCidrMaskSize,omitempty"`
 	Ipv4Node                 string            `json:"ipv4Node,omitempty"`

pkg/apis/kops/v1alpha1/networking.go

@@ -197,6 +197,7 @@ type CiliumNetworkingSpec struct {
 
 	AccessLog                string            `json:"accessLog,omitempty"`
 	AgentLabels              []string          `json:"agentLabels,omitempty"`
+	AgentPrometheusPort      int               `json:"agentPrometheusPort,omitempty"`
 	AllowLocalhost           string            `json:"allowLocalhost,omitempty"`
 	AutoIpv6NodeRoutes       bool              `json:"autoIpv6NodeRoutes,omitempty"`
 	BPFRoot                  string            `json:"bpfRoot,omitempty"`
@@ -209,6 +210,7 @@ type CiliumNetworkingSpec struct {
 	DisableIpv4              bool              `json:"disableIpv4,omitempty"`
 	DisableK8sServices       bool              `json:"disableK8sServices,omitempty"`
 	EnablePolicy             string            `json:"enablePolicy,omitempty"`
+	EnablePrometheusMetrics  bool              `json:"enablePrometheusMetrics,omitempty"`
 	EnableTracing            bool              `json:"enableTracing,omitempty"`
 	EnvoyLog                 string            `json:"envoyLog,omitempty"`
 	Ipv4ClusterCIDRMaskSize  int               `json:"ipv4ClusterCidrMaskSize,omitempty"`

pkg/apis/kops/v1alpha1/zz_generated.conversion.go

@@ -1250,6 +1250,7 @@ func autoConvert_v1alpha1_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in *
 	out.Version = in.Version
 	out.AccessLog = in.AccessLog
 	out.AgentLabels = in.AgentLabels
+	out.AgentPrometheusPort = in.AgentPrometheusPort
 	out.AllowLocalhost = in.AllowLocalhost
 	out.AutoIpv6NodeRoutes = in.AutoIpv6NodeRoutes
 	out.BPFRoot = in.BPFRoot
@@ -1262,6 +1263,7 @@ func autoConvert_v1alpha1_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in *
 	out.DisableIpv4 = in.DisableIpv4
 	out.DisableK8sServices = in.DisableK8sServices
 	out.EnablePolicy = in.EnablePolicy
+	out.EnablePrometheusMetrics = in.EnablePrometheusMetrics
 	out.EnableTracing = in.EnableTracing
 	out.EnvoyLog = in.EnvoyLog
 	out.Ipv4ClusterCIDRMaskSize = in.Ipv4ClusterCIDRMaskSize
@@ -1326,6 +1328,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha1_CiliumNetworkingSpec(in *
 	out.Version = in.Version
 	out.AccessLog = in.AccessLog
 	out.AgentLabels = in.AgentLabels
+	out.AgentPrometheusPort = in.AgentPrometheusPort
 	out.AllowLocalhost = in.AllowLocalhost
 	out.AutoIpv6NodeRoutes = in.AutoIpv6NodeRoutes
 	out.BPFRoot = in.BPFRoot
@@ -1339,6 +1342,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha1_CiliumNetworkingSpec(in *
 	out.DisableK8sServices = in.DisableK8sServices
 	out.EnablePolicy = in.EnablePolicy
 	out.EnableTracing = in.EnableTracing
+	out.EnablePrometheusMetrics = in.EnablePrometheusMetrics
 	out.EnvoyLog = in.EnvoyLog
 	out.Ipv4ClusterCIDRMaskSize = in.Ipv4ClusterCIDRMaskSize
 	out.Ipv4Node = in.Ipv4Node

pkg/apis/kops/v1alpha2/networking.go

@@ -198,6 +198,7 @@ type CiliumNetworkingSpec struct {
 
 	AccessLog                string            `json:"accessLog,omitempty"`
 	AgentLabels              []string          `json:"agentLabels,omitempty"`
+	AgentPrometheusPort      int               `json:"agentPrometheusPort,omitempty"`
 	AllowLocalhost           string            `json:"allowLocalhost,omitempty"`
 	AutoIpv6NodeRoutes       bool              `json:"autoIpv6NodeRoutes,omitempty"`
 	BPFRoot                  string            `json:"bpfRoot,omitempty"`
@@ -210,6 +211,7 @@ type CiliumNetworkingSpec struct {
 	DisableIpv4              bool              `json:"disableIpv4,omitempty"`
 	DisableK8sServices       bool              `json:"disableK8sServices,omitempty"`
 	EnablePolicy             string            `json:"enablePolicy,omitempty"`
+	EnablePrometheusMetrics  bool              `json:"enablePrometheusMetrics,omitempty"`
 	EnableTracing            bool              `json:"enableTracing,omitempty"`
 	EnvoyLog                 string            `json:"envoyLog,omitempty"`
 	Ipv4ClusterCIDRMaskSize  int               `json:"ipv4ClusterCidrMaskSize,omitempty"`

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -1292,6 +1292,7 @@ func autoConvert_v1alpha2_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in *
 	out.Version = in.Version
 	out.AccessLog = in.AccessLog
 	out.AgentLabels = in.AgentLabels
+	out.AgentPrometheusPort = in.AgentPrometheusPort
 	out.AllowLocalhost = in.AllowLocalhost
 	out.AutoIpv6NodeRoutes = in.AutoIpv6NodeRoutes
 	out.BPFRoot = in.BPFRoot
@@ -1304,6 +1305,7 @@ func autoConvert_v1alpha2_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in *
 	out.DisableIpv4 = in.DisableIpv4
 	out.DisableK8sServices = in.DisableK8sServices
 	out.EnablePolicy = in.EnablePolicy
+	out.EnablePrometheusMetrics = in.EnablePrometheusMetrics
 	out.EnableTracing = in.EnableTracing
 	out.EnvoyLog = in.EnvoyLog
 	out.Ipv4ClusterCIDRMaskSize = in.Ipv4ClusterCIDRMaskSize
@@ -1368,6 +1370,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha2_CiliumNetworkingSpec(in *
 	out.Version = in.Version
 	out.AccessLog = in.AccessLog
 	out.AgentLabels = in.AgentLabels
+	out.AgentPrometheusPort = in.AgentPrometheusPort
 	out.AllowLocalhost = in.AllowLocalhost
 	out.AutoIpv6NodeRoutes = in.AutoIpv6NodeRoutes
 	out.BPFRoot = in.BPFRoot
@@ -1381,6 +1384,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha2_CiliumNetworkingSpec(in *
 	out.DisableK8sServices = in.DisableK8sServices
 	out.EnablePolicy = in.EnablePolicy
 	out.EnableTracing = in.EnableTracing
+	out.EnablePrometheusMetrics = in.EnablePrometheusMetrics
 	out.EnvoyLog = in.EnvoyLog
 	out.Ipv4ClusterCIDRMaskSize = in.Ipv4ClusterCIDRMaskSize
 	out.Ipv4Node = in.Ipv4Node

upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template

@@ -20,6 +20,15 @@ data:
   identity-allocation-mode: crd
   # If you want to run cilium in debug mode change this value to true
   debug: "{{- if .Debug -}}true{{- else -}}false{{- end -}}"
+  {{ if .EnablePrometheusMetrics }}
+  # If you want metrics enabled in all of your Cilium agents, set the port for
+  # which the Cilium agents will have their metrics exposed.
+  # This option deprecates the "prometheus-serve-addr" in the
+  # "cilium-metrics-config" ConfigMap
+  # NOTE that this will open the port on ALL nodes where Cilium pods are
+  # scheduled.
+  prometheus-serve-addr: ":{{- or .AgentPrometheusPort "9090" }}"
+  {{ end }}
   # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4
   # address.
   enable-ipv4: "{{- if or  (.EnableIpv4) (and (not (.EnableIpv4)) (not (.EnableIpv6))) -}}true{{- else -}}false{{- end -}}"
@@ -385,6 +394,13 @@ spec:
           successThreshold: 1
           timeoutSeconds: 5
         name: cilium-agent
+        {{ if .EnablePrometheusMetrics }}
+        ports:
+        - containerPort: {{ or .AgentPrometheusPort "9090" }}
+          hostPort: {{ or .AgentPrometheusPort "9090" }}
+          name: prometheus
+          protocol: TCP
+        {{ end }}
         readinessProbe:
           exec:
             command:
@@ -543,6 +559,10 @@ spec:
       - args:
         - --debug=$(CILIUM_DEBUG)
         - --identity-allocation-mode=$(CILIUM_IDENTITY_ALLOCATION_MODE)
+{{ with .Networking.Cilium }}
+        {{ if .EnablePrometheusMetrics }}
+        - --enable-metrics
+        {{ end }}
         command:
         - cilium-operator
         env:
@@ -622,12 +642,17 @@ spec:
               key: AWS_DEFAULT_REGION
               name: cilium-aws
               optional: true
-
-{{ with .Networking.Cilium }}
         image: "docker.io/cilium/operator:{{ .Version }}"
-{{ end }}
         imagePullPolicy: IfNotPresent
         name: cilium-operator
+        {{ if .EnablePrometheusMetrics }}
+        ports:
+        - containerPort: 6942
+          hostPort: 6942
+          name: prometheus
+          protocol: TCP
+        {{ end }}
+{{ end }}
         livenessProbe:
           httpGet:
             path: /healthz

upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template

@@ -20,6 +20,15 @@ data:
   identity-allocation-mode: crd
   # If you want to run cilium in debug mode change this value to true
   debug: "{{- if .Debug -}}true{{- else -}}false{{- end -}}"
+  {{ if .EnablePrometheusMetrics }}
+  # If you want metrics enabled in all of your Cilium agents, set the port for
+  # which the Cilium agents will have their metrics exposed.
+  # This option deprecates the "prometheus-serve-addr" in the
+  # "cilium-metrics-config" ConfigMap
+  # NOTE that this will open the port on ALL nodes where Cilium pods are
+  # scheduled.
+  prometheus-serve-addr: ":{{- or .AgentPrometheusPort "9090" }}"
+  {{ end }}
   # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4
   # address.
   enable-ipv4: "{{- if or  (.EnableIpv4) (and (not (.EnableIpv4)) (not (.EnableIpv6))) -}}true{{- else -}}false{{- end -}}"
@@ -385,6 +394,13 @@ spec:
           successThreshold: 1
           timeoutSeconds: 5
         name: cilium-agent
+        {{ if .EnablePrometheusMetrics }}
+        ports:
+        - containerPort: {{ or .AgentPrometheusPort "9090" }}
+          hostPort: {{ or .AgentPrometheusPort "9090" }}
+          name: prometheus
+          protocol: TCP
+        {{ end }}
         readinessProbe:
           exec:
             command:
@@ -541,6 +557,10 @@ spec:
       containers:
       - args:
         - --debug=$(CILIUM_DEBUG)
+{{ with .Networking.Cilium }}
+        {{ if .EnablePrometheusMetrics }}
+        - --enable-metrics
+        {{ end }}
         command:
         - cilium-operator
         env:
@@ -614,12 +634,17 @@ spec:
               key: AWS_DEFAULT_REGION
               name: cilium-aws
               optional: true
-
-{{ with .Networking.Cilium }}
         image: "docker.io/cilium/operator:{{ .Version }}"
-{{ end }}
         imagePullPolicy: IfNotPresent
         name: cilium-operator
+        {{ if .EnablePrometheusMetrics }}
+        ports:
+        - containerPort: 6942
+          hostPort: 6942
+          name: prometheus
+          protocol: TCP
+        {{ end }}
+{{ end }}
         livenessProbe:
           httpGet:
             path: /healthz