diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index c466f8336e..a4ebca853f 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -2547,6 +2547,8 @@ spec: items: type: string type: array + agentPrometheusPort: + type: integer allowLocalhost: type: string autoDirectNodeRoutes: @@ -2593,6 +2595,8 @@ spec: type: boolean enablePolicy: type: string + enablePrometheusMetrics: + type: boolean enableTracing: type: boolean enableipv4: diff --git a/pkg/apis/kops/networking.go b/pkg/apis/kops/networking.go index eed7809648..3e0119f381 100644 --- a/pkg/apis/kops/networking.go +++ b/pkg/apis/kops/networking.go @@ -200,6 +200,7 @@ type CiliumNetworkingSpec struct { AccessLog string `json:"accessLog,omitempty"` AgentLabels []string `json:"agentLabels,omitempty"` + AgentPrometheusPort int `json:"agentPrometheusPort,omitempty"` AllowLocalhost string `json:"allowLocalhost,omitempty"` AutoIpv6NodeRoutes bool `json:"autoIpv6NodeRoutes,omitempty"` BPFRoot string `json:"bpfRoot,omitempty"` @@ -213,6 +214,7 @@ type CiliumNetworkingSpec struct { DisableK8sServices bool `json:"disableK8sServices,omitempty"` EnablePolicy string `json:"enablePolicy,omitempty"` EnableTracing bool `json:"enableTracing,omitempty"` + EnablePrometheusMetrics bool `json:"enablePrometheusMetrics,omitempty"` EnvoyLog string `json:"envoyLog,omitempty"` Ipv4ClusterCIDRMaskSize int `json:"ipv4ClusterCidrMaskSize,omitempty"` Ipv4Node string `json:"ipv4Node,omitempty"` diff --git a/pkg/apis/kops/v1alpha1/networking.go b/pkg/apis/kops/v1alpha1/networking.go index 9332c48cb9..a125bf8a62 100644 --- a/pkg/apis/kops/v1alpha1/networking.go +++ b/pkg/apis/kops/v1alpha1/networking.go @@ -197,6 +197,7 @@ type CiliumNetworkingSpec struct { AccessLog string `json:"accessLog,omitempty"` AgentLabels []string `json:"agentLabels,omitempty"` + AgentPrometheusPort int `json:"agentPrometheusPort,omitempty"` AllowLocalhost string `json:"allowLocalhost,omitempty"` AutoIpv6NodeRoutes bool `json:"autoIpv6NodeRoutes,omitempty"` BPFRoot string `json:"bpfRoot,omitempty"` @@ -209,6 +210,7 @@ type CiliumNetworkingSpec struct { DisableIpv4 bool `json:"disableIpv4,omitempty"` DisableK8sServices bool `json:"disableK8sServices,omitempty"` EnablePolicy string `json:"enablePolicy,omitempty"` + EnablePrometheusMetrics bool `json:"enablePrometheusMetrics,omitempty"` EnableTracing bool `json:"enableTracing,omitempty"` EnvoyLog string `json:"envoyLog,omitempty"` Ipv4ClusterCIDRMaskSize int `json:"ipv4ClusterCidrMaskSize,omitempty"` diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index 5d2eca016a..096bfe0c54 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -1250,6 +1250,7 @@ func autoConvert_v1alpha1_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in * out.Version = in.Version out.AccessLog = in.AccessLog out.AgentLabels = in.AgentLabels + out.AgentPrometheusPort = in.AgentPrometheusPort out.AllowLocalhost = in.AllowLocalhost out.AutoIpv6NodeRoutes = in.AutoIpv6NodeRoutes out.BPFRoot = in.BPFRoot @@ -1262,6 +1263,7 @@ func autoConvert_v1alpha1_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in * out.DisableIpv4 = in.DisableIpv4 out.DisableK8sServices = in.DisableK8sServices out.EnablePolicy = in.EnablePolicy + out.EnablePrometheusMetrics = in.EnablePrometheusMetrics out.EnableTracing = in.EnableTracing out.EnvoyLog = in.EnvoyLog out.Ipv4ClusterCIDRMaskSize = in.Ipv4ClusterCIDRMaskSize @@ -1326,6 +1328,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha1_CiliumNetworkingSpec(in * out.Version = in.Version out.AccessLog = in.AccessLog out.AgentLabels = in.AgentLabels + out.AgentPrometheusPort = in.AgentPrometheusPort out.AllowLocalhost = in.AllowLocalhost out.AutoIpv6NodeRoutes = in.AutoIpv6NodeRoutes out.BPFRoot = in.BPFRoot @@ -1339,6 +1342,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha1_CiliumNetworkingSpec(in * out.DisableK8sServices = in.DisableK8sServices out.EnablePolicy = in.EnablePolicy out.EnableTracing = in.EnableTracing + out.EnablePrometheusMetrics = in.EnablePrometheusMetrics out.EnvoyLog = in.EnvoyLog out.Ipv4ClusterCIDRMaskSize = in.Ipv4ClusterCIDRMaskSize out.Ipv4Node = in.Ipv4Node diff --git a/pkg/apis/kops/v1alpha2/networking.go b/pkg/apis/kops/v1alpha2/networking.go index d176c5c432..57739b11c4 100644 --- a/pkg/apis/kops/v1alpha2/networking.go +++ b/pkg/apis/kops/v1alpha2/networking.go @@ -198,6 +198,7 @@ type CiliumNetworkingSpec struct { AccessLog string `json:"accessLog,omitempty"` AgentLabels []string `json:"agentLabels,omitempty"` + AgentPrometheusPort int `json:"agentPrometheusPort,omitempty"` AllowLocalhost string `json:"allowLocalhost,omitempty"` AutoIpv6NodeRoutes bool `json:"autoIpv6NodeRoutes,omitempty"` BPFRoot string `json:"bpfRoot,omitempty"` @@ -210,6 +211,7 @@ type CiliumNetworkingSpec struct { DisableIpv4 bool `json:"disableIpv4,omitempty"` DisableK8sServices bool `json:"disableK8sServices,omitempty"` EnablePolicy string `json:"enablePolicy,omitempty"` + EnablePrometheusMetrics bool `json:"enablePrometheusMetrics,omitempty"` EnableTracing bool `json:"enableTracing,omitempty"` EnvoyLog string `json:"envoyLog,omitempty"` Ipv4ClusterCIDRMaskSize int `json:"ipv4ClusterCidrMaskSize,omitempty"` diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 3eaa237e3d..aa7ee793d4 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -1292,6 +1292,7 @@ func autoConvert_v1alpha2_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in * out.Version = in.Version out.AccessLog = in.AccessLog out.AgentLabels = in.AgentLabels + out.AgentPrometheusPort = in.AgentPrometheusPort out.AllowLocalhost = in.AllowLocalhost out.AutoIpv6NodeRoutes = in.AutoIpv6NodeRoutes out.BPFRoot = in.BPFRoot @@ -1304,6 +1305,7 @@ func autoConvert_v1alpha2_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in * out.DisableIpv4 = in.DisableIpv4 out.DisableK8sServices = in.DisableK8sServices out.EnablePolicy = in.EnablePolicy + out.EnablePrometheusMetrics = in.EnablePrometheusMetrics out.EnableTracing = in.EnableTracing out.EnvoyLog = in.EnvoyLog out.Ipv4ClusterCIDRMaskSize = in.Ipv4ClusterCIDRMaskSize @@ -1368,6 +1370,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha2_CiliumNetworkingSpec(in * out.Version = in.Version out.AccessLog = in.AccessLog out.AgentLabels = in.AgentLabels + out.AgentPrometheusPort = in.AgentPrometheusPort out.AllowLocalhost = in.AllowLocalhost out.AutoIpv6NodeRoutes = in.AutoIpv6NodeRoutes out.BPFRoot = in.BPFRoot @@ -1381,6 +1384,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha2_CiliumNetworkingSpec(in * out.DisableK8sServices = in.DisableK8sServices out.EnablePolicy = in.EnablePolicy out.EnableTracing = in.EnableTracing + out.EnablePrometheusMetrics = in.EnablePrometheusMetrics out.EnvoyLog = in.EnvoyLog out.Ipv4ClusterCIDRMaskSize = in.Ipv4ClusterCIDRMaskSize out.Ipv4Node = in.Ipv4Node diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template index bba3b264a8..c0d3d3667e 100644 --- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template @@ -20,6 +20,15 @@ data: identity-allocation-mode: crd # If you want to run cilium in debug mode change this value to true debug: "{{- if .Debug -}}true{{- else -}}false{{- end -}}" + {{ if .EnablePrometheusMetrics }} + # If you want metrics enabled in all of your Cilium agents, set the port for + # which the Cilium agents will have their metrics exposed. + # This option deprecates the "prometheus-serve-addr" in the + # "cilium-metrics-config" ConfigMap + # NOTE that this will open the port on ALL nodes where Cilium pods are + # scheduled. + prometheus-serve-addr: ":{{- or .AgentPrometheusPort "9090" }}" + {{ end }} # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 # address. enable-ipv4: "{{- if or (.EnableIpv4) (and (not (.EnableIpv4)) (not (.EnableIpv6))) -}}true{{- else -}}false{{- end -}}" @@ -385,6 +394,13 @@ spec: successThreshold: 1 timeoutSeconds: 5 name: cilium-agent + {{ if .EnablePrometheusMetrics }} + ports: + - containerPort: {{ or .AgentPrometheusPort "9090" }} + hostPort: {{ or .AgentPrometheusPort "9090" }} + name: prometheus + protocol: TCP + {{ end }} readinessProbe: exec: command: @@ -543,6 +559,10 @@ spec: - args: - --debug=$(CILIUM_DEBUG) - --identity-allocation-mode=$(CILIUM_IDENTITY_ALLOCATION_MODE) +{{ with .Networking.Cilium }} + {{ if .EnablePrometheusMetrics }} + - --enable-metrics + {{ end }} command: - cilium-operator env: @@ -622,12 +642,17 @@ spec: key: AWS_DEFAULT_REGION name: cilium-aws optional: true - -{{ with .Networking.Cilium }} image: "docker.io/cilium/operator:{{ .Version }}" -{{ end }} imagePullPolicy: IfNotPresent name: cilium-operator + {{ if .EnablePrometheusMetrics }} + ports: + - containerPort: 6942 + hostPort: 6942 + name: prometheus + protocol: TCP + {{ end }} +{{ end }} livenessProbe: httpGet: path: /healthz diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template index d7cc314a0a..aa8b382961 100644 --- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template @@ -20,6 +20,15 @@ data: identity-allocation-mode: crd # If you want to run cilium in debug mode change this value to true debug: "{{- if .Debug -}}true{{- else -}}false{{- end -}}" + {{ if .EnablePrometheusMetrics }} + # If you want metrics enabled in all of your Cilium agents, set the port for + # which the Cilium agents will have their metrics exposed. + # This option deprecates the "prometheus-serve-addr" in the + # "cilium-metrics-config" ConfigMap + # NOTE that this will open the port on ALL nodes where Cilium pods are + # scheduled. + prometheus-serve-addr: ":{{- or .AgentPrometheusPort "9090" }}" + {{ end }} # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 # address. enable-ipv4: "{{- if or (.EnableIpv4) (and (not (.EnableIpv4)) (not (.EnableIpv6))) -}}true{{- else -}}false{{- end -}}" @@ -385,6 +394,13 @@ spec: successThreshold: 1 timeoutSeconds: 5 name: cilium-agent + {{ if .EnablePrometheusMetrics }} + ports: + - containerPort: {{ or .AgentPrometheusPort "9090" }} + hostPort: {{ or .AgentPrometheusPort "9090" }} + name: prometheus + protocol: TCP + {{ end }} readinessProbe: exec: command: @@ -541,6 +557,10 @@ spec: containers: - args: - --debug=$(CILIUM_DEBUG) +{{ with .Networking.Cilium }} + {{ if .EnablePrometheusMetrics }} + - --enable-metrics + {{ end }} command: - cilium-operator env: @@ -614,12 +634,17 @@ spec: key: AWS_DEFAULT_REGION name: cilium-aws optional: true - -{{ with .Networking.Cilium }} image: "docker.io/cilium/operator:{{ .Version }}" -{{ end }} imagePullPolicy: IfNotPresent name: cilium-operator + {{ if .EnablePrometheusMetrics }} + ports: + - containerPort: 6942 + hostPort: 6942 + name: prometheus + protocol: TCP + {{ end }} +{{ end }} livenessProbe: httpGet: path: /healthz