Jaeger injector mutating webhook (#5276)

* Jaeger injector mutating webhook

Closes #5231. This is based off of the `alex/sep-tracing` branch.

This webhook injects the `LINKERD2_PROXY_TRACE_COLLECTOR_SVC_ADDR`,
`LINKERD2_PROXY_TRACE_COLLECTOR_SVC_NAME` and
`LINKERD2_PROXY_TRACE_ATTRIBUTES_PATH` environment vars into the proxy
spec when a pod is created, as well as the podinfo volume and its mount.
If any of these are found to be present already in the pod spec, it
exits without applying a patch.

The `values.yaml` file has been expanded to include config for this
webhook. In particular, one can define a `namespaceSelector` and/or a
`objectSelector` to filter which pods will this webhook act on.

The config entries in `values.yam` for `collectorSvcAddr` and
`collectorSvcAccount` can be overriden with the
`config.linkerd.io/trace-collector` and
`config.alpha.linkerd.io/trace-collector-service-account` annotation at
the namespace or pod spec level.

## How to test:
```bash
docker build . -t ghcr.io/linkerd/jaeger-webhook:0.0.1 -f
jaeger/proxy-mutator/Dockerfile
k3d image import ghcr.io/linkerd/jaeger-webhook:0.0.1
bin/helm-build
linkerd install
helm install jaeger jaeger/charts/jaeger
linkerd inject https://run.linkerd.io/emojivoto.yml | kubectl apply -f -
kubectl -n emojivoto get po -l app=emoji-svc -oyaml | grep -A1 TRACE
```

## Reinvocation policy
The webhookconfig resource is configured with `reinvocationPolicy:
IfNeeded` so that if the tracing injector gets triggered before the
proxy injector, it will get triggered a second time after the proxy
injector runs so it can act on the injected proxy. By default this won't
be necessary because the webhooks run in alphabetical order (this is not
documented in k8s docs though) so
`linkerd-proxy-injector-webhook-config` will run before
`linkerd-proxy-mutator-webhook-config`. In order to test the
reinvocation mechanism, you can change the name of the former so it gets
called first.

I versioned the webhook image as `0.0.1`, but we can decide to align
that with linkerd's main version tag.
This commit is contained in:
Alejandro Pedraza 2020-11-27 12:25:28 -05:00 committed by GitHub
parent 62e208b99f
commit 6fb35b0af7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 508 additions and 4 deletions

View File

@ -20,3 +20,4 @@ else
"$bindir"/build-cli-bin
fi
"$bindir"/docker-build-grafana
"$bindir"/docker-build-jaeger-webhook

21
bin/docker-build-jaeger-webhook Executable file
View File

@ -0,0 +1,21 @@
#!/usr/bin/env bash
set -eu
if [ $# -ne 0 ]; then
echo "no arguments allowed for ${0##*/}, given: $*" >&2
exit 64
fi
bindir=$( cd "${BASH_SOURCE[0]%/*}" && pwd )
rootdir=$( cd "$bindir"/.. && pwd )
# shellcheck source=_docker.sh
. "$bindir"/_docker.sh
# shellcheck source=_tag.sh
. "$bindir"/_tag.sh
dockerfile=$rootdir/jaeger/proxy-mutator/Dockerfile
tag=$(head_root_tag)
docker_build jaeger-webhook "$tag" "$dockerfile" --build-arg LINKERD_VERSION="$tag"

View File

@ -6,6 +6,7 @@ setValues() {
sed -i "s/$1/$2/" charts/linkerd2/values.yaml
sed -i "s/$1/$2/" charts/linkerd2-cni/values.yaml
sed -i "s/$1/$2/" charts/linkerd2-multicluster/values.yaml
sed -i "s/$1/$2/" jaeger/charts/jaeger/values.yaml
}
showErr() {

View File

@ -79,7 +79,8 @@ fi
"$bin" version
rm -f load_fail
for img in proxy controller web grafana cli-bin debug cni-plugin ; do
for img in proxy controller web grafana cli-bin debug cni-plugin jaeger-webhook; do
printf 'Importing %s...\n' $img
if [ $archive ]; then
param="image-archives/$img.tar"
else

3
go.sum
View File

@ -77,8 +77,10 @@ github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
@ -975,6 +977,7 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA=
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

View File

@ -0,0 +1,75 @@
---
###
### Proxy Mutator
###
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app.kubernetes.io/name: proxy-mutator
app.kubernetes.io/part-of: Linkerd
component: proxy-mutator
name: proxy-mutator
namespace: {{.Values.namespace}}
spec:
replicas: 1
selector:
matchLabels:
component: proxy-mutator
template:
metadata:
annotations:
checksum/config: {{ include (print $.Template.BasePath "/rbac.yaml") . | sha256sum }}
labels:
component: proxy-mutator
spec:
containers:
- args:
- -collector-svc-addr={{.Values.collectorSvcAddr}}
- -collector-svc-account={{.Values.collectorSvcAccount}}
- -log-level={{.Values.webhook.logLevel}}
image: {{.Values.webhook.image.name}}:{{default .Values.webhook.image.version .Values.cliVersion}}
imagePullPolicy: {{.Values.webhook.image.pullPolicy}}
livenessProbe:
httpGet:
path: /ping
port: 9995
initialDelaySeconds: 10
name: proxy-mutator
ports:
- containerPort: 8443
name: proxy-mutator
- containerPort: 9995
name: admin-http
readinessProbe:
failureThreshold: 7
httpGet:
path: /ready
port: 9995
volumeMounts:
- mountPath: /var/run/linkerd/tls
name: tls
readOnly: true
serviceAccountName: proxy-mutator
volumes:
- name: tls
secret:
secretName: proxy-mutator-k8s-tls
---
kind: Service
apiVersion: v1
metadata:
name: proxy-mutator
namespace: {{.Values.namespace}}
labels:
component: proxy-mutator
spec:
type: ClusterIP
selector:
component: proxy-mutator
ports:
- name: proxy-mutator
port: 443
targetPort: proxy-mutator

View File

@ -18,3 +18,84 @@ apiVersion: v1
metadata:
name: jaeger
namespace: {{.Values.namespace}}
---
###
### Proxy Mutator RBAC
###
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: linkerd-jaeger-{{.Values.namespace}}-proxy-mutator
rules:
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["get", "list"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: linkerd-jaeger-{{.Values.namespace}}-proxy-mutator
subjects:
- kind: ServiceAccount
name: proxy-mutator
namespace: {{.Values.namespace}}
apiGroup: ""
roleRef:
kind: ClusterRole
name: linkerd-jaeger-{{.Values.namespace}}-proxy-mutator
apiGroup: rbac.authorization.k8s.io
---
kind: ServiceAccount
apiVersion: v1
metadata:
name: proxy-mutator
namespace: {{.Values.namespace}}
---
{{- $host := printf "proxy-mutator.%s.svc" .Values.namespace }}
{{- $ca := genSelfSignedCert $host (list) (list $host) 365 }}
{{- if (not .Values.webhook.externalSecret) }}
kind: Secret
apiVersion: v1
metadata:
name: proxy-mutator-k8s-tls
namespace: {{ .Values.namespace }}
type: kubernetes.io/tls
data:
tls.crt: {{ ternary (b64enc (trim $ca.Cert)) (b64enc (trim .Values.webhook.crtPEM)) (empty .Values.webhook.crtPEM) }}
tls.key: {{ ternary (b64enc (trim $ca.Key)) (b64enc (trim .Values.webhook.keyPEM)) (empty .Values.webhook.keyPEM) }}
---
{{- end }}
apiVersion: admissionregistration.k8s.io/v1beta1
kind: MutatingWebhookConfiguration
metadata:
name: linkerd-proxy-mutator-webhook-config
webhooks:
- name: proxy-mutator.linkerd.io
{{- if .Values.webhook.namespaceSelector }}
namespaceSelector:
{{ toYaml .Values.webhook.namespaceSelector | trim | indent 4 -}}
{{- end }}
{{- if .Values.webhook.objectSelector }}
objectSelector:
{{ toYaml .Values.webhook.objectSelector | trim | indent 4 -}}
{{- end }}
clientConfig:
service:
name: proxy-mutator
namespace: {{ .Values.namespace }}
path: "/"
{{- if and (.Values.webhook.externalSecret) (empty .Values.webhook.caBundle) }}
{{- fail "If webhook.externalSecret is true then you need to provide webook.caBundle" }}
{{- end }}
caBundle: {{ ternary (b64enc (trim $ca.Cert)) (b64enc (trim .Values.webhook.caBundle)) (empty .Values.webhook.caBundle) }}
failurePolicy: {{.Values.webhook.failurePolicy}}
reinvocationPolicy: IfNeeded
rules:
- operations: [ "CREATE" ]
apiGroups: [""]
apiVersions: ["v1"]
resources: ["pods"]
sideEffects: None

View File

@ -8,8 +8,42 @@ collector:
version: 0.1.11
pullPolicy: Always
# resources:
collectorSvcAddr: collector.linkerd-jaeger:55678
collectorSvcAccount: collector
jaeger:
image:
name: jaegertracing/all-in-one
version: 1.19.2
pullPolicy: Always
linkerdVersion: &linkerd_version linkerdVersionValue
webhook:
externalSecret: false
# if empty, Helm will auto-generate these fields
crtPEM: |
keyPEM: |
# if empty, Helm will auto-generate this field, unless externalSecret is set to true.
caBundle: |
failurePolicy: Ignore
image:
name: ghcr.io/linkerd/jaeger-webhook
version: *linkerd_version
pullPolicy: IfNotPresent
logLevel: info
namespaceSelector:
#matchExpressions:
#- key: runlevel
# operator: NotIn
# values: ["0","1"]
objectSelector:
#matchLabels:
# foo: bar

View File

@ -22,6 +22,7 @@ import (
var (
templatesJaeger = []string{
"templates/namespace.yaml",
"templates/proxy-mutator.yaml",
"templates/rbac.yaml",
"templates/tracing.yaml",
}

View File

@ -0,0 +1,32 @@
ARG BUILDPLATFORM=linux/amd64
# Precompile key slow-to-build dependencies
FROM --platform=$BUILDPLATFORM golang:1.14.2-alpine as go-deps
WORKDIR /linkerd-build
COPY go.mod go.sum ./
COPY bin/install-deps bin/
RUN go mod download
ARG TARGETARCH
RUN ./bin/install-deps $TARGETARCH
## compile controller service
FROM go-deps as golang
WORKDIR /linkerd-build
COPY jaeger jaeger
COPY controller/gen controller/gen
COPY pkg pkg
COPY controller controller
COPY charts/partials charts/partials
ARG TARGETARCH
RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH go build -o /out/proxy-mutator -tags prod -mod=readonly -ldflags "-s -w" ./jaeger/proxy-mutator/cmd
## package runtime
FROM scratch
ENV PATH=$PATH:/go/bin
COPY --from=golang /out/proxy-mutator /go/bin/proxy-mutator
ARG LINKERD_VERSION
ENV LINKERD_CONTAINER_VERSION_OVERRIDE=${LINKERD_VERSION}
ENTRYPOINT ["/go/bin/proxy-mutator"]

View File

@ -0,0 +1,37 @@
package main
import (
"context"
"flag"
"fmt"
"os"
"github.com/linkerd/linkerd2/controller/k8s"
"github.com/linkerd/linkerd2/controller/webhook"
"github.com/linkerd/linkerd2/jaeger/proxy-mutator/mutator"
"github.com/linkerd/linkerd2/pkg/flags"
)
func main() {
cmd := flag.NewFlagSet("proxy-mutator", flag.ExitOnError)
metricsAddr := cmd.String("metrics-addr", fmt.Sprintf(":%d", 9995),
"address to serve scrapable metrics on")
addr := cmd.String("addr", ":8443", "address to serve on")
kubeconfig := cmd.String("kubeconfig", "", "path to kubeconfig")
collectorSvcAddr := cmd.String("collector-svc-addr", "",
"collector service address for the proxies to send trace data")
collectorSvcAccount := cmd.String("collector-svc-account", "",
"service account associated with the collector instance")
flags.ConfigureAndParse(cmd, os.Args[1:])
webhook.Launch(
context.Background(),
[]k8s.APIResource{k8s.NS},
mutator.Mutate(*collectorSvcAddr, *collectorSvcAccount),
"linkerd-proxy-mutator",
*metricsAddr,
*addr,
*kubeconfig,
)
}

View File

@ -0,0 +1,53 @@
package mutator
const tpl = `[
{
"op": "add",
"path": "/spec/containers/{{.ProxyIndex}}/env/-",
"value": {
"name": "LINKERD2_PROXY_TRACE_ATTRIBUTES_PATH",
"value": "/var/run/linkerd/podinfo/labels"
}
},
{
"op": "add",
"path": "/spec/containers/{{.ProxyIndex}}/env/-",
"value": {
"name": "LINKERD2_PROXY_TRACE_COLLECTOR_SVC_ADDR",
"value": "{{.CollectorSvcAddr}}"
}
},
{
"op": "add",
"path": "/spec/containers/{{.ProxyIndex}}/env/-",
"value": {
"name": "LINKERD2_PROXY_TRACE_COLLECTOR_SVC_NAME",
"value": "{{.CollectorSvcAccount}}.serviceaccount.identity.$(_l5d_ns).$(_l5d_trustdomain)"
}
},
{
"op": "add",
"path": "/spec/containers/{{.ProxyIndex}}/volumeMounts/-",
"value": {
"mountPath": "var/run/linkerd/podinfo",
"name": "podinfo"
}
},
{
"op": "add",
"path": "/spec/volumes/-",
"value": {
"downwardAPI": {
"items": [
{
"fieldRef": {
"fieldPath": "metadata.labels"
},
"path": "labels"
}
]
},
"name": "podinfo"
}
}
]`

View File

@ -0,0 +1,144 @@
package mutator
import (
"bytes"
"context"
"fmt"
"html/template"
"strings"
"github.com/linkerd/linkerd2/controller/k8s"
"github.com/linkerd/linkerd2/controller/webhook"
labels "github.com/linkerd/linkerd2/pkg/k8s"
"github.com/prometheus/common/log"
admissionv1beta1 "k8s.io/api/admission/v1beta1"
corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/yaml"
)
const (
collectorSvcAddrAnnotation = labels.ProxyConfigAnnotationsPrefix + "/trace-collector"
collectorSvcAccountAnnotation = labels.ProxyConfigAnnotationsPrefixAlpha +
"/trace-collector-service-account"
)
// Params holds the values used in the patch template
type Params struct {
ProxyIndex int
CollectorSvcAddr string
CollectorSvcAccount string
}
// Mutate returns an AdmissionResponse containing the patch, if any, to apply
// to the proxy
func Mutate(collectorSvcAddr, collectorSvcAccount string) webhook.Handler {
return func(
ctx context.Context,
api *k8s.API,
request *admissionv1beta1.AdmissionRequest,
recorder record.EventRecorder,
) (*admissionv1beta1.AdmissionResponse, error) {
log.Debugf("request object bytes: %s", request.Object.Raw)
admissionResponse := &admissionv1beta1.AdmissionResponse{
UID: request.UID,
Allowed: true,
}
if collectorSvcAddr == "" {
return admissionResponse, nil
}
var pod *corev1.Pod
if err := yaml.Unmarshal(request.Object.Raw, &pod); err != nil {
return nil, err
}
params := Params{
ProxyIndex: getProxyContainerIndex(pod.Spec.Containers),
CollectorSvcAddr: collectorSvcAddr,
CollectorSvcAccount: collectorSvcAccount,
}
if params.ProxyIndex < 0 || alreadyMutated(pod, params.ProxyIndex) {
return admissionResponse, nil
}
namespace, err := api.NS().Lister().Get(request.Namespace)
if err != nil {
return nil, err
}
applyOverrides(namespace, pod, &params)
ammendSvcAccount(pod.Namespace, &params)
t, err := template.New("tpl").Parse(tpl)
if err != nil {
return nil, err
}
var patchJSON bytes.Buffer
if err = t.Execute(&patchJSON, params); err != nil {
return nil, err
}
patchType := admissionv1beta1.PatchTypeJSONPatch
admissionResponse.Patch = patchJSON.Bytes()
admissionResponse.PatchType = &patchType
return admissionResponse, nil
}
}
func getProxyContainerIndex(containers []corev1.Container) int {
for i, c := range containers {
if c.Name == labels.ProxyContainerName {
return i
}
}
return -1
}
func alreadyMutated(pod *corev1.Pod, proxyIndex int) bool {
for _, v := range pod.Spec.Volumes {
if v.DownwardAPI != nil && v.Name == "podinfo" {
return true
}
}
for _, mount := range pod.Spec.Containers[proxyIndex].VolumeMounts {
if mount.Name == "podinfo" && mount.MountPath == "var/run/linkerd/podinfo" {
return true
}
}
for _, env := range pod.Spec.Containers[proxyIndex].Env {
if env.Name == "LINKERD2_PROXY_TRACE_ATTRIBUTES_PATH" ||
env.Name == "LINKERD2_PROXY_TRACE_COLLECTOR_SVC_ADDR" ||
env.Name == "LINKERD2_PROXY_TRACE_COLLECTOR_SVC_NAME" {
return true
}
}
return false
}
func applyOverrides(ns *corev1.Namespace, pod *corev1.Pod, params *Params) {
ann := ns.GetAnnotations()
if ann == nil {
ann = map[string]string{}
}
for k, v := range pod.Annotations {
ann[k] = v
}
if override, ok := ann[collectorSvcAddrAnnotation]; ok {
params.CollectorSvcAddr = override
}
if override, ok := ann[collectorSvcAccountAnnotation]; ok {
params.CollectorSvcAccount = override
}
}
func ammendSvcAccount(ns string, params *Params) {
hostAndPort := strings.Split(params.CollectorSvcAddr, ":")
hostname := strings.Split(hostAndPort[0], ".")
if len(hostname) > 1 {
ns = hostname[1]
}
params.CollectorSvcAccount = fmt.Sprintf("%s.%s", params.CollectorSvcAccount, ns)
}

View File

@ -6,16 +6,23 @@ import (
"github.com/linkerd/linkerd2/jaeger/static"
"github.com/linkerd/linkerd2/pkg/charts"
l5dcharts "github.com/linkerd/linkerd2/pkg/charts/linkerd2"
"github.com/linkerd/linkerd2/pkg/version"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/chartutil"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/yaml"
)
// Values represents the values of jaeger template
type Values struct {
Namespace string `json:"namespace"`
Collector collector `json:"collector"`
Jaeger jaeger `json:"jaeger"`
Namespace string `json:"namespace"`
CliVersion string `json:"cliVersion"`
Collector collector `json:"collector"`
CollectorSvcAddr string `json:"collectorSvcAddr"`
CollectorSvcAccount string `json:"collectorSvcAccount"`
Jaeger jaeger `json:"jaeger"`
LinkerdVersion string `json:"linkerdVersion"`
Webhook webhook `json:"webhook"`
}
type collector struct {
@ -28,6 +35,18 @@ type jaeger struct {
Image l5dcharts.Image `json:"image"`
}
type webhook struct {
ExternalSecret bool `json:"externalSecret"`
CrtPEM string `json:"crtPEM"`
KeyPEM string `json:"keyPEM"`
CaBundle string `json:"caBundle"`
FailurePolicy string `json:"failurePolicy"`
Image l5dcharts.Image `json:"image"`
LogLevel string `json:"logLevel"`
NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector"`
ObjectSelector *metav1.LabelSelector `json:"objectSelector"`
}
// NewValues returns a new instance of the Values type.
// TODO: Add HA logic
func NewValues() (*Values, error) {
@ -36,6 +55,7 @@ func NewValues() (*Values, error) {
if err != nil {
return nil, err
}
v.CliVersion = version.Version
return v, nil
}