First operator integration: CoreDNS

Hidden behind a feature-flag, but when the UseAddonOperators feature
flag is set, we now use the cluster-addons CoreDNS operator instead of
our built-in manifests.

channels/operators/coredns.addons.x-k8s.io/0.1.0-kops.1/manifest.yaml

@@ -0,0 +1,266 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.2.5
+  creationTimestamp: null
+  name: coredns.addons.x-k8s.io
+spec:
+  group: addons.x-k8s.io
+  names:
+    kind: CoreDNS
+    listKind: CoreDNSList
+    plural: coredns
+    singular: coredns
+  scope: Namespaced
+  subresources:
+    status: {}
+  validation:
+    openAPIV3Schema:
+      description: CoreDNS is the Schema for the coredns API
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+          type: string
+        metadata:
+          type: object
+        spec:
+          description: CoreDNSSpec defines the desired state of CoreDNS
+          properties:
+            channel:
+              description: 'Channel specifies a channel that can be used to resolve a specific addon, eg: stable It will be ignored if Version is specified'
+              type: string
+            corefile:
+              type: string
+            dnsDomain:
+              type: string
+            dnsIP:
+              type: string
+            patches:
+              items:
+                type: object
+              type: array
+            version:
+              description: Version specifies the exact addon version to be deployed, eg 1.2.3 It should not be specified if Channel is specified
+              type: string
+          type: object
+        status:
+          description: CoreDNSStatus defines the observed state of CoreDNS
+          properties:
+            errors:
+              items:
+                type: string
+              type: array
+            healthy:
+              type: boolean
+          required:
+          - healthy
+          type: object
+      type: object
+  version: v1alpha1
+  versions:
+  - name: v1alpha1
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+
+---
+
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: coredns-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-app: coredns-operator
+  name: coredns-operator
+  namespace: coredns-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: coredns-operator
+rules:
+- apiGroups:
+  - addons.x-k8s.io
+  resources:
+  - coredns
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - addons.x-k8s.io
+  resources:
+  - coredns/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - serviceaccounts
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resourceNames:
+  - coredns
+  resources:
+  - configmaps
+  - serviceaccounts
+  - services
+  verbs:
+  - delete
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - serviceaccounts
+  - services
+  verbs:
+  - create
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - deployments
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - apps
+  - extensions
+  resourceNames:
+  - coredns
+  resources:
+  - deployments
+  verbs:
+  - delete
+  - patch
+  - update
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - deployments
+  verbs:
+  - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    addonmanager.kubernetes.io/mode: EnsureExists
+    kubernetes.io/bootstrapping: rbac-defaults
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: CoreDNS
+  name: system:coredns
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  - services
+  - pods
+  - namespaces
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    k8s-app: coredns-operator
+  name: coredns-system:coredns-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: coredns-operator
+subjects:
+- kind: ServiceAccount
+  name: coredns-operator
+  namespace: coredns-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  labels:
+    addonmanager.kubernetes.io/mode: EnsureExists
+    kubernetes.io/bootstrapping: rbac-defaults
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: CoreDNS
+  name: system:coredns
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:coredns
+subjects:
+- kind: ServiceAccount
+  name: coredns
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    k8s-app: coredns-operator
+  name: coredns-operator
+  namespace: coredns-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      k8s-app: coredns-operator
+  template:
+    metadata:
+      labels:
+        k8s-app: coredns-operator
+    spec:
+      containers:
+      - args:
+        - --enable-leader-election=false
+        - --rbac-mode=ignore
+        command:
+        - /manager
+        image: justinsb/coredns-operator:latest
+        name: manager
+        resources:
+          limits:
+            cpu: 100m
+            memory: 30Mi
+          requests:
+            cpu: 100m
+            memory: 20Mi
+      serviceAccountName: coredns-operator
+      terminationGracePeriodSeconds: 10

channels/src/operators/coredns.addons.x-k8s.io/kustomization.yaml

@@ -0,0 +1,20 @@
+namespace: kube-system
+
+namePrefix: coredns-operator-
+
+# Labels to add to all resources and selectors.
+commonLabels:
+  k8s-app: kube-dns
+
+bases:
+- https://github.com/kubernetes-sigs/cluster-addons/coredns/config/crd/
+- https://github.com/kubernetes-sigs/cluster-addons/coredns/config/rbac/
+- https://github.com/kubernetes-sigs/cluster-addons/coredns/config/manager/
+
+images:
+  - name: controller
+    newName: justinsb/coredns-operator
+    newTag: latest
+
+patches:
+- resources.yaml

channels/src/operators/coredns.addons.x-k8s.io/resources.yaml

@@ -0,0 +1,14 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        resources:
+          limits:
+            cpu: null
+            memory: 100Mi

pkg/apis/kops/channel.go

@@ -86,15 +86,16 @@ type ChannelImageSpec struct {
 	KubernetesVersion string `json:"kubernetesVersion,omitempty"`
 }
 
-// LoadChannel loads a Channel object from the specified VFS location
-func LoadChannel(location string) (*Channel, error) {
+// ResolveChannel maps a channel to an absolute URL (possibly a VFS URL)
+// If the channel is the well-known "none" value, we return (nil, nil)
+func ResolveChannel(location string) (*url.URL, error) {
 	if location == "none" {
-		return &Channel{}, nil
+		return nil, nil
 	}
 
 	u, err := url.Parse(location)
 	if err != nil {
-		return nil, fmt.Errorf("invalid channel: %q", location)
+		return nil, fmt.Errorf("invalid channel location: %q", location)
 	}
 
 	if !u.IsAbs() {
@@ -106,7 +107,22 @@ func LoadChannel(location string) (*Channel, error) {
 		u = base.ResolveReference(u)
 	}
 
-	resolved := u.String()
+	return u, nil
+}
+
+// LoadChannel loads a Channel object from the specified VFS location
+func LoadChannel(location string) (*Channel, error) {
+	resolvedURL, err := ResolveChannel(location)
+	if err != nil {
+		return nil, err
+	}
+
+	if resolvedURL == nil {
+		return &Channel{}, nil
+	}
+
+	resolved := resolvedURL.String()
+
 	klog.V(2).Infof("Loading channel from %q", resolved)
 	channelBytes, err := vfs.Context.ReadFile(resolved)
 	if err != nil {

pkg/featureflag/featureflag.go

@@ -99,6 +99,8 @@ var (
 	KopsControllerStateStore = New("KopsControllerStateStore", Bool(false))
 	// APIServerNodes enables ability to provision nodes that only run the kube-apiserver
 	APIServerNodes = New("APIServerNodes", Bool(false))
+	// UseAddonOperators activates experimental addon operator support
+	UseAddonOperators = New("UseAddonOperators", Bool(false))
 )
 
 // FeatureFlag defines a feature flag

pkg/kubemanifest/manifest.go

@@ -31,6 +31,11 @@ type Object struct {
 	data map[string]interface{}
 }
 
+// NewObject returns an Object wrapping the provided data
+func NewObject(data map[string]interface{}) *Object {
+	return &Object{data: data}
+}
+
 // ObjectList describes a list of objects, allowing us to add bulk-methods
 type ObjectList []*Object
 

pkg/kubemanifest/visitor.go

@@ -48,6 +48,10 @@ type Visitor interface {
 }
 
 func visit(visitor Visitor, data interface{}, path []string, mutator func(interface{})) error {
+	if data == nil {
+		return nil
+	}
+
 	switch data := data.(type) {
 	case string:
 		err := visitor.VisitString(path, data, func(v string) {

pkg/wellknownoperators/BUILD.bazel

@@ -0,0 +1,16 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["operators.go"],
+    importpath = "k8s.io/kops/pkg/wellknownoperators",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//channels/pkg/api:go_default_library",
+        "//pkg/apis/kops:go_default_library",
+        "//pkg/featureflag:go_default_library",
+        "//pkg/kubemanifest:go_default_library",
+        "//upup/pkg/fi:go_default_library",
+        "//util/pkg/vfs:go_default_library",
+    ],
+)

pkg/wellknownoperators/operators.go

@@ -0,0 +1,103 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package wellknownoperators
+
+import (
+	"fmt"
+	"net/url"
+	"path"
+
+	channelsapi "k8s.io/kops/channels/pkg/api"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/featureflag"
+	"k8s.io/kops/pkg/kubemanifest"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/util/pkg/vfs"
+)
+
+type WellKnownAddon struct {
+	Manifest []byte
+	Spec     channelsapi.AddonSpec
+}
+
+type Builder struct {
+	Cluster *kops.Cluster
+}
+
+func (b *Builder) Build() ([]*WellKnownAddon, kubemanifest.ObjectList, error) {
+	if !featureflag.UseAddonOperators.Enabled() {
+		return nil, nil, nil
+	}
+
+	var addons []*WellKnownAddon
+	var crds kubemanifest.ObjectList
+
+	if b.Cluster.Spec.KubeDNS != nil && b.Cluster.Spec.KubeDNS.Provider == "CoreDNS" {
+		// TODO: Check that we haven't manually loaded a CoreDNS operator
+		// TODO: Check that we haven't manually created a CoreDNS CRD
+
+		key := "coredns.addons.x-k8s.io"
+		version := "0.1.0-kops.1"
+		id := ""
+
+		location := path.Join("operators", key, version, "manifest.yaml")
+		channelURL, err := kops.ResolveChannel(b.Cluster.Spec.Channel)
+		if err != nil {
+			return nil, nil, fmt.Errorf("error resolving channel %q: %v", b.Cluster.Spec.Channel, err)
+		}
+
+		locationURL := channelURL.ResolveReference(&url.URL{Path: location}).String()
+
+		manifestBytes, err := vfs.Context.ReadFile(locationURL)
+		if err != nil {
+			return nil, nil, fmt.Errorf("error reading operator manifest %q: %v", locationURL, err)
+		}
+
+		addon := &WellKnownAddon{
+			Manifest: manifestBytes,
+			Spec: channelsapi.AddonSpec{
+				Name:     fi.String(key),
+				Version:  fi.String(version),
+				Selector: map[string]string{"k8s-addon": key},
+				Manifest: fi.String(location),
+				Id:       id,
+			},
+		}
+		addons = append(addons, addon)
+
+		{
+			metadata := map[string]interface{}{
+				"namespace": "kube-system",
+				"name":      "coredns",
+			}
+			spec := map[string]interface{}{
+				"dnsDomain": b.Cluster.Spec.KubeDNS.Domain,
+				"dnsIP":     b.Cluster.Spec.KubeDNS.ServerIP,
+			}
+
+			crd := kubemanifest.NewObject(map[string]interface{}{
+				"apiVersion": "addons.x-k8s.io/v1alpha1",
+				"kind":       "CoreDNS",
+				"metadata":   metadata,
+				"spec":       spec,
+			})
+			crds = append(crds, crd)
+		}
+	}
+
+	return addons, crds, nil
+}

upup/pkg/fi/cloudup/bootstrapchannelbuilder/BUILD.bazel

@@ -21,6 +21,7 @@ go_library(
         "//pkg/model/components/addonmanifests/dnscontroller:go_default_library",
         "//pkg/model/iam:go_default_library",
         "//pkg/templates:go_default_library",
+        "//pkg/wellknownoperators:go_default_library",
         "//upup/pkg/fi:go_default_library",
         "//upup/pkg/fi/fitasks:go_default_library",
         "//upup/pkg/fi/utils:go_default_library",

upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go

@@ -33,6 +33,7 @@ import (
 	"k8s.io/kops/pkg/model/components/addonmanifests/dnscontroller"
 	"k8s.io/kops/pkg/model/iam"
 	"k8s.io/kops/pkg/templates"
+	"k8s.io/kops/pkg/wellknownoperators"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/fitasks"
 	"k8s.io/kops/upup/pkg/fi/utils"
@@ -146,6 +147,57 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error {
 		})
 	}
 
+	if featureflag.UseAddonOperators.Enabled() {
+		ob := &wellknownoperators.Builder{
+			Cluster: b.Cluster,
+		}
+
+		wellKnownAddons, crds, err := ob.Build()
+		if err != nil {
+			return fmt.Errorf("error building well-known operators: %v", err)
+		}
+
+		for _, a := range wellKnownAddons {
+			key := *a.Spec.Name
+			if a.Spec.Id != "" {
+				key = key + "-" + a.Spec.Id
+			}
+			name := b.Cluster.ObjectMeta.Name + "-addons-" + key
+			manifestPath := "addons/" + *a.Spec.Manifest
+
+			// Go through any transforms that are best expressed as code
+			manifestBytes, err := addonmanifests.RemapAddonManifest(&a.Spec, b.KopsModelContext, b.assetBuilder, a.Manifest)
+			if err != nil {
+				klog.Infof("invalid manifest: %s", string(a.Manifest))
+				return fmt.Errorf("error remapping manifest %s: %v", manifestPath, err)
+			}
+
+			// Trim whitespace
+			manifestBytes = []byte(strings.TrimSpace(string(manifestBytes)))
+
+			rawManifest := string(manifestBytes)
+			klog.V(4).Infof("Manifest %v", rawManifest)
+
+			manifestHash, err := utils.HashString(rawManifest)
+			klog.V(4).Infof("hash %s", manifestHash)
+			if err != nil {
+				return fmt.Errorf("error hashing manifest: %v", err)
+			}
+			a.Spec.ManifestHash = manifestHash
+
+			c.AddTask(&fitasks.ManagedFile{
+				Contents:  fi.NewBytesResource(manifestBytes),
+				Lifecycle: b.Lifecycle,
+				Location:  fi.String(manifestPath),
+				Name:      fi.String(name),
+			})
+
+			addons.Spec.Addons = append(addons.Spec.Addons, &a.Spec)
+		}
+
+		b.ClusterAddons = append(b.ClusterAddons, crds...)
+	}
+
 	if b.ClusterAddons != nil {
 		key := "cluster-addons.kops.k8s.io"
 		version := "0.0.0"
@@ -293,7 +345,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 		}
 	}
 
-	if kubeDNS.Provider == "CoreDNS" {
+	if kubeDNS.Provider == "CoreDNS" && !featureflag.UseAddonOperators.Enabled() {
 		{
 			key := "coredns.addons.k8s.io"
 			version := "1.8.3-kops.3"