diff --git a/channels/operators/coredns.addons.x-k8s.io/0.1.0-kops.1/manifest.yaml b/channels/operators/coredns.addons.x-k8s.io/0.1.0-kops.1/manifest.yaml new file mode 100644 index 0000000000..28142c03fd --- /dev/null +++ b/channels/operators/coredns.addons.x-k8s.io/0.1.0-kops.1/manifest.yaml @@ -0,0 +1,266 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.5 + creationTimestamp: null + name: coredns.addons.x-k8s.io +spec: + group: addons.x-k8s.io + names: + kind: CoreDNS + listKind: CoreDNSList + plural: coredns + singular: coredns + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: CoreDNS is the Schema for the coredns API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: CoreDNSSpec defines the desired state of CoreDNS + properties: + channel: + description: 'Channel specifies a channel that can be used to resolve a specific addon, eg: stable It will be ignored if Version is specified' + type: string + corefile: + type: string + dnsDomain: + type: string + dnsIP: + type: string + patches: + items: + type: object + type: array + version: + description: Version specifies the exact addon version to be deployed, eg 1.2.3 It should not be specified if Channel is specified + type: string + type: object + status: + description: CoreDNSStatus defines the observed state of CoreDNS + properties: + errors: + items: + type: string + type: array + healthy: + type: boolean + required: + - healthy + type: object + type: object + version: v1alpha1 + versions: + - name: v1alpha1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- + +apiVersion: v1 +kind: Namespace +metadata: + name: coredns-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: coredns-operator + name: coredns-operator + namespace: coredns-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: coredns-operator +rules: +- apiGroups: + - addons.x-k8s.io + resources: + - coredns + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - addons.x-k8s.io + resources: + - coredns/status + verbs: + - get + - patch + - update +- apiGroups: + - "" + resources: + - configmaps + - serviceaccounts + - services + verbs: + - get + - list + - watch +- apiGroups: + - "" + resourceNames: + - coredns + resources: + - configmaps + - serviceaccounts + - services + verbs: + - delete + - patch + - update +- apiGroups: + - "" + resources: + - configmaps + - serviceaccounts + - services + verbs: + - create +- apiGroups: + - apps + - extensions + resources: + - deployments + verbs: + - get + - list + - watch +- apiGroups: + - apps + - extensions + resourceNames: + - coredns + resources: + - deployments + verbs: + - delete + - patch + - update +- apiGroups: + - apps + - extensions + resources: + - deployments + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + addonmanager.kubernetes.io/mode: EnsureExists + kubernetes.io/bootstrapping: rbac-defaults + kubernetes.io/cluster-service: "true" + kubernetes.io/name: CoreDNS + name: system:coredns +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-app: coredns-operator + name: coredns-system:coredns-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: coredns-operator +subjects: +- kind: ServiceAccount + name: coredns-operator + namespace: coredns-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + addonmanager.kubernetes.io/mode: EnsureExists + kubernetes.io/bootstrapping: rbac-defaults + kubernetes.io/cluster-service: "true" + kubernetes.io/name: CoreDNS + name: system:coredns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns +subjects: +- kind: ServiceAccount + name: coredns + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + k8s-app: coredns-operator + name: coredns-operator + namespace: coredns-system +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: coredns-operator + template: + metadata: + labels: + k8s-app: coredns-operator + spec: + containers: + - args: + - --enable-leader-election=false + - --rbac-mode=ignore + command: + - /manager + image: justinsb/coredns-operator:latest + name: manager + resources: + limits: + cpu: 100m + memory: 30Mi + requests: + cpu: 100m + memory: 20Mi + serviceAccountName: coredns-operator + terminationGracePeriodSeconds: 10 diff --git a/channels/src/operators/coredns.addons.x-k8s.io/kustomization.yaml b/channels/src/operators/coredns.addons.x-k8s.io/kustomization.yaml new file mode 100644 index 0000000000..3d7df95925 --- /dev/null +++ b/channels/src/operators/coredns.addons.x-k8s.io/kustomization.yaml @@ -0,0 +1,20 @@ +namespace: kube-system + +namePrefix: coredns-operator- + +# Labels to add to all resources and selectors. +commonLabels: + k8s-app: kube-dns + +bases: +- https://github.com/kubernetes-sigs/cluster-addons/coredns/config/crd/ +- https://github.com/kubernetes-sigs/cluster-addons/coredns/config/rbac/ +- https://github.com/kubernetes-sigs/cluster-addons/coredns/config/manager/ + +images: + - name: controller + newName: justinsb/coredns-operator + newTag: latest + +patches: +- resources.yaml \ No newline at end of file diff --git a/channels/src/operators/coredns.addons.x-k8s.io/resources.yaml b/channels/src/operators/coredns.addons.x-k8s.io/resources.yaml new file mode 100644 index 0000000000..ae10aca95c --- /dev/null +++ b/channels/src/operators/coredns.addons.x-k8s.io/resources.yaml @@ -0,0 +1,14 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + resources: + limits: + cpu: null + memory: 100Mi diff --git a/pkg/apis/kops/channel.go b/pkg/apis/kops/channel.go index e9fc120de5..9048ce4882 100644 --- a/pkg/apis/kops/channel.go +++ b/pkg/apis/kops/channel.go @@ -86,15 +86,16 @@ type ChannelImageSpec struct { KubernetesVersion string `json:"kubernetesVersion,omitempty"` } -// LoadChannel loads a Channel object from the specified VFS location -func LoadChannel(location string) (*Channel, error) { +// ResolveChannel maps a channel to an absolute URL (possibly a VFS URL) +// If the channel is the well-known "none" value, we return (nil, nil) +func ResolveChannel(location string) (*url.URL, error) { if location == "none" { - return &Channel{}, nil + return nil, nil } u, err := url.Parse(location) if err != nil { - return nil, fmt.Errorf("invalid channel: %q", location) + return nil, fmt.Errorf("invalid channel location: %q", location) } if !u.IsAbs() { @@ -106,7 +107,22 @@ func LoadChannel(location string) (*Channel, error) { u = base.ResolveReference(u) } - resolved := u.String() + return u, nil +} + +// LoadChannel loads a Channel object from the specified VFS location +func LoadChannel(location string) (*Channel, error) { + resolvedURL, err := ResolveChannel(location) + if err != nil { + return nil, err + } + + if resolvedURL == nil { + return &Channel{}, nil + } + + resolved := resolvedURL.String() + klog.V(2).Infof("Loading channel from %q", resolved) channelBytes, err := vfs.Context.ReadFile(resolved) if err != nil { diff --git a/pkg/featureflag/featureflag.go b/pkg/featureflag/featureflag.go index dde82d6cf8..a773adeafc 100644 --- a/pkg/featureflag/featureflag.go +++ b/pkg/featureflag/featureflag.go @@ -99,6 +99,8 @@ var ( KopsControllerStateStore = New("KopsControllerStateStore", Bool(false)) // APIServerNodes enables ability to provision nodes that only run the kube-apiserver APIServerNodes = New("APIServerNodes", Bool(false)) + // UseAddonOperators activates experimental addon operator support + UseAddonOperators = New("UseAddonOperators", Bool(false)) ) // FeatureFlag defines a feature flag diff --git a/pkg/kubemanifest/manifest.go b/pkg/kubemanifest/manifest.go index c6beaa0149..8a5e5ef66f 100644 --- a/pkg/kubemanifest/manifest.go +++ b/pkg/kubemanifest/manifest.go @@ -31,6 +31,11 @@ type Object struct { data map[string]interface{} } +// NewObject returns an Object wrapping the provided data +func NewObject(data map[string]interface{}) *Object { + return &Object{data: data} +} + // ObjectList describes a list of objects, allowing us to add bulk-methods type ObjectList []*Object diff --git a/pkg/kubemanifest/visitor.go b/pkg/kubemanifest/visitor.go index 33745a9581..b674140eea 100644 --- a/pkg/kubemanifest/visitor.go +++ b/pkg/kubemanifest/visitor.go @@ -48,6 +48,10 @@ type Visitor interface { } func visit(visitor Visitor, data interface{}, path []string, mutator func(interface{})) error { + if data == nil { + return nil + } + switch data := data.(type) { case string: err := visitor.VisitString(path, data, func(v string) { diff --git a/pkg/wellknownoperators/BUILD.bazel b/pkg/wellknownoperators/BUILD.bazel new file mode 100644 index 0000000000..874f93e31a --- /dev/null +++ b/pkg/wellknownoperators/BUILD.bazel @@ -0,0 +1,16 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["operators.go"], + importpath = "k8s.io/kops/pkg/wellknownoperators", + visibility = ["//visibility:public"], + deps = [ + "//channels/pkg/api:go_default_library", + "//pkg/apis/kops:go_default_library", + "//pkg/featureflag:go_default_library", + "//pkg/kubemanifest:go_default_library", + "//upup/pkg/fi:go_default_library", + "//util/pkg/vfs:go_default_library", + ], +) diff --git a/pkg/wellknownoperators/operators.go b/pkg/wellknownoperators/operators.go new file mode 100644 index 0000000000..d53129245d --- /dev/null +++ b/pkg/wellknownoperators/operators.go @@ -0,0 +1,103 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wellknownoperators + +import ( + "fmt" + "net/url" + "path" + + channelsapi "k8s.io/kops/channels/pkg/api" + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/featureflag" + "k8s.io/kops/pkg/kubemanifest" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/util/pkg/vfs" +) + +type WellKnownAddon struct { + Manifest []byte + Spec channelsapi.AddonSpec +} + +type Builder struct { + Cluster *kops.Cluster +} + +func (b *Builder) Build() ([]*WellKnownAddon, kubemanifest.ObjectList, error) { + if !featureflag.UseAddonOperators.Enabled() { + return nil, nil, nil + } + + var addons []*WellKnownAddon + var crds kubemanifest.ObjectList + + if b.Cluster.Spec.KubeDNS != nil && b.Cluster.Spec.KubeDNS.Provider == "CoreDNS" { + // TODO: Check that we haven't manually loaded a CoreDNS operator + // TODO: Check that we haven't manually created a CoreDNS CRD + + key := "coredns.addons.x-k8s.io" + version := "0.1.0-kops.1" + id := "" + + location := path.Join("operators", key, version, "manifest.yaml") + channelURL, err := kops.ResolveChannel(b.Cluster.Spec.Channel) + if err != nil { + return nil, nil, fmt.Errorf("error resolving channel %q: %v", b.Cluster.Spec.Channel, err) + } + + locationURL := channelURL.ResolveReference(&url.URL{Path: location}).String() + + manifestBytes, err := vfs.Context.ReadFile(locationURL) + if err != nil { + return nil, nil, fmt.Errorf("error reading operator manifest %q: %v", locationURL, err) + } + + addon := &WellKnownAddon{ + Manifest: manifestBytes, + Spec: channelsapi.AddonSpec{ + Name: fi.String(key), + Version: fi.String(version), + Selector: map[string]string{"k8s-addon": key}, + Manifest: fi.String(location), + Id: id, + }, + } + addons = append(addons, addon) + + { + metadata := map[string]interface{}{ + "namespace": "kube-system", + "name": "coredns", + } + spec := map[string]interface{}{ + "dnsDomain": b.Cluster.Spec.KubeDNS.Domain, + "dnsIP": b.Cluster.Spec.KubeDNS.ServerIP, + } + + crd := kubemanifest.NewObject(map[string]interface{}{ + "apiVersion": "addons.x-k8s.io/v1alpha1", + "kind": "CoreDNS", + "metadata": metadata, + "spec": spec, + }) + crds = append(crds, crd) + } + } + + return addons, crds, nil +} diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/BUILD.bazel b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/BUILD.bazel index 12c06c3336..6548184c65 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/BUILD.bazel +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/BUILD.bazel @@ -21,6 +21,7 @@ go_library( "//pkg/model/components/addonmanifests/dnscontroller:go_default_library", "//pkg/model/iam:go_default_library", "//pkg/templates:go_default_library", + "//pkg/wellknownoperators:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/fitasks:go_default_library", "//upup/pkg/fi/utils:go_default_library", diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go index 06cfbcf296..6c307bb887 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go @@ -33,6 +33,7 @@ import ( "k8s.io/kops/pkg/model/components/addonmanifests/dnscontroller" "k8s.io/kops/pkg/model/iam" "k8s.io/kops/pkg/templates" + "k8s.io/kops/pkg/wellknownoperators" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/fitasks" "k8s.io/kops/upup/pkg/fi/utils" @@ -146,6 +147,57 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error { }) } + if featureflag.UseAddonOperators.Enabled() { + ob := &wellknownoperators.Builder{ + Cluster: b.Cluster, + } + + wellKnownAddons, crds, err := ob.Build() + if err != nil { + return fmt.Errorf("error building well-known operators: %v", err) + } + + for _, a := range wellKnownAddons { + key := *a.Spec.Name + if a.Spec.Id != "" { + key = key + "-" + a.Spec.Id + } + name := b.Cluster.ObjectMeta.Name + "-addons-" + key + manifestPath := "addons/" + *a.Spec.Manifest + + // Go through any transforms that are best expressed as code + manifestBytes, err := addonmanifests.RemapAddonManifest(&a.Spec, b.KopsModelContext, b.assetBuilder, a.Manifest) + if err != nil { + klog.Infof("invalid manifest: %s", string(a.Manifest)) + return fmt.Errorf("error remapping manifest %s: %v", manifestPath, err) + } + + // Trim whitespace + manifestBytes = []byte(strings.TrimSpace(string(manifestBytes))) + + rawManifest := string(manifestBytes) + klog.V(4).Infof("Manifest %v", rawManifest) + + manifestHash, err := utils.HashString(rawManifest) + klog.V(4).Infof("hash %s", manifestHash) + if err != nil { + return fmt.Errorf("error hashing manifest: %v", err) + } + a.Spec.ManifestHash = manifestHash + + c.AddTask(&fitasks.ManagedFile{ + Contents: fi.NewBytesResource(manifestBytes), + Lifecycle: b.Lifecycle, + Location: fi.String(manifestPath), + Name: fi.String(name), + }) + + addons.Spec.Addons = append(addons.Spec.Addons, &a.Spec) + } + + b.ClusterAddons = append(b.ClusterAddons, crds...) + } + if b.ClusterAddons != nil { key := "cluster-addons.kops.k8s.io" version := "0.0.0" @@ -293,7 +345,7 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann } } - if kubeDNS.Provider == "CoreDNS" { + if kubeDNS.Provider == "CoreDNS" && !featureflag.UseAddonOperators.Enabled() { { key := "coredns.addons.k8s.io" version := "1.8.3-kops.3"