868 lines
27 KiB
Go
868 lines
27 KiB
Go
/*
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"reflect"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/go-logr/logr"
|
|
reconcilehelper "github.com/kubeflow/kubeflow/components/common/reconcilehelper"
|
|
"github.com/kubeflow/kubeflow/components/notebook-controller/api/v1beta1"
|
|
"github.com/kubeflow/kubeflow/components/notebook-controller/pkg/metrics"
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
apierrs "k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/apimachinery/pkg/util/intstr"
|
|
"k8s.io/client-go/tools/record"
|
|
ctrl "sigs.k8s.io/controller-runtime"
|
|
"sigs.k8s.io/controller-runtime/pkg/builder"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/event"
|
|
"sigs.k8s.io/controller-runtime/pkg/handler"
|
|
"sigs.k8s.io/controller-runtime/pkg/predicate"
|
|
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
|
"sigs.k8s.io/controller-runtime/pkg/source"
|
|
)
|
|
|
|
const DefaultContainerPort = 8888
|
|
const DefaultServingPort = 80
|
|
const AnnotationRewriteURI = "notebooks.kubeflow.org/http-rewrite-uri"
|
|
const AnnotationHeadersRequestSet = "notebooks.kubeflow.org/http-headers-request-set"
|
|
|
|
const PrefixEnvVar = "NB_PREFIX"
|
|
|
|
// The default fsGroup of PodSecurityContext.
|
|
// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#podsecuritycontext-v1-core
|
|
const DefaultFSGroup = int64(100)
|
|
|
|
/*
|
|
We generally want to ignore (not requeue) NotFound errors, since we'll get a
|
|
reconciliation request once the object exists, and requeuing in the meantime
|
|
won't help.
|
|
*/
|
|
func ignoreNotFound(err error) error {
|
|
if apierrs.IsNotFound(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
|
|
// NotebookReconciler reconciles a Notebook object
|
|
type NotebookReconciler struct {
|
|
client.Client
|
|
Log logr.Logger
|
|
Scheme *runtime.Scheme
|
|
Metrics *metrics.Metrics
|
|
EventRecorder record.EventRecorder
|
|
}
|
|
|
|
// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch
|
|
// +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;patch
|
|
// +kubebuilder:rbac:groups=core,resources=services,verbs="*"
|
|
// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs="*"
|
|
// +kubebuilder:rbac:groups=kubeflow.org,resources=notebooks;notebooks/status;notebooks/finalizers,verbs="*"
|
|
// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices,verbs="*"
|
|
|
|
func (r *NotebookReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
|
log := r.Log.WithValues("notebook", req.NamespacedName)
|
|
log.Info("Reconciliation loop started")
|
|
|
|
// TODO(yanniszark): Can we avoid reconciling Events and Notebook in the same queue?
|
|
event := &corev1.Event{}
|
|
var getEventErr error
|
|
getEventErr = r.Get(ctx, req.NamespacedName, event)
|
|
if getEventErr == nil {
|
|
log.Info("Found event for Notebook. Re-emitting...")
|
|
|
|
// Find the Notebook that corresponds to the triggered event
|
|
involvedNotebook := &v1beta1.Notebook{}
|
|
nbName, err := nbNameFromInvolvedObject(r.Client, &event.InvolvedObject)
|
|
if err != nil {
|
|
return ctrl.Result{}, err
|
|
}
|
|
|
|
involvedNotebookKey := types.NamespacedName{Name: nbName, Namespace: req.Namespace}
|
|
if err := r.Get(ctx, involvedNotebookKey, involvedNotebook); err != nil {
|
|
log.Error(err, "unable to fetch Notebook by looking at event")
|
|
return ctrl.Result{}, ignoreNotFound(err)
|
|
}
|
|
|
|
// re-emit the event in the Notebook CR
|
|
log.Info("Emitting Notebook Event.", "Event", event)
|
|
r.EventRecorder.Eventf(involvedNotebook, event.Type, event.Reason,
|
|
"Reissued from %s/%s: %s", strings.ToLower(event.InvolvedObject.Kind), event.InvolvedObject.Name, event.Message)
|
|
return ctrl.Result{}, nil
|
|
}
|
|
|
|
if getEventErr != nil && !apierrs.IsNotFound(getEventErr) {
|
|
return ctrl.Result{}, getEventErr
|
|
}
|
|
// If not found, continue. Is not an event.
|
|
instance := &v1beta1.Notebook{}
|
|
if err := r.Get(ctx, req.NamespacedName, instance); err != nil {
|
|
log.Error(err, "unable to fetch Notebook")
|
|
return ctrl.Result{}, ignoreNotFound(err)
|
|
}
|
|
|
|
// jupyter-web-app deletes objects using foreground deletion policy, Notebook CR will stay until all owned objects are deleted
|
|
// reconcile loop might keep on trying to recreate the resources that the API server tries to delete.
|
|
// so when Notebook CR is terminating, reconcile loop should do nothing
|
|
|
|
if !instance.DeletionTimestamp.IsZero() {
|
|
return ctrl.Result{}, nil
|
|
}
|
|
|
|
// Reconcile StatefulSet
|
|
ss := generateStatefulSet(instance)
|
|
if err := ctrl.SetControllerReference(instance, ss, r.Scheme); err != nil {
|
|
return ctrl.Result{}, err
|
|
}
|
|
// Check if the StatefulSet already exists
|
|
foundStateful := &appsv1.StatefulSet{}
|
|
justCreated := false
|
|
err := r.Get(ctx, types.NamespacedName{Name: ss.Name, Namespace: ss.Namespace}, foundStateful)
|
|
if err != nil && apierrs.IsNotFound(err) {
|
|
log.Info("Creating StatefulSet", "namespace", ss.Namespace, "name", ss.Name)
|
|
r.Metrics.NotebookCreation.WithLabelValues(ss.Namespace).Inc()
|
|
err = r.Create(ctx, ss)
|
|
justCreated = true
|
|
if err != nil {
|
|
log.Error(err, "unable to create Statefulset")
|
|
r.Metrics.NotebookFailCreation.WithLabelValues(ss.Namespace).Inc()
|
|
return ctrl.Result{}, err
|
|
}
|
|
} else if err != nil {
|
|
log.Error(err, "error getting Statefulset")
|
|
return ctrl.Result{}, err
|
|
}
|
|
// Update the foundStateful object and write the result back if there are any changes
|
|
if !justCreated && reconcilehelper.CopyStatefulSetFields(ss, foundStateful) {
|
|
log.Info("Updating StatefulSet", "namespace", ss.Namespace, "name", ss.Name)
|
|
err = r.Update(ctx, foundStateful)
|
|
if err != nil {
|
|
log.Error(err, "unable to update Statefulset")
|
|
return ctrl.Result{}, err
|
|
}
|
|
}
|
|
|
|
// Reconcile service
|
|
service := generateService(instance)
|
|
if err := ctrl.SetControllerReference(instance, service, r.Scheme); err != nil {
|
|
return ctrl.Result{}, err
|
|
}
|
|
// Check if the Service already exists
|
|
foundService := &corev1.Service{}
|
|
justCreated = false
|
|
err = r.Get(ctx, types.NamespacedName{Name: service.Name, Namespace: service.Namespace}, foundService)
|
|
if err != nil && apierrs.IsNotFound(err) {
|
|
log.Info("Creating Service", "namespace", service.Namespace, "name", service.Name)
|
|
err = r.Create(ctx, service)
|
|
justCreated = true
|
|
if err != nil {
|
|
log.Error(err, "unable to create Service")
|
|
return ctrl.Result{}, err
|
|
}
|
|
} else if err != nil {
|
|
log.Error(err, "error getting Service")
|
|
return ctrl.Result{}, err
|
|
}
|
|
// Update the foundService object and write the result back if there are any changes
|
|
if !justCreated && reconcilehelper.CopyServiceFields(service, foundService) {
|
|
log.Info("Updating Service\n", "namespace", service.Namespace, "name", service.Name)
|
|
err = r.Update(ctx, foundService)
|
|
if err != nil {
|
|
log.Error(err, "unable to update Service")
|
|
return ctrl.Result{}, err
|
|
}
|
|
}
|
|
|
|
// Reconcile virtual service if we use ISTIO.
|
|
if os.Getenv("USE_ISTIO") == "true" {
|
|
err = r.reconcileVirtualService(instance)
|
|
if err != nil {
|
|
return ctrl.Result{}, err
|
|
}
|
|
}
|
|
|
|
foundPod := &corev1.Pod{}
|
|
err = r.Get(ctx, types.NamespacedName{Name: ss.Name + "-0", Namespace: ss.Namespace}, foundPod)
|
|
if err != nil && apierrs.IsNotFound(err) {
|
|
log.Info(fmt.Sprintf("No Pods are currently running for Notebook Server: %s in namespace: %s.", instance.Name, instance.Namespace))
|
|
} else if err != nil {
|
|
return ctrl.Result{}, err
|
|
}
|
|
|
|
// Update Notebook CR status
|
|
err = updateNotebookStatus(r, instance, foundStateful, foundPod, req)
|
|
if err != nil {
|
|
return ctrl.Result{}, err
|
|
}
|
|
|
|
return ctrl.Result{}, nil
|
|
}
|
|
|
|
func updateNotebookStatus(r *NotebookReconciler, nb *v1beta1.Notebook,
|
|
sts *appsv1.StatefulSet, pod *corev1.Pod, req ctrl.Request) error {
|
|
|
|
log := r.Log.WithValues("notebook", req.NamespacedName)
|
|
ctx := context.Background()
|
|
|
|
status, err := createNotebookStatus(r, nb, sts, pod, req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
log.Info("Updating Notebook CR Status", "status", status)
|
|
nb.Status = status
|
|
return r.Status().Update(ctx, nb)
|
|
}
|
|
|
|
func createNotebookStatus(r *NotebookReconciler, nb *v1beta1.Notebook,
|
|
sts *appsv1.StatefulSet, pod *corev1.Pod, req ctrl.Request) (v1beta1.NotebookStatus, error) {
|
|
|
|
log := r.Log.WithValues("notebook", req.NamespacedName)
|
|
|
|
// Initialize Notebook CR Status
|
|
log.Info("Initializing Notebook CR Status")
|
|
status := v1beta1.NotebookStatus{
|
|
Conditions: make([]v1beta1.NotebookCondition, 0),
|
|
ReadyReplicas: sts.Status.ReadyReplicas,
|
|
ContainerState: corev1.ContainerState{},
|
|
}
|
|
|
|
// Update the status based on the Pod's status
|
|
if reflect.DeepEqual(pod.Status, corev1.PodStatus{}) {
|
|
log.Info("No pod.Status found. Won't update notebook conditions and containerState")
|
|
return status, nil
|
|
}
|
|
|
|
// Update status of the CR using the ContainerState of
|
|
// the container that has the same name as the CR.
|
|
// If no container of same name is found, the state of the CR is not updated.
|
|
notebookContainerFound := false
|
|
log.Info("Calculating Notebook's containerState")
|
|
for i := range pod.Status.ContainerStatuses {
|
|
if pod.Status.ContainerStatuses[i].Name != nb.Name {
|
|
continue
|
|
}
|
|
|
|
if pod.Status.ContainerStatuses[i].State == nb.Status.ContainerState {
|
|
continue
|
|
}
|
|
|
|
// Update Notebook CR's status.ContainerState
|
|
cs := pod.Status.ContainerStatuses[i].State
|
|
log.Info("Updating Notebook CR state: ", "state", cs)
|
|
|
|
status.ContainerState = cs
|
|
notebookContainerFound = true
|
|
break
|
|
}
|
|
|
|
if !notebookContainerFound {
|
|
log.Error(nil, "Could not find container with the same name as Notebook "+
|
|
"in containerStates of Pod. Will not update notebook's "+
|
|
"status.containerState ")
|
|
}
|
|
|
|
// Mirroring pod condition
|
|
notebookConditions := []v1beta1.NotebookCondition{}
|
|
log.Info("Calculating Notebook's Conditions")
|
|
for i := range pod.Status.Conditions {
|
|
condition := PodCondToNotebookCond(pod.Status.Conditions[i])
|
|
notebookConditions = append(notebookConditions, condition)
|
|
}
|
|
|
|
status.Conditions = notebookConditions
|
|
|
|
return status, nil
|
|
}
|
|
|
|
func PodCondToNotebookCond(podc corev1.PodCondition) v1beta1.NotebookCondition {
|
|
|
|
condition := v1beta1.NotebookCondition{}
|
|
|
|
if len(podc.Type) > 0 {
|
|
condition.Type = string(podc.Type)
|
|
}
|
|
|
|
if len(podc.Status) > 0 {
|
|
condition.Status = string(podc.Status)
|
|
}
|
|
|
|
if len(podc.Message) > 0 {
|
|
condition.Message = podc.Message
|
|
}
|
|
|
|
if len(podc.Reason) > 0 {
|
|
condition.Reason = podc.Reason
|
|
}
|
|
|
|
// check if podc.LastProbeTime is null. If so initialize
|
|
// the field with metav1.Now()
|
|
check := podc.LastProbeTime.Time.Equal(time.Time{})
|
|
if !check {
|
|
condition.LastProbeTime = podc.LastProbeTime
|
|
} else {
|
|
condition.LastProbeTime = metav1.Now()
|
|
}
|
|
|
|
// check if podc.LastTransitionTime is null. If so initialize
|
|
// the field with metav1.Now()
|
|
check = podc.LastTransitionTime.Time.Equal(time.Time{})
|
|
if !check {
|
|
condition.LastTransitionTime = podc.LastTransitionTime
|
|
} else {
|
|
condition.LastTransitionTime = metav1.Now()
|
|
}
|
|
|
|
return condition
|
|
}
|
|
|
|
func setPrefixEnvVar(instance *v1beta1.Notebook, container *corev1.Container) {
|
|
prefix := "/notebook/" + instance.Namespace + "/" + instance.Name
|
|
|
|
for _, envVar := range container.Env {
|
|
if envVar.Name == PrefixEnvVar {
|
|
envVar.Value = prefix
|
|
return
|
|
}
|
|
}
|
|
|
|
container.Env = append(container.Env, corev1.EnvVar{
|
|
Name: PrefixEnvVar,
|
|
Value: prefix,
|
|
})
|
|
}
|
|
|
|
func generateStatefulSet(instance *v1beta1.Notebook) *appsv1.StatefulSet {
|
|
replicas := int32(1)
|
|
if metav1.HasAnnotation(instance.ObjectMeta, "kubeflow-resource-stopped") {
|
|
replicas = 0
|
|
}
|
|
|
|
ss := &appsv1.StatefulSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: instance.Name,
|
|
Namespace: instance.Namespace,
|
|
},
|
|
Spec: appsv1.StatefulSetSpec{
|
|
Replicas: &replicas,
|
|
Selector: &metav1.LabelSelector{
|
|
MatchLabels: map[string]string{
|
|
"statefulset": instance.Name,
|
|
},
|
|
},
|
|
Template: corev1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{
|
|
"statefulset": instance.Name,
|
|
"notebook-name": instance.Name,
|
|
},
|
|
Annotations: map[string]string{},
|
|
},
|
|
Spec: *instance.Spec.Template.Spec.DeepCopy(),
|
|
},
|
|
},
|
|
}
|
|
|
|
// copy all of the Notebook labels to the pod including poddefault related labels
|
|
l := &ss.Spec.Template.ObjectMeta.Labels
|
|
for k, v := range instance.ObjectMeta.Labels {
|
|
(*l)[k] = v
|
|
}
|
|
|
|
// copy all of the Notebook annotations to the pod.
|
|
a := &ss.Spec.Template.ObjectMeta.Annotations
|
|
for k, v := range instance.ObjectMeta.Annotations {
|
|
if !strings.Contains(k, "kubectl") && !strings.Contains(k, "notebook") {
|
|
(*a)[k] = v
|
|
}
|
|
}
|
|
|
|
podSpec := &ss.Spec.Template.Spec
|
|
container := &podSpec.Containers[0]
|
|
if container.WorkingDir == "" {
|
|
container.WorkingDir = "/home/jovyan"
|
|
}
|
|
if container.Ports == nil {
|
|
container.Ports = []corev1.ContainerPort{
|
|
{
|
|
ContainerPort: DefaultContainerPort,
|
|
Name: "notebook-port",
|
|
Protocol: "TCP",
|
|
},
|
|
}
|
|
}
|
|
|
|
setPrefixEnvVar(instance, container)
|
|
|
|
// For some platforms (like OpenShift), adding fsGroup: 100 is troublesome.
|
|
// This allows for those platforms to bypass the automatic addition of the fsGroup
|
|
// and will allow for the Pod Security Policy controller to make an appropriate choice
|
|
// https://github.com/kubernetes-sigs/controller-runtime/issues/4617
|
|
if value, exists := os.LookupEnv("ADD_FSGROUP"); !exists || value == "true" {
|
|
if podSpec.SecurityContext == nil {
|
|
fsGroup := DefaultFSGroup
|
|
podSpec.SecurityContext = &corev1.PodSecurityContext{
|
|
FSGroup: &fsGroup,
|
|
}
|
|
}
|
|
}
|
|
return ss
|
|
}
|
|
|
|
func generateService(instance *v1beta1.Notebook) *corev1.Service {
|
|
// Define the desired Service object
|
|
port := DefaultContainerPort
|
|
containerPorts := instance.Spec.Template.Spec.Containers[0].Ports
|
|
if containerPorts != nil {
|
|
port = int(containerPorts[0].ContainerPort)
|
|
}
|
|
svc := &corev1.Service{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: instance.Name,
|
|
Namespace: instance.Namespace,
|
|
},
|
|
Spec: corev1.ServiceSpec{
|
|
Type: "ClusterIP",
|
|
Selector: map[string]string{"statefulset": instance.Name},
|
|
Ports: []corev1.ServicePort{
|
|
{
|
|
// Make port name follow Istio pattern so it can be managed by istio rbac
|
|
Name: "http-" + instance.Name,
|
|
Port: DefaultServingPort,
|
|
TargetPort: intstr.FromInt(port),
|
|
Protocol: "TCP",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
return svc
|
|
}
|
|
|
|
func virtualServiceName(kfName string, namespace string) string {
|
|
return fmt.Sprintf("notebook-%s-%s", namespace, kfName)
|
|
}
|
|
|
|
func virtualServiceNameWithSuffix(kfName string, namespace string, suffix string) string {
|
|
return fmt.Sprintf("notebook-%s-%s-%s", namespace, kfName, suffix)
|
|
}
|
|
|
|
func generateVirtualServiceForNotebookRedirect(name string, namespace string, prefix string, istioHostNotebook string, istioGateway string) (*unstructured.Unstructured, error) {
|
|
vsvc := &unstructured.Unstructured{}
|
|
vsvc.SetAPIVersion("networking.istio.io/v1alpha3")
|
|
vsvc.SetKind("VirtualService")
|
|
vsvc.SetName(virtualServiceNameWithSuffix(name, namespace, "notebook-redirect"))
|
|
vsvc.SetNamespace(namespace)
|
|
|
|
istioHost := os.Getenv("ISTIO_HOST")
|
|
if len(istioHost) == 0 {
|
|
istioHost = "*"
|
|
}
|
|
if err := unstructured.SetNestedStringSlice(vsvc.Object, []string{istioHost}, "spec", "hosts"); err != nil {
|
|
return nil, fmt.Errorf("Set .spec.hosts error: %v", err)
|
|
|
|
}
|
|
|
|
if err := unstructured.SetNestedStringSlice(vsvc.Object, []string{istioGateway},
|
|
"spec", "gateways"); err != nil {
|
|
return nil, fmt.Errorf("set .spec.gateways error: %v", err)
|
|
}
|
|
|
|
// the http section of the istio VirtualService spec
|
|
http := []interface{}{
|
|
map[string]interface{}{
|
|
"match": []interface{}{
|
|
map[string]interface{}{
|
|
"uri": map[string]interface{}{
|
|
"prefix": prefix,
|
|
},
|
|
},
|
|
},
|
|
"redirect": map[string]interface{}{
|
|
"authority": istioHostNotebook,
|
|
"redirectCode": int64(307),
|
|
},
|
|
},
|
|
}
|
|
|
|
// add http section to istio VirtualService spec
|
|
if err := unstructured.SetNestedSlice(vsvc.Object, http, "spec", "http"); err != nil {
|
|
return nil, fmt.Errorf("set .spec.http error: %v", err)
|
|
}
|
|
|
|
return vsvc, nil
|
|
}
|
|
|
|
func generateVirtualServiceForAuthRedirect(name string, namespace string, prefix string, istioHostNotebook string, istioGateway string) (*unstructured.Unstructured, error) {
|
|
vsvc := &unstructured.Unstructured{}
|
|
vsvc.SetAPIVersion("networking.istio.io/v1alpha3")
|
|
vsvc.SetKind("VirtualService")
|
|
vsvc.SetName(virtualServiceNameWithSuffix(name, namespace, "auth-redirect"))
|
|
vsvc.SetNamespace(namespace)
|
|
|
|
if err := unstructured.SetNestedStringSlice(vsvc.Object, []string{istioHostNotebook}, "spec", "hosts"); err != nil {
|
|
return nil, fmt.Errorf("set .spec.hosts error: %v", err)
|
|
}
|
|
|
|
if err := unstructured.SetNestedStringSlice(vsvc.Object, []string{istioGateway},
|
|
"spec", "gateways"); err != nil {
|
|
return nil, fmt.Errorf("set .spec.gateways error: %v", err)
|
|
}
|
|
|
|
istioHostAuth := os.Getenv("ISTIO_HOST_AUTH")
|
|
if len(istioHostAuth) == 0 {
|
|
return nil, fmt.Errorf("invalid ISTIO_HOST_AUTH: When ISTIO_USE_NOTEBOOK_SUBDOMAINS is set, the ISTIO_HOST_AUTH environment variable must be defined.")
|
|
}
|
|
|
|
istioAuthPath := os.Getenv("ISTIO_AUTH_PATH")
|
|
if len(istioAuthPath) == 0 {
|
|
istioAuthPath = "/oauth2/"
|
|
}
|
|
|
|
// the http section of the istio VirtualService spec
|
|
http := []interface{}{
|
|
map[string]interface{}{
|
|
"match": []interface{}{
|
|
map[string]interface{}{
|
|
"uri": map[string]interface{}{
|
|
"prefix": istioAuthPath,
|
|
},
|
|
},
|
|
},
|
|
"redirect": map[string]interface{}{
|
|
"authority": istioHostAuth,
|
|
"redirectCode": int64(307),
|
|
},
|
|
},
|
|
}
|
|
|
|
// add http section to istio VirtualService spec
|
|
if err := unstructured.SetNestedSlice(vsvc.Object, http, "spec", "http"); err != nil {
|
|
return nil, fmt.Errorf("set .spec.http error: %v", err)
|
|
}
|
|
|
|
return vsvc, nil
|
|
}
|
|
|
|
func generateVirtualServices(instance *v1beta1.Notebook) ([]*unstructured.Unstructured, error) {
|
|
name := instance.Name
|
|
namespace := instance.Namespace
|
|
clusterDomain := "cluster.local"
|
|
prefix := fmt.Sprintf("/notebook/%s/%s/", namespace, name)
|
|
|
|
// unpack annotations from Notebook resource
|
|
annotations := make(map[string]string)
|
|
for k, v := range instance.ObjectMeta.Annotations {
|
|
annotations[k] = v
|
|
}
|
|
|
|
rewrite := fmt.Sprintf("/notebook/%s/%s/", namespace, name)
|
|
// If AnnotationRewriteURI is present, use this value for "rewrite"
|
|
if _, ok := annotations[AnnotationRewriteURI]; ok && len(annotations[AnnotationRewriteURI]) > 0 {
|
|
rewrite = annotations[AnnotationRewriteURI]
|
|
}
|
|
|
|
if clusterDomainFromEnv, ok := os.LookupEnv("CLUSTER_DOMAIN"); ok {
|
|
clusterDomain = clusterDomainFromEnv
|
|
}
|
|
service := fmt.Sprintf("%s.%s.svc.%s", name, namespace, clusterDomain)
|
|
|
|
vsvc := &unstructured.Unstructured{}
|
|
vsvcs := []*unstructured.Unstructured{vsvc}
|
|
|
|
vsvc.SetAPIVersion("networking.istio.io/v1alpha3")
|
|
vsvc.SetKind("VirtualService")
|
|
vsvc.SetName(virtualServiceName(name, namespace))
|
|
vsvc.SetNamespace(namespace)
|
|
|
|
istioGateway := os.Getenv("ISTIO_GATEWAY")
|
|
if len(istioGateway) == 0 {
|
|
istioGateway = "kubeflow/kubeflow-gateway"
|
|
}
|
|
if err := unstructured.SetNestedStringSlice(vsvc.Object, []string{istioGateway},
|
|
"spec", "gateways"); err != nil {
|
|
return nil, fmt.Errorf("set .spec.gateways error: %v", err)
|
|
}
|
|
|
|
istioHost := os.Getenv("ISTIO_HOST")
|
|
if os.Getenv("ISTIO_USE_NOTEBOOK_SUBDOMAINS") == "true" {
|
|
istioHost = os.Getenv("ISTIO_HOST_NOTEBOOK")
|
|
if !strings.Contains(istioHost, "${NAMESPACE}") {
|
|
return nil, errors.New("invalid ISTIO_HOST_NOTEBOOK: When ISTIO_USE_NOTEBOOK_SUBDOMAINS is set, the placeholder ${NAMESPACE} must be defined in ISTIO_HOST_NOTEBOOK.")
|
|
}
|
|
istioHost = strings.ReplaceAll(istioHost, "${NAMESPACE}", namespace)
|
|
|
|
vsvcForNotebookRedirect, err := generateVirtualServiceForNotebookRedirect(name, namespace, prefix, istioHost, istioGateway)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("generate virtual service for notebook redirect error: %v", err)
|
|
}
|
|
|
|
vsvcForAuthRedirect, err := generateVirtualServiceForAuthRedirect(name, namespace, prefix, istioHost, istioGateway)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("generate virtual service for auth redirect error: %v", err)
|
|
}
|
|
|
|
vsvcs = append(vsvcs, vsvcForAuthRedirect, vsvcForNotebookRedirect)
|
|
} else if len(istioHost) == 0 {
|
|
istioHost = "*"
|
|
}
|
|
if err := unstructured.SetNestedStringSlice(vsvc.Object, []string{istioHost}, "spec", "hosts"); err != nil {
|
|
return nil, fmt.Errorf("Set .spec.hosts error: %v", err)
|
|
|
|
}
|
|
|
|
headersRequestSet := make(map[string]string)
|
|
// If AnnotationHeadersRequestSet is present, use its values in "headers.request.set"
|
|
if _, ok := annotations[AnnotationHeadersRequestSet]; ok && len(annotations[AnnotationHeadersRequestSet]) > 0 {
|
|
requestHeadersBytes := []byte(annotations[AnnotationHeadersRequestSet])
|
|
if err := json.Unmarshal(requestHeadersBytes, &headersRequestSet); err != nil {
|
|
// if JSON decoding fails, set an empty map
|
|
headersRequestSet = make(map[string]string)
|
|
}
|
|
}
|
|
// cast from map[string]string, as SetNestedSlice needs map[string]interface{}
|
|
headersRequestSetInterface := make(map[string]interface{})
|
|
for key, element := range headersRequestSet {
|
|
headersRequestSetInterface[key] = element
|
|
}
|
|
|
|
// the http section of the istio VirtualService spec
|
|
http := []interface{}{
|
|
map[string]interface{}{
|
|
"headers": map[string]interface{}{
|
|
"request": map[string]interface{}{
|
|
"set": headersRequestSetInterface,
|
|
"remove": []interface{}{"Cookie"},
|
|
},
|
|
},
|
|
"match": []interface{}{
|
|
map[string]interface{}{
|
|
"uri": map[string]interface{}{
|
|
"prefix": prefix,
|
|
},
|
|
},
|
|
},
|
|
"rewrite": map[string]interface{}{
|
|
"uri": rewrite,
|
|
},
|
|
"route": []interface{}{
|
|
map[string]interface{}{
|
|
"destination": map[string]interface{}{
|
|
"host": service,
|
|
"port": map[string]interface{}{
|
|
"number": int64(DefaultServingPort),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
// add http section to istio VirtualService spec
|
|
if err := unstructured.SetNestedSlice(vsvc.Object, http, "spec", "http"); err != nil {
|
|
return nil, fmt.Errorf("set .spec.http error: %v", err)
|
|
}
|
|
|
|
return vsvcs, nil
|
|
|
|
}
|
|
|
|
func (r *NotebookReconciler) reconcileVirtualService(instance *v1beta1.Notebook) error {
|
|
log := r.Log.WithValues("notebook", instance.Namespace)
|
|
virtualServices, err := generateVirtualServices(instance)
|
|
if err != nil {
|
|
log.Info("Unable to generate VirtualService...", err)
|
|
return err
|
|
}
|
|
|
|
for _, virtualService := range virtualServices {
|
|
|
|
if err := ctrl.SetControllerReference(instance, virtualService, r.Scheme); err != nil {
|
|
return err
|
|
}
|
|
// Check if the virtual service already exists.
|
|
foundVirtual := &unstructured.Unstructured{}
|
|
justCreated := false
|
|
foundVirtual.SetAPIVersion("networking.istio.io/v1alpha3")
|
|
foundVirtual.SetKind("VirtualService")
|
|
virtualServiceName := virtualService.GetName()
|
|
virtualServiceNamespace := virtualService.GetNamespace()
|
|
err = r.Get(context.TODO(), types.NamespacedName{Name: virtualServiceName, Namespace: virtualServiceNamespace}, foundVirtual)
|
|
if err != nil && apierrs.IsNotFound(err) {
|
|
log.Info("Creating virtual service", "namespace", virtualServiceNamespace, "name",
|
|
virtualServiceName)
|
|
err = r.Create(context.TODO(), virtualService)
|
|
justCreated = true
|
|
if err != nil {
|
|
return err
|
|
}
|
|
} else if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !justCreated && reconcilehelper.CopyVirtualService(virtualService, foundVirtual) {
|
|
log.Info("Updating virtual service", "namespace", virtualServiceNamespace, "name",
|
|
virtualServiceName)
|
|
err = r.Update(context.TODO(), foundVirtual)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func isStsOrPodEvent(event *corev1.Event) bool {
|
|
return event.InvolvedObject.Kind == "Pod" || event.InvolvedObject.Kind == "StatefulSet"
|
|
}
|
|
|
|
func nbNameFromInvolvedObject(c client.Client, object *corev1.ObjectReference) (string, error) {
|
|
name, namespace := object.Name, object.Namespace
|
|
|
|
if object.Kind == "StatefulSet" {
|
|
return name, nil
|
|
}
|
|
if object.Kind == "Pod" {
|
|
pod := &corev1.Pod{}
|
|
err := c.Get(
|
|
context.TODO(),
|
|
types.NamespacedName{
|
|
Namespace: namespace,
|
|
Name: name,
|
|
},
|
|
pod,
|
|
)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if nbName, ok := pod.Labels["notebook-name"]; ok {
|
|
return nbName, nil
|
|
}
|
|
}
|
|
return "", fmt.Errorf("object isn't related to a Notebook")
|
|
}
|
|
|
|
func nbNameExists(client client.Client, nbName string, namespace string) bool {
|
|
if err := client.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: nbName}, &v1beta1.Notebook{}); err != nil {
|
|
// If error != NotFound, trigger the reconcile call anyway to avoid loosing a potential relevant event
|
|
return !apierrs.IsNotFound(err)
|
|
}
|
|
return true
|
|
}
|
|
|
|
// predNBPodIsLabeled filters pods not containing the "notebook-name" label key
|
|
func predNBPodIsLabeled() predicate.Funcs {
|
|
// Documented at
|
|
// https://github.com/kubernetes-sigs/controller-runtime/blob/ce8bdd3d81ab410ff23255e9ad3554f613c5183c/pkg/predicate/predicate_test.go#L884
|
|
checkNBLabel := func() func(object client.Object) bool {
|
|
return func(object client.Object) bool {
|
|
_, labelExists := object.GetLabels()["notebook-name"]
|
|
return labelExists
|
|
}
|
|
}
|
|
|
|
return predicate.NewPredicateFuncs(checkNBLabel())
|
|
}
|
|
|
|
// predNBEvents filters events not coming from Pod or STS, and coming from
|
|
// unknown NBs
|
|
func predNBEvents(r *NotebookReconciler) predicate.Funcs {
|
|
checkEvent := func() func(object client.Object) bool {
|
|
return func(object client.Object) bool {
|
|
event := object.(*corev1.Event)
|
|
nbName, err := nbNameFromInvolvedObject(r.Client, &event.InvolvedObject)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
return isStsOrPodEvent(event) && nbNameExists(r.Client, nbName, object.GetNamespace())
|
|
}
|
|
}
|
|
|
|
predicates := predicate.NewPredicateFuncs(checkEvent())
|
|
|
|
// Do not reconcile when an event gets deleted
|
|
predicates.DeleteFunc = func(e event.DeleteEvent) bool {
|
|
return false
|
|
}
|
|
|
|
return predicates
|
|
}
|
|
|
|
// SetupWithManager sets up the controller with the Manager.
|
|
func (r *NotebookReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
|
|
|
// Map function to convert pod events to reconciliation requests
|
|
mapPodToRequest := func(object client.Object) []reconcile.Request {
|
|
return []reconcile.Request{
|
|
{NamespacedName: types.NamespacedName{
|
|
Name: object.GetLabels()["notebook-name"],
|
|
Namespace: object.GetNamespace(),
|
|
}},
|
|
}
|
|
}
|
|
|
|
// Map function to convert namespace events to reconciliation requests
|
|
mapEventToRequest := func(object client.Object) []reconcile.Request {
|
|
return []reconcile.Request{
|
|
{NamespacedName: types.NamespacedName{
|
|
Name: object.GetName(),
|
|
Namespace: object.GetNamespace(),
|
|
}},
|
|
}
|
|
}
|
|
|
|
builder := ctrl.NewControllerManagedBy(mgr).
|
|
For(&v1beta1.Notebook{}).
|
|
Owns(&appsv1.StatefulSet{}).
|
|
Owns(&corev1.Service{}).
|
|
Watches(
|
|
&source.Kind{Type: &corev1.Pod{}},
|
|
handler.EnqueueRequestsFromMapFunc(mapPodToRequest),
|
|
builder.WithPredicates(predNBPodIsLabeled())).
|
|
Watches(
|
|
&source.Kind{Type: &corev1.Event{}},
|
|
handler.EnqueueRequestsFromMapFunc(mapEventToRequest),
|
|
builder.WithPredicates(predNBEvents(r)))
|
|
// watch Istio virtual service
|
|
if os.Getenv("USE_ISTIO") == "true" {
|
|
virtualService := &unstructured.Unstructured{}
|
|
virtualService.SetAPIVersion("networking.istio.io/v1alpha3")
|
|
virtualService.SetKind("VirtualService")
|
|
builder.Owns(virtualService)
|
|
}
|
|
|
|
err := builder.Complete(r)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|