342 lines
10 KiB
Go
342 lines
10 KiB
Go
/*
|
|
Copyright 2021 The Karmada Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package karmada
|
|
|
|
import (
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
|
|
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/client-go/kubernetes"
|
|
"k8s.io/client-go/rest"
|
|
"k8s.io/klog/v2"
|
|
apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
|
|
"sigs.k8s.io/yaml"
|
|
|
|
clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
|
|
bootstrapagent "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/bootstraptoken/agent"
|
|
"github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/bootstraptoken/clusterinfo"
|
|
"github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/options"
|
|
"github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils"
|
|
"github.com/karmada-io/karmada/pkg/karmadactl/util"
|
|
"github.com/karmada-io/karmada/pkg/karmadactl/util/apiclient"
|
|
tokenutil "github.com/karmada-io/karmada/pkg/karmadactl/util/bootstraptoken"
|
|
)
|
|
|
|
const (
|
|
clusterProxyAdminRole = "cluster-proxy-admin"
|
|
clusterProxyAdminUser = "system:admin"
|
|
|
|
aggregatedApiserverServiceName = "karmada-aggregated-apiserver"
|
|
)
|
|
|
|
// InitKarmadaResources Initialize karmada resource
|
|
func InitKarmadaResources(dir, caBase64, systemNamespace string) error {
|
|
restConfig, err := apiclient.RestConfig("", filepath.Join(dir, options.KarmadaKubeConfigName))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
clientSet, err := apiclient.NewClientSet(restConfig)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// create namespace
|
|
if err := util.CreateOrUpdateNamespace(clientSet, util.NewNamespace(systemNamespace)); err != nil {
|
|
klog.Exitln(err)
|
|
}
|
|
|
|
// New CRDsClient
|
|
crdClient, err := apiclient.NewCRDsClient(restConfig)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Initialize crd
|
|
basesCRDsFiles := utils.ListFiles(dir + "/crds/bases")
|
|
klog.Infof("Initialize karmada bases crd resource `%s/crds/bases`", dir)
|
|
for _, v := range basesCRDsFiles {
|
|
if path.Ext(v) != ".yaml" {
|
|
continue
|
|
}
|
|
if err := createCRDs(crdClient, v); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
patchesCRDsFiles := utils.ListFiles(dir + "/crds/patches")
|
|
klog.Infof("Initialize karmada patches crd resource `%s/crds/patches`", dir)
|
|
for _, v := range patchesCRDsFiles {
|
|
if path.Ext(v) != ".yaml" {
|
|
continue
|
|
}
|
|
if err := patchCRDs(crdClient, caBase64, v); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// create webhook configuration
|
|
// https://github.com/karmada-io/karmada/blob/master/artifacts/deploy/webhook-configuration.yaml
|
|
klog.Info("Create MutatingWebhookConfiguration mutating-config.")
|
|
if err = createOrUpdateMutatingWebhookConfiguration(clientSet, mutatingConfig(caBase64, systemNamespace)); err != nil {
|
|
klog.Exitln(err)
|
|
}
|
|
|
|
klog.Info("Create ValidatingWebhookConfiguration validating-config.")
|
|
if err = createOrUpdateValidatingWebhookConfiguration(clientSet, validatingConfig(caBase64, systemNamespace)); err != nil {
|
|
klog.Exitln(err)
|
|
}
|
|
|
|
// karmada-aggregated-apiserver
|
|
klog.Info("Create Service 'karmada-aggregated-apiserver' and APIService 'v1alpha1.cluster.karmada.io'.")
|
|
if err = initAggregatedAPIService(clientSet, restConfig, systemNamespace, caBase64); err != nil {
|
|
klog.Exitln(err)
|
|
}
|
|
|
|
if err = createExtraResources(clientSet, dir); err != nil {
|
|
klog.Exitln(err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// InitKarmadaBootstrapToken create initial bootstrap token
|
|
func InitKarmadaBootstrapToken(dir string) (string, error) {
|
|
restConfig, err := apiclient.RestConfig("", filepath.Join(dir, options.KarmadaKubeConfigName))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
clientSet, err := apiclient.NewClientSet(restConfig)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
// Create initial bootstrap token
|
|
klog.Info("Initialize karmada bootstrap token")
|
|
bootstrapToken, err := tokenutil.GenerateRandomBootstrapToken(&metav1.Duration{Duration: tokenutil.DefaultTokenDuration}, "", tokenutil.DefaultGroups, tokenutil.DefaultUsages)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if err := tokenutil.CreateNewToken(clientSet, bootstrapToken); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
tokenStr := bootstrapToken.Token.ID + "." + bootstrapToken.Token.Secret
|
|
|
|
registerCommand, err := tokenutil.GenerateRegisterCommand(filepath.Join(dir, options.KarmadaKubeConfigName), "", tokenStr, "")
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to get register command, err: %w", err)
|
|
}
|
|
|
|
return registerCommand, nil
|
|
}
|
|
|
|
func createExtraResources(clientSet *kubernetes.Clientset, dir string) error {
|
|
// grant view clusterrole with karmada resource permission
|
|
if err := grantKarmadaPermissionToViewClusterRole(clientSet); err != nil {
|
|
return err
|
|
}
|
|
|
|
// grant edit clusterrole with karmada resource permission
|
|
if err := grantKarmadaPermissionToEditClusterRole(clientSet); err != nil {
|
|
return err
|
|
}
|
|
|
|
// grant proxy permission to "system:admin".
|
|
if err := grantProxyPermissionToAdmin(clientSet); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Create the cluster-info ConfigMap with the associated RBAC rules
|
|
if err := clusterinfo.CreateBootstrapConfigMapIfNotExists(clientSet, filepath.Join(dir, options.KarmadaKubeConfigName)); err != nil {
|
|
return fmt.Errorf("error creating bootstrap ConfigMap: %v", err)
|
|
}
|
|
|
|
if err := clusterinfo.CreateClusterInfoRBACRules(clientSet); err != nil {
|
|
return fmt.Errorf("error creating clusterinfo RBAC rules: %v", err)
|
|
}
|
|
|
|
// grant limited access permission to 'karmada-agent'
|
|
if err := grantAccessPermissionToAgent(clientSet); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := bootstrapagent.AllowBootstrapTokensToPostCSRs(clientSet); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := bootstrapagent.AutoApproveKarmadaAgentBootstrapTokens(clientSet); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := bootstrapagent.AutoApproveAgentCertificateRotation(clientSet); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func crdPatchesResources(filename, caBundle string) ([]byte, error) {
|
|
data, err := os.ReadFile(filename)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
re, err := regexp.Compile("{{caBundle}}")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
repl := re.ReplaceAllString(string(data), caBundle)
|
|
|
|
return []byte(repl), nil
|
|
}
|
|
|
|
// createCRDs create crd resource
|
|
func createCRDs(crdClient *clientset.Clientset, filename string) error {
|
|
obj := apiextensionsv1.CustomResourceDefinition{}
|
|
data, err := os.ReadFile(filename)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
jsonByte, err := yaml.YAMLToJSON(data)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := json.Unmarshal(jsonByte, &obj); err != nil {
|
|
klog.Errorln("Error convert json byte to apiExtensionsV1 CustomResourceDefinition struct.")
|
|
return err
|
|
}
|
|
|
|
klog.Infof("Attempting to create CRD")
|
|
crd := crdClient.ApiextensionsV1().CustomResourceDefinitions()
|
|
if _, err := crd.Create(context.TODO(), &obj, metav1.CreateOptions{}); err != nil {
|
|
if !apierrors.IsAlreadyExists(err) {
|
|
return err
|
|
}
|
|
klog.Infof("CRD %s already exists, skipping.", obj.Name)
|
|
return nil
|
|
}
|
|
|
|
klog.Infof("Create CRD %s successfully.", obj.Name)
|
|
return nil
|
|
}
|
|
|
|
// patchCRDs patch crd resource
|
|
func patchCRDs(crdClient *clientset.Clientset, caBundle, filename string) error {
|
|
data, err := crdPatchesResources(filename, caBundle)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
jsonByte, err := yaml.YAMLToJSON(data)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
name := getName(path.Base(filename), "_", ".") + ".work.karmada.io"
|
|
crd := crdClient.ApiextensionsV1().CustomResourceDefinitions()
|
|
if _, err := crd.Patch(context.TODO(), name, types.StrategicMergePatchType, jsonByte, metav1.PatchOptions{}); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func getName(str, start, end string) string {
|
|
n := strings.LastIndex(str, start) + 1
|
|
str = string([]byte(str)[n:])
|
|
m := strings.Index(str, end)
|
|
str = string([]byte(str)[:m])
|
|
return str
|
|
}
|
|
|
|
func initAggregatedAPIService(clientSet *kubernetes.Clientset, restConfig *rest.Config, systemNamespace, caBase64 string) error {
|
|
// https://github.com/karmada-io/karmada/blob/master/artifacts/deploy/karmada-aggregated-apiserver-apiservice.yaml
|
|
caBytes, err := base64.StdEncoding.DecodeString(caBase64)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to decode caBase64: %+v", err)
|
|
}
|
|
aaService := &corev1.Service{
|
|
TypeMeta: metav1.TypeMeta{
|
|
APIVersion: "v1",
|
|
Kind: "Service",
|
|
},
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: aggregatedApiserverServiceName,
|
|
Namespace: systemNamespace,
|
|
},
|
|
Spec: corev1.ServiceSpec{
|
|
Type: corev1.ServiceTypeExternalName,
|
|
ExternalName: fmt.Sprintf("%s.%s.svc", aggregatedApiserverServiceName, systemNamespace),
|
|
},
|
|
}
|
|
if err := util.CreateOrUpdateService(clientSet, aaService); err != nil {
|
|
return err
|
|
}
|
|
|
|
// new apiRegistrationClient
|
|
apiRegistrationClient, err := apiclient.NewAPIRegistrationClient(restConfig)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
aaAPIServiceObjName := "v1alpha1.cluster.karmada.io"
|
|
aaAPIService := &apiregistrationv1.APIService{
|
|
TypeMeta: metav1.TypeMeta{
|
|
APIVersion: "v1",
|
|
Kind: "APIService",
|
|
},
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: aaAPIServiceObjName,
|
|
Labels: map[string]string{"app": "karmada-aggregated-apiserver", "apiserver": "true"},
|
|
},
|
|
Spec: apiregistrationv1.APIServiceSpec{
|
|
CABundle: caBytes,
|
|
Group: clusterv1alpha1.GroupName,
|
|
GroupPriorityMinimum: 2000,
|
|
Service: &apiregistrationv1.ServiceReference{
|
|
Name: aaService.Name,
|
|
Namespace: aaService.Namespace,
|
|
},
|
|
Version: clusterv1alpha1.GroupVersion.Version,
|
|
VersionPriority: 10,
|
|
},
|
|
}
|
|
|
|
if err = util.CreateOrUpdateAPIService(apiRegistrationClient, aaAPIService); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := WaitAPIServiceReady(apiRegistrationClient, aaAPIServiceObjName, 120*time.Second); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|