Initial strucutre

Signed-off-by: Joe Nathan Abellard <contact@jabellard.com> Update crds Signed-off-by: Joe Nathan Abellard <contact@jabellard.com> Remove debug logs Signed-off-by: Joe Nathan Abellard <contact@jabellard.com> Remove debug logs Signed-off-by: Joe Nathan Abellard <contact@jabellard.com> Remove debug logs Signed-off-by: Joe Nathan Abellard <contact@jabellard.com> Remove debug logs Signed-off-by: Joe Nathan Abellard <contact@jabellard.com> Fix linting errors Signed-off-by: Joe Nathan Abellard <contact@jabellard.com> Address comments Signed-off-by: Joe Nathan Abellard <contact@jabellard.com> Update api Signed-off-by: Joe Nathan Abellard <contact@jabellard.com> Update api Signed-off-by: Joe Nathan Abellard <contact@jabellard.com> Address comments Signed-off-by: Joe Nathan Abellard <contact@jabellard.com> Address comments Signed-off-by: Joe Nathan Abellard <contact@jabellard.com> Address comments Signed-off-by: Joe Nathan Abellard <contact@jabellard.com>
2024-07-12 14:57:27 -04:00 · 2024-07-12 14:57:27 -04:00 · 1ae5bd1924
parent 7f0961f06c
commit 1ae5bd1924
8 changed files with 221 additions and 40 deletions
--- a/charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml
+++ b/charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml
@ -1730,6 +1730,34 @@ spec:
                        type: object
                    type: object
                type: object
+              crdTarball:
+                description: |-
+                  CRDTarball specifies the source from which the Karmada CRD tarball should be downloaded, along with the download policy to use.
+                  If not set, the operator will download the tarball from a GitHub release.
+                  By default, it will download the tarball of the same version as the operator itself.
+                  For instance, if the operator's version is v1.10.0, the tarball will be downloaded from the following location:
+                  https://github.com/karmada-io/karmada/releases/download/v1.10.0/crds.tar.gz
+                  By default, the operator will only attempt to download the tarball if it's not yet present in the local cache.
+                properties:
+                  crdDownloadPolicy:
+                    default: IfNotPresent
+                    description: |-
+                      CRDDownloadPolicy specifies a policy that should be used to download the CRD tarball.
+                      Valid values are "Always" and "IfNotPresent".
+                      Defaults to "IfNotPresent".
+                    enum:
+                    - Always
+                    - IfNotPresent
+                    type: string
+                  httpSource:
+                    description: HTTPSource specifies how to download the CRD tarball
+                      via either HTTP or HTTPS protocol.
+                    properties:
+                      url:
+                        description: URL specifies the URL of the CRD tarball resource.
+                        type: string
+                    type: object
+                type: object
              featureGates:
                additionalProperties:
                  type: boolean
--- a/operator/config/crds/operator.karmada.io_karmadas.yaml
+++ b/operator/config/crds/operator.karmada.io_karmadas.yaml
@ -1730,6 +1730,34 @@ spec:
                        type: object
                    type: object
                type: object
+              crdTarball:
+                description: |-
+                  CRDTarball specifies the source from which the Karmada CRD tarball should be downloaded, along with the download policy to use.
+                  If not set, the operator will download the tarball from a GitHub release.
+                  By default, it will download the tarball of the same version as the operator itself.
+                  For instance, if the operator's version is v1.10.0, the tarball will be downloaded from the following location:
+                  https://github.com/karmada-io/karmada/releases/download/v1.10.0/crds.tar.gz
+                  By default, the operator will only attempt to download the tarball if it's not yet present in the local cache.
+                properties:
+                  crdDownloadPolicy:
+                    default: IfNotPresent
+                    description: |-
+                      CRDDownloadPolicy specifies a policy that should be used to download the CRD tarball.
+                      Valid values are "Always" and "IfNotPresent".
+                      Defaults to "IfNotPresent".
+                    enum:
+                    - Always
+                    - IfNotPresent
+                    type: string
+                  httpSource:
+                    description: HTTPSource specifies how to download the CRD tarball
+                      via either HTTP or HTTPS protocol.
+                    properties:
+                      url:
+                        description: URL specifies the URL of the CRD tarball resource.
+                        type: string
+                    type: object
+                type: object
              featureGates:
                additionalProperties:
                  type: boolean
--- a/operator/pkg/apis/operator/v1alpha1/type.go
+++ b/operator/pkg/apis/operator/v1alpha1/type.go
@ -44,6 +44,38 @@ type Karmada struct {
 	Status KarmadaStatus `json:"status,omitempty"`
 }

+// CRDDownloadPolicy specifies a policy for how the operator will download the Karmada CRD tarball
+type CRDDownloadPolicy string
+
+const (
+	// DownloadAlways instructs the Karmada operator to always download the CRD tarball from a remote location.
+	DownloadAlways CRDDownloadPolicy = "Always"
+
+	// DownloadIfNotPresent instructs the Karmada operator to download the CRDs tarball from a remote location only if it is not yet present in the local cache.
+	DownloadIfNotPresent CRDDownloadPolicy = "IfNotPresent"
+)
+
+// HTTPSource specifies how to download the CRD tarball via either HTTP or HTTPS protocol.
+type HTTPSource struct {
+	// URL specifies the URL of the CRD tarball resource.
+	URL string `json:"url,omitempty"`
+}
+
+// CRDTarball specifies the source from which the Karmada CRD tarball should be downloaded, along with the download policy to use.
+type CRDTarball struct {
+	// HTTPSource specifies how to download the CRD tarball via either HTTP or HTTPS protocol.
+	// +optional
+	HTTPSource *HTTPSource `json:"httpSource,omitempty"`
+
+	// CRDDownloadPolicy specifies a policy that should be used to download the CRD tarball.
+	// Valid values are "Always" and "IfNotPresent".
+	// Defaults to "IfNotPresent".
+	// +kubebuilder:validation:Enum=Always;IfNotPresent
+	// +kubebuilder:default=IfNotPresent
+	// +optional
+	CRDDownloadPolicy *CRDDownloadPolicy `json:"crdDownloadPolicy,omitempty"`
+}
+
 // KarmadaSpec is the specification of the desired behavior of the Karmada.
 type KarmadaSpec struct {
 	// HostCluster represents the cluster where to install the Karmada control plane.
@ -72,6 +104,15 @@ type KarmadaSpec struct {
 	// More info: https://github.com/karmada-io/karmada/blob/master/pkg/features/features.go
 	// +optional
 	FeatureGates map[string]bool `json:"featureGates,omitempty"`
+
+	// CRDTarball specifies the source from which the Karmada CRD tarball should be downloaded, along with the download policy to use.
+	// If not set, the operator will download the tarball from a GitHub release.
+	// By default, it will download the tarball of the same version as the operator itself.
+	// For instance, if the operator's version is v1.10.0, the tarball will be downloaded from the following location:
+	// https://github.com/karmada-io/karmada/releases/download/v1.10.0/crds.tar.gz
+	// By default, the operator will only attempt to download the tarball if it's not yet present in the local cache.
+	// +optional
+	CRDTarball *CRDTarball `json:"crdTarball,omitempty"`
 }

 // ImageRegistry represents an image registry as well as the
--- a/operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go
+++ b/operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go
@ -27,6 +27,32 @@ import (
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )

+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CRDTarball) DeepCopyInto(out *CRDTarball) {
+	*out = *in
+	if in.HTTPSource != nil {
+		in, out := &in.HTTPSource, &out.HTTPSource
+		*out = new(HTTPSource)
+		**out = **in
+	}
+	if in.CRDDownloadPolicy != nil {
+		in, out := &in.CRDDownloadPolicy, &out.CRDDownloadPolicy
+		*out = new(CRDDownloadPolicy)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CRDTarball.
+func (in *CRDTarball) DeepCopy() *CRDTarball {
+	if in == nil {
+		return nil
+	}
+	out := new(CRDTarball)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *CommonSettings) DeepCopyInto(out *CommonSettings) {
 	*out = *in
@ -126,6 +152,22 @@ func (in *ExternalEtcd) DeepCopy() *ExternalEtcd {
 	return out
 }

+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *HTTPSource) DeepCopyInto(out *HTTPSource) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPSource.
+func (in *HTTPSource) DeepCopy() *HTTPSource {
+	if in == nil {
+		return nil
+	}
+	out := new(HTTPSource)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *HostCluster) DeepCopyInto(out *HostCluster) {
 	*out = *in
@ -559,6 +601,11 @@ func (in *KarmadaSpec) DeepCopyInto(out *KarmadaSpec) {
 			(*out)[key] = val
 		}
 	}
+	if in.CRDTarball != nil {
+		in, out := &in.CRDTarball, &out.CRDTarball
+		*out = new(CRDTarball)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }

--- a/operator/pkg/init.go
+++ b/operator/pkg/init.go
@ -48,7 +48,7 @@ type InitOptions struct {
 	Namespace      string
 	Kubeconfig     *rest.Config
 	KarmadaVersion string
-	CRDRemoteURL   string
+	CRDTarball     operatorv1alpha1.CRDTarball
 	KarmadaDataDir string
 	Karmada        *operatorv1alpha1.Karmada
 }
@ -60,9 +60,9 @@ func (opt *InitOptions) Validate() error {
 	if len(opt.Name) == 0 || len(opt.Namespace) == 0 {
 		return errors.New("unexpected empty name or namespace")
 	}
-	if len(opt.CRDRemoteURL) > 0 {
-		if _, err := url.Parse(opt.CRDRemoteURL); err != nil {
-			return fmt.Errorf("unexpected invalid crds remote url %s", opt.CRDRemoteURL)
+	if opt.CRDTarball.HTTPSource != nil {
+		if _, err := url.Parse(opt.CRDTarball.HTTPSource.URL); err != nil {
+			return fmt.Errorf("unexpected invalid crds remote url %s", opt.CRDTarball.HTTPSource.URL)
 		}
 	}
 	if !util.IsInCluster(opt.Karmada.Spec.HostCluster) && opt.Karmada.Spec.Components.KarmadaAPIServer.ServiceType == corev1.ServiceTypeClusterIP {
@ -102,7 +102,7 @@ type initData struct {
 	remoteClient        clientset.Interface
 	karmadaClient       clientset.Interface
 	dnsDomain           string
-	CRDRemoteURL        string
+	CRDTarball          operatorv1alpha1.CRDTarball
 	karmadaDataDir      string
 	privateRegistry     string
 	featureGates        map[string]bool
@ -183,7 +183,7 @@ func newRunData(opt *InitOptions) (*initData, error) {
 		karmadaVersion:      version,
 		controlplaneAddress: address,
 		remoteClient:        remoteClient,
-		CRDRemoteURL:        opt.CRDRemoteURL,
+		CRDTarball:          opt.CRDTarball,
 		karmadaDataDir:      opt.KarmadaDataDir,
 		privateRegistry:     privateRegistry,
 		components:          opt.Karmada.Spec.Components,
@ -235,8 +235,8 @@ func (data *initData) DataDir() string {
 	return data.karmadaDataDir
 }

-func (data *initData) CrdsRemoteURL() string {
-	return data.CRDRemoteURL
+func (data *initData) CrdTarball() operatorv1alpha1.CRDTarball {
+	return data.CRDTarball
 }

 func (data *initData) KarmadaVersion() string {
@ -268,8 +268,14 @@ func defaultJobInitOptions() *InitOptions {
 	// set defaults for karmada.
 	operatorscheme.Scheme.Default(karmada)

+	defaultDownloadPolicy := operatorv1alpha1.DownloadIfNotPresent
 	return &InitOptions{
-		CRDRemoteURL:   fmt.Sprintf(defaultCrdURL, operatorv1alpha1.DefaultKarmadaImageVersion),
+		CRDTarball: operatorv1alpha1.CRDTarball{
+			CRDDownloadPolicy: &defaultDownloadPolicy,
+			HTTPSource: &operatorv1alpha1.HTTPSource{
+				URL: fmt.Sprintf(defaultCrdURL, operatorv1alpha1.DefaultKarmadaImageVersion),
+			},
+		},
 		KarmadaVersion: operatorv1alpha1.DefaultKarmadaImageVersion,
 		KarmadaDataDir: constants.KarmadaDataDir,
 		Karmada:        karmada,
@ -282,6 +288,9 @@ func NewInitOptWithKarmada(karmada *operatorv1alpha1.Karmada) InitOpt {
 		o.Karmada = karmada
 		o.Name = karmada.GetName()
 		o.Namespace = karmada.GetNamespace()
+		if karmada.Spec.CRDTarball != nil {
+			o.CRDTarball = *karmada.Spec.CRDTarball
+		}
 	}
 }

--- a/operator/pkg/tasks/init/crd.go
+++ b/operator/pkg/tasks/init/crd.go
@ -17,6 +17,8 @@ limitations under the License.
 package tasks

 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"os"
@ -25,6 +27,7 @@ import (

 	"k8s.io/klog/v2"

+	operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
 	"github.com/karmada-io/karmada/operator/pkg/util"
 	"github.com/karmada-io/karmada/operator/pkg/workflow"
 )
@ -60,7 +63,11 @@ func runPrepareCrds(r workflow.RunData) error {
 		return errors.New("prepare-crds task invoked with an invalid data struct")
 	}

-	crdsDir := path.Join(data.DataDir(), data.KarmadaVersion())
+	crdsDir, err := getCrdsDir(data)
+	if err != nil {
+		return fmt.Errorf("[prepare-crds] failed to get CRD dir, err: %w", err)
+	}
+
 	klog.V(4).InfoS("[prepare-crds] Running prepare-crds task", "karmada", klog.KObj(data))
 	klog.V(2).InfoS("[prepare-crds] Using crd folder", "folder", crdsDir, "karmada", klog.KObj(data))

@ -73,7 +80,17 @@ func skipCrdsDownload(r workflow.RunData) (bool, error) {
 		return false, errors.New("prepare-crds task invoked with an invalid data struct")
 	}

-	crdsDir := path.Join(data.DataDir(), data.KarmadaVersion())
+	crdTarball := data.CrdTarball()
+	if crdTarball.CRDDownloadPolicy != nil && *crdTarball.CRDDownloadPolicy == operatorv1alpha1.DownloadAlways {
+		klog.V(2).InfoS("[skipCrdsDownload] CrdDownloadPolicy is 'Always', skipping cache check")
+		return false, nil
+	}
+
+	crdsDir, err := getCrdsDir(data)
+	if err != nil {
+		return false, fmt.Errorf("[skipCrdsDownload] failed to get CRD dir, err: %w", err)
+	}
+
 	if exist, err := util.PathExists(crdsDir); !exist || err != nil {
 		return false, err
 	}
@ -82,7 +99,6 @@ func skipCrdsDownload(r workflow.RunData) (bool, error) {
 		return false, nil
 	}

-	klog.V(2).InfoS("[download-crds] Skip download crd yaml files, the crd tar exists on disk", "karmada", klog.KObj(data))
 	return true, nil
 }

@ -92,31 +108,33 @@ func runCrdsDownload(r workflow.RunData) error {
 		return errors.New("download-crds task invoked with an invalid data struct")
 	}

-	var (
-		crdsDir     = path.Join(data.DataDir(), data.KarmadaVersion())
-		crdsTarPath = path.Join(crdsDir, crdsFileSuffix)
-	)
+	crdsDir, err := getCrdsDir(data)
+	if err != nil {
+		return fmt.Errorf("[download-crds] failed to get CRD dir, err: %w", err)
+	}
+	crdsTarPath := path.Join(crdsDir, crdsFileSuffix)

 	exist, err := util.PathExists(crdsDir)
 	if err != nil {
 		return err
 	}
-	if !exist {
-		if err := os.MkdirAll(crdsDir, 0700); err != nil {
-			return err
+
+	if exist {
+		if err := os.RemoveAll(crdsDir); err != nil {
+			return fmt.Errorf("failed to delete CRDs directory, err: %w", err)
 		}
 	}

-	if !existCrdsTar(crdsDir) {
-		err := util.DownloadFile(data.CrdsRemoteURL(), crdsTarPath)
-		if err != nil {
-			return fmt.Errorf("failed to download crd tar, err: %w", err)
-		}
-	} else {
-		klog.V(2).InfoS("[download-crds] The crd tar exists on disk", "path", crdsDir, "karmada", klog.KObj(data))
+	if err := os.MkdirAll(crdsDir, 0700); err != nil {
+		return fmt.Errorf("failed to create CRDs directory, err: %w", err)
 	}

-	klog.V(2).InfoS("[download-crds] Successfully downloaded crd package for remote url", "karmada", klog.KObj(data))
+	crdTarball := data.CrdTarball()
+	if err := util.DownloadFile(crdTarball.HTTPSource.URL, crdsTarPath); err != nil {
+		return fmt.Errorf("failed to download CRD tar, err: %w", err)
+	}
+
+	klog.V(2).InfoS("[download-crds] Successfully downloaded crd package", "karmada", klog.KObj(data))
 	return nil
 }

@ -126,15 +144,16 @@ func runUnpack(r workflow.RunData) error {
 		return errors.New("unpack task invoked with an invalid data struct")
 	}

-	var (
-		crdsDir     = path.Join(data.DataDir(), data.KarmadaVersion())
-		crdsTarPath = path.Join(crdsDir, crdsFileSuffix)
-		crdsPath    = path.Join(crdsDir, crdPathSuffix)
-	)
+	crdsDir, err := getCrdsDir(data)
+	if err != nil {
+		return fmt.Errorf("[unpack] failed to get CRD dir, err: %w", err)
+	}
+	crdsTarPath := path.Join(crdsDir, crdsFileSuffix)
+	crdsPath := path.Join(crdsDir, crdPathSuffix)

-	// TODO: check whether crd yaml is valid.
 	exist, _ := util.PathExists(crdsPath)
 	if !exist {
+		klog.V(2).InfoS("[runUnpack] CRD yaml files do not exist, unpacking tar file", "unpackDir", crdsDir)
 		if err := util.Unpack(crdsTarPath, crdsDir); err != nil {
 			return fmt.Errorf("[unpack] failed to unpack crd tar, err: %w", err)
 		}
@ -148,6 +167,7 @@ func runUnpack(r workflow.RunData) error {

 func existCrdsTar(crdsDir string) bool {
 	files := util.ListFiles(crdsDir)
+	klog.V(2).InfoS("[existCrdsTar] Checking for CRD tar file in directory", "directory", crdsDir)

 	for _, file := range files {
 		if strings.Contains(file.Name(), crdsFileSuffix) && file.Size() > 0 {
@ -156,3 +176,11 @@ func existCrdsTar(crdsDir string) bool {
 	}
 	return false
 }
+
+func getCrdsDir(data InitData) (string, error) {
+	crdTarball := data.CrdTarball()
+	key := strings.TrimSpace(crdTarball.HTTPSource.URL)
+	hash := sha256.Sum256([]byte(key))
+	hashedKey := hex.EncodeToString(hash[:])
+	return path.Join(data.DataDir(), "cache", hashedKey), nil
+}
--- a/operator/pkg/tasks/init/data.go
+++ b/operator/pkg/tasks/init/data.go
@ -35,7 +35,7 @@ type InitData interface {
 	RemoteClient() clientset.Interface
 	KarmadaClient() clientset.Interface
 	DataDir() string
-	CrdsRemoteURL() string
+	CrdTarball() operatorv1alpha1.CRDTarball
 	KarmadaVersion() string
 	Components() *operatorv1alpha1.KarmadaComponents
 	FeatureGates() map[string]bool
--- a/operator/pkg/tasks/init/karmadaresource.go
+++ b/operator/pkg/tasks/init/karmadaresource.go
@ -101,12 +101,12 @@ func runCrds(r workflow.RunData) error {
 		return errors.New("crds task invoked with an invalid data struct")
 	}

-	var (
-		crdsDir       = path.Join(data.DataDir(), data.KarmadaVersion())
-		crdsPath      = path.Join(crdsDir, "crds/bases")
-		crdsPatchPath = path.Join(crdsDir, "crds/patches")
-	)
-
+	crdsDir, err := getCrdsDir(data)
+	if err != nil {
+		return fmt.Errorf("failed to get CRD dir, err: %w", err)
+	}
+	crdsPath := path.Join(crdsDir, "crds/bases")
+	crdsPatchPath := path.Join(crdsDir, "crds/patches")
 	crdsClient, err := apiclient.NewCRDsClient(data.ControlplaneConfig())
 	if err != nil {
 		return err