From 1ae5bd19240c92cde9d76ed875943106c6bdc19f Mon Sep 17 00:00:00 2001 From: Joe Nathan Abellard Date: Fri, 12 Jul 2024 14:57:27 -0400 Subject: [PATCH] Initial strucutre Signed-off-by: Joe Nathan Abellard Update crds Signed-off-by: Joe Nathan Abellard Remove debug logs Signed-off-by: Joe Nathan Abellard Remove debug logs Signed-off-by: Joe Nathan Abellard Remove debug logs Signed-off-by: Joe Nathan Abellard Remove debug logs Signed-off-by: Joe Nathan Abellard Fix linting errors Signed-off-by: Joe Nathan Abellard Address comments Signed-off-by: Joe Nathan Abellard Update api Signed-off-by: Joe Nathan Abellard Update api Signed-off-by: Joe Nathan Abellard Address comments Signed-off-by: Joe Nathan Abellard Address comments Signed-off-by: Joe Nathan Abellard Address comments Signed-off-by: Joe Nathan Abellard --- .../crds/operator.karmada.io_karmadas.yaml | 28 +++++++ .../crds/operator.karmada.io_karmadas.yaml | 28 +++++++ operator/pkg/apis/operator/v1alpha1/type.go | 41 ++++++++++ .../v1alpha1/zz_generated.deepcopy.go | 47 ++++++++++++ operator/pkg/init.go | 27 ++++--- operator/pkg/tasks/init/crd.go | 76 +++++++++++++------ operator/pkg/tasks/init/data.go | 2 +- operator/pkg/tasks/init/karmadaresource.go | 12 +-- 8 files changed, 221 insertions(+), 40 deletions(-) diff --git a/charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml b/charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml index bed6b2796..2125afde7 100644 --- a/charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml +++ b/charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml @@ -1730,6 +1730,34 @@ spec: type: object type: object type: object + crdTarball: + description: |- + CRDTarball specifies the source from which the Karmada CRD tarball should be downloaded, along with the download policy to use. + If not set, the operator will download the tarball from a GitHub release. + By default, it will download the tarball of the same version as the operator itself. + For instance, if the operator's version is v1.10.0, the tarball will be downloaded from the following location: + https://github.com/karmada-io/karmada/releases/download/v1.10.0/crds.tar.gz + By default, the operator will only attempt to download the tarball if it's not yet present in the local cache. + properties: + crdDownloadPolicy: + default: IfNotPresent + description: |- + CRDDownloadPolicy specifies a policy that should be used to download the CRD tarball. + Valid values are "Always" and "IfNotPresent". + Defaults to "IfNotPresent". + enum: + - Always + - IfNotPresent + type: string + httpSource: + description: HTTPSource specifies how to download the CRD tarball + via either HTTP or HTTPS protocol. + properties: + url: + description: URL specifies the URL of the CRD tarball resource. + type: string + type: object + type: object featureGates: additionalProperties: type: boolean diff --git a/operator/config/crds/operator.karmada.io_karmadas.yaml b/operator/config/crds/operator.karmada.io_karmadas.yaml index bed6b2796..2125afde7 100644 --- a/operator/config/crds/operator.karmada.io_karmadas.yaml +++ b/operator/config/crds/operator.karmada.io_karmadas.yaml @@ -1730,6 +1730,34 @@ spec: type: object type: object type: object + crdTarball: + description: |- + CRDTarball specifies the source from which the Karmada CRD tarball should be downloaded, along with the download policy to use. + If not set, the operator will download the tarball from a GitHub release. + By default, it will download the tarball of the same version as the operator itself. + For instance, if the operator's version is v1.10.0, the tarball will be downloaded from the following location: + https://github.com/karmada-io/karmada/releases/download/v1.10.0/crds.tar.gz + By default, the operator will only attempt to download the tarball if it's not yet present in the local cache. + properties: + crdDownloadPolicy: + default: IfNotPresent + description: |- + CRDDownloadPolicy specifies a policy that should be used to download the CRD tarball. + Valid values are "Always" and "IfNotPresent". + Defaults to "IfNotPresent". + enum: + - Always + - IfNotPresent + type: string + httpSource: + description: HTTPSource specifies how to download the CRD tarball + via either HTTP or HTTPS protocol. + properties: + url: + description: URL specifies the URL of the CRD tarball resource. + type: string + type: object + type: object featureGates: additionalProperties: type: boolean diff --git a/operator/pkg/apis/operator/v1alpha1/type.go b/operator/pkg/apis/operator/v1alpha1/type.go index fbe5dfd72..dd2ebb518 100644 --- a/operator/pkg/apis/operator/v1alpha1/type.go +++ b/operator/pkg/apis/operator/v1alpha1/type.go @@ -44,6 +44,38 @@ type Karmada struct { Status KarmadaStatus `json:"status,omitempty"` } +// CRDDownloadPolicy specifies a policy for how the operator will download the Karmada CRD tarball +type CRDDownloadPolicy string + +const ( + // DownloadAlways instructs the Karmada operator to always download the CRD tarball from a remote location. + DownloadAlways CRDDownloadPolicy = "Always" + + // DownloadIfNotPresent instructs the Karmada operator to download the CRDs tarball from a remote location only if it is not yet present in the local cache. + DownloadIfNotPresent CRDDownloadPolicy = "IfNotPresent" +) + +// HTTPSource specifies how to download the CRD tarball via either HTTP or HTTPS protocol. +type HTTPSource struct { + // URL specifies the URL of the CRD tarball resource. + URL string `json:"url,omitempty"` +} + +// CRDTarball specifies the source from which the Karmada CRD tarball should be downloaded, along with the download policy to use. +type CRDTarball struct { + // HTTPSource specifies how to download the CRD tarball via either HTTP or HTTPS protocol. + // +optional + HTTPSource *HTTPSource `json:"httpSource,omitempty"` + + // CRDDownloadPolicy specifies a policy that should be used to download the CRD tarball. + // Valid values are "Always" and "IfNotPresent". + // Defaults to "IfNotPresent". + // +kubebuilder:validation:Enum=Always;IfNotPresent + // +kubebuilder:default=IfNotPresent + // +optional + CRDDownloadPolicy *CRDDownloadPolicy `json:"crdDownloadPolicy,omitempty"` +} + // KarmadaSpec is the specification of the desired behavior of the Karmada. type KarmadaSpec struct { // HostCluster represents the cluster where to install the Karmada control plane. @@ -72,6 +104,15 @@ type KarmadaSpec struct { // More info: https://github.com/karmada-io/karmada/blob/master/pkg/features/features.go // +optional FeatureGates map[string]bool `json:"featureGates,omitempty"` + + // CRDTarball specifies the source from which the Karmada CRD tarball should be downloaded, along with the download policy to use. + // If not set, the operator will download the tarball from a GitHub release. + // By default, it will download the tarball of the same version as the operator itself. + // For instance, if the operator's version is v1.10.0, the tarball will be downloaded from the following location: + // https://github.com/karmada-io/karmada/releases/download/v1.10.0/crds.tar.gz + // By default, the operator will only attempt to download the tarball if it's not yet present in the local cache. + // +optional + CRDTarball *CRDTarball `json:"crdTarball,omitempty"` } // ImageRegistry represents an image registry as well as the diff --git a/operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go b/operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go index 7e0139e61..465f0cf1a 100644 --- a/operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go @@ -27,6 +27,32 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CRDTarball) DeepCopyInto(out *CRDTarball) { + *out = *in + if in.HTTPSource != nil { + in, out := &in.HTTPSource, &out.HTTPSource + *out = new(HTTPSource) + **out = **in + } + if in.CRDDownloadPolicy != nil { + in, out := &in.CRDDownloadPolicy, &out.CRDDownloadPolicy + *out = new(CRDDownloadPolicy) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CRDTarball. +func (in *CRDTarball) DeepCopy() *CRDTarball { + if in == nil { + return nil + } + out := new(CRDTarball) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CommonSettings) DeepCopyInto(out *CommonSettings) { *out = *in @@ -126,6 +152,22 @@ func (in *ExternalEtcd) DeepCopy() *ExternalEtcd { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HTTPSource) DeepCopyInto(out *HTTPSource) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPSource. +func (in *HTTPSource) DeepCopy() *HTTPSource { + if in == nil { + return nil + } + out := new(HTTPSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HostCluster) DeepCopyInto(out *HostCluster) { *out = *in @@ -559,6 +601,11 @@ func (in *KarmadaSpec) DeepCopyInto(out *KarmadaSpec) { (*out)[key] = val } } + if in.CRDTarball != nil { + in, out := &in.CRDTarball, &out.CRDTarball + *out = new(CRDTarball) + (*in).DeepCopyInto(*out) + } return } diff --git a/operator/pkg/init.go b/operator/pkg/init.go index f6bde11f7..31a0bce99 100644 --- a/operator/pkg/init.go +++ b/operator/pkg/init.go @@ -48,7 +48,7 @@ type InitOptions struct { Namespace string Kubeconfig *rest.Config KarmadaVersion string - CRDRemoteURL string + CRDTarball operatorv1alpha1.CRDTarball KarmadaDataDir string Karmada *operatorv1alpha1.Karmada } @@ -60,9 +60,9 @@ func (opt *InitOptions) Validate() error { if len(opt.Name) == 0 || len(opt.Namespace) == 0 { return errors.New("unexpected empty name or namespace") } - if len(opt.CRDRemoteURL) > 0 { - if _, err := url.Parse(opt.CRDRemoteURL); err != nil { - return fmt.Errorf("unexpected invalid crds remote url %s", opt.CRDRemoteURL) + if opt.CRDTarball.HTTPSource != nil { + if _, err := url.Parse(opt.CRDTarball.HTTPSource.URL); err != nil { + return fmt.Errorf("unexpected invalid crds remote url %s", opt.CRDTarball.HTTPSource.URL) } } if !util.IsInCluster(opt.Karmada.Spec.HostCluster) && opt.Karmada.Spec.Components.KarmadaAPIServer.ServiceType == corev1.ServiceTypeClusterIP { @@ -102,7 +102,7 @@ type initData struct { remoteClient clientset.Interface karmadaClient clientset.Interface dnsDomain string - CRDRemoteURL string + CRDTarball operatorv1alpha1.CRDTarball karmadaDataDir string privateRegistry string featureGates map[string]bool @@ -183,7 +183,7 @@ func newRunData(opt *InitOptions) (*initData, error) { karmadaVersion: version, controlplaneAddress: address, remoteClient: remoteClient, - CRDRemoteURL: opt.CRDRemoteURL, + CRDTarball: opt.CRDTarball, karmadaDataDir: opt.KarmadaDataDir, privateRegistry: privateRegistry, components: opt.Karmada.Spec.Components, @@ -235,8 +235,8 @@ func (data *initData) DataDir() string { return data.karmadaDataDir } -func (data *initData) CrdsRemoteURL() string { - return data.CRDRemoteURL +func (data *initData) CrdTarball() operatorv1alpha1.CRDTarball { + return data.CRDTarball } func (data *initData) KarmadaVersion() string { @@ -268,8 +268,14 @@ func defaultJobInitOptions() *InitOptions { // set defaults for karmada. operatorscheme.Scheme.Default(karmada) + defaultDownloadPolicy := operatorv1alpha1.DownloadIfNotPresent return &InitOptions{ - CRDRemoteURL: fmt.Sprintf(defaultCrdURL, operatorv1alpha1.DefaultKarmadaImageVersion), + CRDTarball: operatorv1alpha1.CRDTarball{ + CRDDownloadPolicy: &defaultDownloadPolicy, + HTTPSource: &operatorv1alpha1.HTTPSource{ + URL: fmt.Sprintf(defaultCrdURL, operatorv1alpha1.DefaultKarmadaImageVersion), + }, + }, KarmadaVersion: operatorv1alpha1.DefaultKarmadaImageVersion, KarmadaDataDir: constants.KarmadaDataDir, Karmada: karmada, @@ -282,6 +288,9 @@ func NewInitOptWithKarmada(karmada *operatorv1alpha1.Karmada) InitOpt { o.Karmada = karmada o.Name = karmada.GetName() o.Namespace = karmada.GetNamespace() + if karmada.Spec.CRDTarball != nil { + o.CRDTarball = *karmada.Spec.CRDTarball + } } } diff --git a/operator/pkg/tasks/init/crd.go b/operator/pkg/tasks/init/crd.go index 197895424..431ea840f 100644 --- a/operator/pkg/tasks/init/crd.go +++ b/operator/pkg/tasks/init/crd.go @@ -17,6 +17,8 @@ limitations under the License. package tasks import ( + "crypto/sha256" + "encoding/hex" "errors" "fmt" "os" @@ -25,6 +27,7 @@ import ( "k8s.io/klog/v2" + operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1" "github.com/karmada-io/karmada/operator/pkg/util" "github.com/karmada-io/karmada/operator/pkg/workflow" ) @@ -60,7 +63,11 @@ func runPrepareCrds(r workflow.RunData) error { return errors.New("prepare-crds task invoked with an invalid data struct") } - crdsDir := path.Join(data.DataDir(), data.KarmadaVersion()) + crdsDir, err := getCrdsDir(data) + if err != nil { + return fmt.Errorf("[prepare-crds] failed to get CRD dir, err: %w", err) + } + klog.V(4).InfoS("[prepare-crds] Running prepare-crds task", "karmada", klog.KObj(data)) klog.V(2).InfoS("[prepare-crds] Using crd folder", "folder", crdsDir, "karmada", klog.KObj(data)) @@ -73,7 +80,17 @@ func skipCrdsDownload(r workflow.RunData) (bool, error) { return false, errors.New("prepare-crds task invoked with an invalid data struct") } - crdsDir := path.Join(data.DataDir(), data.KarmadaVersion()) + crdTarball := data.CrdTarball() + if crdTarball.CRDDownloadPolicy != nil && *crdTarball.CRDDownloadPolicy == operatorv1alpha1.DownloadAlways { + klog.V(2).InfoS("[skipCrdsDownload] CrdDownloadPolicy is 'Always', skipping cache check") + return false, nil + } + + crdsDir, err := getCrdsDir(data) + if err != nil { + return false, fmt.Errorf("[skipCrdsDownload] failed to get CRD dir, err: %w", err) + } + if exist, err := util.PathExists(crdsDir); !exist || err != nil { return false, err } @@ -82,7 +99,6 @@ func skipCrdsDownload(r workflow.RunData) (bool, error) { return false, nil } - klog.V(2).InfoS("[download-crds] Skip download crd yaml files, the crd tar exists on disk", "karmada", klog.KObj(data)) return true, nil } @@ -92,31 +108,33 @@ func runCrdsDownload(r workflow.RunData) error { return errors.New("download-crds task invoked with an invalid data struct") } - var ( - crdsDir = path.Join(data.DataDir(), data.KarmadaVersion()) - crdsTarPath = path.Join(crdsDir, crdsFileSuffix) - ) + crdsDir, err := getCrdsDir(data) + if err != nil { + return fmt.Errorf("[download-crds] failed to get CRD dir, err: %w", err) + } + crdsTarPath := path.Join(crdsDir, crdsFileSuffix) exist, err := util.PathExists(crdsDir) if err != nil { return err } - if !exist { - if err := os.MkdirAll(crdsDir, 0700); err != nil { - return err + + if exist { + if err := os.RemoveAll(crdsDir); err != nil { + return fmt.Errorf("failed to delete CRDs directory, err: %w", err) } } - if !existCrdsTar(crdsDir) { - err := util.DownloadFile(data.CrdsRemoteURL(), crdsTarPath) - if err != nil { - return fmt.Errorf("failed to download crd tar, err: %w", err) - } - } else { - klog.V(2).InfoS("[download-crds] The crd tar exists on disk", "path", crdsDir, "karmada", klog.KObj(data)) + if err := os.MkdirAll(crdsDir, 0700); err != nil { + return fmt.Errorf("failed to create CRDs directory, err: %w", err) } - klog.V(2).InfoS("[download-crds] Successfully downloaded crd package for remote url", "karmada", klog.KObj(data)) + crdTarball := data.CrdTarball() + if err := util.DownloadFile(crdTarball.HTTPSource.URL, crdsTarPath); err != nil { + return fmt.Errorf("failed to download CRD tar, err: %w", err) + } + + klog.V(2).InfoS("[download-crds] Successfully downloaded crd package", "karmada", klog.KObj(data)) return nil } @@ -126,15 +144,16 @@ func runUnpack(r workflow.RunData) error { return errors.New("unpack task invoked with an invalid data struct") } - var ( - crdsDir = path.Join(data.DataDir(), data.KarmadaVersion()) - crdsTarPath = path.Join(crdsDir, crdsFileSuffix) - crdsPath = path.Join(crdsDir, crdPathSuffix) - ) + crdsDir, err := getCrdsDir(data) + if err != nil { + return fmt.Errorf("[unpack] failed to get CRD dir, err: %w", err) + } + crdsTarPath := path.Join(crdsDir, crdsFileSuffix) + crdsPath := path.Join(crdsDir, crdPathSuffix) - // TODO: check whether crd yaml is valid. exist, _ := util.PathExists(crdsPath) if !exist { + klog.V(2).InfoS("[runUnpack] CRD yaml files do not exist, unpacking tar file", "unpackDir", crdsDir) if err := util.Unpack(crdsTarPath, crdsDir); err != nil { return fmt.Errorf("[unpack] failed to unpack crd tar, err: %w", err) } @@ -148,6 +167,7 @@ func runUnpack(r workflow.RunData) error { func existCrdsTar(crdsDir string) bool { files := util.ListFiles(crdsDir) + klog.V(2).InfoS("[existCrdsTar] Checking for CRD tar file in directory", "directory", crdsDir) for _, file := range files { if strings.Contains(file.Name(), crdsFileSuffix) && file.Size() > 0 { @@ -156,3 +176,11 @@ func existCrdsTar(crdsDir string) bool { } return false } + +func getCrdsDir(data InitData) (string, error) { + crdTarball := data.CrdTarball() + key := strings.TrimSpace(crdTarball.HTTPSource.URL) + hash := sha256.Sum256([]byte(key)) + hashedKey := hex.EncodeToString(hash[:]) + return path.Join(data.DataDir(), "cache", hashedKey), nil +} diff --git a/operator/pkg/tasks/init/data.go b/operator/pkg/tasks/init/data.go index 6ce0a2491..12dd256c0 100644 --- a/operator/pkg/tasks/init/data.go +++ b/operator/pkg/tasks/init/data.go @@ -35,7 +35,7 @@ type InitData interface { RemoteClient() clientset.Interface KarmadaClient() clientset.Interface DataDir() string - CrdsRemoteURL() string + CrdTarball() operatorv1alpha1.CRDTarball KarmadaVersion() string Components() *operatorv1alpha1.KarmadaComponents FeatureGates() map[string]bool diff --git a/operator/pkg/tasks/init/karmadaresource.go b/operator/pkg/tasks/init/karmadaresource.go index fb8a421fd..f0c6dd5e0 100644 --- a/operator/pkg/tasks/init/karmadaresource.go +++ b/operator/pkg/tasks/init/karmadaresource.go @@ -101,12 +101,12 @@ func runCrds(r workflow.RunData) error { return errors.New("crds task invoked with an invalid data struct") } - var ( - crdsDir = path.Join(data.DataDir(), data.KarmadaVersion()) - crdsPath = path.Join(crdsDir, "crds/bases") - crdsPatchPath = path.Join(crdsDir, "crds/patches") - ) - + crdsDir, err := getCrdsDir(data) + if err != nil { + return fmt.Errorf("failed to get CRD dir, err: %w", err) + } + crdsPath := path.Join(crdsDir, "crds/bases") + crdsPatchPath := path.Join(crdsDir, "crds/patches") crdsClient, err := apiclient.NewCRDsClient(data.ControlplaneConfig()) if err != nil { return err