package olm import ( "encoding/base64" "encoding/json" "fmt" "io/ioutil" "os" "path/filepath" "regexp" "sort" "strconv" "strings" "time" "github.com/Masterminds/semver/v3" "github.com/artifacthub/hub/internal/hub" "github.com/artifacthub/hub/internal/pkg" "github.com/artifacthub/hub/internal/tracker/source" "github.com/ghodss/yaml" "github.com/operator-framework/api/pkg/manifests" operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" ) const ( changesAnnotation = "artifacthub.io/changes" imagesWhitelistAnnotation = "artifacthub.io/imagesWhitelist" installAnnotation = "artifacthub.io/install" licenseAnnotation = "artifacthub.io/license" prereleaseAnnotation = "artifacthub.io/prerelease" recommendationsAnnotation = "artifacthub.io/recommendations" securityUpdatesAnnotation = "artifacthub.io/containsSecurityUpdates" ) var ( // channelVersionRE is a regexp used to extract the version from the // channel CurrentCSVName. channelVersionRE = regexp.MustCompile(`^[A-Za-z0-9_-]+\.v?(.*)$`) ) // TrackerSource is a hub.TrackerSource implementation for OLM repositories. type TrackerSource struct { i *hub.TrackerSourceInput } // NewTrackerSource creates a new TrackerSource instance. func NewTrackerSource(i *hub.TrackerSourceInput) *TrackerSource { return &TrackerSource{i} } // GetPackagesAvailable implements the TrackerSource interface. func (s *TrackerSource) GetPackagesAvailable() (map[string]*hub.Package, error) { packagesAvailable := make(map[string]*hub.Package) // Walk the path provided looking for available packages err := filepath.Walk(s.i.BasePath, func(pkgPath string, info os.FileInfo, err error) error { // Return ASAP if context is cancelled select { case <-s.i.Svc.Ctx.Done(): return s.i.Svc.Ctx.Err() default: } // If an error is raised while visiting a path or the path is not a // directory, we skip it if err != nil || !info.IsDir() { return nil } // Get package manifest manifest, err := getManifest(pkgPath) if err != nil { s.warn(fmt.Errorf("error getting package manifest: %w", err)) return nil } if manifest == nil { // Package manifest not found, not a package path return nil } // Get package versions pkgName := manifest.PackageName versionsUnfiltered, err := ioutil.ReadDir(pkgPath) if err != nil { s.warn(fmt.Errorf("error reading package %s versions: %w", pkgName, err)) return nil } var versions []os.FileInfo for _, entryV := range versionsUnfiltered { if !entryV.IsDir() { continue } if _, err := semver.StrictNewVersion(entryV.Name()); err != nil { s.warn(fmt.Errorf("invalid package %s version (%s): %w", pkgName, entryV.Name(), err)) continue } else { versions = append(versions, entryV) } } sort.Slice(versions, func(i, j int) bool { vi, _ := semver.NewVersion(versions[i].Name()) vj, _ := semver.NewVersion(versions[j].Name()) return vj.LessThan(vi) }) // Process package versions for _, entryV := range versions { // Get package version CSV version := entryV.Name() pkgVersionPath := filepath.Join(pkgPath, version) csv, csvData, err := getCSV(pkgVersionPath) if err != nil { s.warn(fmt.Errorf("error getting package %s version %s csv: %w", pkgName, version, err)) continue } // Prepare and store package version p := s.preparePackage(s.i.Repository, manifest, csv, csvData) packagesAvailable[pkg.BuildKey(p)] = p } return nil }) if err != nil { return nil, err } return packagesAvailable, nil } // preparePackage prepares a package version using the package manifest and csv. func (s *TrackerSource) preparePackage( r *hub.Repository, manifest *manifests.PackageManifest, csv *operatorsv1alpha1.ClusterServiceVersion, csvData []byte, ) *hub.Package { // Prepare package from manifest and csv p := &hub.Package{ Name: manifest.PackageName, DisplayName: csv.Spec.DisplayName, Description: csv.Annotations["description"], Keywords: csv.Spec.Keywords, Readme: csv.Spec.Description, Version: csv.Spec.Version.String(), IsOperator: true, Capabilities: csv.Annotations["capabilities"], DefaultChannel: manifest.DefaultChannelName, License: csv.Annotations[licenseAnnotation], Provider: csv.Spec.Provider.Name, ContainersImages: getContainersImages(csv, csvData), Install: csv.Annotations[installAnnotation], Repository: r, } // TS ts, err := time.Parse(time.RFC3339, csv.Annotations["createdAt"]) if err == nil { p.TS = ts.Unix() } else { // Try alternative layout ts, err = time.Parse("2006-01-02 15:04:05", csv.Annotations["createdAt"]) if err == nil { p.TS = ts.Unix() } } // Keywords for _, category := range strings.Split(csv.Annotations["categories"], ",") { if strings.Trim(strings.ToLower(category), " ") == "ai/machine learning" { p.Keywords = append(p.Keywords, []string{"AI", "Machine Learning"}...) } else { p.Keywords = append(p.Keywords, strings.Trim(category, " ")) } } // Links for _, link := range csv.Spec.Links { p.Links = append(p.Links, &hub.Link{ Name: link.Name, URL: link.URL, }) } if csv.Annotations["repository"] != "" { p.Links = append(p.Links, &hub.Link{ Name: "source", URL: csv.Annotations["repository"], }) } // Store logo when available if len(csv.Spec.Icon) > 0 && csv.Spec.Icon[0].Data != "" { data, err := base64.StdEncoding.DecodeString(csv.Spec.Icon[0].Data) if err != nil { s.warn(fmt.Errorf("error decoding package %s logo image: %w", p.Name, err)) } else { p.LogoImageID, err = s.i.Svc.Is.SaveImage(s.i.Svc.Ctx, data) if err != nil { s.warn(fmt.Errorf("error saving package %s image: %w", p.Name, err)) } } } // Maintainers for _, maintainer := range csv.Spec.Maintainers { p.Maintainers = append(p.Maintainers, &hub.Maintainer{ Name: maintainer.Name, Email: maintainer.Email, }) } // Channels for _, channel := range manifest.Channels { matches := channelVersionRE.FindStringSubmatch(channel.CurrentCSVName) if len(matches) != 2 { continue } version := matches[1] p.Channels = append(p.Channels, &hub.Channel{ Name: channel.Name, Version: version, }) } // CRDs crds := make([]interface{}, 0, len(csv.Spec.CustomResourceDefinitions.Owned)) for _, crd := range csv.Spec.CustomResourceDefinitions.Owned { crds = append(crds, map[string]interface{}{ "name": crd.Name, "version": crd.Version, "kind": crd.Kind, "displayName": crd.DisplayName, "description": crd.Description, }) } if len(crds) > 0 { p.CRDs = crds } var crdsExamples []interface{} if err := json.Unmarshal([]byte(csv.Annotations["alm-examples"]), &crdsExamples); err == nil { p.CRDsExamples = crdsExamples } // Recommendations if v, ok := csv.Annotations[recommendationsAnnotation]; ok { var recommendations []*hub.Recommendation if err := yaml.Unmarshal([]byte(v), &recommendations); err == nil { p.Recommendations = recommendations } } // Misc var isGlobalOperator bool for _, e := range csv.Spec.InstallModes { if e.Type == operatorsv1alpha1.InstallModeTypeAllNamespaces && e.Supported { isGlobalOperator = true } } changes, err := source.ParseChangesAnnotation(csv.Annotations[changesAnnotation]) if err == nil { p.Changes = changes } containsSecurityUpdates, err := strconv.ParseBool(csv.Annotations[securityUpdatesAnnotation]) if err == nil { p.ContainsSecurityUpdates = containsSecurityUpdates } prerelease, err := strconv.ParseBool(csv.Annotations[prereleaseAnnotation]) if err == nil { p.Prerelease = prerelease } p.Data = map[string]interface{}{ "isGlobalOperator": isGlobalOperator, } return p } // warn is a helper that sends the error provided to the errors collector and // logs it as a warning. func (s *TrackerSource) warn(err error) { s.i.Svc.Logger.Warn().Err(err).Send() s.i.Svc.Ec.Append(s.i.Repository.RepositoryID, err.Error()) } // getManifest reads and parses the package manifest. func getManifest(pkgPath string) (*manifests.PackageManifest, error) { // Locate manifest file matches, err := filepath.Glob(filepath.Join(pkgPath, "*package.yaml")) if err != nil { return nil, fmt.Errorf("error locating manifest file: %w", err) } if len(matches) != 1 { return nil, nil } manifestPath := matches[0] // Read and parse manifest file manifestData, err := ioutil.ReadFile(manifestPath) if err != nil { return nil, fmt.Errorf("error reading manifest file: %w", err) } manifest := &manifests.PackageManifest{} if err = yaml.Unmarshal(manifestData, &manifest); err != nil { return nil, fmt.Errorf("error unmarshaling manifest file: %w", err) } return manifest, nil } // getCSV reads, parses and validates the cluster service version file of the // given package version. func getCSV(path string) (*operatorsv1alpha1.ClusterServiceVersion, []byte, error) { // Locate cluster service version file matches, err := filepath.Glob(filepath.Join(path, "*.clusterserviceversion.yaml")) if err != nil { return nil, nil, fmt.Errorf("error locating csv file: %w", err) } if len(matches) != 1 { return nil, nil, fmt.Errorf("csv file not found") } csvPath := matches[0] // Read and parse cluster service version file csvData, err := ioutil.ReadFile(csvPath) if err != nil { return nil, nil, fmt.Errorf("error reading csv file: %w", err) } csv := &operatorsv1alpha1.ClusterServiceVersion{} if err = yaml.Unmarshal(csvData, &csv); err != nil { return nil, nil, fmt.Errorf("error unmarshaling csv file: %w", err) } return csv, csvData, nil } // getContainersImages returns all containers images declared in the csv data // provided. func getContainersImages(csv *operatorsv1alpha1.ClusterServiceVersion, csvData []byte) []*hub.ContainerImage { var images []*hub.ContainerImage // Container image annotation if containerImage, ok := csv.Annotations["containerImage"]; ok && containerImage != "" { images = append(images, &hub.ContainerImage{Image: containerImage}) } // Related images type Spec struct { RelatedImages []*hub.ContainerImage `json:"relatedImages"` } type CSV struct { Spec Spec `json:"spec"` } csvRI := &CSV{} if err := yaml.Unmarshal(csvData, &csvRI); err == nil { images = append(images, csvRI.Spec.RelatedImages...) } var imagesWhitelist []string if err := yaml.Unmarshal([]byte(csv.Annotations[imagesWhitelistAnnotation]), &imagesWhitelist); err == nil { for _, image := range images { if contains(imagesWhitelist, image.Image) { image.Whitelisted = true } } } return images } // contains is a helper to check if a list contains the string provided. func contains(l []string, e string) bool { for _, x := range l { if x == e { return true } } return false }