hub/cmd/tracker/main.go

package main

import (
	"context"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"github.com/artifacthub/hub/internal/authz"
	"github.com/artifacthub/hub/internal/hub"
	"github.com/artifacthub/hub/internal/pkg"
	"github.com/artifacthub/hub/internal/repo"
	"github.com/artifacthub/hub/internal/tracker"
	"github.com/artifacthub/hub/internal/tracker/falco"
	"github.com/artifacthub/hub/internal/tracker/generic"
	"github.com/artifacthub/hub/internal/tracker/helm"
	"github.com/artifacthub/hub/internal/tracker/krew"
	"github.com/artifacthub/hub/internal/tracker/olm"
	"github.com/artifacthub/hub/internal/util"
	"github.com/rs/zerolog/log"
	"golang.org/x/time/rate"
)

const (
	githubMaxRequestsPerHour = 5000
	cloudNativeSecurityHub   = "https://github.com/falcosecurity/cloud-native-security-hub/resources/falco"
)

func main() {
	// Setup configuration and logger
	cfg, err := util.SetupConfig("tracker")
	if err != nil {
		log.Fatal().Err(err).Msg("configuration setup failed")
	}
	fields := map[string]interface{}{"cmd": "tracker"}
	if err := util.SetupLogger(cfg, fields); err != nil {
		log.Fatal().Err(err).Msg("logger setup failed")
	}

	// Shutdown gracefully when SIGINT or SIGTERM signal is received
	log.Info().Int("pid", os.Getpid()).Msg("tracker started")
	ctx, cancel := context.WithCancel(context.Background())
	shutdown := make(chan os.Signal, 1)
	signal.Notify(shutdown, os.Interrupt, syscall.SIGTERM)
	go func() {
		<-shutdown
		cancel()
		log.Info().Msg("tracker shutting down..")
	}()

	// Check required external tools are available
	if _, err := exec.LookPath("opm"); err != nil {
		log.Fatal().Err(err).Msg("opm not found")
	}

	// Setup services
	db, err := util.SetupDB(cfg)
	if err != nil {
		log.Fatal().Err(err).Msg("database setup failed")
	}
	az, err := authz.NewAuthorizer(db)
	if err != nil {
		log.Fatal().Err(err).Msg("authorizer setup failed")
	}
	rm := repo.NewManager(cfg, db, az)
	pm := pkg.NewManager(db)
	is, err := util.SetupImageStore(cfg, db)
	if err != nil {
		log.Fatal().Err(err).Msg("image store setup failed")
	}
	repos, err := tracker.GetRepositories(ctx, cfg, rm)
	if err != nil {
		log.Fatal().Err(err).Msg("error getting repositories")
	}
	ec := tracker.NewDBErrorsCollector(rm, repos)
	githubRL := rate.NewLimiter(rate.Every(1*time.Hour), githubMaxRequestsPerHour)
	go func() {
		<-time.After(1 * time.Hour)
		githubRL.SetLimit(rate.Every(1 * time.Hour / githubMaxRequestsPerHour))
	}()
	svc := &tracker.Services{
		Ctx:      ctx,
		Cfg:      cfg,
		Rm:       rm,
		Pm:       pm,
		Is:       is,
		Ec:       ec,
		Hc:       &http.Client{Timeout: 10 * time.Second},
		GithubRL: githubRL,
	}

	// Track registered repositories
	cfg.SetDefault("tracker.concurrency", 1)
	limiter := make(chan struct{}, cfg.GetInt("tracker.concurrency"))
	var wg sync.WaitGroup
L:
	for _, r := range repos {
		select {
		case <-ctx.Done():
			break L
		default:
		}

		limiter <- struct{}{}
		wg.Add(1)
		go func(r *hub.Repository) {
			defer func() {
				<-limiter
				wg.Done()
			}()
			var t tracker.Tracker
			switch r.Kind {
			case hub.Falco:
				// Temporary solution to maintain backwards compatibility with
				// the only Falco rules repository registered at the moment in
				// artifacthub.io using the structure and metadata format used
				// by the cloud native security hub.
				if r.URL == cloudNativeSecurityHub {
					t = falco.NewTracker(svc, r)
				} else {
					t = generic.NewTracker(svc, r)
				}
			case hub.Helm:
				t = helm.NewTracker(svc, r)
			case hub.Krew:
				t = krew.NewTracker(svc, r)
			case hub.OLM:
				t = olm.NewTracker(svc, r)
			case hub.OPA, hub.TBAction:
				t = generic.NewTracker(svc, r)
			}
			if err := tracker.TrackRepository(ctx, cfg, rm, t, r); err != nil {
				svc.Ec.Append(r.RepositoryID, err)
				log.Error().Err(err).Str("repo", r.Name).Str("kind", hub.GetKindName(r.Kind)).Send()
			}
		}(r)
	}
	wg.Wait()
	ec.Flush()
	log.Info().Msg("tracker finished")
}