internal/helm: optimize dependency manager
This commit starts with the optimization of the `DepenendencyManager`, ensuring the chart indexes are lazy loaded, and replacing the (limitless) concurrency with a configurable number of workers with a default of 1. Signed-off-by: Hidde Beydals <hello@hidde.co>
This commit is contained in:
parent
44c1863334
commit
d60131d16b
|
@ -28,6 +28,7 @@ import (
|
||||||
"github.com/Masterminds/semver/v3"
|
"github.com/Masterminds/semver/v3"
|
||||||
securejoin "github.com/cyphar/filepath-securejoin"
|
securejoin "github.com/cyphar/filepath-securejoin"
|
||||||
"golang.org/x/sync/errgroup"
|
"golang.org/x/sync/errgroup"
|
||||||
|
"golang.org/x/sync/semaphore"
|
||||||
helmchart "helm.sh/helm/v3/pkg/chart"
|
helmchart "helm.sh/helm/v3/pkg/chart"
|
||||||
"helm.sh/helm/v3/pkg/chart/loader"
|
"helm.sh/helm/v3/pkg/chart/loader"
|
||||||
)
|
)
|
||||||
|
@ -58,38 +59,51 @@ type DependencyManager struct {
|
||||||
// Dependencies contains a list of dependencies, and the respective
|
// Dependencies contains a list of dependencies, and the respective
|
||||||
// repository the dependency can be found at.
|
// repository the dependency can be found at.
|
||||||
Dependencies []*DependencyWithRepository
|
Dependencies []*DependencyWithRepository
|
||||||
|
// Workers is the number of concurrent chart-add operations during
|
||||||
|
// Build. Defaults to 1 (non-concurrent).
|
||||||
|
Workers int64
|
||||||
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build compiles and builds the dependencies of the Chart.
|
// Build compiles and builds the dependencies of the Chart with the
|
||||||
|
// configured number of Workers.
|
||||||
func (dm *DependencyManager) Build(ctx context.Context) error {
|
func (dm *DependencyManager) Build(ctx context.Context) error {
|
||||||
if len(dm.Dependencies) == 0 {
|
if len(dm.Dependencies) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
errs, ctx := errgroup.WithContext(ctx)
|
workers := dm.Workers
|
||||||
for _, i := range dm.Dependencies {
|
if workers <= 0 {
|
||||||
item := i
|
workers = 1
|
||||||
errs.Go(func() error {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
var err error
|
|
||||||
switch item.Repository {
|
|
||||||
case nil:
|
|
||||||
err = dm.addLocalDependency(item)
|
|
||||||
default:
|
|
||||||
err = dm.addRemoteDependency(item)
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return errs.Wait()
|
defer func() {
|
||||||
|
for _, dep := range dm.Dependencies {
|
||||||
|
dep.Repository.UnloadIndex()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
group, groupCtx := errgroup.WithContext(ctx)
|
||||||
|
group.Go(func() error {
|
||||||
|
sem := semaphore.NewWeighted(workers)
|
||||||
|
for _, dep := range dm.Dependencies {
|
||||||
|
dep := dep
|
||||||
|
if err := sem.Acquire(groupCtx, 1); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
group.Go(func() error {
|
||||||
|
defer sem.Release(1)
|
||||||
|
if dep.Repository == nil {
|
||||||
|
return dm.addLocalDependency(dep)
|
||||||
|
}
|
||||||
|
return dm.addRemoteDependency(dep)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
return group.Wait()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (dm *DependencyManager) addLocalDependency(dpr *DependencyWithRepository) error {
|
func (dm *DependencyManager) addLocalDependency(dpr *DependencyWithRepository) error {
|
||||||
|
@ -136,7 +150,18 @@ func (dm *DependencyManager) addLocalDependency(dpr *DependencyWithRepository) e
|
||||||
|
|
||||||
func (dm *DependencyManager) addRemoteDependency(dpr *DependencyWithRepository) error {
|
func (dm *DependencyManager) addRemoteDependency(dpr *DependencyWithRepository) error {
|
||||||
if dpr.Repository == nil {
|
if dpr.Repository == nil {
|
||||||
return fmt.Errorf("no ChartRepository given for '%s' dependency", dpr.Dependency.Name)
|
return fmt.Errorf("no HelmRepository for '%s' dependency", dpr.Dependency.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !dpr.Repository.HasIndex() {
|
||||||
|
if !dpr.Repository.HasCacheFile() {
|
||||||
|
if _, err := dpr.Repository.CacheIndex(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := dpr.Repository.LoadFromCache(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
chartVer, err := dpr.Repository.Get(dpr.Dependency.Name, dpr.Dependency.Version)
|
chartVer, err := dpr.Repository.Get(dpr.Dependency.Name, dpr.Dependency.Version)
|
||||||
|
@ -157,7 +182,6 @@ func (dm *DependencyManager) addRemoteDependency(dpr *DependencyWithRepository)
|
||||||
dm.mu.Lock()
|
dm.mu.Lock()
|
||||||
dm.Chart.AddDependency(ch)
|
dm.Chart.AddDependency(ch)
|
||||||
dm.mu.Unlock()
|
dm.mu.Unlock()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -182,13 +182,12 @@ func TestBuild_WithRemoteChart(t *testing.T) {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
i := repo.NewIndexFile()
|
i := repo.NewIndexFile()
|
||||||
i.Add(&helmchart.Metadata{Name: chartName, Version: chartVersion}, fmt.Sprintf("%s-%s.tgz", chartName, chartVersion), "http://example.com/charts", "sha256:1234567890")
|
i.MustAdd(&helmchart.Metadata{Name: chartName, Version: chartVersion}, fmt.Sprintf("%s-%s.tgz", chartName, chartVersion), "http://example.com/charts", "sha256:1234567890")
|
||||||
mg := mockGetter{response: b}
|
mg := mockGetter{response: b}
|
||||||
cr := &ChartRepository{
|
cr := newChartRepository()
|
||||||
URL: remoteDepFixture.Repository,
|
cr.URL = remoteDepFixture.Repository
|
||||||
Index: i,
|
cr.Index = i
|
||||||
Client: &mg,
|
cr.Client = &mg
|
||||||
}
|
|
||||||
dm := DependencyManager{
|
dm := DependencyManager{
|
||||||
Chart: &chart,
|
Chart: &chart,
|
||||||
Dependencies: []*DependencyWithRepository{
|
Dependencies: []*DependencyWithRepository{
|
||||||
|
|
Loading…
Reference in New Issue