libgit2: Remove references to libgit2 from code

Signed-off-by: Paulo Gomes <paulo.gomes@weave.works>
This commit is contained in:
Paulo Gomes 2022-12-09 14:45:58 +00:00
parent aad4060461
commit 5ffa6a5a68
No known key found for this signature in database
GPG Key ID: 9995233870E99BEE
13 changed files with 106 additions and 1609 deletions

View File

@ -1 +1 @@
build/libgit2/
build/

File diff suppressed because it is too large Load Diff

View File

@ -2,11 +2,6 @@ ARG BASE_VARIANT=alpine
ARG GO_VERSION=1.19
ARG XX_VERSION=1.1.2
ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2-only
ARG LIBGIT2_TAG=v0.4.0
FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} AS libgit2-libs
FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as gostable
@ -17,12 +12,18 @@ FROM gostable AS go-linux
# These will be used at current arch to yield execute the cross compilations.
FROM go-${TARGETOS} AS build-base
RUN apk add --no-cache clang lld pkgconfig
RUN apk add --no-cache clang lld
COPY --from=xx / /
# build-go-mod can still be cached at build platform architecture.
FROM build-base as build-go-mod
FROM build-base as build
ARG TARGETPLATFORM
# Some dependencies have to installed
# for the target platform: https://github.com/tonistiigi/xx#go--cgo
RUN xx-apk add musl-dev gcc clang lld
# Configure workspace
WORKDIR /workspace
@ -37,21 +38,6 @@ COPY go.sum go.sum
# Cache modules
RUN go mod download
# Build stage install per target platform
# dependency and effectively cross compile the application.
FROM build-go-mod as build
ARG TARGETPLATFORM
COPY --from=libgit2-libs /usr/local/ /usr/local/
# Some dependencies have to installed
# for the target platform: https://github.com/tonistiigi/xx#go--cgo
RUN xx-apk add musl-dev gcc clang lld
WORKDIR /workspace
# Copy source code
COPY main.go main.go
COPY controllers/ controllers/
@ -60,11 +46,13 @@ COPY internal/ internal/
ARG TARGETPLATFORM
ARG TARGETARCH
# Reasons why CGO is in use:
# - The SHA1 implementation (sha1cd) used by go-git depends on CGO for
# performance reasons. See: https://github.com/pjbgf/sha1cd/issues/15
ENV CGO_ENABLED=1
# Instead of using xx-go, (cross) compile with vanilla go leveraging musl tool chain.
RUN export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig" && \
export CGO_LDFLAGS="$(pkg-config --static --libs --cflags libgit2) -static -fuse-ld=lld" && \
RUN export CGO_LDFLAGS="-static -fuse-ld=lld" && \
xx-go build \
-ldflags "-s -w" \
-tags 'netgo,osusergo,static_build' \
@ -73,7 +61,6 @@ RUN export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig" && \
# Ensure that the binary was cross-compiled correctly to the target platform.
RUN xx-verify --static /source-controller
FROM alpine:3.16
ARG TARGETPLATFORM
@ -82,7 +69,6 @@ RUN apk --no-cache add ca-certificates \
# Copy over binary from build
COPY --from=build /source-controller /usr/local/bin/
COPY ATTRIBUTIONS.md /
USER 65534:65534
ENTRYPOINT [ "source-controller" ]

View File

@ -2,10 +2,6 @@
IMG ?= fluxcd/source-controller
TAG ?= latest
# Base image used to build the Go binary
LIBGIT2_IMG ?= ghcr.io/fluxcd/golang-with-libgit2-only
LIBGIT2_TAG ?= v0.4.0
# Allows for defining additional Go test args, e.g. '-tags integration'.
GO_TEST_ARGS ?= -race
@ -39,14 +35,6 @@ ENVTEST_BIN_VERSION ?= 1.24.0
# each fuzzer should run for.
FUZZ_TIME ?= 1m
# Caches libgit2 versions per tag, "forcing" rebuild only when needed.
LIBGIT2_PATH := $(BUILD_DIR)/libgit2/$(LIBGIT2_TAG)
LIBGIT2_LIB_PATH := $(LIBGIT2_PATH)/lib
LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.a
export CGO_ENABLED=1
export PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig
export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libgit2 2>/dev/null)
GO_STATIC_FLAGS=-ldflags "-s -w" -tags 'netgo,osusergo,static_build$(addprefix ,,$(GO_TAGS))'
# API (doc) generation utilities
@ -75,11 +63,11 @@ endif
all: build
build: check-deps $(LIBGIT2) ## Build manager binary
build: check-deps ## Build manager binary
go build $(GO_STATIC_FLAGS) -o $(BUILD_DIR)/bin/manager main.go
KUBEBUILDER_ASSETS?="$(shell $(ENVTEST) --arch=$(ENVTEST_ARCH) use -i $(ENVTEST_KUBERNETES_VERSION) --bin-dir=$(ENVTEST_ASSETS_DIR) -p path)"
test: $(LIBGIT2) install-envtest test-api check-deps ## Run all tests
test: install-envtest test-api check-deps ## Run all tests
HTTPS_PROXY="" HTTP_PROXY="" \
KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) \
GIT_CONFIG_GLOBAL=/dev/null \
@ -88,7 +76,7 @@ test: $(LIBGIT2) install-envtest test-api check-deps ## Run all tests
$(GO_TEST_ARGS) \
-coverprofile cover.out
test-ctrl: $(LIBGIT2) install-envtest test-api check-deps ## Run controller tests
test-ctrl: install-envtest test-api check-deps ## Run controller tests
HTTPS_PROXY="" HTTP_PROXY="" \
KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) \
GIT_CONFIG_GLOBAL=/dev/null \
@ -105,7 +93,7 @@ endif
test-api: ## Run api tests
cd api; go test $(GO_TEST_ARGS) ./... -coverprofile cover.out
run: $(LIBGIT2) generate fmt vet manifests ## Run against the configured Kubernetes cluster in ~/.kube/config
run: generate fmt vet manifests ## Run against the configured Kubernetes cluster in ~/.kube/config
go run $(GO_STATIC_FLAGS) ./main.go
install: manifests ## Install CRDs into a cluster
@ -139,7 +127,7 @@ fmt: ## Run go fmt against code
go fmt ./...
cd api; go fmt ./...
vet: $(LIBGIT2) ## Run go vet against code
vet: ## Run go vet against code
go vet ./...
cd api; go vet ./...
@ -148,8 +136,6 @@ generate: controller-gen ## Generate API code
docker-build: ## Build the Docker image
docker buildx build \
--build-arg LIBGIT2_IMG=$(LIBGIT2_IMG) \
--build-arg LIBGIT2_TAG=$(LIBGIT2_TAG) \
--platform=$(BUILD_PLATFORMS) \
-t $(IMG):$(TAG) \
$(BUILD_ARGS) .
@ -182,15 +168,6 @@ install-envtest: setup-envtest ## Download envtest binaries locally.
# setup-envtest sets anything below k8s to 0555
chmod -R u+w $(BUILD_DIR)/testbin
libgit2: $(LIBGIT2) ## Detect or download libgit2 library
COSIGN = $(GOBIN)/cosign
$(LIBGIT2):
$(call go-install-tool,$(COSIGN),github.com/sigstore/cosign/cmd/cosign@latest)
IMG=$(LIBGIT2_IMG) TAG=$(LIBGIT2_TAG) PATH=$(PATH):$(GOBIN) ./hack/install-libraries.sh
.PHONY: help
help: ## Display this help menu
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
@ -202,20 +179,6 @@ e2e:
./hack/ci/e2e.sh
verify: update-attributions fmt vet manifests api-docs tidy
ifneq ($(shell grep -o 'LIBGIT2_IMG ?= \w.*' Makefile | cut -d ' ' -f 3):$(shell grep -o 'LIBGIT2_TAG ?= \w.*' Makefile | cut -d ' ' -f 3), \
$(shell grep -o "LIBGIT2_IMG=\w.*" Dockerfile | cut -d'=' -f2):$(shell grep -o "LIBGIT2_TAG=\w.*" Dockerfile | cut -d'=' -f2))
@{ \
echo "LIBGIT2_IMG and LIBGIT2_TAG must match in both Makefile and Dockerfile"; \
exit 1; \
}
endif
ifneq ($(shell grep -o 'LIBGIT2_TAG ?= \w.*' Makefile | cut -d ' ' -f 3), $(shell grep -o "LIBGIT2_TAG=.*" tests/fuzz/oss_fuzz_prebuild.sh | sed 's;LIBGIT2_TAG="$${LIBGIT2_TAG:-;;g' | sed 's;}";;g'))
@{ \
echo "LIBGIT2_TAG must match in both Makefile and tests/fuzz/oss_fuzz_prebuild.sh"; \
exit 1; \
}
endif
@if [ ! "$$(git status --porcelain --untracked-files=no)" = "" ]; then \
echo "working directory is dirty:"; \
git --no-pager diff; \
@ -236,7 +199,7 @@ rm -rf $$TMP_DIR ;\
endef
# Build fuzzers used by oss-fuzz.
fuzz-build: $(LIBGIT2)
fuzz-build:
rm -rf $(shell pwd)/build/fuzz/
mkdir -p $(shell pwd)/build/fuzz/out/
@ -260,15 +223,3 @@ fuzz-native:
KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) \
FUZZ_TIME=$(FUZZ_TIME) \
./tests/fuzz/native_go_run.sh
# Creates an env file that can be used to load all source-controller's dependencies
# this is handy when you want to run adhoc debug sessions on tests or start the
# controller in a new debug session.
env: $(LIBGIT2)
echo 'GO_ENABLED="1"' > $(BUILD_DIR)/.env
echo 'PKG_CONFIG_PATH="$(PKG_CONFIG_PATH)"' >> $(BUILD_DIR)/.env
echo 'LIBRARY_PATH="$(LIBRARY_PATH)"' >> $(BUILD_DIR)/.env
echo 'CGO_CFLAGS="$(CGO_CFLAGS)"' >> $(BUILD_DIR)/.env
echo 'CGO_LDFLAGS="$(CGO_LDFLAGS)"' >> $(BUILD_DIR)/.env
echo 'KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS)' >> $(BUILD_DIR)/.env
echo 'GIT_CONFIG_GLOBAL=/dev/null' >> $(BUILD_DIR)/.env

View File

@ -1,24 +1,8 @@
apiVersion: source.toolkit.fluxcd.io/v1beta1
kind: GitRepository
metadata:
name: large-repo-go-git
name: large-repo
spec:
gitImplementation: go-git
interval: 10m
timeout: 2m
url: https://github.com/hashgraph/hedera-mirror-node.git
ref:
branch: main
ignore: |
/*
!/charts
---
apiVersion: source.toolkit.fluxcd.io/v1beta1
kind: GitRepository
metadata:
name: large-repo-libgit2
spec:
gitImplementation: libgit2
interval: 10m
timeout: 2m
url: https://github.com/hashgraph/hedera-mirror-node.git

View File

@ -45,7 +45,6 @@ import (
"github.com/fluxcd/pkg/apis/meta"
"github.com/fluxcd/pkg/git"
"github.com/fluxcd/pkg/git/gogit"
"github.com/fluxcd/pkg/git/libgit2"
"github.com/fluxcd/pkg/git/repository"
"github.com/fluxcd/pkg/runtime/conditions"
helper "github.com/fluxcd/pkg/runtime/controller"
@ -116,9 +115,6 @@ type GitRepositoryReconciler struct {
Storage *Storage
ControllerName string
// Libgit2TransportInitialized lets the reconciler know whether
// libgit2 transport was intialized successfully.
Libgit2TransportInitialized func() bool
requeueDependency time.Duration
features map[string]bool
@ -423,18 +419,6 @@ func (r *GitRepositoryReconciler) reconcileStorage(ctx context.Context,
// change, it short-circuits the whole reconciliation with an early return.
func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
obj *sourcev1.GitRepository, commit *git.Commit, includes *artifactSet, dir string) (sreconcile.Result, error) {
gitImplementation := obj.Spec.GitImplementation
if goGitOnly, _ := r.features[features.ForceGoGitImplementation]; goGitOnly {
gitImplementation = sourcev1.GoGitImplementation
}
// Exit early, if we need to use libgit2 AND managed transport hasn't been intialized.
if !r.Libgit2TransportInitialized() && gitImplementation == sourcev1.LibGit2Implementation {
return sreconcile.ResultEmpty, serror.NewStalling(
errors.New("libgit2 managed transport not initialized"), "Libgit2TransportNotEnabled",
)
}
// Remove previously failed source verification status conditions. The
// failing verification should be recalculated. But an existing successful
// verification need not be removed as it indicates verification of previous
@ -505,7 +489,7 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
optimizedClone = true
}
c, err := r.gitCheckout(ctx, obj, authOpts, dir, optimizedClone, gitImplementation)
c, err := r.gitCheckout(ctx, obj, authOpts, dir, optimizedClone)
if err != nil {
return sreconcile.ResultEmpty, err
}
@ -539,7 +523,7 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
// If we can't skip the reconciliation, checkout again without any
// optimization.
c, err := r.gitCheckout(ctx, obj, authOpts, dir, false, gitImplementation)
c, err := r.gitCheckout(ctx, obj, authOpts, dir, false)
if err != nil {
return sreconcile.ResultEmpty, err
}
@ -732,7 +716,7 @@ func (r *GitRepositoryReconciler) reconcileInclude(ctx context.Context,
// performs a git checkout.
func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context,
obj *sourcev1.GitRepository, authOpts *git.AuthOptions, dir string,
optimized bool, gitImplementation string) (*git.Commit, error) {
optimized bool) (*git.Commit, error) {
// Configure checkout strategy.
cloneOpts := repository.CloneOptions{
RecurseSubmodules: obj.Spec.RecurseSubmodules,
@ -757,28 +741,15 @@ func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context,
gitCtx, cancel := context.WithTimeout(ctx, obj.Spec.Timeout.Duration)
defer cancel()
var gitReader repository.Reader
var err error
switch gitImplementation {
case sourcev1.LibGit2Implementation:
clientOpts := []libgit2.ClientOption{libgit2.WithDiskStorage()}
if authOpts.Transport == git.HTTP {
clientOpts = append(clientOpts, libgit2.WithInsecureCredentialsOverHTTP())
}
gitReader, err = libgit2.NewClient(dir, authOpts, clientOpts...)
case sourcev1.GoGitImplementation:
clientOpts := []gogit.ClientOption{gogit.WithDiskStorage()}
if authOpts.Transport == git.HTTP {
clientOpts = append(clientOpts, gogit.WithInsecureCredentialsOverHTTP())
}
gitReader, err = gogit.NewClient(dir, authOpts, clientOpts...)
default:
err = fmt.Errorf("invalid Git implementation: %s", gitImplementation)
clientOpts := []gogit.ClientOption{gogit.WithDiskStorage()}
if authOpts.Transport == git.HTTP {
clientOpts = append(clientOpts, gogit.WithInsecureCredentialsOverHTTP())
}
gitReader, err := gogit.NewClient(dir, authOpts, clientOpts...)
if err != nil {
e := serror.NewGeneric(
fmt.Errorf("failed to create Git client for implementation '%s': %w", gitImplementation, err),
fmt.Errorf("failed to create Git client: %w", err),
sourcev1.GitOperationFailedReason,
)
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())

View File

@ -57,7 +57,6 @@ import (
"github.com/fluxcd/pkg/testserver"
"github.com/fluxcd/pkg/git"
"github.com/fluxcd/pkg/git/libgit2/transport"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
serror "github.com/fluxcd/source-controller/internal/error"
"github.com/fluxcd/source-controller/internal/features"
@ -145,14 +144,6 @@ Oomb3gD/TRf/nAdVED+k81GdLzciYdUGtI71/qI47G0nMBluLRE=
`
)
var (
testGitImplementations = []string{sourcev1.GoGitImplementation, sourcev1.LibGit2Implementation}
)
func mockTransportNotInitialized() bool {
return false
}
func TestGitRepositoryReconciler_Reconcile(t *testing.T) {
g := NewWithT(t)
@ -240,15 +231,14 @@ func TestGitRepositoryReconciler_reconcileSource_authStrategy(t *testing.T) {
}
tests := []struct {
name string
skipForImplementation string
protocol string
server options
secret *corev1.Secret
beforeFunc func(obj *sourcev1.GitRepository)
want sreconcile.Result
wantErr bool
assertConditions []metav1.Condition
name string
protocol string
server options
secret *corev1.Secret
beforeFunc func(obj *sourcev1.GitRepository)
want sreconcile.Result
wantErr bool
assertConditions []metav1.Condition
}{
{
name: "HTTP without secretRef makes ArtifactOutdated=True",
@ -310,9 +300,8 @@ func TestGitRepositoryReconciler_reconcileSource_authStrategy(t *testing.T) {
},
},
{
name: "HTTPS with invalid CAFile secret makes CheckoutFailed=True and returns error",
skipForImplementation: sourcev1.LibGit2Implementation,
protocol: "https",
name: "HTTPS with invalid CAFile secret makes CheckoutFailed=True and returns error",
protocol: "https",
server: options{
publicKey: tlsPublicKey,
privateKey: tlsPrivateKey,
@ -339,31 +328,6 @@ func TestGitRepositoryReconciler_reconcileSource_authStrategy(t *testing.T) {
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.GitOperationFailedReason, "x509: "),
},
},
{
name: "HTTPS with invalid CAFile secret makes CheckoutFailed=True and returns error",
skipForImplementation: sourcev1.GoGitImplementation,
protocol: "https",
server: options{
publicKey: tlsPublicKey,
privateKey: tlsPrivateKey,
ca: tlsCA,
},
secret: &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "invalid-ca",
},
Data: map[string][]byte{
"caFile": []byte("invalid"),
},
},
beforeFunc: func(obj *sourcev1.GitRepository) {
obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "invalid-ca"}
},
wantErr: true,
assertConditions: []metav1.Condition{
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.GitOperationFailedReason, "failed to checkout and determine revision: unable to fetch-connect to remote '<url>': PEM CA bundle could not be appended to x509 certificate pool"),
},
},
{
name: "SSH with private key secret makes ArtifactOutdated=True",
protocol: "ssh",
@ -503,82 +467,36 @@ func TestGitRepositoryReconciler_reconcileSource_authStrategy(t *testing.T) {
Storage: testStorage,
features: map[string]bool{
features.OptimizedGitClones: true,
// Ensure that both implementations are tested.
features.ForceGoGitImplementation: false,
},
Libgit2TransportInitialized: transport.Enabled,
}
for _, i := range testGitImplementations {
t.Run(i, func(t *testing.T) {
g := NewWithT(t)
t.Run(sourcev1.GoGitImplementation, func(t *testing.T) {
g := NewWithT(t)
if tt.skipForImplementation == i {
t.Skipf("Skipped for Git implementation %q", i)
}
tmpDir := t.TempDir()
tmpDir := t.TempDir()
obj := obj.DeepCopy()
obj := obj.DeepCopy()
obj.Spec.GitImplementation = i
head, _ := localRepo.Head()
assertConditions := tt.assertConditions
for k := range assertConditions {
assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "<commit>", head.Hash().String())
assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "<url>", obj.Spec.URL)
}
head, _ := localRepo.Head()
assertConditions := tt.assertConditions
for k := range assertConditions {
assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "<commit>", head.Hash().String())
assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "<url>", obj.Spec.URL)
}
var commit git.Commit
var includes artifactSet
var commit git.Commit
var includes artifactSet
got, err := r.reconcileSource(context.TODO(), obj, &commit, &includes, tmpDir)
g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.assertConditions))
g.Expect(err != nil).To(Equal(tt.wantErr))
g.Expect(got).To(Equal(tt.want))
g.Expect(commit).ToNot(BeNil())
})
}
got, err := r.reconcileSource(context.TODO(), obj, &commit, &includes, tmpDir)
g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.assertConditions))
g.Expect(err != nil).To(Equal(tt.wantErr))
g.Expect(got).To(Equal(tt.want))
g.Expect(commit).ToNot(BeNil())
})
})
}
}
func TestGitRepositoryReconciler_reconcileSource_libgit2TransportUninitialized(t *testing.T) {
g := NewWithT(t)
r := &GitRepositoryReconciler{
Client: fakeclient.NewClientBuilder().WithScheme(runtime.NewScheme()).Build(),
EventRecorder: record.NewFakeRecorder(32),
Storage: testStorage,
features: map[string]bool{
features.ForceGoGitImplementation: false,
},
Libgit2TransportInitialized: mockTransportNotInitialized,
}
obj := &sourcev1.GitRepository{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "libgit2-transport",
},
Spec: sourcev1.GitRepositorySpec{
Interval: metav1.Duration{Duration: interval},
Timeout: &metav1.Duration{Duration: timeout},
Reference: &sourcev1.GitRepositoryRef{
Branch: git.DefaultBranch,
},
GitImplementation: sourcev1.LibGit2Implementation,
},
}
tmpDir := t.TempDir()
var commit git.Commit
var includes artifactSet
_, err := r.reconcileSource(ctx, obj, &commit, &includes, tmpDir)
g.Expect(err).To(HaveOccurred())
g.Expect(err).To(BeAssignableToTypeOf(&serror.Stalling{}))
g.Expect(err.Error()).To(Equal("libgit2 managed transport not initialized"))
}
func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T) {
g := NewWithT(t)
@ -586,14 +504,13 @@ func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T)
tags := []string{"non-semver-tag", "v0.1.0", "0.2.0", "v0.2.1", "v1.0.0-alpha", "v1.1.0", "v2.0.0"}
tests := []struct {
name string
skipForImplementation string
reference *sourcev1.GitRepositoryRef
beforeFunc func(obj *sourcev1.GitRepository, latestRev string)
want sreconcile.Result
wantErr bool
wantRevision string
wantArtifactOutdated bool
name string
reference *sourcev1.GitRepositoryRef
beforeFunc func(obj *sourcev1.GitRepository, latestRev string)
want sreconcile.Result
wantErr bool
wantRevision string
wantArtifactOutdated bool
}{
{
name: "Nil reference (default branch)",
@ -620,8 +537,7 @@ func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T)
wantArtifactOutdated: true,
},
{
name: "Branch commit",
skipForImplementation: sourcev1.LibGit2Implementation,
name: "Branch commit",
reference: &sourcev1.GitRepositoryRef{
Branch: "staging",
Commit: "<commit>",
@ -630,17 +546,6 @@ func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T)
wantRevision: "staging/<commit>",
wantArtifactOutdated: true,
},
{
name: "Branch commit",
skipForImplementation: sourcev1.GoGitImplementation,
reference: &sourcev1.GitRepositoryRef{
Branch: "staging",
Commit: "<commit>",
},
want: sreconcile.ResultSuccess,
wantRevision: "HEAD/<commit>",
wantArtifactOutdated: true,
},
{
name: "SemVer",
reference: &sourcev1.GitRepositoryRef{
@ -738,10 +643,7 @@ func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T)
Storage: testStorage,
features: map[string]bool{
features.OptimizedGitClones: true,
// Ensure that both implementations are tested.
features.ForceGoGitImplementation: false,
},
Libgit2TransportInitialized: transport.Enabled,
}
for _, tt := range tests {
@ -762,38 +664,30 @@ func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T)
obj.Spec.Reference.Commit = headRef.Hash().String()
}
for _, i := range testGitImplementations {
t.Run(i, func(t *testing.T) {
g := NewWithT(t)
t.Run(sourcev1.GoGitImplementation, func(t *testing.T) {
g := NewWithT(t)
if tt.skipForImplementation == i {
t.Skipf("Skipped for Git implementation %q", i)
}
tmpDir := t.TempDir()
obj := obj.DeepCopy()
tmpDir := t.TempDir()
if tt.beforeFunc != nil {
tt.beforeFunc(obj, headRef.Hash().String())
}
obj := obj.DeepCopy()
obj.Spec.GitImplementation = i
if tt.beforeFunc != nil {
tt.beforeFunc(obj, headRef.Hash().String())
}
var commit git.Commit
var includes artifactSet
got, err := r.reconcileSource(ctx, obj, &commit, &includes, tmpDir)
if err != nil {
println(err.Error())
}
g.Expect(err != nil).To(Equal(tt.wantErr))
g.Expect(got).To(Equal(tt.want))
if tt.wantRevision != "" && !tt.wantErr {
revision := strings.ReplaceAll(tt.wantRevision, "<commit>", headRef.Hash().String())
g.Expect(commit.String()).To(Equal(revision))
g.Expect(conditions.IsTrue(obj, sourcev1.ArtifactOutdatedCondition)).To(Equal(tt.wantArtifactOutdated))
}
})
}
var commit git.Commit
var includes artifactSet
got, err := r.reconcileSource(ctx, obj, &commit, &includes, tmpDir)
if err != nil {
println(err.Error())
}
g.Expect(err != nil).To(Equal(tt.wantErr))
g.Expect(got).To(Equal(tt.want))
if tt.wantRevision != "" && !tt.wantErr {
revision := strings.ReplaceAll(tt.wantRevision, "<commit>", headRef.Hash().String())
g.Expect(commit.String()).To(Equal(revision))
g.Expect(conditions.IsTrue(obj, sourcev1.ArtifactOutdatedCondition)).To(Equal(tt.wantArtifactOutdated))
}
})
})
}
}
@ -1605,11 +1499,10 @@ func TestGitRepositoryReconciler_ConditionsUpdate(t *testing.T) {
builder := fakeclient.NewClientBuilder().WithScheme(testEnv.GetScheme()).WithObjects(obj)
r := &GitRepositoryReconciler{
Client: builder.Build(),
EventRecorder: record.NewFakeRecorder(32),
Storage: testStorage,
features: features.FeatureGates(),
Libgit2TransportInitialized: transport.Enabled,
Client: builder.Build(),
EventRecorder: record.NewFakeRecorder(32),
Storage: testStorage,
features: features.FeatureGates(),
}
key := client.ObjectKeyFromObject(obj)

View File

@ -37,7 +37,6 @@ import (
ctrl "sigs.k8s.io/controller-runtime"
dcontext "github.com/distribution/distribution/v3/context"
"github.com/fluxcd/pkg/git/libgit2/transport"
"github.com/fluxcd/pkg/runtime/controller"
"github.com/fluxcd/pkg/runtime/testenv"
"github.com/fluxcd/pkg/testserver"
@ -48,7 +47,6 @@ import (
dockerRegistry "github.com/distribution/distribution/v3/registry"
_ "github.com/distribution/distribution/v3/registry/auth/htpasswd"
_ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory"
git2go "github.com/libgit2/git2go/v34"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
"github.com/fluxcd/source-controller/internal/cache"
@ -203,8 +201,6 @@ func setupRegistryServer(ctx context.Context, workspaceDir string, opts registry
}
func TestMain(m *testing.M) {
mustHaveNoThreadSupport()
initTestTLS()
utilruntime.Must(sourcev1.AddToScheme(scheme.Scheme))
@ -237,10 +233,6 @@ func TestMain(m *testing.M) {
panic(fmt.Sprintf("Failed to create a test registry server: %v", err))
}
if err = transport.InitManagedTransport(); err != nil {
panic(fmt.Sprintf("Failed to initialize libgit2 managed transport: %v", err))
}
if err := (&GitRepositoryReconciler{
Client: testEnv,
EventRecorder: record.NewFakeRecorder(32),
@ -248,10 +240,7 @@ func TestMain(m *testing.M) {
Storage: testStorage,
features: map[string]bool{
features.OptimizedGitClones: true,
// Ensure that both implementations are used during tests.
features.ForceGoGitImplementation: false,
},
Libgit2TransportInitialized: transport.Enabled,
}).SetupWithManager(testEnv); err != nil {
panic(fmt.Sprintf("Failed to start GitRepositoryReconciler: %v", err))
}
@ -378,22 +367,3 @@ func randStringRunes(n int) string {
func int64p(i int64) *int64 {
return &i
}
// This provides a regression assurance for image-automation-controller/#339.
// Validates that:
// - libgit2 was built with no support for threads.
// - git2go accepts libgit2 built with no support for threads.
//
// The logic below does the validation of the former, whilst
// referring to git2go forces its init() execution, which is
// where any validation to that effect resides.
//
// git2go does not support threadless libgit2 by default,
// hence a fork is being used which disables such validation.
//
// TODO: extract logic into pkg.
func mustHaveNoThreadSupport() {
if git2go.Features()&git2go.FeatureThreads != 0 {
panic("libgit2 must not be build with thread support")
}
}

View File

@ -421,7 +421,9 @@ string
<td>
<em>(Optional)</em>
<p>GitImplementation specifies which Git client library implementation to
use. Defaults to &lsquo;go-git&rsquo;, valid values are (&lsquo;go-git&rsquo;, &lsquo;libgit2&rsquo;).</p>
use. Defaults to &lsquo;go-git&rsquo;, valid values are (&lsquo;go-git&rsquo;, &lsquo;libgit2&rsquo;).
Deprecated: gitImplementation is deprecated now that &lsquo;go-git&rsquo; is the
only supported implementation.</p>
</td>
</tr>
<tr>
@ -1845,7 +1847,9 @@ string
<td>
<em>(Optional)</em>
<p>GitImplementation specifies which Git client library implementation to
use. Defaults to &lsquo;go-git&rsquo;, valid values are (&lsquo;go-git&rsquo;, &lsquo;libgit2&rsquo;).</p>
use. Defaults to &lsquo;go-git&rsquo;, valid values are (&lsquo;go-git&rsquo;, &lsquo;libgit2&rsquo;).
Deprecated: gitImplementation is deprecated now that &lsquo;go-git&rsquo; is the
only supported implementation.</p>
</td>
</tr>
<tr>

13
go.mod
View File

@ -4,14 +4,6 @@ go 1.18
replace github.com/fluxcd/source-controller/api => ./api
// Flux has its own git2go fork to enable changes in behaviour for improved
// reliability.
//
// For more information refer to:
// - fluxcd/image-automation-controller/#339.
// - libgit2/git2go#918.
replace github.com/libgit2/git2go/v34 => github.com/fluxcd/git2go/v34 v34.0.0
// Fix CVE-2022-1996 (for v2, Go Modules incompatible)
replace github.com/emicklei/go-restful => github.com/emicklei/go-restful v2.16.0+incompatible
@ -33,8 +25,7 @@ require (
github.com/fluxcd/pkg/apis/event v0.2.0
github.com/fluxcd/pkg/apis/meta v0.18.0
github.com/fluxcd/pkg/git v0.7.0
github.com/fluxcd/pkg/git/gogit v0.2.0
github.com/fluxcd/pkg/git/libgit2 v0.3.0
github.com/fluxcd/pkg/git/gogit v0.2.1-0.20221128153352-fbb8b9f29f38
github.com/fluxcd/pkg/gittestserver v0.8.0
github.com/fluxcd/pkg/helmtestserver v0.10.0
github.com/fluxcd/pkg/lockedfile v0.1.0
@ -52,7 +43,6 @@ require (
github.com/google/go-containerregistry v0.12.1
github.com/google/go-containerregistry/pkg/authn/k8schain v0.0.0-20221114162634-781782aa2757
github.com/google/uuid v1.3.0
github.com/libgit2/git2go/v34 v34.0.0
github.com/minio/minio-go/v7 v7.0.43
github.com/onsi/gomega v1.24.1
github.com/otiai10/copy v1.9.0
@ -178,7 +168,6 @@ require (
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/fluxcd/gitkit v0.6.0 // indirect
github.com/fluxcd/pkg/apis/acl v0.1.0 // indirect
github.com/fluxcd/pkg/http/transport v0.1.0 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/fullstorydev/grpcurl v1.8.7 // indirect
github.com/go-chi/chi v4.1.2+incompatible // indirect

12
go.sum
View File

@ -502,8 +502,6 @@ github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw
github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0=
github.com/fluxcd/git2go/v34 v34.0.0 h1:08PEpdbsLO4sUTvFKuxmt6xBowaqObro9veChBWFwa8=
github.com/fluxcd/git2go/v34 v34.0.0/go.mod h1:blVco2jDAw6YTXkErMMqzHLcAjKkwF0aWIRHBqiJkZ0=
github.com/fluxcd/gitkit v0.6.0 h1:iNg5LTx6ePo+Pl0ZwqHTAkhbUHxGVSY3YCxCdw7VIFg=
github.com/fluxcd/gitkit v0.6.0/go.mod h1:svOHuKi0fO9HoawdK4HfHAJJseZDHHjk7I3ihnCIqNo=
github.com/fluxcd/go-git/v5 v5.0.0-20221104190732-329fd6659b10 h1:au798417R1iWtgcgKe3Dg495mexQmuxelL+NebAtexE=
@ -516,16 +514,12 @@ github.com/fluxcd/pkg/apis/meta v0.18.0 h1:s0LeulWcQ4DxVX6805vgDTxlA6bAYk+Lq1QHS
github.com/fluxcd/pkg/apis/meta v0.18.0/go.mod h1:pYvXRFi1UKNNrGR34jw3uqOnMXw9X6dTkML8j5Z7tis=
github.com/fluxcd/pkg/git v0.7.0 h1:sQHRpFMcOzEdqlyGMjFv2LKMdcoE5xeUr2UcRrsLRG8=
github.com/fluxcd/pkg/git v0.7.0/go.mod h1:3deiLPws4DSQ3hqwtQd7Dt66GXTN/4RcT/yHAljXaHo=
github.com/fluxcd/pkg/git/gogit v0.2.0 h1:vhFzk2Pky4tDZwisx8+26YZumRDPxERnkV8l2dbLSoo=
github.com/fluxcd/pkg/git/gogit v0.2.0/go.mod h1:d1RIwl6DVdU8/2dBIhw6n7GNokIKqs+b9cKc/8tz7ew=
github.com/fluxcd/pkg/git/libgit2 v0.3.0 h1:fgw/Cm7Id2AxanMab3qU+UuvdnhfwRMbbWF27xu413o=
github.com/fluxcd/pkg/git/libgit2 v0.3.0/go.mod h1:JHeZBeq+jbUpRpP5OoEYXua0EF5dyqAlbEfYcXY228A=
github.com/fluxcd/pkg/git/gogit v0.2.1-0.20221128153352-fbb8b9f29f38 h1:FGDncVUlMfK7eZiw0m/mWa6CGczGeGomw8jJ0uX2zIw=
github.com/fluxcd/pkg/git/gogit v0.2.1-0.20221128153352-fbb8b9f29f38/go.mod h1:d1RIwl6DVdU8/2dBIhw6n7GNokIKqs+b9cKc/8tz7ew=
github.com/fluxcd/pkg/gittestserver v0.8.0 h1:YrYe63KScKlLxx0GAiQthx2XqHDx0vKitIIx4JnDtIo=
github.com/fluxcd/pkg/gittestserver v0.8.0/go.mod h1:/LI/xKMrnQbIsTDnTyABQ71iaYhFIZ8fb4cvY7WAlBU=
github.com/fluxcd/pkg/helmtestserver v0.10.0 h1:/cnx1D6/cln7v06iXZMLeshl82mV1wv3OVZoBE9Bbm4=
github.com/fluxcd/pkg/helmtestserver v0.10.0/go.mod h1:sJwcAw/4ENiJO6i0HJSMV8ypp7k+U4amDwH7RJ57ScY=
github.com/fluxcd/pkg/http/transport v0.1.0 h1:tzRprpCAUkPFlF/9VMLSbw95jcAKCbimDC//VULQXnY=
github.com/fluxcd/pkg/http/transport v0.1.0/go.mod h1:BNXQYy3ifMTc3DaOu3GlIR6j5DtTwMEyvTr2EA7XP8w=
github.com/fluxcd/pkg/lockedfile v0.1.0 h1:YsYFAkd6wawMCcD74ikadAKXA4s2sukdxrn7w8RB5eo=
github.com/fluxcd/pkg/lockedfile v0.1.0/go.mod h1:EJLan8t9MiOcgTs8+puDjbE6I/KAfHbdvIy9VUgIjm8=
github.com/fluxcd/pkg/masktoken v0.2.0 h1:HoSPTk4l1fz5Fevs2vVRvZGru33blfMwWSZKsHdfG/0=
@ -1736,7 +1730,6 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200930160638-afb6bcd081ae/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
@ -1975,7 +1968,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

View File

@ -141,9 +141,7 @@ kubectl -n source-system wait helmchart/helmchart-bucket --for=condition=ready -
echo "Run large Git repo tests"
kubectl -n source-system apply -f "${ROOT_DIR}/config/testdata/git/large-repo.yaml"
kubectl -n source-system wait gitrepository/large-repo-go-git --for=condition=ready --timeout=2m15s
kubectl -n source-system wait gitrepository/large-repo-libgit2 --for=condition=ready --timeout=2m15s
kubectl -n source-system wait gitrepository/large-repo --for=condition=ready --timeout=2m15s
echo "Run HelmChart from OCI registry tests"
kubectl -n source-system apply -f "${ROOT_DIR}/config/testdata/helmchart-from-oci/source.yaml"

19
main.go
View File

@ -34,7 +34,6 @@ import (
ctrl "sigs.k8s.io/controller-runtime"
"github.com/fluxcd/pkg/git"
"github.com/fluxcd/pkg/git/libgit2/transport"
"github.com/fluxcd/pkg/runtime/client"
helper "github.com/fluxcd/pkg/runtime/controller"
"github.com/fluxcd/pkg/runtime/events"
@ -204,20 +203,12 @@ func main() {
}
storage := mustInitStorage(storagePath, storageAdvAddr, artifactRetentionTTL, artifactRetentionRecords, setupLog)
if gogitOnly, _ := features.Enabled(features.ForceGoGitImplementation); !gogitOnly {
if err = transport.InitManagedTransport(); err != nil {
// Log the error, but don't exit so as to not block reconcilers that are healthy.
setupLog.Error(err, "unable to initialize libgit2 managed transport")
}
}
if err = (&controllers.GitRepositoryReconciler{
Client: mgr.GetClient(),
EventRecorder: eventRecorder,
Metrics: metricsH,
Storage: storage,
ControllerName: controllerName,
Libgit2TransportInitialized: transport.Enabled,
Client: mgr.GetClient(),
EventRecorder: eventRecorder,
Metrics: metricsH,
Storage: storage,
ControllerName: controllerName,
}).SetupWithManagerAndOptions(mgr, controllers.GitRepositoryReconcilerOptions{
MaxConcurrentReconciles: concurrent,
DependencyRequeueInterval: requeueDependency,