fluxcd
/
source-controller
mirror of https://github.com/fluxcd/source-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #977 from fluxcd/remove-libgit2 

Remove libgit2 and git2go from codebase
This commit is contained in:
Paulo Gomes 2022-12-12 15:54:40 +00:00 committed by GitHub
parent aad4060461 cc75764412
commit f7a64b1b81
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 128 additions and 2113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.dockerignore
View File

@ -1 +1 @@
build/libgit2/ build/

1232
ATTRIBUTIONS.md
View File

File diff suppressed because it is too large Load Diff

28
DEVELOPMENT.md
View File

@ -13,27 +13,6 @@ There are a number of dependencies required to be able to run the controller and
- [Install Docker](https://docs.docker.com/engine/install/) - [Install Docker](https://docs.docker.com/engine/install/)
- (Optional) [Install Kubebuilder](https://book.kubebuilder.io/quick-start.html#installation) - (Optional) [Install Kubebuilder](https://book.kubebuilder.io/quick-start.html#installation)
The [libgit2](https://libgit2.org/) dependency is now automatically managed by the Makefile logic.
However, it depends on [pkg-config](https://freedesktop.org/wiki/Software/pkg-config/) being installed:
### macOS
```console
$ # Ensure pkg-config is installed
$ brew install pkg-config
```
### Linux
```console
$ # Ensure pkg-config is installed
$ pacman -S pkgconf
```
**Note:** Example shown is for Arch Linux, but likewise procedure can be
followed using any other package manager. Some distributions may have slight
variation of package names (e.g. `apt install -y pkg-config`).
In addition to the above, the following dependencies are also used by some of the `make` targets: In addition to the above, the following dependencies are also used by some of the `make` targets:
- `controller-gen` (v0.7.0) - `controller-gen` (v0.7.0)
@ -149,18 +128,11 @@ Create a `.vscode/launch.json` file:
"type": "go", "type": "go",
"request": "launch", "request": "launch",
"mode": "auto", "mode": "auto",
"envFile": "${workspaceFolder}/build/.env",
"program": "${workspaceFolder}/main.go" "program": "${workspaceFolder}/main.go"
} }
] ]
} }
``` ```
Create the environment file containing details on how to load
`libgit2` dependencies:
```bash
make env
```
Start debugging by either clicking `Run` > `Start Debugging` or using Start debugging by either clicking `Run` > `Start Debugging` or using
the relevant shortcut. the relevant shortcut.

40
Dockerfile
View File

@ -2,11 +2,6 @@ ARG BASE_VARIANT=alpine
ARG GO_VERSION=1.19 ARG GO_VERSION=1.19
ARG XX_VERSION=1.1.2 ARG XX_VERSION=1.1.2
ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2-only
ARG LIBGIT2_TAG=v0.4.0
FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} AS libgit2-libs
FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as gostable FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as gostable
@ -17,12 +12,18 @@ FROM gostable AS go-linux
# These will be used at current arch to yield execute the cross compilations. # These will be used at current arch to yield execute the cross compilations.
FROM go-${TARGETOS} AS build-base FROM go-${TARGETOS} AS build-base
RUN apk add --no-cache clang lld pkgconfig RUN apk add --no-cache clang lld
COPY --from=xx / / COPY --from=xx / /
# build-go-mod can still be cached at build platform architecture. # build-go-mod can still be cached at build platform architecture.
FROM build-base as build-go-mod FROM build-base as build
ARG TARGETPLATFORM
# Some dependencies have to installed
# for the target platform: https://github.com/tonistiigi/xx#go--cgo
RUN xx-apk add musl-dev gcc clang lld
# Configure workspace # Configure workspace
WORKDIR /workspace WORKDIR /workspace
@ -37,21 +38,6 @@ COPY go.sum go.sum
# Cache modules # Cache modules
RUN go mod download RUN go mod download
# Build stage install per target platform
# dependency and effectively cross compile the application.
FROM build-go-mod as build
ARG TARGETPLATFORM
COPY --from=libgit2-libs /usr/local/ /usr/local/
# Some dependencies have to installed
# for the target platform: https://github.com/tonistiigi/xx#go--cgo
RUN xx-apk add musl-dev gcc clang lld
WORKDIR /workspace
# Copy source code # Copy source code
COPY main.go main.go COPY main.go main.go
COPY controllers/ controllers/ COPY controllers/ controllers/
@ -60,11 +46,13 @@ COPY internal/ internal/
ARG TARGETPLATFORM ARG TARGETPLATFORM
ARG TARGETARCH ARG TARGETARCH
# Reasons why CGO is in use:
# - The SHA1 implementation (sha1cd) used by go-git depends on CGO for
# performance reasons. See: https://github.com/pjbgf/sha1cd/issues/15
ENV CGO_ENABLED=1 ENV CGO_ENABLED=1
# Instead of using xx-go, (cross) compile with vanilla go leveraging musl tool chain. RUN export CGO_LDFLAGS="-static -fuse-ld=lld" && \
RUN export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig" && \
export CGO_LDFLAGS="$(pkg-config --static --libs --cflags libgit2) -static -fuse-ld=lld" && \
xx-go build \ xx-go build \
-ldflags "-s -w" \ -ldflags "-s -w" \
-tags 'netgo,osusergo,static_build' \ -tags 'netgo,osusergo,static_build' \
@ -73,7 +61,6 @@ RUN export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig" && \
# Ensure that the binary was cross-compiled correctly to the target platform. # Ensure that the binary was cross-compiled correctly to the target platform.
RUN xx-verify --static /source-controller RUN xx-verify --static /source-controller
FROM alpine:3.16 FROM alpine:3.16
ARG TARGETPLATFORM ARG TARGETPLATFORM
@ -82,7 +69,6 @@ RUN apk --no-cache add ca-certificates \
# Copy over binary from build # Copy over binary from build
COPY --from=build /source-controller /usr/local/bin/ COPY --from=build /source-controller /usr/local/bin/
COPY ATTRIBUTIONS.md /
USER 65534:65534 USER 65534:65534
ENTRYPOINT [ "source-controller" ] ENTRYPOINT [ "source-controller" ]

66
Makefile
View File

@ -2,10 +2,6 @@
IMG ?= fluxcd/source-controller IMG ?= fluxcd/source-controller
TAG ?= latest TAG ?= latest
# Base image used to build the Go binary
LIBGIT2_IMG ?= ghcr.io/fluxcd/golang-with-libgit2-only
LIBGIT2_TAG ?= v0.4.0
# Allows for defining additional Go test args, e.g. '-tags integration'. # Allows for defining additional Go test args, e.g. '-tags integration'.
GO_TEST_ARGS ?= -race GO_TEST_ARGS ?= -race
@ -39,14 +35,6 @@ ENVTEST_BIN_VERSION ?= 1.24.0
# each fuzzer should run for. # each fuzzer should run for.
FUZZ_TIME ?= 1m FUZZ_TIME ?= 1m
# Caches libgit2 versions per tag, "forcing" rebuild only when needed.
LIBGIT2_PATH := $(BUILD_DIR)/libgit2/$(LIBGIT2_TAG)
LIBGIT2_LIB_PATH := $(LIBGIT2_PATH)/lib
LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.a
export CGO_ENABLED=1
export PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig
export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libgit2 2>/dev/null)
GO_STATIC_FLAGS=-ldflags "-s -w" -tags 'netgo,osusergo,static_build$(addprefix ,,$(GO_TAGS))' GO_STATIC_FLAGS=-ldflags "-s -w" -tags 'netgo,osusergo,static_build$(addprefix ,,$(GO_TAGS))'
# API (doc) generation utilities # API (doc) generation utilities
@ -75,11 +63,11 @@ endif
all: build all: build
build: check-deps $(LIBGIT2) ## Build manager binary build: check-deps ## Build manager binary
go build $(GO_STATIC_FLAGS) -o $(BUILD_DIR)/bin/manager main.go go build $(GO_STATIC_FLAGS) -o $(BUILD_DIR)/bin/manager main.go
KUBEBUILDER_ASSETS?="$(shell $(ENVTEST) --arch=$(ENVTEST_ARCH) use -i $(ENVTEST_KUBERNETES_VERSION) --bin-dir=$(ENVTEST_ASSETS_DIR) -p path)" KUBEBUILDER_ASSETS?="$(shell $(ENVTEST) --arch=$(ENVTEST_ARCH) use -i $(ENVTEST_KUBERNETES_VERSION) --bin-dir=$(ENVTEST_ASSETS_DIR) -p path)"
test: $(LIBGIT2) install-envtest test-api check-deps ## Run all tests test: install-envtest test-api check-deps ## Run all tests
HTTPS_PROXY="" HTTP_PROXY="" \ HTTPS_PROXY="" HTTP_PROXY="" \
KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) \ KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) \
GIT_CONFIG_GLOBAL=/dev/null \ GIT_CONFIG_GLOBAL=/dev/null \
@ -88,7 +76,7 @@ test: $(LIBGIT2) install-envtest test-api check-deps ## Run all tests
$(GO_TEST_ARGS) \ $(GO_TEST_ARGS) \
-coverprofile cover.out -coverprofile cover.out
test-ctrl: $(LIBGIT2) install-envtest test-api check-deps ## Run controller tests test-ctrl: install-envtest test-api check-deps ## Run controller tests
HTTPS_PROXY="" HTTP_PROXY="" \ HTTPS_PROXY="" HTTP_PROXY="" \
KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) \ KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) \
GIT_CONFIG_GLOBAL=/dev/null \ GIT_CONFIG_GLOBAL=/dev/null \
@ -105,7 +93,7 @@ endif
test-api: ## Run api tests test-api: ## Run api tests
cd api; go test $(GO_TEST_ARGS) ./... -coverprofile cover.out cd api; go test $(GO_TEST_ARGS) ./... -coverprofile cover.out
run: $(LIBGIT2) generate fmt vet manifests ## Run against the configured Kubernetes cluster in ~/.kube/config run: generate fmt vet manifests ## Run against the configured Kubernetes cluster in ~/.kube/config
go run $(GO_STATIC_FLAGS) ./main.go go run $(GO_STATIC_FLAGS) ./main.go
install: manifests ## Install CRDs into a cluster install: manifests ## Install CRDs into a cluster
@ -139,7 +127,7 @@ fmt: ## Run go fmt against code
go fmt ./... go fmt ./...
cd api; go fmt ./... cd api; go fmt ./...
vet: $(LIBGIT2) ## Run go vet against code vet: ## Run go vet against code
go vet ./... go vet ./...
cd api; go vet ./... cd api; go vet ./...
@ -148,8 +136,6 @@ generate: controller-gen ## Generate API code
docker-build: ## Build the Docker image docker-build: ## Build the Docker image
docker buildx build \ docker buildx build \
--build-arg LIBGIT2_IMG=$(LIBGIT2_IMG) \
--build-arg LIBGIT2_TAG=$(LIBGIT2_TAG) \
--platform=$(BUILD_PLATFORMS) \ --platform=$(BUILD_PLATFORMS) \
-t $(IMG):$(TAG) \ -t $(IMG):$(TAG) \
$(BUILD_ARGS) . $(BUILD_ARGS) .
@ -182,40 +168,14 @@ install-envtest: setup-envtest ## Download envtest binaries locally.
# setup-envtest sets anything below k8s to 0555 # setup-envtest sets anything below k8s to 0555
chmod -R u+w $(BUILD_DIR)/testbin chmod -R u+w $(BUILD_DIR)/testbin
libgit2: $(LIBGIT2) ## Detect or download libgit2 library
COSIGN = $(GOBIN)/cosign
$(LIBGIT2):
$(call go-install-tool,$(COSIGN),github.com/sigstore/cosign/cmd/cosign@latest)
IMG=$(LIBGIT2_IMG) TAG=$(LIBGIT2_TAG) PATH=$(PATH):$(GOBIN) ./hack/install-libraries.sh
.PHONY: help .PHONY: help
help: ## Display this help menu help: ## Display this help menu
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
update-attributions:
./hack/update-attributions.sh
e2e: e2e:
./hack/ci/e2e.sh ./hack/ci/e2e.sh
verify: update-attributions fmt vet manifests api-docs tidy verify: fmt vet manifests api-docs tidy
ifneq ($(shell grep -o 'LIBGIT2_IMG ?= \w.*' Makefile | cut -d ' ' -f 3):$(shell grep -o 'LIBGIT2_TAG ?= \w.*' Makefile | cut -d ' ' -f 3), \
$(shell grep -o "LIBGIT2_IMG=\w.*" Dockerfile | cut -d'=' -f2):$(shell grep -o "LIBGIT2_TAG=\w.*" Dockerfile | cut -d'=' -f2))
@{ \
echo "LIBGIT2_IMG and LIBGIT2_TAG must match in both Makefile and Dockerfile"; \
exit 1; \
}
endif
ifneq ($(shell grep -o 'LIBGIT2_TAG ?= \w.*' Makefile | cut -d ' ' -f 3), $(shell grep -o "LIBGIT2_TAG=.*" tests/fuzz/oss_fuzz_prebuild.sh | sed 's;LIBGIT2_TAG="$${LIBGIT2_TAG:-;;g' | sed 's;}";;g'))
@{ \
echo "LIBGIT2_TAG must match in both Makefile and tests/fuzz/oss_fuzz_prebuild.sh"; \
exit 1; \
}
endif
@if [ ! "$$(git status --porcelain --untracked-files=no)" = "" ]; then \ @if [ ! "$$(git status --porcelain --untracked-files=no)" = "" ]; then \
echo "working directory is dirty:"; \ echo "working directory is dirty:"; \
git --no-pager diff; \ git --no-pager diff; \
@ -236,7 +196,7 @@ rm -rf $$TMP_DIR ;\
endef endef
# Build fuzzers used by oss-fuzz. # Build fuzzers used by oss-fuzz.
fuzz-build: $(LIBGIT2) fuzz-build:
rm -rf $(shell pwd)/build/fuzz/ rm -rf $(shell pwd)/build/fuzz/
mkdir -p $(shell pwd)/build/fuzz/out/ mkdir -p $(shell pwd)/build/fuzz/out/
@ -260,15 +220,3 @@ fuzz-native:
KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) \ KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) \
FUZZ_TIME=$(FUZZ_TIME) \ FUZZ_TIME=$(FUZZ_TIME) \
./tests/fuzz/native_go_run.sh ./tests/fuzz/native_go_run.sh
# Creates an env file that can be used to load all source-controller's dependencies
# this is handy when you want to run adhoc debug sessions on tests or start the
# controller in a new debug session.
env: $(LIBGIT2)
echo 'GO_ENABLED="1"' > $(BUILD_DIR)/.env
echo 'PKG_CONFIG_PATH="$(PKG_CONFIG_PATH)"' >> $(BUILD_DIR)/.env
echo 'LIBRARY_PATH="$(LIBRARY_PATH)"' >> $(BUILD_DIR)/.env
echo 'CGO_CFLAGS="$(CGO_CFLAGS)"' >> $(BUILD_DIR)/.env
echo 'CGO_LDFLAGS="$(CGO_LDFLAGS)"' >> $(BUILD_DIR)/.env
echo 'KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS)' >> $(BUILD_DIR)/.env
echo 'GIT_CONFIG_GLOBAL=/dev/null' >> $(BUILD_DIR)/.env

2
api/v1beta2/gitrepository_types.go
View File

@ -97,6 +97,8 @@ type GitRepositorySpec struct {
// GitImplementation specifies which Git client library implementation to // GitImplementation specifies which Git client library implementation to
// use. Defaults to 'go-git', valid values are ('go-git', 'libgit2'). // use. Defaults to 'go-git', valid values are ('go-git', 'libgit2').
// Deprecated: gitImplementation is deprecated now that 'go-git' is the
// only supported implementation.
// +kubebuilder:validation:Enum=go-git;libgit2 // +kubebuilder:validation:Enum=go-git;libgit2
// +kubebuilder:default:=go-git // +kubebuilder:default:=go-git
// +optional // +optional

7
config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml
View File

@ -411,9 +411,10 @@ spec:
type: object type: object
gitImplementation: gitImplementation:
default: go-git default: go-git
description: GitImplementation specifies which Git client library description: 'GitImplementation specifies which Git client library
implementation to use. Defaults to 'go-git', valid values are ('go-git', implementation to use. Defaults to ''go-git'', valid values are
'libgit2'). (''go-git'', ''libgit2''). Deprecated: gitImplementation is deprecated
now that ''go-git'' is the only supported implementation.'
enum: enum:
- go-git - go-git
- libgit2 - libgit2

18
config/testdata/git/large-repo.yaml vendored
View File

@ -1,24 +1,8 @@
apiVersion: source.toolkit.fluxcd.io/v1beta1 apiVersion: source.toolkit.fluxcd.io/v1beta1
kind: GitRepository kind: GitRepository
metadata: metadata:
name: large-repo-go-git name: large-repo
spec: spec:
gitImplementation: go-git
interval: 10m
timeout: 2m
url: https://github.com/hashgraph/hedera-mirror-node.git
ref:
branch: main
ignore: |
/*
!/charts
---
apiVersion: source.toolkit.fluxcd.io/v1beta1
kind: GitRepository
metadata:
name: large-repo-libgit2
spec:
gitImplementation: libgit2
interval: 10m interval: 10m
timeout: 2m timeout: 2m
url: https://github.com/hashgraph/hedera-mirror-node.git url: https://github.com/hashgraph/hedera-mirror-node.git

47
controllers/gitrepository_controller.go
View File

@ -45,7 +45,6 @@ import (
"github.com/fluxcd/pkg/apis/meta" "github.com/fluxcd/pkg/apis/meta"
"github.com/fluxcd/pkg/git" "github.com/fluxcd/pkg/git"
"github.com/fluxcd/pkg/git/gogit" "github.com/fluxcd/pkg/git/gogit"
"github.com/fluxcd/pkg/git/libgit2"
"github.com/fluxcd/pkg/git/repository" "github.com/fluxcd/pkg/git/repository"
"github.com/fluxcd/pkg/runtime/conditions" "github.com/fluxcd/pkg/runtime/conditions"
helper "github.com/fluxcd/pkg/runtime/controller" helper "github.com/fluxcd/pkg/runtime/controller"
@ -116,9 +115,6 @@ type GitRepositoryReconciler struct {
Storage *Storage Storage *Storage
ControllerName string ControllerName string
// Libgit2TransportInitialized lets the reconciler know whether
// libgit2 transport was intialized successfully.
Libgit2TransportInitialized func() bool
requeueDependency time.Duration requeueDependency time.Duration
features map[string]bool features map[string]bool
@ -423,18 +419,6 @@ func (r *GitRepositoryReconciler) reconcileStorage(ctx context.Context,
// change, it short-circuits the whole reconciliation with an early return. // change, it short-circuits the whole reconciliation with an early return.
func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
obj *sourcev1.GitRepository, commit *git.Commit, includes *artifactSet, dir string) (sreconcile.Result, error) { obj *sourcev1.GitRepository, commit *git.Commit, includes *artifactSet, dir string) (sreconcile.Result, error) {
gitImplementation := obj.Spec.GitImplementation
if goGitOnly, _ := r.features[features.ForceGoGitImplementation]; goGitOnly {
gitImplementation = sourcev1.GoGitImplementation
}
// Exit early, if we need to use libgit2 AND managed transport hasn't been intialized.
if !r.Libgit2TransportInitialized() && gitImplementation == sourcev1.LibGit2Implementation {
return sreconcile.ResultEmpty, serror.NewStalling(
errors.New("libgit2 managed transport not initialized"), "Libgit2TransportNotEnabled",
)
}
// Remove previously failed source verification status conditions. The // Remove previously failed source verification status conditions. The
// failing verification should be recalculated. But an existing successful // failing verification should be recalculated. But an existing successful
// verification need not be removed as it indicates verification of previous // verification need not be removed as it indicates verification of previous
@ -505,7 +489,7 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
optimizedClone = true optimizedClone = true
} }
c, err := r.gitCheckout(ctx, obj, authOpts, dir, optimizedClone, gitImplementation) c, err := r.gitCheckout(ctx, obj, authOpts, dir, optimizedClone)
if err != nil { if err != nil {
return sreconcile.ResultEmpty, err return sreconcile.ResultEmpty, err
} }
@ -539,7 +523,7 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
// If we can't skip the reconciliation, checkout again without any // If we can't skip the reconciliation, checkout again without any
// optimization. // optimization.
c, err := r.gitCheckout(ctx, obj, authOpts, dir, false, gitImplementation) c, err := r.gitCheckout(ctx, obj, authOpts, dir, false)
if err != nil { if err != nil {
return sreconcile.ResultEmpty, err return sreconcile.ResultEmpty, err
} }
@ -732,7 +716,7 @@ func (r *GitRepositoryReconciler) reconcileInclude(ctx context.Context,
// performs a git checkout. // performs a git checkout.
func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context, func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context,
obj *sourcev1.GitRepository, authOpts *git.AuthOptions, dir string, obj *sourcev1.GitRepository, authOpts *git.AuthOptions, dir string,
optimized bool, gitImplementation string) (*git.Commit, error) { optimized bool) (*git.Commit, error) {
// Configure checkout strategy. // Configure checkout strategy.
cloneOpts := repository.CloneOptions{ cloneOpts := repository.CloneOptions{
RecurseSubmodules: obj.Spec.RecurseSubmodules, RecurseSubmodules: obj.Spec.RecurseSubmodules,
@ -757,28 +741,15 @@ func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context,
gitCtx, cancel := context.WithTimeout(ctx, obj.Spec.Timeout.Duration) gitCtx, cancel := context.WithTimeout(ctx, obj.Spec.Timeout.Duration)
defer cancel() defer cancel()
var gitReader repository.Reader clientOpts := []gogit.ClientOption{gogit.WithDiskStorage()}
var err error if authOpts.Transport == git.HTTP {
clientOpts = append(clientOpts, gogit.WithInsecureCredentialsOverHTTP())
switch gitImplementation {
case sourcev1.LibGit2Implementation:
clientOpts := []libgit2.ClientOption{libgit2.WithDiskStorage()}
if authOpts.Transport == git.HTTP {
clientOpts = append(clientOpts, libgit2.WithInsecureCredentialsOverHTTP())
}
gitReader, err = libgit2.NewClient(dir, authOpts, clientOpts...)
case sourcev1.GoGitImplementation:
clientOpts := []gogit.ClientOption{gogit.WithDiskStorage()}
if authOpts.Transport == git.HTTP {
clientOpts = append(clientOpts, gogit.WithInsecureCredentialsOverHTTP())
}
gitReader, err = gogit.NewClient(dir, authOpts, clientOpts...)
default:
err = fmt.Errorf("invalid Git implementation: %s", gitImplementation)
} }
gitReader, err := gogit.NewClient(dir, authOpts, clientOpts...)
if err != nil { if err != nil {
e := serror.NewGeneric( e := serror.NewGeneric(
fmt.Errorf("failed to create Git client for implementation '%s': %w", gitImplementation, err), fmt.Errorf("failed to create Git client: %w", err),
sourcev1.GitOperationFailedReason, sourcev1.GitOperationFailedReason,
) )
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())

229
controllers/gitrepository_controller_test.go
View File

@ -57,7 +57,6 @@ import (
"github.com/fluxcd/pkg/testserver" "github.com/fluxcd/pkg/testserver"
"github.com/fluxcd/pkg/git" "github.com/fluxcd/pkg/git"
"github.com/fluxcd/pkg/git/libgit2/transport"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2" sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
serror "github.com/fluxcd/source-controller/internal/error" serror "github.com/fluxcd/source-controller/internal/error"
"github.com/fluxcd/source-controller/internal/features" "github.com/fluxcd/source-controller/internal/features"
@ -145,14 +144,6 @@ Oomb3gD/TRf/nAdVED+k81GdLzciYdUGtI71/qI47G0nMBluLRE=
` `
) )
var (
testGitImplementations = []string{sourcev1.GoGitImplementation, sourcev1.LibGit2Implementation}
)
func mockTransportNotInitialized() bool {
return false
}
func TestGitRepositoryReconciler_Reconcile(t *testing.T) { func TestGitRepositoryReconciler_Reconcile(t *testing.T) {
g := NewWithT(t) g := NewWithT(t)
@ -240,15 +231,14 @@ func TestGitRepositoryReconciler_reconcileSource_authStrategy(t *testing.T) {
} }
tests := []struct { tests := []struct {
name string name string
skipForImplementation string protocol string
protocol string server options
server options secret *corev1.Secret
secret *corev1.Secret beforeFunc func(obj *sourcev1.GitRepository)
beforeFunc func(obj *sourcev1.GitRepository) want sreconcile.Result
want sreconcile.Result wantErr bool
wantErr bool assertConditions []metav1.Condition
assertConditions []metav1.Condition
}{ }{
{ {
name: "HTTP without secretRef makes ArtifactOutdated=True", name: "HTTP without secretRef makes ArtifactOutdated=True",
@ -310,9 +300,8 @@ func TestGitRepositoryReconciler_reconcileSource_authStrategy(t *testing.T) {
}, },
}, },
{ {
name: "HTTPS with invalid CAFile secret makes CheckoutFailed=True and returns error", name: "HTTPS with invalid CAFile secret makes CheckoutFailed=True and returns error",
skipForImplementation: sourcev1.LibGit2Implementation, protocol: "https",
protocol: "https",
server: options{ server: options{
publicKey: tlsPublicKey, publicKey: tlsPublicKey,
privateKey: tlsPrivateKey, privateKey: tlsPrivateKey,
@ -339,31 +328,6 @@ func TestGitRepositoryReconciler_reconcileSource_authStrategy(t *testing.T) {
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.GitOperationFailedReason, "x509: "), *conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.GitOperationFailedReason, "x509: "),
}, },
}, },
{
name: "HTTPS with invalid CAFile secret makes CheckoutFailed=True and returns error",
skipForImplementation: sourcev1.GoGitImplementation,
protocol: "https",
server: options{
publicKey: tlsPublicKey,
privateKey: tlsPrivateKey,
ca: tlsCA,
},
secret: &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "invalid-ca",
},
Data: map[string][]byte{
"caFile": []byte("invalid"),
},
},
beforeFunc: func(obj *sourcev1.GitRepository) {
obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "invalid-ca"}
},
wantErr: true,
assertConditions: []metav1.Condition{
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.GitOperationFailedReason, "failed to checkout and determine revision: unable to fetch-connect to remote '<url>': PEM CA bundle could not be appended to x509 certificate pool"),
},
},
{ {
name: "SSH with private key secret makes ArtifactOutdated=True", name: "SSH with private key secret makes ArtifactOutdated=True",
protocol: "ssh", protocol: "ssh",
@ -503,82 +467,36 @@ func TestGitRepositoryReconciler_reconcileSource_authStrategy(t *testing.T) {
Storage: testStorage, Storage: testStorage,
features: map[string]bool{ features: map[string]bool{
features.OptimizedGitClones: true, features.OptimizedGitClones: true,
// Ensure that both implementations are tested.
features.ForceGoGitImplementation: false,
}, },
Libgit2TransportInitialized: transport.Enabled,
} }
for _, i := range testGitImplementations { t.Run(sourcev1.GoGitImplementation, func(t *testing.T) {
t.Run(i, func(t *testing.T) { g := NewWithT(t)
g := NewWithT(t)
if tt.skipForImplementation == i { tmpDir := t.TempDir()
t.Skipf("Skipped for Git implementation %q", i)
}
tmpDir := t.TempDir() obj := obj.DeepCopy()
obj := obj.DeepCopy() head, _ := localRepo.Head()
obj.Spec.GitImplementation = i assertConditions := tt.assertConditions
for k := range assertConditions {
assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "<commit>", head.Hash().String())
assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "<url>", obj.Spec.URL)
}
head, _ := localRepo.Head() var commit git.Commit
assertConditions := tt.assertConditions var includes artifactSet
for k := range assertConditions {
assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "<commit>", head.Hash().String())
assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "<url>", obj.Spec.URL)
}
var commit git.Commit got, err := r.reconcileSource(context.TODO(), obj, &commit, &includes, tmpDir)
var includes artifactSet g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.assertConditions))
g.Expect(err != nil).To(Equal(tt.wantErr))
got, err := r.reconcileSource(context.TODO(), obj, &commit, &includes, tmpDir) g.Expect(got).To(Equal(tt.want))
g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.assertConditions)) g.Expect(commit).ToNot(BeNil())
g.Expect(err != nil).To(Equal(tt.wantErr)) })
g.Expect(got).To(Equal(tt.want))
g.Expect(commit).ToNot(BeNil())
})
}
}) })
} }
} }
func TestGitRepositoryReconciler_reconcileSource_libgit2TransportUninitialized(t *testing.T) {
g := NewWithT(t)
r := &GitRepositoryReconciler{
Client: fakeclient.NewClientBuilder().WithScheme(runtime.NewScheme()).Build(),
EventRecorder: record.NewFakeRecorder(32),
Storage: testStorage,
features: map[string]bool{
features.ForceGoGitImplementation: false,
},
Libgit2TransportInitialized: mockTransportNotInitialized,
}
obj := &sourcev1.GitRepository{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "libgit2-transport",
},
Spec: sourcev1.GitRepositorySpec{
Interval: metav1.Duration{Duration: interval},
Timeout: &metav1.Duration{Duration: timeout},
Reference: &sourcev1.GitRepositoryRef{
Branch: git.DefaultBranch,
},
GitImplementation: sourcev1.LibGit2Implementation,
},
}
tmpDir := t.TempDir()
var commit git.Commit
var includes artifactSet
_, err := r.reconcileSource(ctx, obj, &commit, &includes, tmpDir)
g.Expect(err).To(HaveOccurred())
g.Expect(err).To(BeAssignableToTypeOf(&serror.Stalling{}))
g.Expect(err.Error()).To(Equal("libgit2 managed transport not initialized"))
}
func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T) { func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T) {
g := NewWithT(t) g := NewWithT(t)
@ -586,14 +504,13 @@ func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T)
tags := []string{"non-semver-tag", "v0.1.0", "0.2.0", "v0.2.1", "v1.0.0-alpha", "v1.1.0", "v2.0.0"} tags := []string{"non-semver-tag", "v0.1.0", "0.2.0", "v0.2.1", "v1.0.0-alpha", "v1.1.0", "v2.0.0"}
tests := []struct { tests := []struct {
name string name string
skipForImplementation string reference *sourcev1.GitRepositoryRef
reference *sourcev1.GitRepositoryRef beforeFunc func(obj *sourcev1.GitRepository, latestRev string)
beforeFunc func(obj *sourcev1.GitRepository, latestRev string) want sreconcile.Result
want sreconcile.Result wantErr bool
wantErr bool wantRevision string
wantRevision string wantArtifactOutdated bool
wantArtifactOutdated bool
}{ }{
{ {
name: "Nil reference (default branch)", name: "Nil reference (default branch)",
@ -620,8 +537,7 @@ func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T)
wantArtifactOutdated: true, wantArtifactOutdated: true,
}, },
{ {
name: "Branch commit", name: "Branch commit",
skipForImplementation: sourcev1.LibGit2Implementation,
reference: &sourcev1.GitRepositoryRef{ reference: &sourcev1.GitRepositoryRef{
Branch: "staging", Branch: "staging",
Commit: "<commit>", Commit: "<commit>",
@ -630,17 +546,6 @@ func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T)
wantRevision: "staging/<commit>", wantRevision: "staging/<commit>",
wantArtifactOutdated: true, wantArtifactOutdated: true,
}, },
{
name: "Branch commit",
skipForImplementation: sourcev1.GoGitImplementation,
reference: &sourcev1.GitRepositoryRef{
Branch: "staging",
Commit: "<commit>",
},
want: sreconcile.ResultSuccess,
wantRevision: "HEAD/<commit>",
wantArtifactOutdated: true,
},
{ {
name: "SemVer", name: "SemVer",
reference: &sourcev1.GitRepositoryRef{ reference: &sourcev1.GitRepositoryRef{
@ -738,10 +643,7 @@ func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T)
Storage: testStorage, Storage: testStorage,
features: map[string]bool{ features: map[string]bool{
features.OptimizedGitClones: true, features.OptimizedGitClones: true,
// Ensure that both implementations are tested.
features.ForceGoGitImplementation: false,
}, },
Libgit2TransportInitialized: transport.Enabled,
} }
for _, tt := range tests { for _, tt := range tests {
@ -762,38 +664,30 @@ func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T)
obj.Spec.Reference.Commit = headRef.Hash().String() obj.Spec.Reference.Commit = headRef.Hash().String()
} }
for _, i := range testGitImplementations { t.Run(sourcev1.GoGitImplementation, func(t *testing.T) {
t.Run(i, func(t *testing.T) { g := NewWithT(t)
g := NewWithT(t)
if tt.skipForImplementation == i { tmpDir := t.TempDir()
t.Skipf("Skipped for Git implementation %q", i) obj := obj.DeepCopy()
}
tmpDir := t.TempDir() if tt.beforeFunc != nil {
tt.beforeFunc(obj, headRef.Hash().String())
}
obj := obj.DeepCopy() var commit git.Commit
obj.Spec.GitImplementation = i var includes artifactSet
got, err := r.reconcileSource(ctx, obj, &commit, &includes, tmpDir)
if tt.beforeFunc != nil { if err != nil {
tt.beforeFunc(obj, headRef.Hash().String()) println(err.Error())
} }
g.Expect(err != nil).To(Equal(tt.wantErr))
var commit git.Commit g.Expect(got).To(Equal(tt.want))
var includes artifactSet if tt.wantRevision != "" && !tt.wantErr {
got, err := r.reconcileSource(ctx, obj, &commit, &includes, tmpDir) revision := strings.ReplaceAll(tt.wantRevision, "<commit>", headRef.Hash().String())
if err != nil { g.Expect(commit.String()).To(Equal(revision))
println(err.Error()) g.Expect(conditions.IsTrue(obj, sourcev1.ArtifactOutdatedCondition)).To(Equal(tt.wantArtifactOutdated))
} }
g.Expect(err != nil).To(Equal(tt.wantErr)) })
g.Expect(got).To(Equal(tt.want))
if tt.wantRevision != "" && !tt.wantErr {
revision := strings.ReplaceAll(tt.wantRevision, "<commit>", headRef.Hash().String())
g.Expect(commit.String()).To(Equal(revision))
g.Expect(conditions.IsTrue(obj, sourcev1.ArtifactOutdatedCondition)).To(Equal(tt.wantArtifactOutdated))
}
})
}
}) })
} }
} }
@ -1605,11 +1499,10 @@ func TestGitRepositoryReconciler_ConditionsUpdate(t *testing.T) {
builder := fakeclient.NewClientBuilder().WithScheme(testEnv.GetScheme()).WithObjects(obj) builder := fakeclient.NewClientBuilder().WithScheme(testEnv.GetScheme()).WithObjects(obj)
r := &GitRepositoryReconciler{ r := &GitRepositoryReconciler{
Client: builder.Build(), Client: builder.Build(),
EventRecorder: record.NewFakeRecorder(32), EventRecorder: record.NewFakeRecorder(32),
Storage: testStorage, Storage: testStorage,
features: features.FeatureGates(), features: features.FeatureGates(),
Libgit2TransportInitialized: transport.Enabled,
} }
key := client.ObjectKeyFromObject(obj) key := client.ObjectKeyFromObject(obj)

30
controllers/suite_test.go
View File

@ -37,7 +37,6 @@ import (
ctrl "sigs.k8s.io/controller-runtime" ctrl "sigs.k8s.io/controller-runtime"
dcontext "github.com/distribution/distribution/v3/context" dcontext "github.com/distribution/distribution/v3/context"
"github.com/fluxcd/pkg/git/libgit2/transport"
"github.com/fluxcd/pkg/runtime/controller" "github.com/fluxcd/pkg/runtime/controller"
"github.com/fluxcd/pkg/runtime/testenv" "github.com/fluxcd/pkg/runtime/testenv"
"github.com/fluxcd/pkg/testserver" "github.com/fluxcd/pkg/testserver"
@ -48,7 +47,6 @@ import (
dockerRegistry "github.com/distribution/distribution/v3/registry" dockerRegistry "github.com/distribution/distribution/v3/registry"
_ "github.com/distribution/distribution/v3/registry/auth/htpasswd" _ "github.com/distribution/distribution/v3/registry/auth/htpasswd"
_ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory" _ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory"
git2go "github.com/libgit2/git2go/v34"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2" sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
"github.com/fluxcd/source-controller/internal/cache" "github.com/fluxcd/source-controller/internal/cache"
@ -203,8 +201,6 @@ func setupRegistryServer(ctx context.Context, workspaceDir string, opts registry
} }
func TestMain(m *testing.M) { func TestMain(m *testing.M) {
mustHaveNoThreadSupport()
initTestTLS() initTestTLS()
utilruntime.Must(sourcev1.AddToScheme(scheme.Scheme)) utilruntime.Must(sourcev1.AddToScheme(scheme.Scheme))
@ -237,10 +233,6 @@ func TestMain(m *testing.M) {
panic(fmt.Sprintf("Failed to create a test registry server: %v", err)) panic(fmt.Sprintf("Failed to create a test registry server: %v", err))
} }
if err = transport.InitManagedTransport(); err != nil {
panic(fmt.Sprintf("Failed to initialize libgit2 managed transport: %v", err))
}
if err := (&GitRepositoryReconciler{ if err := (&GitRepositoryReconciler{
Client: testEnv, Client: testEnv,
EventRecorder: record.NewFakeRecorder(32), EventRecorder: record.NewFakeRecorder(32),
@ -248,10 +240,7 @@ func TestMain(m *testing.M) {
Storage: testStorage, Storage: testStorage,
features: map[string]bool{ features: map[string]bool{
features.OptimizedGitClones: true, features.OptimizedGitClones: true,
// Ensure that both implementations are used during tests.
features.ForceGoGitImplementation: false,
}, },
Libgit2TransportInitialized: transport.Enabled,
}).SetupWithManager(testEnv); err != nil { }).SetupWithManager(testEnv); err != nil {
panic(fmt.Sprintf("Failed to start GitRepositoryReconciler: %v", err)) panic(fmt.Sprintf("Failed to start GitRepositoryReconciler: %v", err))
} }
@ -378,22 +367,3 @@ func randStringRunes(n int) string {
func int64p(i int64) *int64 { func int64p(i int64) *int64 {
return &i return &i
} }
// This provides a regression assurance for image-automation-controller/#339.
// Validates that:
// - libgit2 was built with no support for threads.
// - git2go accepts libgit2 built with no support for threads.
//
// The logic below does the validation of the former, whilst
// referring to git2go forces its init() execution, which is
// where any validation to that effect resides.
//
// git2go does not support threadless libgit2 by default,
// hence a fork is being used which disables such validation.
//
// TODO: extract logic into pkg.
func mustHaveNoThreadSupport() {
if git2go.Features()&git2go.FeatureThreads != 0 {
panic("libgit2 must not be build with thread support")
}
}

8
docs/api/source.md
View File

@ -421,7 +421,9 @@ string
<td> <td>
<em>(Optional)</em> <em>(Optional)</em>
<p>GitImplementation specifies which Git client library implementation to <p>GitImplementation specifies which Git client library implementation to
use. Defaults to &lsquo;go-git&rsquo;, valid values are (&lsquo;go-git&rsquo;, &lsquo;libgit2&rsquo;).</p> use. Defaults to &lsquo;go-git&rsquo;, valid values are (&lsquo;go-git&rsquo;, &lsquo;libgit2&rsquo;).
Deprecated: gitImplementation is deprecated now that &lsquo;go-git&rsquo; is the
only supported implementation.</p>
</td> </td>
</tr> </tr>
<tr> <tr>
@ -1845,7 +1847,9 @@ string
<td> <td>
<em>(Optional)</em> <em>(Optional)</em>
<p>GitImplementation specifies which Git client library implementation to <p>GitImplementation specifies which Git client library implementation to
use. Defaults to &lsquo;go-git&rsquo;, valid values are (&lsquo;go-git&rsquo;, &lsquo;libgit2&rsquo;).</p> use. Defaults to &lsquo;go-git&rsquo;, valid values are (&lsquo;go-git&rsquo;, &lsquo;libgit2&rsquo;).
Deprecated: gitImplementation is deprecated now that &lsquo;go-git&rsquo; is the
only supported implementation.</p>
</td> </td>
</tr> </tr>
<tr> <tr>

45
docs/spec/v1beta2/gitrepositories.md
View File

@ -228,8 +228,7 @@ spec:
branch: <branch-name> branch: <branch-name>
``` ```
Using the [`go-git` Git implementation](#git-implementation), this will perform This will perform a shallow clone to only fetch the specified branch.
a shallow clone to only fetch the specified branch.
#### Tag example #### Tag example
@ -284,9 +283,9 @@ spec:
commit: "<commit SHA>" commit: "<commit SHA>"
``` ```
This field takes precedence over all other fields. Using the [`go-git` Git This field takes precedence over all other fields. It can be combined with
implementation](#git-implementation), it can be combined with `.spec.ref.branch` `.spec.ref.branch` to perform a shallow clone of the branch, in which the
to perform a shallow clone of the branch, in which the commit must exist: commit must exist:
```yaml ```yaml
--- ---
@ -385,32 +384,13 @@ resume.
### Git implementation ### Git implementation
**Note:** `libgit2` is being deprecated, as its use is known to cause controllers `.spec.gitImplementation` is deprecated and its value ignored, the git
to panic when running over long periods of time, or when under high GC pressure. implementation used across Flux is go-git.
A new opt-out feature gate `ForceGoGitImplementation` was introduced, which will
use `go-git` regardless of the value defined at `.spec.gitImplementation`.
This can be disabled by starting the controller with the additional flag below:
`--feature-gates=ForceGoGitImplementation=false`.
`.spec.gitImplementation` is an optional field to change the client library
implementation used for Git operations (e.g. clone, checkout). The default
value is `go-git`.
Unless you need support for a specific Git wire protocol functionality not
supported by the default implementation (as documented below), changing the
implementation is generally not recommended as it can come with its own set of
drawbacks. For example, not being able to make use of shallow clones forces the
controller to fetch the whole Git history tree instead of a specific one,
resulting in an increase of disk space and traffic usage.
**Note:** The `libgit2` implementation does not support shallow clones or
Git submodules.
#### Optimized Git clones #### Optimized Git clones
Optimized Git clones decreases resource utilization for GitRepository Optimized Git clones decreases resource utilization for GitRepository
reconciliations. It supports both `go-git` and `libgit2` implementations reconciliations.
when cloning repositories using branches or tags.
When enabled, it avoids full Git clone operations by first checking whether When enabled, it avoids full Git clone operations by first checking whether
the revision of the last stored artifact is still the head of the remote the revision of the last stored artifact is still the head of the remote
@ -428,20 +408,13 @@ not affected by this functionality.
#### Proxy support #### Proxy support
When a proxy is configured in the source-controller Pod through the appropriate When a proxy is configured in the source-controller Pod through the appropriate
environment variables, for example `HTTPS_PROXY`, `NO_PROXY`, etc. There may be environment variables, for example `HTTPS_PROXY`, `NO_PROXY`, etc.
some limitations in the proxy support based on the Git implementation.
| Git Implementation | HTTP_PROXY | HTTPS_PROXY | NO_PROXY | Self-signed Certs |
|--------------------|------------|-------------|----------|-------------------|
| `go-git` | true | true | true | false |n
| `libgit2` | true | true | true | true |
### Recurse submodules ### Recurse submodules
`.spec.recurseSubmodules` is an optional field to enable the initialization of `.spec.recurseSubmodules` is an optional field to enable the initialization of
all submodules within the cloned Git repository, using their default settings. all submodules within the cloned Git repository, using their default settings.
This option is only available when using the (default) `go-git` [Git This option defaults to `false`.
implementation](#git-implementation), and defaults to `false`.
Note that for most Git providers (e.g. GitHub and GitLab), deploy keys can not Note that for most Git providers (e.g. GitHub and GitLab), deploy keys can not
be used as reusing a key across multiple repositories is not allowed. You have be used as reusing a key across multiple repositories is not allowed. You have

13
go.mod
View File

@ -4,14 +4,6 @@ go 1.18
replace github.com/fluxcd/source-controller/api => ./api replace github.com/fluxcd/source-controller/api => ./api
// Flux has its own git2go fork to enable changes in behaviour for improved
// reliability.
//
// For more information refer to:
// - fluxcd/image-automation-controller/#339.
// - libgit2/git2go#918.
replace github.com/libgit2/git2go/v34 => github.com/fluxcd/git2go/v34 v34.0.0
// Fix CVE-2022-1996 (for v2, Go Modules incompatible) // Fix CVE-2022-1996 (for v2, Go Modules incompatible)
replace github.com/emicklei/go-restful => github.com/emicklei/go-restful v2.16.0+incompatible replace github.com/emicklei/go-restful => github.com/emicklei/go-restful v2.16.0+incompatible
@ -33,8 +25,7 @@ require (
github.com/fluxcd/pkg/apis/event v0.2.0 github.com/fluxcd/pkg/apis/event v0.2.0
github.com/fluxcd/pkg/apis/meta v0.18.0 github.com/fluxcd/pkg/apis/meta v0.18.0
github.com/fluxcd/pkg/git v0.7.0 github.com/fluxcd/pkg/git v0.7.0
github.com/fluxcd/pkg/git/gogit v0.2.0 github.com/fluxcd/pkg/git/gogit v0.2.1-0.20221128153352-fbb8b9f29f38
github.com/fluxcd/pkg/git/libgit2 v0.3.0
github.com/fluxcd/pkg/gittestserver v0.8.0 github.com/fluxcd/pkg/gittestserver v0.8.0
github.com/fluxcd/pkg/helmtestserver v0.10.0 github.com/fluxcd/pkg/helmtestserver v0.10.0
github.com/fluxcd/pkg/lockedfile v0.1.0 github.com/fluxcd/pkg/lockedfile v0.1.0
@ -52,7 +43,6 @@ require (
github.com/google/go-containerregistry v0.12.1 github.com/google/go-containerregistry v0.12.1
github.com/google/go-containerregistry/pkg/authn/k8schain v0.0.0-20221114162634-781782aa2757 github.com/google/go-containerregistry/pkg/authn/k8schain v0.0.0-20221114162634-781782aa2757
github.com/google/uuid v1.3.0 github.com/google/uuid v1.3.0
github.com/libgit2/git2go/v34 v34.0.0
github.com/minio/minio-go/v7 v7.0.43 github.com/minio/minio-go/v7 v7.0.43
github.com/onsi/gomega v1.24.1 github.com/onsi/gomega v1.24.1
github.com/otiai10/copy v1.9.0 github.com/otiai10/copy v1.9.0
@ -178,7 +168,6 @@ require (
github.com/felixge/httpsnoop v1.0.3 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/fluxcd/gitkit v0.6.0 // indirect github.com/fluxcd/gitkit v0.6.0 // indirect
github.com/fluxcd/pkg/apis/acl v0.1.0 // indirect github.com/fluxcd/pkg/apis/acl v0.1.0 // indirect
github.com/fluxcd/pkg/http/transport v0.1.0 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/fullstorydev/grpcurl v1.8.7 // indirect github.com/fullstorydev/grpcurl v1.8.7 // indirect
github.com/go-chi/chi v4.1.2+incompatible // indirect github.com/go-chi/chi v4.1.2+incompatible // indirect

12
go.sum
View File

@ -502,8 +502,6 @@ github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw
github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0=
github.com/fluxcd/git2go/v34 v34.0.0 h1:08PEpdbsLO4sUTvFKuxmt6xBowaqObro9veChBWFwa8=
github.com/fluxcd/git2go/v34 v34.0.0/go.mod h1:blVco2jDAw6YTXkErMMqzHLcAjKkwF0aWIRHBqiJkZ0=
github.com/fluxcd/gitkit v0.6.0 h1:iNg5LTx6ePo+Pl0ZwqHTAkhbUHxGVSY3YCxCdw7VIFg= github.com/fluxcd/gitkit v0.6.0 h1:iNg5LTx6ePo+Pl0ZwqHTAkhbUHxGVSY3YCxCdw7VIFg=
github.com/fluxcd/gitkit v0.6.0/go.mod h1:svOHuKi0fO9HoawdK4HfHAJJseZDHHjk7I3ihnCIqNo= github.com/fluxcd/gitkit v0.6.0/go.mod h1:svOHuKi0fO9HoawdK4HfHAJJseZDHHjk7I3ihnCIqNo=
github.com/fluxcd/go-git/v5 v5.0.0-20221104190732-329fd6659b10 h1:au798417R1iWtgcgKe3Dg495mexQmuxelL+NebAtexE= github.com/fluxcd/go-git/v5 v5.0.0-20221104190732-329fd6659b10 h1:au798417R1iWtgcgKe3Dg495mexQmuxelL+NebAtexE=
@ -516,16 +514,12 @@ github.com/fluxcd/pkg/apis/meta v0.18.0 h1:s0LeulWcQ4DxVX6805vgDTxlA6bAYk+Lq1QHS
github.com/fluxcd/pkg/apis/meta v0.18.0/go.mod h1:pYvXRFi1UKNNrGR34jw3uqOnMXw9X6dTkML8j5Z7tis= github.com/fluxcd/pkg/apis/meta v0.18.0/go.mod h1:pYvXRFi1UKNNrGR34jw3uqOnMXw9X6dTkML8j5Z7tis=
github.com/fluxcd/pkg/git v0.7.0 h1:sQHRpFMcOzEdqlyGMjFv2LKMdcoE5xeUr2UcRrsLRG8= github.com/fluxcd/pkg/git v0.7.0 h1:sQHRpFMcOzEdqlyGMjFv2LKMdcoE5xeUr2UcRrsLRG8=
github.com/fluxcd/pkg/git v0.7.0/go.mod h1:3deiLPws4DSQ3hqwtQd7Dt66GXTN/4RcT/yHAljXaHo= github.com/fluxcd/pkg/git v0.7.0/go.mod h1:3deiLPws4DSQ3hqwtQd7Dt66GXTN/4RcT/yHAljXaHo=
github.com/fluxcd/pkg/git/gogit v0.2.0 h1:vhFzk2Pky4tDZwisx8+26YZumRDPxERnkV8l2dbLSoo= github.com/fluxcd/pkg/git/gogit v0.2.1-0.20221128153352-fbb8b9f29f38 h1:FGDncVUlMfK7eZiw0m/mWa6CGczGeGomw8jJ0uX2zIw=
github.com/fluxcd/pkg/git/gogit v0.2.0/go.mod h1:d1RIwl6DVdU8/2dBIhw6n7GNokIKqs+b9cKc/8tz7ew= github.com/fluxcd/pkg/git/gogit v0.2.1-0.20221128153352-fbb8b9f29f38/go.mod h1:d1RIwl6DVdU8/2dBIhw6n7GNokIKqs+b9cKc/8tz7ew=
github.com/fluxcd/pkg/git/libgit2 v0.3.0 h1:fgw/Cm7Id2AxanMab3qU+UuvdnhfwRMbbWF27xu413o=
github.com/fluxcd/pkg/git/libgit2 v0.3.0/go.mod h1:JHeZBeq+jbUpRpP5OoEYXua0EF5dyqAlbEfYcXY228A=
github.com/fluxcd/pkg/gittestserver v0.8.0 h1:YrYe63KScKlLxx0GAiQthx2XqHDx0vKitIIx4JnDtIo= github.com/fluxcd/pkg/gittestserver v0.8.0 h1:YrYe63KScKlLxx0GAiQthx2XqHDx0vKitIIx4JnDtIo=
github.com/fluxcd/pkg/gittestserver v0.8.0/go.mod h1:/LI/xKMrnQbIsTDnTyABQ71iaYhFIZ8fb4cvY7WAlBU= github.com/fluxcd/pkg/gittestserver v0.8.0/go.mod h1:/LI/xKMrnQbIsTDnTyABQ71iaYhFIZ8fb4cvY7WAlBU=
github.com/fluxcd/pkg/helmtestserver v0.10.0 h1:/cnx1D6/cln7v06iXZMLeshl82mV1wv3OVZoBE9Bbm4= github.com/fluxcd/pkg/helmtestserver v0.10.0 h1:/cnx1D6/cln7v06iXZMLeshl82mV1wv3OVZoBE9Bbm4=
github.com/fluxcd/pkg/helmtestserver v0.10.0/go.mod h1:sJwcAw/4ENiJO6i0HJSMV8ypp7k+U4amDwH7RJ57ScY= github.com/fluxcd/pkg/helmtestserver v0.10.0/go.mod h1:sJwcAw/4ENiJO6i0HJSMV8ypp7k+U4amDwH7RJ57ScY=
github.com/fluxcd/pkg/http/transport v0.1.0 h1:tzRprpCAUkPFlF/9VMLSbw95jcAKCbimDC//VULQXnY=
github.com/fluxcd/pkg/http/transport v0.1.0/go.mod h1:BNXQYy3ifMTc3DaOu3GlIR6j5DtTwMEyvTr2EA7XP8w=
github.com/fluxcd/pkg/lockedfile v0.1.0 h1:YsYFAkd6wawMCcD74ikadAKXA4s2sukdxrn7w8RB5eo= github.com/fluxcd/pkg/lockedfile v0.1.0 h1:YsYFAkd6wawMCcD74ikadAKXA4s2sukdxrn7w8RB5eo=
github.com/fluxcd/pkg/lockedfile v0.1.0/go.mod h1:EJLan8t9MiOcgTs8+puDjbE6I/KAfHbdvIy9VUgIjm8= github.com/fluxcd/pkg/lockedfile v0.1.0/go.mod h1:EJLan8t9MiOcgTs8+puDjbE6I/KAfHbdvIy9VUgIjm8=
github.com/fluxcd/pkg/masktoken v0.2.0 h1:HoSPTk4l1fz5Fevs2vVRvZGru33blfMwWSZKsHdfG/0= github.com/fluxcd/pkg/masktoken v0.2.0 h1:HoSPTk4l1fz5Fevs2vVRvZGru33blfMwWSZKsHdfG/0=
@ -1736,7 +1730,6 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200930160638-afb6bcd081ae/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200930160638-afb6bcd081ae/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
@ -1975,7 +1968,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

4
hack/ci/e2e.sh
View File

@ -141,9 +141,7 @@ kubectl -n source-system wait helmchart/helmchart-bucket --for=condition=ready -
echo "Run large Git repo tests" echo "Run large Git repo tests"
kubectl -n source-system apply -f "${ROOT_DIR}/config/testdata/git/large-repo.yaml" kubectl -n source-system apply -f "${ROOT_DIR}/config/testdata/git/large-repo.yaml"
kubectl -n source-system wait gitrepository/large-repo-go-git --for=condition=ready --timeout=2m15s kubectl -n source-system wait gitrepository/large-repo --for=condition=ready --timeout=2m15s
kubectl -n source-system wait gitrepository/large-repo-libgit2 --for=condition=ready --timeout=2m15s
echo "Run HelmChart from OCI registry tests" echo "Run HelmChart from OCI registry tests"
kubectl -n source-system apply -f "${ROOT_DIR}/config/testdata/helmchart-from-oci/source.yaml" kubectl -n source-system apply -f "${ROOT_DIR}/config/testdata/helmchart-from-oci/source.yaml"

158
hack/install-libraries.sh
View File

@ -1,158 +0,0 @@
#!/usr/bin/env bash
set -euxo pipefail
IMG="${IMG:-}"
TAG="${TAG:-}"
IMG_TAG="${IMG}:${TAG}"
DOWNLOAD_URL="https://github.com/fluxcd/golang-with-libgit2/releases/download/${TAG}"
SKIP_COSIGN_VERIFICATION="${SKIP_COSIGN_VERIFICATION:-false}"
TMP_DIR=$(mktemp -d)
function cleanup(){
rm -rf "${TMP_DIR}"
}
trap cleanup EXIT
fatal() {
echo '[ERROR] ' "$@" >&2
exit 1
}
download() {
[[ $# -eq 2 ]] || fatal 'download needs exactly 2 arguments'
curl -o "$1" -sfL "$2"
[[ $? -eq 0 ]] || fatal 'Download failed'
}
download_files() {
[[ $# -eq 1 ]] || fatal 'download_files needs exactly 1 arguments'
FILE_NAMES="checksums.txt checksums.txt.sig checksums.txt.pem $1"
for FILE_NAME in ${FILE_NAMES}; do
download "${TMP_DIR}/${FILE_NAME}" "${DOWNLOAD_URL}/${FILE_NAME}"
done
}
cosign_verify(){
[[ $# -eq 3 ]] || fatal 'cosign_verify needs exactly 3 arguments'
COSIGN_EXPERIMENTAL=1 cosign verify-blob --cert "$1" --signature "$2" "$3"
[[ $? -eq 0 ]] || fatal 'signature verification failed'
}
assure_provenance() {
[[ $# -eq 1 ]] || fatal 'assure_provenance needs exactly 1 arguments'
if "${SKIP_COSIGN_VERIFICATION}"; then
echo 'Skipping cosign verification...'
else
cosign_verify "${TMP_DIR}/checksums.txt.pem" \
"${TMP_DIR}/checksums.txt.sig" \
"${TMP_DIR}/checksums.txt"
fi
pushd "${TMP_DIR}" || exit
if command -v sha256sum; then
grep "$1" "checksums.txt" | sha256sum --check
else
grep "$1" "checksums.txt" | shasum -a 256 --check
fi
popd || exit
[[ $? -eq 0 ]] || fatal 'integrity verification failed'
}
extract_libraries(){
[[ $# -eq 2 ]] || fatal 'extract_libraries needs exactly 2 arguments'
tar -xf "${TMP_DIR}/$1"
rm "${TMP_DIR}/$1"
mv "${2}" "${TAG}"
mv "${TAG}/" "./build/libgit2"
}
fix_pkgconfigs(){
NEW_DIR="$(/bin/pwd)/build/libgit2/${TAG}"
# Update the prefix paths included in the .pc files.
if [[ $OSTYPE == 'darwin'* ]]; then
# https://github.com/fluxcd/golang-with-libgit2/blob/v0.1.4/.github/workflows/release.yaml#L158
INSTALLED_DIR="/Users/runner/work/golang-with-libgit2/golang-with-libgit2/build/darwin-libgit2-only"
# This will make it easier to update to the location in which they will be used.
# sed has a sight different behaviour in MacOS
# NB: Some macOS users may override their sed with gsed. If gsed is the PATH, use that instead.
if command -v gsed &> /dev/null; then
find "${NEW_DIR}" -type f -name "*.pc" | xargs -I {} gsed -i "s;${INSTALLED_DIR};${NEW_DIR};g" {}
else
find "${NEW_DIR}" -type f -name "*.pc" | xargs -I {} sed -i "" "s;${INSTALLED_DIR};${NEW_DIR};g" {}
fi
else
# https://github.com/fluxcd/golang-with-libgit2/blob/v0.1.4/.github/workflows/release.yaml#L52
INSTALLED_DIR="/home/runner/work/golang-with-libgit2/golang-with-libgit2/build/build_libgit2_only"
find "${NEW_DIR}" -type f -name "*.pc" | xargs -I {} sed -i "s;${INSTALLED_DIR};${NEW_DIR};g" {}
fi
}
extract_from_image(){
PLATFORM=$1
DIR=$2
id=$(docker create --platform="${PLATFORM}" "${IMG_TAG}" sh)
docker cp "${id}":/usr/local - > output.tar.gz
docker rm -v "${id}"
tar -xf output.tar.gz "local/${DIR}"
rm output.tar.gz
NEW_DIR="$(/bin/pwd)/build/libgit2/${TAG}"
INSTALLED_DIR="/usr/local/${DIR}"
mv "local/${DIR}" "${TAG}"
rm -rf "local"
mv "${TAG}/" "./build/libgit2"
# Update the prefix paths included in the .pc files.
# This will make it easier to update to the location in which they will be used.
find "${NEW_DIR}" -type f -name "*.pc" | xargs -I {} sed -i "s;${INSTALLED_DIR};${NEW_DIR};g" {}
}
install_libraries(){
if [ -d "./build/libgit2/${TAG}" ]; then
echo "Skipping: libgit2 ${TAG} already installed"
exit 0
fi
mkdir -p "./build/libgit2"
# Linux ARM support is still based on the container image libraries.
if [[ $OSTYPE == 'linux'* ]]; then
if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
extract_from_image "linux/arm64" "aarch64-alpine-linux-musl"
fix_pkgconfigs "aarch64-alpine-linux-musl"
exit 0
fi
fi
FILE_NAME="linux-$(uname -m)-libgit2-only.tar.gz"
DIR="linux-libgit2-only"
if [[ $OSTYPE == 'darwin'* ]]; then
FILE_NAME="darwin-libgit2-only.tar.gz"
DIR="darwin-libgit2-only"
fi
download_files "${FILE_NAME}"
assure_provenance "${FILE_NAME}"
extract_libraries "${FILE_NAME}" "${DIR}"
fix_pkgconfigs
}
install_libraries

25
hack/update-attributions.sh
View File

@ -1,25 +0,0 @@
#!/usr/bin/env bash
set -eoux pipefail
SPLIT="***"
cat <<EOF > ATTRIBUTIONS.md
# Attributions
This application uses Open Source components. You can find the source
code of their open source projects along with license information below.
We acknowledge and are grateful to these developers for their contributions
to open source.
## libgit2
Libgit2 was obtained in source-code form from its github repository:
https://github.com/libgit2/libgit2/
No changes were made to its original source code.
Copyright notice (https://raw.githubusercontent.com/libgit2/libgit2/main/COPYING):
$(curl --max-time 5 -L https://raw.githubusercontent.com/libgit2/libgit2/main/COPYING)
EOF

20
internal/features/features.go
View File

@ -23,36 +23,18 @@ import feathelper "github.com/fluxcd/pkg/runtime/features"
const ( const (
// OptimizedGitClones decreases resource utilization for GitRepository // OptimizedGitClones decreases resource utilization for GitRepository
// reconciliations. It supports both go-git and libgit2 implementations // reconciliations.
// when cloning repositories using branches or tags.
// //
// When enabled, avoids full clone operations by first checking whether // When enabled, avoids full clone operations by first checking whether
// the last revision is still the same at the target repository, // the last revision is still the same at the target repository,
// and if that is so, skips the reconciliation. // and if that is so, skips the reconciliation.
OptimizedGitClones = "OptimizedGitClones" OptimizedGitClones = "OptimizedGitClones"
// ForceGoGitImplementation ignores the value set for gitImplementation
// and ensures that go-git is used for all GitRepository objects.
//
// Libgit2 is built in C and we use the Go bindings provided by git2go
// to cross the C-GO chasm. Unfortunately, when libgit2 is being used the
// controllers are known to panic over long periods of time, or when
// under high GC pressure.
//
// This feature gate enables the gradual deprecation of libgit2 in favour
// of go-git, which so far is the most stable of the pair.
//
// When enabled, libgit2 won't be initialized, nor will any git2go CGO
// code be called.
ForceGoGitImplementation = "ForceGoGitImplementation"
) )
var features = map[string]bool{ var features = map[string]bool{
// OptimizedGitClones // OptimizedGitClones
// opt-out from v0.25 // opt-out from v0.25
OptimizedGitClones: true, OptimizedGitClones: true,
// ForceGoGitImplementation
// opt-out from v0.32
ForceGoGitImplementation: true,
} }
// DefaultFeatureGates contains a list of all supported feature gates and // DefaultFeatureGates contains a list of all supported feature gates and

19
main.go
View File

@ -34,7 +34,6 @@ import (
ctrl "sigs.k8s.io/controller-runtime" ctrl "sigs.k8s.io/controller-runtime"
"github.com/fluxcd/pkg/git" "github.com/fluxcd/pkg/git"
"github.com/fluxcd/pkg/git/libgit2/transport"
"github.com/fluxcd/pkg/runtime/client" "github.com/fluxcd/pkg/runtime/client"
helper "github.com/fluxcd/pkg/runtime/controller" helper "github.com/fluxcd/pkg/runtime/controller"
"github.com/fluxcd/pkg/runtime/events" "github.com/fluxcd/pkg/runtime/events"
@ -204,20 +203,12 @@ func main() {
} }
storage := mustInitStorage(storagePath, storageAdvAddr, artifactRetentionTTL, artifactRetentionRecords, setupLog) storage := mustInitStorage(storagePath, storageAdvAddr, artifactRetentionTTL, artifactRetentionRecords, setupLog)
if gogitOnly, _ := features.Enabled(features.ForceGoGitImplementation); !gogitOnly {
if err = transport.InitManagedTransport(); err != nil {
// Log the error, but don't exit so as to not block reconcilers that are healthy.
setupLog.Error(err, "unable to initialize libgit2 managed transport")
}
}
if err = (&controllers.GitRepositoryReconciler{ if err = (&controllers.GitRepositoryReconciler{
Client: mgr.GetClient(), Client: mgr.GetClient(),
EventRecorder: eventRecorder, EventRecorder: eventRecorder,
Metrics: metricsH, Metrics: metricsH,
Storage: storage, Storage: storage,
ControllerName: controllerName, ControllerName: controllerName,
Libgit2TransportInitialized: transport.Enabled,
}).SetupWithManagerAndOptions(mgr, controllers.GitRepositoryReconcilerOptions{ }).SetupWithManagerAndOptions(mgr, controllers.GitRepositoryReconcilerOptions{
MaxConcurrentReconciles: concurrent, MaxConcurrentReconciles: concurrent,
DependencyRequeueInterval: requeueDependency, DependencyRequeueInterval: requeueDependency,

8
tests/fuzz/Dockerfile.builder
View File

@ -1,9 +1,9 @@
FROM gcr.io/oss-fuzz-base/base-builder-go FROM gcr.io/oss-fuzz-base/base-builder-go
RUN apt-get update && apt-get install -y cmake pkg-config ENV SRC=$GOPATH/src/github.com/fluxcd/source-controller
ENV FLUX_CI=true
COPY ./ $GOPATH/src/github.com/fluxcd/source-controller/ COPY ./ $SRC
COPY ./tests/fuzz/oss_fuzz_build.sh $SRC/build.sh RUN wget https://raw.githubusercontent.com/google/oss-fuzz/master/projects/fluxcd/build.sh -O $SRC/build.sh
COPY tests/fuzz/compile_native_go_fuzzer /usr/local/bin/
WORKDIR $SRC WORKDIR $SRC

62
tests/fuzz/compile_native_go_fuzzer
View File

@ -1,62 +0,0 @@
#!/bin/bash -eux
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################
# This is a copy of the upstream script which is only needed to link
# additional static libraries. Orignal source:
#
# https://github.com/google/oss-fuzz/blob/9e8dd47cb902545efc60a5580126adc36d70bae3/infra/base-images/base-builder/compile_native_go_fuzzer
function build_native_go_fuzzer() {
fuzzer=$1
function=$2
path=$3
tags="-tags gofuzz"
if [[ $SANITIZER == *coverage* ]]; then
current_dir=$(pwd)
mkdir $OUT/rawfuzzers || true
cd $abs_file_dir
go test -c -run $fuzzer -o $OUT/$fuzzer -cover
cp "${fuzzer_filename}" "${OUT}/rawfuzzers/${fuzzer}"
cd $current_dir
else
go-118-fuzz-build -o $fuzzer.a -func $function $abs_file_dir
# TODO: upstream support for linking $ADDITIONAL_LIBS
$CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -o $OUT/$fuzzer \
$ADDITIONAL_LIBS
fi
}
path=$1
function=$2
fuzzer=$3
tags="-tags gofuzz"
# Get absolute path.
abs_file_dir=$(go list $tags -f {{.Dir}} $path)
# TODO(adamkorcz): Get rid of "-r" flag here.
fuzzer_filename=$(grep -r -l --include='*.go' -s "$function" "${abs_file_dir}")
# Test if file contains a line with "func $function" and "testing.F".
if [ $(grep -r "func $function" $fuzzer_filename | grep "testing.F" | wc -l) -eq 1 ]
then
build_native_go_fuzzer $fuzzer $function $abs_file_dir
else
echo "Could not find the function: func ${function}(f *testing.F)"
fi

80
tests/fuzz/oss_fuzz_build.sh
View File

@ -1,80 +0,0 @@
#!/usr/bin/env bash
# Copyright 2022 The Flux authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -euxo pipefail
# This file aims for:
# - Dynamically discover and build all fuzz tests within the repository.
# - Work for both local make fuzz-smoketest and the upstream oss-fuzz.
GOPATH="${GOPATH:-/root/go}"
GO_SRC="${GOPATH}/src"
PROJECT_PATH="github.com/fluxcd/source-controller"
# install_deps installs all dependencies needed for upstream oss-fuzz.
# Unfortunately we can't pin versions here, as we want to always
# have the latest, so that we can reproduce errors occuring upstream.
install_deps(){
if ! command -v go-118-fuzz-build &> /dev/null; then
go install github.com/AdamKorcz/go-118-fuzz-build@latest
fi
}
install_deps
cd "${GO_SRC}/${PROJECT_PATH}"
# Ensure any project-specific requirements are catered for ahead of
# the generic build process.
if [ -f "tests/fuzz/oss_fuzz_prebuild.sh" ]; then
. tests/fuzz/oss_fuzz_prebuild.sh
fi
modules=$(find . -mindepth 1 -maxdepth 4 -type f -name 'go.mod' | cut -c 3- | sed 's|/[^/]*$$||' | sort -u | sed 's;/go.mod;;g' | sed 's;go.mod;.;g')
for module in ${modules}; do
cd "${GO_SRC}/${PROJECT_PATH}/${module}"
test_files=$(grep -r --include='**_test.go' --files-with-matches 'func Fuzz' . || echo "")
if [ -z "${test_files}" ]; then
continue
fi
go get github.com/AdamKorcz/go-118-fuzz-build/testing
# Iterate through all Go Fuzz targets, compiling each into a fuzzer.
for file in ${test_files}; do
# If the subdir is a module, skip this file, as it will be handled
# at the next iteration of the outer loop.
if [ -f "$(dirname "${file}")/go.mod" ]; then
continue
fi
targets=$(grep -oP 'func \K(Fuzz\w*)' "${file}")
for target_name in ${targets}; do
# Transform module path into module name (e.g. git/libgit2 to git_libgit2).
module_name="$(echo ${module} | tr / _)_"
# Compose fuzzer name based on the lowercase version of the func names.
# The module name is added after the fuzz prefix, for better discoverability.
fuzzer_name=$(echo "${target_name}" | tr '[:upper:]' '[:lower:]' | sed "s;fuzz_;fuzz_${module_name//._/};g")
target_dir=$(dirname "${file}")
echo "Building ${file}.${target_name} into ${fuzzer_name}"
compile_native_go_fuzzer "${target_dir}" "${target_name}" "${fuzzer_name}"
done
done
done

33
tests/fuzz/oss_fuzz_postbuild.sh
View File

@ -1,33 +0,0 @@
#!/usr/bin/env bash
# Copyright 2022 The Flux authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -euxo pipefail
# This file is executed by upstream oss-fuzz after its building process.
# Use it for unsetting any environment variables that may impact other building
# processes.
if [[ -n "${PRE_LIB_FUZZING_ENGINE}" ]]; then
export LIB_FUZZING_ENGINE="${PRE_LIB_FUZZING_ENGINE}"
fi
unset TARGET_DIR
unset CGO_ENABLED
unset LIBRARY_PATH
unset PKG_CONFIG_PATH
unset CGO_CFLAGS
unset CGO_LDFLAGS
unset PRE_LIB_FUZZING_ENGINE

51
tests/fuzz/oss_fuzz_prebuild.sh
View File

@ -23,54 +23,3 @@ set -euxo pipefail
# for traversing into ascending dirs, therefore we copy those contents here: # for traversing into ascending dirs, therefore we copy those contents here:
mkdir -p controllers/testdata/crd mkdir -p controllers/testdata/crd
cp config/crd/bases/*.yaml controllers/testdata/crd/ cp config/crd/bases/*.yaml controllers/testdata/crd/
# libgit2, cmake and pkg-config are requirements to support libgit2.
LIBGIT2_TAG="${LIBGIT2_TAG:-v0.4.0}"
# Avoid updating apt get and installing dependencies, if they are already in place.
if (! command -v cmake &> /dev/null) || (! command -v pkg-config &> /dev/null) then
apt-get update && apt-get install -y cmake pkg-config
fi
export TARGET_DIR="$(/bin/pwd)/build/libgit2/${LIBGIT2_TAG}"
# For most cases, libgit2 will already be present.
# The exception being at the oss-fuzz integration.
if [ ! -d "${TARGET_DIR}" ]; then
curl --connect-timeout 2 --retry 3 --retry-delay 1 --retry-max-time 30 \
-o output.tar.gz -LO "https://github.com/fluxcd/golang-with-libgit2/releases/download/${LIBGIT2_TAG}/linux-$(uname -m)-libgit2-only.tar.gz"
DIR=linux-libgit2-only
NEW_DIR="$(/bin/pwd)/build/libgit2/${LIBGIT2_TAG}"
INSTALLED_DIR="/home/runner/work/golang-with-libgit2/golang-with-libgit2/build/${DIR}"
mkdir -p ./build/libgit2
tar -xf output.tar.gz
rm output.tar.gz
mv "${DIR}" "${LIBGIT2_TAG}"
mv "${LIBGIT2_TAG}/" "./build/libgit2"
# Update the prefix paths included in the .pc files.
# This will make it easier to update to the location in which they will be used.
find "${NEW_DIR}" -type f -name "*.pc" | xargs -I {} sed -i "s;${INSTALLED_DIR};${NEW_DIR};g" {}
fi
export CGO_ENABLED=1
export LIBRARY_PATH="${TARGET_DIR}/lib"
export PKG_CONFIG_PATH="${TARGET_DIR}/lib/pkgconfig"
export CGO_CFLAGS="-I${TARGET_DIR}/include"
export CGO_LDFLAGS="$(pkg-config --libs --static --cflags libgit2)"
# Temporary hack whilst libgit2 is still in use.
# Enables the fuzzing compilation to link libgit2.
#
# After building the fuzzers, the value of
# LIB_FUZZING_ENGINE is reset to what it was before
# it to avoid side effects onto other repositories.
#
# For context refer to:
# https://github.com/google/oss-fuzz/pull/9063
export PRE_LIB_FUZZING_ENGINE="${LIB_FUZZING_ENGINE}"
export LIB_FUZZING_ENGINE="${LIB_FUZZING_ENGINE} -Wl,--start-group ${TARGET_DIR}/lib/libgit2.a"

2
tests/fuzz/oss_fuzz_run.sh
View File

@ -17,4 +17,4 @@
set -euxo pipefail set -euxo pipefail
# run each fuzzer once to ensure they are working properly # run each fuzzer once to ensure they are working properly
find /out -type f -name "fuzz*" -exec echo {} -runs=1 \; | bash -e find /out -type f -iname "fuzz*" -exec echo {} -runs=1 \; | bash -e