Merge pull request #977 from fluxcd/remove-libgit2
Remove libgit2 and git2go from codebase
This commit is contained in:
Paulo Gomes
GitHub
commit
f7a64b1b81
|
|
@ -1 +1 @@
|
|||
build/libgit2/
|
||||
build/
|
||||
|
|
|
|||
1232
ATTRIBUTIONS.md
1232
ATTRIBUTIONS.md
File diff suppressed because it is too large
Load Diff
|
|
@ -13,27 +13,6 @@ There are a number of dependencies required to be able to run the controller and
|
|||
- [Install Docker](https://docs.docker.com/engine/install/)
|
||||
- (Optional) [Install Kubebuilder](https://book.kubebuilder.io/quick-start.html#installation)
|
||||
|
||||
The [libgit2](https://libgit2.org/) dependency is now automatically managed by the Makefile logic.
|
||||
However, it depends on [pkg-config](https://freedesktop.org/wiki/Software/pkg-config/) being installed:
|
||||
|
||||
### macOS
|
||||
|
||||
```console
|
||||
$ # Ensure pkg-config is installed
|
||||
$ brew install pkg-config
|
||||
```
|
||||
|
||||
### Linux
|
||||
|
||||
```console
|
||||
$ # Ensure pkg-config is installed
|
||||
$ pacman -S pkgconf
|
||||
```
|
||||
|
||||
**Note:** Example shown is for Arch Linux, but likewise procedure can be
|
||||
followed using any other package manager. Some distributions may have slight
|
||||
variation of package names (e.g. `apt install -y pkg-config`).
|
||||
|
||||
In addition to the above, the following dependencies are also used by some of the `make` targets:
|
||||
|
||||
- `controller-gen` (v0.7.0)
|
||||
|
|
@ -149,18 +128,11 @@ Create a `.vscode/launch.json` file:
|
|||
"type": "go",
|
||||
"request": "launch",
|
||||
"mode": "auto",
|
||||
"envFile": "${workspaceFolder}/build/.env",
|
||||
"program": "${workspaceFolder}/main.go"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Create the environment file containing details on how to load
|
||||
`libgit2` dependencies:
|
||||
```bash
|
||||
make env
|
||||
```
|
||||
|
||||
Start debugging by either clicking `Run` > `Start Debugging` or using
|
||||
the relevant shortcut.
|
||||
|
|
|
|||
40
Dockerfile
40
Dockerfile
|
|
@ -2,11 +2,6 @@ ARG BASE_VARIANT=alpine
|
|||
ARG GO_VERSION=1.19
|
||||
ARG XX_VERSION=1.1.2
|
||||
|
||||
ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2-only
|
||||
ARG LIBGIT2_TAG=v0.4.0
|
||||
|
||||
FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} AS libgit2-libs
|
||||
|
||||
FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
|
||||
|
||||
FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as gostable
|
||||
|
|
@ -17,12 +12,18 @@ FROM gostable AS go-linux
|
|||
# These will be used at current arch to yield execute the cross compilations.
|
||||
FROM go-${TARGETOS} AS build-base
|
||||
|
||||
RUN apk add --no-cache clang lld pkgconfig
|
||||
RUN apk add --no-cache clang lld
|
||||
|
||||
COPY --from=xx / /
|
||||
|
||||
# build-go-mod can still be cached at build platform architecture.
|
||||
FROM build-base as build-go-mod
|
||||
FROM build-base as build
|
||||
|
||||
ARG TARGETPLATFORM
|
||||
|
||||
# Some dependencies have to installed
|
||||
# for the target platform: https://github.com/tonistiigi/xx#go--cgo
|
||||
RUN xx-apk add musl-dev gcc clang lld
|
||||
|
||||
# Configure workspace
|
||||
WORKDIR /workspace
|
||||
|
|
@ -37,21 +38,6 @@ COPY go.sum go.sum
|
|||
# Cache modules
|
||||
RUN go mod download
|
||||
|
||||
|
||||
# Build stage install per target platform
|
||||
# dependency and effectively cross compile the application.
|
||||
FROM build-go-mod as build
|
||||
|
||||
ARG TARGETPLATFORM
|
||||
|
||||
COPY --from=libgit2-libs /usr/local/ /usr/local/
|
||||
|
||||
# Some dependencies have to installed
|
||||
# for the target platform: https://github.com/tonistiigi/xx#go--cgo
|
||||
RUN xx-apk add musl-dev gcc clang lld
|
||||
|
||||
WORKDIR /workspace
|
||||
|
||||
# Copy source code
|
||||
COPY main.go main.go
|
||||
COPY controllers/ controllers/
|
||||
|
|
@ -60,11 +46,13 @@ COPY internal/ internal/
|
|||
|
||||
ARG TARGETPLATFORM
|
||||
ARG TARGETARCH
|
||||
|
||||
# Reasons why CGO is in use:
|
||||
# - The SHA1 implementation (sha1cd) used by go-git depends on CGO for
|
||||
# performance reasons. See: https://github.com/pjbgf/sha1cd/issues/15
|
||||
ENV CGO_ENABLED=1
|
||||
|
||||
# Instead of using xx-go, (cross) compile with vanilla go leveraging musl tool chain.
|
||||
RUN export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig" && \
|
||||
export CGO_LDFLAGS="$(pkg-config --static --libs --cflags libgit2) -static -fuse-ld=lld" && \
|
||||
RUN export CGO_LDFLAGS="-static -fuse-ld=lld" && \
|
||||
xx-go build \
|
||||
-ldflags "-s -w" \
|
||||
-tags 'netgo,osusergo,static_build' \
|
||||
|
|
@ -73,7 +61,6 @@ RUN export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig" && \
|
|||
# Ensure that the binary was cross-compiled correctly to the target platform.
|
||||
RUN xx-verify --static /source-controller
|
||||
|
||||
|
||||
FROM alpine:3.16
|
||||
|
||||
ARG TARGETPLATFORM
|
||||
|
|
@ -82,7 +69,6 @@ RUN apk --no-cache add ca-certificates \
|
|||
|
||||
# Copy over binary from build
|
||||
COPY --from=build /source-controller /usr/local/bin/
|
||||
COPY ATTRIBUTIONS.md /
|
||||
|
||||
USER 65534:65534
|
||||
ENTRYPOINT [ "source-controller" ]
|
||||
|
|
|
|||
66
Makefile
66
Makefile
|
|
@ -2,10 +2,6 @@
|
|||
IMG ?= fluxcd/source-controller
|
||||
TAG ?= latest
|
||||
|
||||
# Base image used to build the Go binary
|
||||
LIBGIT2_IMG ?= ghcr.io/fluxcd/golang-with-libgit2-only
|
||||
LIBGIT2_TAG ?= v0.4.0
|
||||
|
||||
# Allows for defining additional Go test args, e.g. '-tags integration'.
|
||||
GO_TEST_ARGS ?= -race
|
||||
|
||||
|
|
@ -39,14 +35,6 @@ ENVTEST_BIN_VERSION ?= 1.24.0
|
|||
# each fuzzer should run for.
|
||||
FUZZ_TIME ?= 1m
|
||||
|
||||
# Caches libgit2 versions per tag, "forcing" rebuild only when needed.
|
||||
LIBGIT2_PATH := $(BUILD_DIR)/libgit2/$(LIBGIT2_TAG)
|
||||
LIBGIT2_LIB_PATH := $(LIBGIT2_PATH)/lib
|
||||
LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.a
|
||||
|
||||
export CGO_ENABLED=1
|
||||
export PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig
|
||||
export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libgit2 2>/dev/null)
|
||||
GO_STATIC_FLAGS=-ldflags "-s -w" -tags 'netgo,osusergo,static_build$(addprefix ,,$(GO_TAGS))'
|
||||
|
||||
# API (doc) generation utilities
|
||||
|
|
@ -75,11 +63,11 @@ endif
|
|||
|
||||
all: build
|
||||
|
||||
build: check-deps $(LIBGIT2) ## Build manager binary
|
||||
build: check-deps ## Build manager binary
|
||||
go build $(GO_STATIC_FLAGS) -o $(BUILD_DIR)/bin/manager main.go
|
||||
|
||||
KUBEBUILDER_ASSETS?="$(shell $(ENVTEST) --arch=$(ENVTEST_ARCH) use -i $(ENVTEST_KUBERNETES_VERSION) --bin-dir=$(ENVTEST_ASSETS_DIR) -p path)"
|
||||
test: $(LIBGIT2) install-envtest test-api check-deps ## Run all tests
|
||||
test: install-envtest test-api check-deps ## Run all tests
|
||||
HTTPS_PROXY="" HTTP_PROXY="" \
|
||||
KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) \
|
||||
GIT_CONFIG_GLOBAL=/dev/null \
|
||||
|
|
@ -88,7 +76,7 @@ test: $(LIBGIT2) install-envtest test-api check-deps ## Run all tests
|
|||
$(GO_TEST_ARGS) \
|
||||
-coverprofile cover.out
|
||||
|
||||
test-ctrl: $(LIBGIT2) install-envtest test-api check-deps ## Run controller tests
|
||||
test-ctrl: install-envtest test-api check-deps ## Run controller tests
|
||||
HTTPS_PROXY="" HTTP_PROXY="" \
|
||||
KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) \
|
||||
GIT_CONFIG_GLOBAL=/dev/null \
|
||||
|
|
@ -105,7 +93,7 @@ endif
|
|||
test-api: ## Run api tests
|
||||
cd api; go test $(GO_TEST_ARGS) ./... -coverprofile cover.out
|
||||
|
||||
run: $(LIBGIT2) generate fmt vet manifests ## Run against the configured Kubernetes cluster in ~/.kube/config
|
||||
run: generate fmt vet manifests ## Run against the configured Kubernetes cluster in ~/.kube/config
|
||||
go run $(GO_STATIC_FLAGS) ./main.go
|
||||
|
||||
install: manifests ## Install CRDs into a cluster
|
||||
|
|
@ -139,7 +127,7 @@ fmt: ## Run go fmt against code
|
|||
go fmt ./...
|
||||
cd api; go fmt ./...
|
||||
|
||||
vet: $(LIBGIT2) ## Run go vet against code
|
||||
vet: ## Run go vet against code
|
||||
go vet ./...
|
||||
cd api; go vet ./...
|
||||
|
||||
|
|
@ -148,8 +136,6 @@ generate: controller-gen ## Generate API code
|
|||
|
||||
docker-build: ## Build the Docker image
|
||||
docker buildx build \
|
||||
--build-arg LIBGIT2_IMG=$(LIBGIT2_IMG) \
|
||||
--build-arg LIBGIT2_TAG=$(LIBGIT2_TAG) \
|
||||
--platform=$(BUILD_PLATFORMS) \
|
||||
-t $(IMG):$(TAG) \
|
||||
$(BUILD_ARGS) .
|
||||
|
|
@ -182,40 +168,14 @@ install-envtest: setup-envtest ## Download envtest binaries locally.
|
|||
# setup-envtest sets anything below k8s to 0555
|
||||
chmod -R u+w $(BUILD_DIR)/testbin
|
||||
|
||||
libgit2: $(LIBGIT2) ## Detect or download libgit2 library
|
||||
|
||||
COSIGN = $(GOBIN)/cosign
|
||||
$(LIBGIT2):
|
||||
$(call go-install-tool,$(COSIGN),github.com/sigstore/cosign/cmd/cosign@latest)
|
||||
|
||||
IMG=$(LIBGIT2_IMG) TAG=$(LIBGIT2_TAG) PATH=$(PATH):$(GOBIN) ./hack/install-libraries.sh
|
||||
|
||||
|
||||
.PHONY: help
|
||||
help: ## Display this help menu
|
||||
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
|
||||
|
||||
update-attributions:
|
||||
./hack/update-attributions.sh
|
||||
|
||||
e2e:
|
||||
./hack/ci/e2e.sh
|
||||
|
||||
verify: update-attributions fmt vet manifests api-docs tidy
|
||||
ifneq ($(shell grep -o 'LIBGIT2_IMG ?= \w.*' Makefile | cut -d ' ' -f 3):$(shell grep -o 'LIBGIT2_TAG ?= \w.*' Makefile | cut -d ' ' -f 3), \
|
||||
$(shell grep -o "LIBGIT2_IMG=\w.*" Dockerfile | cut -d'=' -f2):$(shell grep -o "LIBGIT2_TAG=\w.*" Dockerfile | cut -d'=' -f2))
|
||||
@{ \
|
||||
echo "LIBGIT2_IMG and LIBGIT2_TAG must match in both Makefile and Dockerfile"; \
|
||||
exit 1; \
|
||||
}
|
||||
endif
|
||||
ifneq ($(shell grep -o 'LIBGIT2_TAG ?= \w.*' Makefile | cut -d ' ' -f 3), $(shell grep -o "LIBGIT2_TAG=.*" tests/fuzz/oss_fuzz_prebuild.sh | sed 's;LIBGIT2_TAG="$${LIBGIT2_TAG:-;;g' | sed 's;}";;g'))
|
||||
@{ \
|
||||
echo "LIBGIT2_TAG must match in both Makefile and tests/fuzz/oss_fuzz_prebuild.sh"; \
|
||||
exit 1; \
|
||||
}
|
||||
endif
|
||||
|
||||
verify: fmt vet manifests api-docs tidy
|
||||
@if [ ! "$$(git status --porcelain --untracked-files=no)" = "" ]; then \
|
||||
echo "working directory is dirty:"; \
|
||||
git --no-pager diff; \
|
||||
|
|
@ -236,7 +196,7 @@ rm -rf $$TMP_DIR ;\
|
|||
endef
|
||||
|
||||
# Build fuzzers used by oss-fuzz.
|
||||
fuzz-build: $(LIBGIT2)
|
||||
fuzz-build:
|
||||
rm -rf $(shell pwd)/build/fuzz/
|
||||
mkdir -p $(shell pwd)/build/fuzz/out/
|
||||
|
||||
|
|
@ -260,15 +220,3 @@ fuzz-native:
|
|||
KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) \
|
||||
FUZZ_TIME=$(FUZZ_TIME) \
|
||||
./tests/fuzz/native_go_run.sh
|
||||
|
||||
# Creates an env file that can be used to load all source-controller's dependencies
|
||||
# this is handy when you want to run adhoc debug sessions on tests or start the
|
||||
# controller in a new debug session.
|
||||
env: $(LIBGIT2)
|
||||
echo 'GO_ENABLED="1"' > $(BUILD_DIR)/.env
|
||||
echo 'PKG_CONFIG_PATH="$(PKG_CONFIG_PATH)"' >> $(BUILD_DIR)/.env
|
||||
echo 'LIBRARY_PATH="$(LIBRARY_PATH)"' >> $(BUILD_DIR)/.env
|
||||
echo 'CGO_CFLAGS="$(CGO_CFLAGS)"' >> $(BUILD_DIR)/.env
|
||||
echo 'CGO_LDFLAGS="$(CGO_LDFLAGS)"' >> $(BUILD_DIR)/.env
|
||||
echo 'KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS)' >> $(BUILD_DIR)/.env
|
||||
echo 'GIT_CONFIG_GLOBAL=/dev/null' >> $(BUILD_DIR)/.env
|
||||
|
|
|
|||
|
|
@ -97,6 +97,8 @@ type GitRepositorySpec struct {
|
|||
|
||||
// GitImplementation specifies which Git client library implementation to
|
||||
// use. Defaults to 'go-git', valid values are ('go-git', 'libgit2').
|
||||
// Deprecated: gitImplementation is deprecated now that 'go-git' is the
|
||||
// only supported implementation.
|
||||
// +kubebuilder:validation:Enum=go-git;libgit2
|
||||
// +kubebuilder:default:=go-git
|
||||
// +optional
|
||||
|
|
|
|||
|
|
@ -411,9 +411,10 @@ spec:
|
|||
type: object
|
||||
gitImplementation:
|
||||
default: go-git
|
||||
description: GitImplementation specifies which Git client library
|
||||
implementation to use. Defaults to 'go-git', valid values are ('go-git',
|
||||
'libgit2').
|
||||
description: 'GitImplementation specifies which Git client library
|
||||
implementation to use. Defaults to ''go-git'', valid values are
|
||||
(''go-git'', ''libgit2''). Deprecated: gitImplementation is deprecated
|
||||
now that ''go-git'' is the only supported implementation.'
|
||||
enum:
|
||||
- go-git
|
||||
- libgit2
|
||||
|
|
|
|||
|
|
@ -1,24 +1,8 @@
|
|||
apiVersion: source.toolkit.fluxcd.io/v1beta1
|
||||
kind: GitRepository
|
||||
metadata:
|
||||
name: large-repo-go-git
|
||||
name: large-repo
|
||||
spec:
|
||||
gitImplementation: go-git
|
||||
interval: 10m
|
||||
timeout: 2m
|
||||
url: https://github.com/hashgraph/hedera-mirror-node.git
|
||||
ref:
|
||||
branch: main
|
||||
ignore: |
|
||||
/*
|
||||
!/charts
|
||||
---
|
||||
apiVersion: source.toolkit.fluxcd.io/v1beta1
|
||||
kind: GitRepository
|
||||
metadata:
|
||||
name: large-repo-libgit2
|
||||
spec:
|
||||
gitImplementation: libgit2
|
||||
interval: 10m
|
||||
timeout: 2m
|
||||
url: https://github.com/hashgraph/hedera-mirror-node.git
|
||||
|
|
|
|||
|
|
@ -45,7 +45,6 @@ import (
|
|||
"github.com/fluxcd/pkg/apis/meta"
|
||||
"github.com/fluxcd/pkg/git"
|
||||
"github.com/fluxcd/pkg/git/gogit"
|
||||
"github.com/fluxcd/pkg/git/libgit2"
|
||||
"github.com/fluxcd/pkg/git/repository"
|
||||
"github.com/fluxcd/pkg/runtime/conditions"
|
||||
helper "github.com/fluxcd/pkg/runtime/controller"
|
||||
|
|
@ -116,9 +115,6 @@ type GitRepositoryReconciler struct {
|
|||
|
||||
Storage *Storage
|
||||
ControllerName string
|
||||
// Libgit2TransportInitialized lets the reconciler know whether
|
||||
// libgit2 transport was intialized successfully.
|
||||
Libgit2TransportInitialized func() bool
|
||||
|
||||
requeueDependency time.Duration
|
||||
features map[string]bool
|
||||
|
|
@ -423,18 +419,6 @@ func (r *GitRepositoryReconciler) reconcileStorage(ctx context.Context,
|
|||
// change, it short-circuits the whole reconciliation with an early return.
|
||||
func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
|
||||
obj *sourcev1.GitRepository, commit *git.Commit, includes *artifactSet, dir string) (sreconcile.Result, error) {
|
||||
gitImplementation := obj.Spec.GitImplementation
|
||||
if goGitOnly, _ := r.features[features.ForceGoGitImplementation]; goGitOnly {
|
||||
gitImplementation = sourcev1.GoGitImplementation
|
||||
}
|
||||
|
||||
// Exit early, if we need to use libgit2 AND managed transport hasn't been intialized.
|
||||
if !r.Libgit2TransportInitialized() && gitImplementation == sourcev1.LibGit2Implementation {
|
||||
return sreconcile.ResultEmpty, serror.NewStalling(
|
||||
errors.New("libgit2 managed transport not initialized"), "Libgit2TransportNotEnabled",
|
||||
)
|
||||
}
|
||||
|
||||
// Remove previously failed source verification status conditions. The
|
||||
// failing verification should be recalculated. But an existing successful
|
||||
// verification need not be removed as it indicates verification of previous
|
||||
|
|
@ -505,7 +489,7 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
|
|||
optimizedClone = true
|
||||
}
|
||||
|
||||
c, err := r.gitCheckout(ctx, obj, authOpts, dir, optimizedClone, gitImplementation)
|
||||
c, err := r.gitCheckout(ctx, obj, authOpts, dir, optimizedClone)
|
||||
if err != nil {
|
||||
return sreconcile.ResultEmpty, err
|
||||
}
|
||||
|
|
@ -539,7 +523,7 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
|
|||
|
||||
// If we can't skip the reconciliation, checkout again without any
|
||||
// optimization.
|
||||
c, err := r.gitCheckout(ctx, obj, authOpts, dir, false, gitImplementation)
|
||||
c, err := r.gitCheckout(ctx, obj, authOpts, dir, false)
|
||||
if err != nil {
|
||||
return sreconcile.ResultEmpty, err
|
||||
}
|
||||
|
|
@ -732,7 +716,7 @@ func (r *GitRepositoryReconciler) reconcileInclude(ctx context.Context,
|
|||
// performs a git checkout.
|
||||
func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context,
|
||||
obj *sourcev1.GitRepository, authOpts *git.AuthOptions, dir string,
|
||||
optimized bool, gitImplementation string) (*git.Commit, error) {
|
||||
optimized bool) (*git.Commit, error) {
|
||||
// Configure checkout strategy.
|
||||
cloneOpts := repository.CloneOptions{
|
||||
RecurseSubmodules: obj.Spec.RecurseSubmodules,
|
||||
|
|
@ -757,28 +741,15 @@ func (r *GitRepositoryReconciler) gitCheckout(ctx context.Context,
|
|||
gitCtx, cancel := context.WithTimeout(ctx, obj.Spec.Timeout.Duration)
|
||||
defer cancel()
|
||||
|
||||
var gitReader repository.Reader
|
||||
var err error
|
||||
|
||||
switch gitImplementation {
|
||||
case sourcev1.LibGit2Implementation:
|
||||
clientOpts := []libgit2.ClientOption{libgit2.WithDiskStorage()}
|
||||
if authOpts.Transport == git.HTTP {
|
||||
clientOpts = append(clientOpts, libgit2.WithInsecureCredentialsOverHTTP())
|
||||
}
|
||||
gitReader, err = libgit2.NewClient(dir, authOpts, clientOpts...)
|
||||
case sourcev1.GoGitImplementation:
|
||||
clientOpts := []gogit.ClientOption{gogit.WithDiskStorage()}
|
||||
if authOpts.Transport == git.HTTP {
|
||||
clientOpts = append(clientOpts, gogit.WithInsecureCredentialsOverHTTP())
|
||||
}
|
||||
gitReader, err = gogit.NewClient(dir, authOpts, clientOpts...)
|
||||
default:
|
||||
err = fmt.Errorf("invalid Git implementation: %s", gitImplementation)
|
||||
clientOpts := []gogit.ClientOption{gogit.WithDiskStorage()}
|
||||
if authOpts.Transport == git.HTTP {
|
||||
clientOpts = append(clientOpts, gogit.WithInsecureCredentialsOverHTTP())
|
||||
}
|
||||
|
||||
gitReader, err := gogit.NewClient(dir, authOpts, clientOpts...)
|
||||
if err != nil {
|
||||
e := serror.NewGeneric(
|
||||
fmt.Errorf("failed to create Git client for implementation '%s': %w", gitImplementation, err),
|
||||
fmt.Errorf("failed to create Git client: %w", err),
|
||||
sourcev1.GitOperationFailedReason,
|
||||
)
|
||||
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
|
||||
|
|
|
|||
|
|
@ -57,7 +57,6 @@ import (
|
|||
"github.com/fluxcd/pkg/testserver"
|
||||
|
||||
"github.com/fluxcd/pkg/git"
|
||||
"github.com/fluxcd/pkg/git/libgit2/transport"
|
||||
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
|
||||
serror "github.com/fluxcd/source-controller/internal/error"
|
||||
"github.com/fluxcd/source-controller/internal/features"
|
||||
|
|
@ -145,14 +144,6 @@ Oomb3gD/TRf/nAdVED+k81GdLzciYdUGtI71/qI47G0nMBluLRE=
|
|||
`
|
||||
)
|
||||
|
||||
var (
|
||||
testGitImplementations = []string{sourcev1.GoGitImplementation, sourcev1.LibGit2Implementation}
|
||||
)
|
||||
|
||||
func mockTransportNotInitialized() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func TestGitRepositoryReconciler_Reconcile(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
|
||||
|
|
@ -240,15 +231,14 @@ func TestGitRepositoryReconciler_reconcileSource_authStrategy(t *testing.T) {
|
|||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
skipForImplementation string
|
||||
protocol string
|
||||
server options
|
||||
secret *corev1.Secret
|
||||
beforeFunc func(obj *sourcev1.GitRepository)
|
||||
want sreconcile.Result
|
||||
wantErr bool
|
||||
assertConditions []metav1.Condition
|
||||
name string
|
||||
protocol string
|
||||
server options
|
||||
secret *corev1.Secret
|
||||
beforeFunc func(obj *sourcev1.GitRepository)
|
||||
want sreconcile.Result
|
||||
wantErr bool
|
||||
assertConditions []metav1.Condition
|
||||
}{
|
||||
{
|
||||
name: "HTTP without secretRef makes ArtifactOutdated=True",
|
||||
|
|
@ -310,9 +300,8 @@ func TestGitRepositoryReconciler_reconcileSource_authStrategy(t *testing.T) {
|
|||
},
|
||||
},
|
||||
{
|
||||
name: "HTTPS with invalid CAFile secret makes CheckoutFailed=True and returns error",
|
||||
skipForImplementation: sourcev1.LibGit2Implementation,
|
||||
protocol: "https",
|
||||
name: "HTTPS with invalid CAFile secret makes CheckoutFailed=True and returns error",
|
||||
protocol: "https",
|
||||
server: options{
|
||||
publicKey: tlsPublicKey,
|
||||
privateKey: tlsPrivateKey,
|
||||
|
|
@ -339,31 +328,6 @@ func TestGitRepositoryReconciler_reconcileSource_authStrategy(t *testing.T) {
|
|||
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.GitOperationFailedReason, "x509: "),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "HTTPS with invalid CAFile secret makes CheckoutFailed=True and returns error",
|
||||
skipForImplementation: sourcev1.GoGitImplementation,
|
||||
protocol: "https",
|
||||
server: options{
|
||||
publicKey: tlsPublicKey,
|
||||
privateKey: tlsPrivateKey,
|
||||
ca: tlsCA,
|
||||
},
|
||||
secret: &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "invalid-ca",
|
||||
},
|
||||
Data: map[string][]byte{
|
||||
"caFile": []byte("invalid"),
|
||||
},
|
||||
},
|
||||
beforeFunc: func(obj *sourcev1.GitRepository) {
|
||||
obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "invalid-ca"}
|
||||
},
|
||||
wantErr: true,
|
||||
assertConditions: []metav1.Condition{
|
||||
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.GitOperationFailedReason, "failed to checkout and determine revision: unable to fetch-connect to remote '<url>': PEM CA bundle could not be appended to x509 certificate pool"),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "SSH with private key secret makes ArtifactOutdated=True",
|
||||
protocol: "ssh",
|
||||
|
|
@ -503,82 +467,36 @@ func TestGitRepositoryReconciler_reconcileSource_authStrategy(t *testing.T) {
|
|||
Storage: testStorage,
|
||||
features: map[string]bool{
|
||||
features.OptimizedGitClones: true,
|
||||
// Ensure that both implementations are tested.
|
||||
features.ForceGoGitImplementation: false,
|
||||
},
|
||||
Libgit2TransportInitialized: transport.Enabled,
|
||||
}
|
||||
|
||||
for _, i := range testGitImplementations {
|
||||
t.Run(i, func(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
t.Run(sourcev1.GoGitImplementation, func(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
|
||||
if tt.skipForImplementation == i {
|
||||
t.Skipf("Skipped for Git implementation %q", i)
|
||||
}
|
||||
tmpDir := t.TempDir()
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
obj := obj.DeepCopy()
|
||||
|
||||
obj := obj.DeepCopy()
|
||||
obj.Spec.GitImplementation = i
|
||||
head, _ := localRepo.Head()
|
||||
assertConditions := tt.assertConditions
|
||||
for k := range assertConditions {
|
||||
assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "<commit>", head.Hash().String())
|
||||
assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "<url>", obj.Spec.URL)
|
||||
}
|
||||
|
||||
head, _ := localRepo.Head()
|
||||
assertConditions := tt.assertConditions
|
||||
for k := range assertConditions {
|
||||
assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "<commit>", head.Hash().String())
|
||||
assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "<url>", obj.Spec.URL)
|
||||
}
|
||||
var commit git.Commit
|
||||
var includes artifactSet
|
||||
|
||||
var commit git.Commit
|
||||
var includes artifactSet
|
||||
|
||||
got, err := r.reconcileSource(context.TODO(), obj, &commit, &includes, tmpDir)
|
||||
g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.assertConditions))
|
||||
g.Expect(err != nil).To(Equal(tt.wantErr))
|
||||
g.Expect(got).To(Equal(tt.want))
|
||||
g.Expect(commit).ToNot(BeNil())
|
||||
})
|
||||
}
|
||||
got, err := r.reconcileSource(context.TODO(), obj, &commit, &includes, tmpDir)
|
||||
g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.assertConditions))
|
||||
g.Expect(err != nil).To(Equal(tt.wantErr))
|
||||
g.Expect(got).To(Equal(tt.want))
|
||||
g.Expect(commit).ToNot(BeNil())
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitRepositoryReconciler_reconcileSource_libgit2TransportUninitialized(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
|
||||
r := &GitRepositoryReconciler{
|
||||
Client: fakeclient.NewClientBuilder().WithScheme(runtime.NewScheme()).Build(),
|
||||
EventRecorder: record.NewFakeRecorder(32),
|
||||
Storage: testStorage,
|
||||
features: map[string]bool{
|
||||
features.ForceGoGitImplementation: false,
|
||||
},
|
||||
Libgit2TransportInitialized: mockTransportNotInitialized,
|
||||
}
|
||||
|
||||
obj := &sourcev1.GitRepository{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
GenerateName: "libgit2-transport",
|
||||
},
|
||||
Spec: sourcev1.GitRepositorySpec{
|
||||
Interval: metav1.Duration{Duration: interval},
|
||||
Timeout: &metav1.Duration{Duration: timeout},
|
||||
Reference: &sourcev1.GitRepositoryRef{
|
||||
Branch: git.DefaultBranch,
|
||||
},
|
||||
GitImplementation: sourcev1.LibGit2Implementation,
|
||||
},
|
||||
}
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
var commit git.Commit
|
||||
var includes artifactSet
|
||||
_, err := r.reconcileSource(ctx, obj, &commit, &includes, tmpDir)
|
||||
g.Expect(err).To(HaveOccurred())
|
||||
g.Expect(err).To(BeAssignableToTypeOf(&serror.Stalling{}))
|
||||
g.Expect(err.Error()).To(Equal("libgit2 managed transport not initialized"))
|
||||
}
|
||||
|
||||
func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
|
||||
|
|
@ -586,14 +504,13 @@ func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T)
|
|||
tags := []string{"non-semver-tag", "v0.1.0", "0.2.0", "v0.2.1", "v1.0.0-alpha", "v1.1.0", "v2.0.0"}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
skipForImplementation string
|
||||
reference *sourcev1.GitRepositoryRef
|
||||
beforeFunc func(obj *sourcev1.GitRepository, latestRev string)
|
||||
want sreconcile.Result
|
||||
wantErr bool
|
||||
wantRevision string
|
||||
wantArtifactOutdated bool
|
||||
name string
|
||||
reference *sourcev1.GitRepositoryRef
|
||||
beforeFunc func(obj *sourcev1.GitRepository, latestRev string)
|
||||
want sreconcile.Result
|
||||
wantErr bool
|
||||
wantRevision string
|
||||
wantArtifactOutdated bool
|
||||
}{
|
||||
{
|
||||
name: "Nil reference (default branch)",
|
||||
|
|
@ -620,8 +537,7 @@ func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T)
|
|||
wantArtifactOutdated: true,
|
||||
},
|
||||
{
|
||||
name: "Branch commit",
|
||||
skipForImplementation: sourcev1.LibGit2Implementation,
|
||||
name: "Branch commit",
|
||||
reference: &sourcev1.GitRepositoryRef{
|
||||
Branch: "staging",
|
||||
Commit: "<commit>",
|
||||
|
|
@ -630,17 +546,6 @@ func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T)
|
|||
wantRevision: "staging/<commit>",
|
||||
wantArtifactOutdated: true,
|
||||
},
|
||||
{
|
||||
name: "Branch commit",
|
||||
skipForImplementation: sourcev1.GoGitImplementation,
|
||||
reference: &sourcev1.GitRepositoryRef{
|
||||
Branch: "staging",
|
||||
Commit: "<commit>",
|
||||
},
|
||||
want: sreconcile.ResultSuccess,
|
||||
wantRevision: "HEAD/<commit>",
|
||||
wantArtifactOutdated: true,
|
||||
},
|
||||
{
|
||||
name: "SemVer",
|
||||
reference: &sourcev1.GitRepositoryRef{
|
||||
|
|
@ -738,10 +643,7 @@ func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T)
|
|||
Storage: testStorage,
|
||||
features: map[string]bool{
|
||||
features.OptimizedGitClones: true,
|
||||
// Ensure that both implementations are tested.
|
||||
features.ForceGoGitImplementation: false,
|
||||
},
|
||||
Libgit2TransportInitialized: transport.Enabled,
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
|
@ -762,38 +664,30 @@ func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T)
|
|||
obj.Spec.Reference.Commit = headRef.Hash().String()
|
||||
}
|
||||
|
||||
for _, i := range testGitImplementations {
|
||||
t.Run(i, func(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
t.Run(sourcev1.GoGitImplementation, func(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
|
||||
if tt.skipForImplementation == i {
|
||||
t.Skipf("Skipped for Git implementation %q", i)
|
||||
}
|
||||
tmpDir := t.TempDir()
|
||||
obj := obj.DeepCopy()
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
if tt.beforeFunc != nil {
|
||||
tt.beforeFunc(obj, headRef.Hash().String())
|
||||
}
|
||||
|
||||
obj := obj.DeepCopy()
|
||||
obj.Spec.GitImplementation = i
|
||||
|
||||
if tt.beforeFunc != nil {
|
||||
tt.beforeFunc(obj, headRef.Hash().String())
|
||||
}
|
||||
|
||||
var commit git.Commit
|
||||
var includes artifactSet
|
||||
got, err := r.reconcileSource(ctx, obj, &commit, &includes, tmpDir)
|
||||
if err != nil {
|
||||
println(err.Error())
|
||||
}
|
||||
g.Expect(err != nil).To(Equal(tt.wantErr))
|
||||
g.Expect(got).To(Equal(tt.want))
|
||||
if tt.wantRevision != "" && !tt.wantErr {
|
||||
revision := strings.ReplaceAll(tt.wantRevision, "<commit>", headRef.Hash().String())
|
||||
g.Expect(commit.String()).To(Equal(revision))
|
||||
g.Expect(conditions.IsTrue(obj, sourcev1.ArtifactOutdatedCondition)).To(Equal(tt.wantArtifactOutdated))
|
||||
}
|
||||
})
|
||||
}
|
||||
var commit git.Commit
|
||||
var includes artifactSet
|
||||
got, err := r.reconcileSource(ctx, obj, &commit, &includes, tmpDir)
|
||||
if err != nil {
|
||||
println(err.Error())
|
||||
}
|
||||
g.Expect(err != nil).To(Equal(tt.wantErr))
|
||||
g.Expect(got).To(Equal(tt.want))
|
||||
if tt.wantRevision != "" && !tt.wantErr {
|
||||
revision := strings.ReplaceAll(tt.wantRevision, "<commit>", headRef.Hash().String())
|
||||
g.Expect(commit.String()).To(Equal(revision))
|
||||
g.Expect(conditions.IsTrue(obj, sourcev1.ArtifactOutdatedCondition)).To(Equal(tt.wantArtifactOutdated))
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -1605,11 +1499,10 @@ func TestGitRepositoryReconciler_ConditionsUpdate(t *testing.T) {
|
|||
builder := fakeclient.NewClientBuilder().WithScheme(testEnv.GetScheme()).WithObjects(obj)
|
||||
|
||||
r := &GitRepositoryReconciler{
|
||||
Client: builder.Build(),
|
||||
EventRecorder: record.NewFakeRecorder(32),
|
||||
Storage: testStorage,
|
||||
features: features.FeatureGates(),
|
||||
Libgit2TransportInitialized: transport.Enabled,
|
||||
Client: builder.Build(),
|
||||
EventRecorder: record.NewFakeRecorder(32),
|
||||
Storage: testStorage,
|
||||
features: features.FeatureGates(),
|
||||
}
|
||||
|
||||
key := client.ObjectKeyFromObject(obj)
|
||||
|
|
|
|||
|
|
@ -37,7 +37,6 @@ import (
|
|||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
|
||||
dcontext "github.com/distribution/distribution/v3/context"
|
||||
"github.com/fluxcd/pkg/git/libgit2/transport"
|
||||
"github.com/fluxcd/pkg/runtime/controller"
|
||||
"github.com/fluxcd/pkg/runtime/testenv"
|
||||
"github.com/fluxcd/pkg/testserver"
|
||||
|
|
@ -48,7 +47,6 @@ import (
|
|||
dockerRegistry "github.com/distribution/distribution/v3/registry"
|
||||
_ "github.com/distribution/distribution/v3/registry/auth/htpasswd"
|
||||
_ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory"
|
||||
git2go "github.com/libgit2/git2go/v34"
|
||||
|
||||
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
|
||||
"github.com/fluxcd/source-controller/internal/cache"
|
||||
|
|
@ -203,8 +201,6 @@ func setupRegistryServer(ctx context.Context, workspaceDir string, opts registry
|
|||
}
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
mustHaveNoThreadSupport()
|
||||
|
||||
initTestTLS()
|
||||
|
||||
utilruntime.Must(sourcev1.AddToScheme(scheme.Scheme))
|
||||
|
|
@ -237,10 +233,6 @@ func TestMain(m *testing.M) {
|
|||
panic(fmt.Sprintf("Failed to create a test registry server: %v", err))
|
||||
}
|
||||
|
||||
if err = transport.InitManagedTransport(); err != nil {
|
||||
panic(fmt.Sprintf("Failed to initialize libgit2 managed transport: %v", err))
|
||||
}
|
||||
|
||||
if err := (&GitRepositoryReconciler{
|
||||
Client: testEnv,
|
||||
EventRecorder: record.NewFakeRecorder(32),
|
||||
|
|
@ -248,10 +240,7 @@ func TestMain(m *testing.M) {
|
|||
Storage: testStorage,
|
||||
features: map[string]bool{
|
||||
features.OptimizedGitClones: true,
|
||||
// Ensure that both implementations are used during tests.
|
||||
features.ForceGoGitImplementation: false,
|
||||
},
|
||||
Libgit2TransportInitialized: transport.Enabled,
|
||||
}).SetupWithManager(testEnv); err != nil {
|
||||
panic(fmt.Sprintf("Failed to start GitRepositoryReconciler: %v", err))
|
||||
}
|
||||
|
|
@ -378,22 +367,3 @@ func randStringRunes(n int) string {
|
|||
func int64p(i int64) *int64 {
|
||||
return &i
|
||||
}
|
||||
|
||||
// This provides a regression assurance for image-automation-controller/#339.
|
||||
// Validates that:
|
||||
// - libgit2 was built with no support for threads.
|
||||
// - git2go accepts libgit2 built with no support for threads.
|
||||
//
|
||||
// The logic below does the validation of the former, whilst
|
||||
// referring to git2go forces its init() execution, which is
|
||||
// where any validation to that effect resides.
|
||||
//
|
||||
// git2go does not support threadless libgit2 by default,
|
||||
// hence a fork is being used which disables such validation.
|
||||
//
|
||||
// TODO: extract logic into pkg.
|
||||
func mustHaveNoThreadSupport() {
|
||||
if git2go.Features()&git2go.FeatureThreads != 0 {
|
||||
panic("libgit2 must not be build with thread support")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -421,7 +421,9 @@ string
|
|||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>GitImplementation specifies which Git client library implementation to
|
||||
use. Defaults to ‘go-git’, valid values are (‘go-git’, ‘libgit2’).</p>
|
||||
use. Defaults to ‘go-git’, valid values are (‘go-git’, ‘libgit2’).
|
||||
Deprecated: gitImplementation is deprecated now that ‘go-git’ is the
|
||||
only supported implementation.</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
|
@ -1845,7 +1847,9 @@ string
|
|||
<td>
|
||||
<em>(Optional)</em>
|
||||
<p>GitImplementation specifies which Git client library implementation to
|
||||
use. Defaults to ‘go-git’, valid values are (‘go-git’, ‘libgit2’).</p>
|
||||
use. Defaults to ‘go-git’, valid values are (‘go-git’, ‘libgit2’).
|
||||
Deprecated: gitImplementation is deprecated now that ‘go-git’ is the
|
||||
only supported implementation.</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
|
|
|||
|
|
@ -228,8 +228,7 @@ spec:
|
|||
branch: <branch-name>
|
||||
```
|
||||
|
||||
Using the [`go-git` Git implementation](#git-implementation), this will perform
|
||||
a shallow clone to only fetch the specified branch.
|
||||
This will perform a shallow clone to only fetch the specified branch.
|
||||
|
||||
#### Tag example
|
||||
|
||||
|
|
@ -284,9 +283,9 @@ spec:
|
|||
commit: "<commit SHA>"
|
||||
```
|
||||
|
||||
This field takes precedence over all other fields. Using the [`go-git` Git
|
||||
implementation](#git-implementation), it can be combined with `.spec.ref.branch`
|
||||
to perform a shallow clone of the branch, in which the commit must exist:
|
||||
This field takes precedence over all other fields. It can be combined with
|
||||
`.spec.ref.branch` to perform a shallow clone of the branch, in which the
|
||||
commit must exist:
|
||||
|
||||
```yaml
|
||||
---
|
||||
|
|
@ -385,32 +384,13 @@ resume.
|
|||
|
||||
### Git implementation
|
||||
|
||||
**Note:** `libgit2` is being deprecated, as its use is known to cause controllers
|
||||
to panic when running over long periods of time, or when under high GC pressure.
|
||||
A new opt-out feature gate `ForceGoGitImplementation` was introduced, which will
|
||||
use `go-git` regardless of the value defined at `.spec.gitImplementation`.
|
||||
This can be disabled by starting the controller with the additional flag below:
|
||||
`--feature-gates=ForceGoGitImplementation=false`.
|
||||
|
||||
`.spec.gitImplementation` is an optional field to change the client library
|
||||
implementation used for Git operations (e.g. clone, checkout). The default
|
||||
value is `go-git`.
|
||||
|
||||
Unless you need support for a specific Git wire protocol functionality not
|
||||
supported by the default implementation (as documented below), changing the
|
||||
implementation is generally not recommended as it can come with its own set of
|
||||
drawbacks. For example, not being able to make use of shallow clones forces the
|
||||
controller to fetch the whole Git history tree instead of a specific one,
|
||||
resulting in an increase of disk space and traffic usage.
|
||||
|
||||
**Note:** The `libgit2` implementation does not support shallow clones or
|
||||
Git submodules.
|
||||
`.spec.gitImplementation` is deprecated and its value ignored, the git
|
||||
implementation used across Flux is go-git.
|
||||
|
||||
#### Optimized Git clones
|
||||
|
||||
Optimized Git clones decreases resource utilization for GitRepository
|
||||
reconciliations. It supports both `go-git` and `libgit2` implementations
|
||||
when cloning repositories using branches or tags.
|
||||
reconciliations.
|
||||
|
||||
When enabled, it avoids full Git clone operations by first checking whether
|
||||
the revision of the last stored artifact is still the head of the remote
|
||||
|
|
@ -428,20 +408,13 @@ not affected by this functionality.
|
|||
#### Proxy support
|
||||
|
||||
When a proxy is configured in the source-controller Pod through the appropriate
|
||||
environment variables, for example `HTTPS_PROXY`, `NO_PROXY`, etc. There may be
|
||||
some limitations in the proxy support based on the Git implementation.
|
||||
|
||||
| Git Implementation | HTTP_PROXY | HTTPS_PROXY | NO_PROXY | Self-signed Certs |
|
||||
|--------------------|------------|-------------|----------|-------------------|
|
||||
| `go-git` | true | true | true | false |n
|
||||
| `libgit2` | true | true | true | true |
|
||||
environment variables, for example `HTTPS_PROXY`, `NO_PROXY`, etc.
|
||||
|
||||
### Recurse submodules
|
||||
|
||||
`.spec.recurseSubmodules` is an optional field to enable the initialization of
|
||||
all submodules within the cloned Git repository, using their default settings.
|
||||
This option is only available when using the (default) `go-git` [Git
|
||||
implementation](#git-implementation), and defaults to `false`.
|
||||
This option defaults to `false`.
|
||||
|
||||
Note that for most Git providers (e.g. GitHub and GitLab), deploy keys can not
|
||||
be used as reusing a key across multiple repositories is not allowed. You have
|
||||
|
|
|
|||
13
go.mod
13
go.mod
|
|
@ -4,14 +4,6 @@ go 1.18
|
|||
|
||||
replace github.com/fluxcd/source-controller/api => ./api
|
||||
|
||||
// Flux has its own git2go fork to enable changes in behaviour for improved
|
||||
// reliability.
|
||||
//
|
||||
// For more information refer to:
|
||||
// - fluxcd/image-automation-controller/#339.
|
||||
// - libgit2/git2go#918.
|
||||
replace github.com/libgit2/git2go/v34 => github.com/fluxcd/git2go/v34 v34.0.0
|
||||
|
||||
// Fix CVE-2022-1996 (for v2, Go Modules incompatible)
|
||||
replace github.com/emicklei/go-restful => github.com/emicklei/go-restful v2.16.0+incompatible
|
||||
|
||||
|
|
@ -33,8 +25,7 @@ require (
|
|||
github.com/fluxcd/pkg/apis/event v0.2.0
|
||||
github.com/fluxcd/pkg/apis/meta v0.18.0
|
||||
github.com/fluxcd/pkg/git v0.7.0
|
||||
github.com/fluxcd/pkg/git/gogit v0.2.0
|
||||
github.com/fluxcd/pkg/git/libgit2 v0.3.0
|
||||
github.com/fluxcd/pkg/git/gogit v0.2.1-0.20221128153352-fbb8b9f29f38
|
||||
github.com/fluxcd/pkg/gittestserver v0.8.0
|
||||
github.com/fluxcd/pkg/helmtestserver v0.10.0
|
||||
github.com/fluxcd/pkg/lockedfile v0.1.0
|
||||
|
|
@ -52,7 +43,6 @@ require (
|
|||
github.com/google/go-containerregistry v0.12.1
|
||||
github.com/google/go-containerregistry/pkg/authn/k8schain v0.0.0-20221114162634-781782aa2757
|
||||
github.com/google/uuid v1.3.0
|
||||
github.com/libgit2/git2go/v34 v34.0.0
|
||||
github.com/minio/minio-go/v7 v7.0.43
|
||||
github.com/onsi/gomega v1.24.1
|
||||
github.com/otiai10/copy v1.9.0
|
||||
|
|
@ -178,7 +168,6 @@ require (
|
|||
github.com/felixge/httpsnoop v1.0.3 // indirect
|
||||
github.com/fluxcd/gitkit v0.6.0 // indirect
|
||||
github.com/fluxcd/pkg/apis/acl v0.1.0 // indirect
|
||||
github.com/fluxcd/pkg/http/transport v0.1.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.6.0 // indirect
|
||||
github.com/fullstorydev/grpcurl v1.8.7 // indirect
|
||||
github.com/go-chi/chi v4.1.2+incompatible // indirect
|
||||
|
|
|
|||
12
go.sum
12
go.sum
|
|
@ -502,8 +502,6 @@ github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw
|
|||
github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
|
||||
github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0=
|
||||
github.com/fluxcd/git2go/v34 v34.0.0 h1:08PEpdbsLO4sUTvFKuxmt6xBowaqObro9veChBWFwa8=
|
||||
github.com/fluxcd/git2go/v34 v34.0.0/go.mod h1:blVco2jDAw6YTXkErMMqzHLcAjKkwF0aWIRHBqiJkZ0=
|
||||
github.com/fluxcd/gitkit v0.6.0 h1:iNg5LTx6ePo+Pl0ZwqHTAkhbUHxGVSY3YCxCdw7VIFg=
|
||||
github.com/fluxcd/gitkit v0.6.0/go.mod h1:svOHuKi0fO9HoawdK4HfHAJJseZDHHjk7I3ihnCIqNo=
|
||||
github.com/fluxcd/go-git/v5 v5.0.0-20221104190732-329fd6659b10 h1:au798417R1iWtgcgKe3Dg495mexQmuxelL+NebAtexE=
|
||||
|
|
@ -516,16 +514,12 @@ github.com/fluxcd/pkg/apis/meta v0.18.0 h1:s0LeulWcQ4DxVX6805vgDTxlA6bAYk+Lq1QHS
|
|||
github.com/fluxcd/pkg/apis/meta v0.18.0/go.mod h1:pYvXRFi1UKNNrGR34jw3uqOnMXw9X6dTkML8j5Z7tis=
|
||||
github.com/fluxcd/pkg/git v0.7.0 h1:sQHRpFMcOzEdqlyGMjFv2LKMdcoE5xeUr2UcRrsLRG8=
|
||||
github.com/fluxcd/pkg/git v0.7.0/go.mod h1:3deiLPws4DSQ3hqwtQd7Dt66GXTN/4RcT/yHAljXaHo=
|
||||
github.com/fluxcd/pkg/git/gogit v0.2.0 h1:vhFzk2Pky4tDZwisx8+26YZumRDPxERnkV8l2dbLSoo=
|
||||
github.com/fluxcd/pkg/git/gogit v0.2.0/go.mod h1:d1RIwl6DVdU8/2dBIhw6n7GNokIKqs+b9cKc/8tz7ew=
|
||||
github.com/fluxcd/pkg/git/libgit2 v0.3.0 h1:fgw/Cm7Id2AxanMab3qU+UuvdnhfwRMbbWF27xu413o=
|
||||
github.com/fluxcd/pkg/git/libgit2 v0.3.0/go.mod h1:JHeZBeq+jbUpRpP5OoEYXua0EF5dyqAlbEfYcXY228A=
|
||||
github.com/fluxcd/pkg/git/gogit v0.2.1-0.20221128153352-fbb8b9f29f38 h1:FGDncVUlMfK7eZiw0m/mWa6CGczGeGomw8jJ0uX2zIw=
|
||||
github.com/fluxcd/pkg/git/gogit v0.2.1-0.20221128153352-fbb8b9f29f38/go.mod h1:d1RIwl6DVdU8/2dBIhw6n7GNokIKqs+b9cKc/8tz7ew=
|
||||
github.com/fluxcd/pkg/gittestserver v0.8.0 h1:YrYe63KScKlLxx0GAiQthx2XqHDx0vKitIIx4JnDtIo=
|
||||
github.com/fluxcd/pkg/gittestserver v0.8.0/go.mod h1:/LI/xKMrnQbIsTDnTyABQ71iaYhFIZ8fb4cvY7WAlBU=
|
||||
github.com/fluxcd/pkg/helmtestserver v0.10.0 h1:/cnx1D6/cln7v06iXZMLeshl82mV1wv3OVZoBE9Bbm4=
|
||||
github.com/fluxcd/pkg/helmtestserver v0.10.0/go.mod h1:sJwcAw/4ENiJO6i0HJSMV8ypp7k+U4amDwH7RJ57ScY=
|
||||
github.com/fluxcd/pkg/http/transport v0.1.0 h1:tzRprpCAUkPFlF/9VMLSbw95jcAKCbimDC//VULQXnY=
|
||||
github.com/fluxcd/pkg/http/transport v0.1.0/go.mod h1:BNXQYy3ifMTc3DaOu3GlIR6j5DtTwMEyvTr2EA7XP8w=
|
||||
github.com/fluxcd/pkg/lockedfile v0.1.0 h1:YsYFAkd6wawMCcD74ikadAKXA4s2sukdxrn7w8RB5eo=
|
||||
github.com/fluxcd/pkg/lockedfile v0.1.0/go.mod h1:EJLan8t9MiOcgTs8+puDjbE6I/KAfHbdvIy9VUgIjm8=
|
||||
github.com/fluxcd/pkg/masktoken v0.2.0 h1:HoSPTk4l1fz5Fevs2vVRvZGru33blfMwWSZKsHdfG/0=
|
||||
|
|
@ -1736,7 +1730,6 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
|
|||
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200930160638-afb6bcd081ae/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
||||
golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
||||
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
|
|
@ -1975,7 +1968,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
|
|
|
|||
|
|
@ -141,9 +141,7 @@ kubectl -n source-system wait helmchart/helmchart-bucket --for=condition=ready -
|
|||
|
||||
echo "Run large Git repo tests"
|
||||
kubectl -n source-system apply -f "${ROOT_DIR}/config/testdata/git/large-repo.yaml"
|
||||
kubectl -n source-system wait gitrepository/large-repo-go-git --for=condition=ready --timeout=2m15s
|
||||
kubectl -n source-system wait gitrepository/large-repo-libgit2 --for=condition=ready --timeout=2m15s
|
||||
|
||||
kubectl -n source-system wait gitrepository/large-repo --for=condition=ready --timeout=2m15s
|
||||
|
||||
echo "Run HelmChart from OCI registry tests"
|
||||
kubectl -n source-system apply -f "${ROOT_DIR}/config/testdata/helmchart-from-oci/source.yaml"
|
||||
|
|
|
|||
|
|
@ -1,158 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euxo pipefail
|
||||
|
||||
IMG="${IMG:-}"
|
||||
TAG="${TAG:-}"
|
||||
IMG_TAG="${IMG}:${TAG}"
|
||||
DOWNLOAD_URL="https://github.com/fluxcd/golang-with-libgit2/releases/download/${TAG}"
|
||||
SKIP_COSIGN_VERIFICATION="${SKIP_COSIGN_VERIFICATION:-false}"
|
||||
|
||||
TMP_DIR=$(mktemp -d)
|
||||
|
||||
function cleanup(){
|
||||
rm -rf "${TMP_DIR}"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
fatal() {
|
||||
echo '[ERROR] ' "$@" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
download() {
|
||||
[[ $# -eq 2 ]] || fatal 'download needs exactly 2 arguments'
|
||||
|
||||
curl -o "$1" -sfL "$2"
|
||||
|
||||
[[ $? -eq 0 ]] || fatal 'Download failed'
|
||||
}
|
||||
|
||||
download_files() {
|
||||
[[ $# -eq 1 ]] || fatal 'download_files needs exactly 1 arguments'
|
||||
|
||||
FILE_NAMES="checksums.txt checksums.txt.sig checksums.txt.pem $1"
|
||||
|
||||
for FILE_NAME in ${FILE_NAMES}; do
|
||||
download "${TMP_DIR}/${FILE_NAME}" "${DOWNLOAD_URL}/${FILE_NAME}"
|
||||
done
|
||||
}
|
||||
|
||||
cosign_verify(){
|
||||
[[ $# -eq 3 ]] || fatal 'cosign_verify needs exactly 3 arguments'
|
||||
|
||||
COSIGN_EXPERIMENTAL=1 cosign verify-blob --cert "$1" --signature "$2" "$3"
|
||||
|
||||
[[ $? -eq 0 ]] || fatal 'signature verification failed'
|
||||
}
|
||||
|
||||
assure_provenance() {
|
||||
[[ $# -eq 1 ]] || fatal 'assure_provenance needs exactly 1 arguments'
|
||||
|
||||
if "${SKIP_COSIGN_VERIFICATION}"; then
|
||||
echo 'Skipping cosign verification...'
|
||||
else
|
||||
cosign_verify "${TMP_DIR}/checksums.txt.pem" \
|
||||
"${TMP_DIR}/checksums.txt.sig" \
|
||||
"${TMP_DIR}/checksums.txt"
|
||||
fi
|
||||
|
||||
pushd "${TMP_DIR}" || exit
|
||||
if command -v sha256sum; then
|
||||
grep "$1" "checksums.txt" | sha256sum --check
|
||||
else
|
||||
grep "$1" "checksums.txt" | shasum -a 256 --check
|
||||
fi
|
||||
popd || exit
|
||||
|
||||
[[ $? -eq 0 ]] || fatal 'integrity verification failed'
|
||||
}
|
||||
|
||||
extract_libraries(){
|
||||
[[ $# -eq 2 ]] || fatal 'extract_libraries needs exactly 2 arguments'
|
||||
|
||||
tar -xf "${TMP_DIR}/$1"
|
||||
|
||||
rm "${TMP_DIR}/$1"
|
||||
mv "${2}" "${TAG}"
|
||||
mv "${TAG}/" "./build/libgit2"
|
||||
}
|
||||
|
||||
fix_pkgconfigs(){
|
||||
NEW_DIR="$(/bin/pwd)/build/libgit2/${TAG}"
|
||||
|
||||
# Update the prefix paths included in the .pc files.
|
||||
if [[ $OSTYPE == 'darwin'* ]]; then
|
||||
# https://github.com/fluxcd/golang-with-libgit2/blob/v0.1.4/.github/workflows/release.yaml#L158
|
||||
INSTALLED_DIR="/Users/runner/work/golang-with-libgit2/golang-with-libgit2/build/darwin-libgit2-only"
|
||||
|
||||
# This will make it easier to update to the location in which they will be used.
|
||||
# sed has a sight different behaviour in MacOS
|
||||
# NB: Some macOS users may override their sed with gsed. If gsed is the PATH, use that instead.
|
||||
if command -v gsed &> /dev/null; then
|
||||
find "${NEW_DIR}" -type f -name "*.pc" | xargs -I {} gsed -i "s;${INSTALLED_DIR};${NEW_DIR};g" {}
|
||||
else
|
||||
find "${NEW_DIR}" -type f -name "*.pc" | xargs -I {} sed -i "" "s;${INSTALLED_DIR};${NEW_DIR};g" {}
|
||||
fi
|
||||
else
|
||||
# https://github.com/fluxcd/golang-with-libgit2/blob/v0.1.4/.github/workflows/release.yaml#L52
|
||||
INSTALLED_DIR="/home/runner/work/golang-with-libgit2/golang-with-libgit2/build/build_libgit2_only"
|
||||
|
||||
find "${NEW_DIR}" -type f -name "*.pc" | xargs -I {} sed -i "s;${INSTALLED_DIR};${NEW_DIR};g" {}
|
||||
fi
|
||||
}
|
||||
|
||||
extract_from_image(){
|
||||
PLATFORM=$1
|
||||
DIR=$2
|
||||
|
||||
id=$(docker create --platform="${PLATFORM}" "${IMG_TAG}" sh)
|
||||
docker cp "${id}":/usr/local - > output.tar.gz
|
||||
docker rm -v "${id}"
|
||||
|
||||
tar -xf output.tar.gz "local/${DIR}"
|
||||
rm output.tar.gz
|
||||
|
||||
NEW_DIR="$(/bin/pwd)/build/libgit2/${TAG}"
|
||||
INSTALLED_DIR="/usr/local/${DIR}"
|
||||
|
||||
mv "local/${DIR}" "${TAG}"
|
||||
rm -rf "local"
|
||||
mv "${TAG}/" "./build/libgit2"
|
||||
|
||||
# Update the prefix paths included in the .pc files.
|
||||
# This will make it easier to update to the location in which they will be used.
|
||||
find "${NEW_DIR}" -type f -name "*.pc" | xargs -I {} sed -i "s;${INSTALLED_DIR};${NEW_DIR};g" {}
|
||||
}
|
||||
|
||||
install_libraries(){
|
||||
if [ -d "./build/libgit2/${TAG}" ]; then
|
||||
echo "Skipping: libgit2 ${TAG} already installed"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mkdir -p "./build/libgit2"
|
||||
|
||||
# Linux ARM support is still based on the container image libraries.
|
||||
if [[ $OSTYPE == 'linux'* ]]; then
|
||||
if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
|
||||
extract_from_image "linux/arm64" "aarch64-alpine-linux-musl"
|
||||
fix_pkgconfigs "aarch64-alpine-linux-musl"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
FILE_NAME="linux-$(uname -m)-libgit2-only.tar.gz"
|
||||
DIR="linux-libgit2-only"
|
||||
if [[ $OSTYPE == 'darwin'* ]]; then
|
||||
FILE_NAME="darwin-libgit2-only.tar.gz"
|
||||
DIR="darwin-libgit2-only"
|
||||
fi
|
||||
|
||||
download_files "${FILE_NAME}"
|
||||
assure_provenance "${FILE_NAME}"
|
||||
extract_libraries "${FILE_NAME}" "${DIR}"
|
||||
fix_pkgconfigs
|
||||
}
|
||||
|
||||
install_libraries
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -eoux pipefail
|
||||
|
||||
SPLIT="***"
|
||||
|
||||
cat <<EOF > ATTRIBUTIONS.md
|
||||
# Attributions
|
||||
|
||||
This application uses Open Source components. You can find the source
|
||||
code of their open source projects along with license information below.
|
||||
We acknowledge and are grateful to these developers for their contributions
|
||||
to open source.
|
||||
|
||||
## libgit2
|
||||
|
||||
Libgit2 was obtained in source-code form from its github repository:
|
||||
https://github.com/libgit2/libgit2/
|
||||
|
||||
No changes were made to its original source code.
|
||||
|
||||
Copyright notice (https://raw.githubusercontent.com/libgit2/libgit2/main/COPYING):
|
||||
|
||||
$(curl --max-time 5 -L https://raw.githubusercontent.com/libgit2/libgit2/main/COPYING)
|
||||
EOF
|
||||
|
|
@ -23,36 +23,18 @@ import feathelper "github.com/fluxcd/pkg/runtime/features"
|
|||
|
||||
const (
|
||||
// OptimizedGitClones decreases resource utilization for GitRepository
|
||||
// reconciliations. It supports both go-git and libgit2 implementations
|
||||
// when cloning repositories using branches or tags.
|
||||
// reconciliations.
|
||||
//
|
||||
// When enabled, avoids full clone operations by first checking whether
|
||||
// the last revision is still the same at the target repository,
|
||||
// and if that is so, skips the reconciliation.
|
||||
OptimizedGitClones = "OptimizedGitClones"
|
||||
// ForceGoGitImplementation ignores the value set for gitImplementation
|
||||
// and ensures that go-git is used for all GitRepository objects.
|
||||
//
|
||||
// Libgit2 is built in C and we use the Go bindings provided by git2go
|
||||
// to cross the C-GO chasm. Unfortunately, when libgit2 is being used the
|
||||
// controllers are known to panic over long periods of time, or when
|
||||
// under high GC pressure.
|
||||
//
|
||||
// This feature gate enables the gradual deprecation of libgit2 in favour
|
||||
// of go-git, which so far is the most stable of the pair.
|
||||
//
|
||||
// When enabled, libgit2 won't be initialized, nor will any git2go CGO
|
||||
// code be called.
|
||||
ForceGoGitImplementation = "ForceGoGitImplementation"
|
||||
)
|
||||
|
||||
var features = map[string]bool{
|
||||
// OptimizedGitClones
|
||||
// opt-out from v0.25
|
||||
OptimizedGitClones: true,
|
||||
// ForceGoGitImplementation
|
||||
// opt-out from v0.32
|
||||
ForceGoGitImplementation: true,
|
||||
}
|
||||
|
||||
// DefaultFeatureGates contains a list of all supported feature gates and
|
||||
|
|
|
|||
19
main.go
19
main.go
|
|
@ -34,7 +34,6 @@ import (
|
|||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
|
||||
"github.com/fluxcd/pkg/git"
|
||||
"github.com/fluxcd/pkg/git/libgit2/transport"
|
||||
"github.com/fluxcd/pkg/runtime/client"
|
||||
helper "github.com/fluxcd/pkg/runtime/controller"
|
||||
"github.com/fluxcd/pkg/runtime/events"
|
||||
|
|
@ -204,20 +203,12 @@ func main() {
|
|||
}
|
||||
storage := mustInitStorage(storagePath, storageAdvAddr, artifactRetentionTTL, artifactRetentionRecords, setupLog)
|
||||
|
||||
if gogitOnly, _ := features.Enabled(features.ForceGoGitImplementation); !gogitOnly {
|
||||
if err = transport.InitManagedTransport(); err != nil {
|
||||
// Log the error, but don't exit so as to not block reconcilers that are healthy.
|
||||
setupLog.Error(err, "unable to initialize libgit2 managed transport")
|
||||
}
|
||||
}
|
||||
|
||||
if err = (&controllers.GitRepositoryReconciler{
|
||||
Client: mgr.GetClient(),
|
||||
EventRecorder: eventRecorder,
|
||||
Metrics: metricsH,
|
||||
Storage: storage,
|
||||
ControllerName: controllerName,
|
||||
Libgit2TransportInitialized: transport.Enabled,
|
||||
Client: mgr.GetClient(),
|
||||
EventRecorder: eventRecorder,
|
||||
Metrics: metricsH,
|
||||
Storage: storage,
|
||||
ControllerName: controllerName,
|
||||
}).SetupWithManagerAndOptions(mgr, controllers.GitRepositoryReconcilerOptions{
|
||||
MaxConcurrentReconciles: concurrent,
|
||||
DependencyRequeueInterval: requeueDependency,
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
FROM gcr.io/oss-fuzz-base/base-builder-go
|
||||
|
||||
RUN apt-get update && apt-get install -y cmake pkg-config
|
||||
ENV SRC=$GOPATH/src/github.com/fluxcd/source-controller
|
||||
ENV FLUX_CI=true
|
||||
|
||||
COPY ./ $GOPATH/src/github.com/fluxcd/source-controller/
|
||||
COPY ./tests/fuzz/oss_fuzz_build.sh $SRC/build.sh
|
||||
COPY tests/fuzz/compile_native_go_fuzzer /usr/local/bin/
|
||||
COPY ./ $SRC
|
||||
RUN wget https://raw.githubusercontent.com/google/oss-fuzz/master/projects/fluxcd/build.sh -O $SRC/build.sh
|
||||
|
||||
WORKDIR $SRC
|
||||
|
|
|
|||
|
|
@ -1,62 +0,0 @@
|
|||
#!/bin/bash -eux
|
||||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
################################################################################
|
||||
|
||||
# This is a copy of the upstream script which is only needed to link
|
||||
# additional static libraries. Orignal source:
|
||||
#
|
||||
# https://github.com/google/oss-fuzz/blob/9e8dd47cb902545efc60a5580126adc36d70bae3/infra/base-images/base-builder/compile_native_go_fuzzer
|
||||
|
||||
function build_native_go_fuzzer() {
|
||||
fuzzer=$1
|
||||
function=$2
|
||||
path=$3
|
||||
tags="-tags gofuzz"
|
||||
|
||||
if [[ $SANITIZER == *coverage* ]]; then
|
||||
current_dir=$(pwd)
|
||||
mkdir $OUT/rawfuzzers || true
|
||||
cd $abs_file_dir
|
||||
go test -c -run $fuzzer -o $OUT/$fuzzer -cover
|
||||
cp "${fuzzer_filename}" "${OUT}/rawfuzzers/${fuzzer}"
|
||||
cd $current_dir
|
||||
else
|
||||
go-118-fuzz-build -o $fuzzer.a -func $function $abs_file_dir
|
||||
# TODO: upstream support for linking $ADDITIONAL_LIBS
|
||||
$CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -o $OUT/$fuzzer \
|
||||
$ADDITIONAL_LIBS
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
path=$1
|
||||
function=$2
|
||||
fuzzer=$3
|
||||
tags="-tags gofuzz"
|
||||
|
||||
# Get absolute path.
|
||||
abs_file_dir=$(go list $tags -f {{.Dir}} $path)
|
||||
|
||||
# TODO(adamkorcz): Get rid of "-r" flag here.
|
||||
fuzzer_filename=$(grep -r -l --include='*.go' -s "$function" "${abs_file_dir}")
|
||||
|
||||
# Test if file contains a line with "func $function" and "testing.F".
|
||||
if [ $(grep -r "func $function" $fuzzer_filename | grep "testing.F" | wc -l) -eq 1 ]
|
||||
then
|
||||
build_native_go_fuzzer $fuzzer $function $abs_file_dir
|
||||
else
|
||||
echo "Could not find the function: func ${function}(f *testing.F)"
|
||||
fi
|
||||
|
|
@ -1,80 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2022 The Flux authors
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -euxo pipefail
|
||||
|
||||
# This file aims for:
|
||||
# - Dynamically discover and build all fuzz tests within the repository.
|
||||
# - Work for both local make fuzz-smoketest and the upstream oss-fuzz.
|
||||
|
||||
GOPATH="${GOPATH:-/root/go}"
|
||||
GO_SRC="${GOPATH}/src"
|
||||
PROJECT_PATH="github.com/fluxcd/source-controller"
|
||||
|
||||
# install_deps installs all dependencies needed for upstream oss-fuzz.
|
||||
# Unfortunately we can't pin versions here, as we want to always
|
||||
# have the latest, so that we can reproduce errors occuring upstream.
|
||||
install_deps(){
|
||||
if ! command -v go-118-fuzz-build &> /dev/null; then
|
||||
go install github.com/AdamKorcz/go-118-fuzz-build@latest
|
||||
fi
|
||||
}
|
||||
|
||||
install_deps
|
||||
|
||||
cd "${GO_SRC}/${PROJECT_PATH}"
|
||||
|
||||
# Ensure any project-specific requirements are catered for ahead of
|
||||
# the generic build process.
|
||||
if [ -f "tests/fuzz/oss_fuzz_prebuild.sh" ]; then
|
||||
. tests/fuzz/oss_fuzz_prebuild.sh
|
||||
fi
|
||||
|
||||
modules=$(find . -mindepth 1 -maxdepth 4 -type f -name 'go.mod' | cut -c 3- | sed 's|/[^/]*$$||' | sort -u | sed 's;/go.mod;;g' | sed 's;go.mod;.;g')
|
||||
|
||||
for module in ${modules}; do
|
||||
|
||||
cd "${GO_SRC}/${PROJECT_PATH}/${module}"
|
||||
|
||||
test_files=$(grep -r --include='**_test.go' --files-with-matches 'func Fuzz' . || echo "")
|
||||
if [ -z "${test_files}" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
go get github.com/AdamKorcz/go-118-fuzz-build/testing
|
||||
|
||||
# Iterate through all Go Fuzz targets, compiling each into a fuzzer.
|
||||
for file in ${test_files}; do
|
||||
# If the subdir is a module, skip this file, as it will be handled
|
||||
# at the next iteration of the outer loop.
|
||||
if [ -f "$(dirname "${file}")/go.mod" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
targets=$(grep -oP 'func \K(Fuzz\w*)' "${file}")
|
||||
for target_name in ${targets}; do
|
||||
# Transform module path into module name (e.g. git/libgit2 to git_libgit2).
|
||||
module_name="$(echo ${module} | tr / _)_"
|
||||
# Compose fuzzer name based on the lowercase version of the func names.
|
||||
# The module name is added after the fuzz prefix, for better discoverability.
|
||||
fuzzer_name=$(echo "${target_name}" | tr '[:upper:]' '[:lower:]' | sed "s;fuzz_;fuzz_${module_name//._/};g")
|
||||
target_dir=$(dirname "${file}")
|
||||
|
||||
echo "Building ${file}.${target_name} into ${fuzzer_name}"
|
||||
compile_native_go_fuzzer "${target_dir}" "${target_name}" "${fuzzer_name}"
|
||||
done
|
||||
done
|
||||
done
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2022 The Flux authors
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -euxo pipefail
|
||||
|
||||
# This file is executed by upstream oss-fuzz after its building process.
|
||||
# Use it for unsetting any environment variables that may impact other building
|
||||
# processes.
|
||||
|
||||
if [[ -n "${PRE_LIB_FUZZING_ENGINE}" ]]; then
|
||||
export LIB_FUZZING_ENGINE="${PRE_LIB_FUZZING_ENGINE}"
|
||||
fi
|
||||
|
||||
unset TARGET_DIR
|
||||
unset CGO_ENABLED
|
||||
unset LIBRARY_PATH
|
||||
unset PKG_CONFIG_PATH
|
||||
unset CGO_CFLAGS
|
||||
unset CGO_LDFLAGS
|
||||
unset PRE_LIB_FUZZING_ENGINE
|
||||
|
|
@ -23,54 +23,3 @@ set -euxo pipefail
|
|||
# for traversing into ascending dirs, therefore we copy those contents here:
|
||||
mkdir -p controllers/testdata/crd
|
||||
cp config/crd/bases/*.yaml controllers/testdata/crd/
|
||||
|
||||
# libgit2, cmake and pkg-config are requirements to support libgit2.
|
||||
LIBGIT2_TAG="${LIBGIT2_TAG:-v0.4.0}"
|
||||
|
||||
# Avoid updating apt get and installing dependencies, if they are already in place.
|
||||
if (! command -v cmake &> /dev/null) || (! command -v pkg-config &> /dev/null) then
|
||||
apt-get update && apt-get install -y cmake pkg-config
|
||||
fi
|
||||
|
||||
export TARGET_DIR="$(/bin/pwd)/build/libgit2/${LIBGIT2_TAG}"
|
||||
|
||||
# For most cases, libgit2 will already be present.
|
||||
# The exception being at the oss-fuzz integration.
|
||||
if [ ! -d "${TARGET_DIR}" ]; then
|
||||
curl --connect-timeout 2 --retry 3 --retry-delay 1 --retry-max-time 30 \
|
||||
-o output.tar.gz -LO "https://github.com/fluxcd/golang-with-libgit2/releases/download/${LIBGIT2_TAG}/linux-$(uname -m)-libgit2-only.tar.gz"
|
||||
|
||||
DIR=linux-libgit2-only
|
||||
NEW_DIR="$(/bin/pwd)/build/libgit2/${LIBGIT2_TAG}"
|
||||
INSTALLED_DIR="/home/runner/work/golang-with-libgit2/golang-with-libgit2/build/${DIR}"
|
||||
|
||||
mkdir -p ./build/libgit2
|
||||
|
||||
tar -xf output.tar.gz
|
||||
rm output.tar.gz
|
||||
mv "${DIR}" "${LIBGIT2_TAG}"
|
||||
mv "${LIBGIT2_TAG}/" "./build/libgit2"
|
||||
|
||||
# Update the prefix paths included in the .pc files.
|
||||
# This will make it easier to update to the location in which they will be used.
|
||||
find "${NEW_DIR}" -type f -name "*.pc" | xargs -I {} sed -i "s;${INSTALLED_DIR};${NEW_DIR};g" {}
|
||||
fi
|
||||
|
||||
export CGO_ENABLED=1
|
||||
export LIBRARY_PATH="${TARGET_DIR}/lib"
|
||||
export PKG_CONFIG_PATH="${TARGET_DIR}/lib/pkgconfig"
|
||||
export CGO_CFLAGS="-I${TARGET_DIR}/include"
|
||||
export CGO_LDFLAGS="$(pkg-config --libs --static --cflags libgit2)"
|
||||
|
||||
# Temporary hack whilst libgit2 is still in use.
|
||||
# Enables the fuzzing compilation to link libgit2.
|
||||
#
|
||||
# After building the fuzzers, the value of
|
||||
# LIB_FUZZING_ENGINE is reset to what it was before
|
||||
# it to avoid side effects onto other repositories.
|
||||
#
|
||||
# For context refer to:
|
||||
# https://github.com/google/oss-fuzz/pull/9063
|
||||
export PRE_LIB_FUZZING_ENGINE="${LIB_FUZZING_ENGINE}"
|
||||
|
||||
export LIB_FUZZING_ENGINE="${LIB_FUZZING_ENGINE} -Wl,--start-group ${TARGET_DIR}/lib/libgit2.a"
|
||||
|
|
|
|||
|
|
@ -17,4 +17,4 @@
|
|||
set -euxo pipefail
|
||||
|
||||
# run each fuzzer once to ensure they are working properly
|
||||
find /out -type f -name "fuzz*" -exec echo {} -runs=1 \; | bash -e
|
||||
find /out -type f -iname "fuzz*" -exec echo {} -runs=1 \; | bash -e
|
||||
|
|
|
|||
Loading…
Reference in New Issue