fluxcd
/
source-controller
mirror of https://github.com/fluxcd/source-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update github.com/libgit2/git2go to v31.6.1 

This commit updates `github.com/libgit2/git2go` to `v31.6.1` (with
`libgit2` `1.1.1`), and changes the container image build process so
that it makes use of `ghcr.io/hiddeco/golang-with-libgit2`.

This image provides a pre-build dynamic `libgit2` dependency linked
against OpenSSL and LibSSH2 (without gcrypt), and a set of cross-compile
build tools (see
[rationale](https://github.com/hiddeco/golang-with-libgit2#rationale) and
[usage](https://github.co/hiddeco/golang-with-libgit2#usage) for more
detailed information).

The linked set of dependency should solve most known issues around
unsupport private key types, but does not resolve the issues with ECDSA*
and ED25519 hostkeys yet. Solving this requires a newer version of
`libgit2` (`>=1.2.0`), which currently does not seem to work properly
with `git2go/v32`.

Some small changes have been made to the `libgit2` package to address
(future) deprecations.

Signed-off-by: Hidde Beydals <hello@hidde.co>
This commit is contained in:
Hidde Beydals 2021-09-10 12:39:40 +02:00 committed by pa250194
parent 7c0d4c070e
commit 869c7960e3
12 changed files with 151 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.dockerignore Normal file
View File

@ -0,0 +1 @@
hack/libgit2/

15
.github/actions/run-tests/Dockerfile vendored
View File

@ -1,17 +1,4 @@
FROM golang:1.16-buster as builder
# Up-to-date libgit2 dependencies are only available in
# unstable, as libssh2 in testing/bullseye has been linked
# against gcrypt which causes issues with PKCS* formats.
# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271
RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \
&& echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list
RUN set -eux; \
apt-get update \
&& apt-get install -y libgit2-dev/unstable \
&& apt-get clean \
&& apt-get autoremove --purge -y \
&& rm -rf /var/lib/apt/lists/*
FROM ghcr.io/hiddeco/golang-with-libgit2:dev as builder
# Use the GitHub Actions uid:gid combination for proper fs permissions
RUN groupadd -g 116 test && \

3
.gitignore vendored
View File

@ -15,3 +15,6 @@
# vendor/
bin/
config/release/
# Exclude all libgit2 related files
hack/libgit2/

78
Dockerfile
View File

@ -1,69 +1,57 @@
FROM golang:1.16-buster as builder
# Up-to-date libgit2 dependencies are only available in
# unstable, as libssh2 in testing/bullseye has been linked
# against gcrypt which causes issues with PKCS* formats.
# Explicitly listing all build dependencies is required because
# they can only be automagically found for AMD64 builds.
# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271
RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \
&& echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list
RUN set -eux; \
apt-get update \
&& apt-get install -y \
libgit2-dev/unstable \
zlib1g-dev/unstable \
libssh2-1-dev/unstable \
libpcre3-dev/unstable \
&& apt-get clean \
&& apt-get autoremove --purge -y \
&& rm -rf /var/lib/apt/lists/*
ARG BASE_IMG=ghcr.io/hiddeco/golang-with-libgit2
ARG BASE_TAG=dev
FROM ${BASE_IMG}:${BASE_TAG} AS build
# Configure workspace
WORKDIR /workspace
# copy api submodule
# Copy api submodule
COPY api/ api/
# copy modules manifests
# Copy modules manifests
COPY go.mod go.mod
COPY go.sum go.sum
# cache modules
# Cache modules
RUN go mod download
# copy source code
# Copy source code
COPY main.go main.go
COPY controllers/ controllers/
COPY pkg/ pkg/
COPY internal/ internal/
# build without specifing the arch
RUN CGO_ENABLED=1 go build -o source-controller main.go
# Build the binary
ENV CGO_ENABLED=1
ARG TARGETPLATFORM
RUN xx-go build -o source-controller -trimpath \
main.go
FROM debian:buster-slim as controller
FROM debian:bullseye-slim as controller
# link repo to the GitHub Container Registry image
# Link repo to the GitHub Container Registry image
LABEL org.opencontainers.image.source="https://github.com/fluxcd/source-controller"
# Up-to-date libgit2 dependencies are only available in
# unstable, as libssh2 in testing/bullseye has been linked
# against gcrypt which causes issues with PKCS* formats.
# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271
RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \
&& echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list
RUN set -eux; \
apt-get update \
&& apt-get install -y \
ca-certificates \
libgit2-1.1 \
&& apt-get clean \
&& apt-get autoremove --purge -y \
&& rm -rf /var/lib/apt/lists/*
COPY --from=builder /workspace/source-controller /usr/local/bin/
# Configure user
RUN groupadd controller && \
useradd --gid controller --shell /bin/sh --create-home controller
# Copy libgit2
COPY --from=build /libgit2/lib/* /usr/local/lib/
RUN ldconfig
# Upgrade packages and install runtime dependencies
RUN echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list \
&& echo "deb-src http://deb.debian.org/debian sid main" >> /etc/apt/sources.list \
&& apt update \
&& apt install --no-install-recommends -y zlib1g/sid libssl1.1/sid libssh2-1/sid \
&& apt install --no-install-recommends -y ca-certificates \
&& apt clean \
&& apt autoremove --purge -y \
&& rm -rf /var/lib/apt/lists/*
# Copy over binary from build
COPY --from=build /workspace/source-controller /usr/local/bin/
USER controller
ENTRYPOINT [ "source-controller" ]

153
Makefile
View File

@ -1,10 +1,32 @@
# Image URL to use all building/pushing image targets
IMG ?= fluxcd/source-controller:latest
# Base image used to build the Go binary
BASE_IMG ?= ghcr.io/hiddeco/golang-with-libgit2
BASE_TAG ?= dev
# Produce CRDs that work back to Kubernetes 1.16
CRD_OPTIONS ?= crd:crdVersions=v1
ENVTEST_BIN_VERSION?=1.19.2
KUBEBUILDER_ASSETS?=$(shell $(SETUP_ENVTEST) use -i $(ENVTEST_BIN_VERSION) -p path)
# Repository root based on Git metadata
REPOSITORY_ROOT := $(shell git rev-parse --show-toplevel)
# Dependency versions
LIBGIT2_VERSION ?= 1.1.1
ENVTEST_BIN_VERSION ?= 1.19.2
KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use -i $(ENVTEST_BIN_VERSION) -p path)
# libgit2 related magical paths
# These are used to determine if the target libgit2 version is already available on
# the system, or where they should be installed to
SYSTEM_LIBGIT2_VERSION := $(shell pkg-config --modversion libgit2 2>/dev/null)
LIBGIT2_PATH := $(REPOSITORY_ROOT)/hack/libgit2
LIBGIT2_LIB_PATH := $(LIBGIT2_PATH)/lib
LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.so.$(LIBGIT2_VERSION)
# API (doc) generation utilities
CONTROLLER_GEN_VERSION ?= v0.5.0
GEN_API_REF_DOCS_VERSION ?= 0.3.0
# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
ifeq (,$(shell go env GOBIN))
@ -13,121 +35,130 @@ else
GOBIN=$(shell go env GOBIN)
endif
all: manager
all: build
# Run tests
test: generate fmt vet manifests api-docs setup-envtest
KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) go test ./... -coverprofile cover.out
cd api; go test ./... -coverprofile cover.out
# Build manager binary
manager: generate fmt vet
build: $(LIBGIT2) ## Build manager binary
PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig/ \
go build -o bin/manager main.go
# Run against the configured Kubernetes cluster in ~/.kube/config
run: generate fmt vet manifests
test: $(LIBGIT2) test-api ## Run tests
LD_LIBRARY_PATH=$(LIBGIT2_LIB_PATH) \
PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig/ \
go test ./... -coverprofile cover.out
test-api: ## Run api tests
cd api; go test ./... -coverprofile cover.out
run: $(LIBGIT2) generate fmt vet manifests ## Run against the configured Kubernetes cluster in ~/.kube/config
LD_LIBRARY_PATH=$(LIBGIT2_LIB_PATH) \
go run ./main.go
# Install CRDs into a cluster
install: manifests
install: manifests ## Install CRDs into a cluster
kustomize build config/crd | kubectl apply -f -
# Uninstall CRDs from a cluster
uninstall: manifests
uninstall: manifests ## Uninstall CRDs from a cluster
kustomize build config/crd | kubectl delete -f -
# Deploy controller in the configured Kubernetes cluster in ~/.kube/config
deploy: manifests
deploy: manifests ## Deploy controller in the configured Kubernetes cluster in ~/.kube/config
cd config/manager && kustomize edit set image fluxcd/source-controller=${IMG}
kustomize build config/default | kubectl apply -f -
# Deploy controller dev image in the configured Kubernetes cluster in ~/.kube/config
dev-deploy:
dev-deploy: ## Deploy controller dev image in the configured Kubernetes cluster in ~/.kube/config
mkdir -p config/dev && cp config/default/* config/dev
cd config/dev && kustomize edit set image fluxcd/source-controller=${IMG}
kustomize build config/dev | kubectl apply -f -
rm -rf config/dev
# Generate manifests e.g. CRD, RBAC etc.
manifests: controller-gen
manifests: controller-gen ## Generate manifests, e.g. CRD, RBAC, etc.
$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role paths="./..." output:crd:artifacts:config="config/crd/bases"
cd api; $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role paths="./..." output:crd:artifacts:config="../config/crd/bases"
# Generate API reference documentation
api-docs: gen-crd-api-reference-docs
api-docs: gen-crd-api-reference-docs ## Generate API reference documentation
$(API_REF_GEN) -api-dir=./api/v1beta1 -config=./hack/api-docs/config.json -template-dir=./hack/api-docs/template -out-file=./docs/api/source.md
# Run go mod tidy
tidy:
tidy: ## Run go mod tidy
go mod tidy
cd api; go mod tidy
# Run go fmt against code
fmt:
fmt: ## Run go fmt against code
go fmt ./...
cd api; go fmt ./...
# Run go vet against code
vet:
vet: ## Run go vet against code
PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig \
go vet ./...
cd api; go vet ./...
# Generate code
generate: controller-gen
generate: controller-gen ## Generate API code
cd api; $(CONTROLLER_GEN) object:headerFile="../hack/boilerplate.go.txt" paths="./..."
# Build the docker image
docker-build:
docker build . -t ${IMG}
docker-build: ## Build the docker image
docker build \
--build-arg BASE_IMG=$(BASE_IMG) \
--build-arg BASE_TAG=$(BASE_TAG) \
-t ${IMG} .
# Push the docker image
docker-push:
docker-push: ## Push docker image
docker push ${IMG}
# Find or download controller-gen
controller-gen:
controller-gen: ## Find or download controller-gen
ifeq (, $(shell which controller-gen))
@{ \
set -e ;\
CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
cd $$CONTROLLER_GEN_TMP_DIR ;\
go mod init tmp ;\
go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.5.0 ;\
rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
set -e; \
CONTROLLER_GEN_TMP_DIR=$$(mktemp -d); \
cd $$CONTROLLER_GEN_TMP_DIR; \
go mod init tmp; \
go get sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_GEN_VERSION); \
rm -rf $$CONTROLLER_GEN_TMP_DIR; \
}
CONTROLLER_GEN=$(GOBIN)/controller-gen
else
CONTROLLER_GEN=$(shell which controller-gen)
endif
# Find or download gen-crd-api-reference-docs
gen-crd-api-reference-docs:
gen-crd-api-reference-docs: ## Find or download gen-crd-api-reference-docs
ifeq (, $(shell which gen-crd-api-reference-docs))
@{ \
set -e ;\
API_REF_GEN_TMP_DIR=$$(mktemp -d) ;\
cd $$API_REF_GEN_TMP_DIR ;\
go mod init tmp ;\
go get github.com/ahmetb/gen-crd-api-reference-docs@v0.3.0 ;\
rm -rf $$API_REF_GEN_TMP_DIR ;\
set -e; \
API_REF_GEN_TMP_DIR=$$(mktemp -d); \
cd $$API_REF_GEN_TMP_DIR; \
go mod init tmp; \
go get github.com/ahmetb/gen-crd-api-reference-docs@$(GEN_API_REF_DOCS_VERSION); \
rm -rf $$API_REF_GEN_TMP_DIR; \
}
API_REF_GEN=$(GOBIN)/gen-crd-api-reference-docs
else
API_REF_GEN=$(shell which gen-crd-api-reference-docs)
endif
# Find or download setup-envtest
setup-envtest:
setup-envtest: ## Find or download setup-envtest
ifeq (, $(shell which setup-envtest))
@{ \
set -e ;\
SETUP_ENVTEST_TMP_DIR=$$(mktemp -d) ;\
cd $$SETUP_ENVTEST_TMP_DIR ;\
go mod init tmp ;\
go get sigs.k8s.io/controller-runtime/tools/setup-envtest@latest ;\
rm -rf $$SETUP_ENVTEST_TMP_DIR ;\
set -e; \
SETUP_ENVTEST_TMP_DIR=$$(mktemp -d); \
cd $$SETUP_ENVTEST_TMP_DIR; \
go mod init tmp; \
go get sigs.k8s.io/controller-runtime/tools/setup-envtest@latest; \
rm -rf $$SETUP_ENVTEST_TMP_DIR; \
}
SETUP_ENVTEST=$(GOBIN)/setup-envtest
else
SETUP_ENVTEST=$(shell which setup-envtest)
endif
libgit2: $(LIBGIT2) ## Detect or download libgit2 library
$(LIBGIT2):
ifeq ($(LIBGIT2_VERSION),$(SYSTEM_LIBGIT2_VERSION))
else
@{ \
set -e; \
mkdir -p $(LIBGIT2_PATH); \
docker cp $(shell docker create --rm $(BASE_IMG):$(BASE_TAG)):/libgit2/Makefile $(LIBGIT2_PATH); \
INSTALL_PREFIX=$(LIBGIT2_PATH) make -C $(LIBGIT2_PATH); \
}
endif
.PHONY: help
help: ## Display this help menu
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

2
go.mod
View File

@ -24,7 +24,7 @@ require (
github.com/go-logr/logr v0.4.0
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/googleapis/gax-go/v2 v2.1.0 // indirect
github.com/libgit2/git2go/v31 v31.4.14
github.com/libgit2/git2go/v31 v31.6.1
github.com/minio/minio-go/v7 v7.0.10
github.com/onsi/ginkgo v1.16.4
github.com/onsi/gomega v1.14.0

4
go.sum
View File

@ -608,8 +608,8 @@ github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6Fm
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E=
github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/libgit2/git2go/v31 v31.4.14 h1:6GOd3965D9e/+gjxCwZF4eQ+vB9kKB4yKFqdQr6XZ2E=
github.com/libgit2/git2go/v31 v31.4.14/go.mod h1:c/rkJcBcUFx6wHaT++UwNpKvIsmPNqCeQ/vzO4DrEec=
github.com/libgit2/git2go/v31 v31.6.1 h1:FnKHHDDBgltSsu9RpKuL4rSR8dQ1JTf9dfvFhZ1y7Aw=
github.com/libgit2/git2go/v31 v31.6.1/go.mod h1:c/rkJcBcUFx6wHaT++UwNpKvIsmPNqCeQ/vzO4DrEec=
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=

1
internal/fs/rename.go
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !windows
// +build !windows
package fs

1
internal/fs/rename_windows.go
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build windows
// +build windows
package fs

2
pkg/git/libgit2/checkout.go
View File

@ -156,7 +156,7 @@ func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, auth *g
if err != nil {
return nil, "", fmt.Errorf("git worktree error: %w", err)
}
err = repo.CheckoutTree(tree, &git2go.CheckoutOpts{
err = repo.CheckoutTree(tree, &git2go.CheckoutOptions{
Strategy: git2go.CheckoutForce,
})
if err != nil {

7
pkg/git/libgit2/checkout_test.go
View File

@ -32,7 +32,7 @@ import (
func TestCheckoutTagSemVer_Checkout(t *testing.T) {
certCallback := func(cert *git2go.Certificate, valid bool, hostname string) git2go.ErrorCode {
return 0
return git2go.ErrorCodeOK
}
auth := &git.Auth{CertCallback: certCallback}
@ -57,9 +57,10 @@ func TestCheckoutTagSemVer_Checkout(t *testing.T) {
if _, err := io.Copy(h, f); err != nil {
t.Error(err)
}
const expectedHash = "2bd1707542a11f987ee24698dcc095a9f57639f401133ef6a29da97bf8f3f302"
fileHash := hex.EncodeToString(h.Sum(nil))
if fileHash != "2bd1707542a11f987ee24698dcc095a9f57639f401133ef6a29da97bf8f3f302" {
t.Errorf("expected files not checked out. Expected hash %s, got %s", "2bd1707542a11f987ee24698dcc095a9f57639f401133ef6a29da97bf8f3f302", fileHash)
if fileHash != expectedHash {
t.Errorf("expected files not checked out. Expected hash %s, got %s", expectedHash, fileHash)
}
semVer := CheckoutSemVer{

22
pkg/git/libgit2/transport.go
View File

@ -66,8 +66,8 @@ func (s *BasicAuth) Method(secret corev1.Secret) (*git.Auth, error) {
password = string(d)
}
if username != "" && password != "" {
credCallback = func(url string, usernameFromURL string, allowedTypes git2go.CredType) (*git2go.Cred, error) {
cred, err := git2go.NewCredUserpassPlaintext(username, password)
credCallback = func(url string, usernameFromURL string, allowedTypes git2go.CredentialType) (*git2go.Credential, error) {
cred, err := git2go.NewCredentialUserpassPlaintext(username, password)
if err != nil {
return nil, err
}
@ -81,7 +81,7 @@ func (s *BasicAuth) Method(secret corev1.Secret) (*git.Auth, error) {
roots := x509.NewCertPool()
ok := roots.AppendCertsFromPEM(caFile)
if !ok {
return git2go.ErrCertificate
return git2go.ErrorCodeCertificate
}
opts := x509.VerifyOptions{
@ -90,9 +90,9 @@ func (s *BasicAuth) Method(secret corev1.Secret) (*git.Auth, error) {
}
_, err := cert.X509.Verify(opts)
if err != nil {
return git2go.ErrCertificate
return git2go.ErrorCodeCertificate
}
return git2go.ErrOk
return git2go.ErrorCodeOK
}
}
@ -137,8 +137,8 @@ func (s *PublicKeyAuth) Method(secret corev1.Secret) (*git.Auth, error) {
user = git.DefaultPublicKeyAuthUser
}
credCallback := func(url string, usernameFromURL string, allowedTypes git2go.CredType) (*git2go.Cred, error) {
cred, err := git2go.NewCredSshKeyFromMemory(user, "", string(identity), string(password))
credCallback := func(url string, usernameFromURL string, allowedTypes git2go.CredentialType) (*git2go.Credential, error) {
cred, err := git2go.NewCredentialSSHKeyFromMemory(user, "", string(identity), string(password))
if err != nil {
return nil, err
}
@ -157,20 +157,20 @@ func (s *PublicKeyAuth) Method(secret corev1.Secret) (*git.Auth, error) {
// Check if the configured host matches the hostname given to
// the callback.
if host != hostname {
return git2go.ErrUser
return git2go.ErrorCodeUser
}
// We are now certain that the configured host and the hostname
// given to the callback match. Use the configured host (that
// includes the port), and normalize it so we can check if there
// includes the port), and normalize it, so we can check if there
// is an entry for the hostname _and_ port.
host = knownhosts.Normalize(s.host)
for _, k := range kk {
if k.matches(host, cert.Hostkey) {
return git2go.ErrOk
return git2go.ErrorCodeOK
}
}
return git2go.ErrCertificate
return git2go.ErrorCodeCertificate
}
return &git.Auth{CredCallback: credCallback, CertCallback: certCallback}, nil