fluxcd
/
source-controller
mirror of https://github.com/fluxcd/source-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #853 from pjbgf/remove-musl 

Remove MUSL and enable threadless libgit2 support
This commit is contained in:
Stefan Prodan 2022-08-01 12:03:03 +03:00 committed by GitHub
parent d9dfb9d79a f51f9d3305
commit 574bb4c5c5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 44 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
Dockerfile
View File

@ -3,7 +3,7 @@ ARG GO_VERSION=1.18
ARG XX_VERSION=1.1.2
ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2-only
ARG LIBGIT2_TAG=v0.1.4
ARG LIBGIT2_TAG=v0.2.0
FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} AS libgit2-libs
@ -37,22 +37,6 @@ COPY go.sum go.sum
# Cache modules
RUN go mod download
# The musl-tool-chain layer is an adhoc solution
# for the problem in which xx gets confused during compilation
# and a) looks for gold linker and then b) cannot find musl's dynamic linker.
FROM --platform=$BUILDPLATFORM alpine as musl-tool-chain
COPY --from=xx / /
RUN apk add bash curl tar
WORKDIR /workspace
COPY hack/download-musl.sh .
ARG TARGETPLATFORM
ARG TARGETARCH
RUN ROOT_DIR="$(pwd)" TARGET_ARCH="$(xx-info alpine-arch)" ENV_FILE=true \
./download-musl.sh
# Build stage install per target platform
# dependency and effectively cross compile the application.
@ -64,7 +48,7 @@ COPY --from=libgit2-libs /usr/local/ /usr/local/
# Some dependencies have to installed
# for the target platform: https://github.com/tonistiigi/xx#go--cgo
RUN xx-apk add musl-dev gcc lld
RUN xx-apk add musl-dev gcc clang lld
WORKDIR /workspace
@ -74,17 +58,14 @@ COPY controllers/ controllers/
COPY pkg/ pkg/
COPY internal/ internal/
COPY --from=musl-tool-chain /workspace/build /workspace/build
ARG TARGETPLATFORM
ARG TARGETARCH
ENV CGO_ENABLED=1
# Instead of using xx-go, (cross) compile with vanilla go leveraging musl tool chain.
RUN export $(cat build/musl/$(xx-info alpine-arch).env | xargs) && \
export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig" && \
export CGO_LDFLAGS="$(pkg-config --static --libs --cflags libgit2) -static" && \
GOARCH=$TARGETARCH go build \
RUN export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig" && \
export CGO_LDFLAGS="$(pkg-config --static --libs --cflags libgit2) -static -fuse-ld=lld" && \
xx-go build \
-ldflags "-s -w" \
-tags 'netgo,osusergo,static_build' \
-o /source-controller -trimpath main.go;

20
Makefile
View File

@ -4,7 +4,7 @@ TAG ?= latest
# Base image used to build the Go binary
LIBGIT2_IMG ?= ghcr.io/fluxcd/golang-with-libgit2-only
LIBGIT2_TAG ?= v0.1.4
LIBGIT2_TAG ?= v0.2.0
# Allows for defining additional Go test args, e.g. '-tags integration'.
GO_TEST_ARGS ?= -race
@ -33,24 +33,12 @@ ENVTEST_BIN_VERSION ?= 1.19.2
LIBGIT2_PATH := $(BUILD_DIR)/libgit2/$(LIBGIT2_TAG)
LIBGIT2_LIB_PATH := $(LIBGIT2_PATH)/lib
LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.a
MUSL-CC =
export CGO_ENABLED=1
export PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig
export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libgit2 2>/dev/null)
GO_STATIC_FLAGS=-ldflags "-s -w" -tags 'netgo,osusergo,static_build$(addprefix ,,$(GO_TAGS))'
ifeq ($(shell uname -s),Linux)
ifneq ($(shell uname -m),x86_64)
MUSL-PREFIX=$(BUILD_DIR)/musl/$(shell uname -m)-linux-musl-native/bin/$(shell uname -m)-linux-musl
MUSL-CC=$(MUSL-PREFIX)-gcc
export CC=$(MUSL-PREFIX)-gcc
export CXX=$(MUSL-PREFIX)-g++
export AR=$(MUSL-PREFIX)-ar
GO_STATIC_FLAGS=-ldflags "-s -w -extldflags \"-static\"" -tags 'netgo,osusergo,static_build$(addprefix ,,$(GO_TAGS))'
endif
endif
# API (doc) generation utilities
CONTROLLER_GEN_VERSION ?= v0.7.0
GEN_API_REF_DOCS_VERSION ?= v0.3.0
@ -179,15 +167,11 @@ install-envtest: setup-envtest ## Download envtest binaries locally.
libgit2: $(LIBGIT2) ## Detect or download libgit2 library
COSIGN = $(GOBIN)/cosign
$(LIBGIT2): $(MUSL-CC)
$(LIBGIT2):
$(call go-install-tool,$(COSIGN),github.com/sigstore/cosign/cmd/cosign@latest)
IMG=$(LIBGIT2_IMG) TAG=$(LIBGIT2_TAG) PATH=$(PATH):$(GOBIN) ./hack/install-libraries.sh
$(MUSL-CC):
ifneq ($(shell uname -s),Darwin)
./hack/download-musl.sh
endif
.PHONY: help
help: ## Display this help menu

22
controllers/suite_test.go
View File

@ -45,6 +45,7 @@ import (
dockerRegistry "github.com/distribution/distribution/v3/registry"
_ "github.com/distribution/distribution/v3/registry/auth/htpasswd"
_ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory"
git2go "github.com/libgit2/git2go/v33"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
"github.com/fluxcd/source-controller/internal/cache"
@ -179,6 +180,8 @@ func setupRegistryServer(ctx context.Context) (*registryClientTestServer, error)
}
func TestMain(m *testing.M) {
mustHaveNoThreadSupport()
initTestTLS()
utilruntime.Must(sourcev1.AddToScheme(scheme.Scheme))
@ -333,3 +336,22 @@ func randStringRunes(n int) string {
func int64p(i int64) *int64 {
return &i
}
// This provides a regression assurance for image-automation-controller/#339.
// Validates that:
// - libgit2 was built with no support for threads.
// - git2go accepts libgit2 built with no support for threads.
//
// The logic below does the validation of the former, whilst
// referring to git2go forces its init() execution, which is
// where any validation to that effect resides.
//
// git2go does not support threadless libgit2 by default,
// hence a fork is being used which disables such validation.
//
// TODO: extract logic into pkg.
func mustHaveNoThreadSupport() {
if git2go.Features()&git2go.FeatureThreads != 0 {
panic("libgit2 must not be build with thread support")
}
}

10
go.mod
View File

@ -4,6 +4,16 @@ go 1.18
replace github.com/fluxcd/source-controller/api => ./api
// A temporary fork of git2go was created to enable use
// of libgit2 without thread support to fix:
// fluxcd/image-automation-controller/#339.
//
// This can be removed once libgit2/git2go#918 is merged.
//
// The fork automatically releases new patches based on upstream:
// https://github.com/pjbgf/git2go/commit/d72e39cdc20f7fe014ba73072b01ba7b569e9253
replace github.com/libgit2/git2go/v33 => github.com/pjbgf/git2go/v33 v33.0.9-nothread-check
require (
cloud.google.com/go/storage v1.23.0
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.22.0

4
go.sum
View File

@ -593,8 +593,6 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs=
github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/libgit2/git2go/v33 v33.0.9 h1:4ch2DJed6IhJO28BEohkUoGvxLsRzUjxljoNFJ6/O78=
github.com/libgit2/git2go/v33 v33.0.9/go.mod h1:KdpqkU+6+++4oHna/MIOgx4GCQ92IPCdpVRMRI80J+4=
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
@ -719,6 +717,8 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI=
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
github.com/pjbgf/git2go/v33 v33.0.9-nothread-check h1:gSK7FaLECIM3VSuBOAsVZQtWd+51iTB5lv9RyxhOYMk=
github.com/pjbgf/git2go/v33 v33.0.9-nothread-check/go.mod h1:KdpqkU+6+++4oHna/MIOgx4GCQ92IPCdpVRMRI80J+4=
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI=
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=

71
hack/download-musl.sh
View File

@ -1,71 +0,0 @@
#!/usr/bin/env bash
set -eoux pipefail
MUSL_X86_64_FILENAME=x86_64-linux-musl-native.tgz
MUSL_X86_64_SHA512=44d441ad9aa11a06feddf3daa4c9f53ad7d9ca37af1f5a61379aca07793703d179410cea723c1b7fca94c4de19a321228bdb3656bc5cbdb5e3bea8e2d6dac6c7
MUSL_AARCH64_FILENAME=aarch64-linux-musl-native.tgz
MUSL_AARCH64_SHA512=16d544e09845c9dbba50f29e0cb04dd661e17eb63c56acad6a67fd2a78aa7596b792477c7177d3cd56d408a27dc291a90507df882f2b099c0f25511ce08fd3b5
MUSL_XX86_64_FILENAME=x86_64-linux-musl-cross.tgz
MUSL_XX86_64_SHA512=52abd1a56e670952116e35d1a62e048a9b6160471d988e16fa0e1611923dd108a581d2e00874af5eb04e4968b1ba32e0eb449a1f15c3e4d5240ebe09caf5a9f3
MUSL_XAARCH64_FILENAME=aarch64-linux-musl-cross.tgz
MUSL_XAARCH64_SHA512=8695ff86979cdf30fbbcd33061711f5b1ebc3c48a87822b9ca56cde6d3a22abd4dab30fdcd1789ac27c6febbaeb9e5bde59d79d66552fae53d54cc1377a19272
MUSL_XARMV7_FILENAME=armv7l-linux-musleabihf-cross.tgz
MUSL_XARMV7_SHA512=1bb399a61da425faac521df9b8d303e60ad101f6c7827469e0b4bc685ce1f3dedc606ac7b1e8e34d79f762a3bfe3e8ab479a97e97d9f36fbd9fc5dc9d7ed6fd1
TARGET_ARCH="${TARGET_ARCH:-$(uname -m)}"
ENV_FILE="${ENV_FILE:-false}"
MUSL_FILENAME=""
MUSL_SHA512=""
ROOT_DIR="${ROOT_DIR:-$(git rev-parse --show-toplevel)}"
MUSL_DIR="${ROOT_DIR}/build/musl"
if [ "${TARGET_ARCH}" = "$(uname -m)" ]; then
MUSL_FILENAME="${MUSL_X86_64_FILENAME}"
MUSL_SHA512="${MUSL_X86_64_SHA512}"
MUSL_PREFIX="${TARGET_ARCH}-linux-musl-native/bin/${TARGET_ARCH}-linux-musl"
if [ "${TARGET_ARCH}" = "arm64" ] || [ "${TARGET_ARCH}" = "aarch64" ]; then
MUSL_FILENAME="${MUSL_AARCH64_FILENAME}"
MUSL_SHA512="${MUSL_AARCH64_SHA512}"
fi
else
MUSL_FILENAME="${MUSL_XX86_64_FILENAME}"
MUSL_SHA512="${MUSL_XX86_64_SHA512}"
MUSL_PREFIX="${TARGET_ARCH}-linux-musl-cross/bin/${TARGET_ARCH}-linux-musl"
if [ "${TARGET_ARCH}" = "arm64" ] || [ "${TARGET_ARCH}" = "aarch64" ]; then
MUSL_FILENAME="${MUSL_XAARCH64_FILENAME}"
MUSL_SHA512="${MUSL_XAARCH64_SHA512}"
elif [ "${TARGET_ARCH}" = "arm" ] || [ "${TARGET_ARCH}" = "armv7" ]; then
MUSL_FILENAME="${MUSL_XARMV7_FILENAME}"
MUSL_SHA512="${MUSL_XARMV7_SHA512}"
MUSL_PREFIX=armv7l-linux-musleabihf-cross/bin/armv7l-linux-musleabihf
fi
fi
mkdir -p "${MUSL_DIR}"
if "${ENV_FILE}"; then
cat<<EOF > "${MUSL_DIR}/${TARGET_ARCH}.env"
CC="$(pwd)/build/musl/${MUSL_PREFIX}-gcc"
CXX="$(pwd)/build/musl/${MUSL_PREFIX}-g++"
AR="$(pwd)/build/musl/${MUSL_PREFIX}-ar"
EOF
fi
MUSL_AARCH64_URL="https://more.musl.cc/11.2.1/x86_64-linux-musl/${MUSL_FILENAME}"
if [ ! -f "${MUSL_DIR}/bin" ]; then
TARGET_FILE="${MUSL_DIR}/${MUSL_FILENAME}"
curl -o "${TARGET_FILE}" -LO "${MUSL_AARCH64_URL}"
if ! echo "${MUSL_SHA512} ${TARGET_FILE}" | sha512sum; then
echo "Checksum failed for ${MUSL_FILENAME}."
rm -rf "${MUSL_DIR}"
exit 1
fi
tar xzf "${TARGET_FILE}" -C "${MUSL_DIR}"
rm "${TARGET_FILE}"
fi

4
hack/install-libraries.sh
View File

@ -79,7 +79,7 @@ fix_pkgconfigs(){
# Update the prefix paths included in the .pc files.
if [[ $OSTYPE == 'darwin'* ]]; then
# https://github.com/fluxcd/golang-with-libgit2/blob/v0.1.4/.github/workflows/release.yaml#L158
INSTALLED_DIR="/Users/runner/work/golang-with-libgit2/golang-with-libgit2/build/libgit2-darwin-amd64"
INSTALLED_DIR="/Users/runner/work/golang-with-libgit2/golang-with-libgit2/build/darwin-libgit2-only"
# This will make it easier to update to the location in which they will be used.
# sed has a sight different behaviour in MacOS
@ -137,7 +137,7 @@ install_libraries(){
fi
fi
FILE_NAME="linux-x86_64-libgit2-only.tar.gz"
FILE_NAME="linux-$(uname -m)-libgit2-only.tar.gz"
DIR="linux-libgit2-only"
if [[ $OSTYPE == 'darwin'* ]]; then
FILE_NAME="darwin-libgit2-only.tar.gz"

2
tests/fuzz/oss_fuzz_build.sh
View File

@ -16,7 +16,7 @@
set -euxo pipefail
LIBGIT2_TAG="${LIBGIT2_TAG:-v0.1.4}"
LIBGIT2_TAG="${LIBGIT2_TAG:-v0.2.0}"
GOPATH="${GOPATH:-/root/go}"
GO_SRC="${GOPATH}/src"
PROJECT_PATH="github.com/fluxcd/source-controller"