Update libgit2 to 1.1.1-6

Fix issues developing in amd64, arm64 and apple silicon

Signed-off-by: Paulo Gomes <paulo.gomes@weave.works>

.gitignore

@@ -17,8 +17,5 @@ bin/
 testbin/
 config/release/
 
-# Exclude all libgit2 related files
-hack/libgit2/
-
 # Exclude temporary build files
 build/

Dockerfile

@@ -84,8 +84,7 @@ ENV CGO_ENABLED=1
 RUN export $(cat build/musl/$(xx-info alpine-arch).env | xargs) && \
     export LIBRARY_PATH="/usr/local/$(xx-info triple):/usr/local/$(xx-info triple)/lib64" && \
     export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig:/usr/local/$(xx-info triple)/lib64/pkgconfig" && \
-    export FLAGS="$(pkg-config --static --libs --cflags libssh2 openssl libgit2)" && \
+    export CGO_LDFLAGS="$(pkg-config --static --libs --cflags libssh2 openssl libgit2) -static" && \
-    export CGO_LDFLAGS="${FLAGS} -static" && \
     GOARCH=$TARGETARCH go build  \
         -ldflags "-s -w" \
         -tags 'netgo,osusergo,static_build' \

Makefile

@@ -17,48 +17,60 @@ CRD_OPTIONS ?= crd:crdVersions=v1
 
 # Repository root based on Git metadata
 REPOSITORY_ROOT := $(shell git rev-parse --show-toplevel)
+BUILD_DIR := $(REPOSITORY_ROOT)/build
 
 # Other dependency versions
 ENVTEST_BIN_VERSION ?= 1.19.2
 
 # Caches libgit2 versions per tag, "forcing" rebuild only when needed.
-LIBGIT2_PATH := $(REPOSITORY_ROOT)/build/libgit2/$(LIBGIT2_TAG)
+LIBGIT2_PATH := $(BUILD_DIR)/libgit2/$(LIBGIT2_TAG)
 LIBGIT2_LIB_PATH := $(LIBGIT2_PATH)/lib
 LIBGIT2_LIB64_PATH := $(LIBGIT2_PATH)/lib64
 LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.a
 MUSL-CC =
 
 export CGO_ENABLED=1
-export PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig:$(LIBGIT2_LIB64_PATH)/pkgconfig
+export PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig
-export LD_LIBRARY_PATH=$(LIBGIT2_LIB_PATH):$(LIBGIT2_LIB64_PATH)
+export LIBRARY_PATH=$(LIBGIT2_LIB_PATH)
 export CGO_CFLAGS=-I$(LIBGIT2_PATH)/include -I$(LIBGIT2_PATH)/include/openssl
 
+
 ifeq ($(shell uname -s),Darwin)
-	export CGO_LDFLAGS=-L$(LIBGIT2_LIB_PATH) -lssh2 -lssl -lcrypto -lgit2
+export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libssh2 openssl libgit2)
+GO_STATIC_FLAGS=-ldflags "-s -w" -tags 'netgo,osusergo,static_build'
 else
+export PKG_CONFIG_PATH:=$(PKG_CONFIG_PATH):$(LIBGIT2_LIB64_PATH)/pkgconfig
+export LIBRARY_PATH:=$(LIBRARY_PATH):$(LIBGIT2_LIB64_PATH)
 export CGO_LDFLAGS=$(shell PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) pkg-config --libs --static --cflags libssh2 openssl libgit2)
 endif
 
 
 ifeq ($(shell uname -s),Linux)
-	MUSL-PREFIX=$(REPOSITORY_ROOT)/build/musl/$(shell uname -m)-linux-musl-native/bin/$(shell uname -m)-linux-musl
+ifeq ($(shell uname -m),x86_64)
+# Linux x86_64 seem to be able to cope with the static libraries 
+# by having only musl-dev installed, without the need of using musl toolchain.
+	GO_STATIC_FLAGS=-ldflags "-s -w" -tags 'netgo,osusergo,static_build'
+else
+	MUSL-PREFIX=$(BUILD_DIR)/musl/$(shell uname -m)-linux-musl-native/bin/$(shell uname -m)-linux-musl
 	MUSL-CC=$(MUSL-PREFIX)-gcc
 	export CC=$(MUSL-PREFIX)-gcc
 	export CXX=$(MUSL-PREFIX)-g++
 	export AR=$(MUSL-PREFIX)-ar
+	GO_STATIC_FLAGS=-ldflags "-s -w -extldflags \"-static\"" -tags 'netgo,osusergo,static_build'
+endif
 endif
 
 # API (doc) generation utilities
 CONTROLLER_GEN_VERSION ?= v0.7.0
 GEN_API_REF_DOCS_VERSION ?= v0.3.0
 
-# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
+# If gobin not set, create one on ./build and add to path.
 ifeq (,$(shell go env GOBIN))
-GOBIN=$(shell go env GOPATH)/bin
+export GOBIN=$(BUILD_DIR)/gobin
 else
-GOBIN=$(shell go env GOBIN)
+export GOBIN=$(shell go env GOBIN)
 endif
-
+export PATH:=${GOBIN}:${PATH}
 
 # Architecture to use envtest with
 ifeq ($(shell uname -m),x86_64)
@@ -67,24 +79,31 @@ else
 ENVTEST_ARCH ?= arm64
 endif
 
+ifeq ($(shell uname -s),Darwin)
+# Envtest only supports darwin-amd64
+ENVTEST_ARCH=amd64
+endif
+
 all: build
 
-build: $(LIBGIT2) ## Build manager binary
+build: check-deps $(LIBGIT2) ## Build manager binary
-	go build -o bin/manager main.go
+	go build $(GO_STATIC_FLAGS) -o $(BUILD_DIR)/bin/manager main.go
 
 KUBEBUILDER_ASSETS?="$(shell $(ENVTEST) --arch=$(ENVTEST_ARCH) use -i $(ENVTEST_KUBERNETES_VERSION) --bin-dir=$(ENVTEST_ASSETS_DIR) -p path)"
-test: $(LIBGIT2) install-envtest test-api  ## Run tests
+test: $(LIBGIT2) install-envtest test-api check-deps ## Run tests
 	KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) \
-	go test ./... \
+	go test $(GO_STATIC_FLAGS) ./... -coverprofile cover.out
-		-ldflags "-s -w" \
+
-		-coverprofile cover.out \
+check-deps:
-	 	-tags 'netgo,osusergo,static_build'
+ifeq ($(shell uname -s),Darwin)
+	if ! command -v pkg-config &> /dev/null; then echo "pkg-config is required"; exit 1; fi
+endif
 
 test-api: ## Run api tests
 	cd api; go test ./... -coverprofile cover.out
 
 run: $(LIBGIT2) generate fmt vet manifests  ## Run against the configured Kubernetes cluster in ~/.kube/config
-	go run ./main.go
+	go run $(GO_STATIC_FLAGS) ./main.go
 
 install: manifests  ## Install CRDs into a cluster
 	kustomize build config/crd | kubectl apply -f -
@@ -136,23 +155,23 @@ docker-push:  ## Push Docker image
 	docker push $(IMG):$(TAG)
 
 # Find or download controller-gen
-CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
+CONTROLLER_GEN = $(GOBIN)/controller-gen
 .PHONY: controller-gen
 controller-gen: ## Download controller-gen locally if necessary.
 	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.7.0)
 
 # Find or download gen-crd-api-reference-docs
-GEN_CRD_API_REFERENCE_DOCS = $(shell pwd)/bin/gen-crd-api-reference-docs
+GEN_CRD_API_REFERENCE_DOCS = $(GOBIN)/gen-crd-api-reference-docs
 .PHONY: gen-crd-api-reference-docs
 gen-crd-api-reference-docs: ## Download gen-crd-api-reference-docs locally if necessary
 	$(call go-install-tool,$(GEN_CRD_API_REFERENCE_DOCS),github.com/ahmetb/gen-crd-api-reference-docs@v0.3.0)
 
-ENVTEST = $(shell pwd)/bin/setup-envtest
+ENVTEST = $(GOBIN)/setup-envtest
 .PHONY: envtest
 setup-envtest: ## Download setup-envtest locally if necessary.
 	$(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest)
 
-ENVTEST_ASSETS_DIR=$(shell pwd)/testbin
+ENVTEST_ASSETS_DIR=$(BUILD_DIR)/testbin
 ENVTEST_KUBERNETES_VERSION?=latest
 install-envtest: setup-envtest ## Download envtest binaries locally.
 	mkdir -p ${ENVTEST_ASSETS_DIR}
@@ -188,7 +207,6 @@ ifneq (, $(shell git status --porcelain --untracked-files=no))
 endif
 
 # go-install-tool will 'go install' any package $2 and install it to $1.
-PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
 define go-install-tool
 @[ -f $(1) ] || { \
 set -e ;\
@@ -196,7 +214,7 @@ TMP_DIR=$$(mktemp -d) ;\
 cd $$TMP_DIR ;\
 go mod init tmp ;\
 echo "Downloading $(2)" ;\
-GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
+go install $(2) ;\
 rm -rf $$TMP_DIR ;\
 }
 endef

hack/download-musl.sh

@@ -26,7 +26,7 @@ MUSL_DIR="${ROOT_DIR}/build/musl"
 if [ "${TARGET_ARCH}" = "$(uname -m)" ]; then
     MUSL_FILENAME="${MUSL_X86_64_FILENAME}"
     MUSL_SHA512="${MUSL_X86_64_SHA512}"
-    MUSL_PREFIX=$(xx-info alpine-arch)-linux-musl-native/bin/$(xx-info alpine-arch)-linux-musl
+    MUSL_PREFIX="${TARGET_ARCH}-linux-musl-native/bin/${TARGET_ARCH}-linux-musl"
     if [ "${TARGET_ARCH}" = "arm64" ] || [ "${TARGET_ARCH}" = "aarch64" ]; then
         MUSL_FILENAME="${MUSL_AARCH64_FILENAME}"
         MUSL_SHA512="${MUSL_AARCH64_SHA512}"
@@ -34,7 +34,7 @@ if [ "${TARGET_ARCH}" = "$(uname -m)" ]; then
 else
     MUSL_FILENAME="${MUSL_XX86_64_FILENAME}"
     MUSL_SHA512="${MUSL_XX86_64_SHA512}"
-    MUSL_PREFIX=$(xx-info alpine-arch)-linux-musl-cross/bin/$(xx-info alpine-arch)-linux-musl
+    MUSL_PREFIX="${TARGET_ARCH}-linux-musl-cross/bin/${TARGET_ARCH}-linux-musl"
     if [ "${TARGET_ARCH}" = "arm64" ] || [ "${TARGET_ARCH}" = "aarch64" ]; then
         MUSL_FILENAME="${MUSL_XAARCH64_FILENAME}"
         MUSL_SHA512="${MUSL_XAARCH64_SHA512}"

hack/install-libraries.sh

@@ -27,20 +27,13 @@ function setup() {
     NEW_DIR="$(/bin/pwd)/build/libgit2/${TAG}"
     INSTALLED_DIR="/usr/local/${DIR}"
 
-    mkdir -p "./build/libgit2"
-
     mv "local/${DIR}" "${TAG}"
     rm -rf "local"
     mv "${TAG}/" "./build/libgit2"
 
     # Update the prefix paths included in the .pc files.
     # This will make it easier to update to the location in which they will be used.
-    if [[ $OSTYPE == 'darwin'* ]]; then    
-        # sed has a sight different behaviour in MacOS
-        find "${NEW_DIR}" -type f -name "*.pc" | xargs -I {} sed -i "" "s;${INSTALLED_DIR};${NEW_DIR};g" {}
-    else
     find "${NEW_DIR}" -type f -name "*.pc" | xargs -I {} sed -i "s;${INSTALLED_DIR};${NEW_DIR};g" {}
-    fi
 }
 
 function setup_current() {
@@ -49,6 +42,29 @@ function setup_current() {
         exit 0
     fi
 
+    mkdir -p "./build/libgit2"
+    if [[ $OSTYPE == 'darwin'* ]]; then
+        # For MacOS development environments, download the amd64 static libraries released from from golang-with-libgit2.
+
+        #TODO: update URL with official URL + TAG:
+        curl -o output.tar.gz -LO "https://github.com/pjbgf/golang-with-libgit2/releases/download/1.1.1-6/darwin-libs.tar.gz"
+       
+        DIR=libgit2-darwin
+        NEW_DIR="$(/bin/pwd)/build/libgit2/${TAG}"
+        INSTALLED_DIR="/Users/runner/work/golang-with-libgit2/golang-with-libgit2/build/${DIR}-amd64"
+
+        tar -xf output.tar.gz
+        mv "${DIR}" "${TAG}"
+        mv "${TAG}/" "./build/libgit2"
+
+        sed -i "" "s;-L/Applications/Xcode_12.4.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk/usr/lib ;;g" "$(/bin/pwd)/build/libgit2/${TAG}/lib/pkgconfig/libgit2.pc"
+
+        # Update the prefix paths included in the .pc files.
+        # This will make it easier to update to the location in which they will be used.
+        # sed has a sight different behaviour in MacOS
+        find "${NEW_DIR}" -type f -name "*.pc" | xargs -I {} sed -i "" "s;${INSTALLED_DIR};${NEW_DIR};g" {}
+    else
+        # for linux development environments, use the static libraries from the official container images.
         DIR="x86_64-alpine-linux-musl"
         PLATFORM="linux/amd64"
 
@@ -61,6 +77,7 @@ function setup_current() {
         fi
         
         setup "${PLATFORM}" "${DIR}"
+    fi
 }
 
 setup_current