Convert from glibc-based binaries to musl-based binaries

The intent of the previous implementation was to avoid libc, but it turns out that just invoking a syscall without libc is complicated (see https://github.com/docker-library/hello-world/pull/62#issuecomment-568573535 for details).

On the other hand, my personal machine can cross-compile all of musl in ~30s per architecture, which is pretty reasonable, and the resulting binary sizes are only around ~10k each, and I was able to do so successfully for every architecture we currently support.

Dockerfile.build

@@ -1,30 +1,50 @@
 # explicitly use Debian for maximum cross-architecture compatibility
-FROM debian:stretch-slim
+FROM debian:buster-slim
 
-RUN dpkg --add-architecture i386
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
+RUN set -eux; \
+	apt-get update; \
+	apt-get install -y --no-install-recommends \
+		ca-certificates \
+		gnupg dirmngr \
+		wget \
+		\
 		gcc \
 		libc6-dev \
 		make \
 		\
-		libc6-dev:i386 \
-		libgcc-6-dev:i386 \
-		\
 		libc6-dev-arm64-cross \
 		libc6-dev-armel-cross \
 		libc6-dev-armhf-cross \
+		libc6-dev-i386-cross \
 		libc6-dev-ppc64el-cross \
 		libc6-dev-s390x-cross \
 		\
 		gcc-aarch64-linux-gnu \
 		gcc-arm-linux-gnueabi \
 		gcc-arm-linux-gnueabihf \
+		gcc-i686-linux-gnu \
 		gcc-powerpc64le-linux-gnu \
 		gcc-s390x-linux-gnu \
 		\
 		file \
-	&& rm -rf /var/lib/apt/lists/*
+	; \
+	rm -rf /var/lib/apt/lists/*
+
+# https://www.musl-libc.org/download.html
+ENV MUSL_VERSION 1.1.24
+RUN set -eux; \
+	wget -O musl.tgz.asc "https://www.musl-libc.org/releases/musl-$MUSL_VERSION.tar.gz.asc"; \
+	wget -O musl.tgz "https://www.musl-libc.org/releases/musl-$MUSL_VERSION.tar.gz"; \
+	\
+	export GNUPGHOME="$(mktemp -d)"; \
+	gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys '836489290BB6B70F99FFDA0556BCDB593020450F'; \
+	gpg --batch --verify musl.tgz.asc musl.tgz; \
+	gpgconf --kill all; \
+	rm -rf "$GNUPGHOME" musl.tgz.asc; \
+	\
+	mkdir /usr/local/src/musl; \
+	tar --extract --file musl.tgz --directory /usr/local/src/musl --strip-components 1; \
+	rm musl.tgz
 
 WORKDIR /usr/src/hello
 COPY . .
@@ -32,44 +52,38 @@ COPY . .
 RUN set -ex; \
 	make clean all test \
 		TARGET_ARCH='amd64' \
-		CC='x86_64-linux-gnu-gcc' \
-		STRIP='x86_64-linux-gnu-strip'
+		CROSS_COMPILE='x86_64-linux-gnu-'
 
 RUN set -ex; \
 	make clean all \
 		TARGET_ARCH='arm32v5' \
-		CC='arm-linux-gnueabi-gcc' \
-		STRIP='arm-linux-gnueabi-strip'
+		CROSS_COMPILE='arm-linux-gnueabi-'
 
 RUN set -ex; \
 	make clean all \
 		TARGET_ARCH='arm32v7' \
-		CC='arm-linux-gnueabihf-gcc' \
-		STRIP='arm-linux-gnueabihf-strip'
+		CROSS_COMPILE='arm-linux-gnueabihf-'
 
 RUN set -ex; \
 	make clean all \
 		TARGET_ARCH='arm64v8' \
-		CC='aarch64-linux-gnu-gcc' \
-		STRIP='aarch64-linux-gnu-strip'
+		CROSS_COMPILE='aarch64-linux-gnu-'
 
 RUN set -ex; \
 	make clean all test \
 		TARGET_ARCH='i386' \
-		CC='gcc -m32 -L/usr/lib/gcc/i686-linux-gnu/6' \
-		STRIP='x86_64-linux-gnu-strip'
+		CROSS_COMPILE='i686-linux-gnu-'
 
 RUN set -ex; \
 	make clean all \
 		TARGET_ARCH='ppc64le' \
-		CC='powerpc64le-linux-gnu-gcc' \
-		STRIP='powerpc64le-linux-gnu-strip'
+		CROSS_COMPILE='powerpc64le-linux-gnu-' \
+		CFLAGS+='-mlong-double-64'
 
 RUN set -ex; \
 	make clean all \
 		TARGET_ARCH='s390x' \
-		CC='s390x-linux-gnu-gcc' \
-		STRIP='s390x-linux-gnu-strip'
+		CROSS_COMPILE='s390x-linux-gnu-'
 
 RUN find \( -name 'hello' -or -name 'hello.txt' \) -exec file '{}' + -exec ls -lh '{}' +
 

Makefile

@@ -1,16 +1,30 @@
 TARGET_ARCH := amd64
 C_TARGETS := $(addsuffix hello, $(wildcard $(TARGET_ARCH)/*/))
 
-CC := gcc
-CFLAGS := -static -Os -nostartfiles -fno-asynchronous-unwind-tables
-STRIP := strip
+export CFLAGS := -Os -fdata-sections -ffunction-sections -s
+STRIP := $(CROSS_COMPILE)strip
 
 .PHONY: all
 all: $(C_TARGETS)
 
-$(C_TARGETS): hello.c
-	$(CC) $(CFLAGS) -o '$@' -D DOCKER_IMAGE='"$(notdir $(@D))"' -D DOCKER_GREETING="\"$$(cat 'greetings/$(notdir $(@D)).txt')\"" -D DOCKER_ARCH='"$(TARGET_ARCH)"' '$<'
-	$(STRIP) -R .comment -s '$@'
+MUSL_SRC := /usr/local/src/musl
+MUSL_DIR := $(CURDIR)/musl/$(TARGET_ARCH)
+MUSL_PREFIX := $(MUSL_DIR)/prefix
+MUSL_GCC := $(MUSL_PREFIX)/bin/musl-gcc
+
+$(MUSL_GCC):
+	mkdir -p '$(MUSL_DIR)'
+	cd '$(MUSL_DIR)' && '$(MUSL_SRC)/configure' --disable-shared --prefix='$(MUSL_PREFIX)' > /dev/null
+	$(MAKE) -C '$(MUSL_DIR)' -j '$(shell nproc)' install > /dev/null
+
+$(C_TARGETS): hello.c $(MUSL_GCC)
+	$(MUSL_GCC) $(CFLAGS) -Wl,--gc-sections -static \
+		-o '$@' \
+		-D DOCKER_IMAGE='"$(notdir $(@D))"' \
+		-D DOCKER_GREETING="\"$$(cat 'greetings/$(notdir $(@D)).txt')\"" \
+		-D DOCKER_ARCH='"$(TARGET_ARCH)"' \
+		'$<'
+	$(STRIP) --strip-all --remove-section=.comment '$@'
 	@if [ '$(TARGET_ARCH)' = 'amd64' ]; then \
 		for winVariant in \
 			nanoserver-1809 \
@@ -27,7 +41,7 @@ $(C_TARGETS): hello.c
 
 .PHONY: clean
 clean:
-	-rm -vrf $(C_TARGETS)
+	-rm -vrf $(C_TARGETS) $(MUSL_DIR)
 
 .PHONY: test
 test: $(C_TARGETS)

hello.c

@@ -1,5 +1,5 @@
-//#include <unistd.h>
 #include <sys/syscall.h>
+#include <unistd.h>
 
 #ifndef DOCKER_IMAGE
 	#define DOCKER_IMAGE "hello-world"
@@ -37,10 +37,11 @@ const char message[] =
 	" https://docs.docker.com/get-started/\n"
 	"\n";
 
-void _start() {
+int main() {
 	//write(1, message, sizeof(message) - 1);
 	syscall(SYS_write, 1, message, sizeof(message) - 1);
 
 	//_exit(0);
-	syscall(SYS_exit, 0);
+	//syscall(SYS_exit, 0);
+	return 0;
 }

update.sh

@@ -7,7 +7,7 @@ set -x
 
 docker build -f Dockerfile.build -t hello-world:build .
 
-rm -rf */hello */nanoserver*/hello.txt
+find */ \( -name hello -or -name hello.txt \) -delete
 docker run --rm hello-world:build sh -c 'find \( -name hello -or -name hello.txt \) -print0 | xargs -0 tar --create' | tar --extract --verbose
 
 find -name hello -type f -exec dirname '{}' ';' | xargs -n1 -i'{}' cp Dockerfile-linux.template '{}/Dockerfile'