From fbb575f95480488e58904087e67c173f3e50619f Mon Sep 17 00:00:00 2001 From: Gaius Date: Tue, 27 Jun 2023 17:24:09 +0800 Subject: [PATCH] chore: add trainer to Makefile and shell (#2488) Signed-off-by: Gaius --- Makefile | 42 +++++++++++++++++++++++++++++---- build/images/trainer/Dockerfile | 34 ++++++++++++++++++++++++++ hack/build.sh | 17 +++++++++++++ hack/docker-build.sh | 3 +++ hack/docker-push.sh | 3 +++ hack/gen-buildx.sh | 2 +- hack/install.sh | 19 +++++++++++++++ hack/kind-load.sh | 3 +++ 8 files changed, 118 insertions(+), 5 deletions(-) create mode 100644 build/images/trainer/Dockerfile diff --git a/Makefile b/Makefile index b5ac7fa97..ef2c7bf42 100644 --- a/Makefile +++ b/Makefile @@ -34,12 +34,12 @@ build-dirs: .PHONY: build-dirs # Build dragonlfy. -docker-build: docker-build-dfdaemon docker-build-scheduler docker-build-manager +docker-build: docker-build-dfdaemon docker-build-scheduler docker-build-manager docker-build-trainer @echo "Build image done." .PHONY: docker-build # Push dragonfly images. -docker-push: docker-push-dfdaemon docker-push-scheduler docker-push-manager +docker-push: docker-push-dfdaemon docker-push-scheduler docker-push-manager docker-build-trainer @echo "Push image done." .PHONY: docker-push @@ -61,6 +61,12 @@ docker-build-manager: ./hack/docker-build.sh manager .PHONY: docker-build-manager +# Build trainer image. +docker-build-trainer: + @echo "Begin to use docker build trainer image." + ./hack/docker-build.sh trainer +.PHONY: docker-build-trainer + # Build testing tools image. docker-build-testing-tools: build-dirs @echo "Begin to testing tools image." @@ -85,8 +91,14 @@ docker-push-manager: docker-build-manager ./hack/docker-push.sh manager .PHONY: docker-push-manager +# Push trainer image. +docker-push-trainer: docker-build-trainer + @echo "Begin to push trainer docker image." + ./hack/docker-push.sh trainer +.PHONY: docker-push-trainer + # Build dragonfly. -build: build-manager build-scheduler build-dfget build-dfcache build-dfstore +build: build-manager build-scheduler build-trainer build-dfget build-dfcache build-dfstore .PHONY: build # Build dfget. @@ -149,6 +161,12 @@ build-manager-console: build-dirs ./hack/build.sh manager-console .PHONY: build-manager-console +# Build trainer. +build-trainer: build-dirs + @echo "Begin to build trainer." + ./hack/build.sh trainer +.PHONY: build-trainer + # Install dfget. install-dfget: @echo "Begin to install dfget." @@ -167,6 +185,12 @@ install-manager: ./hack/install.sh install manager .PHONY: install-manager +# Install trainer. +install-trainer: + @echo "Begin to install trainer." + ./hack/install.sh install trainer +.PHONY: install-trainer + # Build rpm dfget. build-rpm-dfget: build-linux-dfget @echo "Begin to build rpm dfget." @@ -349,7 +373,7 @@ clean-e2e-test: .PHONY: clean-e2e-test # Kind load dragonlfy. -kind-load: kind-load-scheduler kind-load-dfdaemon kind-load-manager kind-load-testing-tools +kind-load: kind-load-scheduler kind-load-dfdaemon kind-load-manager kind-load-trainer kind-load-testing-tools @echo "Kind load image done." .PHONY: kind-load @@ -368,6 +392,11 @@ kind-load-manager: @./hack/kind-load.sh manager .PHONY: kind-load-manager +# Run kind load docker trainer. +kind-load-trainer: + @./hack/kind-load.sh trainer +.PHONY: kind-load-trainer + # Run kind load docker testing tools. kind-load-testing-tools: @./hack/kind-load.sh no-content-length @@ -412,9 +441,11 @@ help: @echo "make docker-build-dfdaemon build dfdaemon image" @echo "make docker-build-scheduler build scheduler image" @echo "make docker-build-manager build manager image" + @echo "make docker-build-trainer build trainer image" @echo "make docker-push-dfdaemon push dfdaemon image" @echo "make docker-push-scheduler push scheduler image" @echo "make docker-push-manager push manager image" + @echo "make docker-push-trainer push trainer image" @echo "make build build dragonfly" @echo "make build-dfget build dfget" @echo "make build-linux-dfget build linux dfget" @@ -426,11 +457,13 @@ help: @echo "make build-manager build manager" @echo "make build-manager-server build manager server" @echo "make build-manager-console build manager console" + @echo "make build-trainer build trainer" @echo "make build-e2e-sha256sum build sha256sum test tool" @echo "make build-e2e-download-grpc-test build download grpc test tool" @echo "make install-dfget install dfget" @echo "make install-scheduler install scheduler" @echo "make install-manager install manager" + @echo "make install-trainer install trainer" @echo "make build-rpm-dfget build rpm dfget" @echo "make build-rpm-dfcache build rpm dfcache" @echo "make build-rpm-dfstore build rpm dfstore" @@ -452,6 +485,7 @@ help: @echo "make kind-load-scheduler kind load scheduler docker image" @echo "make kind-load-dfdaemon kind load dfdaemon docker image" @echo "make kind-load-manager kind load manager docker image" + @echo "make kind-load-trainer kind load trainer docker image" @echo "make kind-load-testing-tools kind load testing tools docker image" @echo "make lint run code lint" @echo "make markdownlint run markdown lint" diff --git a/build/images/trainer/Dockerfile b/build/images/trainer/Dockerfile new file mode 100644 index 000000000..f9d0e2181 --- /dev/null +++ b/build/images/trainer/Dockerfile @@ -0,0 +1,34 @@ +ARG BASE_IMAGE=alpine:3.16 + +FROM golang:1.20.1-alpine3.16 as builder + +ARG GOPROXY +ARG GOTAGS +ARG GOGCFLAGS + +WORKDIR /go/src/d7y.io/dragonfly/v2 + +RUN apk --no-cache add bash make gcc libc-dev git + +COPY . /go/src/d7y.io/dragonfly/v2 + +RUN make build-trainer && make install-trainer + +FROM ${BASE_IMAGE} as health + +ENV GRPC_HEALTH_PROBE_VERSION v0.4.16 + +RUN wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /bin/grpc_health_probe + +FROM ${BASE_IMAGE} + +ENV PATH=/opt/dragonfly/bin:$PATH +RUN echo "hosts: files dns" > /etc/nsswitch.conf + +COPY --from=builder /opt/dragonfly/bin/trainer /opt/dragonfly/bin/trainer +COPY --from=health /bin/grpc_health_probe /bin/grpc_health_probe + +EXPOSE 8002 + +ENTRYPOINT ["/opt/dragonfly/bin/trainer"] diff --git a/hack/build.sh b/hack/build.sh index f7afe8c23..bfe44942c 100755 --- a/hack/build.sh +++ b/hack/build.sh @@ -9,6 +9,7 @@ DFCACHE_BINARY_NAME=dfcache DFSTORE_BINARY_NAME=dfstore SCHEDULER_BINARY_NAME=scheduler MANAGER_BINARY_NAME=manager +TRAINER_BINARY_NAME=trainer PKG=d7y.io/dragonfly/v2 BUILD_IMAGE=golang:1.20.1-alpine3.16 @@ -69,6 +70,10 @@ build-manager-local() { build-local ${MANAGER_BINARY_NAME} manager } +build-trainer-local() { + build-local ${TRAINER_BINARY_NAME} trainer +} + build-docker() { cd "${BUILD_SOURCE_HOME}" || return docker run \ @@ -124,6 +129,10 @@ build-manager-console() { cp -r $CONSOLE_ASSETS $MANAGER_ASSETS_DIR } +build-trainer-docker() { + build-docker ${TRAINER_BINARY_NAME} trainer +} + main() { create-dirs if [[ "1" == "${USE_DOCKER}" ]]; then @@ -141,6 +150,9 @@ main() { scheduler) build-scheduler-docker ;; + trainer) + build-trainer-docker + ;; manager) build-manager-docker ;; @@ -153,6 +165,7 @@ main() { build-dfstore-docker build-scheduler-docker build-manager-docker + build-trainer-docker ;; esac else @@ -170,6 +183,9 @@ main() { scheduler) build-scheduler-local ;; + trainer) + build-trainer-local + ;; manager) build-manager-local ;; @@ -182,6 +198,7 @@ main() { build-dfstore-local build-scheduler-local build-manager-local + build-trainer-local ;; esac fi diff --git a/hack/docker-build.sh b/hack/docker-build.sh index 63b36ece6..ec71c9913 100755 --- a/hack/docker-build.sh +++ b/hack/docker-build.sh @@ -55,6 +55,9 @@ main() { manager) git-submodule docker-build manager + ;; + trainer) + docker-build trainer esac } diff --git a/hack/docker-push.sh b/hack/docker-push.sh index c88ac2c6a..6dd28d3fc 100755 --- a/hack/docker-push.sh +++ b/hack/docker-push.sh @@ -23,6 +23,9 @@ main() { ;; manager) docker-push manager + ;; + trainer) + docker-push trainer esac } diff --git a/hack/gen-buildx.sh b/hack/gen-buildx.sh index ccbbc6024..c5e1d9046 100755 --- a/hack/gen-buildx.sh +++ b/hack/gen-buildx.sh @@ -1,6 +1,6 @@ #!/bin/bash -components="dfdaemon scheduler manager" +components="dfdaemon scheduler manager trainer" set -x diff --git a/hack/install.sh b/hack/install.sh index 21a3f913c..09856ca3c 100755 --- a/hack/install.sh +++ b/hack/install.sh @@ -8,6 +8,7 @@ BIN_DIR="../bin" DFGET_BINARY_NAME=dfget SCHEDULER_BINARY_NAME=scheduler MANAGER_BINARY_NAME=manager +TRAINER_BINARY_NAME=trainer curDir=$(cd "$(dirname "$0")" && pwd) cd "${curDir}" || return @@ -24,6 +25,9 @@ install() { ;; manager) install-manager + ;; + trainer) + install-trainer esac } @@ -72,6 +76,21 @@ uninstall-manager() { test -e /usr/local/bin/manager && unlink /usr/local/bin/manager } +install-trainer() { + local bin="${INSTALL_HOME}/${INSTALL_BIN_PATH}" + echo "install: ${bin}" + mkdir -p "${bin}" + + cp "${BIN_DIR}/${GOOS}_${GOARCH}/${TRAINER_BINARY_NAME}" "${bin}" + + createLink "${bin}/${TRAINER_BINARY_NAME}" /usr/local/bin/trainer +} + +uninstall-trainer() { + echo "unlink /usr/local/bin/trainer" + test -e /usr/local/bin/trainer && unlink /usr/local/bin/trainer +} + createLink() { srcPath="$1" linkPath="$2" diff --git a/hack/kind-load.sh b/hack/kind-load.sh index d727d2632..733222cbd 100755 --- a/hack/kind-load.sh +++ b/hack/kind-load.sh @@ -22,6 +22,9 @@ main() { manager) kind-load manager ;; + trainer) + kind-load trainer + ;; no-content-length) kind-load no-content-length esac