chore: add trainer to Makefile and shell (#2488)

Signed-off-by: Gaius <gaius.qi@gmail.com>

Makefile

@@ -34,12 +34,12 @@ build-dirs:
 .PHONY: build-dirs
 
 # Build dragonlfy.
-docker-build: docker-build-dfdaemon docker-build-scheduler docker-build-manager
+docker-build: docker-build-dfdaemon docker-build-scheduler docker-build-manager docker-build-trainer
 	@echo "Build image done."
 .PHONY: docker-build
 
 # Push dragonfly images.
-docker-push: docker-push-dfdaemon docker-push-scheduler docker-push-manager
+docker-push: docker-push-dfdaemon docker-push-scheduler docker-push-manager docker-build-trainer
 	@echo "Push image done."
 .PHONY: docker-push
 
@@ -61,6 +61,12 @@ docker-build-manager:
 	./hack/docker-build.sh manager
 .PHONY: docker-build-manager
 
+# Build trainer image.
+docker-build-trainer:
+	@echo "Begin to use docker build trainer image."
+	./hack/docker-build.sh trainer
+.PHONY: docker-build-trainer
+
 # Build testing tools image.
 docker-build-testing-tools: build-dirs
 	@echo "Begin to testing tools image."
@@ -85,8 +91,14 @@ docker-push-manager: docker-build-manager
 	./hack/docker-push.sh manager
 .PHONY: docker-push-manager
 
+# Push trainer image.
+docker-push-trainer: docker-build-trainer
+	@echo "Begin to push trainer docker image."
+	./hack/docker-push.sh trainer
+.PHONY: docker-push-trainer
+
 # Build dragonfly.
-build: build-manager build-scheduler build-dfget build-dfcache build-dfstore
+build: build-manager build-scheduler build-trainer build-dfget build-dfcache build-dfstore
 .PHONY: build
 
 # Build dfget.
@@ -149,6 +161,12 @@ build-manager-console: build-dirs
 	./hack/build.sh manager-console
 .PHONY: build-manager-console
 
+# Build trainer.
+build-trainer: build-dirs
+	@echo "Begin to build trainer."
+	./hack/build.sh trainer
+.PHONY: build-trainer
+
 # Install dfget.
 install-dfget:
 	@echo "Begin to install dfget."
@@ -167,6 +185,12 @@ install-manager:
 	./hack/install.sh install manager
 .PHONY: install-manager
 
+# Install trainer.
+install-trainer:
+	@echo "Begin to install trainer."
+	./hack/install.sh install trainer 
+.PHONY: install-trainer
+
 # Build rpm dfget.
 build-rpm-dfget: build-linux-dfget
 	@echo "Begin to build rpm dfget."
@@ -349,7 +373,7 @@ clean-e2e-test:
 .PHONY: clean-e2e-test
 
 # Kind load dragonlfy.
-kind-load: kind-load-scheduler kind-load-dfdaemon kind-load-manager kind-load-testing-tools
+kind-load: kind-load-scheduler kind-load-dfdaemon kind-load-manager kind-load-trainer kind-load-testing-tools
 	@echo "Kind load image done."
 .PHONY: kind-load
 
@@ -368,6 +392,11 @@ kind-load-manager:
 	@./hack/kind-load.sh manager
 .PHONY: kind-load-manager
 
+# Run kind load docker trainer.
+kind-load-trainer:
+	@./hack/kind-load.sh trainer
+.PHONY: kind-load-trainer
+
 # Run kind load docker testing tools.
 kind-load-testing-tools:
 	@./hack/kind-load.sh no-content-length
@@ -412,9 +441,11 @@ help:
 	@echo "make docker-build-dfdaemon          build dfdaemon image"
 	@echo "make docker-build-scheduler         build scheduler image"
 	@echo "make docker-build-manager           build manager image"
+	@echo "make docker-build-trainer           build trainer image"
 	@echo "make docker-push-dfdaemon           push dfdaemon image"
 	@echo "make docker-push-scheduler          push scheduler image"
 	@echo "make docker-push-manager            push manager image"
+	@echo "make docker-push-trainer            push trainer image"
 	@echo "make build                          build dragonfly"
 	@echo "make build-dfget                    build dfget"
 	@echo "make build-linux-dfget              build linux dfget"
@@ -426,11 +457,13 @@ help:
 	@echo "make build-manager                  build manager"
 	@echo "make build-manager-server           build manager server"
 	@echo "make build-manager-console          build manager console"
+	@echo "make build-trainer                  build trainer"
 	@echo "make build-e2e-sha256sum            build sha256sum test tool"
 	@echo "make build-e2e-download-grpc-test   build download grpc test tool"
 	@echo "make install-dfget                  install dfget"
 	@echo "make install-scheduler              install scheduler"
 	@echo "make install-manager                install manager"
+	@echo "make install-trainer                install trainer"
 	@echo "make build-rpm-dfget                build rpm dfget"
 	@echo "make build-rpm-dfcache              build rpm dfcache"
 	@echo "make build-rpm-dfstore              build rpm dfstore"
@@ -452,6 +485,7 @@ help:
 	@echo "make kind-load-scheduler            kind load scheduler docker image"
 	@echo "make kind-load-dfdaemon             kind load dfdaemon docker image"
 	@echo "make kind-load-manager              kind load manager docker image"
+	@echo "make kind-load-trainer              kind load trainer docker image"
 	@echo "make kind-load-testing-tools        kind load testing tools docker image"
 	@echo "make lint                           run code lint"
 	@echo "make markdownlint                   run markdown lint"

build/images/trainer/Dockerfile

@@ -0,0 +1,34 @@
+ARG BASE_IMAGE=alpine:3.16
+
+FROM golang:1.20.1-alpine3.16 as builder
+
+ARG GOPROXY
+ARG GOTAGS
+ARG GOGCFLAGS
+
+WORKDIR /go/src/d7y.io/dragonfly/v2
+
+RUN apk --no-cache add bash make gcc libc-dev git
+
+COPY . /go/src/d7y.io/dragonfly/v2
+
+RUN make build-trainer && make install-trainer
+
+FROM ${BASE_IMAGE} as health
+
+ENV GRPC_HEALTH_PROBE_VERSION v0.4.16
+
+RUN wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
+    chmod +x /bin/grpc_health_probe
+
+FROM ${BASE_IMAGE}
+
+ENV PATH=/opt/dragonfly/bin:$PATH
+RUN echo "hosts: files dns" > /etc/nsswitch.conf
+
+COPY --from=builder /opt/dragonfly/bin/trainer /opt/dragonfly/bin/trainer
+COPY --from=health /bin/grpc_health_probe /bin/grpc_health_probe
+
+EXPOSE 8002
+
+ENTRYPOINT ["/opt/dragonfly/bin/trainer"]

hack/build.sh

@@ -9,6 +9,7 @@ DFCACHE_BINARY_NAME=dfcache
 DFSTORE_BINARY_NAME=dfstore
 SCHEDULER_BINARY_NAME=scheduler
 MANAGER_BINARY_NAME=manager
+TRAINER_BINARY_NAME=trainer
 
 PKG=d7y.io/dragonfly/v2
 BUILD_IMAGE=golang:1.20.1-alpine3.16
@@ -69,6 +70,10 @@ build-manager-local() {
     build-local ${MANAGER_BINARY_NAME} manager
 }
 
+build-trainer-local() {
+    build-local ${TRAINER_BINARY_NAME} trainer
+}
+
 build-docker() {
     cd "${BUILD_SOURCE_HOME}" || return
     docker run \
@@ -124,6 +129,10 @@ build-manager-console() {
     cp -r $CONSOLE_ASSETS $MANAGER_ASSETS_DIR
 }
 
+build-trainer-docker() {
+    build-docker ${TRAINER_BINARY_NAME} trainer
+}
+
 main() {
     create-dirs
     if [[ "1" == "${USE_DOCKER}" ]]; then
@@ -141,6 +150,9 @@ main() {
         scheduler)
             build-scheduler-docker
             ;;
+        trainer)
+            build-trainer-docker
+            ;;
         manager)
             build-manager-docker
             ;;
@@ -153,6 +165,7 @@ main() {
             build-dfstore-docker
             build-scheduler-docker
             build-manager-docker
+            build-trainer-docker
             ;;
         esac
     else
@@ -170,6 +183,9 @@ main() {
         scheduler)
             build-scheduler-local
             ;;
+        trainer)
+            build-trainer-local
+            ;;
         manager)
             build-manager-local
             ;;
@@ -182,6 +198,7 @@ main() {
             build-dfstore-local
             build-scheduler-local
             build-manager-local
+            build-trainer-local
             ;;
         esac
     fi

hack/docker-build.sh

@@ -55,6 +55,9 @@ main() {
     manager)
         git-submodule
         docker-build manager
+        ;;
+    trainer)
+        docker-build trainer
     esac
 }
 

hack/docker-push.sh

@@ -23,6 +23,9 @@ main() {
         ;;
     manager)
         docker-push manager
+        ;;
+    trainer)
+        docker-push trainer
     esac
 }
 

hack/gen-buildx.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-components="dfdaemon scheduler manager"
+components="dfdaemon scheduler manager trainer"
 
 set -x
 

hack/install.sh

@@ -8,6 +8,7 @@ BIN_DIR="../bin"
 DFGET_BINARY_NAME=dfget
 SCHEDULER_BINARY_NAME=scheduler
 MANAGER_BINARY_NAME=manager
+TRAINER_BINARY_NAME=trainer
 
 curDir=$(cd "$(dirname "$0")" && pwd)
 cd "${curDir}" || return
@@ -24,6 +25,9 @@ install() {
         ;;
     manager)
         install-manager
+        ;;
+    trainer)
+        install-trainer
     esac
 }
 
@@ -72,6 +76,21 @@ uninstall-manager() {
     test -e /usr/local/bin/manager && unlink /usr/local/bin/manager
 }
 
+install-trainer() {
+    local bin="${INSTALL_HOME}/${INSTALL_BIN_PATH}"
+    echo "install: ${bin}"
+    mkdir -p "${bin}"
+
+    cp "${BIN_DIR}/${GOOS}_${GOARCH}/${TRAINER_BINARY_NAME}" "${bin}"
+
+    createLink "${bin}/${TRAINER_BINARY_NAME}" /usr/local/bin/trainer
+}
+
+uninstall-trainer() {
+    echo "unlink /usr/local/bin/trainer"
+    test -e /usr/local/bin/trainer && unlink /usr/local/bin/trainer
+}
+
 createLink() {
     srcPath="$1"
     linkPath="$2"

hack/kind-load.sh

@@ -22,6 +22,9 @@ main() {
     manager)
         kind-load manager
         ;;
+    trainer)
+        kind-load trainer
+        ;;
     no-content-length)
         kind-load no-content-length
     esac